Bypass Polars with most Nothing columns

noxfile.py

@@ -77,6 +77,7 @@ def coverage(s: Session):
         ["ruff", "check", "."],
         ["ruff", "format", "--check", "."],
         ["cargo", "clippy", "--locked", "--", "-D", "warnings"],
+        ["cargo", "fmt", "--check"],
     ],
 )
 def lint(s: Session, command: list[str]):

src/data_frame.rs

@@ -1157,7 +1157,13 @@ impl PyDataFrame {
                     .agg([all().as_expr().gather(Expr::Literal(LiteralValue::Series(
                         SpecialEq::new(Series::new("".into(), &indexes)),
                     )))])
-                    .explode(cols(non_group_columns), ExplodeOptions { empty_as_null: false, keep_nulls: false })
+                    .explode(
+                        cols(non_group_columns),
+                        ExplodeOptions {
+                            empty_as_null: false,
+                            keep_nulls: false,
+                        },
+                    )
                     .sort(["_index"], Default::default())
                     .drop(cols(["_index"]))
                     .collect();

src/data_type.rs

@@ -150,7 +150,9 @@ impl DataType {
     /// Returns Float32 if either operand is Float32 and neither is Float64.
     /// Returns Float64 otherwise (including when both are integers).
     pub fn promote_to_float(self, other: DataType) -> DataType {
-        if self == DataType::Float64 || other == DataType::Float64 {
+        if self == DataType::Nothing || other == DataType::Nothing {
+            DataType::Nothing
+        } else if self == DataType::Float64 || other == DataType::Float64 {
             DataType::Float64
         } else if self == DataType::Float32 || other == DataType::Float32 {
             DataType::Float32

src/function/branch.rs

@@ -70,11 +70,16 @@ impl IfElse {
 
 impl Function for IfElse {
     fn to_polars(&self) -> Expr {
-        ternary_expr(
-            self.condition.to_polars(),
-            self.then_branch.to_polars(),
-            self.else_branch.to_polars(),
-        )
+        if self.condition.expression_type().data_type() == crate::data_type::DataType::Nothing {
+            // WORKAROUND: Polars ternary_expr crashes when condition has DataType::Null
+            lit(NULL)
+        } else {
+            ternary_expr(
+                self.condition.to_polars(),
+                self.then_branch.to_polars(),
+                self.else_branch.to_polars(),
+            )
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {

src/function/elementwise_extrema.rs

@@ -65,6 +65,11 @@ impl PMax {
 
 impl Function for PMax {
     fn to_polars(&self) -> Expr {
+        if self.expression_type.data_type() == crate::data_type::DataType::Nothing {
+            // WORKAROUND: Polars max_horizontal on DataType::Null columns may not preserve Null dtype
+            return lit(NULL);
+        }
+
         if self.arguments.len() == 1 {
             return self.arguments[0].to_polars();
         }
@@ -168,6 +173,11 @@ impl PMin {
 
 impl Function for PMin {
     fn to_polars(&self) -> Expr {
+        if self.expression_type.data_type() == crate::data_type::DataType::Nothing {
+            // WORKAROUND: Polars min_horizontal on DataType::Null columns may not preserve Null dtype
+            return lit(NULL);
+        }
+
         if self.arguments.len() == 1 {
             return self.arguments[0].to_polars();
         }

src/function/finite.rs

@@ -96,16 +96,26 @@ impl IsNan {
 
         require_numeric(arg_type, "is_nan", "argument")?;
 
+        let expression_type = if arg_type.data_type() == DataType::Nothing {
+            arg_type
+        } else {
+            arg_type.with_data_type(DataType::Boolean)
+        };
         Ok(Arc::new(IsNan {
             argument: Arc::new(typed_arg),
-            expression_type: arg_type.with_data_type(DataType::Boolean),
+            expression_type,
         }) as Arc<dyn Function>)
     }
 }
 
 impl Function for IsNan {
     fn to_polars(&self) -> Expr {
-        self.argument.to_polars().is_nan()
+        if self.argument.expression_type().data_type() == DataType::Nothing {
+            // WORKAROUND: Polars is_nan() crashes on DataType::Null (Nothing) columns
+            lit(NULL)
+        } else {
+            self.argument.to_polars().is_nan()
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {
@@ -160,16 +170,26 @@ impl IsFinite {
 
         require_numeric(arg_type, "is_finite", "argument")?;
 
+        let expression_type = if arg_type.data_type() == DataType::Nothing {
+            arg_type
+        } else {
+            arg_type.with_data_type(DataType::Boolean)
+        };
         Ok(Arc::new(IsFinite {
             argument: Arc::new(typed_arg),
-            expression_type: arg_type.with_data_type(DataType::Boolean),
+            expression_type,
         }) as Arc<dyn Function>)
     }
 }
 
 impl Function for IsFinite {
     fn to_polars(&self) -> Expr {
-        self.argument.to_polars().is_finite()
+        if self.argument.expression_type().data_type() == DataType::Nothing {
+            // WORKAROUND: Polars is_finite() crashes on DataType::Null (Nothing) columns
+            lit(NULL)
+        } else {
+            self.argument.to_polars().is_finite()
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {

src/function/interp.rs

@@ -130,8 +130,16 @@ impl Interp {
         require_numeric(ts_type, "interp", "ts")?;
         require_numeric(ys_type, "interp", "ys")?;
 
-        let result_type = t_type.with_data_type(crate::data_type::DataType::Float64);
         let float64 = crate::data_type::DataType::Float64;
+        let nothing = crate::data_type::DataType::Nothing;
+        let result_type = if t_type.data_type() == nothing
+            || ts_type.data_type() == nothing
+            || ys_type.data_type() == nothing
+        {
+            t_type.with_data_type(nothing)
+        } else {
+            t_type.with_data_type(float64)
+        };
 
         Ok(Arc::new(Interp {
             t: Arc::new(t.cast_if_needed(float64)),
@@ -144,16 +152,16 @@ impl Interp {
 
 impl Function for Interp {
     fn to_polars(&self) -> Expr {
-        apply_multiple(
-            interpolate,
-            &[
-                self.t.to_polars(),
-                self.ts.to_polars(),
-                self.ys.to_polars(),
-            ],
-            |_, fields| Ok(fields[0].clone()),
-            true,
-        )
+        if self.expression_type.data_type() == crate::data_type::DataType::Nothing {
+            lit(NULL)
+        } else {
+            apply_multiple(
+                interpolate,
+                &[self.t.to_polars(), self.ts.to_polars(), self.ys.to_polars()],
+                |_, fields| Ok(fields[0].clone()),
+                true,
+            )
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {

src/function/log.rs

@@ -54,7 +54,14 @@ macro_rules! impl_log_function {
 
         impl Function for $name {
             fn to_polars(&self) -> Expr {
-                self.argument.to_polars().log(Expr::from($base))
+                if self.argument.expression_type().data_type()
+                    == crate::data_type::DataType::Nothing
+                {
+                    // WORKAROUND: Polars returns Unknown(Float) for log on empty DataType::Null columns
+                    lit(NULL)
+                } else {
+                    self.argument.to_polars().log(Expr::from($base))
+                }
             }
 
             fn substitute(

src/function/logical.rs

@@ -42,7 +42,13 @@ impl Any {
 
 impl Function for Any {
     fn to_polars(&self) -> Expr {
-        self.argument.to_polars().any(false)
+        if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing {
+            // WORKAROUND: Polars any() crashes on DataType::Null (Nothing) columns;
+            // any(nothing) = nothing per three-value logic
+            lit(NULL)
+        } else {
+            self.argument.to_polars().any(false)
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {
@@ -107,7 +113,13 @@ impl All {
 
 impl Function for All {
     fn to_polars(&self) -> Expr {
-        self.argument.to_polars().all(false)
+        if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing {
+            // WORKAROUND: Polars all() crashes on DataType::Null (Nothing) columns;
+            // all(nothing) = nothing per three-value logic
+            lit(NULL)
+        } else {
+            self.argument.to_polars().all(false)
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {

src/function/power.rs

@@ -6,8 +6,8 @@ use polars::prelude::*;
 
 use crate::expression::Expression;
 use crate::typed_expression::{
-    promote_expression_types, require_numeric, DataFrameType, ExpressionType, LiteralType,
-    Function, TypedExpression, ValidationError,
+    promote_expression_types, require_numeric, DataFrameType, ExpressionType, Function,
+    LiteralType, TypedExpression, ValidationError,
 };
 
 #[derive(Debug, Clone, PartialEq)]
@@ -45,7 +45,12 @@ impl Sqrt {
 
 impl Function for Sqrt {
     fn to_polars(&self) -> Expr {
-        self.argument.to_polars().sqrt()
+        if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing {
+            // WORKAROUND: Polars returns Unknown(Float) for sqrt on empty DataType::Null columns
+            lit(NULL)
+        } else {
+            self.argument.to_polars().sqrt()
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {
@@ -111,7 +116,12 @@ impl Exp {
 
 impl Function for Exp {
     fn to_polars(&self) -> Expr {
-        self.argument.to_polars().exp()
+        if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing {
+            // WORKAROUND: Polars returns Float64 for exp() on DataType::Null (Nothing) columns
+            lit(NULL)
+        } else {
+            self.argument.to_polars().exp()
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {
@@ -171,8 +181,7 @@ impl Pow {
         require_numeric(base_type, "pow", "base")?;
         require_numeric(exponent_type, "pow", "exponent")?;
 
-        let promoted_type =
-            promote_expression_types(base_type, exponent_type, "pow")?;
+        let promoted_type = promote_expression_types(base_type, exponent_type, "pow")?;
 
         // Check wholeness on the original exponent type, before any casting
         let exponent_is_whole = match exponent_type {
@@ -207,7 +216,12 @@ impl Pow {
 
 impl Function for Pow {
     fn to_polars(&self) -> Expr {
-        self.base.to_polars().pow(self.exponent.to_polars())
+        if self.expression_type.data_type() == crate::data_type::DataType::Nothing {
+            // WORKAROUND: Polars pow crashes on DataType::Null (Nothing) columns
+            lit(NULL)
+        } else {
+            self.base.to_polars().pow(self.exponent.to_polars())
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {

src/function/round.rs

@@ -43,7 +43,14 @@ macro_rules! impl_round_function {
 
         impl Function for $name {
             fn to_polars(&self) -> Expr {
-                self.argument.to_polars().$polars_method()
+                if self.argument.expression_type().data_type()
+                    == crate::data_type::DataType::Nothing
+                {
+                    // WORKAROUND: Polars crashes on DataType::Null columns for rounding functions
+                    lit(NULL)
+                } else {
+                    self.argument.to_polars().$polars_method()
+                }
             }
 
             fn substitute(

src/function/sign.rs

@@ -41,7 +41,12 @@ impl Abs {
 
 impl Function for Abs {
     fn to_polars(&self) -> Expr {
-        self.argument.to_polars().abs()
+        if self.argument.expression_type().data_type() == crate::data_type::DataType::Nothing {
+            // WORKAROUND: Polars abs crashes on DataType::Null columns
+            lit(NULL)
+        } else {
+            self.argument.to_polars().abs()
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {

src/function/trapz.rs

@@ -99,26 +99,33 @@ impl Trapz {
         require_array(y_type, "trapz", "y")?;
 
         let float64 = crate::data_type::DataType::Float64;
+        let nothing = crate::data_type::DataType::Nothing;
+        let expression_type = if t_type.data_type() == nothing || y_type.data_type() == nothing {
+            ExpressionType::Scalar(nothing)
+        } else {
+            ExpressionType::Scalar(float64)
+        };
 
         Ok(Arc::new(Trapz {
             t: Arc::new(t.cast_if_needed(float64)),
             y: Arc::new(y.cast_if_needed(float64)),
-            expression_type: ExpressionType::Scalar(float64),
+            expression_type,
         }) as Arc<dyn Function>)
     }
 }
 
 impl Function for Trapz {
     fn to_polars(&self) -> Expr {
-        apply_multiple(
-            compute_trapz,
-            &[
-                self.t.to_polars(),
-                self.y.to_polars(),
-            ],
-            |_, fields| Ok(fields[0].clone()),
-            true,
-        )
+        if self.expression_type.data_type() == crate::data_type::DataType::Nothing {
+            lit(NULL)
+        } else {
+            apply_multiple(
+                compute_trapz,
+                &[self.t.to_polars(), self.y.to_polars()],
+                |_, fields| Ok(fields[0].clone()),
+                true,
+            )
+        }
     }
 
     fn substitute(&self, substitutions: &HashMap<&str, TypedExpression>) -> Arc<dyn Function> {

src/function/trigonometry.rs

@@ -47,7 +47,14 @@ macro_rules! impl_trig_function {
 
         impl Function for $name {
             fn to_polars(&self) -> Expr {
-                self.argument.to_polars().$polars_method()
+                if self.argument.expression_type().data_type()
+                    == crate::data_type::DataType::Nothing
+                {
+                    // WORKAROUND: Polars crashes on DataType::Null columns for trigonometry functions
+                    lit(NULL)
+                } else {
+                    self.argument.to_polars().$polars_method()
+                }
             }
 
             fn substitute(