|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one or more |
| 2 | +// contributor license agreements. See the NOTICE file distributed with |
| 3 | +// this work for additional information regarding copyright ownership. |
| 4 | +// The ASF licenses this file to You under the Apache License, Version 2.0 |
| 5 | +// (the "License"); you may not use this file except in compliance with |
| 6 | +// the License. You may obtain a copy of the License at |
| 7 | +// |
| 8 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +// |
| 10 | +// Unless required by applicable law or agreed to in writing, software |
| 11 | +// distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +// See the License for the specific language governing permissions and |
| 14 | +// limitations under the License. |
| 15 | + |
| 16 | +use std::{ |
| 17 | + any::Any, |
| 18 | + fmt::{Debug, Display, Formatter}, |
| 19 | + hash::{Hash, Hasher}, |
| 20 | + sync::Arc, |
| 21 | +}; |
| 22 | + |
| 23 | +use arrow::{ |
| 24 | + array::{Float64Array, RecordBatch}, |
| 25 | + datatypes::{DataType, Schema}, |
| 26 | +}; |
| 27 | +use datafusion::{ |
| 28 | + common::Result, |
| 29 | + logical_expr::ColumnarValue, |
| 30 | + physical_expr::{PhysicalExpr, PhysicalExprRef}, |
| 31 | +}; |
| 32 | +use parking_lot::Mutex; |
| 33 | +use rand::{SeedableRng, rngs::StdRng}; |
| 34 | +use rand_distr::{Distribution, StandardNormal}; |
| 35 | + |
| 36 | +use crate::down_cast_any_ref; |
| 37 | + |
| 38 | +/// Returns random values with independent and identically distributed (i.i.d.) |
| 39 | +/// samples drawn from the standard normal distribution. |
| 40 | +/// |
| 41 | +/// Spark-compatible semantics: |
| 42 | +/// - RNG is seeded with `seed + partition_id` |
| 43 | +/// - RNG state advances for each row (stateful across batches) |
| 44 | +/// |
| 45 | +/// Note: the underlying RNG/gaussian implementation is not intended to |
| 46 | +/// reproduce Spark's exact output sequence for a given seed/partition. |
| 47 | +pub struct SparkRandnExpr { |
| 48 | + seed: i64, |
| 49 | + partition_id: usize, |
| 50 | + rng: Mutex<StdRng>, |
| 51 | +} |
| 52 | + |
| 53 | +impl SparkRandnExpr { |
| 54 | + pub fn new(seed: i64, partition_id: usize) -> Self { |
| 55 | + let effective_seed = (seed as u64).wrapping_add(partition_id as u64); |
| 56 | + Self { |
| 57 | + seed, |
| 58 | + partition_id, |
| 59 | + rng: Mutex::new(StdRng::seed_from_u64(effective_seed)), |
| 60 | + } |
| 61 | + } |
| 62 | +} |
| 63 | + |
| 64 | +impl Display for SparkRandnExpr { |
| 65 | + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { |
| 66 | + write!( |
| 67 | + f, |
| 68 | + "Randn(seed={}, partition={})", |
| 69 | + self.seed, self.partition_id |
| 70 | + ) |
| 71 | + } |
| 72 | +} |
| 73 | + |
| 74 | +impl Debug for SparkRandnExpr { |
| 75 | + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { |
| 76 | + write!( |
| 77 | + f, |
| 78 | + "Randn(seed={}, partition={})", |
| 79 | + self.seed, self.partition_id |
| 80 | + ) |
| 81 | + } |
| 82 | +} |
| 83 | + |
| 84 | +impl PartialEq for SparkRandnExpr { |
| 85 | + fn eq(&self, other: &Self) -> bool { |
| 86 | + self.seed == other.seed && self.partition_id == other.partition_id |
| 87 | + } |
| 88 | +} |
| 89 | + |
| 90 | +impl Eq for SparkRandnExpr {} |
| 91 | + |
| 92 | +impl Hash for SparkRandnExpr { |
| 93 | + fn hash<H: Hasher>(&self, state: &mut H) { |
| 94 | + self.seed.hash(state); |
| 95 | + self.partition_id.hash(state); |
| 96 | + } |
| 97 | +} |
| 98 | + |
| 99 | +impl PhysicalExpr for SparkRandnExpr { |
| 100 | + fn as_any(&self) -> &dyn Any { |
| 101 | + self |
| 102 | + } |
| 103 | + |
| 104 | + fn data_type(&self, _input_schema: &Schema) -> Result<DataType> { |
| 105 | + Ok(DataType::Float64) |
| 106 | + } |
| 107 | + |
| 108 | + fn nullable(&self, _input_schema: &Schema) -> Result<bool> { |
| 109 | + Ok(false) |
| 110 | + } |
| 111 | + |
| 112 | + fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> { |
| 113 | + let num_rows = batch.num_rows(); |
| 114 | + let mut rng = self.rng.lock(); |
| 115 | + let values = |
| 116 | + Float64Array::from_iter_values(StandardNormal.sample_iter(&mut *rng).take(num_rows)); |
| 117 | + Ok(ColumnarValue::Array(Arc::new(values))) |
| 118 | + } |
| 119 | + |
| 120 | + fn children(&self) -> Vec<&PhysicalExprRef> { |
| 121 | + vec![] |
| 122 | + } |
| 123 | + |
| 124 | + fn with_new_children( |
| 125 | + self: Arc<Self>, |
| 126 | + _children: Vec<PhysicalExprRef>, |
| 127 | + ) -> Result<PhysicalExprRef> { |
| 128 | + Ok(Arc::new(Self::new(self.seed, self.partition_id))) |
| 129 | + } |
| 130 | + |
| 131 | + fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { |
| 132 | + write!(f, "randn({})", self.seed) |
| 133 | + } |
| 134 | +} |
| 135 | + |
| 136 | +impl PartialEq<dyn Any> for SparkRandnExpr { |
| 137 | + fn eq(&self, other: &dyn Any) -> bool { |
| 138 | + down_cast_any_ref(other) |
| 139 | + .downcast_ref::<Self>() |
| 140 | + .map(|other| self.seed == other.seed && self.partition_id == other.partition_id) |
| 141 | + .unwrap_or(false) |
| 142 | + } |
| 143 | +} |
| 144 | + |
| 145 | +#[cfg(test)] |
| 146 | +mod tests { |
| 147 | + use std::sync::Arc; |
| 148 | + |
| 149 | + use arrow::{array::RecordBatch, datatypes::Schema}; |
| 150 | + use datafusion::common::{Result, cast::as_float64_array}; |
| 151 | + |
| 152 | + use super::*; |
| 153 | + |
| 154 | + fn create_empty_batch(num_rows: usize) -> RecordBatch { |
| 155 | + let schema = Arc::new(Schema::empty()); |
| 156 | + RecordBatch::try_new_with_options( |
| 157 | + schema, |
| 158 | + vec![], |
| 159 | + &arrow::array::RecordBatchOptions::new().with_row_count(Some(num_rows)), |
| 160 | + ) |
| 161 | + .expect("Failed to create empty batch") |
| 162 | + } |
| 163 | + |
| 164 | + #[test] |
| 165 | + fn test_randn_generates_different_values_per_row() -> Result<()> { |
| 166 | + let expr = SparkRandnExpr::new(42, 0); |
| 167 | + let batch = create_empty_batch(5); |
| 168 | + |
| 169 | + let result = expr.evaluate(&batch)?; |
| 170 | + let array = result.into_array(5)?; |
| 171 | + let float_arr = as_float64_array(&array)?; |
| 172 | + |
| 173 | + // Values should not be constant across rows, which verifies a value is |
| 174 | + // generated per row rather than a single value being broadcast. |
| 175 | + // (Individual samples are allowed to repeat, so we don't require all |
| 176 | + // values to be distinct.) |
| 177 | + let values: Vec<f64> = (0..5).map(|i| float_arr.value(i)).collect(); |
| 178 | + assert!( |
| 179 | + values.iter().any(|&v| v != values[0]), |
| 180 | + "Expected per-row values, but all rows were identical: {values:?}" |
| 181 | + ); |
| 182 | + |
| 183 | + Ok(()) |
| 184 | + } |
| 185 | + |
| 186 | + #[test] |
| 187 | + fn test_randn_reproducible_with_same_seed() -> Result<()> { |
| 188 | + let expr1 = SparkRandnExpr::new(42, 0); |
| 189 | + let expr2 = SparkRandnExpr::new(42, 0); |
| 190 | + let batch = create_empty_batch(5); |
| 191 | + |
| 192 | + let result1 = expr1.evaluate(&batch)?; |
| 193 | + let result2 = expr2.evaluate(&batch)?; |
| 194 | + |
| 195 | + let arr1_binding = result1.into_array(5)?; |
| 196 | + let arr2_binding = result2.into_array(5)?; |
| 197 | + let arr1 = as_float64_array(&arr1_binding)?; |
| 198 | + let arr2 = as_float64_array(&arr2_binding)?; |
| 199 | + |
| 200 | + for i in 0..5 { |
| 201 | + assert_eq!( |
| 202 | + arr1.value(i), |
| 203 | + arr2.value(i), |
| 204 | + "Same seed should produce same values" |
| 205 | + ); |
| 206 | + } |
| 207 | + |
| 208 | + Ok(()) |
| 209 | + } |
| 210 | + |
| 211 | + #[test] |
| 212 | + fn test_randn_different_seeds_produce_different_values() -> Result<()> { |
| 213 | + let expr1 = SparkRandnExpr::new(42, 0); |
| 214 | + let expr2 = SparkRandnExpr::new(123, 0); |
| 215 | + let batch = create_empty_batch(5); |
| 216 | + |
| 217 | + let result1 = expr1.evaluate(&batch)?; |
| 218 | + let result2 = expr2.evaluate(&batch)?; |
| 219 | + |
| 220 | + let arr1_binding = result1.into_array(5)?; |
| 221 | + let arr2_binding = result2.into_array(5)?; |
| 222 | + let arr1 = as_float64_array(&arr1_binding)?; |
| 223 | + let arr2 = as_float64_array(&arr2_binding)?; |
| 224 | + |
| 225 | + // At least one value should be different |
| 226 | + let any_different = (0..5).any(|i| arr1.value(i) != arr2.value(i)); |
| 227 | + assert!( |
| 228 | + any_different, |
| 229 | + "Different seeds should produce different values" |
| 230 | + ); |
| 231 | + |
| 232 | + Ok(()) |
| 233 | + } |
| 234 | + |
| 235 | + #[test] |
| 236 | + fn test_randn_different_partitions_produce_different_values() -> Result<()> { |
| 237 | + let expr1 = SparkRandnExpr::new(42, 0); |
| 238 | + let expr2 = SparkRandnExpr::new(42, 1); |
| 239 | + let batch = create_empty_batch(5); |
| 240 | + |
| 241 | + let result1 = expr1.evaluate(&batch)?; |
| 242 | + let result2 = expr2.evaluate(&batch)?; |
| 243 | + |
| 244 | + let arr1_binding = result1.into_array(5)?; |
| 245 | + let arr2_binding = result2.into_array(5)?; |
| 246 | + let arr1 = as_float64_array(&arr1_binding)?; |
| 247 | + let arr2 = as_float64_array(&arr2_binding)?; |
| 248 | + |
| 249 | + // At least one value should be different |
| 250 | + let any_different = (0..5).any(|i| arr1.value(i) != arr2.value(i)); |
| 251 | + assert!( |
| 252 | + any_different, |
| 253 | + "Different partitions should produce different values" |
| 254 | + ); |
| 255 | + |
| 256 | + Ok(()) |
| 257 | + } |
| 258 | + |
| 259 | + #[test] |
| 260 | + fn test_randn_stateful_across_batches() -> Result<()> { |
| 261 | + let expr = SparkRandnExpr::new(42, 0); |
| 262 | + let batch1 = create_empty_batch(3); |
| 263 | + let batch2 = create_empty_batch(3); |
| 264 | + |
| 265 | + // Evaluate two batches sequentially |
| 266 | + let result1 = expr.evaluate(&batch1)?; |
| 267 | + let result2 = expr.evaluate(&batch2)?; |
| 268 | + |
| 269 | + let arr1_binding = result1.into_array(3)?; |
| 270 | + let arr2_binding = result2.into_array(3)?; |
| 271 | + let arr1 = as_float64_array(&arr1_binding)?; |
| 272 | + let arr2 = as_float64_array(&arr2_binding)?; |
| 273 | + |
| 274 | + // Collect all values |
| 275 | + let values1: Vec<f64> = (0..3).map(|i| arr1.value(i)).collect(); |
| 276 | + let values2: Vec<f64> = (0..3).map(|i| arr2.value(i)).collect(); |
| 277 | + |
| 278 | + // Second batch should continue from where first left off (not restart) |
| 279 | + // So values should be different between batches |
| 280 | + assert_ne!(values1, values2, "Batches should have different values"); |
| 281 | + |
| 282 | + // Compare with fresh expr that evaluates both batches together |
| 283 | + let expr_fresh = SparkRandnExpr::new(42, 0); |
| 284 | + let batch_combined = create_empty_batch(6); |
| 285 | + let result_combined = expr_fresh.evaluate(&batch_combined)?; |
| 286 | + let arr_combined_binding = result_combined.into_array(6)?; |
| 287 | + let arr_combined = as_float64_array(&arr_combined_binding)?; |
| 288 | + |
| 289 | + // First 3 values should match values1, next 3 should match values2 |
| 290 | + for i in 0..3 { |
| 291 | + assert_eq!( |
| 292 | + arr_combined.value(i), |
| 293 | + values1[i], |
| 294 | + "First batch values should match" |
| 295 | + ); |
| 296 | + assert_eq!( |
| 297 | + arr_combined.value(i + 3), |
| 298 | + values2[i], |
| 299 | + "Second batch values should match continuation" |
| 300 | + ); |
| 301 | + } |
| 302 | + |
| 303 | + Ok(()) |
| 304 | + } |
| 305 | +} |
0 commit comments