From aeaf8b462a8f6eca69e066a11c3a22802ba820e7 Mon Sep 17 00:00:00 2001 From: linfeng <33561138+lyne7-sc@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:57:56 +0800 Subject: [PATCH 1/2] perf: avoid identity take in GenerateExec --- native-engine/datafusion-ext-plans/src/generate_exec.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/native-engine/datafusion-ext-plans/src/generate_exec.rs b/native-engine/datafusion-ext-plans/src/generate_exec.rs index ae975237f..1e4c28750 100644 --- a/native-engine/datafusion-ext-plans/src/generate_exec.rs +++ b/native-engine/datafusion-ext-plans/src/generate_exec.rs @@ -274,9 +274,13 @@ fn execute_generate( let generated_ids = generated_ids.finish(); let child_outputs = take_cols(&child_outputs, child_output_row_ids)?; let generated_outputs = match generated_outputs { - Some(generated_outputs) => { + Some(generated_outputs) if outer => { take_cols(&generated_outputs.cols, generated_ids)? } + // Without outer rows, generated_ids is always the identity + // sequence 0..generated_outputs.len(). Reuse the generated + // arrays instead of copying them through Arrow take. + Some(generated_outputs) => generated_outputs.cols, None => generator_output_schema .fields() .iter() From c26475cdcaea30acd7dbfd3aa6853bec7f4f3729 Mon Sep 17 00:00:00 2001 From: linfeng Date: Thu, 3 Sep 2026 21:35:16 +0800 Subject: [PATCH 2/2] docs: clarify GenerateExec identity range comment --- native-engine/datafusion-ext-plans/src/generate_exec.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/native-engine/datafusion-ext-plans/src/generate_exec.rs b/native-engine/datafusion-ext-plans/src/generate_exec.rs index 1e4c28750..7874491e6 100644 --- a/native-engine/datafusion-ext-plans/src/generate_exec.rs +++ b/native-engine/datafusion-ext-plans/src/generate_exec.rs @@ -277,9 +277,9 @@ fn execute_generate( Some(generated_outputs) if outer => { take_cols(&generated_outputs.cols, generated_ids)? } - // Without outer rows, generated_ids is always the identity - // sequence 0..generated_outputs.len(). Reuse the generated - // arrays instead of copying them through Arrow take. + // When outer is false, generated_ids is the identity sequence + // 0..generated_outputs.row_ids.len(). Reuse the generated arrays + // instead of copying them through Arrow take. Some(generated_outputs) => generated_outputs.cols, None => generator_output_schema .fields()