From 00ed35010d4bada38eed1e58610f1b6423350adc Mon Sep 17 00:00:00 2001 From: Pranav Walimbe Date: Tue, 25 Aug 2026 16:04:19 -0700 Subject: [PATCH] perf: locate the q2 zone by name before reading its boundary The zone predicate prunes nothing because every row group's z_name statistics span the full name range, so Polars fetched all 1386 MB of boundary geometry to return one 169 KB polygon. Reading the name column alone costs 2.4 MB and identifies the file and row, after which a slice skips every other row group. DuckDB already performs this late materialisation on the same declarative query. SedonaDB and Polars do not. --- bench/spatial_bench/queries/pycanopy/q02.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/bench/spatial_bench/queries/pycanopy/q02.py b/bench/spatial_bench/queries/pycanopy/q02.py index 3c2c901..409a6c6 100644 --- a/bench/spatial_bench/queries/pycanopy/q02.py +++ b/bench/spatial_bench/queries/pycanopy/q02.py @@ -17,13 +17,24 @@ def pycanopy(data_paths: dict[str, str]) -> pl.DataFrame: - # The zone filter belongs in the scan so Polars decodes one boundary instead of every one + # Zone row group statistics span the whole name range so the predicate prunes no bytes + names = ( + pl.scan_parquet( + data_paths["zone"], storage_options=STORAGE_OPTIONS, include_file_paths="_file" + ) + .select(["z_name", "_file"]) + .collect() + ) + hit = names.with_row_index("_row").filter(pl.col("z_name") == ZONE_NAME).head(1) + zone_file = hit["_file"][0] + # Row offset inside its own file lets the slice skip every other row group + local_row = hit["_row"][0] - int((names["_file"] == zone_file).arg_max()) + zone, trip = pl.collect_all( [ - pl.scan_parquet(data_paths["zone"], storage_options=STORAGE_OPTIONS) - .filter(pl.col("z_name") == ZONE_NAME) - .select(["z_name", "z_boundary"]) - .head(1), + pl.scan_parquet(zone_file, storage_options=STORAGE_OPTIONS) + .select(["z_boundary"]) + .slice(local_row, 1), pl.scan_parquet(data_paths["trip"], storage_options=STORAGE_OPTIONS).select( ["t_pickuploc"] ),