From 8172873a04c19394b56ff78c5a6f0b03de9cbd64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Jun 2026 14:53:29 +1000 Subject: [PATCH 1/2] chore(deps): bump the all-other-cargo-deps group across 1 directory with 6 updates (#22975) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the all-other-cargo-deps group with 5 updates in the / directory: | Package | From | To | | --- | --- | --- | | [insta](https://github.com/mitsuhiko/insta) | `1.47.2` | `1.48.0` | | [memchr](https://github.com/BurntSushi/memchr) | `2.8.1` | `2.8.2` | | [regex](https://github.com/rust-lang/regex) | `1.12.3` | `1.12.4` | | [uuid](https://github.com/uuid-rs/uuid) | `1.23.2` | `1.23.3` | | [stabby](https://github.com/ZettaScaleLabs/stabby) | `72.1.2` | `72.1.8` | Updates `insta` from 1.47.2 to 1.48.0
Release notes

Sourced from insta's releases.

1.48.0

Release Notes

Install cargo-insta 1.48.0

Install prebuilt binaries via shell script

curl --proto '=https' --tlsv1.2 -LsSf
https://github.com/mitsuhiko/insta/releases/download/1.48.0/cargo-insta-installer.sh
| sh

Install prebuilt binaries via powershell script

powershell -ExecutionPolicy Bypass -c "irm
https://github.com/mitsuhiko/insta/releases/download/1.48.0/cargo-insta-installer.ps1
| iex"

Download cargo-insta 1.48.0

File Platform Checksum
cargo-insta-aarch64-apple-darwin.tar.xz Apple Silicon macOS checksum
cargo-insta-x86_64-apple-darwin.tar.xz Intel macOS checksum
cargo-insta-x86_64-pc-windows-msvc.zip x64 Windows checksum
cargo-insta-x86_64-unknown-linux-gnu.tar.xz x64 Linux checksum
cargo-insta-x86_64-unknown-linux-musl.tar.xz x64 MUSL Linux checksum
Changelog

Sourced from insta's changelog.

1.48.0

Commits

Updates `memchr` from 2.8.1 to 2.8.2
Commits

Updates `regex` from 1.12.3 to 1.12.4
Changelog

Sourced from regex's changelog.

1.12.4 (2025-06-09)

This release includes a performance optimization for compilation of regexes with very large character classes.

Improvements:

Commits

Updates `uuid` from 1.23.2 to 1.23.3
Release notes

Sourced from uuid's releases.

v1.23.3

What's Changed

Full Changelog: https://github.com/uuid-rs/uuid/compare/v1.23.2...v1.23.3

Commits

Updates `stabby` from 72.1.2 to 72.1.8
Changelog

Sourced from stabby's changelog.

72.1.8 (api=3.0.3, abi=2.0.0)

72.1.4 (api=3.0.2, abi=2.0.0)

Commits

Updates `regex-syntax` from 0.8.10 to 0.8.11
Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 165 ++++++++++++++++++++++++++++------------------------- 1 file changed, 86 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e70bf18ae5b..fdca3237b71d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -506,7 +506,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -517,7 +517,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -867,7 +867,7 @@ checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1318,7 +1318,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1653,7 +1653,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn", ] [[package]] @@ -1664,7 +1664,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2329,7 +2329,7 @@ version = "54.0.0" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2776,7 +2776,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2817,7 +2817,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2855,7 +2855,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3112,7 +3112,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3705,9 +3705,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.47.2" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" dependencies = [ "console", "globset", @@ -3715,6 +3715,7 @@ dependencies = [ "regex", "serde", "similar", + "strip-ansi-escapes", "tempfile", "walkdir", ] @@ -3797,7 +3798,7 @@ checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4055,9 +4056,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "mimalloc" @@ -4486,7 +4487,7 @@ dependencies = [ "regex", "regex-syntax", "structmeta", - "syn 2.0.117", + "syn", ] [[package]] @@ -4626,7 +4627,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4705,7 +4706,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4780,7 +4781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn", ] [[package]] @@ -4826,7 +4827,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.117", + "syn", "tempfile", ] @@ -4840,7 +4841,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5090,7 +5091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5139,14 +5140,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -5173,9 +5174,9 @@ checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "regress" @@ -5274,7 +5275,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.117", + "syn", "unicode-ident", ] @@ -5286,7 +5287,7 @@ checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14" dependencies = [ "quote", "rand 0.8.6", - "syn 2.0.117", + "syn", ] [[package]] @@ -5463,7 +5464,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.117", + "syn", ] [[package]] @@ -5538,7 +5539,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5549,7 +5550,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5574,7 +5575,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5586,7 +5587,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.117", + "syn", ] [[package]] @@ -5629,7 +5630,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5823,14 +5824,14 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] name = "stabby" -version = "72.1.2" +version = "72.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec9e9da673d4db1d470fa36cf4483ad5b1fdea349a392d400fea5d3673a9c5ca" +checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec" dependencies = [ "rustversion", "stabby-abi", @@ -5838,9 +5839,9 @@ dependencies = [ [[package]] name = "stabby-abi" -version = "72.1.2" +version = "72.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a281b17b3cf11531b7dc4e5f1c6be27db86a06e19c477e7a88fa4ee1b6daf3" +checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4" dependencies = [ "rustc_version", "rustversion", @@ -5850,15 +5851,14 @@ dependencies = [ [[package]] name = "stabby-macros" -version = "72.1.2" +version = "72.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "605b39114a0c132d77ffdd7d179491323dbaa8369e7dcbcdf3da09d0b43c13cf" +checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "rand 0.8.6", - "syn 1.0.109", + "syn", ] [[package]] @@ -5891,6 +5891,15 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -5906,7 +5915,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.117", + "syn", ] [[package]] @@ -5917,7 +5926,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5935,7 +5944,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5970,7 +5979,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.117", + "syn", "typify", "walkdir", ] @@ -5981,17 +5990,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.117" @@ -6020,7 +6018,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6119,7 +6117,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6231,7 +6229,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6433,7 +6431,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6517,7 +6515,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.117", + "syn", "thiserror", "unicode-ident", ] @@ -6535,7 +6533,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.117", + "syn", "typify-impl", ] @@ -6674,9 +6672,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.2" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -6701,6 +6699,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -6804,7 +6811,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wasm-bindgen-shared", ] @@ -6847,7 +6854,7 @@ checksum = "caf0ca1bd612b988616bac1ab34c4e4290ef18f7148a1d8b7f31c150080e9295" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7020,7 +7027,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7031,7 +7038,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7278,7 +7285,7 @@ dependencies = [ "heck", "indexmap 2.14.0", "prettyplease", - "syn 2.0.117", + "syn", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -7294,7 +7301,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -7383,7 +7390,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -7404,7 +7411,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7424,7 +7431,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -7464,7 +7471,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] From 7bb6e152bee1df3cd8215176742bfc1cdf300e52 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Wed, 17 Jun 2026 07:38:25 +0200 Subject: [PATCH 2/2] Remove redundant `collect_stat` and `target_partitions` on `ListingOptions` (#22969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #. ## Rationale for this change Something that was spotted during the review of: - https://github.com/apache/datafusion/pull/22657 `ListingOptions::target_partitions` and `ListingOptions::collect_stat` duplicate `SessionConfig`'s `execution.target_partitions` and `execution.collect_statistics`. After some investigation, I think they only live on `ListingOptions` for historical reasons: when the struct was added (#1010 5 years ago), `TableProvider::scan` had no access to the session, so the values had to be copied onto the table at build time. Once #2660 passed `SessionState` into `scan`, the fields became redundant (and had already drifted — `scan` read them from the session config while `list_files_for_scan` read the stale copy). This PR makes `SessionConfig` the single source of truth. ## What changes are included in this PR? - Remove `target_partitions`/`collect_stat` fields, their builders, and `with_session_config_options` from `ListingOptions`. - `ListingTable` now reads both values from the session config at scan time. - Reserve proto tags 8/9 in `ListingTableScanNode` and drop the related (de)serialization. - Update benchmarks, factory, and test call sites. ## Are these changes tested? Yes, by existing tests ## Are there any user-facing changes? Yes, breaking: the removed fields/builders require configuring `SessionConfig` instead, and the two proto fields no longer round-trip. --------- Co-authored-by: Andrew Lamb --- benchmarks/src/bin/external_aggr.rs | 4 +- benchmarks/src/imdb/run.rs | 5 +- benchmarks/src/sort_pushdown.rs | 7 +- benchmarks/src/sort_tpch.rs | 4 +- benchmarks/src/tpcds/run.rs | 5 +- benchmarks/src/tpch/run.rs | 6 +- datafusion/catalog-listing/src/config.rs | 3 +- datafusion/catalog-listing/src/options.rs | 57 -------------- datafusion/catalog-listing/src/table.rs | 44 +++++------ datafusion/common/src/config.rs | 3 +- datafusion/core/benches/sql_query_with_io.rs | 6 +- .../src/datasource/file_format/options.rs | 15 ++-- .../core/src/datasource/listing/table.rs | 77 +++++++++++-------- .../src/datasource/listing_table_factory.rs | 5 +- .../core/tests/parquet/file_statistics.rs | 15 ++-- datafusion/core/tests/sql/path_partition.rs | 3 +- .../proto-models/proto/datafusion.proto | 70 ++++++++--------- .../proto-models/src/generated/pbjson.rs | 38 --------- .../proto-models/src/generated/prost.rs | 4 - datafusion/proto/src/logical_plan/mod.rs | 4 - .../test_files/information_schema.slt | 2 +- datafusion/sqllogictest/test_files/window.slt | 2 +- .../library-user-guide/upgrading/55.0.0.md | 37 +++++++++ docs/source/user-guide/configs.md | 2 +- 24 files changed, 171 insertions(+), 247 deletions(-) diff --git a/benchmarks/src/bin/external_aggr.rs b/benchmarks/src/bin/external_aggr.rs index a6e322c7fabc..42f25c2cb010 100644 --- a/benchmarks/src/bin/external_aggr.rs +++ b/benchmarks/src/bin/external_aggr.rs @@ -318,9 +318,7 @@ impl ExternalAggrConfig { ); let extension = DEFAULT_PARQUET_EXTENSION; - let options = ListingOptions::new(format) - .with_file_extension(extension) - .with_collect_stat(state.config().collect_statistics()); + let options = ListingOptions::new(format).with_file_extension(extension); let table_path = ListingTableUrl::parse(path)?; let config = ListingTableConfig::new(table_path).with_listing_options(options); diff --git a/benchmarks/src/imdb/run.rs b/benchmarks/src/imdb/run.rs index 6d3b5c6bafb4..e0e302e46684 100644 --- a/benchmarks/src/imdb/run.rs +++ b/benchmarks/src/imdb/run.rs @@ -425,7 +425,6 @@ impl RunOpt { let table_format = self.file_format.as_str(); // Obtain a snapshot of the SessionState - let state = ctx.state(); let (format, path, extension): (Arc, String, &'static str) = match table_format { // dbgen creates .tbl ('|' delimited) files without header @@ -458,9 +457,7 @@ impl RunOpt { } }; - let options = ListingOptions::new(format) - .with_file_extension(extension) - .with_collect_stat(state.config().collect_statistics()); + let options = ListingOptions::new(format).with_file_extension(extension); let table_path = ListingTableUrl::parse(path)?; let config = ListingTableConfig::new(table_path).with_listing_options(options); diff --git a/benchmarks/src/sort_pushdown.rs b/benchmarks/src/sort_pushdown.rs index 8e34706ac140..86f1c0f5c111 100644 --- a/benchmarks/src/sort_pushdown.rs +++ b/benchmarks/src/sort_pushdown.rs @@ -162,7 +162,8 @@ impl RunOpt { let config = self.common.config()?; let rt = self.common.build_runtime()?; let state = SessionStateBuilder::new() - .with_config(config) + // Always collect statistics for sort pushdown + .with_config(config.with_collect_statistics(true)) .with_runtime_env(rt) .with_default_features() .build(); @@ -255,9 +256,7 @@ impl RunOpt { ); let extension = DEFAULT_PARQUET_EXTENSION; - let options = ListingOptions::new(format) - .with_file_extension(extension) - .with_collect_stat(true); // Always collect statistics for sort pushdown + let options = ListingOptions::new(format).with_file_extension(extension); let table_path = ListingTableUrl::parse(path)?; let schema = options.infer_schema(&state, &table_path).await?; diff --git a/benchmarks/src/sort_tpch.rs b/benchmarks/src/sort_tpch.rs index 206911c45add..338afec0e80e 100644 --- a/benchmarks/src/sort_tpch.rs +++ b/benchmarks/src/sort_tpch.rs @@ -333,9 +333,7 @@ impl RunOpt { ); let extension = DEFAULT_PARQUET_EXTENSION; - let options = ListingOptions::new(format) - .with_file_extension(extension) - .with_collect_stat(state.config().collect_statistics()); + let options = ListingOptions::new(format).with_file_extension(extension); let table_path = ListingTableUrl::parse(path)?; let schema = options.infer_schema(&state, &table_path).await?; diff --git a/benchmarks/src/tpcds/run.rs b/benchmarks/src/tpcds/run.rs index cc059575f452..2e0274c935de 100644 --- a/benchmarks/src/tpcds/run.rs +++ b/benchmarks/src/tpcds/run.rs @@ -364,7 +364,6 @@ impl RunOpt { table: &str, ) -> Result> { let path = self.path.to_str().unwrap(); - let target_partitions = self.partitions(); // Obtain a snapshot of the SessionState let state = ctx.state(); @@ -380,9 +379,7 @@ impl RunOpt { let table_path = ListingTableUrl::parse(path)?; let options = ListingOptions::new(Arc::new(format)) - .with_file_extension(DEFAULT_PARQUET_EXTENSION) - .with_target_partitions(target_partitions) - .with_collect_stat(state.config().collect_statistics()); + .with_file_extension(DEFAULT_PARQUET_EXTENSION); let schema = options.infer_schema(&state, &table_path).await?; let constraints = table_constraints(table, schema.as_ref()); diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs index 3e5a6026924e..422bcec9ea06 100644 --- a/benchmarks/src/tpch/run.rs +++ b/benchmarks/src/tpch/run.rs @@ -283,7 +283,6 @@ impl RunOpt { ) -> Result> { let path = self.path.to_str().unwrap(); let table_format = self.file_format.as_str(); - let target_partitions = self.partitions(); // Obtain a snapshot of the SessionState let state = ctx.state(); @@ -320,10 +319,7 @@ impl RunOpt { }; let table_path = ListingTableUrl::parse(path)?; - let options = ListingOptions::new(format) - .with_file_extension(extension) - .with_target_partitions(target_partitions) - .with_collect_stat(state.config().collect_statistics()); + let options = ListingOptions::new(format).with_file_extension(extension); let schema = match table_format { "parquet" => options.infer_schema(&state, &table_path).await?, diff --git a/datafusion/catalog-listing/src/config.rs b/datafusion/catalog-listing/src/config.rs index ca4d2abfcd73..2b83c8ec92b2 100644 --- a/datafusion/catalog-listing/src/config.rs +++ b/datafusion/catalog-listing/src/config.rs @@ -152,8 +152,7 @@ impl ListingTableConfig { /// # use datafusion_datasource_parquet::file_format::ParquetFormat; /// # let table_paths = ListingTableUrl::parse("file:///path/to/data").unwrap(); /// let options = ListingOptions::new(Arc::new(ParquetFormat::default())) - /// .with_file_extension(".parquet") - /// .with_collect_stat(true); + /// .with_file_extension(".parquet"); /// /// let config = ListingTableConfig::new(table_paths).with_listing_options(options); /// // Configure file format and options diff --git a/datafusion/catalog-listing/src/options.rs b/datafusion/catalog-listing/src/options.rs index 0ab15e05abba..55840eb0e312 100644 --- a/datafusion/catalog-listing/src/options.rs +++ b/datafusion/catalog-listing/src/options.rs @@ -20,7 +20,6 @@ use datafusion_catalog::Session; use datafusion_common::plan_err; use datafusion_datasource::ListingTableUrl; use datafusion_datasource::file_format::FileFormat; -use datafusion_execution::config::SessionConfig; use datafusion_expr::SortExpr; use futures::StreamExt; use futures::TryStreamExt; @@ -38,13 +37,6 @@ pub struct ListingOptions { /// The expected partition column names in the folder structure. /// See [Self::with_table_partition_cols] for details pub table_partition_cols: Vec<(String, DataType)>, - /// Set true to try to guess statistics from the files. - /// This can add a lot of overhead as it will usually require files - /// to be opened and at least partially parsed. - pub collect_stat: bool, - /// Group files to avoid that the number of partitions exceeds - /// this limit - pub target_partitions: usize, /// Optional pre-known sort order(s). Must be `SortExpr`s. /// /// DataFusion may take advantage of this ordering to omit sorts @@ -68,30 +60,15 @@ impl ListingOptions { /// Default values: /// - use default file extension filter /// - no input partition to discover - /// - one target partition - /// - do not collect statistics pub fn new(format: Arc) -> Self { Self { file_extension: format.get_ext(), format, table_partition_cols: vec![], - collect_stat: false, - target_partitions: 1, file_sort_order: vec![], } } - /// Set options from [`SessionConfig`] and returns self. - /// - /// Currently this sets `target_partitions` and `collect_stat` - /// but if more options are added in the future that need to be coordinated - /// they will be synchronized through this method. - pub fn with_session_config_options(mut self, config: &SessionConfig) -> Self { - self = self.with_target_partitions(config.target_partitions()); - self = self.with_collect_stat(config.collect_statistics()); - self - } - /// Set file extension on [`ListingOptions`] and returns self. /// /// # Example @@ -205,40 +182,6 @@ impl ListingOptions { self } - /// Set stat collection on [`ListingOptions`] and returns self. - /// - /// ``` - /// # use std::sync::Arc; - /// # use datafusion_catalog_listing::ListingOptions; - /// # use datafusion_datasource_parquet::file_format::ParquetFormat; - /// - /// let listing_options = - /// ListingOptions::new(Arc::new(ParquetFormat::default())).with_collect_stat(true); - /// - /// assert_eq!(listing_options.collect_stat, true); - /// ``` - pub fn with_collect_stat(mut self, collect_stat: bool) -> Self { - self.collect_stat = collect_stat; - self - } - - /// Set number of target partitions on [`ListingOptions`] and returns self. - /// - /// ``` - /// # use std::sync::Arc; - /// # use datafusion_catalog_listing::ListingOptions; - /// # use datafusion_datasource_parquet::file_format::ParquetFormat; - /// - /// let listing_options = - /// ListingOptions::new(Arc::new(ParquetFormat::default())).with_target_partitions(8); - /// - /// assert_eq!(listing_options.target_partitions, 8); - /// ``` - pub fn with_target_partitions(mut self, target_partitions: usize) -> Self { - self.target_partitions = target_partitions; - self - } - /// Set file sort order on [`ListingOptions`] and returns self. /// /// ``` diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index dd3675bd2b39..c0303bc8fb6b 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -533,7 +533,7 @@ impl TableProvider for ListingTable { &self.table_schema, &partitioned_file_lists, output_ordering, - self.options.target_partitions, + state.config().target_partitions(), ) }) }) @@ -541,7 +541,7 @@ impl TableProvider for ListingTable { { Some(Err(e)) => log::debug!("failed to split file groups by statistics: {e}"), Some(Ok(new_groups)) => { - if new_groups.len() <= self.options.target_partitions { + if new_groups.len() <= state.config().target_partitions() { partitioned_file_lists = new_groups; } else { log::debug!( @@ -724,7 +724,7 @@ impl ListingTable { let files = file_list .map(|part_file| async { let part_file = part_file?; - let (statistics, ordering) = if self.options.collect_stat { + let (statistics, ordering) = if ctx.config().collect_statistics() { self.do_collect_statistics_and_ordering(ctx, &store, &part_file) .await? } else { @@ -738,7 +738,7 @@ impl ListingTable { .buffer_unordered(ctx.config_options().execution.meta_fetch_concurrency); let (file_group, inexact_stats) = - get_files_with_limit(files, limit, self.options.collect_stat).await?; + get_files_with_limit(files, limit, ctx.config().collect_statistics()).await?; // Threshold: 0 = disabled, N > 0 = enabled when distinct_keys >= N // @@ -747,32 +747,32 @@ impl ListingTable { // hash repartitioning for aggregates and joins on partition columns. let threshold = ctx.config_options().optimizer.preserve_file_partitions; - let (file_groups, grouped_by_partition) = if threshold > 0 - && !self.options.table_partition_cols.is_empty() - { - let grouped = - file_group.group_by_partition_values(self.options.target_partitions); - if grouped.len() >= threshold { - (grouped, true) + let (file_groups, grouped_by_partition) = + if threshold > 0 && !self.options.table_partition_cols.is_empty() { + let grouped = file_group + .group_by_partition_values(ctx.config().target_partitions()); + if grouped.len() >= threshold { + (grouped, true) + } else { + let all_files: Vec<_> = + grouped.into_iter().flat_map(|g| g.into_inner()).collect(); + ( + FileGroup::new(all_files) + .split_files(ctx.config().target_partitions()), + false, + ) + } } else { - let all_files: Vec<_> = - grouped.into_iter().flat_map(|g| g.into_inner()).collect(); ( - FileGroup::new(all_files).split_files(self.options.target_partitions), + file_group.split_files(ctx.config().target_partitions()), false, ) - } - } else { - ( - file_group.split_files(self.options.target_partitions), - false, - ) - }; + }; let (file_groups, stats) = compute_all_files_statistics( file_groups, self.schema(), - self.options.collect_stat, + ctx.config().collect_statistics(), inexact_stats, )?; diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 07196d009c54..536afbfed461 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -677,8 +677,7 @@ config_namespace! { pub coalesce_batches: bool, default = true /// Should DataFusion collect statistics when first creating a table. - /// Has no effect after the table is created. Applies to the default - /// `ListingTableProvider` in DataFusion. Defaults to true. + /// Has no effect after the table is created. Defaults to true. pub collect_statistics: bool, default = true /// Number of partitions for query execution. Increasing partitions can increase diff --git a/datafusion/core/benches/sql_query_with_io.rs b/datafusion/core/benches/sql_query_with_io.rs index fc8caf31acd1..c6600e197374 100644 --- a/datafusion/core/benches/sql_query_with_io.rs +++ b/datafusion/core/benches/sql_query_with_io.rs @@ -124,8 +124,10 @@ async fn setup_context(object_store: Arc) -> SessionContext { let table_name = table_name(table_id); let file_format = ParquetFormat::default().with_enable_pruning(true); let options = ListingOptions::new(Arc::new(file_format)) - .with_table_partition_cols(vec![(String::from("partition"), DataType::UInt8)]) - .with_target_partitions(THREADS); + .with_table_partition_cols(vec![( + String::from("partition"), + DataType::UInt8, + )]); // make sure we actually find the data let path = format!("data://my_store/{table_name}/"); diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs index bd0ac3608738..f907d715b8f6 100644 --- a/datafusion/core/src/datasource/file_format/options.rs +++ b/datafusion/core/src/datasource/file_format/options.rs @@ -620,7 +620,7 @@ pub trait ReadOptions<'a> { impl ReadOptions<'_> for CsvReadOptions<'_> { fn to_listing_options( &self, - config: &SessionConfig, + _config: &SessionConfig, table_options: TableOptions, ) -> ListingOptions { let file_format = CsvFormat::default() @@ -639,7 +639,6 @@ impl ReadOptions<'_> for CsvReadOptions<'_> { ListingOptions::new(Arc::new(file_format)) .with_file_extension(self.file_extension) - .with_session_config_options(config) .with_table_partition_cols(self.table_partition_cols.clone()) .with_file_sort_order(self.file_sort_order.clone()) } @@ -660,7 +659,7 @@ impl ReadOptions<'_> for CsvReadOptions<'_> { impl ReadOptions<'_> for ParquetReadOptions<'_> { fn to_listing_options( &self, - config: &SessionConfig, + _config: &SessionConfig, table_options: TableOptions, ) -> ListingOptions { let mut options = table_options.parquet; @@ -685,7 +684,6 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> { .with_file_extension(self.file_extension) .with_table_partition_cols(self.table_partition_cols.clone()) .with_file_sort_order(self.file_sort_order.clone()) - .with_session_config_options(config) } async fn get_resolved_schema( @@ -703,7 +701,7 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> { impl ReadOptions<'_> for JsonReadOptions<'_> { fn to_listing_options( &self, - config: &SessionConfig, + _config: &SessionConfig, table_options: TableOptions, ) -> ListingOptions { let file_format = JsonFormat::default() @@ -714,7 +712,6 @@ impl ReadOptions<'_> for JsonReadOptions<'_> { ListingOptions::new(Arc::new(file_format)) .with_file_extension(self.file_extension) - .with_session_config_options(config) .with_table_partition_cols(self.table_partition_cols.clone()) .with_file_sort_order(self.file_sort_order.clone()) } @@ -735,14 +732,13 @@ impl ReadOptions<'_> for JsonReadOptions<'_> { impl ReadOptions<'_> for AvroReadOptions<'_> { fn to_listing_options( &self, - config: &SessionConfig, + _config: &SessionConfig, _table_options: TableOptions, ) -> ListingOptions { let file_format = AvroFormat; ListingOptions::new(Arc::new(file_format)) .with_file_extension(self.file_extension) - .with_session_config_options(config) .with_table_partition_cols(self.table_partition_cols.clone()) } @@ -761,14 +757,13 @@ impl ReadOptions<'_> for AvroReadOptions<'_> { impl ReadOptions<'_> for ArrowReadOptions<'_> { fn to_listing_options( &self, - config: &SessionConfig, + _config: &SessionConfig, _table_options: TableOptions, ) -> ListingOptions { let file_format = ArrowFormat; ListingOptions::new(Arc::new(file_format)) .with_file_extension(self.file_extension) - .with_session_config_options(config) .with_table_partition_cols(self.table_partition_cols.clone()) } diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index d14ec1f56dce..8c543fdd0af7 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -92,10 +92,8 @@ impl ListingTableConfigExt for ListingTableConfig { file_extension }; - let listing_options = ListingOptions::new(file_format) - .with_file_extension(listing_file_extension) - .with_target_partitions(state.config().target_partitions()) - .with_collect_stat(state.config().collect_statistics()); + let listing_options = + ListingOptions::new(file_format).with_file_extension(listing_file_extension); Ok(self.with_listing_options(listing_options)) } @@ -372,7 +370,9 @@ mod tests { #[tokio::test] async fn read_empty_table() -> Result<()> { - let ctx = SessionContext::new(); + let ctx = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(4), + ); let path = String::from("table/p1=v1/file.json"); register_test_store(&ctx, &[(&path, 100)]); @@ -381,8 +381,7 @@ mod tests { let opt = ListingOptions::new(Arc::new(format)) .with_file_extension(ext) - .with_table_partition_cols(vec![(String::from("p1"), DataType::Utf8)]) - .with_target_partitions(4); + .with_table_partition_cols(vec![(String::from("p1"), DataType::Utf8)]); let table_path = ListingTableUrl::parse("test:///table/")?; let file_schema = @@ -438,12 +437,13 @@ mod tests { output_partitioning: usize, file_ext: Option<&str>, ) -> Result<()> { - let ctx = SessionContext::new(); + let ctx = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(target_partitions), + ); register_test_store(&ctx, &files.iter().map(|f| (*f, 10)).collect::>()); let opt = ListingOptions::new(Arc::new(JsonFormat::default())) - .with_file_extension_opt(file_ext) - .with_target_partitions(target_partitions); + .with_file_extension_opt(file_ext); let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]); @@ -470,12 +470,13 @@ mod tests { output_partitioning: usize, file_ext: Option<&str>, ) -> Result<()> { - let ctx = SessionContext::new(); + let ctx = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(target_partitions), + ); register_test_store(&ctx, &files.iter().map(|f| (*f, 10)).collect::>()); let opt = ListingOptions::new(Arc::new(JsonFormat::default())) - .with_file_extension_opt(file_ext) - .with_target_partitions(target_partitions); + .with_file_extension_opt(file_ext); let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]); @@ -505,7 +506,9 @@ mod tests { output_partitioning: usize, file_ext: Option<&str>, ) -> Result<()> { - let ctx = SessionContext::new(); + let ctx = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(target_partitions), + ); let (store, _) = make_test_store_and_state( &files.iter().map(|f| (*f, 10)).collect::>(), ); @@ -523,9 +526,7 @@ mod tests { let format = JsonFormat::default(); - let opt = ListingOptions::new(Arc::new(format)) - .with_file_extension_opt(file_ext) - .with_target_partitions(target_partitions); + let opt = ListingOptions::new(Arc::new(format)).with_file_extension_opt(file_ext); let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]); @@ -1296,7 +1297,9 @@ mod tests { "bucket/test/other/file5", ]; - let ctx = SessionContext::new(); + let ctx = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(1), + ); register_test_store(&ctx, &files.iter().map(|f| (*f, 10)).collect::>()); let opt = ListingOptions::new(Arc::new(JsonFormat::default())) @@ -1341,10 +1344,12 @@ mod tests { let filename = format!("{}/{}", testdata, "alltypes_plain.parquet"); let table_path = ListingTableUrl::parse(filename)?; - let ctx = SessionContext::new(); - let state = ctx.state(); + let ctx = SessionContext::new_with_config( + SessionConfig::new().with_target_partitions(1), + ); + let mut state = ctx.state(); - // Test 1: Default behavior - stats not collected + // Test 1: Default behavior - stats collected let opt_default = ListingOptions::new(Arc::new(ParquetFormat::default())); let schema_default = opt_default.infer_schema(&state, &table_path).await?; let config_default = ListingTableConfig::new(table_path.clone()) @@ -1356,7 +1361,7 @@ mod tests { let exec_default = table_default.scan(&state, None, &[], None).await?; assert_eq!( exec_default.partition_statistics(None)?.num_rows, - Precision::Absent + Precision::Exact(8) ); // TODO correct byte size: https://github.com/apache/datafusion/issues/14936 @@ -1365,12 +1370,13 @@ mod tests { Precision::Absent ); - // Test 2: Explicitly disable stats - let opt_disabled = ListingOptions::new(Arc::new(ParquetFormat::default())) - .with_collect_stat(false); - let schema_disabled = opt_disabled.infer_schema(&state, &table_path).await?; + // Test 2: Explicitly disable stats via session config + let cfg = state.config_mut(); + cfg.options_mut().execution.collect_statistics = false; + let opt = ListingOptions::new(Arc::new(ParquetFormat::default())); + let schema_disabled = opt.infer_schema(&state, &table_path).await?; let config_disabled = ListingTableConfig::new(table_path.clone()) - .with_listing_options(opt_disabled) + .with_listing_options(opt) .with_schema(schema_disabled); let table_disabled = ListingTable::try_new(config_disabled)?; @@ -1384,12 +1390,13 @@ mod tests { Precision::Absent ); - // Test 3: Explicitly enable stats - let opt_enabled = ListingOptions::new(Arc::new(ParquetFormat::default())) - .with_collect_stat(true); - let schema_enabled = opt_enabled.infer_schema(&state, &table_path).await?; + // Test 3: Re-enable stats via session config + let cfg = state.config_mut(); + cfg.options_mut().execution.collect_statistics = true; + let opt = ListingOptions::new(Arc::new(ParquetFormat::default())); + let schema_enabled = opt.infer_schema(&state, &table_path).await?; let config_enabled = ListingTableConfig::new(table_path) - .with_listing_options(opt_enabled) + .with_listing_options(opt) .with_schema(schema_enabled); let table_enabled = ListingTable::try_new(config_enabled)?; @@ -1456,14 +1463,16 @@ mod tests { #[tokio::test] async fn test_basic_table_scan() -> Result<()> { - let ctx = SessionContext::new(); + let ctx = SessionContext::new_with_config( + SessionConfig::new().with_collect_statistics(false), + ); // Test basic table creation and scanning let path = "table/file.json"; register_test_store(&ctx, &[(path, 10)]); let format = JsonFormat::default(); - let opt = ListingOptions::new(Arc::new(format)).with_collect_stat(false); + let opt = ListingOptions::new(Arc::new(format)); let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]); let table_path = ListingTableUrl::parse("test:///table/")?; diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rs index 349d941cc2bd..34ee6ce53f92 100644 --- a/datafusion/core/src/datasource/listing_table_factory.rs +++ b/datafusion/core/src/datasource/listing_table_factory.rs @@ -81,9 +81,8 @@ impl TableProviderFactory for ListingTableFactory { true => "", false => &get_extension(cmd.location.as_str()), }; - let mut options = ListingOptions::new(file_format) - .with_session_config_options(session_state.config()) - .with_file_extension(file_extension); + let mut options = + ListingOptions::new(file_format).with_file_extension(file_extension); let (provided_schema, table_partition_cols) = if cmd.schema.fields().is_empty() { let infer_parts = session_state diff --git a/datafusion/core/tests/parquet/file_statistics.rs b/datafusion/core/tests/parquet/file_statistics.rs index 3e3b90a348b0..4cca3ae17e1d 100644 --- a/datafusion/core/tests/parquet/file_statistics.rs +++ b/datafusion/core/tests/parquet/file_statistics.rs @@ -52,13 +52,13 @@ async fn check_stats_precision_with_filter_pushdown() { let filename = format!("{}/{}", testdata, "alltypes_plain.parquet"); let table_path = ListingTableUrl::parse(filename).unwrap(); - let opt = - ListingOptions::new(Arc::new(ParquetFormat::default())).with_collect_stat(true); + let opt = ListingOptions::new(Arc::new(ParquetFormat::default())); let table = get_listing_table(&table_path, None, &opt).await; let (_, _, state) = get_cache_runtime_state(); let mut options: ConfigOptions = state.config().options().as_ref().clone(); options.execution.parquet.pushdown_filters = true; + options.execution.collect_statistics = true; // Scan without filter, stats are exact let exec = table.scan(&state, None, &[], None).await.unwrap(); @@ -107,13 +107,16 @@ async fn load_table_stats_with_session_level_cache() { let filename = format!("{}/{}", testdata, "alltypes_plain.parquet"); let table_path = ListingTableUrl::parse(filename).unwrap(); - let (cache1, _, state1) = get_cache_runtime_state(); + let (cache1, _, mut state1) = get_cache_runtime_state(); + let cfg_1 = state1.config_mut(); + cfg_1.options_mut().execution.collect_statistics = true; // Create a separate DefaultFileStatisticsCache - let (cache2, _, state2) = get_cache_runtime_state(); + let (cache2, _, mut state2) = get_cache_runtime_state(); + let cfg_2 = state2.config_mut(); + cfg_2.options_mut().execution.collect_statistics = true; - let opt = - ListingOptions::new(Arc::new(ParquetFormat::default())).with_collect_stat(true); + let opt = ListingOptions::new(Arc::new(ParquetFormat::default())); let table1 = get_listing_table(&table_path, Some(cache1), &opt).await; let table2 = get_listing_table(&table_path, Some(cache2), &opt).await; diff --git a/datafusion/core/tests/sql/path_partition.rs b/datafusion/core/tests/sql/path_partition.rs index 2eff1c262f85..20f49de8aa10 100644 --- a/datafusion/core/tests/sql/path_partition.rs +++ b/datafusion/core/tests/sql/path_partition.rs @@ -609,8 +609,7 @@ async fn create_partitioned_alltypes_parquet_table( .iter() .map(|x| (x.0.to_owned(), x.1.clone())) .collect::>(), - ) - .with_session_config_options(&ctx.copied_config()); + ); let table_path = ListingTableUrl::parse(table_path).unwrap(); let store_path = diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index f13494cf4383..2f5b75e40937 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -92,8 +92,8 @@ message ListingTableScanNode { datafusion_common.Schema schema = 5; repeated LogicalExprNode filters = 6; repeated PartitionColumn table_partition_cols = 7; - bool collect_stat = 8; - uint32 target_partitions = 9; + reserved 8; // was bool collect_stat + reserved 9; // was uint32 target_partitions oneof FileFormatType { datafusion_common.CsvFormat csv = 10; datafusion_common.ParquetFormat parquet = 11; @@ -303,7 +303,7 @@ message FileFormatProto { } message DmlNode{ - enum Type { + enum Type { UPDATE = 0; DELETE = 1; CTAS = 2; @@ -1512,15 +1512,15 @@ message RecursiveQueryNode { } message CteWorkTableScanNode { - string name = 1; - datafusion_common.Schema schema = 2; + string name = 1; + datafusion_common.Schema schema = 2; } message EmptyTableScanNode { - TableReference table_name = 1; - datafusion_common.Schema schema = 2; - ProjectionColumns projection = 3; - repeated LogicalExprNode filters = 4; + TableReference table_name = 1; + datafusion_common.Schema schema = 2; + ProjectionColumns projection = 3; + repeated LogicalExprNode filters = 4; } enum GenerateSeriesName { @@ -1529,44 +1529,44 @@ enum GenerateSeriesName { } message GenerateSeriesArgsContainsNull { - GenerateSeriesName name = 1; + GenerateSeriesName name = 1; } message GenerateSeriesArgsInt64 { - int64 start = 1; - int64 end = 2; - int64 step = 3; - bool include_end = 4; - GenerateSeriesName name = 5; + int64 start = 1; + int64 end = 2; + int64 step = 3; + bool include_end = 4; + GenerateSeriesName name = 5; } message GenerateSeriesArgsTimestamp { - int64 start = 1; - int64 end = 2; - datafusion_common.IntervalMonthDayNanoValue step = 3; - optional string tz = 4; - bool include_end = 5; - GenerateSeriesName name = 6; + int64 start = 1; + int64 end = 2; + datafusion_common.IntervalMonthDayNanoValue step = 3; + optional string tz = 4; + bool include_end = 5; + GenerateSeriesName name = 6; } message GenerateSeriesArgsDate { - int64 start = 1; - int64 end = 2; - datafusion_common.IntervalMonthDayNanoValue step = 3; - bool include_end = 4; - GenerateSeriesName name = 5; + int64 start = 1; + int64 end = 2; + datafusion_common.IntervalMonthDayNanoValue step = 3; + bool include_end = 4; + GenerateSeriesName name = 5; } message GenerateSeriesNode { - datafusion_common.Schema schema = 1; - uint32 target_batch_size = 2; - - oneof args { - GenerateSeriesArgsContainsNull contains_null = 3; - GenerateSeriesArgsInt64 int64_args = 4; - GenerateSeriesArgsTimestamp timestamp_args = 5; - GenerateSeriesArgsDate date_args = 6; - } + datafusion_common.Schema schema = 1; + uint32 target_batch_size = 2; + + oneof args { + GenerateSeriesArgsContainsNull contains_null = 3; + GenerateSeriesArgsInt64 int64_args = 4; + GenerateSeriesArgsTimestamp timestamp_args = 5; + GenerateSeriesArgsDate date_args = 6; + } } message SortMergeJoinExecNode { diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index 2ba5e2505425..733da68fe89c 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -11937,12 +11937,6 @@ impl serde::Serialize for ListingTableScanNode { if !self.table_partition_cols.is_empty() { len += 1; } - if self.collect_stat { - len += 1; - } - if self.target_partitions != 0 { - len += 1; - } if !self.file_sort_order.is_empty() { len += 1; } @@ -11971,12 +11965,6 @@ impl serde::Serialize for ListingTableScanNode { if !self.table_partition_cols.is_empty() { struct_ser.serialize_field("tablePartitionCols", &self.table_partition_cols)?; } - if self.collect_stat { - struct_ser.serialize_field("collectStat", &self.collect_stat)?; - } - if self.target_partitions != 0 { - struct_ser.serialize_field("targetPartitions", &self.target_partitions)?; - } if !self.file_sort_order.is_empty() { struct_ser.serialize_field("fileSortOrder", &self.file_sort_order)?; } @@ -12019,10 +12007,6 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { "filters", "table_partition_cols", "tablePartitionCols", - "collect_stat", - "collectStat", - "target_partitions", - "targetPartitions", "file_sort_order", "fileSortOrder", "csv", @@ -12041,8 +12025,6 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { Schema, Filters, TablePartitionCols, - CollectStat, - TargetPartitions, FileSortOrder, Csv, Parquet, @@ -12077,8 +12059,6 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { "schema" => Ok(GeneratedField::Schema), "filters" => Ok(GeneratedField::Filters), "tablePartitionCols" | "table_partition_cols" => Ok(GeneratedField::TablePartitionCols), - "collectStat" | "collect_stat" => Ok(GeneratedField::CollectStat), - "targetPartitions" | "target_partitions" => Ok(GeneratedField::TargetPartitions), "fileSortOrder" | "file_sort_order" => Ok(GeneratedField::FileSortOrder), "csv" => Ok(GeneratedField::Csv), "parquet" => Ok(GeneratedField::Parquet), @@ -12111,8 +12091,6 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { let mut schema__ = None; let mut filters__ = None; let mut table_partition_cols__ = None; - let mut collect_stat__ = None; - let mut target_partitions__ = None; let mut file_sort_order__ = None; let mut file_format_type__ = None; while let Some(k) = map_.next_key()? { @@ -12159,20 +12137,6 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { } table_partition_cols__ = Some(map_.next_value()?); } - GeneratedField::CollectStat => { - if collect_stat__.is_some() { - return Err(serde::de::Error::duplicate_field("collectStat")); - } - collect_stat__ = Some(map_.next_value()?); - } - GeneratedField::TargetPartitions => { - if target_partitions__.is_some() { - return Err(serde::de::Error::duplicate_field("targetPartitions")); - } - target_partitions__ = - Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) - ; - } GeneratedField::FileSortOrder => { if file_sort_order__.is_some() { return Err(serde::de::Error::duplicate_field("fileSortOrder")); @@ -12224,8 +12188,6 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { schema: schema__, filters: filters__.unwrap_or_default(), table_partition_cols: table_partition_cols__.unwrap_or_default(), - collect_stat: collect_stat__.unwrap_or_default(), - target_partitions: target_partitions__.unwrap_or_default(), file_sort_order: file_sort_order__.unwrap_or_default(), file_format_type: file_format_type__, }) diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index e8fa4599e1f9..4a2edeeb11ec 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -119,10 +119,6 @@ pub struct ListingTableScanNode { pub filters: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "7")] pub table_partition_cols: ::prost::alloc::vec::Vec, - #[prost(bool, tag = "8")] - pub collect_stat: bool, - #[prost(uint32, tag = "9")] - pub target_partitions: u32, #[prost(message, repeated, tag = "13")] pub file_sort_order: ::prost::alloc::vec::Vec, #[prost( diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs index 0e73898c67d8..a0604cb6b03e 100644 --- a/datafusion/proto/src/logical_plan/mod.rs +++ b/datafusion/proto/src/logical_plan/mod.rs @@ -647,8 +647,6 @@ impl AsLogicalPlan for LogicalPlanNode { let options = ListingOptions::new(file_format) .with_file_extension(&scan.file_extension) .with_table_partition_cols(partition_columns) - .with_collect_stat(scan.collect_stat) - .with_target_partitions(scan.target_partitions as usize) .with_file_sort_order(all_sort_orders); let config = @@ -1412,7 +1410,6 @@ impl AsLogicalPlan for LogicalPlanNode { table_name: Some(protobuf::TableReference::from_proto( table_name.clone(), )), - collect_stat: options.collect_stat, file_extension: options.file_extension.clone(), table_partition_cols: partition_columns, paths: listing_table @@ -1423,7 +1420,6 @@ impl AsLogicalPlan for LogicalPlanNode { schema: Some(schema), projection, filters, - target_partitions: options.target_partitions as u32, file_sort_order: exprs_vec, }, )), diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 04ee70b963ce..430d2935157b 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -373,7 +373,7 @@ datafusion.catalog.location NULL Location scanned to load tables for `default` s datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting -datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to true. +datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. datafusion.execution.enable_ansi_mode false Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. datafusion.execution.enable_migration_aggregate true Temporary switch for aggregate stream implementations that are being migrated from `GroupedHashAggregateStream`. When set to true, DataFusion tries the migrated implementations when their preconditions are satisfied. When set to false, grouped aggregation falls back to `GroupedHashAggregateStream`. This option will be removed after the migration is finished. See for details. datafusion.execution.enable_recursive_ctes true Should DataFusion support recursive CTEs diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index 090f44f5628f..22aaf09dff31 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -6084,7 +6084,7 @@ physical_plan 03)----BoundedWindowAggExec: wdw=[sum(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, sum(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, count(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "count(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable List(Int64) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable List(Int64) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 04)------SortPreservingMergeExec: [c1@2 ASC NULLS LAST, c2@3 ASC NULLS LAST], fetch=5 05)--------SortExec: TopK(fetch=5), expr=[c1@2 ASC NULLS LAST, c2@3 ASC NULLS LAST], preserve_partitioning=[true] -06)----------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-0.csv], [WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-1.csv], [WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-2.csv], [WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-3.csv]]}, projection=[c2@1 >= 2 as __common_expr_1, c2@1 >= 2 AND c2@1 < 4 AND c1@0 > 0 as __common_expr_2, c1, c2], file_type=csv, has_header=false +06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-0.csv, WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-1.csv], [WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-2.csv, WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-3.csv]]}, projection=[c2@1 >= 2 as __common_expr_1, c2@1 >= 2 AND c2@1 < 4 AND c1@0 > 0 as __common_expr_2, c1, c2], file_type=csv, has_header=false # FILTER filters out some rows diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md index abe99846c12d..0856f3dc4847 100644 --- a/docs/source/library-user-guide/upgrading/55.0.0.md +++ b/docs/source/library-user-guide/upgrading/55.0.0.md @@ -260,6 +260,43 @@ See [PR #22733](https://github.com/apache/datafusion/pull/22733) for details, including the per-variant size breakdown and benchmark results. +### `ListingOptions::target_partitions` and `collect_stat` removed + +The `target_partitions` and `collect_stat` fields on +`datafusion_catalog_listing::ListingOptions`, their builder methods +(`with_target_partitions`, `with_collect_stat`), and the +`with_session_config_options` helper have been removed. + +`ListingTable` now reads both values directly from the active `SessionConfig` +at scan time instead of from a copy snapshotted onto the table at construction +time. + +**Who is affected:** + +- Code that set `target_partitions` / `collect_stat` per table via + `ListingOptions`, or read those public fields. +- Code that relied on a `ListingTable` freezing these values at construction + time independently of the session config. The table now always reflects the + current `SessionConfig`. + +**Migration guide:** + +Configure these on the `SessionConfig` instead: + +```rust,ignore +// Before +let options = ListingOptions::new(format) + .with_target_partitions(8) + .with_collect_stat(true); + +// After +let config = SessionConfig::new() + .with_target_partitions(8) + .with_collect_statistics(true); +``` + +See [PR #22969](https://github.com/apache/datafusion/pull/22969) for details. + ### Spark map functions now reject duplicate keys by default The Spark-compatibility map-construction functions (`map_from_arrays`, diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index abf1c39510e9..f70daef31721 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -77,7 +77,7 @@ The following configuration settings are available: | datafusion.execution.perfect_hash_join_small_build_threshold | 1024 | A perfect hash join (see `HashJoinExec` for more details) will be considered if the range of keys (max - min) on the build side is < this threshold. This provides a fast path for joins with very small key ranges, bypassing the density check. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | | datafusion.execution.perfect_hash_join_min_key_density | 0.15 | The minimum required density of join keys on the build side to consider a perfect hash join (see `HashJoinExec` for more details). Density is calculated as: `(number of rows) / (max_key - min_key + 1)`. A perfect hash join may be used if the actual key density > this value. Currently only supports cases where build_side.num_rows() < u32::MAX. Support for build_side.num_rows() >= u32::MAX will be added in the future. | | datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | -| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to true. | +| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. | | datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | | datafusion.execution.time_zone | NULL | The default time zone Some functions, e.g. `now` return timestamps in this time zone | | datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. |