From 9f6fea6529bb0d34a88346034b0061e28a19090b Mon Sep 17 00:00:00 2001 From: Shaoyu Zhang Date: Thu, 23 Apr 2026 15:37:15 -0700 Subject: [PATCH 1/9] Hook Azure metadata cache into file info lookups Co-authored-by: Shaoyu Zhang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/azure_blob_filesystem.cpp | 25 ++++-- src/azure_dfs_filesystem.cpp | 23 +++-- src/azure_filesystem.cpp | 116 +++++++++++++++++++++++++- src/include/azure_blob_filesystem.hpp | 3 +- src/include/azure_dfs_filesystem.hpp | 3 +- src/include/azure_filesystem.hpp | 85 ++++++++++++++++++- test/sql/azure.test | 6 +- 7 files changed, 236 insertions(+), 25 deletions(-) diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp index 0c84853..3cdd5dd 100644 --- a/src/azure_blob_filesystem.cpp +++ b/src/azure_blob_filesystem.cpp @@ -75,8 +75,10 @@ AzureBlobContextState::GetBlobContainerClient(const std::string &blobContainerNa //////// AzureBlobStorageFileHandle //////// AzureBlobStorageFileHandle::AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, const AzureOptions &opts, + optional_ptr metadata_cache, Azure::Storage::Blobs::BlockBlobClient blob_client) - : AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, opts), blob_client(std::move(blob_client)) { + : AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, opts, metadata_cache), + blob_client(std::move(blob_client)) { } void AzureBlobStorageFileHandle::StageWriteBuffer() { @@ -138,11 +140,14 @@ unique_ptr AzureBlobStorageFileSystem::CreateHandle(const OpenF auto parsed_url = ParseUrl(info.path); auto storage_context = GetOrCreateStorageContext(opener, info.path, parsed_url); + if (flags.OpenForWriting() || flags.OpenForAppending()) { + InvalidateMetadata(opener, info.path); + } auto container = storage_context->As().GetBlobContainerClient(parsed_url.container); auto blob_client = container.GetBlockBlobClient(parsed_url.path); - auto handle = - make_uniq(*this, info, flags, storage_context->options, std::move(blob_client)); + auto handle = make_uniq(*this, info, flags, storage_context->options, + GetMetadataCache(opener), std::move(blob_client)); if (!handle->PostConstruct()) { return nullptr; } @@ -207,6 +212,7 @@ vector AzureBlobStorageFileSystem::Glob(const string &path, FileOp OpenFileInfo info(result_full_url); info.extended_info = make_shared_ptr(); auto &options = info.extended_info->options; + options.emplace("type", Value("file")); options.emplace("file_size", Value::BIGINT(key.BlobSize)); options.emplace("last_modified", Value::TIMESTAMP(ToTimestamp(key.Details.LastModified))); result.push_back(info); @@ -297,11 +303,7 @@ void AzureBlobStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) { // - doesn't exist, don't create auto set_props = [&](bool is_dir, idx_t length, timestamp_t last_mod) { - afh.is_remote_loaded = true; // always set loaded - afh.file_type = is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR; - afh.length = is_dir ? 0 : length; - afh.last_modified = last_mod; - afh.file_offset = 0; // always reset offset state + afh.SetFileInfo(is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR, length, last_mod); }; auto create_file = [&]() { @@ -382,6 +384,7 @@ void AzureBlobStorageFileSystem::RemoveFile(const string &filename, optional_ptr auto blob_client = container.GetBlockBlobClient(url.path); try { blob_client.Delete(); + InvalidateMetadata(opener, filename); } catch (Azure::Storage::StorageException &e) { throw IOException("AzureBlobStorageFileSystem Delete of %s failed with %s Reason Phrase: %s", filename, e.ErrorCode, e.ReasonPhrase); @@ -393,7 +396,11 @@ bool AzureBlobStorageFileSystem::TryRemoveFile(const string &filename, optional_ auto storage_context = GetOrCreateStorageContext(opener, filename, url); auto container = storage_context->As().GetBlobContainerClient(url.container); auto blob_client = container.GetBlockBlobClient(url.path); - return blob_client.DeleteIfExists().Value.Deleted; + auto removed = blob_client.DeleteIfExists().Value.Deleted; + if (removed) { + InvalidateMetadata(opener, filename); + } + return removed; } void AzureBlobStorageFileSystem::ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out, diff --git a/src/azure_dfs_filesystem.cpp b/src/azure_dfs_filesystem.cpp index 1e7ec83..629ad67 100644 --- a/src/azure_dfs_filesystem.cpp +++ b/src/azure_dfs_filesystem.cpp @@ -73,6 +73,7 @@ static void Walk(const Azure::Storage::Files::DataLake::DataLakeFileSystemClient OpenFileInfo info(elt.Name); info.extended_info = make_shared_ptr(); auto &options = info.extended_info->options; + options.emplace("type", Value("file")); options.emplace("file_size", Value::BIGINT(elt.FileSize)); options.emplace("last_modified", Value::TIMESTAMP(AzureStorageFileSystem::ToTimestamp(elt.LastModified))); @@ -103,8 +104,10 @@ AzureDfsContextState::GetDfsFileSystemClient(const std::string &file_system_name //////// AzureDfsContextState //////// AzureDfsStorageFileHandle::AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, const AzureOptions &options, + optional_ptr metadata_cache, Azure::Storage::Files::DataLake::DataLakeFileClient client) - : AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, options), file_client(std::move(client)) { + : AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, options, metadata_cache), + file_client(std::move(client)) { } void AzureDfsStorageFileHandle::StageWriteBuffer() { @@ -161,6 +164,9 @@ unique_ptr AzureDfsStorageFileSystem::CreateHandle(const OpenFi auto parsed_url = ParseUrl(info.path); auto storage_context = GetOrCreateStorageContext(opener, info.path, parsed_url); + if (flags.OpenForWriting() || flags.OpenForAppending()) { + InvalidateMetadata(opener, info.path); + } auto file_system_client = storage_context->As().GetDfsFileSystemClient(parsed_url.container); // A trailing '/' is only a directory hint, not part of the resource name. Some DFS endpoints (e.g. OneLake) @@ -171,6 +177,7 @@ unique_ptr AzureDfsStorageFileSystem::CreateHandle(const OpenFi } auto handle = make_uniq(*this, info, flags, storage_context->options, + GetMetadataCache(opener), file_system_client.GetFileClient(file_path)); if (!handle->PostConstruct()) { return nullptr; @@ -195,6 +202,7 @@ void AzureDfsStorageFileSystem::CreateDirectory(const string &dirname, optional_ auto storage_context = GetOrCreateStorageContext(opener, dirname, dir_url); auto file_system_client = storage_context->As().GetDfsFileSystemClient(dir_url.container); file_system_client.GetDirectoryClient(dir_url.path).Create(); + InvalidateMetadata(opener, dirname); } bool AzureDfsStorageFileSystem::FileExists(const string &filename, optional_ptr opener) { @@ -209,6 +217,7 @@ void AzureDfsStorageFileSystem::RemoveFile(const string &filename, optional_ptr< auto file_client = file_system_client.GetFileClient(url.path); try { file_client.Delete(); + InvalidateMetadata(opener, filename); } catch (Azure::Storage::StorageException &e) { throw IOException("AzureDfsStorageFileSystem Delete of %s failed with %s Reason Phrase: %s", filename, e.ErrorCode, e.ReasonPhrase); @@ -220,7 +229,11 @@ bool AzureDfsStorageFileSystem::TryRemoveFile(const string &filename, optional_p auto storage_context = GetOrCreateStorageContext(opener, filename, url); auto file_system_client = storage_context->As().GetDfsFileSystemClient(url.container); auto file_client = file_system_client.GetFileClient(url.path); - return file_client.DeleteIfExists().Value.Deleted; + auto removed = file_client.DeleteIfExists().Value.Deleted; + if (removed) { + InvalidateMetadata(opener, filename); + } + return removed; } vector AzureDfsStorageFileSystem::Glob(const string &path, FileOpener *opener) { @@ -323,11 +336,7 @@ void AzureDfsStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) { // - doesn't exist, don't create auto set_props = [&](bool is_dir, idx_t length, timestamp_t last_mod) { - afh.is_remote_loaded = true; // always set loaded - afh.file_type = is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR; - afh.length = is_dir ? 0 : length; - afh.last_modified = last_mod; - afh.file_offset = 0; // always reset offset state + afh.SetFileInfo(is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR, length, last_mod); }; auto create_file = [&]() { diff --git a/src/azure_filesystem.cpp b/src/azure_filesystem.cpp index 8b539e3..4635feb 100644 --- a/src/azure_filesystem.cpp +++ b/src/azure_filesystem.cpp @@ -24,39 +24,151 @@ void AzureContextState::QueryEnd() { is_valid = false; } +static string GetFileType(const Value &value) { + auto type = value.ToString(); + if (!type.empty() && type.front() == '\'' && type.back() == '\'' && type.size() >= 2) { + type = type.substr(1, type.size() - 2); + } + return type; +} + AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, - FileType file_type, const AzureOptions &options) + FileType file_type, const AzureOptions &options, + optional_ptr metadata_cache_p) : FileHandle(fs, info.path, flags), flags(flags), // File info is_remote_loaded(false), file_type(file_type), length(0), last_modified(0), // Read info buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0), // Options - options(options) { + options(options), metadata_cache(metadata_cache_p) { if (!flags.RequireParallelAccess() && !flags.DirectIO()) { read_buffer = duckdb::unique_ptr(new data_t[options.read_buffer_size]); } // Set metadata of file when available, it avoids to invoke to the storage to get them. + bool has_file_type = file_type != FileType::FILE_TYPE_INVALID; + bool has_length = false; + bool has_last_modified = false; if (info.extended_info) { + auto type_entry = info.extended_info->options.find("type"); + if (type_entry != info.extended_info->options.end()) { + auto type = GetFileType(type_entry->second); + if (type == "directory") { + file_type = FileType::FILE_TYPE_DIR; + has_file_type = true; + } else if (type == "file") { + file_type = FileType::FILE_TYPE_REGULAR; + has_file_type = true; + } + } auto entry1 = info.extended_info->options.find("file_size"); if (entry1 != info.extended_info->options.end()) { length = entry1->second.GetValue(); + has_length = true; } auto entry2 = info.extended_info->options.find("last_modified"); if (entry2 != info.extended_info->options.end()) { last_modified = entry2->second.GetValue(); + has_last_modified = true; } } + if (has_file_type && has_last_modified && (file_type == FileType::FILE_TYPE_DIR || has_length)) { + SetFileInfo(file_type, length, last_modified); + } +} + +void AzureFileHandle::SetFileInfo(FileType file_type_p, idx_t length_p, timestamp_t last_modified_p) { + is_remote_loaded = true; + file_type = file_type_p; + length = file_type_p == FileType::FILE_TYPE_DIR ? 0 : length_p; + last_modified = last_modified_p; + file_offset = 0; } bool AzureFileHandle::PostConstruct() { return static_cast(file_system).LoadFileInfo(*this); } +static bool CanUseMetadataCache(const AzureFileHandle &handle) { + return handle.metadata_cache && !handle.flags.OpenForWriting() && !handle.flags.OpenForAppending() && + !handle.flags.ExclusiveCreate(); +} + +static AzureFileInfo GetCacheEntry(const AzureFileHandle &handle) { + AzureFileInfo info; + info.file_type = handle.file_type; + info.length = handle.length; + info.last_modified = handle.last_modified; + return info; +} + +bool AzureStorageFileSystem::ParseAzureMetadataCacheEnabled(optional_ptr opener) { + Value metadata_cache_val; + if (FileOpener::TryGetCurrentSetting(opener, "enable_http_metadata_cache", metadata_cache_val) && + !metadata_cache_val.IsNull()) { + return metadata_cache_val.GetValue(); + } + return false; +} + +optional_ptr AzureStorageFileSystem::GetGlobalMetadataCache() { + lock_guard lock(global_cache_lock); + if (!global_metadata_cache) { + global_metadata_cache = make_uniq(false, true); + } + return global_metadata_cache.get(); +} + +optional_ptr AzureStorageFileSystem::GetMetadataCache(optional_ptr opener) { + auto db = FileOpener::TryGetDatabase(opener); + auto client_context = FileOpener::TryGetClientContext(opener); + if (!db) { + return nullptr; + } + if (ParseAzureMetadataCacheEnabled(opener)) { + return GetGlobalMetadataCache(); + } + if (client_context) { + return client_context->registered_state->GetOrCreate("azure_metadata_cache", true, false) + .get(); + } + return nullptr; +} + +void AzureStorageFileSystem::InvalidateMetadata(optional_ptr opener, const string &path) { + auto metadata_cache = GetMetadataCache(opener); + if (metadata_cache) { + metadata_cache->Erase(path); + } +} + bool AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { try { + if (handle.IsRemoteLoaded()) { + if (CanUseMetadataCache(handle)) { + handle.metadata_cache->Insert(handle.path, GetCacheEntry(handle)); + } + if (handle.flags.ReturnNullIfExists()) { + return false; + } + return true; + } + if (CanUseMetadataCache(handle)) { + AzureFileInfo cached_info; + if (handle.metadata_cache->Find(handle.path, cached_info)) { + handle.SetFileInfo(cached_info.file_type, cached_info.length, cached_info.last_modified); + if (handle.flags.ReturnNullIfExists()) { + return false; + } + return true; + } + } + LoadRemoteFileInfo(handle); + if (CanUseMetadataCache(handle)) { + handle.metadata_cache->Insert(handle.path, GetCacheEntry(handle)); + } if (handle.flags.ReturnNullIfExists()) { return false; } diff --git a/src/include/azure_blob_filesystem.hpp b/src/include/azure_blob_filesystem.hpp index 67bfebd..77e8a8a 100644 --- a/src/include/azure_blob_filesystem.hpp +++ b/src/include/azure_blob_filesystem.hpp @@ -28,7 +28,8 @@ class AzureBlobStorageFileSystem; class AzureBlobStorageFileHandle : public AzureFileHandle { public: AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, - const AzureOptions &options, Azure::Storage::Blobs::BlockBlobClient blob_client); + const AzureOptions &options, optional_ptr metadata_cache, + Azure::Storage::Blobs::BlockBlobClient blob_client); ~AzureBlobStorageFileHandle() override = default; void StageWriteBuffer(); diff --git a/src/include/azure_dfs_filesystem.hpp b/src/include/azure_dfs_filesystem.hpp index c28417d..99bd3ee 100644 --- a/src/include/azure_dfs_filesystem.hpp +++ b/src/include/azure_dfs_filesystem.hpp @@ -28,7 +28,8 @@ class AzureDfsStorageFileSystem; class AzureDfsStorageFileHandle : public AzureFileHandle { public: AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, - const AzureOptions &options, Azure::Storage::Files::DataLake::DataLakeFileClient client); + const AzureOptions &options, optional_ptr metadata_cache, + Azure::Storage::Files::DataLake::DataLakeFileClient client); ~AzureDfsStorageFileHandle() override = default; void StageWriteBuffer(); diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp index b846bb8..40682e0 100644 --- a/src/include/azure_filesystem.hpp +++ b/src/include/azure_filesystem.hpp @@ -4,7 +4,9 @@ #include "duckdb/common/assert.hpp" #include "duckdb/common/file_opener.hpp" #include "duckdb/common/file_system.hpp" +#include "duckdb/common/mutex.hpp" #include "duckdb/common/shared_ptr.hpp" +#include "duckdb/common/unordered_map.hpp" #include "duckdb/logging/file_system_logger.hpp" #include "duckdb/main/client_context_state.hpp" @@ -27,6 +29,76 @@ struct AzureOptions { idx_t write_staged_blocks_per_commit = 0; }; +struct AzureFileInfo { + FileType file_type = FileType::FILE_TYPE_INVALID; + idx_t length = 0; + timestamp_t last_modified; +}; + +class AzureMetadataCache : public ClientContextState { +public: + explicit AzureMetadataCache(bool flush_on_query_end_p, bool shared_p) + : flush_on_query_end(flush_on_query_end_p), shared(shared_p) { + } + + void Insert(const string &path, const AzureFileInfo &val) { + if (shared) { + lock_guard parallel_lock(lock); + map[path] = val; + } else { + map[path] = val; + } + } + + void Erase(const string &path) { + if (shared) { + lock_guard parallel_lock(lock); + map.erase(path); + } else { + map.erase(path); + } + } + + bool Find(const string &path, AzureFileInfo &ret_val) { + if (shared) { + lock_guard parallel_lock(lock); + auto lookup = map.find(path); + if (lookup == map.end()) { + return false; + } + ret_val = lookup->second; + return true; + } + auto lookup = map.find(path); + if (lookup == map.end()) { + return false; + } + ret_val = lookup->second; + return true; + } + + void Clear() { + if (shared) { + lock_guard parallel_lock(lock); + map.clear(); + } else { + map.clear(); + } + } + + void QueryEnd(ClientContext &context) override { + if (flush_on_query_end) { + Clear(); + } + } + +private: + mutex lock; + unordered_map map; + bool flush_on_query_end; + bool shared; +}; + class AzureContextState : public ClientContextState { public: const AzureOptions options; @@ -58,6 +130,7 @@ class AzureStorageFileSystem; class AzureFileHandle : public FileHandle { public: virtual bool PostConstruct(); + void SetFileInfo(FileType file_type_p, idx_t length_p, timestamp_t last_modified_p); bool IsRemoteLoaded() { return is_remote_loaded; @@ -69,7 +142,7 @@ class AzureFileHandle : public FileHandle { protected: AzureFileHandle(AzureStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, FileType file_type, - const AzureOptions &options); + const AzureOptions &options, optional_ptr metadata_cache); public: FileOpenFlags flags; @@ -88,8 +161,8 @@ class AzureFileHandle : public FileHandle { idx_t file_offset; idx_t buffer_start; idx_t buffer_end; - const AzureOptions options; + optional_ptr metadata_cache; }; class AzureStorageFileSystem : public FileSystem { @@ -140,9 +213,17 @@ class AzureStorageFileSystem : public FileSystem { virtual void LoadRemoteFileInfo(AzureFileHandle &handle) = 0; static AzureOptions ParseAzureOptions(optional_ptr opener); + static bool ParseAzureMetadataCacheEnabled(optional_ptr opener); + optional_ptr GetMetadataCache(optional_ptr opener); + void InvalidateMetadata(optional_ptr opener, const string &path); public: static timestamp_t ToTimestamp(const Azure::DateTime &dt); + +private: + optional_ptr GetGlobalMetadataCache(); + mutex global_cache_lock; + duckdb::unique_ptr global_metadata_cache; }; } // namespace duckdb diff --git a/test/sql/azure.test b/test/sql/azure.test index c0d94b1..357ab68 100644 --- a/test/sql/azure.test +++ b/test/sql/azure.test @@ -51,17 +51,17 @@ SET azure_http_stats = true; query II EXPLAIN ANALYZE SELECT sum(l_orderkey) FROM 'az://testing-private/l.parquet'; ---- -analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 3.*GET\: 0.*PUT\: 0.*\#POST\: 0.* +analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: 0.*PUT\: 0.*\#POST\: 0.* # Redoing query should still result in same request count query II EXPLAIN ANALYZE SELECT sum(l_orderkey) FROM 'az://testing-private/l.parquet'; ---- -analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 3.*GET\: 0.*PUT\: 0.*\#POST\: 0.* +analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: 0.*PUT\: 0.*\#POST\: 0.* # Testing public blobs query II EXPLAIN ANALYZE SELECT COUNT(*) FROM "azure://testing-public/l.parquet"; ---- -analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 2.*GET\: 1.*PUT\: 0.*\#POST\: 0.* +analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: 1.*PUT\: 0.*\#POST\: 0.* From e0be81d17e324103fb611d410d7546cb32af02af Mon Sep 17 00:00:00 2001 From: Shaoyu Zhang Date: Thu, 23 Apr 2026 16:02:26 -0700 Subject: [PATCH 2/9] Use shared_mutex for Azure metadata cache Co-authored-by: Shaoyu Zhang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/include/azure_filesystem.hpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp index 40682e0..fb421dd 100644 --- a/src/include/azure_filesystem.hpp +++ b/src/include/azure_filesystem.hpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace duckdb { @@ -43,7 +44,7 @@ class AzureMetadataCache : public ClientContextState { void Insert(const string &path, const AzureFileInfo &val) { if (shared) { - lock_guard parallel_lock(lock); + unique_lock parallel_lock(lock); map[path] = val; } else { map[path] = val; @@ -52,7 +53,7 @@ class AzureMetadataCache : public ClientContextState { void Erase(const string &path) { if (shared) { - lock_guard parallel_lock(lock); + unique_lock parallel_lock(lock); map.erase(path); } else { map.erase(path); @@ -61,7 +62,7 @@ class AzureMetadataCache : public ClientContextState { bool Find(const string &path, AzureFileInfo &ret_val) { if (shared) { - lock_guard parallel_lock(lock); + std::shared_lock parallel_lock(lock); auto lookup = map.find(path); if (lookup == map.end()) { return false; @@ -79,7 +80,7 @@ class AzureMetadataCache : public ClientContextState { void Clear() { if (shared) { - lock_guard parallel_lock(lock); + unique_lock parallel_lock(lock); map.clear(); } else { map.clear(); @@ -93,7 +94,7 @@ class AzureMetadataCache : public ClientContextState { } private: - mutex lock; + std::shared_mutex lock; unordered_map map; bool flush_on_query_end; bool shared; From 09f729b2ab6649818d3947b7421f6baa03f1f002 Mon Sep 17 00:00:00 2001 From: Shaoyu Zhang Date: Fri, 24 Apr 2026 10:37:21 -0700 Subject: [PATCH 3/9] Make Azure metadata cache thread-safe and update Azure tests Co-authored-by: Shaoyu Zhang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/azure_filesystem.cpp | 5 ++-- src/include/azure_filesystem.hpp | 39 ++++++++------------------------ test/sql/azure_glob.test | 4 ++-- 3 files changed, 13 insertions(+), 35 deletions(-) diff --git a/src/azure_filesystem.cpp b/src/azure_filesystem.cpp index 4635feb..b383529 100644 --- a/src/azure_filesystem.cpp +++ b/src/azure_filesystem.cpp @@ -115,7 +115,7 @@ bool AzureStorageFileSystem::ParseAzureMetadataCacheEnabled(optional_ptr AzureStorageFileSystem::GetGlobalMetadataCache() { lock_guard lock(global_cache_lock); if (!global_metadata_cache) { - global_metadata_cache = make_uniq(false, true); + global_metadata_cache = make_uniq(false); } return global_metadata_cache.get(); } @@ -130,8 +130,7 @@ optional_ptr AzureStorageFileSystem::GetMetadataCache(option return GetGlobalMetadataCache(); } if (client_context) { - return client_context->registered_state->GetOrCreate("azure_metadata_cache", true, false) - .get(); + return client_context->registered_state->GetOrCreate("azure_metadata_cache", true).get(); } return nullptr; } diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp index fb421dd..b5f7e1b 100644 --- a/src/include/azure_filesystem.hpp +++ b/src/include/azure_filesystem.hpp @@ -38,38 +38,21 @@ struct AzureFileInfo { class AzureMetadataCache : public ClientContextState { public: - explicit AzureMetadataCache(bool flush_on_query_end_p, bool shared_p) - : flush_on_query_end(flush_on_query_end_p), shared(shared_p) { + explicit AzureMetadataCache(bool flush_on_query_end_p) : flush_on_query_end(flush_on_query_end_p) { } void Insert(const string &path, const AzureFileInfo &val) { - if (shared) { - unique_lock parallel_lock(lock); - map[path] = val; - } else { - map[path] = val; - } + unique_lock parallel_lock(lock); + map[path] = val; } void Erase(const string &path) { - if (shared) { - unique_lock parallel_lock(lock); - map.erase(path); - } else { - map.erase(path); - } + unique_lock parallel_lock(lock); + map.erase(path); } bool Find(const string &path, AzureFileInfo &ret_val) { - if (shared) { - std::shared_lock parallel_lock(lock); - auto lookup = map.find(path); - if (lookup == map.end()) { - return false; - } - ret_val = lookup->second; - return true; - } + std::shared_lock parallel_lock(lock); auto lookup = map.find(path); if (lookup == map.end()) { return false; @@ -79,12 +62,8 @@ class AzureMetadataCache : public ClientContextState { } void Clear() { - if (shared) { - unique_lock parallel_lock(lock); - map.clear(); - } else { - map.clear(); - } + unique_lock parallel_lock(lock); + map.clear(); } void QueryEnd(ClientContext &context) override { @@ -94,10 +73,10 @@ class AzureMetadataCache : public ClientContextState { } private: + // Query-local caches are still shared across parallel tasks within a query. std::shared_mutex lock; unordered_map map; bool flush_on_query_end; - bool shared; }; class AzureContextState : public ClientContextState { diff --git a/test/sql/azure_glob.test b/test/sql/azure_glob.test index 0a0ea06..b80bdc3 100644 --- a/test/sql/azure_glob.test +++ b/test/sql/azure_glob.test @@ -80,9 +80,9 @@ az://testing-public/l.csv az://testing-public/l.parquet az://testing-public/lineitem.csv -# Auto glob: ".../partitioned" -> ".../partitioned/*" via read_csv (and other multi file readers) +# Recursive glob into the nested partitioned CSVs query II -SELECT l_shipmode, COUNT(*) FROM read_csv('az://testing-public/partitioned') GROUP BY l_shipmode ORDER BY l_shipmode; +SELECT l_shipmode, COUNT(*) FROM read_csv('az://testing-public/partitioned/**/*.csv') GROUP BY l_shipmode ORDER BY l_shipmode; ---- AIR 2318 SHIP 2301 From a8bea0648b3766bac7fe202479003e44e05c0d90 Mon Sep 17 00:00:00 2001 From: Shaoyu Zhang Date: Fri, 24 Apr 2026 10:51:01 -0700 Subject: [PATCH 4/9] Fix Azure auto glob for HNS directories Co-authored-by: Shaoyu Zhang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/azure_blob_filesystem.cpp | 6 +----- test/sql/azure_glob.test | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp index 3cdd5dd..af46999 100644 --- a/src/azure_blob_filesystem.cpp +++ b/src/azure_blob_filesystem.cpp @@ -370,11 +370,7 @@ bool AzureBlobStorageFileSystem::DirectoryExists(const string &dirname, optional bool AzureBlobStorageFileSystem::FileExists(const string &filename, optional_ptr opener) { auto handle = OpenFile(filename, FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS, opener); - if (handle != nullptr) { - auto &sfh = handle->Cast(); - return sfh.length >= 0; // aka return true; -- avoid optimizers and shenanigans -- deref handle to be sure - } - return false; + return handle && handle->Cast().GetType() == FileType::FILE_TYPE_REGULAR; } void AzureBlobStorageFileSystem::RemoveFile(const string &filename, optional_ptr opener) { diff --git a/test/sql/azure_glob.test b/test/sql/azure_glob.test index b80bdc3..0a0ea06 100644 --- a/test/sql/azure_glob.test +++ b/test/sql/azure_glob.test @@ -80,9 +80,9 @@ az://testing-public/l.csv az://testing-public/l.parquet az://testing-public/lineitem.csv -# Recursive glob into the nested partitioned CSVs +# Auto glob: ".../partitioned" -> ".../partitioned/*" via read_csv (and other multi file readers) query II -SELECT l_shipmode, COUNT(*) FROM read_csv('az://testing-public/partitioned/**/*.csv') GROUP BY l_shipmode ORDER BY l_shipmode; +SELECT l_shipmode, COUNT(*) FROM read_csv('az://testing-public/partitioned') GROUP BY l_shipmode ORDER BY l_shipmode; ---- AIR 2318 SHIP 2301 From 1cda0f8bcfe1baff1e04fa6682fa4f5c80a530f4 Mon Sep 17 00:00:00 2001 From: Shaoyu Zhang Date: Fri, 24 Apr 2026 11:21:16 -0700 Subject: [PATCH 5/9] Avoid directory short-circuit in Azure Blob glob Co-authored-by: Shaoyu Zhang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/azure_blob_filesystem.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp index af46999..5f019d8 100644 --- a/src/azure_blob_filesystem.cpp +++ b/src/azure_blob_filesystem.cpp @@ -170,7 +170,9 @@ vector AzureBlobStorageFileSystem::Glob(const string &path, FileOp auto first_wildcard_pos = azure_url.path.find_first_of("*[\\"); if (first_wildcard_pos == string::npos) { vector rv; - if (FileExists(path, opener)) { + auto handle = OpenFile(path, FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS, opener); + // Returning directories here suppresses DuckDB's fallback auto-glob path for multi-file readers. + if (handle && handle->Cast().GetType() == FileType::FILE_TYPE_REGULAR) { rv.emplace_back(path); } return rv; @@ -370,7 +372,11 @@ bool AzureBlobStorageFileSystem::DirectoryExists(const string &dirname, optional bool AzureBlobStorageFileSystem::FileExists(const string &filename, optional_ptr opener) { auto handle = OpenFile(filename, FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS, opener); - return handle && handle->Cast().GetType() == FileType::FILE_TYPE_REGULAR; + if (handle != nullptr) { + auto &sfh = handle->Cast(); + return sfh.length >= 0; // aka return true; -- avoid optimizers and shenanigans -- deref handle to be sure + } + return false; } void AzureBlobStorageFileSystem::RemoveFile(const string &filename, optional_ptr opener) { From 5a2002d4365e8c8b22bf2f899e319f1f0154b481 Mon Sep 17 00:00:00 2001 From: Shaoyu Zhang Date: Fri, 24 Apr 2026 12:50:30 -0700 Subject: [PATCH 6/9] Seed Blob glob cache only for non-empty files Co-authored-by: Shaoyu Zhang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/azure_blob_filesystem.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp index 5f019d8..2ea44a8 100644 --- a/src/azure_blob_filesystem.cpp +++ b/src/azure_blob_filesystem.cpp @@ -214,7 +214,12 @@ vector AzureBlobStorageFileSystem::Glob(const string &path, FileOp OpenFileInfo info(result_full_url); info.extended_info = make_shared_ptr(); auto &options = info.extended_info->options; - options.emplace("type", Value("file")); + // Flat Blob listing on HNS accounts can surface directories as zero-length Blob entries. + // Only stamp a file type when the item is clearly a non-empty file so we can seed the cache + // without misclassifying folders. + if (key.BlobSize > 0) { + options.emplace("type", Value("file")); + } options.emplace("file_size", Value::BIGINT(key.BlobSize)); options.emplace("last_modified", Value::TIMESTAMP(ToTimestamp(key.Details.LastModified))); result.push_back(info); From ec764809ccd7b5c27eafd115b8051d331e4b5a38 Mon Sep 17 00:00:00 2001 From: Shaoyu Zhang Date: Fri, 24 Apr 2026 14:37:40 -0700 Subject: [PATCH 7/9] Use portable mutex in Azure metadata cache Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/include/azure_filesystem.hpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp index b5f7e1b..bc19461 100644 --- a/src/include/azure_filesystem.hpp +++ b/src/include/azure_filesystem.hpp @@ -13,8 +13,6 @@ #include #include #include -#include - namespace duckdb { struct AzureOptions { @@ -42,17 +40,17 @@ class AzureMetadataCache : public ClientContextState { } void Insert(const string &path, const AzureFileInfo &val) { - unique_lock parallel_lock(lock); + lock_guard parallel_lock(lock); map[path] = val; } void Erase(const string &path) { - unique_lock parallel_lock(lock); + lock_guard parallel_lock(lock); map.erase(path); } bool Find(const string &path, AzureFileInfo &ret_val) { - std::shared_lock parallel_lock(lock); + lock_guard parallel_lock(lock); auto lookup = map.find(path); if (lookup == map.end()) { return false; @@ -62,7 +60,7 @@ class AzureMetadataCache : public ClientContextState { } void Clear() { - unique_lock parallel_lock(lock); + lock_guard parallel_lock(lock); map.clear(); } @@ -74,7 +72,7 @@ class AzureMetadataCache : public ClientContextState { private: // Query-local caches are still shared across parallel tasks within a query. - std::shared_mutex lock; + mutex lock; unordered_map map; bool flush_on_query_end; }; From 2bd7e5143bc4913d655d67dfefd015ae1bc53366 Mon Sep 17 00:00:00 2001 From: Shaoyu Zhang Date: Thu, 16 Jul 2026 21:45:35 -0700 Subject: [PATCH 8/9] Update Azure HTTP stats expectation --- test/sql/azure.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sql/azure.test b/test/sql/azure.test index 357ab68..81d3fa5 100644 --- a/test/sql/azure.test +++ b/test/sql/azure.test @@ -51,13 +51,13 @@ SET azure_http_stats = true; query II EXPLAIN ANALYZE SELECT sum(l_orderkey) FROM 'az://testing-private/l.parquet'; ---- -analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: 0.*PUT\: 0.*\#POST\: 0.* +analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: (0|2).*PUT\: 0.*\#POST\: 0.* # Redoing query should still result in same request count query II EXPLAIN ANALYZE SELECT sum(l_orderkey) FROM 'az://testing-private/l.parquet'; ---- -analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: 0.*PUT\: 0.*\#POST\: 0.* +analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: (0|2).*PUT\: 0.*\#POST\: 0.* # Testing public blobs query II From 542e30ad38afa01d2022c01a22265fcf41364a25 Mon Sep 17 00:00:00 2001 From: Shaoyu Zhang Date: Thu, 16 Jul 2026 22:34:42 -0700 Subject: [PATCH 9/9] Format Azure DFS handle construction --- src/azure_dfs_filesystem.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/azure_dfs_filesystem.cpp b/src/azure_dfs_filesystem.cpp index 629ad67..9a3a230 100644 --- a/src/azure_dfs_filesystem.cpp +++ b/src/azure_dfs_filesystem.cpp @@ -176,9 +176,9 @@ unique_ptr AzureDfsStorageFileSystem::CreateHandle(const OpenFi file_path.pop_back(); } - auto handle = make_uniq(*this, info, flags, storage_context->options, - GetMetadataCache(opener), - file_system_client.GetFileClient(file_path)); + auto handle = + make_uniq(*this, info, flags, storage_context->options, GetMetadataCache(opener), + file_system_client.GetFileClient(file_path)); if (!handle->PostConstruct()) { return nullptr; }