diff --git a/src/azure_blob_filesystem.cpp b/src/azure_blob_filesystem.cpp index 0c84853..2ea44a8 100644 --- a/src/azure_blob_filesystem.cpp +++ b/src/azure_blob_filesystem.cpp @@ -75,8 +75,10 @@ AzureBlobContextState::GetBlobContainerClient(const std::string &blobContainerNa //////// AzureBlobStorageFileHandle //////// AzureBlobStorageFileHandle::AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, const AzureOptions &opts, + optional_ptr metadata_cache, Azure::Storage::Blobs::BlockBlobClient blob_client) - : AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, opts), blob_client(std::move(blob_client)) { + : AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, opts, metadata_cache), + blob_client(std::move(blob_client)) { } void AzureBlobStorageFileHandle::StageWriteBuffer() { @@ -138,11 +140,14 @@ unique_ptr AzureBlobStorageFileSystem::CreateHandle(const OpenF auto parsed_url = ParseUrl(info.path); auto storage_context = GetOrCreateStorageContext(opener, info.path, parsed_url); + if (flags.OpenForWriting() || flags.OpenForAppending()) { + InvalidateMetadata(opener, info.path); + } auto container = storage_context->As().GetBlobContainerClient(parsed_url.container); auto blob_client = container.GetBlockBlobClient(parsed_url.path); - auto handle = - make_uniq(*this, info, flags, storage_context->options, std::move(blob_client)); + auto handle = make_uniq(*this, info, flags, storage_context->options, + GetMetadataCache(opener), std::move(blob_client)); if (!handle->PostConstruct()) { return nullptr; } @@ -165,7 +170,9 @@ vector AzureBlobStorageFileSystem::Glob(const string &path, FileOp auto first_wildcard_pos = azure_url.path.find_first_of("*[\\"); if (first_wildcard_pos == string::npos) { vector rv; - if (FileExists(path, opener)) { + auto handle = OpenFile(path, FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS, opener); + // Returning directories here suppresses DuckDB's fallback auto-glob path for multi-file readers. + if (handle && handle->Cast().GetType() == FileType::FILE_TYPE_REGULAR) { rv.emplace_back(path); } return rv; @@ -207,6 +214,12 @@ vector AzureBlobStorageFileSystem::Glob(const string &path, FileOp OpenFileInfo info(result_full_url); info.extended_info = make_shared_ptr(); auto &options = info.extended_info->options; + // Flat Blob listing on HNS accounts can surface directories as zero-length Blob entries. + // Only stamp a file type when the item is clearly a non-empty file so we can seed the cache + // without misclassifying folders. + if (key.BlobSize > 0) { + options.emplace("type", Value("file")); + } options.emplace("file_size", Value::BIGINT(key.BlobSize)); options.emplace("last_modified", Value::TIMESTAMP(ToTimestamp(key.Details.LastModified))); result.push_back(info); @@ -297,11 +310,7 @@ void AzureBlobStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) { // - doesn't exist, don't create auto set_props = [&](bool is_dir, idx_t length, timestamp_t last_mod) { - afh.is_remote_loaded = true; // always set loaded - afh.file_type = is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR; - afh.length = is_dir ? 0 : length; - afh.last_modified = last_mod; - afh.file_offset = 0; // always reset offset state + afh.SetFileInfo(is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR, length, last_mod); }; auto create_file = [&]() { @@ -382,6 +391,7 @@ void AzureBlobStorageFileSystem::RemoveFile(const string &filename, optional_ptr auto blob_client = container.GetBlockBlobClient(url.path); try { blob_client.Delete(); + InvalidateMetadata(opener, filename); } catch (Azure::Storage::StorageException &e) { throw IOException("AzureBlobStorageFileSystem Delete of %s failed with %s Reason Phrase: %s", filename, e.ErrorCode, e.ReasonPhrase); @@ -393,7 +403,11 @@ bool AzureBlobStorageFileSystem::TryRemoveFile(const string &filename, optional_ auto storage_context = GetOrCreateStorageContext(opener, filename, url); auto container = storage_context->As().GetBlobContainerClient(url.container); auto blob_client = container.GetBlockBlobClient(url.path); - return blob_client.DeleteIfExists().Value.Deleted; + auto removed = blob_client.DeleteIfExists().Value.Deleted; + if (removed) { + InvalidateMetadata(opener, filename); + } + return removed; } void AzureBlobStorageFileSystem::ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out, diff --git a/src/azure_dfs_filesystem.cpp b/src/azure_dfs_filesystem.cpp index 1e7ec83..9a3a230 100644 --- a/src/azure_dfs_filesystem.cpp +++ b/src/azure_dfs_filesystem.cpp @@ -73,6 +73,7 @@ static void Walk(const Azure::Storage::Files::DataLake::DataLakeFileSystemClient OpenFileInfo info(elt.Name); info.extended_info = make_shared_ptr(); auto &options = info.extended_info->options; + options.emplace("type", Value("file")); options.emplace("file_size", Value::BIGINT(elt.FileSize)); options.emplace("last_modified", Value::TIMESTAMP(AzureStorageFileSystem::ToTimestamp(elt.LastModified))); @@ -103,8 +104,10 @@ AzureDfsContextState::GetDfsFileSystemClient(const std::string &file_system_name //////// AzureDfsContextState //////// AzureDfsStorageFileHandle::AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, const AzureOptions &options, + optional_ptr metadata_cache, Azure::Storage::Files::DataLake::DataLakeFileClient client) - : AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, options), file_client(std::move(client)) { + : AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, options, metadata_cache), + file_client(std::move(client)) { } void AzureDfsStorageFileHandle::StageWriteBuffer() { @@ -161,6 +164,9 @@ unique_ptr AzureDfsStorageFileSystem::CreateHandle(const OpenFi auto parsed_url = ParseUrl(info.path); auto storage_context = GetOrCreateStorageContext(opener, info.path, parsed_url); + if (flags.OpenForWriting() || flags.OpenForAppending()) { + InvalidateMetadata(opener, info.path); + } auto file_system_client = storage_context->As().GetDfsFileSystemClient(parsed_url.container); // A trailing '/' is only a directory hint, not part of the resource name. Some DFS endpoints (e.g. OneLake) @@ -170,8 +176,9 @@ unique_ptr AzureDfsStorageFileSystem::CreateHandle(const OpenFi file_path.pop_back(); } - auto handle = make_uniq(*this, info, flags, storage_context->options, - file_system_client.GetFileClient(file_path)); + auto handle = + make_uniq(*this, info, flags, storage_context->options, GetMetadataCache(opener), + file_system_client.GetFileClient(file_path)); if (!handle->PostConstruct()) { return nullptr; } @@ -195,6 +202,7 @@ void AzureDfsStorageFileSystem::CreateDirectory(const string &dirname, optional_ auto storage_context = GetOrCreateStorageContext(opener, dirname, dir_url); auto file_system_client = storage_context->As().GetDfsFileSystemClient(dir_url.container); file_system_client.GetDirectoryClient(dir_url.path).Create(); + InvalidateMetadata(opener, dirname); } bool AzureDfsStorageFileSystem::FileExists(const string &filename, optional_ptr opener) { @@ -209,6 +217,7 @@ void AzureDfsStorageFileSystem::RemoveFile(const string &filename, optional_ptr< auto file_client = file_system_client.GetFileClient(url.path); try { file_client.Delete(); + InvalidateMetadata(opener, filename); } catch (Azure::Storage::StorageException &e) { throw IOException("AzureDfsStorageFileSystem Delete of %s failed with %s Reason Phrase: %s", filename, e.ErrorCode, e.ReasonPhrase); @@ -220,7 +229,11 @@ bool AzureDfsStorageFileSystem::TryRemoveFile(const string &filename, optional_p auto storage_context = GetOrCreateStorageContext(opener, filename, url); auto file_system_client = storage_context->As().GetDfsFileSystemClient(url.container); auto file_client = file_system_client.GetFileClient(url.path); - return file_client.DeleteIfExists().Value.Deleted; + auto removed = file_client.DeleteIfExists().Value.Deleted; + if (removed) { + InvalidateMetadata(opener, filename); + } + return removed; } vector AzureDfsStorageFileSystem::Glob(const string &path, FileOpener *opener) { @@ -323,11 +336,7 @@ void AzureDfsStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) { // - doesn't exist, don't create auto set_props = [&](bool is_dir, idx_t length, timestamp_t last_mod) { - afh.is_remote_loaded = true; // always set loaded - afh.file_type = is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR; - afh.length = is_dir ? 0 : length; - afh.last_modified = last_mod; - afh.file_offset = 0; // always reset offset state + afh.SetFileInfo(is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR, length, last_mod); }; auto create_file = [&]() { diff --git a/src/azure_filesystem.cpp b/src/azure_filesystem.cpp index 8b539e3..b383529 100644 --- a/src/azure_filesystem.cpp +++ b/src/azure_filesystem.cpp @@ -24,39 +24,150 @@ void AzureContextState::QueryEnd() { is_valid = false; } +static string GetFileType(const Value &value) { + auto type = value.ToString(); + if (!type.empty() && type.front() == '\'' && type.back() == '\'' && type.size() >= 2) { + type = type.substr(1, type.size() - 2); + } + return type; +} + AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, - FileType file_type, const AzureOptions &options) + FileType file_type, const AzureOptions &options, + optional_ptr metadata_cache_p) : FileHandle(fs, info.path, flags), flags(flags), // File info is_remote_loaded(false), file_type(file_type), length(0), last_modified(0), // Read info buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0), // Options - options(options) { + options(options), metadata_cache(metadata_cache_p) { if (!flags.RequireParallelAccess() && !flags.DirectIO()) { read_buffer = duckdb::unique_ptr(new data_t[options.read_buffer_size]); } // Set metadata of file when available, it avoids to invoke to the storage to get them. + bool has_file_type = file_type != FileType::FILE_TYPE_INVALID; + bool has_length = false; + bool has_last_modified = false; if (info.extended_info) { + auto type_entry = info.extended_info->options.find("type"); + if (type_entry != info.extended_info->options.end()) { + auto type = GetFileType(type_entry->second); + if (type == "directory") { + file_type = FileType::FILE_TYPE_DIR; + has_file_type = true; + } else if (type == "file") { + file_type = FileType::FILE_TYPE_REGULAR; + has_file_type = true; + } + } auto entry1 = info.extended_info->options.find("file_size"); if (entry1 != info.extended_info->options.end()) { length = entry1->second.GetValue(); + has_length = true; } auto entry2 = info.extended_info->options.find("last_modified"); if (entry2 != info.extended_info->options.end()) { last_modified = entry2->second.GetValue(); + has_last_modified = true; } } + if (has_file_type && has_last_modified && (file_type == FileType::FILE_TYPE_DIR || has_length)) { + SetFileInfo(file_type, length, last_modified); + } +} + +void AzureFileHandle::SetFileInfo(FileType file_type_p, idx_t length_p, timestamp_t last_modified_p) { + is_remote_loaded = true; + file_type = file_type_p; + length = file_type_p == FileType::FILE_TYPE_DIR ? 0 : length_p; + last_modified = last_modified_p; + file_offset = 0; } bool AzureFileHandle::PostConstruct() { return static_cast(file_system).LoadFileInfo(*this); } +static bool CanUseMetadataCache(const AzureFileHandle &handle) { + return handle.metadata_cache && !handle.flags.OpenForWriting() && !handle.flags.OpenForAppending() && + !handle.flags.ExclusiveCreate(); +} + +static AzureFileInfo GetCacheEntry(const AzureFileHandle &handle) { + AzureFileInfo info; + info.file_type = handle.file_type; + info.length = handle.length; + info.last_modified = handle.last_modified; + return info; +} + +bool AzureStorageFileSystem::ParseAzureMetadataCacheEnabled(optional_ptr opener) { + Value metadata_cache_val; + if (FileOpener::TryGetCurrentSetting(opener, "enable_http_metadata_cache", metadata_cache_val) && + !metadata_cache_val.IsNull()) { + return metadata_cache_val.GetValue(); + } + return false; +} + +optional_ptr AzureStorageFileSystem::GetGlobalMetadataCache() { + lock_guard lock(global_cache_lock); + if (!global_metadata_cache) { + global_metadata_cache = make_uniq(false); + } + return global_metadata_cache.get(); +} + +optional_ptr AzureStorageFileSystem::GetMetadataCache(optional_ptr opener) { + auto db = FileOpener::TryGetDatabase(opener); + auto client_context = FileOpener::TryGetClientContext(opener); + if (!db) { + return nullptr; + } + if (ParseAzureMetadataCacheEnabled(opener)) { + return GetGlobalMetadataCache(); + } + if (client_context) { + return client_context->registered_state->GetOrCreate("azure_metadata_cache", true).get(); + } + return nullptr; +} + +void AzureStorageFileSystem::InvalidateMetadata(optional_ptr opener, const string &path) { + auto metadata_cache = GetMetadataCache(opener); + if (metadata_cache) { + metadata_cache->Erase(path); + } +} + bool AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) { try { + if (handle.IsRemoteLoaded()) { + if (CanUseMetadataCache(handle)) { + handle.metadata_cache->Insert(handle.path, GetCacheEntry(handle)); + } + if (handle.flags.ReturnNullIfExists()) { + return false; + } + return true; + } + if (CanUseMetadataCache(handle)) { + AzureFileInfo cached_info; + if (handle.metadata_cache->Find(handle.path, cached_info)) { + handle.SetFileInfo(cached_info.file_type, cached_info.length, cached_info.last_modified); + if (handle.flags.ReturnNullIfExists()) { + return false; + } + return true; + } + } + LoadRemoteFileInfo(handle); + if (CanUseMetadataCache(handle)) { + handle.metadata_cache->Insert(handle.path, GetCacheEntry(handle)); + } if (handle.flags.ReturnNullIfExists()) { return false; } diff --git a/src/include/azure_blob_filesystem.hpp b/src/include/azure_blob_filesystem.hpp index 67bfebd..77e8a8a 100644 --- a/src/include/azure_blob_filesystem.hpp +++ b/src/include/azure_blob_filesystem.hpp @@ -28,7 +28,8 @@ class AzureBlobStorageFileSystem; class AzureBlobStorageFileHandle : public AzureFileHandle { public: AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, - const AzureOptions &options, Azure::Storage::Blobs::BlockBlobClient blob_client); + const AzureOptions &options, optional_ptr metadata_cache, + Azure::Storage::Blobs::BlockBlobClient blob_client); ~AzureBlobStorageFileHandle() override = default; void StageWriteBuffer(); diff --git a/src/include/azure_dfs_filesystem.hpp b/src/include/azure_dfs_filesystem.hpp index c28417d..99bd3ee 100644 --- a/src/include/azure_dfs_filesystem.hpp +++ b/src/include/azure_dfs_filesystem.hpp @@ -28,7 +28,8 @@ class AzureDfsStorageFileSystem; class AzureDfsStorageFileHandle : public AzureFileHandle { public: AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, - const AzureOptions &options, Azure::Storage::Files::DataLake::DataLakeFileClient client); + const AzureOptions &options, optional_ptr metadata_cache, + Azure::Storage::Files::DataLake::DataLakeFileClient client); ~AzureDfsStorageFileHandle() override = default; void StageWriteBuffer(); diff --git a/src/include/azure_filesystem.hpp b/src/include/azure_filesystem.hpp index b846bb8..bc19461 100644 --- a/src/include/azure_filesystem.hpp +++ b/src/include/azure_filesystem.hpp @@ -4,14 +4,15 @@ #include "duckdb/common/assert.hpp" #include "duckdb/common/file_opener.hpp" #include "duckdb/common/file_system.hpp" +#include "duckdb/common/mutex.hpp" #include "duckdb/common/shared_ptr.hpp" +#include "duckdb/common/unordered_map.hpp" #include "duckdb/logging/file_system_logger.hpp" #include "duckdb/main/client_context_state.hpp" #include #include #include - namespace duckdb { struct AzureOptions { @@ -27,6 +28,55 @@ struct AzureOptions { idx_t write_staged_blocks_per_commit = 0; }; +struct AzureFileInfo { + FileType file_type = FileType::FILE_TYPE_INVALID; + idx_t length = 0; + timestamp_t last_modified; +}; + +class AzureMetadataCache : public ClientContextState { +public: + explicit AzureMetadataCache(bool flush_on_query_end_p) : flush_on_query_end(flush_on_query_end_p) { + } + + void Insert(const string &path, const AzureFileInfo &val) { + lock_guard parallel_lock(lock); + map[path] = val; + } + + void Erase(const string &path) { + lock_guard parallel_lock(lock); + map.erase(path); + } + + bool Find(const string &path, AzureFileInfo &ret_val) { + lock_guard parallel_lock(lock); + auto lookup = map.find(path); + if (lookup == map.end()) { + return false; + } + ret_val = lookup->second; + return true; + } + + void Clear() { + lock_guard parallel_lock(lock); + map.clear(); + } + + void QueryEnd(ClientContext &context) override { + if (flush_on_query_end) { + Clear(); + } + } + +private: + // Query-local caches are still shared across parallel tasks within a query. + mutex lock; + unordered_map map; + bool flush_on_query_end; +}; + class AzureContextState : public ClientContextState { public: const AzureOptions options; @@ -58,6 +108,7 @@ class AzureStorageFileSystem; class AzureFileHandle : public FileHandle { public: virtual bool PostConstruct(); + void SetFileInfo(FileType file_type_p, idx_t length_p, timestamp_t last_modified_p); bool IsRemoteLoaded() { return is_remote_loaded; @@ -69,7 +120,7 @@ class AzureFileHandle : public FileHandle { protected: AzureFileHandle(AzureStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags, FileType file_type, - const AzureOptions &options); + const AzureOptions &options, optional_ptr metadata_cache); public: FileOpenFlags flags; @@ -88,8 +139,8 @@ class AzureFileHandle : public FileHandle { idx_t file_offset; idx_t buffer_start; idx_t buffer_end; - const AzureOptions options; + optional_ptr metadata_cache; }; class AzureStorageFileSystem : public FileSystem { @@ -140,9 +191,17 @@ class AzureStorageFileSystem : public FileSystem { virtual void LoadRemoteFileInfo(AzureFileHandle &handle) = 0; static AzureOptions ParseAzureOptions(optional_ptr opener); + static bool ParseAzureMetadataCacheEnabled(optional_ptr opener); + optional_ptr GetMetadataCache(optional_ptr opener); + void InvalidateMetadata(optional_ptr opener, const string &path); public: static timestamp_t ToTimestamp(const Azure::DateTime &dt); + +private: + optional_ptr GetGlobalMetadataCache(); + mutex global_cache_lock; + duckdb::unique_ptr global_metadata_cache; }; } // namespace duckdb diff --git a/test/sql/azure.test b/test/sql/azure.test index c0d94b1..81d3fa5 100644 --- a/test/sql/azure.test +++ b/test/sql/azure.test @@ -51,17 +51,17 @@ SET azure_http_stats = true; query II EXPLAIN ANALYZE SELECT sum(l_orderkey) FROM 'az://testing-private/l.parquet'; ---- -analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 3.*GET\: 0.*PUT\: 0.*\#POST\: 0.* +analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: (0|2).*PUT\: 0.*\#POST\: 0.* # Redoing query should still result in same request count query II EXPLAIN ANALYZE SELECT sum(l_orderkey) FROM 'az://testing-private/l.parquet'; ---- -analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 3.*GET\: 0.*PUT\: 0.*\#POST\: 0.* +analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: (0|2).*PUT\: 0.*\#POST\: 0.* # Testing public blobs query II EXPLAIN ANALYZE SELECT COUNT(*) FROM "azure://testing-public/l.parquet"; ---- -analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 2.*GET\: 1.*PUT\: 0.*\#POST\: 0.* +analyzed_plan :.*HTTP Stats.*in\: \d[.]\d MiB.*\#HEAD\: 1.*GET\: 1.*PUT\: 0.*\#POST\: 0.*