Skip to content

Azure metadata cache hookup and Azure integration test updates #161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zsy056 wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Azure metadata cache hookup and Azure integration test updates #161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
9f6fea6
Hook Azure metadata cache into file info lookups
Apr 23, 2026
e0be81d
Use shared_mutex for Azure metadata cache
Apr 23, 2026
09f729b
Make Azure metadata cache thread-safe and update Azure tests
Apr 24, 2026
a8bea06
Fix Azure auto glob for HNS directories
Apr 24, 2026
1cda0f8
Avoid directory short-circuit in Azure Blob glob
Apr 24, 2026
5a2002d
Seed Blob glob cache only for non-empty files
Apr 24, 2026
ec76480
Use portable mutex in Azure metadata cache
Apr 24, 2026
2bd7e51
Update Azure HTTP stats expectation
Jul 17, 2026
542e30a
Format Azure DFS handle construction
Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 24 additions & 10 deletions src/azure_blob_filesystem.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,10 @@ AzureBlobContextState::GetBlobContainerClient(const std::string &blobContainerNa
//////// AzureBlobStorageFileHandle ////////
AzureBlobStorageFileHandle::AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, const OpenFileInfo &info,
FileOpenFlags flags, const AzureOptions &opts,
optional_ptr<AzureMetadataCache> metadata_cache,
Azure::Storage::Blobs::BlockBlobClient blob_client)
: AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, opts), blob_client(std::move(blob_client)) {
: AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, opts, metadata_cache),
blob_client(std::move(blob_client)) {
}

void AzureBlobStorageFileHandle::StageWriteBuffer() {
Expand Down Expand Up @@ -138,11 +140,14 @@ unique_ptr<AzureFileHandle> AzureBlobStorageFileSystem::CreateHandle(const OpenF

auto parsed_url = ParseUrl(info.path);
auto storage_context = GetOrCreateStorageContext(opener, info.path, parsed_url);
if (flags.OpenForWriting() || flags.OpenForAppending()) {
InvalidateMetadata(opener, info.path);
}
auto container = storage_context->As<AzureBlobContextState>().GetBlobContainerClient(parsed_url.container);
auto blob_client = container.GetBlockBlobClient(parsed_url.path);

auto handle =
make_uniq<AzureBlobStorageFileHandle>(*this, info, flags, storage_context->options, std::move(blob_client));
auto handle = make_uniq<AzureBlobStorageFileHandle>(*this, info, flags, storage_context->options,
GetMetadataCache(opener), std::move(blob_client));
if (!handle->PostConstruct()) {
return nullptr;
}
Expand All @@ -165,7 +170,9 @@ vector<OpenFileInfo> AzureBlobStorageFileSystem::Glob(const string &path, FileOp
auto first_wildcard_pos = azure_url.path.find_first_of("*[\\");
if (first_wildcard_pos == string::npos) {
vector<OpenFileInfo> rv;
if (FileExists(path, opener)) {
auto handle = OpenFile(path, FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS, opener);
// Returning directories here suppresses DuckDB's fallback auto-glob path for multi-file readers.
if (handle && handle->Cast<AzureBlobStorageFileHandle>().GetType() == FileType::FILE_TYPE_REGULAR) {
rv.emplace_back(path);
}
return rv;
Expand Down Expand Up @@ -207,6 +214,12 @@ vector<OpenFileInfo> AzureBlobStorageFileSystem::Glob(const string &path, FileOp
OpenFileInfo info(result_full_url);
info.extended_info = make_shared_ptr<ExtendedOpenFileInfo>();
auto &options = info.extended_info->options;
// Flat Blob listing on HNS accounts can surface directories as zero-length Blob entries.
// Only stamp a file type when the item is clearly a non-empty file so we can seed the cache
// without misclassifying folders.
if (key.BlobSize > 0) {
options.emplace("type", Value("file"));
}
options.emplace("file_size", Value::BIGINT(key.BlobSize));
options.emplace("last_modified", Value::TIMESTAMP(ToTimestamp(key.Details.LastModified)));
result.push_back(info);
Expand Down Expand Up @@ -297,11 +310,7 @@ void AzureBlobStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) {
// - doesn't exist, don't create

auto set_props = [&](bool is_dir, idx_t length, timestamp_t last_mod) {
afh.is_remote_loaded = true; // always set loaded
afh.file_type = is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR;
afh.length = is_dir ? 0 : length;
afh.last_modified = last_mod;
afh.file_offset = 0; // always reset offset state
afh.SetFileInfo(is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR, length, last_mod);
};

auto create_file = [&]() {
Expand Down Expand Up @@ -382,6 +391,7 @@ void AzureBlobStorageFileSystem::RemoveFile(const string &filename, optional_ptr
auto blob_client = container.GetBlockBlobClient(url.path);
try {
blob_client.Delete();
InvalidateMetadata(opener, filename);
} catch (Azure::Storage::StorageException &e) {
throw IOException("AzureBlobStorageFileSystem Delete of %s failed with %s Reason Phrase: %s", filename,
e.ErrorCode, e.ReasonPhrase);
Expand All @@ -393,7 +403,11 @@ bool AzureBlobStorageFileSystem::TryRemoveFile(const string &filename, optional_
auto storage_context = GetOrCreateStorageContext(opener, filename, url);
auto container = storage_context->As<AzureBlobContextState>().GetBlobContainerClient(url.container);
auto blob_client = container.GetBlockBlobClient(url.path);
return blob_client.DeleteIfExists().Value.Deleted;
auto removed = blob_client.DeleteIfExists().Value.Deleted;
if (removed) {
InvalidateMetadata(opener, filename);
}
return removed;
}

void AzureBlobStorageFileSystem::ReadRange(AzureFileHandle &handle, idx_t file_offset, char *buffer_out,
Expand Down
27 changes: 18 additions & 9 deletions src/azure_dfs_filesystem.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ static void Walk(const Azure::Storage::Files::DataLake::DataLakeFileSystemClient
OpenFileInfo info(elt.Name);
info.extended_info = make_shared_ptr<ExtendedOpenFileInfo>();
auto &options = info.extended_info->options;
options.emplace("type", Value("file"));
options.emplace("file_size", Value::BIGINT(elt.FileSize));
options.emplace("last_modified",
Value::TIMESTAMP(AzureStorageFileSystem::ToTimestamp(elt.LastModified)));
Expand Down Expand Up @@ -103,8 +104,10 @@ AzureDfsContextState::GetDfsFileSystemClient(const std::string &file_system_name
//////// AzureDfsContextState ////////
AzureDfsStorageFileHandle::AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, const OpenFileInfo &info,
FileOpenFlags flags, const AzureOptions &options,
optional_ptr<AzureMetadataCache> metadata_cache,
Azure::Storage::Files::DataLake::DataLakeFileClient client)
: AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, options), file_client(std::move(client)) {
: AzureFileHandle(fs, info, flags, FileType::FILE_TYPE_INVALID, options, metadata_cache),
file_client(std::move(client)) {
}

void AzureDfsStorageFileHandle::StageWriteBuffer() {
Expand Down Expand Up @@ -161,6 +164,9 @@ unique_ptr<AzureFileHandle> AzureDfsStorageFileSystem::CreateHandle(const OpenFi

auto parsed_url = ParseUrl(info.path);
auto storage_context = GetOrCreateStorageContext(opener, info.path, parsed_url);
if (flags.OpenForWriting() || flags.OpenForAppending()) {
InvalidateMetadata(opener, info.path);
}
auto file_system_client = storage_context->As<AzureDfsContextState>().GetDfsFileSystemClient(parsed_url.container);

// A trailing '/' is only a directory hint, not part of the resource name. Some DFS endpoints (e.g. OneLake)
Expand All @@ -170,8 +176,9 @@ unique_ptr<AzureFileHandle> AzureDfsStorageFileSystem::CreateHandle(const OpenFi
file_path.pop_back();
}

auto handle = make_uniq<AzureDfsStorageFileHandle>(*this, info, flags, storage_context->options,
file_system_client.GetFileClient(file_path));
auto handle =
make_uniq<AzureDfsStorageFileHandle>(*this, info, flags, storage_context->options, GetMetadataCache(opener),
file_system_client.GetFileClient(file_path));
if (!handle->PostConstruct()) {
return nullptr;
}
Expand All @@ -195,6 +202,7 @@ void AzureDfsStorageFileSystem::CreateDirectory(const string &dirname, optional_
auto storage_context = GetOrCreateStorageContext(opener, dirname, dir_url);
auto file_system_client = storage_context->As<AzureDfsContextState>().GetDfsFileSystemClient(dir_url.container);
file_system_client.GetDirectoryClient(dir_url.path).Create();
InvalidateMetadata(opener, dirname);
}

bool AzureDfsStorageFileSystem::FileExists(const string &filename, optional_ptr<FileOpener> opener) {
Expand All @@ -209,6 +217,7 @@ void AzureDfsStorageFileSystem::RemoveFile(const string &filename, optional_ptr<
auto file_client = file_system_client.GetFileClient(url.path);
try {
file_client.Delete();
InvalidateMetadata(opener, filename);
} catch (Azure::Storage::StorageException &e) {
throw IOException("AzureDfsStorageFileSystem Delete of %s failed with %s Reason Phrase: %s", filename,
e.ErrorCode, e.ReasonPhrase);
Expand All @@ -220,7 +229,11 @@ bool AzureDfsStorageFileSystem::TryRemoveFile(const string &filename, optional_p
auto storage_context = GetOrCreateStorageContext(opener, filename, url);
auto file_system_client = storage_context->As<AzureDfsContextState>().GetDfsFileSystemClient(url.container);
auto file_client = file_system_client.GetFileClient(url.path);
return file_client.DeleteIfExists().Value.Deleted;
auto removed = file_client.DeleteIfExists().Value.Deleted;
if (removed) {
InvalidateMetadata(opener, filename);
}
return removed;
}

vector<OpenFileInfo> AzureDfsStorageFileSystem::Glob(const string &path, FileOpener *opener) {
Expand Down Expand Up @@ -323,11 +336,7 @@ void AzureDfsStorageFileSystem::LoadRemoteFileInfo(AzureFileHandle &handle) {
// - doesn't exist, don't create

auto set_props = [&](bool is_dir, idx_t length, timestamp_t last_mod) {
afh.is_remote_loaded = true; // always set loaded
afh.file_type = is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR;
afh.length = is_dir ? 0 : length;
afh.last_modified = last_mod;
afh.file_offset = 0; // always reset offset state
afh.SetFileInfo(is_dir ? FileType::FILE_TYPE_DIR : FileType::FILE_TYPE_REGULAR, length, last_mod);
};

auto create_file = [&]() {
Expand Down
115 changes: 113 additions & 2 deletions src/azure_filesystem.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,39 +24,150 @@ void AzureContextState::QueryEnd() {
is_valid = false;
}

static string GetFileType(const Value &value) {
auto type = value.ToString();
if (!type.empty() && type.front() == '\'' && type.back() == '\'' && type.size() >= 2) {
type = type.substr(1, type.size() - 2);
}
return type;
}

AzureFileHandle::AzureFileHandle(AzureStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags,
FileType file_type, const AzureOptions &options)
FileType file_type, const AzureOptions &options,
optional_ptr<AzureMetadataCache> metadata_cache_p)
: FileHandle(fs, info.path, flags), flags(flags),
// File info
is_remote_loaded(false), file_type(file_type), length(0), last_modified(0),
// Read info
buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0),
// Options
options(options) {
options(options), metadata_cache(metadata_cache_p) {
if (!flags.RequireParallelAccess() && !flags.DirectIO()) {
read_buffer = duckdb::unique_ptr<data_t[]>(new data_t[options.read_buffer_size]);
}

// Set metadata of file when available, it avoids to invoke to the storage to get them.
bool has_file_type = file_type != FileType::FILE_TYPE_INVALID;
bool has_length = false;
bool has_last_modified = false;
if (info.extended_info) {
auto type_entry = info.extended_info->options.find("type");
if (type_entry != info.extended_info->options.end()) {
auto type = GetFileType(type_entry->second);
if (type == "directory") {
file_type = FileType::FILE_TYPE_DIR;
has_file_type = true;
} else if (type == "file") {
file_type = FileType::FILE_TYPE_REGULAR;
has_file_type = true;
}
}
auto entry1 = info.extended_info->options.find("file_size");
if (entry1 != info.extended_info->options.end()) {
length = entry1->second.GetValue<uint64_t>();
has_length = true;
}
auto entry2 = info.extended_info->options.find("last_modified");
if (entry2 != info.extended_info->options.end()) {
last_modified = entry2->second.GetValue<timestamp_t>();
has_last_modified = true;
}
}
if (has_file_type && has_last_modified && (file_type == FileType::FILE_TYPE_DIR || has_length)) {
SetFileInfo(file_type, length, last_modified);
}
}

void AzureFileHandle::SetFileInfo(FileType file_type_p, idx_t length_p, timestamp_t last_modified_p) {
is_remote_loaded = true;
file_type = file_type_p;
length = file_type_p == FileType::FILE_TYPE_DIR ? 0 : length_p;
last_modified = last_modified_p;
file_offset = 0;
}

bool AzureFileHandle::PostConstruct() {
return static_cast<AzureStorageFileSystem &>(file_system).LoadFileInfo(*this);
}

static bool CanUseMetadataCache(const AzureFileHandle &handle) {
return handle.metadata_cache && !handle.flags.OpenForWriting() && !handle.flags.OpenForAppending() &&
!handle.flags.ExclusiveCreate();
}

static AzureFileInfo GetCacheEntry(const AzureFileHandle &handle) {
AzureFileInfo info;
info.file_type = handle.file_type;
info.length = handle.length;
info.last_modified = handle.last_modified;
return info;
}

bool AzureStorageFileSystem::ParseAzureMetadataCacheEnabled(optional_ptr<FileOpener> opener) {
Value metadata_cache_val;
if (FileOpener::TryGetCurrentSetting(opener, "enable_http_metadata_cache", metadata_cache_val) &&
!metadata_cache_val.IsNull()) {
return metadata_cache_val.GetValue<bool>();
}
return false;
}

optional_ptr<AzureMetadataCache> AzureStorageFileSystem::GetGlobalMetadataCache() {
lock_guard<mutex> lock(global_cache_lock);
if (!global_metadata_cache) {
global_metadata_cache = make_uniq<AzureMetadataCache>(false);
}
return global_metadata_cache.get();
}

optional_ptr<AzureMetadataCache> AzureStorageFileSystem::GetMetadataCache(optional_ptr<FileOpener> opener) {
auto db = FileOpener::TryGetDatabase(opener);
auto client_context = FileOpener::TryGetClientContext(opener);
if (!db) {
return nullptr;
}
if (ParseAzureMetadataCacheEnabled(opener)) {
return GetGlobalMetadataCache();
}
if (client_context) {
return client_context->registered_state->GetOrCreate<AzureMetadataCache>("azure_metadata_cache", true).get();
}
return nullptr;
}

void AzureStorageFileSystem::InvalidateMetadata(optional_ptr<FileOpener> opener, const string &path) {
auto metadata_cache = GetMetadataCache(opener);
if (metadata_cache) {
metadata_cache->Erase(path);
}
}

bool AzureStorageFileSystem::LoadFileInfo(AzureFileHandle &handle) {
try {
if (handle.IsRemoteLoaded()) {
if (CanUseMetadataCache(handle)) {
handle.metadata_cache->Insert(handle.path, GetCacheEntry(handle));
}
if (handle.flags.ReturnNullIfExists()) {
return false;
}
return true;
}
if (CanUseMetadataCache(handle)) {
AzureFileInfo cached_info;
if (handle.metadata_cache->Find(handle.path, cached_info)) {
handle.SetFileInfo(cached_info.file_type, cached_info.length, cached_info.last_modified);
if (handle.flags.ReturnNullIfExists()) {
return false;
}
return true;
}
}

LoadRemoteFileInfo(handle);
if (CanUseMetadataCache(handle)) {
handle.metadata_cache->Insert(handle.path, GetCacheEntry(handle));
}
if (handle.flags.ReturnNullIfExists()) {
return false;
}
Expand Down
3 changes: 2 additions & 1 deletion src/include/azure_blob_filesystem.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ class AzureBlobStorageFileSystem;
class AzureBlobStorageFileHandle : public AzureFileHandle {
public:
AzureBlobStorageFileHandle(AzureBlobStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags,
const AzureOptions &options, Azure::Storage::Blobs::BlockBlobClient blob_client);
const AzureOptions &options, optional_ptr<AzureMetadataCache> metadata_cache,
Azure::Storage::Blobs::BlockBlobClient blob_client);
~AzureBlobStorageFileHandle() override = default;

void StageWriteBuffer();
Expand Down
3 changes: 2 additions & 1 deletion src/include/azure_dfs_filesystem.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ class AzureDfsStorageFileSystem;
class AzureDfsStorageFileHandle : public AzureFileHandle {
public:
AzureDfsStorageFileHandle(AzureDfsStorageFileSystem &fs, const OpenFileInfo &info, FileOpenFlags flags,
const AzureOptions &options, Azure::Storage::Files::DataLake::DataLakeFileClient client);
const AzureOptions &options, optional_ptr<AzureMetadataCache> metadata_cache,
Azure::Storage::Files::DataLake::DataLakeFileClient client);
~AzureDfsStorageFileHandle() override = default;

void StageWriteBuffer();
Expand Down
Loading
Loading