From d607f613bdf0b5d8cbe2d1e3081022554f92af2c Mon Sep 17 00:00:00 2001 From: ben fleis Date: Tue, 3 Mar 2026 00:39:39 +0100 Subject: [PATCH 1/2] add credential caching; invalidate upon secret change Testing so far is [a] confirming that existing behavior works, and [b] manually checking call counts to oath2. This works via: ``` AZURE_LOG_LEVEL=verbose 2>&1 | tee test.log grep -c test.log ``` Specifically, spn_auth.test drops from 3 -> 1 oauth2 calls, and across the entire test suite (non-slow), oauth2 calls drop from 260 -> 26. So it's definitely working within context. Additionally tested credential invalidations when SECRET replaced, confirmed that the SECRET forces the minting of a new token. --- scripts/env_azure | 2 +- scripts/env_azurite | 10 +-- src/azure_storage_account_client.cpp | 129 +++++++++++++++++++++++---- test/sql/cloud/spn_auth.test | 16 ++++ 4 files changed, 134 insertions(+), 23 deletions(-) diff --git a/scripts/env_azure b/scripts/env_azure index bcb8721..19b6885 100755 --- a/scripts/env_azure +++ b/scripts/env_azure @@ -60,4 +60,4 @@ export AZURE_PROVIDER=cloud shift } -exec "$@" +exec env "$@" diff --git a/scripts/env_azurite b/scripts/env_azurite index 1f91cff..0355d60 100755 --- a/scripts/env_azurite +++ b/scripts/env_azurite @@ -24,12 +24,12 @@ export TEMP_DIR="$AZ_TEMPDIR" # for compatibility, accept --az arg, and fail on --abfss [[ -n "$1" && "$1" = "--abfss" ]] && { - echo "run_azurite can only operate in --az mode, aborting." >&2 - exit + echo "run_azurite can only operate in --az mode, aborting." >&2 + exit } [[ -n "$1" && "$1" = "--az" ]] && { - # NOOP - shift + # NOOP + shift } -exec "$@" +exec env "$@" diff --git a/src/azure_storage_account_client.cpp b/src/azure_storage_account_client.cpp index e50ebf5..b705179 100644 --- a/src/azure_storage_account_client.cpp +++ b/src/azure_storage_account_client.cpp @@ -3,6 +3,7 @@ #include "duckdb/catalog/catalog_transaction.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/file_opener.hpp" +#include "duckdb/common/helper.hpp" #include "duckdb/common/shared_ptr.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/main/client_context.hpp" @@ -225,6 +226,49 @@ CreateAccessTokenCredential(const KeyValueSecret &secret) { return std::make_shared(access_token); } +// Credential cache entry: persists across QueryEnd() calls since we intentionally +// don't override QueryEnd(). The Azure SDK's internal TokenCache handles per-token +// expiry and refresh — we just need to keep the credential object alive. +struct AzureCredentialState : public ClientContextState { + AzureCredentialState(std::shared_ptr cred, string fp) + : credential(std::move(cred)), fingerprint(std::move(fp)) { + } + std::shared_ptr credential; + std::string fingerprint; +}; + +static std::string SecretFingerprint(const KeyValueSecret &secret) { + std::string fp; + // NOTE: this key ordering is safe because secret_map is an ordered tree + for (const auto &kv : secret.secret_map) { + fp += kv.first + "=" + kv.second.ToString() + ";"; + } + return fp; +} + +static std::shared_ptr +GetCachedCredential(optional_ptr opener, const std::string &key, const std::string &fp) { + auto ctx = FileOpener::TryGetClientContext(opener); + if (!ctx) { + return nullptr; + } + auto entry = ctx->registered_state->Get(key); + if (!entry || entry->fingerprint != fp) { + return nullptr; + } + return entry->credential; +} + +static void CacheCredential(optional_ptr opener, const std::string &key, + std::shared_ptr cred, const std::string &fp) { + auto ctx = FileOpener::TryGetClientContext(opener); + if (!ctx) { + return; + } + ctx->registered_state->Remove(key); + ctx->registered_state->Insert(key, make_shared_ptr(std::move(cred), fp)); +} + static std::shared_ptr CreateCurlTransport(const std::string &proxy, const std::string &proxy_username, const std::string &proxy_password) { Azure::Core::Http::CurlTransportOptions curl_transport_options; @@ -388,8 +432,14 @@ static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClientFromCredentialChainProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - // Create credential chain - auto credential = CreateChainedTokenCredential(secret, transport_options); + + auto fp = SecretFingerprint(secret); + auto cache_key = std::string("azure_cred:") + secret.GetName(); + auto credential = GetCachedCredential(opener, cache_key, fp); + if (!credential) { + credential = CreateChainedTokenCredential(secret, transport_options); + CacheCredential(opener, cache_key, credential, fp); + } // Connect to storage account auto account_url = @@ -402,8 +452,14 @@ static Azure::Storage::Files::DataLake::DataLakeServiceClient GetDfsStorageAccountClientFromCredentialChainProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - // Create credential chain - auto credential = CreateChainedTokenCredential(secret, transport_options); + + auto fp = SecretFingerprint(secret); + auto cache_key = std::string("azure_cred:") + secret.GetName(); + auto credential = GetCachedCredential(opener, cache_key, fp); + if (!credential) { + credential = CreateChainedTokenCredential(secret, transport_options); + CacheCredential(opener, cache_key, credential, fp); + } // Connect to storage account auto account_url = @@ -448,37 +504,58 @@ GetManagedIdentityCredential(optional_ptr opener, const KeyValueSecr static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClientFromManagedIdentityProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { - // Connect to (blob) storage account auto transport_options = GetTransportOptions(opener, secret); - auto mi_cred = GetManagedIdentityCredential(opener, secret, transport_options); + + auto fp = SecretFingerprint(secret); + auto cache_key = std::string("azure_cred:") + secret.GetName(); + auto credential = GetCachedCredential(opener, cache_key, fp); + if (!credential) { + credential = GetManagedIdentityCredential(opener, secret, transport_options); + CacheCredential(opener, cache_key, credential, fp); + } + + // Connect to (blob) storage account auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_BLOB_ENDPOINT); auto blob_options = ToBlobClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Blobs::BlobServiceClient(account_url, mi_cred, blob_options); + return Azure::Storage::Blobs::BlobServiceClient(account_url, credential, blob_options); } static Azure::Storage::Files::DataLake::DataLakeServiceClient GetDfsStorageAccountClientFromManagedIdentityProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - auto mi_cred = GetManagedIdentityCredential(opener, secret, transport_options); + + auto fp = SecretFingerprint(secret); + auto cache_key = std::string("azure_cred:") + secret.GetName(); + auto credential = GetCachedCredential(opener, cache_key, fp); + if (!credential) { + credential = GetManagedIdentityCredential(opener, secret, transport_options); + CacheCredential(opener, cache_key, credential, fp); + } // Connect to ADLS storage account auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_DFS_ENDPOINT); auto dfs_options = ToDfsClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, mi_cred, dfs_options); + return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, credential, dfs_options); } static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClientFromServicePrincipalProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - auto token_credential = CreateClientCredential(secret, transport_options); + + auto fp = SecretFingerprint(secret); + auto cache_key = std::string("azure_cred:") + secret.GetName(); + auto token_credential = GetCachedCredential(opener, cache_key, fp); + if (!token_credential) { + token_credential = CreateClientCredential(secret, transport_options); + CacheCredential(opener, cache_key, token_credential, fp); + } auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_BLOB_ENDPOINT); - ; auto blob_options = ToBlobClientOptions(transport_options, GetHttpState(opener)); return Azure::Storage::Blobs::BlobServiceClient(account_url, token_credential, blob_options); } @@ -487,11 +564,17 @@ static Azure::Storage::Files::DataLake::DataLakeServiceClient GetDfsStorageAccountClientFromServicePrincipalProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - auto token_credential = CreateClientCredential(secret, transport_options); + + auto fp = SecretFingerprint(secret); + auto cache_key = std::string("azure_cred:") + secret.GetName(); + auto token_credential = GetCachedCredential(opener, cache_key, fp); + if (!token_credential) { + token_credential = CreateClientCredential(secret, transport_options); + CacheCredential(opener, cache_key, token_credential, fp); + } auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_DFS_ENDPOINT); - ; auto dfs_options = ToDfsClientOptions(transport_options, GetHttpState(opener)); return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, token_credential, dfs_options); } @@ -500,11 +583,17 @@ static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClientFromAccessTokenProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - auto token_credential = CreateAccessTokenCredential(secret); + + auto fp = SecretFingerprint(secret); + auto cache_key = std::string("azure_cred:") + secret.GetName(); + auto token_credential = GetCachedCredential(opener, cache_key, fp); + if (!token_credential) { + token_credential = CreateAccessTokenCredential(secret); + CacheCredential(opener, cache_key, token_credential, fp); + } auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_BLOB_ENDPOINT); - ; auto blob_options = ToBlobClientOptions(transport_options, GetHttpState(opener)); return Azure::Storage::Blobs::BlobServiceClient(account_url, token_credential, blob_options); } @@ -513,11 +602,17 @@ static Azure::Storage::Files::DataLake::DataLakeServiceClient GetDfsStorageAccountClientFromAccessTokenProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - auto token_credential = CreateAccessTokenCredential(secret); + + auto fp = SecretFingerprint(secret); + auto cache_key = std::string("azure_cred:") + secret.GetName(); + auto token_credential = GetCachedCredential(opener, cache_key, fp); + if (!token_credential) { + token_credential = CreateAccessTokenCredential(secret); + CacheCredential(opener, cache_key, token_credential, fp); + } auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_DFS_ENDPOINT); - ; auto dfs_options = ToDfsClientOptions(transport_options, GetHttpState(opener)); return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, token_credential, dfs_options); } diff --git a/test/sql/cloud/spn_auth.test b/test/sql/cloud/spn_auth.test index 5812353..e65fb6f 100644 --- a/test/sql/cloud/spn_auth.test +++ b/test/sql/cloud/spn_auth.test @@ -17,6 +17,10 @@ require-env AZ_STORAGE_ACCOUNT require-env AZ_DATA_DIR +# setup: Validate credential caching in az cli use +statement ok +CALL enable_logging('HTTP'); + statement error SELECT count(*) FROM 'azure://${AZ_DATA_DIR}/l.parquet'; ---- @@ -41,3 +45,15 @@ query I FROM glob('az://${AZ_DATA_DIR}/*.parquet'); ---- az://${AZ_DATA_DIR}/l.parquet + +# TODO: validate that credential call only occurs 1x +# - BLOCKED by lack of HTTP headers in Azure - see +# - demonstrate credential cache by call/header check for /oauth2/ +# - demonstrate invalidation by REPLACE SECRET, call/header check +# - duplicate same to SPN auth +# +# Manual tests: +# - full suite of non slow tests goes from 260 -> 26 calls to Azure's /oauth2/ +# - this test alone goes from 3 -> 1 calls +statement ok +FROM duckdb_logs_parsed('HTTP'); From 2e0a6713126caa6f80b4c120d5c287a592a3b71a Mon Sep 17 00:00:00 2001 From: ben fleis Date: Wed, 4 Mar 2026 23:59:26 +0100 Subject: [PATCH 2/2] feedback + add CACHE_TOKEN_CREDENTIAL flag - rework caching/invalidation a bit - add CACHE_TOKEN_CREDENTIAL flag to disable for e.g. `az account logout` - DRY up cache handling --- README.md | 24 ++++- src/azure_secret.cpp | 8 ++ src/azure_storage_account_client.cpp | 156 +++++++++++---------------- 3 files changed, 94 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 0dc05d6..ac828be 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,28 @@ IDs may be specified. See also [Azure Identity Managed Identity Support](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#managed-identity-support) +### Token Credential Caching + +By default, credential objects are reused across SQL statements so the Azure SDK's internal token +cache survives query boundaries and avoids redundant authentication round-trips. To disable this, +set `CACHE_TOKEN_CREDENTIAL = false`: + +```sql +CREATE SECRET secret_cli ( + TYPE AZURE, + PROVIDER CREDENTIAL_CHAIN, + CHAIN 'cli', + ACCOUNT_NAME '⟨storage account name⟩', + CACHE_TOKEN_CREDENTIAL false +); +``` + +Updating or replacing an Azure secret invalidates its cached credential; the next query +will create a fresh one. However, out-of-band credential changes — such as `az account logout` — +cannot be detected and will not invalidate the cache. Setting `CACHE_TOKEN_CREDENTIAL = false` +disables caching for that secret, so a new credential is created from scratch for each query, +ensuring any behind-the-scenes credential changes are picked up promptly. + ## Supported architectures The extension is tested & distributed for Linux (x64, arm64), MacOS (x64, arm64) and Windows (x64) @@ -122,4 +144,4 @@ cd duckdb_azure GEN=ninja VCPKG_TOOLCHAIN_PATH=$PWD/../vcpkg/scripts/buildsystems/vcpkg.cmake make ``` -Please also refer to our [Build Guide](https://duckdb.org/dev/building) and [Contribution Guide]([CONTRIBUTING.md](https://github.com/duckdb/duckdb/blob/main/CONTRIBUTING.md)). +Please also refer to our [Build Guide](https://duckdb.org/dev/building) and [Contribution Guide](<[CONTRIBUTING.md](https://github.com/duckdb/duckdb/blob/main/CONTRIBUTING.md)>). diff --git a/src/azure_secret.cpp b/src/azure_secret.cpp index 2e39f21..96f79eb 100644 --- a/src/azure_secret.cpp +++ b/src/azure_secret.cpp @@ -76,6 +76,7 @@ static unique_ptr CreateAzureSecretFromCredentialChain(ClientContext // Manage specific secret option CopySecret("chain", input, *result); + CopySecret("cache_token_credential", input, *result); // Redact sensible keys RedactCommonKeys(*result); @@ -103,6 +104,7 @@ static unique_ptr CreateAzureSecretFromManagedIdentity(ClientContext CopySecret("client_id", input, *result); CopySecret("object_id", input, *result); CopySecret("resource_id", input, *result); + CopySecret("cache_token_credential", input, *result); // Redact sensible keys RedactCommonKeys(*result); @@ -131,6 +133,7 @@ static unique_ptr CreateAzureSecretFromServicePrincipal(ClientContex CopySecret("client_id", input, *result); CopySecret("client_secret", input, *result); CopySecret("client_certificate_path", input, *result); + CopySecret("cache_token_credential", input, *result); // Redact sensible keys RedactCommonKeys(*result); @@ -158,6 +161,7 @@ static unique_ptr CreateAzureSecretFromAccessToken(ClientContext &co // Manage specific secret option CopySecret("access_token", input, *result); + CopySecret("cache_token_credential", input, *result); // Redact sensible keys RedactCommonKeys(*result); @@ -196,6 +200,7 @@ void CreateAzureSecretFunctions::Register(ExtensionLoader &loader) { // Register the credential_chain secret provider CreateSecretFunction cred_chain_function = {type, "credential_chain", CreateAzureSecretFromCredentialChain}; cred_chain_function.named_parameters["chain"] = LogicalType::VARCHAR; + cred_chain_function.named_parameters["cache_token_credential"] = LogicalType::BOOLEAN; RegisterCommonSecretParameters(cred_chain_function); loader.RegisterFunction(cred_chain_function); @@ -204,6 +209,7 @@ void CreateAzureSecretFunctions::Register(ExtensionLoader &loader) { managed_identity_function.named_parameters["client_id"] = LogicalType::VARCHAR; managed_identity_function.named_parameters["object_id"] = LogicalType::VARCHAR; managed_identity_function.named_parameters["resource_id"] = LogicalType::VARCHAR; + managed_identity_function.named_parameters["cache_token_credential"] = LogicalType::BOOLEAN; RegisterCommonSecretParameters(managed_identity_function); loader.RegisterFunction(managed_identity_function); @@ -214,12 +220,14 @@ void CreateAzureSecretFunctions::Register(ExtensionLoader &loader) { service_principal_function.named_parameters["client_id"] = LogicalType::VARCHAR; service_principal_function.named_parameters["client_secret"] = LogicalType::VARCHAR; service_principal_function.named_parameters["client_certificate_path"] = LogicalType::VARCHAR; + service_principal_function.named_parameters["cache_token_credential"] = LogicalType::BOOLEAN; RegisterCommonSecretParameters(service_principal_function); loader.RegisterFunction(service_principal_function); // Register the access_token secret provider CreateSecretFunction access_token_function = {type, "access_token", CreateAzureSecretFromAccessToken}; access_token_function.named_parameters["access_token"] = LogicalType::VARCHAR; + access_token_function.named_parameters["cache_token_credential"] = LogicalType::BOOLEAN; RegisterCommonSecretParameters(access_token_function); loader.RegisterFunction(access_token_function); } diff --git a/src/azure_storage_account_client.cpp b/src/azure_storage_account_client.cpp index b705179..9abbb86 100644 --- a/src/azure_storage_account_client.cpp +++ b/src/azure_storage_account_client.cpp @@ -10,6 +10,7 @@ #include "duckdb/main/secret/secret.hpp" #include "duckdb/main/secret/secret_manager.hpp" #include "http_state_policy.hpp" +#include "zstd/common/xxhash.hpp" #include #include @@ -237,13 +238,14 @@ struct AzureCredentialState : public ClientContextState { std::string fingerprint; }; -static std::string SecretFingerprint(const KeyValueSecret &secret) { - std::string fp; +static std::string FingerprintSecret(const KeyValueSecret &secret) { + std::string content; // NOTE: this key ordering is safe because secret_map is an ordered tree for (const auto &kv : secret.secret_map) { - fp += kv.first + "=" + kv.second.ToString() + ";"; + content += kv.first + "=" + kv.second.ToString() + ";"; } - return fp; + auto hash = duckdb_zstd::XXH64(content.c_str(), content.size(), 0); + return std::to_string(hash); } static std::shared_ptr @@ -259,8 +261,9 @@ GetCachedCredential(optional_ptr opener, const std::string &key, con return entry->credential; } -static void CacheCredential(optional_ptr opener, const std::string &key, - std::shared_ptr cred, const std::string &fp) { +static void CacheTokenCredential(optional_ptr opener, const std::string &key, + std::shared_ptr cred, + const std::string &fp) { auto ctx = FileOpener::TryGetClientContext(opener); if (!ctx) { return; @@ -428,44 +431,48 @@ GetDfsStorageAccountClientFromConfigProvider(optional_ptr opener, co return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, dfs_options); } +// Retrieves a cached credential by fingerprint, or creates and caches a new one via create_fn. +// If secret.cache_token_credential==false, then do not cache. (Useful e.g. when using the az cli, +// to where a logout is expected to have effect on next query.) +template +static std::shared_ptr +GetOrCreateCredential(optional_ptr opener, const KeyValueSecret &secret, CreateFn create_fn) { + auto fp = FingerprintSecret(secret); + auto cache_key = std::string("azure_cred:secret:") + secret.GetName(); + auto ctc_val = secret.TryGetValue("cache_token_credential"); + auto use_cache = ctc_val.IsNull() || ctc_val.GetValue(); + auto cred = use_cache ? GetCachedCredential(opener, cache_key, fp) : nullptr; + if (!cred) { + cred = create_fn(); + if (use_cache) { + CacheTokenCredential(opener, cache_key, cred, fp); + } + } + return cred; +} + static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClientFromCredentialChainProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - - auto fp = SecretFingerprint(secret); - auto cache_key = std::string("azure_cred:") + secret.GetName(); - auto credential = GetCachedCredential(opener, cache_key, fp); - if (!credential) { - credential = CreateChainedTokenCredential(secret, transport_options); - CacheCredential(opener, cache_key, credential, fp); - } - - // Connect to storage account + auto cred = GetOrCreateCredential(opener, secret, + [&]() { return CreateChainedTokenCredential(secret, transport_options); }); auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_BLOB_ENDPOINT); auto blob_options = ToBlobClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Blobs::BlobServiceClient(account_url, std::move(credential), blob_options); + return Azure::Storage::Blobs::BlobServiceClient(account_url, std::move(cred), blob_options); } static Azure::Storage::Files::DataLake::DataLakeServiceClient GetDfsStorageAccountClientFromCredentialChainProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - - auto fp = SecretFingerprint(secret); - auto cache_key = std::string("azure_cred:") + secret.GetName(); - auto credential = GetCachedCredential(opener, cache_key, fp); - if (!credential) { - credential = CreateChainedTokenCredential(secret, transport_options); - CacheCredential(opener, cache_key, credential, fp); - } - - // Connect to storage account + auto cred = GetOrCreateCredential(opener, secret, + [&]() { return CreateChainedTokenCredential(secret, transport_options); }); auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_DFS_ENDPOINT); auto dfs_options = ToDfsClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, std::move(credential), dfs_options); + return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, std::move(cred), dfs_options); } static std::shared_ptr @@ -505,116 +512,70 @@ static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClientFromManagedIdentityProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - - auto fp = SecretFingerprint(secret); - auto cache_key = std::string("azure_cred:") + secret.GetName(); - auto credential = GetCachedCredential(opener, cache_key, fp); - if (!credential) { - credential = GetManagedIdentityCredential(opener, secret, transport_options); - CacheCredential(opener, cache_key, credential, fp); - } - - // Connect to (blob) storage account + auto cred = GetOrCreateCredential( + opener, secret, [&]() { return GetManagedIdentityCredential(opener, secret, transport_options); }); auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_BLOB_ENDPOINT); auto blob_options = ToBlobClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Blobs::BlobServiceClient(account_url, credential, blob_options); + return Azure::Storage::Blobs::BlobServiceClient(account_url, cred, blob_options); } static Azure::Storage::Files::DataLake::DataLakeServiceClient GetDfsStorageAccountClientFromManagedIdentityProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - - auto fp = SecretFingerprint(secret); - auto cache_key = std::string("azure_cred:") + secret.GetName(); - auto credential = GetCachedCredential(opener, cache_key, fp); - if (!credential) { - credential = GetManagedIdentityCredential(opener, secret, transport_options); - CacheCredential(opener, cache_key, credential, fp); - } - - // Connect to ADLS storage account + auto cred = GetOrCreateCredential( + opener, secret, [&]() { return GetManagedIdentityCredential(opener, secret, transport_options); }); auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_DFS_ENDPOINT); auto dfs_options = ToDfsClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, credential, dfs_options); + return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, cred, dfs_options); } static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClientFromServicePrincipalProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - - auto fp = SecretFingerprint(secret); - auto cache_key = std::string("azure_cred:") + secret.GetName(); - auto token_credential = GetCachedCredential(opener, cache_key, fp); - if (!token_credential) { - token_credential = CreateClientCredential(secret, transport_options); - CacheCredential(opener, cache_key, token_credential, fp); - } - + auto cred = + GetOrCreateCredential(opener, secret, [&]() { return CreateClientCredential(secret, transport_options); }); auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_BLOB_ENDPOINT); auto blob_options = ToBlobClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Blobs::BlobServiceClient(account_url, token_credential, blob_options); + return Azure::Storage::Blobs::BlobServiceClient(account_url, cred, blob_options); } static Azure::Storage::Files::DataLake::DataLakeServiceClient GetDfsStorageAccountClientFromServicePrincipalProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - - auto fp = SecretFingerprint(secret); - auto cache_key = std::string("azure_cred:") + secret.GetName(); - auto token_credential = GetCachedCredential(opener, cache_key, fp); - if (!token_credential) { - token_credential = CreateClientCredential(secret, transport_options); - CacheCredential(opener, cache_key, token_credential, fp); - } - + auto cred = + GetOrCreateCredential(opener, secret, [&]() { return CreateClientCredential(secret, transport_options); }); auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_DFS_ENDPOINT); auto dfs_options = ToDfsClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, token_credential, dfs_options); + return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, cred, dfs_options); } static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClientFromAccessTokenProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - - auto fp = SecretFingerprint(secret); - auto cache_key = std::string("azure_cred:") + secret.GetName(); - auto token_credential = GetCachedCredential(opener, cache_key, fp); - if (!token_credential) { - token_credential = CreateAccessTokenCredential(secret); - CacheCredential(opener, cache_key, token_credential, fp); - } - + auto cred = GetOrCreateCredential(opener, secret, [&]() { return CreateAccessTokenCredential(secret); }); auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_BLOB_ENDPOINT); auto blob_options = ToBlobClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Blobs::BlobServiceClient(account_url, token_credential, blob_options); + return Azure::Storage::Blobs::BlobServiceClient(account_url, cred, blob_options); } static Azure::Storage::Files::DataLake::DataLakeServiceClient GetDfsStorageAccountClientFromAccessTokenProvider(optional_ptr opener, const KeyValueSecret &secret, const AzureParsedUrl &azure_parsed_url) { auto transport_options = GetTransportOptions(opener, secret); - - auto fp = SecretFingerprint(secret); - auto cache_key = std::string("azure_cred:") + secret.GetName(); - auto token_credential = GetCachedCredential(opener, cache_key, fp); - if (!token_credential) { - token_credential = CreateAccessTokenCredential(secret); - CacheCredential(opener, cache_key, token_credential, fp); - } - + auto cred = GetOrCreateCredential(opener, secret, [&]() { return CreateAccessTokenCredential(secret); }); auto account_url = azure_parsed_url.is_fully_qualified ? AccountUrl(azure_parsed_url) : AccountUrl(secret, DEFAULT_DFS_ENDPOINT); auto dfs_options = ToDfsClientOptions(transport_options, GetHttpState(opener)); - return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, token_credential, dfs_options); + return Azure::Storage::Files::DataLake::DataLakeServiceClient(account_url, cred, dfs_options); } static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(optional_ptr opener, @@ -704,9 +665,18 @@ static Azure::Storage::Blobs::BlobServiceClient GetBlobStorageAccountClient(opti // Credential chain secret equivalent auto credential_chain = TryGetCurrentSetting(opener, "azure_credential_chain"); if (!credential_chain.empty()) { - auto credential = CreateChainedTokenCredential(credential_chain, transport_options); - - return Azure::Storage::Blobs::BlobServiceClient(account_url, std::move(credential), blob_options); + // Settings-based credentials use a singleton cache key; any change to + // azure_credential_chain (the only setting that determines credential type) + // produces a different fingerprint and evicts the cached credential. + auto cache_key = std::string("azure_cred:settings"); + auto hash = duckdb_zstd::XXH64(credential_chain.c_str(), credential_chain.size(), 0); + auto fp = std::to_string(hash); + auto cred = GetCachedCredential(opener, cache_key, fp); + if (!cred) { + cred = CreateChainedTokenCredential(credential_chain, transport_options); + CacheTokenCredential(opener, cache_key, cred, fp); + } + return Azure::Storage::Blobs::BlobServiceClient(account_url, std::move(cred), blob_options); } // Anonymous