From a7dfdbd92bb1c9790a761673fff97072f84e8e6f Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 27 May 2026 06:39:07 +0200 Subject: [PATCH 01/16] Add cfn_* CloudFormation table functions Initial commit lifting the cfn_* functions from the earlier prototype branch onto a clean main. Function names and shape are unchanged from the prototype; rename + outputs-fold + auto-tags + session-id follow as separate commits in this PR. - vcpkg.json: add cloudformation to aws-sdk-cpp features - CMakeLists.txt: add cloudformation to find_package COMPONENTS; add src/cfn_functions.cpp to EXTENSION_SOURCES - src/include/aws_client.hpp: new shared header with BuildClientConfigWithCa + BuildAwsCredentialsProvider declarations - src/aws_secret.cpp: extract BuildClientConfigWithCa to external linkage; add BuildAwsCredentialsProvider factory wrapping the existing DuckDBCustomAWSCredentialsProviderChain - src/aws_extension.cpp: register CfnFunctions in LoadInternal - src/cfn_functions.cpp + src/include/cfn_functions.hpp: cfn_create_stack, cfn_describe_stack, cfn_outputs, cfn_delete_stack - test/sql/cfn_functions.test: bind-time error smoke tests --- CMakeLists.txt | 4 +- src/aws_extension.cpp | 3 + src/aws_secret.cpp | 27 +- src/cfn_functions.cpp | 555 ++++++++++++++++++++++++++++++++++ src/include/aws_client.hpp | 29 ++ src/include/cfn_functions.hpp | 12 + test/sql/cfn_functions.test | 60 ++++ vcpkg.json | 2 +- 8 files changed, 681 insertions(+), 11 deletions(-) create mode 100644 src/cfn_functions.cpp create mode 100644 src/include/aws_client.hpp create mode 100644 src/include/cfn_functions.hpp create mode 100644 test/sql/cfn_functions.test diff --git a/CMakeLists.txt b/CMakeLists.txt index 74abeec..9c1be2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ set(EXTENSION_NAME ${TARGET_NAME}_extension) project(${TARGET_NAME}) include_directories(src/include) -set(EXTENSION_SOURCES src/aws_extension.cpp src/aws_secret.cpp) +set(EXTENSION_SOURCES src/aws_extension.cpp src/aws_secret.cpp src/cfn_functions.cpp) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) set(PARAMETERS "-warnings") @@ -16,7 +16,7 @@ build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES}) # Weirdly we need to manually to this, otherwise linking against # ${AWSSDK_LINK_LIBRARIES} fails for some reason find_package(ZLIB REQUIRED) -find_package(AWSSDK REQUIRED COMPONENTS core sso sts identity-management rds) +find_package(AWSSDK REQUIRED COMPONENTS core sso sts identity-management rds cloudformation) # Build static lib target_include_directories(${EXTENSION_NAME} diff --git a/src/aws_extension.cpp b/src/aws_extension.cpp index 75be7ce..434fec3 100644 --- a/src/aws_extension.cpp +++ b/src/aws_extension.cpp @@ -1,5 +1,6 @@ #include "aws_secret.hpp" #include "aws_extension.hpp" +#include "cfn_functions.hpp" #include "duckdb.hpp" #include "duckdb/common/exception.hpp" @@ -146,6 +147,8 @@ static void LoadInternal(ExtensionLoader &loader) { loader.RegisterFunction(function_set); CreateAwsSecretFunctions::Register(loader); + + CfnFunctions::Register(loader); } void AwsExtension::Load(ExtensionLoader &loader) { diff --git a/src/aws_secret.cpp b/src/aws_secret.cpp index 17e66bf..3bdfaa3 100644 --- a/src/aws_secret.cpp +++ b/src/aws_secret.cpp @@ -1,4 +1,5 @@ #include "aws_secret.hpp" +#include "aws_client.hpp" #include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/exception.hpp" @@ -43,14 +44,8 @@ static struct { const string default_ = "exists"; } CreateAwsSecretValidation; -//! Build a ClientConfiguration with the detected CA file path applied (if any). -//! This is required because libcurl is statically linked from a RHEL-based -//! manylinux image, so its default CA path is /etc/pki/tls/certs/ca-bundle.crt, -//! which does not exist on Debian/Ubuntu/Alpine/etc. Without this, every -//! credentials provider that does its own HTTPS (SSO, STS, RDS, ...) fails the -//! TLS handshake before it can fetch any credentials. -//! See duckdb/duckdb#20652, duckdb/duckdb-aws#131. -static Aws::Client::ClientConfiguration BuildClientConfigWithCa() { +//! See aws_client.hpp for rationale. +Aws::Client::ClientConfiguration BuildClientConfigWithCa() { Aws::Client::ClientConfiguration cfg; if (!SELECTED_CURL_CERT_PATH.empty()) { cfg.caFile = SELECTED_CURL_CERT_PATH; @@ -509,6 +504,22 @@ static unique_ptr CreateAWSSecretFromCredentialChain(ClientContext & return std::move(result); } +std::shared_ptr +BuildAwsCredentialsProvider(const std::string &chain, bool require_credentials, const std::string &profile, + const std::string &assume_role_arn, const std::string &external_id, + const std::string &web_identity_token_file, const std::string &session_name) { + if (chain.empty()) { + if (!profile.empty()) { + return Aws::MakeShared("DuckDBAwsProfile", + profile.c_str()); + } + return Aws::MakeShared("DuckDBAwsDefault"); + } + return Aws::MakeShared("DuckDBCustomChain", chain, require_credentials, + profile, assume_role_arn, external_id, + web_identity_token_file, session_name); +} + void CreateAwsSecretFunctions::InitializeCurlCertificates(DatabaseInstance &db) { for (string &caFile : certFileLocations) { struct stat buf; diff --git a/src/cfn_functions.cpp b/src/cfn_functions.cpp new file mode 100644 index 0000000..3bc645f --- /dev/null +++ b/src/cfn_functions.cpp @@ -0,0 +1,555 @@ +#include "cfn_functions.hpp" +#include "aws_client.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/uuid.hpp" +#include "duckdb/main/extension/extension_loader.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace duckdb { + +namespace { + +struct StringKV { + string key; + string value; +}; + +vector UnpackStringMap(const Value &map_value) { + vector out; + if (map_value.IsNull()) { + return out; + } + auto &children = MapValue::GetChildren(map_value); + for (auto &child : children) { + auto &kv = StructValue::GetChildren(child); + if (kv[0].IsNull()) { + continue; + } + out.push_back({StringValue::Get(kv[0]), kv[1].IsNull() ? string() : StringValue::Get(kv[1])}); + } + return out; +} + +const string *FindCI(const vector &kvs, const string &key) { + auto lowered = StringUtil::Lower(key); + for (auto &kv : kvs) { + if (StringUtil::Lower(kv.key) == lowered) { + return &kv.value; + } + } + return nullptr; +} + +bool LooksLikeUrl(const string &s) { + return StringUtil::StartsWith(s, "http://") || StringUtil::StartsWith(s, "https://"); +} + +//! Coerce a free-form string into a CFN-name-safe fragment ([a-zA-Z0-9-]). +string SanitizeNameFragment(const string &in) { + string out; + for (char c : in) { + if (std::isalnum(static_cast(c)) || c == '-') { + out += c; + } else { + out += '-'; + } + } + return out; +} + +//! Stem of a URL's last path segment: https://host/path/quack.yaml -> "quack". +string UrlBasenameStem(const string &url) { + auto slash = url.find_last_of('/'); + string base = (slash == string::npos) ? url : url.substr(slash + 1); + auto qmark = base.find('?'); + if (qmark != string::npos) { + base = base.substr(0, qmark); + } + auto dot = base.find_last_of('.'); + if (dot != string::npos && dot > 0) { + base = base.substr(0, dot); + } + return SanitizeNameFragment(base); +} + +//! 12 hex characters of randomness from a fresh UUID. Keeping suffixes short +//! ensures the resulting CFN ARN stays within CloudFormation's 128-char +//! StackName API limit even in the longest-named regions. +string ShortRandHex() { + auto uuid_str = UUID::ToString(UUID::GenerateRandomUUID()); + // UUID layout: 8-4-4-4-12. First 12 hex = first segment (8) + second segment (4). + return uuid_str.substr(0, 8) + uuid_str.substr(9, 4); +} + +//! CFN's StackName API parameter is capped at 128 chars. The ARN is +//! `arn:aws:cloudformation:::stack//` — +//! structural+region(≤14)+account(12)+slashes(2)+cfn-uuid(36) ≤ 94, so the +//! name budget is 128 - 94 = 34 chars (worst case across regions). +constexpr idx_t MAX_STACK_NAME_LEN = 34; +constexpr idx_t MAX_PREFIX_LEN = MAX_STACK_NAME_LEN - 12 - 1; // 21: '-<12hex>' + +//! Reserved option keys configure the AWS client; everything else in `options` +//! must match a parameter the template declares. +const std::set RESERVED_OPTION_KEYS = { + "region", "chain", "profile", "assume_role_arn", + "external_id", "web_identity_token_file", "session_name"}; + +} // namespace + +//===--------------------------------------------------------------------===// +// cfn_create_stack(template, name, options) [region :=, tags :=] +//===--------------------------------------------------------------------===// + +struct CfnCreateStackBindData : public TableFunctionData { + string template_arg; + string name_arg; + vector options; + bool has_region_override = false; + string region_override; + vector tags_override; + bool finished = false; +}; + +static unique_ptr CfnCreateStackBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + auto result = make_uniq(); + + if (input.inputs[0].IsNull()) { + throw InvalidInputException("cfn_create_stack: the template argument must not be NULL"); + } + result->template_arg = StringValue::Get(input.inputs[0]); + if (!input.inputs[1].IsNull()) { + result->name_arg = StringValue::Get(input.inputs[1]); + } + result->options = UnpackStringMap(input.inputs[2]); + + for (auto &np : input.named_parameters) { + auto key = StringUtil::Lower(np.first); + if (key == "region") { + if (!np.second.IsNull()) { + result->has_region_override = true; + result->region_override = StringValue::Get(np.second); + } + } else if (key == "tags") { + result->tags_override = UnpackStringMap(np.second); + } + } + + return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + names.emplace_back("handle"); + + return std::move(result); +} + +static void CfnCreateStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CfnCreateStackBindData &)*data_p.bind_data; + if (data.finished) { + return; + } + + // Region: explicit override wins over options['region']. + string region; + if (data.has_region_override) { + region = data.region_override; + } else if (auto *r = FindCI(data.options, "region")) { + region = *r; + } + if (region.empty()) { + throw InvalidInputException( + "cfn_create_stack: region is required - set options['region'] or the region:= named parameter"); + } + + auto opt = [&](const string &key) -> string { + auto *v = FindCI(data.options, key); + return v ? *v : string(); + }; + + auto provider = BuildAwsCredentialsProvider(opt("chain"), /*require_credentials=*/true, opt("profile"), + opt("assume_role_arn"), opt("external_id"), + opt("web_identity_token_file"), opt("session_name")); + auto client_config = BuildClientConfigWithCa(); + client_config.region = region.c_str(); + Aws::CloudFormation::CloudFormationClient cfn(provider, client_config); + + bool template_is_url = LooksLikeUrl(data.template_arg); + + // Ask CloudFormation to parse the template: declared parameters, required + // capabilities, and the Metadata section. + Aws::CloudFormation::Model::GetTemplateSummaryRequest summary_req; + if (template_is_url) { + summary_req.SetTemplateURL(data.template_arg.c_str()); + } else { + summary_req.SetTemplateBody(data.template_arg.c_str()); + } + auto summary_outcome = cfn.GetTemplateSummary(summary_req); + if (!summary_outcome.IsSuccess()) { + const auto &err = summary_outcome.GetError(); + throw IOException("CloudFormation GetTemplateSummary failed: %s - %s", string(err.GetExceptionName().c_str()), + string(err.GetMessage().c_str())); + } + const auto &summary = summary_outcome.GetResult(); + + std::set declared_params; + for (const auto &pd : summary.GetParameters()) { + declared_params.insert(string(pd.GetParameterKey().c_str())); + } + + string metadata_stack_name; + if (!summary.GetMetadata().empty()) { + Aws::Utils::Json::JsonValue meta(summary.GetMetadata()); + if (meta.WasParseSuccessful()) { + auto view = meta.View(); + if (view.KeyExists("StackName")) { + metadata_stack_name = string(view.GetString("StackName").c_str()); + } + } + } + + // Resolve the stack name. Cap autogenerated prefixes so the full ARN + // stays within CloudFormation's 128-char StackName API limit. Reject + // explicit names that exceed the limit. + string stack_name; + if (!data.name_arg.empty()) { + if (data.name_arg.size() > MAX_STACK_NAME_LEN) { + throw InvalidInputException( + "cfn_create_stack: explicit name '%s' is %llu chars; max %llu " + "(to keep the resulting CFN stack ARN within the 128-char API limit)", + data.name_arg, (unsigned long long)data.name_arg.size(), + (unsigned long long)MAX_STACK_NAME_LEN); + } + stack_name = data.name_arg; + } else { + string prefix; + if (!metadata_stack_name.empty()) { + prefix = SanitizeNameFragment(metadata_stack_name); + } else if (template_is_url) { + prefix = UrlBasenameStem(data.template_arg); + } + if (prefix.empty()) { + prefix = "cfn-stack"; + } + if (prefix.size() > MAX_PREFIX_LEN) { + prefix = prefix.substr(0, MAX_PREFIX_LEN); + } + stack_name = prefix + "-" + ShortRandHex(); + } + + // Route every non-reserved option key to a declared template parameter. + Aws::Vector cfn_params; + for (auto &kv : data.options) { + if (RESERVED_OPTION_KEYS.count(StringUtil::Lower(kv.key))) { + continue; + } + if (!declared_params.count(kv.key)) { + throw InvalidInputException( + "cfn_create_stack: option '%s' is not a parameter declared by the template", kv.key); + } + Aws::CloudFormation::Model::Parameter p; + p.SetParameterKey(kv.key.c_str()); + p.SetParameterValue(kv.value.c_str()); + cfn_params.push_back(p); + } + + // Tags: provenance, the metadata stack-name, then caller-supplied extras. + Aws::Vector cfn_tags; + auto add_tag = [&](const string &k, const string &v) { + Aws::CloudFormation::Model::Tag t; + t.SetKey(k.c_str()); + t.SetValue(v.c_str()); + cfn_tags.push_back(t); + }; + add_tag("created-by", "duckdb"); + if (!metadata_stack_name.empty()) { + add_tag("stack-name", metadata_stack_name); + } + for (auto &kv : data.tags_override) { + add_tag(kv.key, kv.value); + } + + Aws::CloudFormation::Model::CreateStackRequest req; + req.SetStackName(stack_name.c_str()); + if (template_is_url) { + req.SetTemplateURL(data.template_arg.c_str()); + } else { + req.SetTemplateBody(data.template_arg.c_str()); + } + if (!cfn_params.empty()) { + req.SetParameters(cfn_params); + } + if (!summary.GetCapabilities().empty()) { + req.SetCapabilities(summary.GetCapabilities()); + } + if (!cfn_tags.empty()) { + req.SetTags(cfn_tags); + } + + auto outcome = cfn.CreateStack(req); + if (!outcome.IsSuccess()) { + const auto &err = outcome.GetError(); + throw IOException("CloudFormation CreateStack failed: %s - %s", string(err.GetExceptionName().c_str()), + string(err.GetMessage().c_str())); + } + string stack_id(outcome.GetResult().GetStackId().c_str()); + + vector keys; + vector values; + keys.emplace_back("stack_name"); + values.emplace_back(stack_name); + keys.emplace_back("stack_id"); + values.emplace_back(stack_id); + keys.emplace_back("region"); + values.emplace_back(region); + auto handle = Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(keys), std::move(values)); + + output.SetValue(0, 0, handle); + output.SetCardinality(1); + data.finished = true; +} + +//===--------------------------------------------------------------------===// +// Shared handle parsing (cfn_describe_stack / cfn_outputs / cfn_delete_stack) +//===--------------------------------------------------------------------===// + +namespace { + +struct CfnHandle { + string stack_ref; // stack_id (ARN) when present, else stack_name + string stack_name; + string stack_id; + string region; +}; + +CfnHandle ParseHandle(const Value &handle_value, const char *fn_name) { + auto kvs = UnpackStringMap(handle_value); + CfnHandle h; + if (auto *v = FindCI(kvs, "stack_id")) { + h.stack_id = *v; + } + if (auto *v = FindCI(kvs, "stack_name")) { + h.stack_name = *v; + } + if (auto *v = FindCI(kvs, "region")) { + h.region = *v; + } + h.stack_ref = !h.stack_id.empty() ? h.stack_id : h.stack_name; + if (h.stack_ref.empty()) { + throw InvalidInputException("%s: handle must contain 'stack_id' or 'stack_name'", fn_name); + } + if (h.region.empty()) { + throw InvalidInputException("%s: handle is missing 'region'", fn_name); + } + return h; +} + +} // namespace + +//===--------------------------------------------------------------------===// +// cfn_describe_stack(handle) +//===--------------------------------------------------------------------===// + +struct CfnDescribeStackBindData : public TableFunctionData { + CfnHandle handle; + bool finished = false; +}; + +static unique_ptr CfnDescribeStackBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + auto result = make_uniq(); + result->handle = ParseHandle(input.inputs[0], "cfn_describe_stack"); + + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("stack_name"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("stack_id"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("status"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("status_reason"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("creation_time"); + + return std::move(result); +} + +static void CfnDescribeStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CfnDescribeStackBindData &)*data_p.bind_data; + if (data.finished) { + return; + } + + auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); + auto cfg = BuildClientConfigWithCa(); + cfg.region = data.handle.region.c_str(); + Aws::CloudFormation::CloudFormationClient cfn(provider, cfg); + + Aws::CloudFormation::Model::DescribeStacksRequest req; + req.SetStackName(data.handle.stack_ref.c_str()); + auto outcome = cfn.DescribeStacks(req); + if (!outcome.IsSuccess()) { + const auto &err = outcome.GetError(); + throw IOException("CloudFormation DescribeStacks failed: %s - %s", string(err.GetExceptionName().c_str()), + string(err.GetMessage().c_str())); + } + const auto &stacks = outcome.GetResult().GetStacks(); + if (stacks.empty()) { + throw IOException("CloudFormation DescribeStacks returned no stack for '%s'", data.handle.stack_ref); + } + const auto &stack = stacks[0]; + + string status( + Aws::CloudFormation::Model::StackStatusMapper::GetNameForStackStatus(stack.GetStackStatus()).c_str()); + string reason(stack.GetStackStatusReason().c_str()); + string created(stack.GetCreationTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + + output.SetValue(0, 0, Value(string(stack.GetStackName().c_str()))); + output.SetValue(1, 0, Value(string(stack.GetStackId().c_str()))); + output.SetValue(2, 0, Value(status)); + output.SetValue(3, 0, reason.empty() ? Value() : Value(reason)); + output.SetValue(4, 0, created.empty() ? Value() : Value(created)); + output.SetCardinality(1); + data.finished = true; +} + +//===--------------------------------------------------------------------===// +// cfn_outputs(handle) +//===--------------------------------------------------------------------===// + +struct CfnOutputsBindData : public TableFunctionData { + CfnHandle handle; + bool finished = false; +}; + +static unique_ptr CfnOutputsBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + auto result = make_uniq(); + result->handle = ParseHandle(input.inputs[0], "cfn_outputs"); + + return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + names.emplace_back("outputs"); + + return std::move(result); +} + +static void CfnOutputsFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CfnOutputsBindData &)*data_p.bind_data; + if (data.finished) { + return; + } + + auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); + auto cfg = BuildClientConfigWithCa(); + cfg.region = data.handle.region.c_str(); + Aws::CloudFormation::CloudFormationClient cfn(provider, cfg); + + Aws::CloudFormation::Model::DescribeStacksRequest req; + req.SetStackName(data.handle.stack_ref.c_str()); + auto outcome = cfn.DescribeStacks(req); + if (!outcome.IsSuccess()) { + const auto &err = outcome.GetError(); + throw IOException("CloudFormation DescribeStacks failed: %s - %s", string(err.GetExceptionName().c_str()), + string(err.GetMessage().c_str())); + } + const auto &stacks = outcome.GetResult().GetStacks(); + if (stacks.empty()) { + throw IOException("CloudFormation DescribeStacks returned no stack for '%s'", data.handle.stack_ref); + } + + vector keys; + vector values; + for (const auto &o : stacks[0].GetOutputs()) { + keys.emplace_back(string(o.GetOutputKey().c_str())); + values.emplace_back(string(o.GetOutputValue().c_str())); + } + auto outputs = Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(keys), std::move(values)); + + output.SetValue(0, 0, outputs); + output.SetCardinality(1); + data.finished = true; +} + +//===--------------------------------------------------------------------===// +// cfn_delete_stack(handle) +//===--------------------------------------------------------------------===// + +struct CfnDeleteStackBindData : public TableFunctionData { + CfnHandle handle; + bool finished = false; +}; + +static unique_ptr CfnDeleteStackBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + auto result = make_uniq(); + result->handle = ParseHandle(input.inputs[0], "cfn_delete_stack"); + + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("stack_id"); + + return std::move(result); +} + +static void CfnDeleteStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CfnDeleteStackBindData &)*data_p.bind_data; + if (data.finished) { + return; + } + + auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); + auto cfg = BuildClientConfigWithCa(); + cfg.region = data.handle.region.c_str(); + Aws::CloudFormation::CloudFormationClient cfn(provider, cfg); + + Aws::CloudFormation::Model::DeleteStackRequest req; + req.SetStackName(data.handle.stack_ref.c_str()); + auto outcome = cfn.DeleteStack(req); + if (!outcome.IsSuccess()) { + const auto &err = outcome.GetError(); + throw IOException("CloudFormation DeleteStack failed: %s - %s", string(err.GetExceptionName().c_str()), + string(err.GetMessage().c_str())); + } + + output.SetValue(0, 0, Value(data.handle.stack_ref)); + output.SetCardinality(1); + data.finished = true; +} + +//===--------------------------------------------------------------------===// +// Registration +//===--------------------------------------------------------------------===// + +void CfnFunctions::Register(ExtensionLoader &loader) { + auto map_vv = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); + + TableFunction create_fn("cfn_create_stack", {LogicalType::VARCHAR, LogicalType::VARCHAR, map_vv}, + CfnCreateStackFun, CfnCreateStackBind); + create_fn.named_parameters["region"] = LogicalType::VARCHAR; + create_fn.named_parameters["tags"] = map_vv; + loader.RegisterFunction(create_fn); + + TableFunction describe_fn("cfn_describe_stack", {map_vv}, CfnDescribeStackFun, CfnDescribeStackBind); + loader.RegisterFunction(describe_fn); + + TableFunction outputs_fn("cfn_outputs", {map_vv}, CfnOutputsFun, CfnOutputsBind); + loader.RegisterFunction(outputs_fn); + + TableFunction delete_fn("cfn_delete_stack", {map_vv}, CfnDeleteStackFun, CfnDeleteStackBind); + loader.RegisterFunction(delete_fn); +} + +} // namespace duckdb diff --git a/src/include/aws_client.hpp b/src/include/aws_client.hpp new file mode 100644 index 0000000..670d8e8 --- /dev/null +++ b/src/include/aws_client.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include + +namespace duckdb { + +//! Build a ClientConfiguration with the detected CA file path applied (if any). +//! Required because libcurl is statically linked from a RHEL-based manylinux +//! image, so its default CA path is /etc/pki/tls/certs/ca-bundle.crt, which +//! does not exist on Debian/Ubuntu/Alpine/etc. Without this, every AWS SDK +//! client that does its own HTTPS fails the TLS handshake. +//! See duckdb/duckdb#20652, duckdb/duckdb-aws#131. +Aws::Client::ClientConfiguration BuildClientConfigWithCa(); + +//! Build a credentials provider from the same opt-set that +//! CREATE SECRET ... credential_chain understands. Falls back to the SDK's +//! DefaultAWSCredentialsProviderChain when no chain/profile is specified. +//! `chain` is a ';'-separated list of provider names; see +//! DuckDBCustomAWSCredentialsProviderChain in aws_secret.cpp for the grammar. +std::shared_ptr +BuildAwsCredentialsProvider(const std::string &chain, bool require_credentials, + const std::string &profile = "", const std::string &assume_role_arn = "", + const std::string &external_id = "", const std::string &web_identity_token_file = "", + const std::string &session_name = ""); + +} // namespace duckdb diff --git a/src/include/cfn_functions.hpp b/src/include/cfn_functions.hpp new file mode 100644 index 0000000..ddc61fc --- /dev/null +++ b/src/include/cfn_functions.hpp @@ -0,0 +1,12 @@ +#pragma once + +namespace duckdb { + +class ExtensionLoader; + +struct CfnFunctions { + //! Register the generic CloudFormation table functions (cfn_*). + static void Register(ExtensionLoader &loader); +}; + +} // namespace duckdb diff --git a/test/sql/cfn_functions.test b/test/sql/cfn_functions.test new file mode 100644 index 0000000..b3b4c38 --- /dev/null +++ b/test/sql/cfn_functions.test @@ -0,0 +1,60 @@ +# name: test/sql/cfn_functions.test +# description: smoke test for the generic cfn_* CloudFormation functions +# group: [sql] + +require no_extension_autoloading "EXPECTED: Test relies on explicit INSTALL and LOAD" + +# Before LOAD aws, the functions do not exist. +statement error +SELECT * FROM cfn_create_stack('x', NULL, MAP {'region': 'us-east-1'}); +---- + +require aws + +# cfn_create_stack: a NULL template is rejected at bind time. +statement error +SELECT * FROM cfn_create_stack(NULL, NULL, MAP {'region': 'us-east-1'}); +---- +template argument must not be NULL + +# cfn_create_stack: region is required (checked before any AWS call). +statement error +SELECT * FROM cfn_create_stack('AWSTemplateFormatVersion: 2010-09-09', NULL, MAP {'InstanceType': 't3.micro'}); +---- +region is required + +# cfn_describe_stack: handle without a stack reference. +statement error +SELECT * FROM cfn_describe_stack(MAP {'region': 'us-east-1'}); +---- +handle must contain 'stack_id' or 'stack_name' + +# cfn_describe_stack: handle missing region. +statement error +SELECT * FROM cfn_describe_stack(MAP {'stack_name': 's'}); +---- +handle is missing 'region' + +# cfn_outputs: handle without a stack reference. +statement error +SELECT * FROM cfn_outputs(MAP {'region': 'us-east-1'}); +---- +handle must contain 'stack_id' or 'stack_name' + +# cfn_outputs: handle missing region. +statement error +SELECT * FROM cfn_outputs(MAP {'stack_name': 's'}); +---- +handle is missing 'region' + +# cfn_delete_stack: handle without a stack reference. +statement error +SELECT * FROM cfn_delete_stack(MAP {'region': 'us-east-1'}); +---- +handle must contain 'stack_id' or 'stack_name' + +# cfn_delete_stack: handle missing region. +statement error +SELECT * FROM cfn_delete_stack(MAP {'stack_name': 's'}); +---- +handle is missing 'region' diff --git a/vcpkg.json b/vcpkg.json index b2800aa..d9d1569 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -4,7 +4,7 @@ "zlib", { "name": "aws-sdk-cpp", - "features": [ "sso", "sts" , "identity-management", "rds"] + "features": [ "sso", "sts" , "identity-management", "rds", "cloudformation"] }, "openssl", "curl" From c77d5531ffca8cccf03dea97d8f44de0cfedd374 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 27 May 2026 06:43:41 +0200 Subject: [PATCH 02/16] Rename cfn_* -> cloudformation_* Function names now mirror the AWS service name (cloudformation), matching how boto3 (boto3.client('cloudformation')) and the AWS CLI (aws cloudformation ...) refer to it. No behavior change, pure rename. - src/cfn_functions.{cpp,hpp} -> src/cloudformation_functions.{cpp,hpp} - test/sql/cfn_functions.test -> test/sql/cloudformation_functions.test - cfn_create_stack -> cloudformation_create_stack - cfn_describe_stack -> cloudformation_describe_stack - cfn_outputs -> cloudformation_outputs - cfn_delete_stack -> cloudformation_delete_stack - CfnFunctions -> CloudFormationFunctions (and all internal Cfn* types) - Fallback autogen stack-name prefix: "cfn-stack" -> "duckdb-aws" (consistent with the planned created-by tag value). --- CMakeLists.txt | 2 +- src/aws_extension.cpp | 4 +- ...tions.cpp => cloudformation_functions.cpp} | 108 +++++++++--------- ...tions.hpp => cloudformation_functions.hpp} | 4 +- test/sql/cfn_functions.test | 60 ---------- test/sql/cloudformation_functions.test | 60 ++++++++++ 6 files changed, 119 insertions(+), 119 deletions(-) rename src/{cfn_functions.cpp => cloudformation_functions.cpp} (80%) rename src/include/{cfn_functions.hpp => cloudformation_functions.hpp} (54%) delete mode 100644 test/sql/cfn_functions.test create mode 100644 test/sql/cloudformation_functions.test diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c1be2d..b9ea3af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ set(EXTENSION_NAME ${TARGET_NAME}_extension) project(${TARGET_NAME}) include_directories(src/include) -set(EXTENSION_SOURCES src/aws_extension.cpp src/aws_secret.cpp src/cfn_functions.cpp) +set(EXTENSION_SOURCES src/aws_extension.cpp src/aws_secret.cpp src/cloudformation_functions.cpp) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) set(PARAMETERS "-warnings") diff --git a/src/aws_extension.cpp b/src/aws_extension.cpp index 434fec3..2c29956 100644 --- a/src/aws_extension.cpp +++ b/src/aws_extension.cpp @@ -1,6 +1,6 @@ #include "aws_secret.hpp" #include "aws_extension.hpp" -#include "cfn_functions.hpp" +#include "cloudformation_functions.hpp" #include "duckdb.hpp" #include "duckdb/common/exception.hpp" @@ -148,7 +148,7 @@ static void LoadInternal(ExtensionLoader &loader) { CreateAwsSecretFunctions::Register(loader); - CfnFunctions::Register(loader); + CloudFormationFunctions::Register(loader); } void AwsExtension::Load(ExtensionLoader &loader) { diff --git a/src/cfn_functions.cpp b/src/cloudformation_functions.cpp similarity index 80% rename from src/cfn_functions.cpp rename to src/cloudformation_functions.cpp index 3bc645f..97754d2 100644 --- a/src/cfn_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -1,4 +1,4 @@ -#include "cfn_functions.hpp" +#include "cloudformation_functions.hpp" #include "aws_client.hpp" #include "duckdb.hpp" @@ -114,10 +114,10 @@ const std::set RESERVED_OPTION_KEYS = { } // namespace //===--------------------------------------------------------------------===// -// cfn_create_stack(template, name, options) [region :=, tags :=] +// cloudformation_create_stack(template, name, options) [region :=, tags :=] //===--------------------------------------------------------------------===// -struct CfnCreateStackBindData : public TableFunctionData { +struct CloudFormationCreateStackBindData : public TableFunctionData { string template_arg; string name_arg; vector options; @@ -127,12 +127,12 @@ struct CfnCreateStackBindData : public TableFunctionData { bool finished = false; }; -static unique_ptr CfnCreateStackBind(ClientContext &context, TableFunctionBindInput &input, +static unique_ptr CloudFormationCreateStackBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { - auto result = make_uniq(); + auto result = make_uniq(); if (input.inputs[0].IsNull()) { - throw InvalidInputException("cfn_create_stack: the template argument must not be NULL"); + throw InvalidInputException("cloudformation_create_stack: the template argument must not be NULL"); } result->template_arg = StringValue::Get(input.inputs[0]); if (!input.inputs[1].IsNull()) { @@ -158,8 +158,8 @@ static unique_ptr CfnCreateStackBind(ClientContext &context, Table return std::move(result); } -static void CfnCreateStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (CfnCreateStackBindData &)*data_p.bind_data; +static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CloudFormationCreateStackBindData &)*data_p.bind_data; if (data.finished) { return; } @@ -173,7 +173,7 @@ static void CfnCreateStackFun(ClientContext &context, TableFunctionInput &data_p } if (region.empty()) { throw InvalidInputException( - "cfn_create_stack: region is required - set options['region'] or the region:= named parameter"); + "cloudformation_create_stack: region is required - set options['region'] or the region:= named parameter"); } auto opt = [&](const string &key) -> string { @@ -229,7 +229,7 @@ static void CfnCreateStackFun(ClientContext &context, TableFunctionInput &data_p if (!data.name_arg.empty()) { if (data.name_arg.size() > MAX_STACK_NAME_LEN) { throw InvalidInputException( - "cfn_create_stack: explicit name '%s' is %llu chars; max %llu " + "cloudformation_create_stack: explicit name '%s' is %llu chars; max %llu " "(to keep the resulting CFN stack ARN within the 128-char API limit)", data.name_arg, (unsigned long long)data.name_arg.size(), (unsigned long long)MAX_STACK_NAME_LEN); @@ -243,7 +243,7 @@ static void CfnCreateStackFun(ClientContext &context, TableFunctionInput &data_p prefix = UrlBasenameStem(data.template_arg); } if (prefix.empty()) { - prefix = "cfn-stack"; + prefix = "duckdb-aws"; } if (prefix.size() > MAX_PREFIX_LEN) { prefix = prefix.substr(0, MAX_PREFIX_LEN); @@ -252,28 +252,28 @@ static void CfnCreateStackFun(ClientContext &context, TableFunctionInput &data_p } // Route every non-reserved option key to a declared template parameter. - Aws::Vector cfn_params; + Aws::Vector cloudformation_params; for (auto &kv : data.options) { if (RESERVED_OPTION_KEYS.count(StringUtil::Lower(kv.key))) { continue; } if (!declared_params.count(kv.key)) { throw InvalidInputException( - "cfn_create_stack: option '%s' is not a parameter declared by the template", kv.key); + "cloudformation_create_stack: option '%s' is not a parameter declared by the template", kv.key); } Aws::CloudFormation::Model::Parameter p; p.SetParameterKey(kv.key.c_str()); p.SetParameterValue(kv.value.c_str()); - cfn_params.push_back(p); + cloudformation_params.push_back(p); } // Tags: provenance, the metadata stack-name, then caller-supplied extras. - Aws::Vector cfn_tags; + Aws::Vector cloudformation_tags; auto add_tag = [&](const string &k, const string &v) { Aws::CloudFormation::Model::Tag t; t.SetKey(k.c_str()); t.SetValue(v.c_str()); - cfn_tags.push_back(t); + cloudformation_tags.push_back(t); }; add_tag("created-by", "duckdb"); if (!metadata_stack_name.empty()) { @@ -290,14 +290,14 @@ static void CfnCreateStackFun(ClientContext &context, TableFunctionInput &data_p } else { req.SetTemplateBody(data.template_arg.c_str()); } - if (!cfn_params.empty()) { - req.SetParameters(cfn_params); + if (!cloudformation_params.empty()) { + req.SetParameters(cloudformation_params); } if (!summary.GetCapabilities().empty()) { req.SetCapabilities(summary.GetCapabilities()); } - if (!cfn_tags.empty()) { - req.SetTags(cfn_tags); + if (!cloudformation_tags.empty()) { + req.SetTags(cloudformation_tags); } auto outcome = cfn.CreateStack(req); @@ -324,21 +324,21 @@ static void CfnCreateStackFun(ClientContext &context, TableFunctionInput &data_p } //===--------------------------------------------------------------------===// -// Shared handle parsing (cfn_describe_stack / cfn_outputs / cfn_delete_stack) +// Shared handle parsing (cloudformation_describe_stack / cloudformation_outputs / cloudformation_delete_stack) //===--------------------------------------------------------------------===// namespace { -struct CfnHandle { +struct CloudFormationHandle { string stack_ref; // stack_id (ARN) when present, else stack_name string stack_name; string stack_id; string region; }; -CfnHandle ParseHandle(const Value &handle_value, const char *fn_name) { +CloudFormationHandle ParseHandle(const Value &handle_value, const char *fn_name) { auto kvs = UnpackStringMap(handle_value); - CfnHandle h; + CloudFormationHandle h; if (auto *v = FindCI(kvs, "stack_id")) { h.stack_id = *v; } @@ -361,18 +361,18 @@ CfnHandle ParseHandle(const Value &handle_value, const char *fn_name) { } // namespace //===--------------------------------------------------------------------===// -// cfn_describe_stack(handle) +// cloudformation_describe_stack(handle) //===--------------------------------------------------------------------===// -struct CfnDescribeStackBindData : public TableFunctionData { - CfnHandle handle; +struct CloudFormationDescribeStackBindData : public TableFunctionData { + CloudFormationHandle handle; bool finished = false; }; -static unique_ptr CfnDescribeStackBind(ClientContext &context, TableFunctionBindInput &input, +static unique_ptr CloudFormationDescribeStackBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { - auto result = make_uniq(); - result->handle = ParseHandle(input.inputs[0], "cfn_describe_stack"); + auto result = make_uniq(); + result->handle = ParseHandle(input.inputs[0], "cloudformation_describe_stack"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("stack_name"); @@ -388,8 +388,8 @@ static unique_ptr CfnDescribeStackBind(ClientContext &context, Tab return std::move(result); } -static void CfnDescribeStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (CfnDescribeStackBindData &)*data_p.bind_data; +static void CloudFormationDescribeStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CloudFormationDescribeStackBindData &)*data_p.bind_data; if (data.finished) { return; } @@ -428,18 +428,18 @@ static void CfnDescribeStackFun(ClientContext &context, TableFunctionInput &data } //===--------------------------------------------------------------------===// -// cfn_outputs(handle) +// cloudformation_outputs(handle) //===--------------------------------------------------------------------===// -struct CfnOutputsBindData : public TableFunctionData { - CfnHandle handle; +struct CloudFormationOutputsBindData : public TableFunctionData { + CloudFormationHandle handle; bool finished = false; }; -static unique_ptr CfnOutputsBind(ClientContext &context, TableFunctionBindInput &input, +static unique_ptr CloudFormationOutputsBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { - auto result = make_uniq(); - result->handle = ParseHandle(input.inputs[0], "cfn_outputs"); + auto result = make_uniq(); + result->handle = ParseHandle(input.inputs[0], "cloudformation_outputs"); return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); names.emplace_back("outputs"); @@ -447,8 +447,8 @@ static unique_ptr CfnOutputsBind(ClientContext &context, TableFunc return std::move(result); } -static void CfnOutputsFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (CfnOutputsBindData &)*data_p.bind_data; +static void CloudFormationOutputsFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CloudFormationOutputsBindData &)*data_p.bind_data; if (data.finished) { return; } @@ -485,18 +485,18 @@ static void CfnOutputsFun(ClientContext &context, TableFunctionInput &data_p, Da } //===--------------------------------------------------------------------===// -// cfn_delete_stack(handle) +// cloudformation_delete_stack(handle) //===--------------------------------------------------------------------===// -struct CfnDeleteStackBindData : public TableFunctionData { - CfnHandle handle; +struct CloudFormationDeleteStackBindData : public TableFunctionData { + CloudFormationHandle handle; bool finished = false; }; -static unique_ptr CfnDeleteStackBind(ClientContext &context, TableFunctionBindInput &input, +static unique_ptr CloudFormationDeleteStackBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { - auto result = make_uniq(); - result->handle = ParseHandle(input.inputs[0], "cfn_delete_stack"); + auto result = make_uniq(); + result->handle = ParseHandle(input.inputs[0], "cloudformation_delete_stack"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("stack_id"); @@ -504,8 +504,8 @@ static unique_ptr CfnDeleteStackBind(ClientContext &context, Table return std::move(result); } -static void CfnDeleteStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (CfnDeleteStackBindData &)*data_p.bind_data; +static void CloudFormationDeleteStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CloudFormationDeleteStackBindData &)*data_p.bind_data; if (data.finished) { return; } @@ -533,22 +533,22 @@ static void CfnDeleteStackFun(ClientContext &context, TableFunctionInput &data_p // Registration //===--------------------------------------------------------------------===// -void CfnFunctions::Register(ExtensionLoader &loader) { +void CloudFormationFunctions::Register(ExtensionLoader &loader) { auto map_vv = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); - TableFunction create_fn("cfn_create_stack", {LogicalType::VARCHAR, LogicalType::VARCHAR, map_vv}, - CfnCreateStackFun, CfnCreateStackBind); + TableFunction create_fn("cloudformation_create_stack", {LogicalType::VARCHAR, LogicalType::VARCHAR, map_vv}, + CloudFormationCreateStackFun, CloudFormationCreateStackBind); create_fn.named_parameters["region"] = LogicalType::VARCHAR; create_fn.named_parameters["tags"] = map_vv; loader.RegisterFunction(create_fn); - TableFunction describe_fn("cfn_describe_stack", {map_vv}, CfnDescribeStackFun, CfnDescribeStackBind); + TableFunction describe_fn("cloudformation_describe_stack", {map_vv}, CloudFormationDescribeStackFun, CloudFormationDescribeStackBind); loader.RegisterFunction(describe_fn); - TableFunction outputs_fn("cfn_outputs", {map_vv}, CfnOutputsFun, CfnOutputsBind); + TableFunction outputs_fn("cloudformation_outputs", {map_vv}, CloudFormationOutputsFun, CloudFormationOutputsBind); loader.RegisterFunction(outputs_fn); - TableFunction delete_fn("cfn_delete_stack", {map_vv}, CfnDeleteStackFun, CfnDeleteStackBind); + TableFunction delete_fn("cloudformation_delete_stack", {map_vv}, CloudFormationDeleteStackFun, CloudFormationDeleteStackBind); loader.RegisterFunction(delete_fn); } diff --git a/src/include/cfn_functions.hpp b/src/include/cloudformation_functions.hpp similarity index 54% rename from src/include/cfn_functions.hpp rename to src/include/cloudformation_functions.hpp index ddc61fc..b66afa0 100644 --- a/src/include/cfn_functions.hpp +++ b/src/include/cloudformation_functions.hpp @@ -4,8 +4,8 @@ namespace duckdb { class ExtensionLoader; -struct CfnFunctions { - //! Register the generic CloudFormation table functions (cfn_*). +struct CloudFormationFunctions { + //! Register the generic CloudFormation table functions (cloudformation_*). static void Register(ExtensionLoader &loader); }; diff --git a/test/sql/cfn_functions.test b/test/sql/cfn_functions.test deleted file mode 100644 index b3b4c38..0000000 --- a/test/sql/cfn_functions.test +++ /dev/null @@ -1,60 +0,0 @@ -# name: test/sql/cfn_functions.test -# description: smoke test for the generic cfn_* CloudFormation functions -# group: [sql] - -require no_extension_autoloading "EXPECTED: Test relies on explicit INSTALL and LOAD" - -# Before LOAD aws, the functions do not exist. -statement error -SELECT * FROM cfn_create_stack('x', NULL, MAP {'region': 'us-east-1'}); ----- - -require aws - -# cfn_create_stack: a NULL template is rejected at bind time. -statement error -SELECT * FROM cfn_create_stack(NULL, NULL, MAP {'region': 'us-east-1'}); ----- -template argument must not be NULL - -# cfn_create_stack: region is required (checked before any AWS call). -statement error -SELECT * FROM cfn_create_stack('AWSTemplateFormatVersion: 2010-09-09', NULL, MAP {'InstanceType': 't3.micro'}); ----- -region is required - -# cfn_describe_stack: handle without a stack reference. -statement error -SELECT * FROM cfn_describe_stack(MAP {'region': 'us-east-1'}); ----- -handle must contain 'stack_id' or 'stack_name' - -# cfn_describe_stack: handle missing region. -statement error -SELECT * FROM cfn_describe_stack(MAP {'stack_name': 's'}); ----- -handle is missing 'region' - -# cfn_outputs: handle without a stack reference. -statement error -SELECT * FROM cfn_outputs(MAP {'region': 'us-east-1'}); ----- -handle must contain 'stack_id' or 'stack_name' - -# cfn_outputs: handle missing region. -statement error -SELECT * FROM cfn_outputs(MAP {'stack_name': 's'}); ----- -handle is missing 'region' - -# cfn_delete_stack: handle without a stack reference. -statement error -SELECT * FROM cfn_delete_stack(MAP {'region': 'us-east-1'}); ----- -handle must contain 'stack_id' or 'stack_name' - -# cfn_delete_stack: handle missing region. -statement error -SELECT * FROM cfn_delete_stack(MAP {'stack_name': 's'}); ----- -handle is missing 'region' diff --git a/test/sql/cloudformation_functions.test b/test/sql/cloudformation_functions.test new file mode 100644 index 0000000..6ae2e15 --- /dev/null +++ b/test/sql/cloudformation_functions.test @@ -0,0 +1,60 @@ +# name: test/sql/cloudformation_functions.test +# description: smoke test for the generic cloudformation_* CloudFormation functions +# group: [sql] + +require no_extension_autoloading "EXPECTED: Test relies on explicit INSTALL and LOAD" + +# Before LOAD aws, the functions do not exist. +statement error +SELECT * FROM cloudformation_create_stack('x', NULL, MAP {'region': 'us-east-1'}); +---- + +require aws + +# cloudformation_create_stack: a NULL template is rejected at bind time. +statement error +SELECT * FROM cloudformation_create_stack(NULL, NULL, MAP {'region': 'us-east-1'}); +---- +template argument must not be NULL + +# cloudformation_create_stack: region is required (checked before any AWS call). +statement error +SELECT * FROM cloudformation_create_stack('AWSTemplateFormatVersion: 2010-09-09', NULL, MAP {'InstanceType': 't3.micro'}); +---- +region is required + +# cloudformation_describe_stack: handle without a stack reference. +statement error +SELECT * FROM cloudformation_describe_stack(MAP {'region': 'us-east-1'}); +---- +handle must contain 'stack_id' or 'stack_name' + +# cloudformation_describe_stack: handle missing region. +statement error +SELECT * FROM cloudformation_describe_stack(MAP {'stack_name': 's'}); +---- +handle is missing 'region' + +# cloudformation_outputs: handle without a stack reference. +statement error +SELECT * FROM cloudformation_outputs(MAP {'region': 'us-east-1'}); +---- +handle must contain 'stack_id' or 'stack_name' + +# cloudformation_outputs: handle missing region. +statement error +SELECT * FROM cloudformation_outputs(MAP {'stack_name': 's'}); +---- +handle is missing 'region' + +# cloudformation_delete_stack: handle without a stack reference. +statement error +SELECT * FROM cloudformation_delete_stack(MAP {'region': 'us-east-1'}); +---- +handle must contain 'stack_id' or 'stack_name' + +# cloudformation_delete_stack: handle missing region. +statement error +SELECT * FROM cloudformation_delete_stack(MAP {'stack_name': 's'}); +---- +handle is missing 'region' From c80456e09ad2c32666885e845c70b76cb0d429b0 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 27 May 2026 06:48:08 +0200 Subject: [PATCH 03/16] Fold cloudformation_outputs into cloudformation_describe_stack cloudformation_describe_stack now returns an extra `outputs` column (MAP(VARCHAR, VARCHAR)) populated from stack.GetOutputs(). The standalone cloudformation_outputs function is removed - it was a second DescribeStacks call returning a subset of the same information. Callers that previously did SELECT outputs FROM cloudformation_outputs(handle); now write SELECT outputs FROM cloudformation_describe_stack(handle); - same shape, one fewer function, one fewer round-trip if you want status + outputs in one query. --- src/cloudformation_functions.cpp | 72 +++++--------------------- test/sql/cloudformation_functions.test | 12 ----- 2 files changed, 12 insertions(+), 72 deletions(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index 97754d2..ee5478c 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -384,6 +384,8 @@ static unique_ptr CloudFormationDescribeStackBind(ClientContext &c names.emplace_back("status_reason"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("creation_time"); + return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + names.emplace_back("outputs"); return std::move(result); } @@ -418,68 +420,21 @@ static void CloudFormationDescribeStackFun(ClientContext &context, TableFunction string reason(stack.GetStackStatusReason().c_str()); string created(stack.GetCreationTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + vector output_keys; + vector output_values; + for (const auto &o : stack.GetOutputs()) { + output_keys.emplace_back(string(o.GetOutputKey().c_str())); + output_values.emplace_back(string(o.GetOutputValue().c_str())); + } + auto outputs = + Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(output_keys), std::move(output_values)); + output.SetValue(0, 0, Value(string(stack.GetStackName().c_str()))); output.SetValue(1, 0, Value(string(stack.GetStackId().c_str()))); output.SetValue(2, 0, Value(status)); output.SetValue(3, 0, reason.empty() ? Value() : Value(reason)); output.SetValue(4, 0, created.empty() ? Value() : Value(created)); - output.SetCardinality(1); - data.finished = true; -} - -//===--------------------------------------------------------------------===// -// cloudformation_outputs(handle) -//===--------------------------------------------------------------------===// - -struct CloudFormationOutputsBindData : public TableFunctionData { - CloudFormationHandle handle; - bool finished = false; -}; - -static unique_ptr CloudFormationOutputsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto result = make_uniq(); - result->handle = ParseHandle(input.inputs[0], "cloudformation_outputs"); - - return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); - names.emplace_back("outputs"); - - return std::move(result); -} - -static void CloudFormationOutputsFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &data = (CloudFormationOutputsBindData &)*data_p.bind_data; - if (data.finished) { - return; - } - - auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); - auto cfg = BuildClientConfigWithCa(); - cfg.region = data.handle.region.c_str(); - Aws::CloudFormation::CloudFormationClient cfn(provider, cfg); - - Aws::CloudFormation::Model::DescribeStacksRequest req; - req.SetStackName(data.handle.stack_ref.c_str()); - auto outcome = cfn.DescribeStacks(req); - if (!outcome.IsSuccess()) { - const auto &err = outcome.GetError(); - throw IOException("CloudFormation DescribeStacks failed: %s - %s", string(err.GetExceptionName().c_str()), - string(err.GetMessage().c_str())); - } - const auto &stacks = outcome.GetResult().GetStacks(); - if (stacks.empty()) { - throw IOException("CloudFormation DescribeStacks returned no stack for '%s'", data.handle.stack_ref); - } - - vector keys; - vector values; - for (const auto &o : stacks[0].GetOutputs()) { - keys.emplace_back(string(o.GetOutputKey().c_str())); - values.emplace_back(string(o.GetOutputValue().c_str())); - } - auto outputs = Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(keys), std::move(values)); - - output.SetValue(0, 0, outputs); + output.SetValue(5, 0, outputs); output.SetCardinality(1); data.finished = true; } @@ -545,9 +500,6 @@ void CloudFormationFunctions::Register(ExtensionLoader &loader) { TableFunction describe_fn("cloudformation_describe_stack", {map_vv}, CloudFormationDescribeStackFun, CloudFormationDescribeStackBind); loader.RegisterFunction(describe_fn); - TableFunction outputs_fn("cloudformation_outputs", {map_vv}, CloudFormationOutputsFun, CloudFormationOutputsBind); - loader.RegisterFunction(outputs_fn); - TableFunction delete_fn("cloudformation_delete_stack", {map_vv}, CloudFormationDeleteStackFun, CloudFormationDeleteStackBind); loader.RegisterFunction(delete_fn); } diff --git a/test/sql/cloudformation_functions.test b/test/sql/cloudformation_functions.test index 6ae2e15..f6973bc 100644 --- a/test/sql/cloudformation_functions.test +++ b/test/sql/cloudformation_functions.test @@ -35,18 +35,6 @@ SELECT * FROM cloudformation_describe_stack(MAP {'stack_name': 's'}); ---- handle is missing 'region' -# cloudformation_outputs: handle without a stack reference. -statement error -SELECT * FROM cloudformation_outputs(MAP {'region': 'us-east-1'}); ----- -handle must contain 'stack_id' or 'stack_name' - -# cloudformation_outputs: handle missing region. -statement error -SELECT * FROM cloudformation_outputs(MAP {'stack_name': 's'}); ----- -handle is missing 'region' - # cloudformation_delete_stack: handle without a stack reference. statement error SELECT * FROM cloudformation_delete_stack(MAP {'region': 'us-east-1'}); From 853cb67ebe936b17b59a45baba6090cd1de7c286 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 27 May 2026 06:52:30 +0200 Subject: [PATCH 04/16] Auto-tag stacks and expose duckdb_aws_session_id() Every stack cloudformation_create_stack produces now carries provenance tags identifying the duckdb-aws extension that created it, the DuckDB host that ran the call, the extension's git short SHA, and a stable per-process session id. Caller-supplied tags via the `tags := ...` named parameter merge on top and override on key collision. Auto-tags applied: created-by = "duckdb-aws" created-by-version = duckdb-version = DuckDB::LibraryVersion() managed-by = "duckdb-aws" duckdb-session-id = stack-name = (only when template has it) Also expose duckdb_aws_session_id() as a scalar SQL function so callers can scope queries / cleanups by the running session, e.g.: cloudformation_destroy_all( tag_filter := MAP {'duckdb-session-id': duckdb_aws_session_id()}) The session id is currently process-scoped and extension-specific. A future DuckDB-core session primitive could replace the local implementation without changing the SQL surface. --- CMakeLists.txt | 14 +++++++ src/cloudformation_functions.cpp | 58 +++++++++++++++++++++----- test/sql/cloudformation_functions.test | 12 ++++++ 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b9ea3af..04271ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,20 @@ set(EXTENSION_NAME ${TARGET_NAME}_extension) project(${TARGET_NAME}) include_directories(src/include) +# Capture the duckdb-aws extension's git short SHA at build time. Surfaced +# at runtime as the `created-by-version` auto-tag on every stack the +# cloudformation_* functions create. +execute_process( + COMMAND git rev-parse --short HEAD + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE DUCKDB_AWS_GIT_SHA + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) +if(NOT DUCKDB_AWS_GIT_SHA) + set(DUCKDB_AWS_GIT_SHA "unknown") +endif() +add_compile_definitions(DUCKDB_AWS_GIT_SHA="${DUCKDB_AWS_GIT_SHA}") + set(EXTENSION_SOURCES src/aws_extension.cpp src/aws_secret.cpp src/cloudformation_functions.cpp) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index ee5478c..4eb24dd 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -5,6 +5,8 @@ #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/types/uuid.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/main/database.hpp" #include "duckdb/main/extension/extension_loader.hpp" #include @@ -98,6 +100,14 @@ string ShortRandHex() { return uuid_str.substr(0, 8) + uuid_str.substr(9, 4); } +//! Per-process session id used for the duckdb-session-id auto-tag on every +//! stack this extension creates. Lazy-initialised on first use, then stable +//! for the lifetime of the duckdb-aws extension load. +const string &SessionId() { + static const string id = ShortRandHex(); + return id; +} + //! CFN's StackName API parameter is capped at 128 chars. The ARN is //! `arn:aws:cloudformation:::stack//` — //! structural+region(≤14)+account(12)+slashes(2)+cfn-uuid(36) ≤ 94, so the @@ -267,20 +277,35 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn cloudformation_params.push_back(p); } - // Tags: provenance, the metadata stack-name, then caller-supplied extras. - Aws::Vector cloudformation_tags; - auto add_tag = [&](const string &k, const string &v) { - Aws::CloudFormation::Model::Tag t; - t.SetKey(k.c_str()); - t.SetValue(v.c_str()); - cloudformation_tags.push_back(t); + // Tags: provenance auto-tags first, then caller-supplied extras (which + // override on key collision because they're applied last). + vector tag_kv; + auto set_tag = [&](const string &k, const string &v) { + for (auto &existing : tag_kv) { + if (existing.key == k) { + existing.value = v; + return; + } + } + tag_kv.push_back({k, v}); }; - add_tag("created-by", "duckdb"); + set_tag("created-by", "duckdb-aws"); + set_tag("created-by-version", DUCKDB_AWS_GIT_SHA); + set_tag("duckdb-version", DuckDB::LibraryVersion()); + set_tag("managed-by", "duckdb-aws"); + set_tag("duckdb-session-id", SessionId()); if (!metadata_stack_name.empty()) { - add_tag("stack-name", metadata_stack_name); + set_tag("stack-name", metadata_stack_name); } for (auto &kv : data.tags_override) { - add_tag(kv.key, kv.value); + set_tag(kv.key, kv.value); + } + Aws::Vector cloudformation_tags; + for (auto &kv : tag_kv) { + Aws::CloudFormation::Model::Tag t; + t.SetKey(kv.key.c_str()); + t.SetValue(kv.value.c_str()); + cloudformation_tags.push_back(t); } Aws::CloudFormation::Model::CreateStackRequest req; @@ -484,6 +509,16 @@ static void CloudFormationDeleteStackFun(ClientContext &context, TableFunctionIn data.finished = true; } +//===--------------------------------------------------------------------===// +// duckdb_aws_session_id() scalar function +//===--------------------------------------------------------------------===// + +static void DuckDBAwsSessionIdFunction(DataChunk &args, ExpressionState &state, Vector &result) { + result.SetVectorType(VectorType::CONSTANT_VECTOR); + auto data = ConstantVector::GetData(result); + data[0] = StringVector::AddString(result, SessionId()); +} + //===--------------------------------------------------------------------===// // Registration //===--------------------------------------------------------------------===// @@ -502,6 +537,9 @@ void CloudFormationFunctions::Register(ExtensionLoader &loader) { TableFunction delete_fn("cloudformation_delete_stack", {map_vv}, CloudFormationDeleteStackFun, CloudFormationDeleteStackBind); loader.RegisterFunction(delete_fn); + + ScalarFunction session_id_fn("duckdb_aws_session_id", {}, LogicalType::VARCHAR, DuckDBAwsSessionIdFunction); + loader.RegisterFunction(session_id_fn); } } // namespace duckdb diff --git a/test/sql/cloudformation_functions.test b/test/sql/cloudformation_functions.test index f6973bc..df84e98 100644 --- a/test/sql/cloudformation_functions.test +++ b/test/sql/cloudformation_functions.test @@ -46,3 +46,15 @@ statement error SELECT * FROM cloudformation_delete_stack(MAP {'stack_name': 's'}); ---- handle is missing 'region' + +# duckdb_aws_session_id returns a non-empty string. +query I +SELECT length(duckdb_aws_session_id()) > 0; +---- +true + +# Session id is stable within a session: repeated calls return the same value. +query I +SELECT duckdb_aws_session_id() = duckdb_aws_session_id(); +---- +true From 55ceaf8062ee7dcb36f4459502f623190197bb77 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 27 May 2026 07:10:54 +0200 Subject: [PATCH 05/16] Enrich describe output, make delete pass-through cloudformation_describe_stack gains three columns: - region (from the input handle, mirrors list's shape) - last_updated_time (from Stack.LastUpdatedTime, nullable) - description (from Stack.Description, nullable) The 8-column common prefix (region, stack_name, stack_id, status, status_reason, creation_time, last_updated_time, description) now matches what cloudformation_list_stacks will return. Describe's 9th column, outputs MAP, remains describe-only because ListStacks doesn't surface outputs. cloudformation_delete_stack now echoes the input handle MAP byte-for-byte instead of returning just stack_id - any extra keys the caller added (annotations, timestamps, custom metadata) survive the call. Composes naturally with create's handle output for audit/log patterns: INSERT INTO deletions (handle) SELECT handle FROM cloudformation_delete_stack(getvariable('h')); --- src/cloudformation_functions.cpp | 36 ++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index 4eb24dd..a296c1d 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -399,6 +399,8 @@ static unique_ptr CloudFormationDescribeStackBind(ClientContext &c auto result = make_uniq(); result->handle = ParseHandle(input.inputs[0], "cloudformation_describe_stack"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("region"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("stack_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -409,6 +411,10 @@ static unique_ptr CloudFormationDescribeStackBind(ClientContext &c names.emplace_back("status_reason"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("creation_time"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("last_updated_time"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("description"); return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); names.emplace_back("outputs"); @@ -444,6 +450,11 @@ static void CloudFormationDescribeStackFun(ClientContext &context, TableFunction Aws::CloudFormation::Model::StackStatusMapper::GetNameForStackStatus(stack.GetStackStatus()).c_str()); string reason(stack.GetStackStatusReason().c_str()); string created(stack.GetCreationTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + string updated; + if (stack.LastUpdatedTimeHasBeenSet()) { + updated = string(stack.GetLastUpdatedTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + } + string description(stack.GetDescription().c_str()); vector output_keys; vector output_values; @@ -454,12 +465,15 @@ static void CloudFormationDescribeStackFun(ClientContext &context, TableFunction auto outputs = Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(output_keys), std::move(output_values)); - output.SetValue(0, 0, Value(string(stack.GetStackName().c_str()))); - output.SetValue(1, 0, Value(string(stack.GetStackId().c_str()))); - output.SetValue(2, 0, Value(status)); - output.SetValue(3, 0, reason.empty() ? Value() : Value(reason)); - output.SetValue(4, 0, created.empty() ? Value() : Value(created)); - output.SetValue(5, 0, outputs); + output.SetValue(0, 0, Value(data.handle.region)); + output.SetValue(1, 0, Value(string(stack.GetStackName().c_str()))); + output.SetValue(2, 0, Value(string(stack.GetStackId().c_str()))); + output.SetValue(3, 0, Value(status)); + output.SetValue(4, 0, reason.empty() ? Value() : Value(reason)); + output.SetValue(5, 0, created.empty() ? Value() : Value(created)); + output.SetValue(6, 0, updated.empty() ? Value() : Value(updated)); + output.SetValue(7, 0, description.empty() ? Value() : Value(description)); + output.SetValue(8, 0, outputs); output.SetCardinality(1); data.finished = true; } @@ -470,6 +484,7 @@ static void CloudFormationDescribeStackFun(ClientContext &context, TableFunction struct CloudFormationDeleteStackBindData : public TableFunctionData { CloudFormationHandle handle; + Value handle_value; // original MAP, echoed back verbatim bool finished = false; }; @@ -477,9 +492,10 @@ static unique_ptr CloudFormationDeleteStackBind(ClientContext &con vector &return_types, vector &names) { auto result = make_uniq(); result->handle = ParseHandle(input.inputs[0], "cloudformation_delete_stack"); + result->handle_value = input.inputs[0]; - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("stack_id"); + return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + names.emplace_back("handle"); return std::move(result); } @@ -504,7 +520,9 @@ static void CloudFormationDeleteStackFun(ClientContext &context, TableFunctionIn string(err.GetMessage().c_str())); } - output.SetValue(0, 0, Value(data.handle.stack_ref)); + // Pass-through: echo the input handle byte-for-byte. Any extra keys the + // caller put in (annotations, timestamps, custom metadata) survive intact. + output.SetValue(0, 0, data.handle_value); output.SetCardinality(1); data.finished = true; } From f8eb71453982be7b701619be572fbc12233aa4cd Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 27 May 2026 07:12:13 +0200 Subject: [PATCH 06/16] Surface stack tags in cloudformation_describe_stack Adds `tags MAP(VARCHAR, VARCHAR)` as a new column between `description` and `outputs`. AWS's DescribeStacks already returns Tags on the Stack object - no extra API call. Useful for verifying that the auto-tags applied by cloudformation_create_stack (created-by=duckdb-aws, created-by-version, duckdb-version, managed-by, duckdb-session-id, stack-name) actually landed: SELECT tags FROM cloudformation_describe_stack(getvariable('h')); And for ad-hoc tag-based queries until cloudformation_destroy_all and friends arrive: SELECT stack_name FROM cloudformation_describe_stack(getvariable('h')) WHERE tags['created-by'] = 'duckdb-aws'; ListStacks's StackSummary doesn't include tags, so this column stays describe-only - same pattern as `outputs`. --- src/cloudformation_functions.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index a296c1d..3b97027 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -416,6 +416,8 @@ static unique_ptr CloudFormationDescribeStackBind(ClientContext &c return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("description"); return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + names.emplace_back("tags"); + return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); names.emplace_back("outputs"); return std::move(result); @@ -456,6 +458,14 @@ static void CloudFormationDescribeStackFun(ClientContext &context, TableFunction } string description(stack.GetDescription().c_str()); + vector tag_keys; + vector tag_values; + for (const auto &t : stack.GetTags()) { + tag_keys.emplace_back(string(t.GetKey().c_str())); + tag_values.emplace_back(string(t.GetValue().c_str())); + } + auto tags = Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(tag_keys), std::move(tag_values)); + vector output_keys; vector output_values; for (const auto &o : stack.GetOutputs()) { @@ -473,7 +483,8 @@ static void CloudFormationDescribeStackFun(ClientContext &context, TableFunction output.SetValue(5, 0, created.empty() ? Value() : Value(created)); output.SetValue(6, 0, updated.empty() ? Value() : Value(updated)); output.SetValue(7, 0, description.empty() ? Value() : Value(description)); - output.SetValue(8, 0, outputs); + output.SetValue(8, 0, tags); + output.SetValue(9, 0, outputs); output.SetCardinality(1); data.finished = true; } From d5a2155e1bc73f78aa0dc2f3b4aaf92807b10751 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 27 May 2026 07:16:57 +0200 Subject: [PATCH 07/16] Add cloudformation_list_stacks Lists every CFN stack in a region. Pagination handled internally (follows NextToken until the result is exhausted). Same 8-column identity-and-state prefix as cloudformation_describe_stack, minus the describe-only `outputs` and `tags` columns (ListStacks's StackSummary doesn't return them). cloudformation_list_stacks( [region := VARCHAR] -- defaults to AWS_REGION -- / AWS_DEFAULT_REGION [status_filter := LIST]) -- passes through to AWS's -- native StackStatusFilter -> (region, stack_name, stack_id, status, status_reason, creation_time, last_updated_time, description) Region resolution: explicit `region :=` wins; otherwise AWS_REGION, then AWS_DEFAULT_REGION. If none resolves, errors out clearly. The `region` output column is the same value across every row in a single call - convenient when UNIONing across regions: SELECT * FROM cloudformation_list_stacks(region := 'us-east-1') UNION ALL SELECT * FROM cloudformation_list_stacks(region := 'eu-west-1'); status_filter values are validated against the SDK's StackStatus enum at bind time; unknown values surface a clear error rather than reaching AWS. --- src/cloudformation_functions.cpp | 160 +++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index 3b97027..54f56da 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,7 @@ #include #include +#include #include namespace duckdb { @@ -538,6 +540,159 @@ static void CloudFormationDeleteStackFun(ClientContext &context, TableFunctionIn data.finished = true; } +//===--------------------------------------------------------------------===// +// cloudformation_list_stacks([region := ...] [status_filter := ...]) +//===--------------------------------------------------------------------===// + +struct CloudFormationListStacksRow { + string region; + string stack_name; + string stack_id; + string status; + string status_reason; + string creation_time; + string last_updated_time; + string description; +}; + +struct CloudFormationListStacksBindData : public TableFunctionData { + string region; + vector status_filter; + bool initialized = false; + vector rows; + idx_t cursor = 0; +}; + +static unique_ptr CloudFormationListStacksBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, + vector &names) { + auto result = make_uniq(); + + for (auto &np : input.named_parameters) { + auto key = StringUtil::Lower(np.first); + if (key == "region") { + if (!np.second.IsNull()) { + result->region = StringValue::Get(np.second); + } + } else if (key == "status_filter") { + if (!np.second.IsNull()) { + auto &children = ListValue::GetChildren(np.second); + for (auto &child : children) { + if (child.IsNull()) { + continue; + } + auto str = StringValue::Get(child); + auto status_enum = + Aws::CloudFormation::Model::StackStatusMapper::GetStackStatusForName(str.c_str()); + if (status_enum == Aws::CloudFormation::Model::StackStatus::NOT_SET) { + throw InvalidInputException( + "cloudformation_list_stacks: unknown stack status '%s' in status_filter", str); + } + result->status_filter.push_back(status_enum); + } + } + } + } + + // Region fallback: AWS_REGION / AWS_DEFAULT_REGION env vars. If neither is + // set and the caller didn't pass `region :=`, surface a clear error. + if (result->region.empty()) { + if (const char *env = std::getenv("AWS_REGION"); env && *env) { + result->region = env; + } else if (env = std::getenv("AWS_DEFAULT_REGION"); env && *env) { + result->region = env; + } + } + if (result->region.empty()) { + throw InvalidInputException( + "cloudformation_list_stacks: no region provided and AWS_REGION / AWS_DEFAULT_REGION " + "environment variables are not set"); + } + + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("region"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("stack_name"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("stack_id"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("status"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("status_reason"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("creation_time"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("last_updated_time"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("description"); + + return std::move(result); +} + +static void CloudFormationListStacksFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CloudFormationListStacksBindData &)*data_p.bind_data; + + if (!data.initialized) { + auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); + auto cfg = BuildClientConfigWithCa(); + cfg.region = data.region.c_str(); + Aws::CloudFormation::CloudFormationClient cfn(provider, cfg); + + Aws::String next_token; + do { + Aws::CloudFormation::Model::ListStacksRequest req; + if (!data.status_filter.empty()) { + req.SetStackStatusFilter(data.status_filter); + } + if (!next_token.empty()) { + req.SetNextToken(next_token); + } + auto outcome = cfn.ListStacks(req); + if (!outcome.IsSuccess()) { + const auto &err = outcome.GetError(); + throw IOException("CloudFormation ListStacks failed: %s - %s", + string(err.GetExceptionName().c_str()), string(err.GetMessage().c_str())); + } + const auto &res = outcome.GetResult(); + for (const auto &s : res.GetStackSummaries()) { + CloudFormationListStacksRow row; + row.region = data.region; + row.stack_name = string(s.GetStackName().c_str()); + row.stack_id = string(s.GetStackId().c_str()); + row.status = string( + Aws::CloudFormation::Model::StackStatusMapper::GetNameForStackStatus(s.GetStackStatus()).c_str()); + row.status_reason = string(s.GetStackStatusReason().c_str()); + row.creation_time = + string(s.GetCreationTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + if (s.LastUpdatedTimeHasBeenSet()) { + row.last_updated_time = + string(s.GetLastUpdatedTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + } + row.description = string(s.GetTemplateDescription().c_str()); + data.rows.push_back(row); + } + next_token = res.GetNextToken(); + } while (!next_token.empty()); + data.initialized = true; + } + + idx_t remaining = data.rows.size() - data.cursor; + idx_t to_emit = std::min(remaining, (idx_t)STANDARD_VECTOR_SIZE); + for (idx_t i = 0; i < to_emit; i++) { + auto &r = data.rows[data.cursor + i]; + output.SetValue(0, i, Value(r.region)); + output.SetValue(1, i, Value(r.stack_name)); + output.SetValue(2, i, Value(r.stack_id)); + output.SetValue(3, i, Value(r.status)); + output.SetValue(4, i, r.status_reason.empty() ? Value() : Value(r.status_reason)); + output.SetValue(5, i, r.creation_time.empty() ? Value() : Value(r.creation_time)); + output.SetValue(6, i, r.last_updated_time.empty() ? Value() : Value(r.last_updated_time)); + output.SetValue(7, i, r.description.empty() ? Value() : Value(r.description)); + } + output.SetCardinality(to_emit); + data.cursor += to_emit; +} + //===--------------------------------------------------------------------===// // duckdb_aws_session_id() scalar function //===--------------------------------------------------------------------===// @@ -567,6 +722,11 @@ void CloudFormationFunctions::Register(ExtensionLoader &loader) { TableFunction delete_fn("cloudformation_delete_stack", {map_vv}, CloudFormationDeleteStackFun, CloudFormationDeleteStackBind); loader.RegisterFunction(delete_fn); + TableFunction list_fn("cloudformation_list_stacks", {}, CloudFormationListStacksFun, CloudFormationListStacksBind); + list_fn.named_parameters["region"] = LogicalType::VARCHAR; + list_fn.named_parameters["status_filter"] = LogicalType::LIST(LogicalType::VARCHAR); + loader.RegisterFunction(list_fn); + ScalarFunction session_id_fn("duckdb_aws_session_id", {}, LogicalType::VARCHAR, DuckDBAwsSessionIdFunction); loader.RegisterFunction(session_id_fn); } From 865860975eeeadb1a689d96847225e63874fe6e3 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 27 May 2026 07:21:40 +0200 Subject: [PATCH 08/16] Fixup c++17 only quirk --- src/cloudformation_functions.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index 54f56da..15df8c0 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -597,9 +597,11 @@ static unique_ptr CloudFormationListStacksBind(ClientContext &cont // Region fallback: AWS_REGION / AWS_DEFAULT_REGION env vars. If neither is // set and the caller didn't pass `region :=`, surface a clear error. if (result->region.empty()) { - if (const char *env = std::getenv("AWS_REGION"); env && *env) { - result->region = env; - } else if (env = std::getenv("AWS_DEFAULT_REGION"); env && *env) { + const char *env = std::getenv("AWS_REGION"); + if (!env || !*env) { + env = std::getenv("AWS_DEFAULT_REGION"); + } + if (env && *env) { result->region = env; } } From 20a906595b22f25d8c26d339ff1cf0ec60d00807 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 27 May 2026 09:27:20 +0200 Subject: [PATCH 09/16] Make region mandatory --- src/cloudformation_functions.cpp | 36 ++++++++++---------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index 15df8c0..68ced68 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -23,7 +23,6 @@ #include #include -#include #include namespace duckdb { @@ -568,13 +567,17 @@ static unique_ptr CloudFormationListStacksBind(ClientContext &cont vector &names) { auto result = make_uniq(); + if (input.inputs[0].IsNull()) { + throw InvalidInputException("cloudformation_list_stacks: region must not be NULL"); + } + result->region = StringValue::Get(input.inputs[0]); + if (result->region.empty()) { + throw InvalidInputException("cloudformation_list_stacks: region must not be empty"); + } + for (auto &np : input.named_parameters) { auto key = StringUtil::Lower(np.first); - if (key == "region") { - if (!np.second.IsNull()) { - result->region = StringValue::Get(np.second); - } - } else if (key == "status_filter") { + if (key == "status_filter") { if (!np.second.IsNull()) { auto &children = ListValue::GetChildren(np.second); for (auto &child : children) { @@ -594,23 +597,6 @@ static unique_ptr CloudFormationListStacksBind(ClientContext &cont } } - // Region fallback: AWS_REGION / AWS_DEFAULT_REGION env vars. If neither is - // set and the caller didn't pass `region :=`, surface a clear error. - if (result->region.empty()) { - const char *env = std::getenv("AWS_REGION"); - if (!env || !*env) { - env = std::getenv("AWS_DEFAULT_REGION"); - } - if (env && *env) { - result->region = env; - } - } - if (result->region.empty()) { - throw InvalidInputException( - "cloudformation_list_stacks: no region provided and AWS_REGION / AWS_DEFAULT_REGION " - "environment variables are not set"); - } - return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("region"); return_types.emplace_back(LogicalType::VARCHAR); @@ -724,8 +710,8 @@ void CloudFormationFunctions::Register(ExtensionLoader &loader) { TableFunction delete_fn("cloudformation_delete_stack", {map_vv}, CloudFormationDeleteStackFun, CloudFormationDeleteStackBind); loader.RegisterFunction(delete_fn); - TableFunction list_fn("cloudformation_list_stacks", {}, CloudFormationListStacksFun, CloudFormationListStacksBind); - list_fn.named_parameters["region"] = LogicalType::VARCHAR; + TableFunction list_fn("cloudformation_list_stacks", {LogicalType::VARCHAR}, CloudFormationListStacksFun, + CloudFormationListStacksBind); list_fn.named_parameters["status_filter"] = LogicalType::LIST(LogicalType::VARCHAR); loader.RegisterFunction(list_fn); From c8235f905c34ba0e6adab1623b892e460f7c8218 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 22 Jun 2026 11:27:35 +0200 Subject: [PATCH 10/16] Move to identifiers --- src/cloudformation_functions.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index 68ced68..59db488 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -152,7 +152,7 @@ static unique_ptr CloudFormationCreateStackBind(ClientContext &con result->options = UnpackStringMap(input.inputs[2]); for (auto &np : input.named_parameters) { - auto key = StringUtil::Lower(np.first); + auto key = StringUtil::Lower(np.first.GetIdentifierName()); if (key == "region") { if (!np.second.IsNull()) { result->has_region_override = true; @@ -576,7 +576,7 @@ static unique_ptr CloudFormationListStacksBind(ClientContext &cont } for (auto &np : input.named_parameters) { - auto key = StringUtil::Lower(np.first); + auto key = StringUtil::Lower(np.first.GetIdentifierName()); if (key == "status_filter") { if (!np.second.IsNull()) { auto &children = ListValue::GetChildren(np.second); From c69b7fedb6c6444ec3aedf0a4bf9b6dd8d8bc141 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 22 Jun 2026 11:28:23 +0200 Subject: [PATCH 11/16] Remove managed-by --- src/cloudformation_functions.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index 59db488..ccc346b 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -293,7 +293,6 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn set_tag("created-by", "duckdb-aws"); set_tag("created-by-version", DUCKDB_AWS_GIT_SHA); set_tag("duckdb-version", DuckDB::LibraryVersion()); - set_tag("managed-by", "duckdb-aws"); set_tag("duckdb-session-id", SessionId()); if (!metadata_stack_name.empty()) { set_tag("stack-name", metadata_stack_name); From e798656425c1f74ee52af32beb1a8087e29ff3ab Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 22 Jun 2026 12:28:20 +0200 Subject: [PATCH 12/16] cfn -> cloudformation_client --- src/cloudformation_functions.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index ccc346b..c0a77f1 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -197,7 +197,7 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn opt("web_identity_token_file"), opt("session_name")); auto client_config = BuildClientConfigWithCa(); client_config.region = region.c_str(); - Aws::CloudFormation::CloudFormationClient cfn(provider, client_config); + Aws::CloudFormation::CloudFormationClient cloudformation_client(provider, client_config); bool template_is_url = LooksLikeUrl(data.template_arg); @@ -209,7 +209,7 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn } else { summary_req.SetTemplateBody(data.template_arg.c_str()); } - auto summary_outcome = cfn.GetTemplateSummary(summary_req); + auto summary_outcome = cloudformation_client.GetTemplateSummary(summary_req); if (!summary_outcome.IsSuccess()) { const auto &err = summary_outcome.GetError(); throw IOException("CloudFormation GetTemplateSummary failed: %s - %s", string(err.GetExceptionName().c_str()), @@ -325,7 +325,7 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn req.SetTags(cloudformation_tags); } - auto outcome = cfn.CreateStack(req); + auto outcome = cloudformation_client.CreateStack(req); if (!outcome.IsSuccess()) { const auto &err = outcome.GetError(); throw IOException("CloudFormation CreateStack failed: %s - %s", string(err.GetExceptionName().c_str()), @@ -432,11 +432,11 @@ static void CloudFormationDescribeStackFun(ClientContext &context, TableFunction auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); auto cfg = BuildClientConfigWithCa(); cfg.region = data.handle.region.c_str(); - Aws::CloudFormation::CloudFormationClient cfn(provider, cfg); + Aws::CloudFormation::CloudFormationClient cloudformation_client(provider, cfg); Aws::CloudFormation::Model::DescribeStacksRequest req; req.SetStackName(data.handle.stack_ref.c_str()); - auto outcome = cfn.DescribeStacks(req); + auto outcome = cloudformation_client.DescribeStacks(req); if (!outcome.IsSuccess()) { const auto &err = outcome.GetError(); throw IOException("CloudFormation DescribeStacks failed: %s - %s", string(err.GetExceptionName().c_str()), @@ -520,11 +520,11 @@ static void CloudFormationDeleteStackFun(ClientContext &context, TableFunctionIn auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); auto cfg = BuildClientConfigWithCa(); cfg.region = data.handle.region.c_str(); - Aws::CloudFormation::CloudFormationClient cfn(provider, cfg); + Aws::CloudFormation::CloudFormationClient cloudformation_client(provider, cfg); Aws::CloudFormation::Model::DeleteStackRequest req; req.SetStackName(data.handle.stack_ref.c_str()); - auto outcome = cfn.DeleteStack(req); + auto outcome = cloudformation_client.DeleteStack(req); if (!outcome.IsSuccess()) { const auto &err = outcome.GetError(); throw IOException("CloudFormation DeleteStack failed: %s - %s", string(err.GetExceptionName().c_str()), @@ -623,7 +623,7 @@ static void CloudFormationListStacksFun(ClientContext &context, TableFunctionInp auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); auto cfg = BuildClientConfigWithCa(); cfg.region = data.region.c_str(); - Aws::CloudFormation::CloudFormationClient cfn(provider, cfg); + Aws::CloudFormation::CloudFormationClient cloudformation_client(provider, cfg); Aws::String next_token; do { @@ -634,7 +634,7 @@ static void CloudFormationListStacksFun(ClientContext &context, TableFunctionInp if (!next_token.empty()) { req.SetNextToken(next_token); } - auto outcome = cfn.ListStacks(req); + auto outcome = cloudformation_client.ListStacks(req); if (!outcome.IsSuccess()) { const auto &err = outcome.GetError(); throw IOException("CloudFormation ListStacks failed: %s - %s", From be6c2e7575afbd86db96b3326177fc5bc11e82f4 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Mon, 22 Jun 2026 12:36:27 +0200 Subject: [PATCH 13/16] More review comments --- src/cloudformation_functions.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index c0a77f1..4475b3c 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -238,6 +238,7 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn // explicit names that exceed the limit. string stack_name; if (!data.name_arg.empty()) { + // Explicit name if (data.name_arg.size() > MAX_STACK_NAME_LEN) { throw InvalidInputException( "cloudformation_create_stack: explicit name '%s' is %llu chars; max %llu " @@ -247,6 +248,7 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn } stack_name = data.name_arg; } else { + // Auto-generate prefix, then append UUID string prefix; if (!metadata_stack_name.empty()) { prefix = SanitizeNameFragment(metadata_stack_name); @@ -272,10 +274,10 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn throw InvalidInputException( "cloudformation_create_stack: option '%s' is not a parameter declared by the template", kv.key); } - Aws::CloudFormation::Model::Parameter p; - p.SetParameterKey(kv.key.c_str()); - p.SetParameterValue(kv.value.c_str()); - cloudformation_params.push_back(p); + Aws::CloudFormation::Model::Parameter param; + param.SetParameterKey(kv.key.c_str()); + param.SetParameterValue(kv.value.c_str()); + cloudformation_params.push_back(param); } // Tags: provenance auto-tags first, then caller-supplied extras (which From 6b110e24bf46928f0797065220d2fb59aaf82861 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 25 Jun 2026 11:44:40 +0200 Subject: [PATCH 14/16] Review: Remove some StringKV to case_insensitive_map_t --- src/cloudformation_functions.cpp | 61 ++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index 4475b3c..636dcdb 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -2,6 +2,7 @@ #include "aws_client.hpp" #include "duckdb.hpp" +#include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/types/uuid.hpp" @@ -50,14 +51,22 @@ vector UnpackStringMap(const Value &map_value) { return out; } -const string *FindCI(const vector &kvs, const string &key) { - auto lowered = StringUtil::Lower(key); - for (auto &kv : kvs) { - if (StringUtil::Lower(kv.key) == lowered) { - return &kv.value; +//! Case-insensitive variant: keys are matched/looked-up case-insensitively but +//! stored with their original case (e.g. for case-sensitive CFN parameter names). +case_insensitive_map_t UnpackStringMapCI(const Value &map_value) { + case_insensitive_map_t out; + if (map_value.IsNull()) { + return out; + } + auto &children = MapValue::GetChildren(map_value); + for (auto &child : children) { + auto &kv = StructValue::GetChildren(child); + if (kv[0].IsNull()) { + continue; } + out[StringValue::Get(kv[0])] = kv[1].IsNull() ? string() : StringValue::Get(kv[1]); } - return nullptr; + return out; } bool LooksLikeUrl(const string &s) { @@ -131,7 +140,7 @@ const std::set RESERVED_OPTION_KEYS = { struct CloudFormationCreateStackBindData : public TableFunctionData { string template_arg; string name_arg; - vector options; + case_insensitive_map_t options; bool has_region_override = false; string region_override; vector tags_override; @@ -149,7 +158,7 @@ static unique_ptr CloudFormationCreateStackBind(ClientContext &con if (!input.inputs[1].IsNull()) { result->name_arg = StringValue::Get(input.inputs[1]); } - result->options = UnpackStringMap(input.inputs[2]); + result->options = UnpackStringMapCI(input.inputs[2]); for (auto &np : input.named_parameters) { auto key = StringUtil::Lower(np.first.GetIdentifierName()); @@ -179,8 +188,8 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn string region; if (data.has_region_override) { region = data.region_override; - } else if (auto *r = FindCI(data.options, "region")) { - region = *r; + } else if (auto it = data.options.find("region"); it != data.options.end()) { + region = it->second; } if (region.empty()) { throw InvalidInputException( @@ -188,8 +197,8 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn } auto opt = [&](const string &key) -> string { - auto *v = FindCI(data.options, key); - return v ? *v : string(); + auto it = data.options.find(key); + return it != data.options.end() ? it->second : string(); }; auto provider = BuildAwsCredentialsProvider(opt("chain"), /*require_credentials=*/true, opt("profile"), @@ -267,16 +276,16 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn // Route every non-reserved option key to a declared template parameter. Aws::Vector cloudformation_params; for (auto &kv : data.options) { - if (RESERVED_OPTION_KEYS.count(StringUtil::Lower(kv.key))) { + if (RESERVED_OPTION_KEYS.count(StringUtil::Lower(kv.first))) { continue; } - if (!declared_params.count(kv.key)) { + if (!declared_params.count(kv.first)) { throw InvalidInputException( - "cloudformation_create_stack: option '%s' is not a parameter declared by the template", kv.key); + "cloudformation_create_stack: option '%s' is not a parameter declared by the template", kv.first); } Aws::CloudFormation::Model::Parameter param; - param.SetParameterKey(kv.key.c_str()); - param.SetParameterValue(kv.value.c_str()); + param.SetParameterKey(kv.first.c_str()); + param.SetParameterValue(kv.second.c_str()); cloudformation_params.push_back(param); } @@ -364,17 +373,15 @@ struct CloudFormationHandle { }; CloudFormationHandle ParseHandle(const Value &handle_value, const char *fn_name) { - auto kvs = UnpackStringMap(handle_value); + auto kvs = UnpackStringMapCI(handle_value); CloudFormationHandle h; - if (auto *v = FindCI(kvs, "stack_id")) { - h.stack_id = *v; - } - if (auto *v = FindCI(kvs, "stack_name")) { - h.stack_name = *v; - } - if (auto *v = FindCI(kvs, "region")) { - h.region = *v; - } + auto get = [&](const string &key) -> string { + auto it = kvs.find(key); + return it != kvs.end() ? it->second : string(); + }; + h.stack_id = get("stack_id"); + h.stack_name = get("stack_name"); + h.region = get("region"); h.stack_ref = !h.stack_id.empty() ? h.stack_id : h.stack_name; if (h.stack_ref.empty()) { throw InvalidInputException("%s: handle must contain 'stack_id' or 'stack_name'", fn_name); From f20c64cd477b4e0c95bd13feb7a27f1003096d2e Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 25 Jun 2026 12:51:32 +0200 Subject: [PATCH 15/16] Fixup test/sql/aws_secret.test --- test/sql/aws_secret.test | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/sql/aws_secret.test b/test/sql/aws_secret.test index c0b6a94..b307de8 100644 --- a/test/sql/aws_secret.test +++ b/test/sql/aws_secret.test @@ -29,22 +29,22 @@ CREATE PERSISTENT SECRET aws_all_s3_params_set ( ); # Now confirm the secrets are correct -query IIIIIII -SELECT * from duckdb_secrets(redact=false); +query IIIIII +SELECT * EXCLUDE secret_string from duckdb_secrets(redact=false); ---- -aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] name=aws_all_s3_params_set;type=s3;provider=credential_chain;serializable=true;scope=s3://,s3n://,s3a://;endpoint=endpoint_override;key_id=key_override;region=region_override;secret=secret_override;session_token=session_override;url_style=style_override;use_ssl=true +aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] restart statement ok set secret_directory='{TEST_DIR}/aws_secret' -query IIIIIII -SELECT * from duckdb_secrets(redact=false); +query IIIIII +SELECT * EXCLUDE secret_string from duckdb_secrets(redact=false); ---- -aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] name=aws_all_s3_params_set;type=s3;provider=credential_chain;serializable=true;scope=s3://,s3n://,s3a://;endpoint=endpoint_override;key_id=key_override;region=region_override;secret=secret_override;session_token=session_override;url_style=style_override;use_ssl=true +aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] -query IIIIIII -SELECT * from duckdb_secrets(); +query IIIIII +SELECT * EXCLUDE secret_string from duckdb_secrets(); ---- -aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] name=aws_all_s3_params_set;type=s3;provider=credential_chain;serializable=true;scope=s3://,s3n://,s3a://;endpoint=endpoint_override;key_id=key_override;region=region_override;secret=redacted;session_token=redacted;url_style=style_override;use_ssl=true \ No newline at end of file +aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] From 2835644f912da179de2d6f4be603f0725bffd7dc Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 25 Jun 2026 14:28:33 +0200 Subject: [PATCH 16/16] StringKV -> InsertionOrderPreservingMap --- src/cloudformation_functions.cpp | 49 ++++++++++++++------------------ 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp index 636dcdb..3f8f3e3 100644 --- a/src/cloudformation_functions.cpp +++ b/src/cloudformation_functions.cpp @@ -4,6 +4,7 @@ #include "duckdb.hpp" #include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/exception.hpp" +#include "duckdb/common/insertion_order_preserving_map.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/types/uuid.hpp" #include "duckdb/function/scalar_function.hpp" @@ -30,13 +31,13 @@ namespace duckdb { namespace { -struct StringKV { - string key; - string value; -}; +//! Case-sensitive, insertion-order-preserving string map. The index map must be +//! spelled out because the type defaults to case-insensitive matching, which is +//! wrong for case-sensitive keys like AWS tag keys. +using OrderedStringMap = InsertionOrderPreservingMap>; -vector UnpackStringMap(const Value &map_value) { - vector out; +OrderedStringMap UnpackStringMap(const Value &map_value) { + OrderedStringMap out; if (map_value.IsNull()) { return out; } @@ -46,7 +47,7 @@ vector UnpackStringMap(const Value &map_value) { if (kv[0].IsNull()) { continue; } - out.push_back({StringValue::Get(kv[0]), kv[1].IsNull() ? string() : StringValue::Get(kv[1])}); + out[StringValue::Get(kv[0])] = kv[1].IsNull() ? string() : StringValue::Get(kv[1]); } return out; } @@ -143,7 +144,7 @@ struct CloudFormationCreateStackBindData : public TableFunctionData { case_insensitive_map_t options; bool has_region_override = false; string region_override; - vector tags_override; + OrderedStringMap tags_override; bool finished = false; }; @@ -290,32 +291,24 @@ static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionIn } // Tags: provenance auto-tags first, then caller-supplied extras (which - // override on key collision because they're applied last). - vector tag_kv; - auto set_tag = [&](const string &k, const string &v) { - for (auto &existing : tag_kv) { - if (existing.key == k) { - existing.value = v; - return; - } - } - tag_kv.push_back({k, v}); - }; - set_tag("created-by", "duckdb-aws"); - set_tag("created-by-version", DUCKDB_AWS_GIT_SHA); - set_tag("duckdb-version", DuckDB::LibraryVersion()); - set_tag("duckdb-session-id", SessionId()); + // override on key collision because they're applied last). operator[] gives + // last-wins override while keeping insertion order. + OrderedStringMap tags; + tags["created-by"] = "duckdb-aws"; + tags["created-by-version"] = DUCKDB_AWS_GIT_SHA; + tags["duckdb-version"] = DuckDB::LibraryVersion(); + tags["duckdb-session-id"] = SessionId(); if (!metadata_stack_name.empty()) { - set_tag("stack-name", metadata_stack_name); + tags["stack-name"] = metadata_stack_name; } for (auto &kv : data.tags_override) { - set_tag(kv.key, kv.value); + tags[kv.first] = kv.second; } Aws::Vector cloudformation_tags; - for (auto &kv : tag_kv) { + for (auto &kv : tags) { Aws::CloudFormation::Model::Tag t; - t.SetKey(kv.key.c_str()); - t.SetValue(kv.value.c_str()); + t.SetKey(kv.first.c_str()); + t.SetValue(kv.second.c_str()); cloudformation_tags.push_back(t); }