diff --git a/CMakeLists.txt b/CMakeLists.txt index 74abeec..04271ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,21 @@ set(EXTENSION_NAME ${TARGET_NAME}_extension) project(${TARGET_NAME}) include_directories(src/include) -set(EXTENSION_SOURCES src/aws_extension.cpp src/aws_secret.cpp) +# Capture the duckdb-aws extension's git short SHA at build time. Surfaced +# at runtime as the `created-by-version` auto-tag on every stack the +# cloudformation_* functions create. +execute_process( + COMMAND git rev-parse --short HEAD + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE DUCKDB_AWS_GIT_SHA + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) +if(NOT DUCKDB_AWS_GIT_SHA) + set(DUCKDB_AWS_GIT_SHA "unknown") +endif() +add_compile_definitions(DUCKDB_AWS_GIT_SHA="${DUCKDB_AWS_GIT_SHA}") + +set(EXTENSION_SOURCES src/aws_extension.cpp src/aws_secret.cpp src/cloudformation_functions.cpp) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) set(PARAMETERS "-warnings") @@ -16,7 +30,7 @@ build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES}) # Weirdly we need to manually to this, otherwise linking against # ${AWSSDK_LINK_LIBRARIES} fails for some reason find_package(ZLIB REQUIRED) -find_package(AWSSDK REQUIRED COMPONENTS core sso sts identity-management rds) +find_package(AWSSDK REQUIRED COMPONENTS core sso sts identity-management rds cloudformation) # Build static lib target_include_directories(${EXTENSION_NAME} diff --git a/src/aws_extension.cpp b/src/aws_extension.cpp index 75be7ce..2c29956 100644 --- a/src/aws_extension.cpp +++ b/src/aws_extension.cpp @@ -1,5 +1,6 @@ #include "aws_secret.hpp" #include "aws_extension.hpp" +#include "cloudformation_functions.hpp" #include "duckdb.hpp" #include "duckdb/common/exception.hpp" @@ -146,6 +147,8 @@ static void LoadInternal(ExtensionLoader &loader) { loader.RegisterFunction(function_set); CreateAwsSecretFunctions::Register(loader); + + CloudFormationFunctions::Register(loader); } void AwsExtension::Load(ExtensionLoader &loader) { diff --git a/src/aws_secret.cpp b/src/aws_secret.cpp index 17e66bf..3bdfaa3 100644 --- a/src/aws_secret.cpp +++ b/src/aws_secret.cpp @@ -1,4 +1,5 @@ #include "aws_secret.hpp" +#include "aws_client.hpp" #include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/exception.hpp" @@ -43,14 +44,8 @@ static struct { const string default_ = "exists"; } CreateAwsSecretValidation; -//! Build a ClientConfiguration with the detected CA file path applied (if any). -//! This is required because libcurl is statically linked from a RHEL-based -//! manylinux image, so its default CA path is /etc/pki/tls/certs/ca-bundle.crt, -//! which does not exist on Debian/Ubuntu/Alpine/etc. Without this, every -//! credentials provider that does its own HTTPS (SSO, STS, RDS, ...) fails the -//! TLS handshake before it can fetch any credentials. -//! See duckdb/duckdb#20652, duckdb/duckdb-aws#131. -static Aws::Client::ClientConfiguration BuildClientConfigWithCa() { +//! See aws_client.hpp for rationale. +Aws::Client::ClientConfiguration BuildClientConfigWithCa() { Aws::Client::ClientConfiguration cfg; if (!SELECTED_CURL_CERT_PATH.empty()) { cfg.caFile = SELECTED_CURL_CERT_PATH; @@ -509,6 +504,22 @@ static unique_ptr CreateAWSSecretFromCredentialChain(ClientContext & return std::move(result); } +std::shared_ptr +BuildAwsCredentialsProvider(const std::string &chain, bool require_credentials, const std::string &profile, + const std::string &assume_role_arn, const std::string &external_id, + const std::string &web_identity_token_file, const std::string &session_name) { + if (chain.empty()) { + if (!profile.empty()) { + return Aws::MakeShared("DuckDBAwsProfile", + profile.c_str()); + } + return Aws::MakeShared("DuckDBAwsDefault"); + } + return Aws::MakeShared("DuckDBCustomChain", chain, require_credentials, + profile, assume_role_arn, external_id, + web_identity_token_file, session_name); +} + void CreateAwsSecretFunctions::InitializeCurlCertificates(DatabaseInstance &db) { for (string &caFile : certFileLocations) { struct stat buf; diff --git a/src/cloudformation_functions.cpp b/src/cloudformation_functions.cpp new file mode 100644 index 0000000..3f8f3e3 --- /dev/null +++ b/src/cloudformation_functions.cpp @@ -0,0 +1,723 @@ +#include "cloudformation_functions.hpp" +#include "aws_client.hpp" + +#include "duckdb.hpp" +#include "duckdb/common/case_insensitive_map.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/common/insertion_order_preserving_map.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/uuid.hpp" +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/main/database.hpp" +#include "duckdb/main/extension/extension_loader.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace duckdb { + +namespace { + +//! Case-sensitive, insertion-order-preserving string map. The index map must be +//! spelled out because the type defaults to case-insensitive matching, which is +//! wrong for case-sensitive keys like AWS tag keys. +using OrderedStringMap = InsertionOrderPreservingMap>; + +OrderedStringMap UnpackStringMap(const Value &map_value) { + OrderedStringMap out; + if (map_value.IsNull()) { + return out; + } + auto &children = MapValue::GetChildren(map_value); + for (auto &child : children) { + auto &kv = StructValue::GetChildren(child); + if (kv[0].IsNull()) { + continue; + } + out[StringValue::Get(kv[0])] = kv[1].IsNull() ? string() : StringValue::Get(kv[1]); + } + return out; +} + +//! Case-insensitive variant: keys are matched/looked-up case-insensitively but +//! stored with their original case (e.g. for case-sensitive CFN parameter names). +case_insensitive_map_t UnpackStringMapCI(const Value &map_value) { + case_insensitive_map_t out; + if (map_value.IsNull()) { + return out; + } + auto &children = MapValue::GetChildren(map_value); + for (auto &child : children) { + auto &kv = StructValue::GetChildren(child); + if (kv[0].IsNull()) { + continue; + } + out[StringValue::Get(kv[0])] = kv[1].IsNull() ? string() : StringValue::Get(kv[1]); + } + return out; +} + +bool LooksLikeUrl(const string &s) { + return StringUtil::StartsWith(s, "http://") || StringUtil::StartsWith(s, "https://"); +} + +//! Coerce a free-form string into a CFN-name-safe fragment ([a-zA-Z0-9-]). +string SanitizeNameFragment(const string &in) { + string out; + for (char c : in) { + if (std::isalnum(static_cast(c)) || c == '-') { + out += c; + } else { + out += '-'; + } + } + return out; +} + +//! Stem of a URL's last path segment: https://host/path/quack.yaml -> "quack". +string UrlBasenameStem(const string &url) { + auto slash = url.find_last_of('/'); + string base = (slash == string::npos) ? url : url.substr(slash + 1); + auto qmark = base.find('?'); + if (qmark != string::npos) { + base = base.substr(0, qmark); + } + auto dot = base.find_last_of('.'); + if (dot != string::npos && dot > 0) { + base = base.substr(0, dot); + } + return SanitizeNameFragment(base); +} + +//! 12 hex characters of randomness from a fresh UUID. Keeping suffixes short +//! ensures the resulting CFN ARN stays within CloudFormation's 128-char +//! StackName API limit even in the longest-named regions. +string ShortRandHex() { + auto uuid_str = UUID::ToString(UUID::GenerateRandomUUID()); + // UUID layout: 8-4-4-4-12. First 12 hex = first segment (8) + second segment (4). + return uuid_str.substr(0, 8) + uuid_str.substr(9, 4); +} + +//! Per-process session id used for the duckdb-session-id auto-tag on every +//! stack this extension creates. Lazy-initialised on first use, then stable +//! for the lifetime of the duckdb-aws extension load. +const string &SessionId() { + static const string id = ShortRandHex(); + return id; +} + +//! CFN's StackName API parameter is capped at 128 chars. The ARN is +//! `arn:aws:cloudformation:::stack//` — +//! structural+region(≤14)+account(12)+slashes(2)+cfn-uuid(36) ≤ 94, so the +//! name budget is 128 - 94 = 34 chars (worst case across regions). +constexpr idx_t MAX_STACK_NAME_LEN = 34; +constexpr idx_t MAX_PREFIX_LEN = MAX_STACK_NAME_LEN - 12 - 1; // 21: '-<12hex>' + +//! Reserved option keys configure the AWS client; everything else in `options` +//! must match a parameter the template declares. +const std::set RESERVED_OPTION_KEYS = { + "region", "chain", "profile", "assume_role_arn", + "external_id", "web_identity_token_file", "session_name"}; + +} // namespace + +//===--------------------------------------------------------------------===// +// cloudformation_create_stack(template, name, options) [region :=, tags :=] +//===--------------------------------------------------------------------===// + +struct CloudFormationCreateStackBindData : public TableFunctionData { + string template_arg; + string name_arg; + case_insensitive_map_t options; + bool has_region_override = false; + string region_override; + OrderedStringMap tags_override; + bool finished = false; +}; + +static unique_ptr CloudFormationCreateStackBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + auto result = make_uniq(); + + if (input.inputs[0].IsNull()) { + throw InvalidInputException("cloudformation_create_stack: the template argument must not be NULL"); + } + result->template_arg = StringValue::Get(input.inputs[0]); + if (!input.inputs[1].IsNull()) { + result->name_arg = StringValue::Get(input.inputs[1]); + } + result->options = UnpackStringMapCI(input.inputs[2]); + + for (auto &np : input.named_parameters) { + auto key = StringUtil::Lower(np.first.GetIdentifierName()); + if (key == "region") { + if (!np.second.IsNull()) { + result->has_region_override = true; + result->region_override = StringValue::Get(np.second); + } + } else if (key == "tags") { + result->tags_override = UnpackStringMap(np.second); + } + } + + return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + names.emplace_back("handle"); + + return std::move(result); +} + +static void CloudFormationCreateStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CloudFormationCreateStackBindData &)*data_p.bind_data; + if (data.finished) { + return; + } + + // Region: explicit override wins over options['region']. + string region; + if (data.has_region_override) { + region = data.region_override; + } else if (auto it = data.options.find("region"); it != data.options.end()) { + region = it->second; + } + if (region.empty()) { + throw InvalidInputException( + "cloudformation_create_stack: region is required - set options['region'] or the region:= named parameter"); + } + + auto opt = [&](const string &key) -> string { + auto it = data.options.find(key); + return it != data.options.end() ? it->second : string(); + }; + + auto provider = BuildAwsCredentialsProvider(opt("chain"), /*require_credentials=*/true, opt("profile"), + opt("assume_role_arn"), opt("external_id"), + opt("web_identity_token_file"), opt("session_name")); + auto client_config = BuildClientConfigWithCa(); + client_config.region = region.c_str(); + Aws::CloudFormation::CloudFormationClient cloudformation_client(provider, client_config); + + bool template_is_url = LooksLikeUrl(data.template_arg); + + // Ask CloudFormation to parse the template: declared parameters, required + // capabilities, and the Metadata section. + Aws::CloudFormation::Model::GetTemplateSummaryRequest summary_req; + if (template_is_url) { + summary_req.SetTemplateURL(data.template_arg.c_str()); + } else { + summary_req.SetTemplateBody(data.template_arg.c_str()); + } + auto summary_outcome = cloudformation_client.GetTemplateSummary(summary_req); + if (!summary_outcome.IsSuccess()) { + const auto &err = summary_outcome.GetError(); + throw IOException("CloudFormation GetTemplateSummary failed: %s - %s", string(err.GetExceptionName().c_str()), + string(err.GetMessage().c_str())); + } + const auto &summary = summary_outcome.GetResult(); + + std::set declared_params; + for (const auto &pd : summary.GetParameters()) { + declared_params.insert(string(pd.GetParameterKey().c_str())); + } + + string metadata_stack_name; + if (!summary.GetMetadata().empty()) { + Aws::Utils::Json::JsonValue meta(summary.GetMetadata()); + if (meta.WasParseSuccessful()) { + auto view = meta.View(); + if (view.KeyExists("StackName")) { + metadata_stack_name = string(view.GetString("StackName").c_str()); + } + } + } + + // Resolve the stack name. Cap autogenerated prefixes so the full ARN + // stays within CloudFormation's 128-char StackName API limit. Reject + // explicit names that exceed the limit. + string stack_name; + if (!data.name_arg.empty()) { + // Explicit name + if (data.name_arg.size() > MAX_STACK_NAME_LEN) { + throw InvalidInputException( + "cloudformation_create_stack: explicit name '%s' is %llu chars; max %llu " + "(to keep the resulting CFN stack ARN within the 128-char API limit)", + data.name_arg, (unsigned long long)data.name_arg.size(), + (unsigned long long)MAX_STACK_NAME_LEN); + } + stack_name = data.name_arg; + } else { + // Auto-generate prefix, then append UUID + string prefix; + if (!metadata_stack_name.empty()) { + prefix = SanitizeNameFragment(metadata_stack_name); + } else if (template_is_url) { + prefix = UrlBasenameStem(data.template_arg); + } + if (prefix.empty()) { + prefix = "duckdb-aws"; + } + if (prefix.size() > MAX_PREFIX_LEN) { + prefix = prefix.substr(0, MAX_PREFIX_LEN); + } + stack_name = prefix + "-" + ShortRandHex(); + } + + // Route every non-reserved option key to a declared template parameter. + Aws::Vector cloudformation_params; + for (auto &kv : data.options) { + if (RESERVED_OPTION_KEYS.count(StringUtil::Lower(kv.first))) { + continue; + } + if (!declared_params.count(kv.first)) { + throw InvalidInputException( + "cloudformation_create_stack: option '%s' is not a parameter declared by the template", kv.first); + } + Aws::CloudFormation::Model::Parameter param; + param.SetParameterKey(kv.first.c_str()); + param.SetParameterValue(kv.second.c_str()); + cloudformation_params.push_back(param); + } + + // Tags: provenance auto-tags first, then caller-supplied extras (which + // override on key collision because they're applied last). operator[] gives + // last-wins override while keeping insertion order. + OrderedStringMap tags; + tags["created-by"] = "duckdb-aws"; + tags["created-by-version"] = DUCKDB_AWS_GIT_SHA; + tags["duckdb-version"] = DuckDB::LibraryVersion(); + tags["duckdb-session-id"] = SessionId(); + if (!metadata_stack_name.empty()) { + tags["stack-name"] = metadata_stack_name; + } + for (auto &kv : data.tags_override) { + tags[kv.first] = kv.second; + } + Aws::Vector cloudformation_tags; + for (auto &kv : tags) { + Aws::CloudFormation::Model::Tag t; + t.SetKey(kv.first.c_str()); + t.SetValue(kv.second.c_str()); + cloudformation_tags.push_back(t); + } + + Aws::CloudFormation::Model::CreateStackRequest req; + req.SetStackName(stack_name.c_str()); + if (template_is_url) { + req.SetTemplateURL(data.template_arg.c_str()); + } else { + req.SetTemplateBody(data.template_arg.c_str()); + } + if (!cloudformation_params.empty()) { + req.SetParameters(cloudformation_params); + } + if (!summary.GetCapabilities().empty()) { + req.SetCapabilities(summary.GetCapabilities()); + } + if (!cloudformation_tags.empty()) { + req.SetTags(cloudformation_tags); + } + + auto outcome = cloudformation_client.CreateStack(req); + if (!outcome.IsSuccess()) { + const auto &err = outcome.GetError(); + throw IOException("CloudFormation CreateStack failed: %s - %s", string(err.GetExceptionName().c_str()), + string(err.GetMessage().c_str())); + } + string stack_id(outcome.GetResult().GetStackId().c_str()); + + vector keys; + vector values; + keys.emplace_back("stack_name"); + values.emplace_back(stack_name); + keys.emplace_back("stack_id"); + values.emplace_back(stack_id); + keys.emplace_back("region"); + values.emplace_back(region); + auto handle = Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(keys), std::move(values)); + + output.SetValue(0, 0, handle); + output.SetCardinality(1); + data.finished = true; +} + +//===--------------------------------------------------------------------===// +// Shared handle parsing (cloudformation_describe_stack / cloudformation_outputs / cloudformation_delete_stack) +//===--------------------------------------------------------------------===// + +namespace { + +struct CloudFormationHandle { + string stack_ref; // stack_id (ARN) when present, else stack_name + string stack_name; + string stack_id; + string region; +}; + +CloudFormationHandle ParseHandle(const Value &handle_value, const char *fn_name) { + auto kvs = UnpackStringMapCI(handle_value); + CloudFormationHandle h; + auto get = [&](const string &key) -> string { + auto it = kvs.find(key); + return it != kvs.end() ? it->second : string(); + }; + h.stack_id = get("stack_id"); + h.stack_name = get("stack_name"); + h.region = get("region"); + h.stack_ref = !h.stack_id.empty() ? h.stack_id : h.stack_name; + if (h.stack_ref.empty()) { + throw InvalidInputException("%s: handle must contain 'stack_id' or 'stack_name'", fn_name); + } + if (h.region.empty()) { + throw InvalidInputException("%s: handle is missing 'region'", fn_name); + } + return h; +} + +} // namespace + +//===--------------------------------------------------------------------===// +// cloudformation_describe_stack(handle) +//===--------------------------------------------------------------------===// + +struct CloudFormationDescribeStackBindData : public TableFunctionData { + CloudFormationHandle handle; + bool finished = false; +}; + +static unique_ptr CloudFormationDescribeStackBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + auto result = make_uniq(); + result->handle = ParseHandle(input.inputs[0], "cloudformation_describe_stack"); + + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("region"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("stack_name"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("stack_id"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("status"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("status_reason"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("creation_time"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("last_updated_time"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("description"); + return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + names.emplace_back("tags"); + return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + names.emplace_back("outputs"); + + return std::move(result); +} + +static void CloudFormationDescribeStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CloudFormationDescribeStackBindData &)*data_p.bind_data; + if (data.finished) { + return; + } + + auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); + auto cfg = BuildClientConfigWithCa(); + cfg.region = data.handle.region.c_str(); + Aws::CloudFormation::CloudFormationClient cloudformation_client(provider, cfg); + + Aws::CloudFormation::Model::DescribeStacksRequest req; + req.SetStackName(data.handle.stack_ref.c_str()); + auto outcome = cloudformation_client.DescribeStacks(req); + if (!outcome.IsSuccess()) { + const auto &err = outcome.GetError(); + throw IOException("CloudFormation DescribeStacks failed: %s - %s", string(err.GetExceptionName().c_str()), + string(err.GetMessage().c_str())); + } + const auto &stacks = outcome.GetResult().GetStacks(); + if (stacks.empty()) { + throw IOException("CloudFormation DescribeStacks returned no stack for '%s'", data.handle.stack_ref); + } + const auto &stack = stacks[0]; + + string status( + Aws::CloudFormation::Model::StackStatusMapper::GetNameForStackStatus(stack.GetStackStatus()).c_str()); + string reason(stack.GetStackStatusReason().c_str()); + string created(stack.GetCreationTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + string updated; + if (stack.LastUpdatedTimeHasBeenSet()) { + updated = string(stack.GetLastUpdatedTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + } + string description(stack.GetDescription().c_str()); + + vector tag_keys; + vector tag_values; + for (const auto &t : stack.GetTags()) { + tag_keys.emplace_back(string(t.GetKey().c_str())); + tag_values.emplace_back(string(t.GetValue().c_str())); + } + auto tags = Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(tag_keys), std::move(tag_values)); + + vector output_keys; + vector output_values; + for (const auto &o : stack.GetOutputs()) { + output_keys.emplace_back(string(o.GetOutputKey().c_str())); + output_values.emplace_back(string(o.GetOutputValue().c_str())); + } + auto outputs = + Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(output_keys), std::move(output_values)); + + output.SetValue(0, 0, Value(data.handle.region)); + output.SetValue(1, 0, Value(string(stack.GetStackName().c_str()))); + output.SetValue(2, 0, Value(string(stack.GetStackId().c_str()))); + output.SetValue(3, 0, Value(status)); + output.SetValue(4, 0, reason.empty() ? Value() : Value(reason)); + output.SetValue(5, 0, created.empty() ? Value() : Value(created)); + output.SetValue(6, 0, updated.empty() ? Value() : Value(updated)); + output.SetValue(7, 0, description.empty() ? Value() : Value(description)); + output.SetValue(8, 0, tags); + output.SetValue(9, 0, outputs); + output.SetCardinality(1); + data.finished = true; +} + +//===--------------------------------------------------------------------===// +// cloudformation_delete_stack(handle) +//===--------------------------------------------------------------------===// + +struct CloudFormationDeleteStackBindData : public TableFunctionData { + CloudFormationHandle handle; + Value handle_value; // original MAP, echoed back verbatim + bool finished = false; +}; + +static unique_ptr CloudFormationDeleteStackBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + auto result = make_uniq(); + result->handle = ParseHandle(input.inputs[0], "cloudformation_delete_stack"); + result->handle_value = input.inputs[0]; + + return_types.emplace_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + names.emplace_back("handle"); + + return std::move(result); +} + +static void CloudFormationDeleteStackFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CloudFormationDeleteStackBindData &)*data_p.bind_data; + if (data.finished) { + return; + } + + auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); + auto cfg = BuildClientConfigWithCa(); + cfg.region = data.handle.region.c_str(); + Aws::CloudFormation::CloudFormationClient cloudformation_client(provider, cfg); + + Aws::CloudFormation::Model::DeleteStackRequest req; + req.SetStackName(data.handle.stack_ref.c_str()); + auto outcome = cloudformation_client.DeleteStack(req); + if (!outcome.IsSuccess()) { + const auto &err = outcome.GetError(); + throw IOException("CloudFormation DeleteStack failed: %s - %s", string(err.GetExceptionName().c_str()), + string(err.GetMessage().c_str())); + } + + // Pass-through: echo the input handle byte-for-byte. Any extra keys the + // caller put in (annotations, timestamps, custom metadata) survive intact. + output.SetValue(0, 0, data.handle_value); + output.SetCardinality(1); + data.finished = true; +} + +//===--------------------------------------------------------------------===// +// cloudformation_list_stacks([region := ...] [status_filter := ...]) +//===--------------------------------------------------------------------===// + +struct CloudFormationListStacksRow { + string region; + string stack_name; + string stack_id; + string status; + string status_reason; + string creation_time; + string last_updated_time; + string description; +}; + +struct CloudFormationListStacksBindData : public TableFunctionData { + string region; + vector status_filter; + bool initialized = false; + vector rows; + idx_t cursor = 0; +}; + +static unique_ptr CloudFormationListStacksBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, + vector &names) { + auto result = make_uniq(); + + if (input.inputs[0].IsNull()) { + throw InvalidInputException("cloudformation_list_stacks: region must not be NULL"); + } + result->region = StringValue::Get(input.inputs[0]); + if (result->region.empty()) { + throw InvalidInputException("cloudformation_list_stacks: region must not be empty"); + } + + for (auto &np : input.named_parameters) { + auto key = StringUtil::Lower(np.first.GetIdentifierName()); + if (key == "status_filter") { + if (!np.second.IsNull()) { + auto &children = ListValue::GetChildren(np.second); + for (auto &child : children) { + if (child.IsNull()) { + continue; + } + auto str = StringValue::Get(child); + auto status_enum = + Aws::CloudFormation::Model::StackStatusMapper::GetStackStatusForName(str.c_str()); + if (status_enum == Aws::CloudFormation::Model::StackStatus::NOT_SET) { + throw InvalidInputException( + "cloudformation_list_stacks: unknown stack status '%s' in status_filter", str); + } + result->status_filter.push_back(status_enum); + } + } + } + } + + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("region"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("stack_name"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("stack_id"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("status"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("status_reason"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("creation_time"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("last_updated_time"); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("description"); + + return std::move(result); +} + +static void CloudFormationListStacksFun(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = (CloudFormationListStacksBindData &)*data_p.bind_data; + + if (!data.initialized) { + auto provider = BuildAwsCredentialsProvider("", /*require_credentials=*/true); + auto cfg = BuildClientConfigWithCa(); + cfg.region = data.region.c_str(); + Aws::CloudFormation::CloudFormationClient cloudformation_client(provider, cfg); + + Aws::String next_token; + do { + Aws::CloudFormation::Model::ListStacksRequest req; + if (!data.status_filter.empty()) { + req.SetStackStatusFilter(data.status_filter); + } + if (!next_token.empty()) { + req.SetNextToken(next_token); + } + auto outcome = cloudformation_client.ListStacks(req); + if (!outcome.IsSuccess()) { + const auto &err = outcome.GetError(); + throw IOException("CloudFormation ListStacks failed: %s - %s", + string(err.GetExceptionName().c_str()), string(err.GetMessage().c_str())); + } + const auto &res = outcome.GetResult(); + for (const auto &s : res.GetStackSummaries()) { + CloudFormationListStacksRow row; + row.region = data.region; + row.stack_name = string(s.GetStackName().c_str()); + row.stack_id = string(s.GetStackId().c_str()); + row.status = string( + Aws::CloudFormation::Model::StackStatusMapper::GetNameForStackStatus(s.GetStackStatus()).c_str()); + row.status_reason = string(s.GetStackStatusReason().c_str()); + row.creation_time = + string(s.GetCreationTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + if (s.LastUpdatedTimeHasBeenSet()) { + row.last_updated_time = + string(s.GetLastUpdatedTime().ToGmtString(Aws::Utils::DateFormat::ISO_8601).c_str()); + } + row.description = string(s.GetTemplateDescription().c_str()); + data.rows.push_back(row); + } + next_token = res.GetNextToken(); + } while (!next_token.empty()); + data.initialized = true; + } + + idx_t remaining = data.rows.size() - data.cursor; + idx_t to_emit = std::min(remaining, (idx_t)STANDARD_VECTOR_SIZE); + for (idx_t i = 0; i < to_emit; i++) { + auto &r = data.rows[data.cursor + i]; + output.SetValue(0, i, Value(r.region)); + output.SetValue(1, i, Value(r.stack_name)); + output.SetValue(2, i, Value(r.stack_id)); + output.SetValue(3, i, Value(r.status)); + output.SetValue(4, i, r.status_reason.empty() ? Value() : Value(r.status_reason)); + output.SetValue(5, i, r.creation_time.empty() ? Value() : Value(r.creation_time)); + output.SetValue(6, i, r.last_updated_time.empty() ? Value() : Value(r.last_updated_time)); + output.SetValue(7, i, r.description.empty() ? Value() : Value(r.description)); + } + output.SetCardinality(to_emit); + data.cursor += to_emit; +} + +//===--------------------------------------------------------------------===// +// duckdb_aws_session_id() scalar function +//===--------------------------------------------------------------------===// + +static void DuckDBAwsSessionIdFunction(DataChunk &args, ExpressionState &state, Vector &result) { + result.SetVectorType(VectorType::CONSTANT_VECTOR); + auto data = ConstantVector::GetData(result); + data[0] = StringVector::AddString(result, SessionId()); +} + +//===--------------------------------------------------------------------===// +// Registration +//===--------------------------------------------------------------------===// + +void CloudFormationFunctions::Register(ExtensionLoader &loader) { + auto map_vv = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); + + TableFunction create_fn("cloudformation_create_stack", {LogicalType::VARCHAR, LogicalType::VARCHAR, map_vv}, + CloudFormationCreateStackFun, CloudFormationCreateStackBind); + create_fn.named_parameters["region"] = LogicalType::VARCHAR; + create_fn.named_parameters["tags"] = map_vv; + loader.RegisterFunction(create_fn); + + TableFunction describe_fn("cloudformation_describe_stack", {map_vv}, CloudFormationDescribeStackFun, CloudFormationDescribeStackBind); + loader.RegisterFunction(describe_fn); + + TableFunction delete_fn("cloudformation_delete_stack", {map_vv}, CloudFormationDeleteStackFun, CloudFormationDeleteStackBind); + loader.RegisterFunction(delete_fn); + + TableFunction list_fn("cloudformation_list_stacks", {LogicalType::VARCHAR}, CloudFormationListStacksFun, + CloudFormationListStacksBind); + list_fn.named_parameters["status_filter"] = LogicalType::LIST(LogicalType::VARCHAR); + loader.RegisterFunction(list_fn); + + ScalarFunction session_id_fn("duckdb_aws_session_id", {}, LogicalType::VARCHAR, DuckDBAwsSessionIdFunction); + loader.RegisterFunction(session_id_fn); +} + +} // namespace duckdb diff --git a/src/include/aws_client.hpp b/src/include/aws_client.hpp new file mode 100644 index 0000000..670d8e8 --- /dev/null +++ b/src/include/aws_client.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include + +namespace duckdb { + +//! Build a ClientConfiguration with the detected CA file path applied (if any). +//! Required because libcurl is statically linked from a RHEL-based manylinux +//! image, so its default CA path is /etc/pki/tls/certs/ca-bundle.crt, which +//! does not exist on Debian/Ubuntu/Alpine/etc. Without this, every AWS SDK +//! client that does its own HTTPS fails the TLS handshake. +//! See duckdb/duckdb#20652, duckdb/duckdb-aws#131. +Aws::Client::ClientConfiguration BuildClientConfigWithCa(); + +//! Build a credentials provider from the same opt-set that +//! CREATE SECRET ... credential_chain understands. Falls back to the SDK's +//! DefaultAWSCredentialsProviderChain when no chain/profile is specified. +//! `chain` is a ';'-separated list of provider names; see +//! DuckDBCustomAWSCredentialsProviderChain in aws_secret.cpp for the grammar. +std::shared_ptr +BuildAwsCredentialsProvider(const std::string &chain, bool require_credentials, + const std::string &profile = "", const std::string &assume_role_arn = "", + const std::string &external_id = "", const std::string &web_identity_token_file = "", + const std::string &session_name = ""); + +} // namespace duckdb diff --git a/src/include/cloudformation_functions.hpp b/src/include/cloudformation_functions.hpp new file mode 100644 index 0000000..b66afa0 --- /dev/null +++ b/src/include/cloudformation_functions.hpp @@ -0,0 +1,12 @@ +#pragma once + +namespace duckdb { + +class ExtensionLoader; + +struct CloudFormationFunctions { + //! Register the generic CloudFormation table functions (cloudformation_*). + static void Register(ExtensionLoader &loader); +}; + +} // namespace duckdb diff --git a/test/sql/aws_secret.test b/test/sql/aws_secret.test index c0b6a94..b307de8 100644 --- a/test/sql/aws_secret.test +++ b/test/sql/aws_secret.test @@ -29,22 +29,22 @@ CREATE PERSISTENT SECRET aws_all_s3_params_set ( ); # Now confirm the secrets are correct -query IIIIIII -SELECT * from duckdb_secrets(redact=false); +query IIIIII +SELECT * EXCLUDE secret_string from duckdb_secrets(redact=false); ---- -aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] name=aws_all_s3_params_set;type=s3;provider=credential_chain;serializable=true;scope=s3://,s3n://,s3a://;endpoint=endpoint_override;key_id=key_override;region=region_override;secret=secret_override;session_token=session_override;url_style=style_override;use_ssl=true +aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] restart statement ok set secret_directory='{TEST_DIR}/aws_secret' -query IIIIIII -SELECT * from duckdb_secrets(redact=false); +query IIIIII +SELECT * EXCLUDE secret_string from duckdb_secrets(redact=false); ---- -aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] name=aws_all_s3_params_set;type=s3;provider=credential_chain;serializable=true;scope=s3://,s3n://,s3a://;endpoint=endpoint_override;key_id=key_override;region=region_override;secret=secret_override;session_token=session_override;url_style=style_override;use_ssl=true +aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] -query IIIIIII -SELECT * from duckdb_secrets(); +query IIIIII +SELECT * EXCLUDE secret_string from duckdb_secrets(); ---- -aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] name=aws_all_s3_params_set;type=s3;provider=credential_chain;serializable=true;scope=s3://,s3n://,s3a://;endpoint=endpoint_override;key_id=key_override;region=region_override;secret=redacted;session_token=redacted;url_style=style_override;use_ssl=true \ No newline at end of file +aws_all_s3_params_set s3 credential_chain 1 local_file ['s3://', 's3n://', 's3a://'] diff --git a/test/sql/cloudformation_functions.test b/test/sql/cloudformation_functions.test new file mode 100644 index 0000000..df84e98 --- /dev/null +++ b/test/sql/cloudformation_functions.test @@ -0,0 +1,60 @@ +# name: test/sql/cloudformation_functions.test +# description: smoke test for the generic cloudformation_* CloudFormation functions +# group: [sql] + +require no_extension_autoloading "EXPECTED: Test relies on explicit INSTALL and LOAD" + +# Before LOAD aws, the functions do not exist. +statement error +SELECT * FROM cloudformation_create_stack('x', NULL, MAP {'region': 'us-east-1'}); +---- + +require aws + +# cloudformation_create_stack: a NULL template is rejected at bind time. +statement error +SELECT * FROM cloudformation_create_stack(NULL, NULL, MAP {'region': 'us-east-1'}); +---- +template argument must not be NULL + +# cloudformation_create_stack: region is required (checked before any AWS call). +statement error +SELECT * FROM cloudformation_create_stack('AWSTemplateFormatVersion: 2010-09-09', NULL, MAP {'InstanceType': 't3.micro'}); +---- +region is required + +# cloudformation_describe_stack: handle without a stack reference. +statement error +SELECT * FROM cloudformation_describe_stack(MAP {'region': 'us-east-1'}); +---- +handle must contain 'stack_id' or 'stack_name' + +# cloudformation_describe_stack: handle missing region. +statement error +SELECT * FROM cloudformation_describe_stack(MAP {'stack_name': 's'}); +---- +handle is missing 'region' + +# cloudformation_delete_stack: handle without a stack reference. +statement error +SELECT * FROM cloudformation_delete_stack(MAP {'region': 'us-east-1'}); +---- +handle must contain 'stack_id' or 'stack_name' + +# cloudformation_delete_stack: handle missing region. +statement error +SELECT * FROM cloudformation_delete_stack(MAP {'stack_name': 's'}); +---- +handle is missing 'region' + +# duckdb_aws_session_id returns a non-empty string. +query I +SELECT length(duckdb_aws_session_id()) > 0; +---- +true + +# Session id is stable within a session: repeated calls return the same value. +query I +SELECT duckdb_aws_session_id() = duckdb_aws_session_id(); +---- +true diff --git a/vcpkg.json b/vcpkg.json index b2800aa..d9d1569 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -4,7 +4,7 @@ "zlib", { "name": "aws-sdk-cpp", - "features": [ "sso", "sts" , "identity-management", "rds"] + "features": [ "sso", "sts" , "identity-management", "rds", "cloudformation"] }, "openssl", "curl"