From 4fa27771b16c85ff0a65f8fb652e4da02d805f2a Mon Sep 17 00:00:00 2001 From: Tmonster Date: Tue, 14 Jul 2026 15:26:02 +0200 Subject: [PATCH 1/5] add rds support and move around some other functions --- CMakeLists.txt | 1 + src/aws_extension.cpp | 4 + src/include/rds/rds_utils.hpp | 53 ++++++++ src/include/utils/utils.hpp | 24 +++- src/rds/rds_storage.cpp | 202 ++++++++++++++++++++++++++++++ src/rds/rds_utils.cpp | 85 +++++++++++++ src/redshift/redshift_storage.cpp | 48 +------ src/utils/utils.cpp | 50 ++++++++ 8 files changed, 419 insertions(+), 48 deletions(-) create mode 100644 src/include/rds/rds_utils.hpp create mode 100644 src/rds/rds_storage.cpp create mode 100644 src/rds/rds_utils.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f7eb80..b3d89a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ endif() add_compile_definitions(DUCKDB_AWS_GIT_SHA="${DUCKDB_AWS_GIT_SHA}") set(EXTENSION_SOURCES src/aws_extension.cpp src/aws_secret.cpp src/cloudformation_functions.cpp + src/rds/rds_utils.cpp src/rds/rds_storage.cpp src/redshift/redshift_utils.cpp src/redshift/redshift_storage.cpp src/utils/utils.cpp) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) diff --git a/src/aws_extension.cpp b/src/aws_extension.cpp index 469c6d7..52a6186 100644 --- a/src/aws_extension.cpp +++ b/src/aws_extension.cpp @@ -1,6 +1,7 @@ #include "aws_secret.hpp" #include "aws_extension.hpp" #include "cloudformation_functions.hpp" +#include "rds/rds_utils.hpp" #include "redshift/redshift_utils.hpp" #include "duckdb.hpp" @@ -25,6 +26,9 @@ static void LoadInternal(ExtensionLoader &loader) { // Makes `ATTACH '' (TYPE redshift, ...)` resolve to the redshift storage extension. Redshift::RegisterStorageExtension(loader); + // Same for `ATTACH '' (TYPE rds, ...)`. + Rds::RegisterStorageExtension(loader); + CloudFormationFunctions::Register(loader); } diff --git a/src/include/rds/rds_utils.hpp b/src/include/rds/rds_utils.hpp new file mode 100644 index 0000000..de38218 --- /dev/null +++ b/src/include/rds/rds_utils.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include "duckdb.hpp" +#include "duckdb/main/secret/secret.hpp" + +#include +#include + +namespace duckdb { + +class ExtensionLoader; + +//! The subset of an RDS DB instance description that we need to open a connection to it. Filled in +//! from the `DescribeDBInstances` API, i.e. what `aws rds describe-db-instances +//! --db-instance-identifier ` reports. +struct RdsInstanceInfo { + string endpoint_address; + int32_t endpoint_port = 0; + //! The engine the instance runs, e.g. 'postgres', 'aurora-postgresql', 'mysql' or 'oracle-se2'. + //! Only the Postgres-flavored ones can be attached; see RdsEngineIsPostgres. + string engine; + string engine_version; + //! The instance's initial database, which may be unset - RDS does not require one. + string db_name; + string master_username; + string status; + bool iam_auth_enabled = false; +}; + +//! Whether an RDS engine speaks the Postgres wire protocol, i.e. whether the postgres extension +//! can connect to it. Covers both 'postgres' (RDS for PostgreSQL) and 'aurora-postgresql'. +bool RdsEngineIsPostgres(const string &engine); + +struct Rds { + //! Register the 'rds' storage extension, which is what makes `ATTACH '' (TYPE rds)` + //! resolve here. + static void RegisterStorageExtension(ExtensionLoader &loader); + + //! Look up a DB instance by identifier via the RDS `DescribeDBInstances` API. Throws when the + //! instance does not exist or exposes no endpoint (e.g. while it is still being created). + static RdsInstanceInfo DescribeDBInstance(const std::shared_ptr &provider, + const string &instance_id, const string ®ion); + + //! Mint a short-lived (15 minute) IAM authentication token for `db_user` on the given endpoint, + //! to be used as the database password. Unlike Redshift's GetClusterCredentialsWithIAM this is + //! not an API call: the token is a request to the endpoint presigned with the AWS identity the + //! provider holds, so the database user must already exist and be granted `rds_iam`. + //! See https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html + static string GenerateAuthToken(const std::shared_ptr &provider, + const string &host, int32_t port, const string ®ion, const string &db_user); +}; + +} // namespace duckdb diff --git a/src/include/utils/utils.hpp b/src/include/utils/utils.hpp index 89b0fcb..d325743 100644 --- a/src/include/utils/utils.hpp +++ b/src/include/utils/utils.hpp @@ -4,6 +4,9 @@ #include "duckdb/main/secret/secret_manager.hpp" #include "duckdb/storage/storage_extension.hpp" +#include +#include + namespace duckdb { //! Resolve the AWS region to use, taking the first of these that yields a value: @@ -30,11 +33,28 @@ bool IsAwsSecretType(const string &type); //! that is present. Throws rather than returning null. unique_ptr FindAwsSecret(ClientContext &context, const string &secret_name); +//! Read a string field out of a key-value secret, "" when the field is not set. +string GetSecretString(const KeyValueSecret &secret, const string &key); + +//! A credentials provider for the AWS identity the secret resolved to. `service` only names the +//! caller in SDK allocation tags. Throws when the secret carries no credentials. +std::shared_ptr CredentialsProviderFromSecret(const KeyValueSecret &secret, + const string &service); + +//! Quote a value for a libpq connection string. Backslashes and single quotes must be escaped +//! with a backslash; quoting the whole value keeps empty values and embedded spaces valid. +string EscapeConnectionValue(const string &value); + //! The name of the postgres extension, for autoloading it. extern const char *const POSTGRES_EXTENSION_NAME; -//! The postgres extension's storage extension, or null when it is not loaded. Redshift speaks the -//! Postgres wire protocol, so attaching one hands off to postgres once the endpoint is resolved. +//! The postgres extension's storage extension, or null when it is not loaded. Redshift and RDS's +//! postgres engines speak the Postgres wire protocol, so attaching one hands off to postgres once +//! the endpoint is resolved. optional_ptr FindPostgresStorageExtension(const DBConfig &config); +//! The postgres storage extension, autoloading the postgres extension if it is not loaded yet. +//! `attach_type` only names the caller in the error thrown when it cannot be loaded. +optional_ptr RequirePostgresStorageExtension(ClientContext &context, const string &attach_type); + } // namespace duckdb diff --git a/src/rds/rds_storage.cpp b/src/rds/rds_storage.cpp new file mode 100644 index 0000000..5b79785 --- /dev/null +++ b/src/rds/rds_storage.cpp @@ -0,0 +1,202 @@ +#include "rds/rds_utils.hpp" + +#include "aws_client.hpp" +#include "utils/utils.hpp" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/main/attached_database.hpp" +#include "duckdb/main/database.hpp" +#include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/main/secret/secret_manager.hpp" +#include "duckdb/main/settings.hpp" +#include "duckdb/parser/parsed_data/attach_info.hpp" +#include "duckdb/storage/storage_extension.hpp" + +namespace duckdb { + +namespace { + +//! ATTACH options that we consume ourselves. Everything else is forwarded to the postgres +//! extension, which rejects options it does not know. +struct RdsAttachOptions { + string secret_name; + string region; + string db_name; + string host; + string port; + string user; +}; + +RdsAttachOptions ParseAttachOptions(AttachOptions &options) { + RdsAttachOptions parsed; + // Consumed options are erased, since postgres throws on any option it does not recognize. + for (auto it = options.options.begin(); it != options.options.end();) { + auto key = StringUtil::Lower(it->first); + auto value = it->second.ToString(); + if (key == "secret") { + // Left in place: postgres looks the secret up by name, and it must find one. See the + // note in RdsAttach. + parsed.secret_name = value; + it++; + continue; + } + if (key == "region") { + parsed.region = value; + } else if (key == "database" || key == "dbname") { + parsed.db_name = value; + } else if (key == "host") { + parsed.host = value; + } else if (key == "port") { + parsed.port = value; + } else if (key == "user") { + parsed.user = value; + } else { + it++; + continue; + } + it = options.options.erase(it); + } + return parsed; +} + +//! `ATTACH '' AS db (TYPE rds, SECRET )`. +//! +//! The DB instance identifier is all the user gives us, so we ask RDS for the rest: +//! DescribeDBInstances - the API behind `aws rds describe-db-instances --db-instance-identifier` - +//! supplies the endpoint host/port, the engine and the default database name, and an IAM auth +//! token signed with the AWS identity behind the secret serves as the database password. Those are +//! assembled into a libpq connection string and handed to the postgres extension's own attach. +//! +//! Only the Postgres-flavored engines can be attached this way; every other engine (MySQL, +//! MariaDB, Oracle, ...) is rejected up front. +unique_ptr RdsAttach(optional_ptr storage_info, ClientContext &context, + AttachedDatabase &db, const string &name, AttachInfo &info, AttachOptions &options) { + if (!Settings::Get(context)) { + throw PermissionException("Attaching RDS databases is disabled through configuration"); + } + + auto instance_id = info.path; + if (instance_id.empty()) { + throw BinderException("No RDS DB instance identifier given. Pass it as the ATTACH path, e.g. " + "ATTACH '' AS db (TYPE rds)"); + } + + auto attach_options = ParseAttachOptions(options); + auto secret_entry = FindAwsSecret(context, attach_options.secret_name); + const auto &secret = dynamic_cast(*secret_entry->secret); + + // An s3 secret's region is the bucket region, which need not be the instance's, so an explicit + // ATTACH region wins over it. Past those two, fall back to the sources CREATE SECRET uses. + auto explicit_region = attach_options.region.empty() ? GetSecretString(secret, "region") : attach_options.region; + auto region = ResolveAwsRegion(context, explicit_region, ""); + if (region.empty()) { + throw InvalidConfigurationException("No AWS region found for the RDS instance. Pass it to ATTACH, e.g. " + "ATTACH '' AS db (TYPE rds, REGION ''), set it on " + "the secret, or configure the AWS_REGION environment variable"); + } + + auto provider = CredentialsProviderFromSecret(secret, "RDS"); + auto instance = Rds::DescribeDBInstance(provider, instance_id, region); + + // RDS runs engines DuckDB cannot speak to; only the Postgres ones have an extension to hand off + // to. MySQL/MariaDB instances would need the mysql extension, which does not take a connection + // string in the shape we build below. + if (!RdsEngineIsPostgres(instance.engine)) { + throw NotImplementedException("RDS instance '%s' runs the '%s' engine, which cannot be attached. Only the " + "Postgres engines ('postgres', 'aurora-postgresql') are supported", + instance_id, instance.engine); + } + + // Resolved after the engine check, so an unsupported engine does not first demand postgres. + auto postgres_extension = RequirePostgresStorageExtension(context, "an RDS instance"); + + // Anything ATTACH pins explicitly wins over what the instance reports - e.g. DATABASE for an + // instance that has no initial database. + auto host = attach_options.host.empty() ? instance.endpoint_address : attach_options.host; + auto db_name = attach_options.db_name.empty() ? instance.db_name : attach_options.db_name; + + auto port = instance.endpoint_port; + if (!attach_options.port.empty()) { + try { + port = std::stoi(attach_options.port); + } catch (const std::exception &) { + throw InvalidInputException("'PORT' must be an integer, got '%s'", attach_options.port); + } + } + + // Unlike Redshift, RDS does not mint the user along with the password: the IAM token is issued + // *for* a database user that must already exist. The instance's master user is the only one we + // can guess at, so anything else has to be named explicitly. + auto user = attach_options.user.empty() ? instance.master_username : attach_options.user; + if (user.empty()) { + throw InvalidConfigurationException("No database user to connect to RDS instance '%s' as, and the instance " + "reports no master username. Pass one to ATTACH, e.g. ATTACH '%s' AS db " + "(TYPE rds, USER '')", + instance_id, instance_id); + } + + // The token is only accepted when the instance has IAM database authentication turned on, and + // the failure it produces otherwise is an opaque password-authentication error. + if (!instance.iam_auth_enabled) { + throw InvalidConfigurationException( + "RDS instance '%s' does not have IAM database authentication enabled, so its AWS identity cannot be used " + "to log in. Enable it on the instance (`aws rds modify-db-instance --db-instance-identifier %s " + "--enable-iam-database-authentication`) and grant the 'rds_iam' role to database user '%s'", + instance_id, instance_id, user); + } + + // The token is signed for a specific host/port/user triple, so it has to be minted from the + // endpoint we are about to connect to rather than the one the instance reported. + auto token = Rds::GenerateAuthToken(provider, host, port, region, user); + + // RDS rejects an IAM login over an unencrypted connection. + string connection_string = + "host=" + EscapeConnectionValue(host) + " port=" + EscapeConnectionValue(to_string(port)) + + " user=" + EscapeConnectionValue(user) + " password=" + EscapeConnectionValue(token) + " sslmode='require'"; + if (!db_name.empty()) { + connection_string += " dbname=" + EscapeConnectionValue(db_name); + } + + // Hand the postgres extension a plain connection string as the attach path. + info.path = connection_string; + + // Postgres must be given a secret name it can resolve: with none it falls back to the implicit + // '__default_postgres' secret, which it probes in the 'local_file' storage - and that throws + // outright when persistent secrets are disabled. Naming the aws/s3 secret we just used is safe, + // because postgres only harvests libpq option names (host, port, user, ...) from a secret and + // an aws/s3 secret holds none of them. + options.options["secret"] = Value(secret.GetName().GetIdentifierName()); + + return postgres_extension->attach(postgres_extension->storage_info.get(), context, db, name, info, options); +} + +unique_ptr RdsCreateTransactionManager(optional_ptr storage_info, + AttachedDatabase &db, Catalog &catalog) { + // RdsAttach returned a PostgresCatalog, so the transaction manager has to come from the same + // place. Attach has already established that the postgres extension is loaded. + auto &db_config = DBConfig::GetConfig(db.GetDatabase()); + auto postgres_extension = FindPostgresStorageExtension(db_config); + if (!postgres_extension || !postgres_extension->create_transaction_manager) { + throw InternalException("RDS attach: the postgres storage extension disappeared after attaching"); + } + return postgres_extension->create_transaction_manager(postgres_extension->storage_info.get(), db, catalog); +} + +class RdsStorageExtension : public StorageExtension { +public: + RdsStorageExtension() { + attach = RdsAttach; + create_transaction_manager = RdsCreateTransactionManager; + } +}; + +} // namespace + +void Rds::RegisterStorageExtension(ExtensionLoader &loader) { + auto &config = DBConfig::GetConfig(loader.GetDatabaseInstance()); + StorageExtension::Register(config, "rds", make_shared_ptr()); +} + +} // namespace duckdb diff --git a/src/rds/rds_utils.cpp b/src/rds/rds_utils.cpp new file mode 100644 index 0000000..f5b49f2 --- /dev/null +++ b/src/rds/rds_utils.cpp @@ -0,0 +1,85 @@ +#include "rds/rds_utils.hpp" + +#include "aws_client.hpp" + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/main/extension/extension_loader.hpp" + +#include +#include + +namespace duckdb { + +namespace { + +Aws::RDS::RDSClient MakeClient(const std::shared_ptr &provider, + const string ®ion) { + Aws::Client::ClientConfiguration config = BuildClientConfigWithCa(); + config.region = region; + return Aws::RDS::RDSClient(provider, config); +} + +} // namespace + +bool RdsEngineIsPostgres(const string &engine) { + auto lower = StringUtil::Lower(engine); + return lower == "postgres" || lower == "aurora-postgresql"; +} + +RdsInstanceInfo Rds::DescribeDBInstance(const std::shared_ptr &provider, + const string &instance_id, const string ®ion) { + auto rds_client = MakeClient(provider, region); + + Aws::RDS::Model::DescribeDBInstancesRequest request; + request.SetDBInstanceIdentifier(instance_id.c_str()); + + auto outcome = rds_client.DescribeDBInstances(request); + if (!outcome.IsSuccess()) { + throw InvalidConfigurationException("DescribeDBInstances failed for RDS instance '%s' in region '%s': %s", + instance_id, region, string(outcome.GetError().GetMessage().c_str())); + } + + // The identifier is unique within an account+region, so a successful lookup by identifier + // returns exactly one instance. + auto &instances = outcome.GetResult().GetDBInstances(); + if (instances.empty()) { + throw InvalidConfigurationException("No RDS instance named '%s' found in region '%s'", instance_id, region); + } + auto &instance = instances.front(); + + RdsInstanceInfo info; + info.endpoint_address = string(instance.GetEndpoint().GetAddress().c_str()); + info.endpoint_port = instance.GetEndpoint().GetPort(); + info.engine = string(instance.GetEngine().c_str()); + info.engine_version = string(instance.GetEngineVersion().c_str()); + info.db_name = string(instance.GetDBName().c_str()); + info.master_username = string(instance.GetMasterUsername().c_str()); + info.status = string(instance.GetDBInstanceStatus().c_str()); + info.iam_auth_enabled = instance.GetIAMDatabaseAuthenticationEnabled(); + + // An instance only has an endpoint once it is available; a stopped/creating instance describes + // fine but cannot be connected to. Report the status rather than letting the connect fail + // against an empty host. + if (info.endpoint_address.empty()) { + throw InvalidConfigurationException("RDS instance '%s' has no endpoint to connect to (instance status: '%s')", + instance_id, info.status); + } + return info; +} + +string Rds::GenerateAuthToken(const std::shared_ptr &provider, const string &host, + int32_t port, const string ®ion, const string &db_user) { + auto rds_client = MakeClient(provider, region); + auto token = + rds_client.GenerateConnectAuthToken(host.c_str(), region.c_str(), static_cast(port), db_user.c_str()); + if (token.empty()) { + throw InvalidConfigurationException( + "Could not generate an RDS IAM authentication token for user '%s' on '%s'. The AWS credentials in the " + "secret could not be resolved", + db_user, host); + } + return string(token.c_str()); +} + +} // namespace duckdb diff --git a/src/redshift/redshift_storage.cpp b/src/redshift/redshift_storage.cpp index dcc854a..c4c021f 100644 --- a/src/redshift/redshift_storage.cpp +++ b/src/redshift/redshift_storage.cpp @@ -30,28 +30,6 @@ struct RedshiftAttachOptions { int duration_seconds = 0; }; -string GetSecretString(const KeyValueSecret &secret, const string &key) { - auto value = secret.TryGetValue(Identifier(key)); - if (value.IsNull()) { - return ""; - } - return value.ToString(); -} - -//! Quote a value for a libpq connection string. Backslashes and single quotes must be escaped -//! with a backslash; quoting the whole value keeps empty values and embedded spaces valid. -string EscapeConnectionValue(const string &value) { - string result = "'"; - for (auto c : value) { - if (c == '\\' || c == '\'') { - result += '\\'; - } - result += c; - } - result += "'"; - return result; -} - RedshiftAttachOptions ParseAttachOptions(AttachOptions &options) { RedshiftAttachOptions parsed; // Consumed options are erased, since postgres throws on any option it does not recognize. @@ -88,19 +66,6 @@ RedshiftAttachOptions ParseAttachOptions(AttachOptions &options) { return parsed; } -std::shared_ptr BuildProvider(const KeyValueSecret &secret) { - auto key_id = GetSecretString(secret, "key_id"); - auto secret_key = GetSecretString(secret, "secret"); - auto session_token = GetSecretString(secret, "session_token"); - if (key_id.empty() || secret_key.empty()) { - throw InvalidConfigurationException( - "Secret \"%s\" holds no AWS credentials (no 'key_id'/'secret'), so it cannot be used to reach Redshift", - secret.GetName().GetIdentifierName()); - } - return Aws::MakeShared("DuckDBRedshift", key_id.c_str(), - secret_key.c_str(), session_token.c_str()); -} - //! `ATTACH '' AS db (TYPE redshift, SECRET )`. //! //! The cluster identifier is all the user gives us, so we ask Redshift for the rest: @@ -137,18 +102,9 @@ unique_ptr RedshiftAttach(optional_ptr storage_in } // Resolve the postgres extension before spending API calls on a connection we cannot open. - auto &db_config = DBConfig::GetConfig(context); - auto postgres_extension = FindPostgresStorageExtension(db_config); - if (!postgres_extension) { - Catalog::TryAutoLoad(context, POSTGRES_EXTENSION_NAME); - postgres_extension = FindPostgresStorageExtension(db_config); - } - if (!postgres_extension || !postgres_extension->attach) { - throw InvalidConfigurationException("Attaching a Redshift cluster requires the postgres extension, which could " - "not be loaded. Run 'INSTALL postgres; LOAD postgres;' and retry"); - } + auto postgres_extension = RequirePostgresStorageExtension(context, "a Redshift cluster"); - auto provider = BuildProvider(secret); + auto provider = CredentialsProviderFromSecret(secret, "Redshift"); // Anything ATTACH pins explicitly wins over what the cluster reports, so when it pins all of // them there is nothing left to discover - skip the call rather than require the caller to diff --git a/src/utils/utils.cpp b/src/utils/utils.cpp index b8abe60..12655a0 100644 --- a/src/utils/utils.cpp +++ b/src/utils/utils.cpp @@ -2,6 +2,7 @@ #include "aws_client.hpp" +#include "duckdb/catalog/catalog.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/main/client_context.hpp" #include "duckdb/main/database.hpp" @@ -105,6 +106,40 @@ unique_ptr FindAwsSecret(ClientContext &context, const string &secr "CREATE SECRET (TYPE aws, PROVIDER credential_chain, REGION '')"); } +string GetSecretString(const KeyValueSecret &secret, const string &key) { + auto value = secret.TryGetValue(Identifier(key)); + if (value.IsNull()) { + return ""; + } + return value.ToString(); +} + +std::shared_ptr CredentialsProviderFromSecret(const KeyValueSecret &secret, + const string &service) { + auto key_id = GetSecretString(secret, "key_id"); + auto secret_key = GetSecretString(secret, "secret"); + auto session_token = GetSecretString(secret, "session_token"); + if (key_id.empty() || secret_key.empty()) { + throw InvalidConfigurationException( + "Secret \"%s\" holds no AWS credentials (no 'key_id'/'secret'), so it cannot be used to reach %s", + secret.GetName().GetIdentifierName(), service); + } + return Aws::MakeShared(service.c_str(), key_id.c_str(), secret_key.c_str(), + session_token.c_str()); +} + +string EscapeConnectionValue(const string &value) { + string result = "'"; + for (auto c : value) { + if (c == '\\' || c == '\'') { + result += '\\'; + } + result += c; + } + result += "'"; + return result; +} + const char *const POSTGRES_EXTENSION_NAME = "postgres"; optional_ptr FindPostgresStorageExtension(const DBConfig &config) { @@ -112,4 +147,19 @@ optional_ptr FindPostgresStorageExtension(const DBConfig &conf return StorageExtension::Find(config, ExtensionHelper::ApplyExtensionAlias(POSTGRES_EXTENSION_NAME)); } +optional_ptr RequirePostgresStorageExtension(ClientContext &context, const string &attach_type) { + auto &db_config = DBConfig::GetConfig(context); + auto postgres_extension = FindPostgresStorageExtension(db_config); + if (!postgres_extension) { + Catalog::TryAutoLoad(context, POSTGRES_EXTENSION_NAME); + postgres_extension = FindPostgresStorageExtension(db_config); + } + if (!postgres_extension || !postgres_extension->attach) { + throw InvalidConfigurationException("Attaching %s requires the postgres extension, which could not be loaded. " + "Run 'INSTALL postgres; LOAD postgres;' and retry", + attach_type); + } + return postgres_extension; +} + } // namespace duckdb From 645f8a51c614745054c42b175c4bff4009f0107e Mon Sep 17 00:00:00 2001 From: Tmonster Date: Tue, 14 Jul 2026 17:36:05 +0200 Subject: [PATCH 2/5] connect to rds storage --- src/rds/rds_storage.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/rds/rds_storage.cpp b/src/rds/rds_storage.cpp index 5b79785..1aca9c1 100644 --- a/src/rds/rds_storage.cpp +++ b/src/rds/rds_storage.cpp @@ -13,6 +13,7 @@ #include "duckdb/main/settings.hpp" #include "duckdb/parser/parsed_data/attach_info.hpp" #include "duckdb/storage/storage_extension.hpp" +#include "duckdb/transaction/transaction_manager.hpp" namespace duckdb { @@ -112,10 +113,15 @@ unique_ptr RdsAttach(optional_ptr storage_info, C // Resolved after the engine check, so an unsupported engine does not first demand postgres. auto postgres_extension = RequirePostgresStorageExtension(context, "an RDS instance"); - // Anything ATTACH pins explicitly wins over what the instance reports - e.g. DATABASE for an - // instance that has no initial database. + // Anything ATTACH pins explicitly wins over what the instance reports. auto host = attach_options.host.empty() ? instance.endpoint_address : attach_options.host; auto db_name = attach_options.db_name.empty() ? instance.db_name : attach_options.db_name; + if (db_name.empty()) { + // An instance created without an initial database reports none, and libpq then defaults + // dbname to the *user* name, which is never a database that exists here. Every Postgres + // engine provisions 'postgres', and the engine check above has established this is one. + db_name = "postgres"; + } auto port = instance.endpoint_port; if (!attach_options.port.empty()) { @@ -152,12 +158,10 @@ unique_ptr RdsAttach(optional_ptr storage_info, C auto token = Rds::GenerateAuthToken(provider, host, port, region, user); // RDS rejects an IAM login over an unencrypted connection. - string connection_string = - "host=" + EscapeConnectionValue(host) + " port=" + EscapeConnectionValue(to_string(port)) + - " user=" + EscapeConnectionValue(user) + " password=" + EscapeConnectionValue(token) + " sslmode='require'"; - if (!db_name.empty()) { - connection_string += " dbname=" + EscapeConnectionValue(db_name); - } + string connection_string = "host=" + EscapeConnectionValue(host) + + " port=" + EscapeConnectionValue(to_string(port)) + + " user=" + EscapeConnectionValue(user) + " password=" + EscapeConnectionValue(token) + + " sslmode='require'" + " dbname=" + EscapeConnectionValue(db_name); // Hand the postgres extension a plain connection string as the attach path. info.path = connection_string; From 920eebb0ec1c6343ad5d0efb9dbae6af1a6a2d6e Mon Sep 17 00:00:00 2001 From: Tmonster Date: Tue, 14 Jul 2026 23:43:28 +0200 Subject: [PATCH 3/5] fix error message if authentication does not work --- src/include/utils/utils.hpp | 7 +++++++ src/rds/rds_storage.cpp | 18 +++++++++++++++++- src/redshift/redshift_storage.cpp | 7 ++++++- src/utils/utils.cpp | 16 ++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/include/utils/utils.hpp b/src/include/utils/utils.hpp index d325743..8951790 100644 --- a/src/include/utils/utils.hpp +++ b/src/include/utils/utils.hpp @@ -57,4 +57,11 @@ optional_ptr FindPostgresStorageExtension(const DBConfig &conf //! `attach_type` only names the caller in the error thrown when it cannot be loaded. optional_ptr RequirePostgresStorageExtension(ClientContext &context, const string &attach_type); +//! The message to report for a failed postgres attach. Postgres quotes the entire connection +//! string back, which buries the server's own diagnostic and puts `password` - for Redshift and +//! RDS a live, AWS-generated credential - in the error and in whatever log it lands in. Returns +//! the server's message alone when the connection got far enough to produce one, and otherwise +//! the full message with the password redacted. +string PostgresAttachErrorMessage(std::exception &ex, const string &password); + } // namespace duckdb diff --git a/src/rds/rds_storage.cpp b/src/rds/rds_storage.cpp index 1aca9c1..6f87e8b 100644 --- a/src/rds/rds_storage.cpp +++ b/src/rds/rds_storage.cpp @@ -173,7 +173,23 @@ unique_ptr RdsAttach(optional_ptr storage_info, C // an aws/s3 secret holds none of them. options.options["secret"] = Value(secret.GetName().GetIdentifierName()); - return postgres_extension->attach(postgres_extension->storage_info.get(), context, db, name, info, options); + try { + return postgres_extension->attach(postgres_extension->storage_info.get(), context, db, name, info, options); + } catch (std::exception &ex) { + auto message = PostgresAttachErrorMessage(ex, token); + auto error_message = StringUtil::Format("Unable to connect to RDS instance '%s': %s", instance_id, message); + if (StringUtil::Contains(message, "password authentication failed")) { + // RDS hands the token to Postgres as an ordinary password, so a database user that has + // not been switched over to IAM authentication rejects it as one - and says nothing + // about IAM, because as far as it knows a password simply did not match. + error_message += StringUtil::Format("\nCheck that user '%s' exists on the instance and has rds_iam " + "privileges (`GRANT rds_iam TO \"%s\";`, run as the master user)." + "\nThe AWS identity behind the secret must also be allowed " + "'rds-db:connect' privileges as that user.", + user, user); + } + throw IOException(error_message); + } } unique_ptr RdsCreateTransactionManager(optional_ptr storage_info, diff --git a/src/redshift/redshift_storage.cpp b/src/redshift/redshift_storage.cpp index c4c021f..2cab5ec 100644 --- a/src/redshift/redshift_storage.cpp +++ b/src/redshift/redshift_storage.cpp @@ -140,7 +140,12 @@ unique_ptr RedshiftAttach(optional_ptr storage_in // ...) from a secret and an aws/s3 secret holds none of them. options.options["secret"] = Value(secret.GetName().GetIdentifierName()); - return postgres_extension->attach(postgres_extension->storage_info.get(), context, db, name, info, options); + try { + return postgres_extension->attach(postgres_extension->storage_info.get(), context, db, name, info, options); + } catch (std::exception &ex) { + auto message = PostgresAttachErrorMessage(ex, credentials.db_password); + throw IOException("Unable to connect to Redshift cluster '%s': %s", cluster_id, message); + } } unique_ptr RedshiftCreateTransactionManager(optional_ptr storage_info, diff --git a/src/utils/utils.cpp b/src/utils/utils.cpp index 12655a0..aaa7314 100644 --- a/src/utils/utils.cpp +++ b/src/utils/utils.cpp @@ -3,7 +3,9 @@ #include "aws_client.hpp" #include "duckdb/catalog/catalog.hpp" +#include "duckdb/common/error_data.hpp" #include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" #include "duckdb/main/client_context.hpp" #include "duckdb/main/database.hpp" #include "duckdb/main/extension_helper.hpp" @@ -162,4 +164,18 @@ optional_ptr RequirePostgresStorageExtension(ClientContext &co return postgres_extension; } +string PostgresAttachErrorMessage(std::exception &ex, const string &password) { + ErrorData error(ex); + auto message = StringUtil::Replace(error.RawMessage(), password, "..."); + // Everything the server says is prefixed with its severity, and everything before that is the + // connection string we passed in - including the password, which is why cutting here is enough + // on its own. A connection that never reached the server has no such message, and there the + // redaction above is what removes the password. + auto server_message = message.find("FATAL:"); + if (server_message != string::npos) { + return message.substr(server_message); + } + return message; +} + } // namespace duckdb From 0981fb9d15e9994be0a4a4c4505e847b75da2d8e Mon Sep 17 00:00:00 2001 From: Tmonster Date: Wed, 15 Jul 2026 13:11:30 +0200 Subject: [PATCH 4/5] fixed test file, do not compile postgres automatically --- extension_config.cmake | 14 ++++---- test/sql/rds/aws_rds_connect.test | 58 +++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 test/sql/rds/aws_rds_connect.test diff --git a/extension_config.cmake b/extension_config.cmake index 10df73d..221884f 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -10,10 +10,10 @@ duckdb_extension_load(aws # Currently Disabled since this will not build on CI. To get CI working you (most likely) # need to copy the vcpkg_ports/libpq in the duckdb/duckdb-postgres repo. # For now commented out so unblock development -duckdb_extension_load(postgres_scanner - DONT_LINK - GIT_URL https://github.com/duckdb/duckdb-postgres - GIT_TAG 74b8d3f71365b351073e9853bfcd70a612bc1488 - SUBMODULES database-connector - APPLY_PATCHES -) +#duckdb_extension_load(postgres_scanner +# DONT_LINK +# GIT_URL https://github.com/duckdb/duckdb-postgres +# GIT_TAG 74b8d3f71365b351073e9853bfcd70a612bc1488 +# SUBMODULES database-connector +# APPLY_PATCHES +#) diff --git a/test/sql/rds/aws_rds_connect.test b/test/sql/rds/aws_rds_connect.test new file mode 100644 index 0000000..feef584 --- /dev/null +++ b/test/sql/rds/aws_rds_connect.test @@ -0,0 +1,58 @@ +# name: test/sql/rds/aws_rds_connect.test +# description: test attaching an RDS instance discovered via DescribeDBInstances + IAM auth +# group: [rds] + +require aws + +require parquet + +# The 'aws' secret type is registered by httpfs. +require httpfs + +require postgres_scanner + +require-env RDS_AVAILABLE 1 + +statement ok +SET allow_persistent_secrets=false + +statement ok +CREATE SECRET rds_creds ( + TYPE aws, + PROVIDER credential_chain, + PROFILE 'personal', + CHAIN 'config', + REGION 'eu-central-1', +); + +statement ok +ATTACH 'attach-connect-test' AS rds_db (TYPE RDS); + +statement ok +select * from (show all tables) where database = 'rds_db'; + +# The dev database should expose exactly 7 tables. +query I +SELECT count(*) FROM (SHOW ALL TABLES) WHERE database = 'rds_db'; +---- +1 + +query IIIIIIII +select * from rds_db.public.dummy_users limit 10; +---- +1 user_1 user_1@example.com BR 21 615.13 false 2026-03-13 06:53:58.607726+00 +2 user_2 user_2@example.com BR 20 1891.62 false 2026-03-17 19:12:22.919463+00 +3 user_3 user_3@example.com BR 41 5543.64 true 2026-05-31 20:39:14.85907+00 +4 user_4 user_4@example.com US 20 9981.93 true 2025-09-13 06:59:40.159487+00 +5 user_5 user_5@example.com BR 23 1413.76 true 2026-03-23 19:24:28.210691+00 +6 user_6 user_6@example.com NL 60 6127.10 true 2025-12-13 21:29:06.845312+00 +7 user_7 user_7@example.com BR 18 6721.71 true 2025-07-23 08:29:01.694829+00 +8 user_8 user_8@example.com DE 55 7432.12 true 2025-07-15 11:48:55.564206+00 +9 user_9 user_9@example.com FR 26 1473.78 true 2026-04-26 21:59:39.039863+00 +10 user_10 user_10@example.com BR 38 395.07 false 2026-02-28 21:43:16.32081+00 + +statement ok +DETACH rds_db + +statement ok +DROP SECRET rds_creds From d37f8f05db92a8b33a634746a97b71c6a6d0e046 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 16 Jul 2026 12:03:03 +0200 Subject: [PATCH 5/5] bump postgres_scanner hash --- extension_config.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extension_config.cmake b/extension_config.cmake index 221884f..8ff60ba 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -13,7 +13,6 @@ duckdb_extension_load(aws #duckdb_extension_load(postgres_scanner # DONT_LINK # GIT_URL https://github.com/duckdb/duckdb-postgres -# GIT_TAG 74b8d3f71365b351073e9853bfcd70a612bc1488 +# GIT_TAG a4e03aad76a002e913e676cce1fc2600f64a614f # SUBMODULES database-connector -# APPLY_PATCHES #)