diff --git a/CMakeLists.txt b/CMakeLists.txt index c3cd7cb25..c2eb7d613 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,6 +87,7 @@ set(EXTENSION_SOURCES src/function/iceberg_scalar_functions.cpp src/function/metadata/iceberg_column_stats.cpp src/function/metadata/iceberg_load_table_response.cpp + src/function/metadata/iceberg_view_metadata.cpp src/function/metadata/iceberg_metadata.cpp src/function/metadata/iceberg_partition_stats.cpp src/function/metadata/iceberg_snapshots.cpp diff --git a/src/catalog/rest/api/catalog_api.cpp b/src/catalog/rest/api/catalog_api.cpp index 56e1fed57..43903fa81 100644 --- a/src/catalog/rest/api/catalog_api.cpp +++ b/src/catalog/rest/api/catalog_api.cpp @@ -561,6 +561,125 @@ rest_api_objects::LoadTableResult IRCAPI::CommitNewTable(ClientContext &context, } } +// ─── View operations ───────────────────────────────────────────────────────── + +vector IRCAPI::GetViews(ClientContext &context, IcebergCatalog &catalog, + const IcebergSchemaEntry &schema) { + vector all_identifiers; + string page_token; + + do { + auto url_builder = catalog.GetBaseUrl(); + url_builder.AddPrefixComponent(catalog.prefix, catalog.prefix_is_one_component); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("namespaces")); + url_builder.AddPathComponent(IRCPathComponent::NamespaceComponent(schema.namespace_items)); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("views")); + if (!page_token.empty()) { + url_builder.SetParam("pageToken", IRCPathComponent::RegularComponent(page_token)); + } + + HTTPHeaders headers(*context.db); + auto response = catalog.auth_handler->Request(RequestType::GET_REQUEST, context, url_builder, headers); + if (!response->Success()) { + if (response->status == HTTPStatusCode::Forbidden_403 || + response->status == HTTPStatusCode::Unauthorized_401 || + response->status == HTTPStatusCode::NotFound_404) { + DUCKDB_LOG_WARNING(context, "GET %s returned status code %s", url_builder.GetURLEncoded(), + EnumUtil::ToString(response->status)); + return all_identifiers; + } + auto url = url_builder.GetURLEncoded(); + ThrowException(url, *response, "GET"); + } + + auto doc = ICUtils::APIResultToDoc(response->body); + auto *root = yyjson_doc_get_root(doc.get()); + auto list_response = rest_api_objects::ListTablesResponse::FromJSON(root); + + if (list_response.has_identifiers) { + all_identifiers.insert(all_identifiers.end(), std::make_move_iterator(list_response.identifiers.begin()), + std::make_move_iterator(list_response.identifiers.end())); + } + + if (list_response.has_next_page_token) { + page_token = list_response.next_page_token.value; + } else { + page_token.clear(); + } + } while (!page_token.empty()); + + return all_identifiers; +} + +APIResult> IRCAPI::GetView(ClientContext &context, + IcebergCatalog &catalog, + const IcebergSchemaEntry &schema, + const string &view_name) { + auto ret = APIResult>(); + + auto url_builder = catalog.GetBaseUrl(); + url_builder.AddPrefixComponent(catalog.prefix, catalog.prefix_is_one_component); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("namespaces")); + url_builder.AddPathComponent(IRCPathComponent::NamespaceComponent(schema.namespace_items)); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("views")); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent(view_name)); + + HTTPHeaders headers(*context.db); + auto response = catalog.auth_handler->Request(RequestType::GET_REQUEST, context, url_builder, headers); + if (response->status != HTTPStatusCode::OK_200) { + std::unique_ptr out_doc; + yyjson_val *error_obj = ICUtils::GetErrorMessage(response->body, out_doc); + if (error_obj == nullptr) { + throw InvalidConfigurationException(response->body); + } + ret.has_error = true; + ret.status_ = response->status; + ret.error_ = rest_api_objects::IcebergErrorResponse::FromJSON(error_obj); + return ret; + } + ret.has_error = false; + auto doc = ICUtils::APIResultToDoc(response->body); + auto *root = yyjson_doc_get_root(doc.get()); + ret.result_ = make_uniq(rest_api_objects::LoadViewResult::FromJSON(root)); + return ret; +} + +void IRCAPI::CommitNewView(ClientContext &context, IcebergCatalog &catalog, const IcebergSchemaEntry &schema, + const string &json_body) { + auto url_builder = catalog.GetBaseUrl(); + url_builder.AddPrefixComponent(catalog.prefix, catalog.prefix_is_one_component); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("namespaces")); + url_builder.AddPathComponent(IRCPathComponent::NamespaceComponent(schema.namespace_items)); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("views")); + + HTTPHeaders headers(*context.db); + headers.Insert("Content-Type", "application/json"); + auto response = catalog.auth_handler->Request(RequestType::POST_REQUEST, context, url_builder, headers, json_body); + if (response->status != HTTPStatusCode::OK_200) { + throw InvalidConfigurationException( + "Request to '%s' returned a non-200 status code (%s), with reason: %s, body: %s", + url_builder.GetURLEncoded(), EnumUtil::ToString(response->status), response->reason, response->body); + } +} + +void IRCAPI::CommitViewDelete(ClientContext &context, IcebergCatalog &catalog, const vector &schema, + const string &view_name) { + auto url_builder = catalog.GetBaseUrl(); + url_builder.AddPrefixComponent(catalog.prefix, catalog.prefix_is_one_component); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("namespaces")); + url_builder.AddPathComponent(IRCPathComponent::NamespaceComponent(schema)); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("views")); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent(view_name)); + + HTTPHeaders headers(*context.db); + auto response = catalog.auth_handler->Request(RequestType::DELETE_REQUEST, context, url_builder, headers); + if (response->status != HTTPStatusCode::NoContent_204 && response->status != HTTPStatusCode::OK_200) { + throw InvalidConfigurationException( + "Request to '%s' returned a non-200 status code (%s), with reason: %s, body: %s", + url_builder.GetURLEncoded(), EnumUtil::ToString(response->status), response->reason, response->body); + } +} + rest_api_objects::CatalogConfig IRCAPI::GetCatalogConfig(ClientContext &context, IcebergCatalog &catalog, const string &warehouse) { auto url_builder = catalog.GetBaseUrl(); diff --git a/src/catalog/rest/catalog_entry/schema/iceberg_schema_entry.cpp b/src/catalog/rest/catalog_entry/schema/iceberg_schema_entry.cpp index 5f9194d91..907377deb 100644 --- a/src/catalog/rest/catalog_entry/schema/iceberg_schema_entry.cpp +++ b/src/catalog/rest/catalog_entry/schema/iceberg_schema_entry.cpp @@ -13,6 +13,9 @@ #include "duckdb/planner/expression_binder/table_function_binder.hpp" #include "duckdb/execution/expression_executor.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" + #include "catalog/rest/catalog_entry/table/iceberg_table_information.hpp" #include "catalog/rest/iceberg_catalog.hpp" #include "catalog/rest/transaction/iceberg_transaction.hpp" @@ -106,30 +109,79 @@ void IcebergSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { } void IcebergSchemaEntry::DropEntry(ClientContext &context, DropInfo &info, bool delete_entry) { - auto table_name = info.name; - // find if info has a table name, if so look for it in - auto table_info_it = tables.GetEntries().find(table_name); - if (table_info_it == tables.GetEntries().end()) { - if (info.if_not_found == OnEntryNotFound::RETURN_NULL) { + auto entry_name = info.name; + + // CASCADE is not part of the Iceberg REST spec — reject before any type-specific handling. + if (info.cascade) { + switch (info.type) { + case CatalogType::VIEW_ENTRY: + throw NotImplementedException("DROP VIEW CASCADE is not supported for Iceberg views currently"); + case CatalogType::TABLE_ENTRY: + throw NotImplementedException( + "DROP TABLE CASCADE is not supported for Iceberg tables currently"); + default: + throw NotImplementedException("DROP %s CASCADE is not supported for Iceberg currently", + CatalogTypeToString(info.type)); + } + } + + switch (info.type) { + case CatalogType::VIEW_ENTRY: { + auto &transaction = IcebergTransaction::Get(context, catalog).Cast(); + auto view_key = IcebergTableInformation::GetTableKey(namespace_items, entry_name); + + // Check if view was created in this transaction — just remove from created_views + if (transaction.created_views.erase(view_key) > 0) { + tables.InvalidateViewCache(entry_name); return; } - throw CatalogException("Table %s does not exist", table_name); + + // Load view entries if not yet loaded, then check if view exists + tables.LoadViewEntries(context); + auto view_it = tables.GetViewEntries().find(entry_name); + if (view_it == tables.GetViewEntries().end()) { + if (info.if_not_found == OnEntryNotFound::RETURN_NULL) { + return; + } + throw CatalogException("View %s does not exist", entry_name); + } + + if (delete_entry) { + tables.InvalidateViewCache(entry_name); + } else { + IcebergTransaction::DeletedViewInfo info; + info.namespace_items = namespace_items; + info.schema_name = name; + info.view_name = entry_name; + transaction.deleted_views.emplace(view_key, std::move(info)); + } + return; } - if (info.cascade) { - throw NotImplementedException("DROP TABLE CASCADE is not supported for Iceberg tables currently"); + case CatalogType::TABLE_ENTRY: { + auto table_info_it = tables.GetEntries().find(entry_name); + if (table_info_it == tables.GetEntries().end()) { + if (info.if_not_found == OnEntryNotFound::RETURN_NULL) { + return; + } + throw CatalogException("Table %s does not exist", entry_name); + } + if (delete_entry) { + // Remove the entry from the catalog + tables.GetEntriesMutable().erase(entry_name); + } else { + // Add the table to the transaction's deleted_tables + auto &transaction = IcebergTransaction::Get(context, catalog).Cast(); + auto &table_info = table_info_it->second; + auto &table = transaction.DeleteTable(*table_info); + //! FIXME: what? + // must init schema versions after copy. Schema versions have a pointer to IcebergTableInformation + // if the IcebergTableInformation is moved, then the pointer is no longer valid. + table.InitSchemaVersions(); + } + return; } - if (delete_entry) { - // Remove the entry from the catalog - tables.GetEntriesMutable().erase(table_name); - } else { - // Add the table to the transaction's deleted_tables - auto &transaction = IcebergTransaction::Get(context, catalog).Cast(); - auto &table_info = table_info_it->second; - auto &table = transaction.DeleteTable(*table_info); - //! FIXME: what? - // must init schema versions after copy. Schema versions have a pointer to IcebergTableInformation - // if the IcebergTableInformation is moved, then the pointer is no longer valid. - table.InitSchemaVersions(); + default: + throw NotImplementedException("DropEntry not implemented for CatalogType '%s'", CatalogTypeToString(info.type)); } } @@ -153,12 +205,57 @@ optional_ptr IcebergSchemaEntry::CreateIndex(CatalogTransaction tr throw NotImplementedException("Create Index"); } -string GetUCCreateView(CreateViewInfo &info) { - throw NotImplementedException("Get Create View"); -} - optional_ptr IcebergSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { - throw NotImplementedException("Create View"); + if (info.sql.empty() && !info.query) { + throw BinderException("Cannot create view in Iceberg without a query"); + } + auto &context = transaction.GetContext(); + + if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { + throw NotImplementedException( + "CREATE OR REPLACE not supported in DuckDB-Iceberg. Please use separate Drop and Create Statements"); + } + + auto existing_entry = GetEntry(transaction, CatalogType::VIEW_ENTRY, info.view_name); + if (existing_entry) { + if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { + // CREATE VIEW IF NOT EXISTS — view already exists, nothing to do. + return existing_entry; + } + // ERROR_ON_CONFLICT + throw CatalogException("View with name \"%s\" already exists", info.view_name); + } + + // Generate default column names if the caller gave us types but no names. + if (info.names.empty() && !info.types.empty()) { + for (idx_t i = 0; i < info.types.size(); i++) { + if (i < info.aliases.size() && !info.aliases[i].empty()) { + info.names.push_back(info.aliases[i]); + } else { + info.names.push_back("col" + to_string(i)); + } + } + } + + // Track the view in the transaction + auto &iceberg_transaction = GetICTransaction(transaction); + auto view_key = IcebergTableInformation::GetTableKey(namespace_items, info.view_name); + + auto view_info = unique_ptr_cast(info.Copy()); + // Preserve the SELECT SQL — ViewCatalogEntry::Initialize() will move the query out, + // so we need the SQL string available at commit time for the REST API request. + if (view_info->query) { + view_info->sql = view_info->query->ToString(); + } + + iceberg_transaction.created_views.erase(view_key); + iceberg_transaction.created_views.emplace(view_key, std::move(view_info)); + + // Invalidate stale caches + tables.InvalidateViewCache(info.view_name); + + // Return a pointer to an owned entry (avoid dangling pointer) + return tables.GetViewEntry(transaction.GetContext(), info.view_name); } optional_ptr IcebergSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { @@ -735,6 +832,10 @@ void IcebergSchemaEntry::Scan(ClientContext &context, CatalogType type, if (!CatalogTypeIsSupported(type)) { return; } + if (type == CatalogType::VIEW_ENTRY) { + GetCatalogSet(type).ScanViews(context, callback); + return; + } GetCatalogSet(type).Scan(context, callback); } void IcebergSchemaEntry::Scan(CatalogType type, const std::function &callback) { @@ -749,13 +850,26 @@ optional_ptr IcebergSchemaEntry::LookupEntry(CatalogTransaction tr } auto &context = transaction.GetContext(); auto &ic_catalog = catalog.Cast(); + + // For VIEW_ENTRY, try the view path + if (type == CatalogType::VIEW_ENTRY) { + auto view_entry = GetCatalogSet(type).GetViewEntry(context, lookup_info.GetEntryName()); + if (view_entry) { + return view_entry; + } + return nullptr; + } + + // For TABLE_ENTRY, use the existing table lookup auto table_entry = GetCatalogSet(type).GetEntry(context, lookup_info); if (!table_entry) { + // Try looking up as a view — DuckDB sometimes looks up views as TABLE_ENTRY + auto view_entry = GetCatalogSet(type).GetViewEntry(context, lookup_info.GetEntryName()); + if (view_entry) { + return view_entry; + } // verify the schema exists if (!IRCAPI::VerifySchemaExistence(context, ic_catalog, name)) { - // set exists to false here - // we would like to throw an error, but this code is also called when listing schemas, - // and throwing an error will abort the listing process. exists = false; return nullptr; } diff --git a/src/catalog/rest/iceberg_catalog.cpp b/src/catalog/rest/iceberg_catalog.cpp index b02c9eabc..2e91835d0 100644 --- a/src/catalog/rest/iceberg_catalog.cpp +++ b/src/catalog/rest/iceberg_catalog.cpp @@ -278,6 +278,7 @@ void IcebergCatalog::AddDefaultSupportedEndpoints() { supported_urls.insert("POST /v1/{prefix}/tables/rename"); // commit updates to multiple tables in an atomic transaction supported_urls.insert("POST /v1/{prefix}/transactions/commit"); + // Views aren't in the spec defaults; catalogs that support them advertise via /v1/config } void IcebergCatalog::AddS3TablesEndpoints() { diff --git a/src/catalog/rest/iceberg_table_set.cpp b/src/catalog/rest/iceberg_table_set.cpp index 45974f877..8fcda479f 100644 --- a/src/catalog/rest/iceberg_table_set.cpp +++ b/src/catalog/rest/iceberg_table_set.cpp @@ -6,12 +6,15 @@ #include "duckdb/common/enums/http_status_code.hpp" #include "duckdb/common/exception/http_exception.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" #include "duckdb/parser/parser.hpp" #include "duckdb/planner/tableref/bound_at_clause.hpp" #include "duckdb/planner/expression_binder/table_function_binder.hpp" #include "catalog/rest/api/catalog_api.hpp" #include "catalog/rest/api/catalog_utils.hpp" +#include "common/iceberg_constants.hpp" #include "iceberg_logging.hpp" #include "catalog/rest/iceberg_catalog.hpp" #include "catalog/rest/catalog_entry/table/iceberg_table_entry.hpp" @@ -359,4 +362,212 @@ optional_ptr IcebergTableSet::GetEntry(ClientContext &context, con return ret; } +// ─── View operations ───────────────────────────────────────────────────────── + +const case_insensitive_map_t> &IcebergTableSet::GetViewEntries() const { + return view_entries; +} + +case_insensitive_map_t> &IcebergTableSet::GetViewEntriesMutable() { + return view_entries; +} + +void IcebergTableSet::InvalidateViewCache(const string &view_name) { + view_entries.erase(view_name); + view_catalog_entries.erase(view_name); +} + +void IcebergTableSet::LoadViewEntries(ClientContext &context) { + auto &iceberg_transaction = IcebergTransaction::Get(context, catalog); + bool views_listed = + iceberg_transaction.listed_view_schemas.find(schema.name) != iceberg_transaction.listed_view_schemas.end(); + if (views_listed) { + return; + } + auto &ic_catalog = catalog.Cast(); + // Only attempt to list views if the endpoint is supported + if (ic_catalog.supported_urls.find("GET /v1/{prefix}/namespaces/{namespace}/views") == + ic_catalog.supported_urls.end()) { + iceberg_transaction.listed_view_schemas.insert(schema.name); + return; + } + auto views = IRCAPI::GetViews(context, ic_catalog, schema); + for (auto &view : views) { + if (view_entries.find(view.name) == view_entries.end()) { + auto info = make_uniq(schema, view.name); + view_entries.emplace(view.name, std::move(info)); + } + } + iceberg_transaction.listed_view_schemas.insert(schema.name); +} + +optional_ptr IcebergTableSet::GetViewEntry(ClientContext &context, const string &view_name) { + lock_guard l(entry_lock); + return GetViewEntryInternal(context, view_name); +} + +optional_ptr IcebergTableSet::GetViewEntryInternal(ClientContext &context, const string &view_name) { + auto &ic_catalog = catalog.Cast(); + auto &iceberg_transaction = IcebergTransaction::Get(context, catalog); + + // Check if view was deleted in this transaction + auto view_key = IcebergTableInformation::GetTableKey(schema.namespace_items, view_name); + if (iceberg_transaction.deleted_views.count(view_key) > 0) { + return nullptr; + } + + // Check if we already have a cached ViewCatalogEntry for this view + auto cached_it = view_catalog_entries.find(view_name); + if (cached_it != view_catalog_entries.end()) { + return cached_it->second.get(); + } + + // Check if the view was created in this transaction + auto created_it = iceberg_transaction.created_views.find(view_key); + if (created_it != iceberg_transaction.created_views.end()) { + auto &view_info = created_it->second; + auto view_entry = make_uniq(catalog, schema, *view_info); + auto result = view_entry.get(); + view_catalog_entries.emplace(view_name, std::move(view_entry)); + return result; + } + + // Check if the view endpoint is supported + if (ic_catalog.supported_urls.find("GET /v1/{prefix}/namespaces/{namespace}/views/{view}") == + ic_catalog.supported_urls.end()) { + return nullptr; + } + + // Load the view from the REST catalog + auto get_view_result = IRCAPI::GetView(context, ic_catalog, schema, view_name); + if (get_view_result.has_error) { + if (get_view_result.status_ == HTTPStatusCode::NotFound_404) { + // View legitimately does not exist in the catalog — let DuckDB report "View ... does not exist". + return nullptr; + } + // 401 / 403 / 500 / etc. — surface the real error rather than silently masking it as "not found". + throw HTTPException(StringUtil::Format("GetView endpoint returned response code %s with message \"%s\"", + EnumUtil::ToString(get_view_result.status_), + get_view_result.error_._error.message)); + } + auto &load_result = *get_view_result.result_; + + // Find a SQL representation with the DuckDB dialect first, fall back to any SQL representation. + string view_sql; + auto &metadata = load_result.metadata; + // Find the current version + for (auto &version : metadata.versions) { + if (version.version_id != metadata.current_version_id) { + continue; + } + // Search representations for DuckDB dialect first + for (auto &repr : version.representations) { + if (!repr.has_sqlview_representation) { + continue; + } + if (repr.sqlview_representation.dialect == IcebergConstants::ViewDuckDBDialect) { + view_sql = repr.sqlview_representation.sql; + break; + } + } + // If no DuckDB dialect found, use the first available SQL representation + if (view_sql.empty()) { + for (auto &repr : version.representations) { + if (repr.has_sqlview_representation) { + DUCKDB_LOG_WARNING(context, + "View '%s' has no representation with dialect '%s'; falling back to " + "dialect '%s' (the SQL may not parse cleanly in DuckDB)", + view_name, IcebergConstants::ViewDuckDBDialect, + repr.sqlview_representation.dialect); + view_sql = repr.sqlview_representation.sql; + break; + } + } + } + break; + } + + if (view_sql.empty()) { + // View exists but has no SQL — create an unbound placeholder so it's visible in catalog + // but fails with a clear error when someone tries to query it + view_sql = StringUtil::Format("SELECT error('View \"%s\" exists in the Iceberg catalog but has no SQL " + "representation that DuckDB can use')", + view_name); + } + + // Parse the SQL text into a SelectStatement + unique_ptr view_query; + try { + view_query = CreateViewInfo::ParseSelect(view_sql); + } catch (std::exception &ex) { + // View exists but its SQL can't be parsed by DuckDB (e.g., Spark/Trino-only syntax). + // Log the parse error for debugging, then fall back to a placeholder that produces + // a clear error if anyone queries it. + DUCKDB_LOG_WARNING(context, + "View '%s' SQL could not be parsed by DuckDB: %s. " + "Replacing with an error() placeholder.", + view_name, ex.what()); + auto error_sql = StringUtil::Format( + "SELECT error('View \"%s\" exists in the Iceberg catalog but its SQL dialect cannot be parsed by DuckDB')", + view_name); + view_query = CreateViewInfo::ParseSelect(error_sql); + } + + auto view_info = make_uniq(schema, view_name); + view_info->query = std::move(view_query); + view_info->sql = view_sql; + + auto view_entry = make_uniq(catalog, schema, *view_info); + auto result = view_entry.get(); + view_catalog_entries.emplace(view_name, std::move(view_entry)); + return result; +} + +void IcebergTableSet::ScanViews(ClientContext &context, const std::function &callback) { + lock_guard lock(entry_lock); + LoadViewEntries(context); + auto &iceberg_transaction = IcebergTransaction::Get(context, catalog); + + // Include views created in this transaction + for (auto &created_view : iceberg_transaction.created_views) { + auto &view_info = created_view.second; + auto cached_it = view_catalog_entries.find(view_info->view_name); + if (cached_it == view_catalog_entries.end()) { + auto view_entry = make_uniq(catalog, schema, *view_info); + view_catalog_entries.emplace(view_info->view_name, std::move(view_entry)); + } + } + + for (auto &view_entry_kv : view_entries) { + auto &view_name = view_entry_kv.first; + // Skip views deleted in this transaction + auto view_key = IcebergTableInformation::GetTableKey(schema.namespace_items, view_name); + if (iceberg_transaction.deleted_views.count(view_key) > 0) { + continue; + } + // Get or create the ViewCatalogEntry + auto cached_it = view_catalog_entries.find(view_name); + if (cached_it != view_catalog_entries.end()) { + callback(*cached_it->second); + } else { + auto entry = GetViewEntryInternal(context, view_name); + if (entry) { + callback(*entry); + } + } + } + + // Also scan views created in this transaction that aren't in view_entries yet + for (auto &created_view : iceberg_transaction.created_views) { + auto &view_name = created_view.second->view_name; + if (view_entries.find(view_name) != view_entries.end()) { + continue; // already scanned above + } + auto cached_it = view_catalog_entries.find(view_name); + if (cached_it != view_catalog_entries.end()) { + callback(*cached_it->second); + } + } +} + } // namespace duckdb diff --git a/src/catalog/rest/transaction/iceberg_transaction.cpp b/src/catalog/rest/transaction/iceberg_transaction.cpp index 6901b08e9..f848ba2c0 100644 --- a/src/catalog/rest/transaction/iceberg_transaction.cpp +++ b/src/catalog/rest/transaction/iceberg_transaction.cpp @@ -1,6 +1,7 @@ #include "catalog/rest/transaction/iceberg_transaction.hpp" #include "duckdb/common/assert.hpp" +#include "duckdb/common/types/timestamp.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" @@ -9,6 +10,7 @@ #include "duckdb/main/client_data.hpp" #include "yyjson.hpp" +#include "common/iceberg_constants.hpp" #include "planning/metadata_io/manifest/iceberg_manifest_reader.hpp" #include "catalog/rest/transaction/iceberg_transaction.hpp" #include "catalog/rest/iceberg_catalog.hpp" @@ -21,6 +23,7 @@ #include "core/metadata/snapshot/iceberg_snapshot.hpp" #include "catalog/rest/iceberg_catalog.hpp" #include "catalog/rest/catalog_entry/schema/iceberg_schema_entry.hpp" +#include "catalog/rest/api/iceberg_type.hpp" #include "planning/metadata_io/avro/avro_scan.hpp" #include "iceberg_logging.hpp" #include "catalog/rest/api/table_update.hpp" @@ -389,7 +392,7 @@ TableTransactionInfo IcebergTransaction::GetTransactionRequest(IcebergTransactio void IcebergTransaction::Commit() { if (transaction_updates.empty() && created_schemas.empty() && deleted_schemas.empty() && - schema_property_updates.empty()) { + schema_property_updates.empty() && created_views.empty() && deleted_views.empty()) { return; } @@ -428,6 +431,8 @@ void IcebergTransaction::Commit() { static_cast(type)); }; } + DoViewDeletes(*temp_con_context); + DoViewCreates(*temp_con_context); DoSchemaDeletes(*temp_con_context); } catch (std::exception &ex) { ErrorData error(ex); @@ -534,6 +539,7 @@ void IcebergTransaction::DoTableRename(IcebergTransactionRenameUpdate &rename_up IRCAPI::CommitTableRename(context, catalog, transaction_json); DropInfo drop_info; + drop_info.type = CatalogType::TABLE_ENTRY; drop_info.name = table_name; drop_info.if_not_found = OnEntryNotFound::THROW_EXCEPTION; schema.DropEntry(context, drop_info, true); @@ -558,6 +564,7 @@ void IcebergTransaction::DoTableDeletes(IcebergTransactionDeleteUpdate &delete_u // remove the table entry from the catalog auto &schema_entry = ic_catalog.schemas.GetEntry(schema_key).Cast(); DropInfo drop_info; + drop_info.type = CatalogType::TABLE_ENTRY; drop_info.name = table_name; drop_info.if_not_found = OnEntryNotFound::RETURN_NULL; schema_entry.DropEntry(context, drop_info, true); @@ -649,6 +656,105 @@ struct ScopedTransaction { } // namespace +void IcebergTransaction::DoViewCreates(ClientContext &context) { + if (created_views.empty()) { + return; + } + auto &ic_catalog = catalog.Cast(); + for (auto &view_entry : created_views) { + auto &view_info = view_entry.second; + auto &view_name = view_info->view_name; + // Look up the schema entry from the catalog by name + auto &schema_entry = ic_catalog.schemas.GetEntry(view_info->schema).Cast(); + + // query may have been moved by ViewCatalogEntry::Initialize(), so use sql field or reconstruct + string view_sql; + if (!view_info->sql.empty()) { + view_sql = view_info->sql; + } else if (view_info->query) { + view_sql = view_info->query->ToString(); + } else { + throw InternalException("Cannot commit view '%s': no SQL representation available", view_name); + } + + // Build the CreateViewRequest JSON body + std::unique_ptr doc_p(yyjson_mut_doc_new(nullptr)); + auto doc = doc_p.get(); + auto root_object = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root_object); + + // name + yyjson_mut_obj_add_strcpy(doc, root_object, "name", view_name.c_str()); + + // schema — the view's output column types + auto schema_obj = yyjson_mut_obj_add_obj(doc, root_object, "schema"); + yyjson_mut_obj_add_strcpy(doc, schema_obj, "type", "struct"); + yyjson_mut_obj_add_int(doc, schema_obj, "schema-id", 0); + auto fields_arr = yyjson_mut_obj_add_arr(doc, schema_obj, "fields"); + for (idx_t i = 0; i < view_info->types.size(); i++) { + auto field_obj = yyjson_mut_arr_add_obj(doc, fields_arr); + auto &col_name = i < view_info->aliases.size() && !view_info->aliases[i].empty() ? view_info->aliases[i] + : view_info->names[i]; + yyjson_mut_obj_add_int(doc, field_obj, "id", i + 1); + yyjson_mut_obj_add_strcpy(doc, field_obj, "name", col_name.c_str()); + yyjson_mut_obj_add_bool(doc, field_obj, "required", false); + yyjson_mut_obj_add_strcpy(doc, field_obj, "type", + IcebergTypeHelper::LogicalTypeToIcebergType(view_info->types[i]).c_str()); + } + auto identifier_field_ids = yyjson_mut_obj_add_arr(doc, schema_obj, "identifier-field-ids"); + (void)identifier_field_ids; + + // view-version + auto version_obj = yyjson_mut_obj_add_obj(doc, root_object, "view-version"); + yyjson_mut_obj_add_int(doc, version_obj, "version-id", 1); + yyjson_mut_obj_add_uint(doc, version_obj, "timestamp-ms", + Timestamp::GetEpochMs(Timestamp::GetCurrentTimestamp())); + yyjson_mut_obj_add_int(doc, version_obj, "schema-id", 0); + + // summary + auto summary_obj = yyjson_mut_obj_add_obj(doc, version_obj, "summary"); + yyjson_mut_obj_add_strcpy(doc, summary_obj, "engine-name", "DuckDB"); + + // default-namespace + auto ns_arr = yyjson_mut_obj_add_arr(doc, version_obj, "default-namespace"); + for (auto &ns_item : schema_entry.namespace_items) { + yyjson_mut_arr_add_strcpy(doc, ns_arr, ns_item.c_str()); + } + + // default-catalog + yyjson_mut_obj_add_strcpy(doc, version_obj, "default-catalog", catalog.GetName().c_str()); + + // representations + auto repr_arr = yyjson_mut_obj_add_arr(doc, version_obj, "representations"); + auto repr_obj = yyjson_mut_arr_add_obj(doc, repr_arr); + yyjson_mut_obj_add_strcpy(doc, repr_obj, "type", IcebergConstants::ViewSQLRepresentationType); + yyjson_mut_obj_add_strcpy(doc, repr_obj, "sql", view_sql.c_str()); + yyjson_mut_obj_add_strcpy(doc, repr_obj, "dialect", IcebergConstants::ViewDuckDBDialect); + + // properties (empty) + yyjson_mut_obj_add_obj(doc, root_object, "properties"); + + auto json_body = JsonDocToString(std::move(doc_p)); + IRCAPI::CommitNewView(context, catalog, schema_entry, json_body); + + // Add a placeholder to the schema's view_entries so subsequent operations can find it + auto placeholder = make_uniq(schema_entry, view_name); + schema_entry.tables.GetViewEntriesMutable().emplace(view_name, std::move(placeholder)); + } + created_views.clear(); +} + +void IcebergTransaction::DoViewDeletes(ClientContext &context) { + auto &ic_catalog = catalog.Cast(); + for (auto &deleted_view : deleted_views) { + auto &info = deleted_view.second; + IRCAPI::CommitViewDelete(context, catalog, info.namespace_items, info.view_name); + auto &schema_entry = ic_catalog.schemas.GetEntry(info.schema_name).Cast(); + schema_entry.tables.InvalidateViewCache(info.view_name); + } + deleted_views.clear(); +} + void IcebergTransaction::CleanupFiles() { // remove any files that were written if (!catalog.attach_options.remove_files_on_delete) { diff --git a/src/function/iceberg_functions.cpp b/src/function/iceberg_functions.cpp index 248dd6267..6b57dfe99 100644 --- a/src/function/iceberg_functions.cpp +++ b/src/function/iceberg_functions.cpp @@ -24,6 +24,7 @@ vector IcebergFunctions::GetTableFunctions(ExtensionLoader &lo functions.push_back(RemoveIcebergSchemaPropertiesFunctions()); functions.push_back(GetIcebergToDuckLakeFunction()); functions.push_back(GetIcebergLoadTableResponseFunction()); + functions.push_back(GetIcebergViewMetadataFunction()); return functions; } diff --git a/src/function/metadata/iceberg_view_metadata.cpp b/src/function/metadata/iceberg_view_metadata.cpp new file mode 100644 index 000000000..13a72c7eb --- /dev/null +++ b/src/function/metadata/iceberg_view_metadata.cpp @@ -0,0 +1,185 @@ +#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" +#include "duckdb/common/http_util.hpp" +#include "duckdb/common/string.hpp" +#include "duckdb/common/vector_operations/vector_operations.hpp" + +#include "function/iceberg_functions.hpp" +#include "common/iceberg_utils.hpp" +#include "catalog/rest/api/catalog_api.hpp" +#include "catalog/rest/api/catalog_utils.hpp" +#include "catalog/rest/api/url_utils.hpp" +#include "catalog/rest/iceberg_catalog.hpp" +#include "catalog/rest/catalog_entry/schema/iceberg_schema_entry.hpp" +#include "rest_catalog/objects/list.hpp" +#include "rest_catalog/objects/load_view_result.hpp" +#include "yyjson.hpp" + +using namespace duckdb_yyjson; + +namespace duckdb { + +//! Table function `iceberg_view_metadata('catalog.schema.view')` — returns the raw result of a +//! LoadView REST call, mirroring `iceberg_load_table_response` for tables. Useful for discovering +//! views on a REST catalog, including views written in a SQL dialect DuckDB can't parse. +struct IcebergViewMetadataBindData : public TableFunctionData { + IcebergCatalog &ic_catalog; + IcebergSchemaEntry &ic_schema; + string view_name; + + IcebergViewMetadataBindData(IcebergCatalog &ic_catalog, IcebergSchemaEntry &ic_schema, string view_name) + : ic_catalog(ic_catalog), ic_schema(ic_schema), view_name(std::move(view_name)) { + } +}; + +struct IcebergViewMetadataGlobalState : public GlobalTableFunctionState { + bool done = false; + + static unique_ptr Init(ClientContext &context, TableFunctionInitInput &input) { + return make_uniq(); + } +}; + +static unique_ptr MakeRequest(ClientContext &context, const IcebergViewMetadataBindData &bind_data) { + auto &ic_catalog = bind_data.ic_catalog; + auto &ic_schema = bind_data.ic_schema; + + auto url_builder = ic_catalog.GetBaseUrl(); + url_builder.AddPrefixComponent(ic_catalog.prefix, ic_catalog.prefix_is_one_component); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("namespaces")); + url_builder.AddPathComponent(IRCPathComponent::NamespaceComponent(ic_schema.namespace_items)); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent("views")); + url_builder.AddPathComponent(IRCPathComponent::RegularComponent(bind_data.view_name)); + + HTTPHeaders headers(*context.db); + unique_ptr response = + ic_catalog.auth_handler->Request(RequestType::GET_REQUEST, context, url_builder, headers); + if (!response->Success()) { + throw IOException("GET request to '%s' failed with status %s: %s", url_builder.GetURLEncoded(), + EnumUtil::ToString(response->status), response->body); + } + return response; +} + +static unique_ptr IcebergViewMetadataBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + auto input_string = input.inputs[0].ToString(); + auto qualified_name = QualifiedName::ParseComponents(input_string); + + if (qualified_name.size() != 3) { + throw InvalidInputException("Expected fully qualified view name (catalog.schema.view), got: %s", input_string); + } + + EntryLookupInfo view_lookup(CatalogType::VIEW_ENTRY, qualified_name[2]); + auto catalog_entry = + Catalog::GetEntry(context, qualified_name[0], qualified_name[1], view_lookup, OnEntryNotFound::THROW_EXCEPTION); + + if (catalog_entry->type != CatalogType::VIEW_ENTRY) { + throw InvalidInputException("'%s' is not a view", input_string); + } + auto &view_entry = catalog_entry->Cast(); + auto &view_catalog = view_entry.ParentCatalog(); + if (view_catalog.GetCatalogType() != "iceberg") { + throw InvalidInputException("View '%s' is not in an Iceberg REST catalog", input_string); + } + + auto &ic_catalog = view_catalog.Cast(); + auto &ic_schema = view_entry.schema.Cast(); + + auto ret = make_uniq(ic_catalog, ic_schema, qualified_name[2]); + + // metadata_location + names.push_back("metadata_location"); + return_types.push_back(LogicalType::VARCHAR); + + // metadata (JSON -> VARIANT) + names.push_back("metadata"); + return_types.push_back(LogicalType::VARIANT()); + + // config + names.push_back("config"); + return_types.push_back(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + + // request_url + names.push_back("request_url"); + return_types.push_back(LogicalType::VARCHAR); + + return std::move(ret); +} + +static void OutputMap(const case_insensitive_map_t &config, Vector &config_vec) { + auto config_count = config.size(); + ListVector::Reserve(config_vec, config_count); + auto &config_key_vec = MapVector::GetKeys(config_vec); + auto &config_val_vec = MapVector::GetValues(config_vec); + idx_t config_idx = 0; + for (auto &kv : config) { + FlatVector::GetData(config_key_vec)[config_idx] = StringVector::AddString(config_key_vec, kv.first); + FlatVector::GetData(config_val_vec)[config_idx] = StringVector::AddString(config_val_vec, kv.second); + config_idx++; + } + ListVector::SetListSize(config_vec, config_count); + auto &config_list_data = FlatVector::GetData(config_vec)[0]; + config_list_data.offset = 0; + config_list_data.length = config_count; +} + +static void IcebergViewMetadataFunction(ClientContext &context, TableFunctionInput &data, DataChunk &output) { + auto &bind_data = data.bind_data->Cast(); + auto &global_state = data.global_state->Cast(); + + if (global_state.done) { + return; + } + global_state.done = true; + + auto response = MakeRequest(context, bind_data); + + // Parse the response using yyjson + auto doc = ICUtils::APIResultToDoc(response->body); + auto *root = yyjson_doc_get_root(doc.get()); + + auto load_result = rest_api_objects::LoadViewResult::FromJSON(root); + + output.SetCardinality(1); + + // metadata_location + auto &metadata_location_vector = output.data[0]; + FlatVector::GetData(metadata_location_vector)[0] = + StringVector::AddString(metadata_location_vector, load_result.metadata_location); + + // metadata (VARIANT) + auto &metadata_vector = output.data[1]; + { + auto *metadata_val = yyjson_obj_get(root, "metadata"); + if (metadata_val) { + auto *json_str = yyjson_val_write(metadata_val, 0, nullptr); + if (json_str) { + Vector json_vec(LogicalType::JSON(), 1); + FlatVector::GetData(json_vec)[0] = StringVector::AddString(json_vec, string(json_str)); + free(json_str); + VectorOperations::Cast(context, json_vec, metadata_vector, 1); + } + } + } + + // config MAP(VARCHAR, VARCHAR) + auto &config_vector = output.data[2]; + OutputMap(load_result.config, config_vector); + + // request_url + auto &request_endpoint_vector = output.data[3]; + FlatVector::GetData(request_endpoint_vector)[0] = + StringVector::AddString(request_endpoint_vector, response->url); +} + +TableFunctionSet IcebergFunctions::GetIcebergViewMetadataFunction() { + TableFunctionSet function_set("iceberg_view_metadata"); + + auto fun = TableFunction({LogicalType::VARCHAR}, IcebergViewMetadataFunction, IcebergViewMetadataBind, + IcebergViewMetadataGlobalState::Init); + function_set.AddFunction(fun); + + return function_set; +} + +} // namespace duckdb diff --git a/src/include/catalog/rest/api/catalog_api.hpp b/src/include/catalog/rest/api/catalog_api.hpp index 747f68a3b..2438ae2e8 100644 --- a/src/include/catalog/rest/api/catalog_api.hpp +++ b/src/include/catalog/rest/api/catalog_api.hpp @@ -7,6 +7,7 @@ #include "catalog/rest/api/url_utils.hpp" #include "rest_catalog/objects/list.hpp" +#include "rest_catalog/objects/load_view_result.hpp" namespace duckdb { @@ -73,6 +74,16 @@ class IRCAPI { const IcebergTableEntry &table); static rest_api_objects::CatalogConfig GetCatalogConfig(ClientContext &context, IcebergCatalog &catalog, const string &warehouse); + + //! View operations + static vector GetViews(ClientContext &context, IcebergCatalog &catalog, + const IcebergSchemaEntry &schema); + static APIResult> + GetView(ClientContext &context, IcebergCatalog &catalog, const IcebergSchemaEntry &schema, const string &view_name); + static void CommitNewView(ClientContext &context, IcebergCatalog &catalog, const IcebergSchemaEntry &schema, + const string &json_body); + static void CommitViewDelete(ClientContext &context, IcebergCatalog &catalog, const vector &schema, + const string &view_name); }; } // namespace duckdb diff --git a/src/include/catalog/rest/iceberg_table_set.hpp b/src/include/catalog/rest/iceberg_table_set.hpp index 610c3a782..1053dd6ed 100644 --- a/src/include/catalog/rest/iceberg_table_set.hpp +++ b/src/include/catalog/rest/iceberg_table_set.hpp @@ -2,6 +2,8 @@ #pragma once #include "duckdb/catalog/catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" #include "catalog/rest/catalog_entry/table/iceberg_table_entry.hpp" #include "catalog/rest/catalog_entry/table/iceberg_table_information.hpp" @@ -34,12 +36,28 @@ class IcebergTableSet { //! or if entry is already filled. Returns False otherwise bool FillEntry(ClientContext &context, IcebergTableInformation &table); + //! View operations + optional_ptr GetViewEntry(ClientContext &context, const string &view_name); + void LoadViewEntries(ClientContext &context); + void ScanViews(ClientContext &context, const std::function &callback); + + const case_insensitive_map_t> &GetViewEntries() const; + case_insensitive_map_t> &GetViewEntriesMutable(); + void InvalidateViewCache(const string &view_name); + public: IcebergSchemaEntry &schema; Catalog &catalog; +private: + //! Internal view lookup — caller must hold entry_lock + optional_ptr GetViewEntryInternal(ClientContext &context, const string &view_name); + private: case_insensitive_map_t> entries; + case_insensitive_map_t> view_entries; + //! Cached ViewCatalogEntry instances for Scan + case_insensitive_map_t> view_catalog_entries; mutex entry_lock; }; diff --git a/src/include/catalog/rest/transaction/iceberg_transaction.hpp b/src/include/catalog/rest/transaction/iceberg_transaction.hpp index 87f98489f..812ed262c 100644 --- a/src/include/catalog/rest/transaction/iceberg_transaction.hpp +++ b/src/include/catalog/rest/transaction/iceberg_transaction.hpp @@ -2,6 +2,7 @@ #pragma once #include "duckdb/transaction/transaction.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" #include "catalog/rest/iceberg_schema_set.hpp" namespace duckdb { @@ -86,6 +87,8 @@ class IcebergTransaction : public Transaction { void DoSchemaCreates(ClientContext &context); void DoSchemaDeletes(ClientContext &context); void DoSchemaPropertyUpdates(ClientContext &context); + void DoViewCreates(ClientContext &context); + void DoViewDeletes(ClientContext &context); IcebergCatalog &GetCatalog(); void DropSecrets(ClientContext &context); TableTransactionInfo GetTransactionRequest(IcebergTransactionAlterUpdate &alter_update, ClientContext &context); @@ -112,12 +115,24 @@ class IcebergTransaction : public Transaction { //! The latest state of a table (either points into 'transaction_updates' or 'tables') case_insensitive_map_t current_table_data; + //! views that have been created in this transaction, to be committed on commit. + //! keyed by view_key (schema_namespace + view_name) + case_insensitive_map_t> created_views; + //! views that have been deleted in this transaction, to be deleted on commit. + struct DeletedViewInfo { + vector namespace_items; + string schema_name; + string view_name; + }; + case_insensitive_map_t deleted_views; + unordered_set created_schemas; unordered_set deleted_schemas; bool called_list_schemas = false; //! Set of schemas that this transaction has listed tables for case_insensitive_set_t listed_schemas; + case_insensitive_set_t listed_view_schemas; case_insensitive_set_t created_secrets; case_insensitive_set_t looked_up_entries; diff --git a/src/include/common/iceberg_constants.hpp b/src/include/common/iceberg_constants.hpp index e6a0d48e6..dbfa61bea 100644 --- a/src/include/common/iceberg_constants.hpp +++ b/src/include/common/iceberg_constants.hpp @@ -14,6 +14,11 @@ namespace duckdb { class IcebergConstants { public: static constexpr const char *DefaultGeometryCRS = "OGC:CRS84"; + //! SQL dialect identifier DuckDB writes/reads for its own Iceberg View representations. + //! Lowercase matches the convention used by other engines in the Iceberg View Spec (e.g. "spark", "trino"). + static constexpr const char *ViewDuckDBDialect = "duckdb"; + //! Representation `type` value for SQL views per the Iceberg View Spec. + static constexpr const char *ViewSQLRepresentationType = "sql"; }; } // namespace duckdb diff --git a/src/include/function/iceberg_functions.hpp b/src/include/function/iceberg_functions.hpp index 0d03f56e5..a4c59a05e 100644 --- a/src/include/function/iceberg_functions.hpp +++ b/src/include/function/iceberg_functions.hpp @@ -30,6 +30,7 @@ class IcebergFunctions { static TableFunctionSet GetIcebergScanFunction(ExtensionLoader &loader); static TableFunctionSet GetIcebergMetadataFunction(); static TableFunctionSet GetIcebergLoadTableResponseFunction(); + static TableFunctionSet GetIcebergViewMetadataFunction(); static TableFunctionSet GetIcebergColumnStatsFunction(); static TableFunctionSet GetIcebergPartitionStatsFunction(); static TableFunctionSet GetIcebergToDuckLakeFunction(); diff --git a/test/sql/local/catalog_custom_setup/fixture/enable_logging_http/test_table_information_requests.test b/test/sql/local/catalog_custom_setup/fixture/enable_logging_http/test_table_information_requests.test index 973a3faf9..001666d57 100644 --- a/test/sql/local/catalog_custom_setup/fixture/enable_logging_http/test_table_information_requests.test +++ b/test/sql/local/catalog_custom_setup/fixture/enable_logging_http/test_table_information_requests.test @@ -52,10 +52,12 @@ select count(*) > 10 from (show all tables); # 1 call to list namespaces # 1 call to list tables in default # 1 call to list tables in level1 namespace (no recursive namespace calls) +# 1 call to list views in default +# 1 call to list views in level1 namespace query I select count(*) from duckdb_logs_parsed('HTTP'); ---- -5 +7 statement ok call truncate_duckdb_logs(); @@ -101,12 +103,13 @@ commit; # 3 calls from above # 5 calls from show all tables above +# 2 calls to list views (default + level1) during show all tables # 1 call to the manifest list # 2 calls to read parquet files query I select count(*) from duckdb_logs_parsed('HTTP'); ---- -9 +11 statement ok call truncate_duckdb_logs(); diff --git a/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_create_view.test b/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_create_view.test new file mode 100644 index 000000000..8721b3a1b --- /dev/null +++ b/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_create_view.test @@ -0,0 +1,142 @@ +# name: test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_create_view.test +# description: test create view, drop view, and querying views in iceberg catalog +# group: [create] + +require-env ICEBERG_SERVER_AVAILABLE + +require-env SECRETS_CREATED_AND_CATALOG_ATTACHED + +require avro + +require parquet + +require iceberg + +require httpfs + +set ignore_error_messages + +statement ok +create schema if not exists my_datalake.default; + +statement ok +use my_datalake.default; + +# Clean up from previous runs +statement ok +drop view if exists test_view_v1; + +statement ok +drop view if exists test_view_v2; + +statement ok +drop table if exists test_view_base_table; + +# Create a base table to back the view +statement ok +create table test_view_base_table (i int, s varchar); + +statement ok +insert into test_view_base_table values (1, 'hello'), (2, 'world'), (3, 'foo'); + +# ─── CREATE VIEW ────────────────────────────────────────────────────────────── + +statement ok +create view test_view_v1 as select i, s from test_view_base_table where i > 1; + +# Query the view +query II +select * from test_view_v1 order by i; +---- +2 world +3 foo + +# ─── CREATE VIEW IF NOT EXISTS ──────────────────────────────────────────────── + +# Should succeed silently (view already exists) +statement ok +create view if not exists test_view_v1 as select 1; + +# Original view should still be there +query II +select * from test_view_v1 order by i; +---- +2 world +3 foo + +# ─── CREATE OR REPLACE VIEW ────────────────────── +# Should fail, CREATE OR REPLACE isn't supported +statement error +create or replace view test_view_v1 as select i * 10 as i_times_10 from test_view_base_table; +---- +:.*CREATE OR REPLACE not supported.* + +# The original view must be untouched by the rejected REPLACE +query II +select * from test_view_v1 order by i; +---- +2 world +3 foo + +# ─── DROP VIEW ──────────────────────────────────────────────────────────────── + +statement ok +drop view test_view_v1; + +# View should no longer exist +statement error +select * from test_view_v1; +---- +:.*does not exist.* + +# DROP VIEW IF EXISTS on non-existent view +statement ok +drop view if exists test_view_v1; + +# DROP VIEW on non-existent view should error +statement error +drop view test_view_v1; +---- +:.*does not exist.* + +# ─── VIEW WITH COLUMN ALIASES ──────────────────────────────────────────────── + +statement ok +create view test_view_v2 (col_a, col_b) as select i, s from test_view_base_table; + +query II +select col_a, col_b from test_view_v2 order by col_a; +---- +1 hello +2 world +3 foo + +# ─── REST 404 path with no cache ────────────────────────────────────────────── +# Cross-attachment query of a view that has never been created. +# Forces IRCAPI::GetView to hit the REST catalog and receive a 404 +# (no transaction or cache state can short-circuit the lookup). + +statement ok +ATTACH '' AS my_datalake_404 (TYPE ICEBERG, CLIENT_ID 'admin', CLIENT_SECRET 'password', ENDPOINT 'http://127.0.0.1:8181'); + +statement ok +use my_datalake_404.default; + +statement error +select * from never_created_view_for_404_test; +---- +:.*does not exist.* + +# Switch back to the primary attachment before cleanup +statement ok +use my_datalake.default; + +statement ok +DETACH my_datalake_404; + +# Clean up +statement ok +drop view test_view_v2; + +statement ok +drop table test_view_base_table; diff --git a/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_features.test b/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_features.test new file mode 100644 index 000000000..3621fe0cb --- /dev/null +++ b/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_features.test @@ -0,0 +1,141 @@ +# name: test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_features.test +# description: iceberg view edge cases — creation-time validation, select *, select * exclude, joins, catalog functions +# group: [create] + +require-env ICEBERG_SERVER_AVAILABLE + +require-env SECRETS_CREATED_AND_CATALOG_ATTACHED + +require avro + +require parquet + +require iceberg + +require httpfs + +statement ok +create schema if not exists my_datalake.default; + +statement ok +use my_datalake.default; + +# Clean up from previous runs +statement ok +drop view if exists v_star; + +statement ok +drop view if exists v_exclude; + +statement ok +drop view if exists v_join; + +statement ok +drop table if exists vf_orders; + +statement ok +drop table if exists vf_customers; + +# Base tables to back the views +statement ok +create table vf_customers (cust_id int, name varchar, region varchar); + +statement ok +insert into vf_customers values (1, 'alice', 'EU'), (2, 'bob', 'US'), (3, 'carol', 'EU'); + +statement ok +create table vf_orders (order_id int, cust_id int, amount int); + +statement ok +insert into vf_orders values (10, 1, 100), (11, 1, 50), (12, 2, 200); + +# ───────────────── view selecting a non-existent column errors at CREATE ─────── +statement error +create view v_bad_col as select nonexistent_column from vf_customers; +---- +:.*not found.* + +# the rejected view must not exist +query I +select count(*) from duckdb_views() where view_name = 'v_bad_col'; +---- +0 + +# ───────────────── view selecting from a non-existent table errors at CREATE ─── +statement error +create view v_bad_tbl as select * from table_that_does_not_exist_vf; +---- +:.*does not exist.* + +# ─── SELECT * view ──────────────────────────────────────────────────────────── +statement ok +create view v_star as select * from vf_customers; + +query III +select * from v_star order by cust_id; +---- +1 alice EU +2 bob US +3 carol EU + +# ───────────────── selecting a non-existent column from a valid view ────────── +statement error +select no_such_col from v_star; +---- +:.*not found.* + +# ─── SELECT * EXCLUDE view ──────────────────────────────────────────────────── +statement ok +create view v_exclude as select * exclude (region) from vf_customers; + +query II +select * from v_exclude order by cust_id; +---- +1 alice +2 bob +3 carol + +# ─── view that joins tables ─────────────────────────────────────────────────── +statement ok +create view v_join as + select c.name, o.order_id, o.amount + from vf_customers c join vf_orders o on c.cust_id = o.cust_id; + +query III +select * from v_join order by order_id; +---- +alice 10 100 +alice 11 50 +bob 12 200 + +# ─── catalog functions must not crash with iceberg views present ────────────── +statement ok +from duckdb_views(); + +statement ok +from duckdb_tables(); + +statement ok +show all tables; + +# the iceberg views should be discoverable via duckdb_views() +query I +select count(*) >= 3 from duckdb_views() where view_name in ('v_star', 'v_exclude', 'v_join'); +---- +true + +# Clean up +statement ok +drop view v_join; + +statement ok +drop view v_exclude; + +statement ok +drop view v_star; + +statement ok +drop table vf_orders; + +statement ok +drop table vf_customers; diff --git a/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_metadata.test b/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_metadata.test new file mode 100644 index 000000000..bcd07e8b2 --- /dev/null +++ b/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_metadata.test @@ -0,0 +1,68 @@ +# name: test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_metadata.test +# description: iceberg_view_metadata table function returns the LoadView REST response +# group: [create] + +require-env ICEBERG_SERVER_AVAILABLE + +require-env SECRETS_CREATED_AND_CATALOG_ATTACHED + +require avro + +require parquet + +require iceberg + +require httpfs + +statement ok +create schema if not exists my_datalake.default; + +statement ok +use my_datalake.default; + +# Clean up from previous runs +statement ok +drop view if exists vm_view; + +statement ok +drop table if exists vm_base; + +statement ok +create table vm_base (a int, b varchar); + +statement ok +insert into vm_base values (1, 'x'), (2, 'y'); + +statement ok +create view vm_view as select a, b from vm_base where a > 0; + +# iceberg_view_metadata returns exactly one row +query I +select count(*) from iceberg_view_metadata('my_datalake.default.vm_view'); +---- +1 + +# metadata VARIANT is populated and the request hit the views endpoint +query II +select metadata is not null, request_url like '%/views/vm_view' from iceberg_view_metadata('my_datalake.default.vm_view'); +---- +true true + +# a non-existent view should raise a clear catalog error +statement error +select * from iceberg_view_metadata('my_datalake.default.this_view_does_not_exist_vm'); +---- +:.*does not exist.* + +# a non-qualified name is rejected +statement error +select * from iceberg_view_metadata('vm_view'); +---- +:.*fully qualified view name.* + +# Clean up +statement ok +drop view vm_view; + +statement ok +drop table vm_base; diff --git a/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_scan_no_deadlock.test b/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_scan_no_deadlock.test new file mode 100644 index 000000000..22f2c565d --- /dev/null +++ b/test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_scan_no_deadlock.test @@ -0,0 +1,74 @@ +# name: test/sql/local/catalog_test_config_setup/catalog_agnostic/create/test_view_scan_no_deadlock.test +# description: test that SHOW ALL TABLES does not deadlock when views exist in the catalog +# group: [create] + +require-env ICEBERG_SERVER_AVAILABLE + +require-env SECRETS_CREATED_AND_CATALOG_ATTACHED + +require avro + +require parquet + +require iceberg + +require httpfs + +set ignore_error_messages + +statement ok +create schema if not exists my_datalake.default; + +statement ok +use my_datalake.default; + +# Clean up from previous runs +statement ok +drop view if exists deadlock_test_view; + +statement ok +drop table if exists deadlock_test_table; + +# Create a table with data and a view +statement ok +create table deadlock_test_table (id int, name varchar); + +statement ok +insert into deadlock_test_table values (1, 'alice'), (2, 'bob'); + +statement ok +create view deadlock_test_view as select id, name from deadlock_test_table where id > 0; + +# Now reconnect — this forces a fresh session where view_catalog_entries cache is empty. +# The view exists in the REST catalog but has never been loaded in this new session. +# SHOW ALL TABLES will call ScanViews which must load the view without deadlocking. + +statement ok +ATTACH '' AS my_datalake2 (TYPE ICEBERG,CLIENT_ID 'admin',CLIENT_SECRET 'password',ENDPOINT 'http://127.0.0.1:8181'); + +statement ok +use my_datalake2.default; + +# This is the critical test: triggers ScanViews on a fresh catalog attachment +# where views exist but are not cached — hits the GetViewEntry path inside ScanViews +statement ok +SHOW ALL TABLES; + +# Direct query on the view through the fresh attachment should also work +query I +select count(*) from deadlock_test_view; +---- +2 + +# Clean up through original attachment +statement ok +use my_datalake.default; + +statement ok +drop view deadlock_test_view; + +statement ok +drop table deadlock_test_table; + +statement ok +DETACH my_datalake2;