V1.4 andium

.github/workflows/MainDistributionPipeline.yml

@@ -14,23 +14,23 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.2
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.3
     with:
       extension_name: iceberg
-      duckdb_version: v1.4.2
-      ci_tools_version: v1.4.2
-      exclude_archs: 'windows_amd64_mingw;wasm_mvp;wasm_eh;wasm_threads'
+      duckdb_version: v1.4.3
+      ci_tools_version: v1.4.3
+      exclude_archs: 'windows_amd64_mingw'
       extra_toolchains: 'python3'
 
   duckdb-stable-deploy:
     name: Deploy extension binaries
     needs: duckdb-stable-build
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.4.2
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.4.3
     secrets: inherit
     with:
       extension_name: iceberg
-      duckdb_version: v1.4.2
-      ci_tools_version: v1.4.2
-      exclude_archs: 'windows_amd64_mingw;wasm_mvp;wasm_eh;wasm_threads'
+      duckdb_version: v1.4.3
+      ci_tools_version: v1.4.3
+      exclude_archs: 'windows_amd64_mingw'
       deploy_latest: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }}
       deploy_versioned: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }}

CMakeLists.txt

@@ -80,6 +80,12 @@ add_subdirectory(src/rest_catalog/objects)
 add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES} ${ALL_OBJECT_FILES})
 
 set(PARAMETERS "-warnings")
+
+if(EMSCRIPTEN)
+  set (DUCKDB_EXTENSION_ICEBERG_LINKED_LIBS "../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-mqtt.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-kinesis.a;../../vcpkg_installed/wasm32-emscripten/lib/liblzma.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-auth.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-s3.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-s3.a;../../vcpkg_installed/wasm32-emscripten/lib/libroaring.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-cal.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-sdkutils.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-sso.a;../../vcpkg_installed/wasm32-emscripten/lib/libs2n.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-common.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-checksums.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-sts.a;../../vcpkg_installed/wasm32-emscripten/lib/libsnappy.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-compression.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-cognito-identity.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-crt-cpp.a;../../vcpkg_installed/wasm32-emscripten/lib/libssl.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-event-stream.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-core.a;../../vcpkg_installed/wasm32-emscripten/lib/libcrypto.a;../../vcpkg_installed/wasm32-emscripten/lib/libz.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-http.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-dynamodb.a;../../vcpkg_installed/wasm32-emscripten/lib/libcurl.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-io.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-identity-management.a;../../vcpkg_installed/wasm32-emscripten/lib/libjansson.a;../../third_party/mbedtls/libduckdb_mbedtls.a")
+endif()
+
+
 build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES} ${ALL_OBJECT_FILES})
 
 if(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten")

Makefile

@@ -25,6 +25,3 @@ data_large: data data_clean
 
 data_clean:
 	rm -rf data/generated
-
-wasm_pre_build_step:
-	jq 'del(.overrides,.dependencies[5])' vcpkg.json | unexpand -t2 > vcpkg.json.tmp && mv vcpkg.json.tmp vcpkg.json

src/aws.cpp

@@ -132,6 +132,7 @@ static string GetPayloadHash(const char *buffer, idx_t buffer_len) {
 std::shared_ptr<Aws::Http::HttpRequest> AWSInput::CreateSignedRequest(Aws::Http::HttpMethod method,
                                                                       const Aws::Http::URI &uri, HTTPHeaders &headers,
                                                                       const string &body) {
+#ifndef EMSCRIPTEN
 	auto request = Aws::Http::CreateHttpRequest(uri, method, Aws::Utils::Stream::DefaultResponseStreamFactoryMethod);
 	request->SetUserAgent(user_agent);
 
@@ -153,10 +154,14 @@ std::shared_ptr<Aws::Http::HttpRequest> AWSInput::CreateSignedRequest(Aws::Http:
 	}
 
 	return request;
+#else
+	return nullptr;
+#endif
 }
 
 unique_ptr<HTTPResponse> AWSInput::ExecuteRequestLegacy(ClientContext &context, Aws::Http::HttpMethod method,
                                                         HTTPHeaders &headers, const string &body) {
+#ifndef EMSCRIPTEN
 	InitAWSAPI();
 	auto clientConfig = BuildClientConfig();
 	auto uri = BuildURI();
@@ -191,6 +196,9 @@ unique_ptr<HTTPResponse> AWSInput::ExecuteRequestLegacy(ClientContext &context,
 		throw HTTPException(*result, result->reason);
 	}
 	return result;
+#else
+	throw NotImplementedException("ExecuteRequestLegacy is not implemented in duckdb-wasm");
+#endif
 }
 
 unique_ptr<HTTPResponse> AWSInput::ExecuteRequest(ClientContext &context, Aws::Http::HttpMethod method,

src/include/rest_catalog/objects/assert_ref_snapshot_id.hpp

@@ -30,6 +30,7 @@ class AssertRefSnapshotId {
 	TableRequirementType type;
 	string ref;
 	int64_t snapshot_id;
+	bool has_snapshot_id = false;
 };
 
 } // namespace rest_api_objects

src/metadata/iceberg_table_metadata.cpp

@@ -334,11 +334,11 @@ string IcebergTableMetadata::GetDataPath() const {
 string IcebergTableMetadata::GetMetadataPath() const {
 	// If write.metadata.path property is set, use it; otherwise use default location + "/metadata"
 	auto metadata_path = table_properties.find("write.metadata.path");
-	// If write.data.path property is set, use it; otherwise use default location + "/data"
+	// If write.data.path property is set, use it; otherwise use default location + "/metadata"
 	if (metadata_path != table_properties.end()) {
 		return metadata_path->second;
 	}
-	return location + "/data";
+	return location + "/metadata";
 }
 
 string IcebergTableMetadata::GetTableProperty(string property_string) const {

src/rest_catalog/objects/assert_ref_snapshot_id.cpp

@@ -50,10 +50,14 @@ string AssertRefSnapshotId::TryFromJSON(yyjson_val *obj) {
 	if (!snapshot_id_val) {
 		return "AssertRefSnapshotId required property 'snapshot-id' is missing";
 	} else {
-		if (yyjson_is_sint(snapshot_id_val)) {
+		if (yyjson_is_null(snapshot_id_val)) {
+			has_snapshot_id = false;
+		} else if (yyjson_is_sint(snapshot_id_val)) {
 			snapshot_id = yyjson_get_sint(snapshot_id_val);
+			has_snapshot_id = true;
 		} else if (yyjson_is_uint(snapshot_id_val)) {
 			snapshot_id = yyjson_get_uint(snapshot_id_val);
+			has_snapshot_id = true;
 		} else {
 			return StringUtil::Format(
 			    "AssertRefSnapshotId property 'snapshot_id' is not of type 'integer', found '%s' instead",

src/storage/irc_catalog.cpp

@@ -240,7 +240,7 @@ void IRCatalog::AddDefaultSupportedEndpoints() {
 	// Rename a table from one identifier to another.
 	supported_urls.insert("POST /v1/{prefix}/tables/rename");
 	// commit updates to multiple tables in an atomic transaction
-	supported_urls.insert("POST /v1/{prefix}/transactions/commit)");
+	supported_urls.insert("POST /v1/{prefix}/transactions/commit");
 }
 
 void IRCatalog::AddS3TablesEndpoints() {
@@ -268,7 +268,7 @@ void IRCatalog::AddS3TablesEndpoints() {
 	// Rename a table from one identifier to another.
 	supported_urls.insert("POST /v1/{prefix}/tables/rename");
 	// commit updates to multiple tables in an atomic transaction
-	supported_urls.insert("POST /v1/{prefix}/transactions/commit)");
+	supported_urls.insert("POST /v1/{prefix}/transactions/commit");
 }
 
 void IRCatalog::GetConfig(ClientContext &context, IcebergEndpointType &endpoint_type) {

src/storage/irc_transaction.cpp

@@ -47,7 +47,11 @@ void CommitTableToJSON(yyjson_mut_doc *doc, yyjson_mut_val *root_object,
 			auto requirement_json = yyjson_mut_arr_add_obj(doc, requirements_array);
 			yyjson_mut_obj_add_strcpy(doc, requirement_json, "type", assert_ref_snapshot_id.type.value.c_str());
 			yyjson_mut_obj_add_strcpy(doc, requirement_json, "ref", assert_ref_snapshot_id.ref.c_str());
-			yyjson_mut_obj_add_uint(doc, requirement_json, "snapshot-id", assert_ref_snapshot_id.snapshot_id);
+			if (assert_ref_snapshot_id.has_snapshot_id) {
+				yyjson_mut_obj_add_uint(doc, requirement_json, "snapshot-id", assert_ref_snapshot_id.snapshot_id);
+			} else {
+				yyjson_mut_obj_add_null(doc, requirement_json, "snapshot-id");
+			}
 		} else if (requirement.has_assert_create) {
 			auto &assert_create = requirement.assert_create;
 			auto requirement_json = yyjson_mut_arr_add_obj(doc, requirements_array);
@@ -230,6 +234,18 @@ static rest_api_objects::TableRequirement CreateAssertRefSnapshotIdRequirement(I
 	auto &res = req.assert_ref_snapshot_id;
 	res.ref = "main";
 	res.snapshot_id = old_snapshot.snapshot_id;
+	res.has_snapshot_id = true;
+	res.type.value = "assert-ref-snapshot-id";
+	return req;
+}
+
+static rest_api_objects::TableRequirement CreateAssertNoSnapshotRequirement() {
+	rest_api_objects::TableRequirement req;
+	req.has_assert_ref_snapshot_id = true;
+
+	auto &res = req.assert_ref_snapshot_id;
+	res.ref = "main";
+	res.has_snapshot_id = false;
 	res.type.value = "assert-ref-snapshot-id";
 	return req;
 }
@@ -307,6 +323,10 @@ TableTransactionInfo IRCTransaction::GetTransactionRequest(ClientContext &contex
 			//! If any changes were made to the state of the table, we should assert that our parent snapshot has
 			//! not changed. We don't want to change the table location if someone has added a snapshot
 			commit_state.table_change.requirements.push_back(CreateAssertRefSnapshotIdRequirement(*current_snapshot));
+		} else if (!info.has_assert_create) {
+			//! If the table had no snapshots and isn't created by this transaction, we should assert that no snapshot
+			//! has been added in the meantime
+			commit_state.table_change.requirements.push_back(CreateAssertNoSnapshotRequirement());
 		}
 
 		transaction.table_changes.push_back(std::move(table_change));

test/sql/local/irc/insert/test_concurrent_insert_new_table.test

@@ -0,0 +1,73 @@
+# name: test/sql/local/irc/insert/test_concurrent_insert_new_table.test
+# description: Test concurrent inserts to a newly created table with no snapshots
+# group: [insert]
+
+require-env ICEBERG_SERVER_AVAILABLE
+
+require avro
+
+require parquet
+
+require iceberg
+
+require httpfs
+
+# Do not ignore 'HTTP' error messages!
+set ignore_error_messages
+
+statement ok
+CREATE SECRET (
+    TYPE S3,
+    KEY_ID 'admin',
+    SECRET 'password',
+    ENDPOINT '127.0.0.1:9000',
+    URL_STYLE 'path',
+    USE_SSL 0
+);
+
+statement ok
+ATTACH '' AS my_datalake (
+    TYPE ICEBERG,
+    CLIENT_ID 'admin',
+    CLIENT_SECRET 'password',
+    ENDPOINT 'http://127.0.0.1:8181'
+);
+
+statement ok
+DROP TABLE IF EXISTS my_datalake.default.concurrent_new_table_test;
+
+statement ok
+CREATE TABLE my_datalake.default.concurrent_new_table_test (id INTEGER, name VARCHAR);
+
+# con1 starts a transaction and inserts
+statement ok con1
+begin
+
+statement ok con1
+INSERT INTO my_datalake.default.concurrent_new_table_test VALUES (1, 'con1');
+
+# con2 starts a transaction and inserts
+statement ok con2
+begin
+
+statement ok con2
+INSERT INTO my_datalake.default.concurrent_new_table_test VALUES (2, 'con2');
+
+# con2 commits first
+statement ok con2
+commit
+
+# con1 should fail because con2 already added a snapshot
+statement error con1
+commit
+----
+<REGEX>:.*Requirement failed: branch main was created concurrently.*
+
+# only con2's data is in the table
+query II
+SELECT * FROM my_datalake.default.concurrent_new_table_test ORDER BY id;
+----
+2	con2
+
+statement ok
+DROP TABLE my_datalake.default.concurrent_new_table_test;

test/sql/local/irc/reads/test_read_nulls.test

@@ -74,7 +74,7 @@ select * from my_datalake.default.test_null_filtering where a is distinct from n
 10	no nulls
 
 query II
-select * from my_datalake.default.test_null_filtering where a is not distinct from null order by a NULLS FIRST;
+select * from my_datalake.default.test_null_filtering where a is not distinct from null order by all NULLS FIRST;
 ----
 NULL	NULL
 NULL	duck

test/sql/local/irc/test_polaris_concurrent_insert.test

@@ -0,0 +1,126 @@
+# name: test/sql/local/irc/test_polaris_concurrent_insert.test
+# description: Test concurrent inserts to a newly created table with no snapshots
+# group: [irc]
+
+require-env POLARIS_CLIENT_ID
+
+require-env POLARIS_CLIENT_SECRET
+
+require-env POLARIS_SERVER_AVAILABLE
+
+require avro
+
+require parquet
+
+require httpfs
+
+require iceberg
+
+require aws
+
+# Do not ignore 'HTTP' error messages!
+set ignore_error_messages
+
+statement ok
+create secret polaris_secret (
+	TYPE ICEBERG,
+	CLIENT_ID '${POLARIS_CLIENT_ID}',
+	CLIENT_SECRET '${POLARIS_CLIENT_SECRET}',
+	ENDPOINT 'http://0.0.0.0:8181/api/catalog'
+);
+
+statement ok
+attach 'quickstart_catalog' as my_datalake (
+	type ICEBERG,
+	ENDPOINT 'http://0.0.0.0:8181/api/catalog'
+);
+
+statement ok
+DROP TABLE IF EXISTS my_datalake.default.concurrent_new_table_test;
+
+statement ok
+CREATE TABLE my_datalake.default.concurrent_new_table_test (id INTEGER, name VARCHAR);
+
+# con1 starts a transaction and inserts
+statement ok con1
+begin
+
+statement ok con1
+INSERT INTO my_datalake.default.concurrent_new_table_test VALUES (1, 'con1');
+
+# con2 starts a transaction and inserts
+statement ok con2
+begin
+
+statement ok con2
+INSERT INTO my_datalake.default.concurrent_new_table_test VALUES (2, 'con2');
+
+# con2 commits first
+statement ok con2
+commit
+
+# con1 should fail because con2 already added a snapshot
+statement error con1
+commit
+----
+<REGEX>:.*Requirement failed: branch main was created concurrently.*
+
+# only con2's data is in the table
+query II
+SELECT * FROM my_datalake.default.concurrent_new_table_test ORDER BY id;
+----
+2	con2
+
+statement ok
+DROP TABLE my_datalake.default.concurrent_new_table_test;
+
+
+# attach with support stage create as false
+# polaris is currently file backed in tests, so folders are not automatically
+# created when creating a table with stage_create=true
+statement ok
+attach or replace 'quickstart_catalog' as my_datalake (
+	type ICEBERG,
+	ENDPOINT 'http://0.0.0.0:8181/api/catalog',
+	SUPPORT_NESTED_NAMESPACES true,
+	support_stage_create false
+);
+
+statement ok
+DROP TABLE IF EXISTS my_datalake.default.concurrent_new_table_test;
+
+statement ok
+CREATE TABLE my_datalake.default.concurrent_new_table_test (id INTEGER, name VARCHAR);
+
+# con1 starts a transaction and inserts
+statement ok con1
+begin
+
+statement ok con1
+INSERT INTO my_datalake.default.concurrent_new_table_test VALUES (1, 'con1');
+
+# con2 starts a transaction and inserts
+statement ok con2
+begin
+
+statement ok con2
+INSERT INTO my_datalake.default.concurrent_new_table_test VALUES (2, 'con2');
+
+# con2 commits first
+statement ok con2
+commit
+
+# con1 should fail because con2 already added a snapshot
+statement error con1
+commit
+----
+<REGEX>:.*Requirement failed: branch main was created concurrently.*
+
+# only con2's data is in the table
+query II
+SELECT * FROM my_datalake.default.concurrent_new_table_test ORDER BY id;
+----
+2	con2
+
+statement ok
+DROP TABLE my_datalake.default.concurrent_new_table_test;