diff --git a/Dockerfile b/Dockerfile index 92cea49..bc9c0a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,11 @@ RUN apt-get update && apt-get install -y build-essential libpq-dev && rm -rf /va COPY pyproject.toml poetry.lock ./ RUN VIRTUAL_ENV=/venv poetry install --no-root --no-directory --all-extras --without dev +ARG DUCKDB_EXTENSION_REPO=https://syncandshare.desy.de/public.php/dav/files/PPGeSD8ceELYbiw + # pre-install duckdb extensions -RUN echo "import duckdb\nfor ext in ['httpfs', 'iceberg', 'avro']: duckdb.install_extension(ext); duckdb.load_extension(ext)" | /venv/bin/python +RUN echo "import duckdb\nfor ext in ['httpfs', 'avro']: duckdb.install_extension(ext); duckdb.load_extension(ext)" | /venv/bin/python +RUN echo "import duckdb; duckdb.connect(config={'allow_unsigned_extensions': 'true'}).install_extension('iceberg', repository_url='$DUCKDB_EXTENSION_REPO');" | /venv/bin/python COPY ampel ampel COPY README.md README.md diff --git a/ampel/lsst/archive/server/display.py b/ampel/lsst/archive/server/display.py index 5a2075a..3b4d4b6 100644 --- a/ampel/lsst/archive/server/display.py +++ b/ampel/lsst/archive/server/display.py @@ -236,7 +236,7 @@ async def list_nights( ): return flatten( alerts.aggregate( - [ # type: ignore[arg-type] + [ SQLExpression("diaSource.visit // 100000 as night").alias("night"), SQLExpression("count(*) as alerts"), SQLExpression("count(distinct diaSource.visit) as visits"), diff --git a/ampel/lsst/archive/server/iceberg.py b/ampel/lsst/archive/server/iceberg.py index 86fac60..b951169 100644 --- a/ampel/lsst/archive/server/iceberg.py +++ b/ampel/lsst/archive/server/iceberg.py @@ -33,8 +33,8 @@ @cache def get_duckdb() -> DuckDBPyConnection: - conn = connect() - for ext in "httpfs", "iceberg": + conn = connect(config={"allow_unsigned_extensions": "true"}) + for ext in "httpfs", "avro", "iceberg": conn.load_extension(ext) conn.execute(f""" CREATE OR REPLACE SECRET secret ( diff --git a/poetry.lock b/poetry.lock index 7f25942..7ee25d6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -473,53 +473,47 @@ websockets = ["websocket-client (>=1.3.0)"] [[package]] name = "duckdb" -version = "1.4.4" +version = "1.5.0.dev293" description = "DuckDB in-process database" optional = false -python-versions = ">=3.9.0" +python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "duckdb-1.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e870a441cb1c41d556205deb665749f26347ed13b3a247b53714f5d589596977"}, - {file = "duckdb-1.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49123b579e4a6323e65139210cd72dddc593a72d840211556b60f9703bda8526"}, - {file = "duckdb-1.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e1933fac5293fea5926b0ee75a55b8cfe7f516d867310a5b251831ab61fe62b"}, - {file = "duckdb-1.4.4-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:707530f6637e91dc4b8125260595299ec9dd157c09f5d16c4186c5988bfbd09a"}, - {file = "duckdb-1.4.4-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:453b115f4777467f35103d8081770ac2f223fb5799178db5b06186e3ab51d1f2"}, - {file = "duckdb-1.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:a3c8542db7ffb128aceb7f3b35502ebaddcd4f73f1227569306cc34bad06680c"}, - {file = "duckdb-1.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5ba684f498d4e924c7e8f30dd157da8da34c8479746c5011b6c0e037e9c60ad2"}, - {file = "duckdb-1.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5536eb952a8aa6ae56469362e344d4e6403cc945a80bc8c5c2ebdd85d85eb64b"}, - {file = "duckdb-1.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:47dd4162da6a2be59a0aef640eb08d6360df1cf83c317dcc127836daaf3b7f7c"}, - {file = "duckdb-1.4.4-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6cb357cfa3403910e79e2eb46c8e445bb1ee2fd62e9e9588c6b999df4256abc1"}, - {file = "duckdb-1.4.4-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c25d5b0febda02b7944e94fdae95aecf952797afc8cb920f677b46a7c251955"}, - {file = "duckdb-1.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:6703dd1bb650025b3771552333d305d62ddd7ff182de121483d4e042ea6e2e00"}, - {file = "duckdb-1.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:bf138201f56e5d6fc276a25138341b3523e2f84733613fc43f02c54465619a95"}, - {file = "duckdb-1.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ddcfd9c6ff234da603a1edd5fd8ae6107f4d042f74951b65f91bc5e2643856b3"}, - {file = "duckdb-1.4.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6792ca647216bd5c4ff16396e4591cfa9b4a72e5ad7cdd312cec6d67e8431a7c"}, - {file = "duckdb-1.4.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1f8d55843cc940e36261689054f7dfb6ce35b1f5b0953b0d355b6adb654b0d52"}, - {file = "duckdb-1.4.4-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c65d15c440c31e06baaebfd2c06d71ce877e132779d309f1edf0a85d23c07e92"}, - {file = "duckdb-1.4.4-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b297eff642503fd435a9de5a9cb7db4eccb6f61d61a55b30d2636023f149855f"}, - {file = "duckdb-1.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:d525de5f282b03aa8be6db86b1abffdceae5f1055113a03d5b50cd2fb8cf2ef8"}, - {file = "duckdb-1.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:50f2eb173c573811b44aba51176da7a4e5c487113982be6a6a1c37337ec5fa57"}, - {file = "duckdb-1.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:337f8b24e89bc2e12dadcfe87b4eb1c00fd920f68ab07bc9b70960d6523b8bc3"}, - {file = "duckdb-1.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0509b39ea7af8cff0198a99d206dca753c62844adab54e545984c2e2c1381616"}, - {file = "duckdb-1.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fb94de6d023de9d79b7edc1ae07ee1d0b4f5fa8a9dcec799650b5befdf7aafec"}, - {file = "duckdb-1.4.4-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d636ceda422e7babd5e2f7275f6a0d1a3405e6a01873f00d38b72118d30c10b"}, - {file = "duckdb-1.4.4-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7df7351328ffb812a4a289732f500d621e7de9942a3a2c9b6d4afcf4c0e72526"}, - {file = "duckdb-1.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:6fb1225a9ea5877421481d59a6c556a9532c32c16c7ae6ca8d127e2b878c9389"}, - {file = "duckdb-1.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:f28a18cc790217e5b347bb91b2cab27aafc557c58d3d8382e04b4fe55d0c3f66"}, - {file = "duckdb-1.4.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:25874f8b1355e96178079e37312c3ba6d61a2354f51319dae860cf21335c3a20"}, - {file = "duckdb-1.4.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:452c5b5d6c349dc5d1154eb2062ee547296fcbd0c20e9df1ed00b5e1809089da"}, - {file = "duckdb-1.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8e5c2d8a0452df55e092959c0bfc8ab8897ac3ea0f754cb3b0ab3e165cd79aff"}, - {file = "duckdb-1.4.4-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1af6e76fe8bd24875dc56dd8e38300d64dc708cd2e772f67b9fbc635cc3066a3"}, - {file = "duckdb-1.4.4-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0440f59e0cd9936a9ebfcf7a13312eda480c79214ffed3878d75947fc3b7d6d"}, - {file = "duckdb-1.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:59c8d76016dde854beab844935b1ec31de358d4053e792988108e995b18c08e7"}, - {file = "duckdb-1.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:53cd6423136ab44383ec9955aefe7599b3fb3dd1fe006161e6396d8167e0e0d4"}, - {file = "duckdb-1.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8097201bc5fd0779d7fcc2f3f4736c349197235f4cb7171622936343a1aa8dbf"}, - {file = "duckdb-1.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cd1be3d48577f5b40eb9706c6b2ae10edfe18e78eb28e31a3b922dcff1183597"}, - {file = "duckdb-1.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e041f2fbd6888da090eca96ac167a7eb62d02f778385dd9155ed859f1c6b6dc8"}, - {file = "duckdb-1.4.4-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7eec0bf271ac622e57b7f6554a27a6e7d1dd2f43d1871f7962c74bcbbede15ba"}, - {file = "duckdb-1.4.4-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5cdc4126ec925edf3112bc656ac9ed23745294b854935fa7a643a216e4455af6"}, - {file = "duckdb-1.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:c9566a4ed834ec7999db5849f53da0a7ee83d86830c33f471bf0211a1148ca12"}, - {file = "duckdb-1.4.4.tar.gz", hash = "sha256:8bba52fd2acb67668a4615ee17ee51814124223de836d9e2fdcbc4c9021b3d3c"}, + {file = "duckdb-1.5.0.dev293-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9f7c2022c1ed0a4e69ef82f193e1040f80587ab9fdbea29967205f4286db77a5"}, + {file = "duckdb-1.5.0.dev293-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ed04f2a767a1f878ad73fefb389a6b143e81c3f4189394f0de0c62bbaa173c0a"}, + {file = "duckdb-1.5.0.dev293-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cbaf0f83b6827368f494a91377349640d6013ffc2272f7135f9f479fb9c3125d"}, + {file = "duckdb-1.5.0.dev293-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48d5c0ffe1bfbd1047bdf2f2b380a2fe35312a9a8d9016d109b780d55857040a"}, + {file = "duckdb-1.5.0.dev293-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4a28cbd18254fe072651c5d6c97c3208faa2cebe92a6238693643210519dc276"}, + {file = "duckdb-1.5.0.dev293-cp310-cp310-win_amd64.whl", hash = "sha256:ef68484d5ae06280d515b99f9963122e1f8a575fee8e490fb93a47ed95037bad"}, + {file = "duckdb-1.5.0.dev293-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f6a6c4edfb13b410dc9e0b9e6cf4cf9d212b6565f9ae5abbac763e4fa7910c59"}, + {file = "duckdb-1.5.0.dev293-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8c90f9274182d4662f616f20ab91172686d54a92ef6e8972eba67c719316680d"}, + {file = "duckdb-1.5.0.dev293-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f88e58ebecaf47b43fa798313922e7fd945f2b802de33d2777ec281ad01c83ad"}, + {file = "duckdb-1.5.0.dev293-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2fcc29864a1a306298fcae451ed1c3a0a40e3517bc1630492ee61bff8f9f02f"}, + {file = "duckdb-1.5.0.dev293-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7ad2dba948325922e4ec3f96870eec01ff3dcce9d3253d9b8e545a6c6fad28b"}, + {file = "duckdb-1.5.0.dev293-cp311-cp311-win_amd64.whl", hash = "sha256:3202df747b18cb1c224d6dbe083d8d497975ed35026d8f1a3e9338d9a53327c9"}, + {file = "duckdb-1.5.0.dev293-cp311-cp311-win_arm64.whl", hash = "sha256:96ad0b9a5fe7001fcb467278698c2035dadb75cbbde0755c1ab7cf544b130df6"}, + {file = "duckdb-1.5.0.dev293-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5fa078e947941bfd05fe5ca50137b8455caa6213a868170a8021055ad517d6af"}, + {file = "duckdb-1.5.0.dev293-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:46ae1f1cb9fe3aa38c4f675c6759286f303528bc803dbb2dfa3560455af0ee53"}, + {file = "duckdb-1.5.0.dev293-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4047c68f7513f39937f339aece69067d679698e661ba4af5c54a4c3ac74ec405"}, + {file = "duckdb-1.5.0.dev293-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e01086f4128c9d56d95e1237ad5269e119d51703da2505e81125c1afeb13607"}, + {file = "duckdb-1.5.0.dev293-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a9696219e220ca31012fad7db1fc5bf6f7a3831f9e25dfc412e769bcf0779ce"}, + {file = "duckdb-1.5.0.dev293-cp312-cp312-win_amd64.whl", hash = "sha256:bd0fa69b04866c7d66382a7cf4060511f868b710414bbbb0843e7ae84cc0543a"}, + {file = "duckdb-1.5.0.dev293-cp312-cp312-win_arm64.whl", hash = "sha256:b633d8cd2467ffd38eb9a912ecc0fc863eb0d168383a41271c02dca35998731e"}, + {file = "duckdb-1.5.0.dev293-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:da46b082180a1204154379f1e848497364e4b97245264e2f07e7fb26f096ac01"}, + {file = "duckdb-1.5.0.dev293-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:db201c04227e0277cdf46fb208f849901ef0c992c247109d7668af9531096623"}, + {file = "duckdb-1.5.0.dev293-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aaf73a66cd2dca228fa2c186638af7b63a55572b6560849283a1a583d0e6b442"}, + {file = "duckdb-1.5.0.dev293-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34c3760d7baf59f20e06167b8bb460acb54de37bccda68cee9111fa07f45d4e5"}, + {file = "duckdb-1.5.0.dev293-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aac789fcb70a3e2bb7b79ccbd77969b2f1579878c37e8d7b32b8b7747dd809c"}, + {file = "duckdb-1.5.0.dev293-cp313-cp313-win_amd64.whl", hash = "sha256:59ab51e2dd3acf3ebc308be8cbbfa4314a91c1050352d1aa83d4f2e6196e9ffb"}, + {file = "duckdb-1.5.0.dev293-cp313-cp313-win_arm64.whl", hash = "sha256:2283255257e528975910cee2b904679ba76fa88f174d308917cffb379d2fd235"}, + {file = "duckdb-1.5.0.dev293-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:27fdbf28e186a54d1d4f926b468a530413f0edef55e5f6e04e61a09b1d0c185a"}, + {file = "duckdb-1.5.0.dev293-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:81a1305f9c65aefffb0c120ff1774c658c2e5cc1058ae1764a0a20185a410bb3"}, + {file = "duckdb-1.5.0.dev293-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca646cd52dae21732328693ddbf76af99c28a6be3f82d5b4c029a70eaa5056e1"}, + {file = "duckdb-1.5.0.dev293-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:877d90dffe05842151c749cea4b037a9f2b61e3cdede14b330e92f638db0d90d"}, + {file = "duckdb-1.5.0.dev293-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0669ada7b102ac073095eea3938a931d74f526cbc079211a6117a4163fcc3125"}, + {file = "duckdb-1.5.0.dev293-cp314-cp314-win_amd64.whl", hash = "sha256:a1a0e741aaf648f687ec160da087c957e16dc86d46bae70feb3a1ceb68d606ea"}, + {file = "duckdb-1.5.0.dev293-cp314-cp314-win_arm64.whl", hash = "sha256:f42b59480d2f2f7d4f2ce802ece664d943d56cbf970992adc162a39dd7625433"}, + {file = "duckdb-1.5.0.dev293.tar.gz", hash = "sha256:e252981f82370382a4751adaec3faf96eead08e42c8f555f34615abe5034c820"}, ] [package.extras] @@ -2669,4 +2663,4 @@ pyiceberg = ["pyiceberg"] [metadata] lock-version = "2.1" python-versions = ">=3.12, <4" -content-hash = "580e22819c0e3c463725ac792c1624adc6a44e25e57d4e8d6aeef314e5ed70bc" +content-hash = "5b403845580b4bc3432132f94a99f67c476cc0bebda4a11f0478ab410bbc75b3" diff --git a/pyproject.toml b/pyproject.toml index 78067fb..2be859b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,8 @@ dependencies = [ "hishel (>=1.1.9,<2.0.0)", "httpx (>=0.28.1,<0.29.0)", "prometheus-fastapi-instrumentator (>=7.1.0,<8.0.0)", - "duckdb (>=1.4.4,<2.0.0)", + # exactly same version as in duckdb-iceberg submodule + "duckdb (==1.5.0.dev293)", "orjson (>=3.11.7,<4.0.0)" ] diff --git a/tests/fixtures.py b/tests/fixtures.py index 25ef367..41ade3c 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -85,7 +85,15 @@ def _alert_table(catalog, warehouse_dir: Path): p.mkdir() # "On some systems, mode is ignored" p.chmod(0o777) - cursor = duckdb.connect() + cursor = duckdb.connect(config={"allow_unsigned_extensions": "true"}) + for ext in "httpfs", "avro": + cursor.install_extension(ext) + cursor.load_extension(ext) + cursor.install_extension( + "iceberg", + repository_url="https://syncandshare.desy.de/public.php/dav/files/PPGeSD8ceELYbiw", + ) + cursor.load_extension("iceberg") cursor.execute(f""" ATTACH 'warehouse' AS iceberg_catalog( TYPE iceberg, AUTHORIZATION_TYPE none,