diff --git a/.github/workflows/CloudTesting.yml b/.github/workflows/CloudTesting.yml index f8f5201..27c79c4 100644 --- a/.github/workflows/CloudTesting.yml +++ b/.github/workflows/CloudTesting.yml @@ -1,7 +1,7 @@ name: Azure (cloud) functional tests -on: +on: push: - branches: [main,v1.5-variegata,v1.4-andium] + branches: [main, v1.5-variegata, v1.4-andium] repository_dispatch: concurrency: @@ -44,10 +44,10 @@ jobs: AZURE_CLIENT_SECRET: op://testing-rw/azure/client_secret AZURE_AUTH_ENV: 1 - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - submodules: 'true' + submodules: true - name: Setup Ccache uses: hendrikmuhs/ccache-action@main @@ -84,7 +84,8 @@ jobs: - name: Test with access token (created via CLI, logged in above) run: | - export AZURE_ACCESS_TOKEN=`az account get-access-token --resource https://storage.azure.com --query accessToken --output tsv` + AZURE_ACCESS_TOKEN="$(az account get-access-token --resource https://storage.azure.com --query accessToken --output tsv)" + export AZURE_ACCESS_TOKEN scripts/env_azure env -u AZURE_CLIENT_SECRET ./build/release/test/unittest "test/sql/cloud/access_token_auth.test" - name: Clear all az credentials from disk @@ -101,7 +102,7 @@ jobs: - name: Run test data integrity check env: - DUCKDB_AZURE_PERSISTENT_SECRET_AVAILABLE: 1 + DUCKDB_AZURE_PERSISTENT_SECRET_AVAILABLE: 1 run: | scripts/env_azure ./build/release/duckdb -c "CREATE PERSISTENT SECRET az_authn_persistent (TYPE AZURE, PROVIDER SERVICE_PRINCIPAL, TENANT_ID '${{secrets.AZURE_TENANT_ID}}', CLIENT_ID '${{secrets.AZURE_CLIENT_ID}}', CLIENT_SECRET '${{secrets.AZURE_CLIENT_SECRET}}', ACCOUNT_NAME '${AZ_STORAGE_ACCOUNT}', SCOPE 'az://')" scripts/env_azure ./build/release/test/unittest "test/sql/test_data_integrity.test" diff --git a/.github/workflows/HighPriorityIssues.yml b/.github/workflows/HighPriorityIssues.yml index 921c1bc..c4c81ed 100644 --- a/.github/workflows/HighPriorityIssues.yml +++ b/.github/workflows/HighPriorityIssues.yml @@ -19,7 +19,7 @@ jobs: - name: Get mirror issue number run: | gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt - echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV + echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> "$GITHUB_ENV" - name: Print whether mirror issue exists run: | diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 99ac281..e2ec8e7 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -1,6 +1,5 @@ - name: Azurite (local) functional tests -on: [push, pull_request,repository_dispatch] +on: [push, pull_request, repository_dispatch] concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} cancel-in-progress: true @@ -8,7 +7,6 @@ defaults: run: shell: bash - env: VCPKG_COMMIT_ID: 84bab45d415d22042bd0b9081aea57f362da3f35 @@ -17,21 +15,21 @@ jobs: name: Azurite tests (Linux) runs-on: ubuntu-latest env: - VCPKG_TARGET_TRIPLET: 'x64-linux' + VCPKG_TARGET_TRIPLET: x64-linux VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake GEN: ninja - AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' - HTTP_PROXY_RUNNING: '1' + AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" + HTTP_PROXY_RUNNING: "1" # AZ protocol vars AZ_STORAGE_ACCOUNT: devstoreaccount1 AZ_DATA_DIR: testing-private AZ_TEMP_DIR: writes steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - submodules: 'true' + submodules: true - uses: actions/setup-node@v4 @@ -97,21 +95,21 @@ jobs: name: Azurite tests (MacOS) runs-on: macos-latest env: - VCPKG_TARGET_TRIPLET: 'x64-osx' - OSX_BUILD_ARCH: 'x86_64' + VCPKG_TARGET_TRIPLET: x64-osx + OSX_BUILD_ARCH: x86_64 GEN: ninja VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' + AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" # AZ protocol vars AZ_STORAGE_ACCOUNT: devstoreaccount1 AZ_DATA_DIR: testing-private AZ_TEMP_DIR: writes steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - submodules: 'true' + submodules: true - name: Install Ninja run: brew install ninja @@ -127,7 +125,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: "3.12" - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 @@ -162,21 +160,21 @@ jobs: name: Azurite tests (Windows) runs-on: windows-latest env: - VCPKG_TARGET_TRIPLET: 'x64-windows' + VCPKG_TARGET_TRIPLET: x64-windows VCPKG_ROOT: ${{ github.workspace }}\vcpkg VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake GEN: ninja - AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' + AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" # AZ protocol vars AZ_STORAGE_ACCOUNT: devstoreaccount1 AZ_DATA_DIR: testing-private AZ_TEMP_DIR: writes steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - submodules: 'true' + submodules: true - uses: actions/setup-node@v4 @@ -192,7 +190,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: "3.12" - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 @@ -206,7 +204,12 @@ jobs: env: EXT_FLAGS: -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + if exist "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvars64.bat" ( + call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + ) else ( + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + ) + move "C:\Program Files\Git\usr\bin\link.exe" "C:\Program Files\Git\usr\bin\link-git.exe" make release - name: Test Extension diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 2824230..ea513d4 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -16,10 +16,10 @@ jobs: name: Build extension binaries uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.5-variegata with: - extension_name: azure - duckdb_version: v1.5.3 - ci_tools_version: v1.5-variegata - exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw" + extension_name: &ext_name azure + duckdb_version: &duckdb_ref v1.5.4 + ci_tools_version: &ci_tools_ref v1.5-variegata + exclude_archs: &exclude_archs wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw duckdb-stable-deploy: name: Deploy extension binaries @@ -27,18 +27,19 @@ jobs: uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.5-variegata secrets: inherit with: - extension_name: azure - duckdb_version: v1.5.3 - ci_tools_version: v1.5-variegata - exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw" - deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/v1.5-variegata' }} + extension_name: *ext_name + duckdb_version: *duckdb_ref + ci_tools_version: *ci_tools_ref + exclude_archs: *exclude_archs + deploy_latest: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }} + deploy_versioned: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }} code-quality-check: name: Code Quality Check uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@v1.5-variegata with: - duckdb_version: v1.5.3 - ci_tools_version: v1.5-variegata - extension_name: azure + extension_name: *ext_name + duckdb_version: *duckdb_ref + ci_tools_version: *ci_tools_ref format_checks: format extra_toolchains: python3 diff --git a/duckdb b/duckdb index a34c90c..10603df 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit a34c90c22926670a6e0a9fa9c483b8b137080768 +Subproject commit 10603df129abb677fe79c1081888c08d1f2be361 diff --git a/extension-ci-tools b/extension-ci-tools index ec20f45..72e76e9 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit ec20f45aabeb9fcfcfa044dda249597f066d4826 +Subproject commit 72e76e99cd7fee45a99739cd118ec2db64e034ec diff --git a/src/azure_dfs_filesystem.cpp b/src/azure_dfs_filesystem.cpp index e5658d3..1f20af6 100644 --- a/src/azure_dfs_filesystem.cpp +++ b/src/azure_dfs_filesystem.cpp @@ -162,6 +162,13 @@ unique_ptr AzureDfsStorageFileSystem::CreateHandle(const OpenFi auto storage_context = GetOrCreateStorageContext(opener, info.path, parsed_url); auto file_system_client = storage_context->As().GetDfsFileSystemClient(parsed_url.container); + // A trailing '/' is only a directory hint, not part of the resource name. Some DFS endpoints (e.g. OneLake) + // reject a GetProperties on a trailing-slash path with 403 AuthenticationFailed, so strip it for the client. + auto file_path = parsed_url.path; + if (file_path.size() > 1 && file_path.back() == '/') { + file_path.pop_back(); + } + auto handle = make_uniq(*this, info, flags, storage_context->options, file_system_client.GetFileClient(parsed_url.path)); if (!handle->PostConstruct()) {