Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions .github/workflows/CloudTesting.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: Azure (cloud) functional tests
on:
on:
push:
branches: [main,v1.5-variegata,v1.4-andium]
branches: [main, v1.5-variegata, v1.4-andium]
repository_dispatch:

concurrency:
Expand Down Expand Up @@ -44,10 +44,10 @@ jobs:
AZURE_CLIENT_SECRET: op://testing-rw/azure/client_secret
AZURE_AUTH_ENV: 1

- uses: actions/checkout@v4
- uses: actions/checkout@v6
with:
fetch-depth: 0
submodules: 'true'
submodules: true

- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
Expand Down Expand Up @@ -84,7 +84,8 @@ jobs:

- name: Test with access token (created via CLI, logged in above)
run: |
export AZURE_ACCESS_TOKEN=`az account get-access-token --resource https://storage.azure.com --query accessToken --output tsv`
AZURE_ACCESS_TOKEN="$(az account get-access-token --resource https://storage.azure.com --query accessToken --output tsv)"
export AZURE_ACCESS_TOKEN
scripts/env_azure env -u AZURE_CLIENT_SECRET ./build/release/test/unittest "test/sql/cloud/access_token_auth.test"

- name: Clear all az credentials from disk
Expand All @@ -101,7 +102,7 @@ jobs:

- name: Run test data integrity check
env:
DUCKDB_AZURE_PERSISTENT_SECRET_AVAILABLE: 1
DUCKDB_AZURE_PERSISTENT_SECRET_AVAILABLE: 1
run: |
scripts/env_azure ./build/release/duckdb -c "CREATE PERSISTENT SECRET az_authn_persistent (TYPE AZURE, PROVIDER SERVICE_PRINCIPAL, TENANT_ID '${{secrets.AZURE_TENANT_ID}}', CLIENT_ID '${{secrets.AZURE_CLIENT_ID}}', CLIENT_SECRET '${{secrets.AZURE_CLIENT_SECRET}}', ACCOUNT_NAME '${AZ_STORAGE_ACCOUNT}', SCOPE 'az://')"
scripts/env_azure ./build/release/test/unittest "test/sql/test_data_integrity.test"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/HighPriorityIssues.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
- name: Get mirror issue number
run: |
gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV
echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> "$GITHUB_ENV"

- name: Print whether mirror issue exists
run: |
Expand Down
43 changes: 23 additions & 20 deletions .github/workflows/LocalTesting.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@

name: Azurite (local) functional tests
on: [push, pull_request,repository_dispatch]
on: [push, pull_request, repository_dispatch]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
cancel-in-progress: true
defaults:
run:
shell: bash


env:
VCPKG_COMMIT_ID: 84bab45d415d22042bd0b9081aea57f362da3f35

Expand All @@ -17,21 +15,21 @@ jobs:
name: Azurite tests (Linux)
runs-on: ubuntu-latest
env:
VCPKG_TARGET_TRIPLET: 'x64-linux'
VCPKG_TARGET_TRIPLET: x64-linux
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
GEN: ninja
AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;'
HTTP_PROXY_RUNNING: '1'
AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
HTTP_PROXY_RUNNING: "1"
# AZ protocol vars
AZ_STORAGE_ACCOUNT: devstoreaccount1
AZ_DATA_DIR: testing-private
AZ_TEMP_DIR: writes

steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
with:
fetch-depth: 0
submodules: 'true'
submodules: true

- uses: actions/setup-node@v4

Expand Down Expand Up @@ -97,21 +95,21 @@ jobs:
name: Azurite tests (MacOS)
runs-on: macos-latest
env:
VCPKG_TARGET_TRIPLET: 'x64-osx'
OSX_BUILD_ARCH: 'x86_64'
VCPKG_TARGET_TRIPLET: x64-osx
OSX_BUILD_ARCH: x86_64
GEN: ninja
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;'
AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
# AZ protocol vars
AZ_STORAGE_ACCOUNT: devstoreaccount1
AZ_DATA_DIR: testing-private
AZ_TEMP_DIR: writes

steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
with:
fetch-depth: 0
submodules: 'true'
submodules: true

- name: Install Ninja
run: brew install ninja
Expand All @@ -127,7 +125,7 @@ jobs:

- uses: actions/setup-python@v5
with:
python-version: '3.12'
python-version: "3.12"

- name: Setup vcpkg
uses: lukka/run-vcpkg@v11.1
Expand Down Expand Up @@ -162,21 +160,21 @@ jobs:
name: Azurite tests (Windows)
runs-on: windows-latest
env:
VCPKG_TARGET_TRIPLET: 'x64-windows'
VCPKG_TARGET_TRIPLET: x64-windows
VCPKG_ROOT: ${{ github.workspace }}\vcpkg
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake
GEN: ninja
AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;'
AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
# AZ protocol vars
AZ_STORAGE_ACCOUNT: devstoreaccount1
AZ_DATA_DIR: testing-private
AZ_TEMP_DIR: writes

steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
with:
fetch-depth: 0
submodules: 'true'
submodules: true

- uses: actions/setup-node@v4

Expand All @@ -192,7 +190,7 @@ jobs:

- uses: actions/setup-python@v5
with:
python-version: '3.12'
python-version: "3.12"

- name: Setup vcpkg
uses: lukka/run-vcpkg@v11.1
Expand All @@ -206,7 +204,12 @@ jobs:
env:
EXT_FLAGS: -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
if exist "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvars64.bat" (
call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
) else (
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
)
move "C:\Program Files\Git\usr\bin\link.exe" "C:\Program Files\Git\usr\bin\link-git.exe"
make release

- name: Test Extension
Expand Down
25 changes: 13 additions & 12 deletions .github/workflows/MainDistributionPipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,30 @@ jobs:
name: Build extension binaries
uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.5-variegata
with:
extension_name: azure
duckdb_version: v1.5.3
ci_tools_version: v1.5-variegata
exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw"
extension_name: &ext_name azure
duckdb_version: &duckdb_ref v1.5.4
ci_tools_version: &ci_tools_ref v1.5-variegata
exclude_archs: &exclude_archs wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw

duckdb-stable-deploy:
name: Deploy extension binaries
needs: duckdb-stable-build
uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.5-variegata
secrets: inherit
with:
extension_name: azure
duckdb_version: v1.5.3
ci_tools_version: v1.5-variegata
exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw"
deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/v1.5-variegata' }}
extension_name: *ext_name
duckdb_version: *duckdb_ref
ci_tools_version: *ci_tools_ref
exclude_archs: *exclude_archs
deploy_latest: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }}
deploy_versioned: ${{ startsWith(github.ref, 'refs/heads/v') || github.ref == 'refs/heads/main' }}

code-quality-check:
name: Code Quality Check
uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@v1.5-variegata
with:
duckdb_version: v1.5.3
ci_tools_version: v1.5-variegata
extension_name: azure
extension_name: *ext_name
duckdb_version: *duckdb_ref
ci_tools_version: *ci_tools_ref
format_checks: format
extra_toolchains: python3
2 changes: 1 addition & 1 deletion duckdb
Submodule duckdb updated 685 files
7 changes: 7 additions & 0 deletions src/azure_dfs_filesystem.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,13 @@ unique_ptr<AzureFileHandle> AzureDfsStorageFileSystem::CreateHandle(const OpenFi
auto storage_context = GetOrCreateStorageContext(opener, info.path, parsed_url);
auto file_system_client = storage_context->As<AzureDfsContextState>().GetDfsFileSystemClient(parsed_url.container);

// A trailing '/' is only a directory hint, not part of the resource name. Some DFS endpoints (e.g. OneLake)
// reject a GetProperties on a trailing-slash path with 403 AuthenticationFailed, so strip it for the client.
auto file_path = parsed_url.path;
if (file_path.size() > 1 && file_path.back() == '/') {
file_path.pop_back();
}

auto handle = make_uniq<AzureDfsStorageFileHandle>(*this, info, flags, storage_context->options,
file_system_client.GetFileClient(parsed_url.path));
if (!handle->PostConstruct()) {
Expand Down
Loading