diff --git a/.artifactignore b/.artifactignore index 1126a160d..614233cd3 100644 --- a/.artifactignore +++ b/.artifactignore @@ -1,2 +1,9 @@ **/* !*.deb +# Also publish build-env/ so downstream consumers (sonic-sairedis, sonic-swss) +# can cascade this repo's dependency declarations: buildenv_setup downloads this +# artifact and reads build-env/upstream-artifacts.yaml + build-env/packages/*.yaml +# from it. Both the directory and its contents must be un-ignored (the leading +# **/* ignores the directory entry too). +!build-env +!build-env/** diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 317c87e8f..9c0bd477e 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -57,129 +57,49 @@ jobs: clean: true - script: | set -ex + # Bootstrap the tool's own runtime deps (buildenv_setup needs PyYAML + requests). sudo apt-get update - sudo apt-get install -qq -y \ - libhiredis-dev \ - libnl-3-dev \ - libnl-genl-3-dev \ - libnl-route-3-dev \ - libnl-nf-3-dev \ - swig - displayName: "Install dependencies" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - ${{ if eq(parameters.arch, 'amd64') }}: - artifact: common-lib - ${{ else }}: - artifact: common-lib.${{ parameters.arch }} - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb - target/debs/${{ parameters.debian_version }}/python3-libyang*.deb - displayName: "Download libyang from ${{ parameters.arch }} common lib" - condition: ne('${{ parameters.debian_version }}', 'trixie') - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - ${{ if eq(parameters.arch, 'amd64') }}: - artifact: common-lib - ${{ else }}: - artifact: common-lib.${{ parameters.arch }} - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb - target/debs/${{ parameters.debian_version }}/python3-libyang*.deb - target/debs/${{ parameters.debian_version }}/libpcre*.deb - displayName: "Download libyang from ${{ parameters.arch }} common lib" - condition: eq('${{ parameters.debian_version }}', 'trixie') - - script: | - set -ex - sudo dpkg -i $(find ./download -name *.deb) - # common-lib doesn't publish python3-libyang (LIBYANG3_PY3 isn't in - # slave.mk's lib-packages), so install the CFFI bindings from PyPI; - # they link against the libyang3 we just installed. - sudo pip3 install --no-build-isolation 'libyang==3.3.0' - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install libyang from common lib" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 142 - artifact: sonic-buildimage.vs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - patterns: | - target/python-wheels/${{ parameters.debian_version }}/sonic_yang_mgmt-1.0-py3-none-any.whl - target/python-wheels/${{ parameters.debian_version }}/sonic_yang_models-1.0-py3-none-any.whl - displayName: "Download yang wheel from latest sonic-buildimage build" - condition: ne('${{ parameters.debian_version }}', 'trixie') - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 142 - artifact: sonic-buildimage.vs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - patterns: | - target/python-wheels/bookworm/sonic_yang_mgmt-1.0-py3-none-any.whl - target/python-wheels/bookworm/sonic_yang_models-1.0-py3-none-any.whl - displayName: "Download yang wheel from latest sonic-buildimage build" - condition: eq('${{ parameters.debian_version }}', 'trixie') - - script: | - set -ex - sudo pip3 install ./download/target/python-wheels/${{ parameters.debian_version }}/sonic_yang_mgmt-1.0-py3-none-any.whl \ - ./download/target/python-wheels/${{ parameters.debian_version }}/sonic_yang_models-1.0-py3-none-any.whl - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install yang wheel from common lib" - condition: ne('${{ parameters.debian_version }}', 'trixie') - - script: | - set -ex - sudo pip3 install ./download/target/python-wheels/bookworm/sonic_yang_mgmt-1.0-py3-none-any.whl \ - ./download/target/python-wheels/bookworm/sonic_yang_models-1.0-py3-none-any.whl - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install yang wheel from common lib" - condition: eq('${{ parameters.debian_version }}', 'trixie') - - script: | - set -ex - rm ../*.deb || true - ./autogen.sh - DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' DEB_CXXFLAGS_APPEND="-coverage -fprofile-abs-path" DEB_LDFLAGS_APPEND="-coverage -fprofile-abs-path" dpkg-buildpackage -Pnopython2 -us -uc -b -j$(nproc) - mv ../*.deb . + sudo apt-get install -qq -y python3-yaml python3-requests + # buildenv_setup lives in this repo's ci/ (sonic-swss-common hosts the shared + # tool), so run it straight from the checkout -- no clone needed here. + # It installs apt/pip deps (libnl comes from stock apt, per base.yaml), + # downloads + installs the libyang3 DEBs and the sonic-yang wheels from + # common-lib (see upstream-artifacts.yaml), and runs the yang-models + # post-install hook. The redis test-config hook is test-scoped, so this Build + # job does NOT run it here; it is applied inline later, after cargo test. + PYTHONPATH=$(Build.SourcesDirectory)/ci python3 -m buildenv_setup \ + --repo-dir $(Build.SourcesDirectory) \ + --scope build \ + --arch ${{ parameters.arch }} \ + --debian-version ${{ parameters.debian_version }} \ + --branch $(BUILD_BRANCH) + displayName: "Set up build environment (buildenv_setup)" + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - script: ./build-env/build.sh displayName: "Compile sonic swss common with coverage enabled" - ${{ if eq(parameters.run_unit_test, true) }}: - script: | set -ex - sudo pip install Pympler==0.8 pytest - sudo apt-get install -y redis-server sudo dpkg -i libswsscommon_*.deb sudo dpkg -i libswsscommon-dev_*.deb sudo dpkg -i python3-swsscommon_*.deb ps aux + # Run the Rust tests BEFORE bringing up the unixsocket redis. redis-server is + # installed (base.yaml) and running on TCP only at this point — the SONiC + # database_config unixsocket is NOT up yet — so the Rust + # `logger_init_without_redis` test (crates/swss-common/tests/logger_fallback.rs) + # correctly sees redis as unreachable and passes. Redis-dependent Rust tests + # spawn their own ephemeral redis via swss-common-testing, so they are fine. cargo test --workspace --all-features cargo test --release --workspace --all-features - sudo sed -i 's/notify-keyspace-events ""/notify-keyspace-events AKE/' /etc/redis/redis.conf - sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf - sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf - sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf - sudo service redis-server restart - sudo mkdir -p /usr/local/yang-models + # Bring up the unixsocket redis for the C++ / pytest tests below. This is the + # same shared script buildenv_setup runs on the VS test host (via the + # configure-redis-for-tests post_install, scope test); the Build stage runs it + # INLINE here because it must happen AFTER cargo, not at build-env setup time. + ./build-env/configure-redis-for-tests.sh ./tests/tests redis-cli FLUSHALL @@ -189,8 +109,6 @@ jobs: make -C goext redis-cli FLUSHALL make -C goext check - - rm -rf $(Build.ArtifactStagingDirectory)/download displayName: "Run swss common unit tests" - publish: $(System.DefaultWorkingDirectory)/ artifact: ${{ parameters.artifact_name }} diff --git a/.azure-pipelines/build-ubuntu-template.yml b/.azure-pipelines/build-ubuntu-template.yml new file mode 100644 index 000000000..59b55d663 --- /dev/null +++ b/.azure-pipelines/build-ubuntu-template.yml @@ -0,0 +1,59 @@ +parameters: +- name: debian_version + type: string + default: bookworm + +jobs: +- job: + displayName: "amd64/ubuntu-22.04" + pool: + vmImage: 'ubuntu-22.04' + + steps: + - script: | + set -ex + # Bootstrap the tool's own runtime deps (buildenv_setup needs PyYAML + requests). + sudo apt-get update + sudo apt-get install -qq -y python3-yaml python3-requests + # buildenv_setup lives in this repo's ci/ (sonic-swss-common hosts the shared + # tool), so run it straight from the checkout. --host-os ubuntu-22.04 selects + # the jammy package variants (swig4.0 + the vmImage build toolchain that the + # sonic-slave-* containers bake in); --debian-version selects which + # common-libs / sonic-buildimage artifacts to fetch (bookworm DEBs/wheels + # installed on jammy, as before). It apt/pip-installs deps (libnl comes from + # stock apt, per base.yaml), downloads + installs the libyang3 DEBs and the + # sonic-yang wheels from common-lib, and runs the yang-models post-install + # hook. The redis test-config hook is test-scoped, so it is not run by this + # Build job (which runs the Bazel test suite, not pytest). + PYTHONPATH=$(Build.SourcesDirectory)/ci python3 -m buildenv_setup \ + --repo-dir $(Build.SourcesDirectory) \ + --scope build \ + --host-os ubuntu-22.04 \ + --debian-version ${{ parameters.debian_version }} \ + --branch $(BUILD_BRANCH) + displayName: "Set up build environment (buildenv_setup)" + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - script: | + set -ex + # Special-case CLI-driven installs that don't fit the packages/*.yaml schema + # (see design doc): compile the gtest sources shipped by libgtest-dev, and + # install the bazelisk binary (provides `bazel`). libgtest-dev/libgmock-dev + + # cmake are apt-installed by buildenv_setup (packages/tooling.yaml, jammy). + cd /usr/src/gtest && sudo cmake . && sudo make + ARCH=$(dpkg --print-architecture) + sudo curl -fsSL -o /usr/local/bin/bazel \ + https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-${ARCH} + sudo chmod 755 /usr/local/bin/bazel + displayName: "Compile gtest + install bazelisk" + - script: | + ./autogen.sh + dpkg-buildpackage -us -uc -Pnopython2 -b -j$(nproc) && cp ../*.deb . + displayName: "Compile sonic swss common" + - script: | + bazel build //... + bazel test //... + displayName: "Compile and test all Bazel targets" + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-swss-common.amd64.ubuntu22_04 + displayName: "Archive swss common debian packages" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 942ddfe7c..863a0dd69 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -52,88 +52,9 @@ stages: - stage: Build jobs: - - job: - displayName: "amd64/ubuntu-22.04" - pool: - vmImage: 'ubuntu-22.04' - - steps: - - script: | - sudo apt-get update - sudo apt-get install -y make libtool m4 autoconf dh-exec debhelper cmake pkg-config nlohmann-json3-dev \ - libhiredis-dev libnl-3-dev libnl-genl-3-dev libnl-route-3-dev libnl-nf-3-dev swig4.0 \ - libpython3-dev libboost-dev libboost-serialization-dev uuid-dev libzmq3-dev - sudo apt-get install -y sudo - sudo apt-get install -y redis-server redis-tools - sudo apt-get install -y python3-pip - sudo pip3 install pytest - sudo apt-get install -y python - sudo apt-get install cmake libgtest-dev libgmock-dev - cd /usr/src/gtest && sudo cmake . && sudo make - ARCH=$(dpkg --print-architecture) - set -x - sudo curl -fsSL -o /usr/local/bin/bazel \ - https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-${ARCH} - sudo chmod 755 /usr/local/bin/bazel - displayName: "Install dependencies" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb - displayName: "Download yang deb from amd64 common lib" - - script: | - set -ex - sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang3_*.deb \ - $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb - # python3-libyang's bookworm .deb pins python3 (>= 3.11~, << 3.12) and - # won't install on Ubuntu 22.04 (python 3.10). Build the Python bindings - # from PyPI instead — they link against the libyang3 we just installed. - # Use --no-build-isolation with apt's python3-cffi to avoid jammy pip - # 22.0.2 picking up cffi 2.0.0 in its build env while Debian's older - # /usr/lib/python3/dist-packages/_cffi_backend.so still wins on sys.path - # (causing a "Version mismatch" Exception inside cffi at build time). - sudo apt-get install -y python3-cffi - sudo pip3 install --no-build-isolation 'libyang==3.3.0' - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install yang deb from common lib" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 142 - artifact: sonic-buildimage.vs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - patterns: | - target/python-wheels/${{ parameters.debian_version }}/sonic_yang_mgmt-1.0-py3-none-any.whl - target/python-wheels/${{ parameters.debian_version }}/sonic_yang_models-1.0-py3-none-any.whl - displayName: "Download yang wheel from latest sonic-buildimage build" - - script: | - set -ex - sudo pip3 install ./download/target/python-wheels/${{ parameters.debian_version }}/sonic_yang_mgmt-1.0-py3-none-any.whl \ - ./download/target/python-wheels/${{ parameters.debian_version }}/sonic_yang_models-1.0-py3-none-any.whl - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install yang wheel" - - script: | - ./autogen.sh - dpkg-buildpackage -us -uc -Pnopython2 -b -j$(nproc) && cp ../*.deb . - displayName: "Compile sonic swss common" - - script: | - bazel build //... - bazel test //... - displayName: "Compile and test all Bazel targets" - - publish: $(System.DefaultWorkingDirectory)/ - artifact: sonic-swss-common.amd64.ubuntu22_04 - displayName: "Archive swss common debian packages" + - template: .azure-pipelines/build-ubuntu-template.yml + parameters: + debian_version: ${{ parameters.debian_version }} - template: .azure-pipelines/build-template.yml parameters: @@ -236,3 +157,27 @@ stages: parameters: log_artifact_name: log debian_version: ${{ parameters.debian_version }} + +- stage: BuildenvSetupUnitTests + # Runs the ci/buildenv_setup unit tests. Independent (dependsOn: []) so it runs in + # parallel and gates nothing else; its own result still contributes to the pipeline. + dependsOn: [] + jobs: + - job: + displayName: "buildenv_setup unit tests" + pool: + vmImage: 'ubuntu-22.04' + steps: + - script: | + sudo pip3 install pytest pyyaml requests + displayName: "Install test dependencies" + - script: | + PYTHONPATH=. python3 -m pytest tests/ -v --junitxml=$(Build.ArtifactStagingDirectory)/buildenv_setup-tests.xml + workingDirectory: ci + displayName: "Run buildenv_setup unit tests" + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFormat: JUnit + testResultsFiles: '$(Build.ArtifactStagingDirectory)/buildenv_setup-tests.xml' + testRunTitle: 'buildenv_setup unit tests' diff --git a/build-env/Dockerfile b/build-env/Dockerfile new file mode 100644 index 000000000..befce4f38 --- /dev/null +++ b/build-env/Dockerfile @@ -0,0 +1,35 @@ +# Local-dev image for building sonic-swss-common. +# +# CI does NOT use this file: CI runs inside `container: sonic-slave-*` and invokes +# buildenv_setup directly (see .azure-pipelines/build-template.yml). This exists +# so a developer can reproduce the CI build environment locally. +# +# Layer ordering (design finding F7): copy ONLY the dependency declarations + the +# buildenv_setup tool, run setup, and do NOT copy the source. build-env/compose.yaml +# mounts the working tree at /workspace at runtime, so editing source never +# invalidates the (heavy) dependency-setup layer below. + +ARG DEBIAN_VERSION=bookworm +FROM sonicdev-microsoft.azurecr.io:443/sonic-slave-${DEBIAN_VERSION}:latest + +ARG DEBIAN_VERSION=bookworm +ARG BUILD_BRANCH=master + +# The buildenv_setup tool (lives in this repo's ci/) + the dep declarations only. +COPY ci/ /opt/buildenv/ci/ +COPY build-env/ /workspace/build-env/ + +# Bootstrap the tool's own runtime deps, then set up the build environment. +# --org-url points at the public SONiC Azure DevOps org for artifact download +# (in CI this comes from $SYSTEM_COLLECTIONURI instead). Set AZURE_DEVOPS_EXT_PAT, +# or use --upstream-staged-dir, if your environment needs auth. +RUN apt-get update && apt-get install -y python3-yaml python3-requests \ + && PYTHONPATH=/opt/buildenv/ci python3 -m buildenv_setup \ + --repo-dir /workspace \ + --scope build \ + --debian-version "${DEBIAN_VERSION}" \ + --branch "${BUILD_BRANCH}" \ + --org-url https://dev.azure.com/mssonic \ + --no-sudo + +WORKDIR /workspace diff --git a/build-env/README.md b/build-env/README.md new file mode 100644 index 000000000..dc52536bc --- /dev/null +++ b/build-env/README.md @@ -0,0 +1,56 @@ +# build-env/ + +Declarative build-environment configuration for sonic-swss-common, consumed by +the shared [`buildenv_setup`](../ci/README.md) tool. This is the single source of +truth for "how to build this repo": what dependencies it needs, which upstream +artifacts it consumes, and the build command itself. + +## Contents + +| Path | Purpose | Cascades to downstream? | +|------|---------|-------------------------| +| `packages/base.yaml` | apt/pip packages needed to build **and link against** libswsscommon; redis + its test-config | **Yes** | +| `packages/tooling.yaml` | Build-stage-only tooling (python test deps, yang-models dir) | No | +| `upstream-artifacts.yaml` | upstream DEBs/wheels to fetch (libyang from common-libs; sonic-yang wheels from sonic-buildimage) | Yes | +| `configure-redis-for-tests.sh` | redis test-config, run via a `post_install` hook | (travels with base.yaml) | +| `build.sh` | canonical build command (autogen + dpkg-buildpackage), used by CI **and** local dev | — | +| `Dockerfile`, `compose.yaml` | local-dev image (CI does not use these) | — | + +> **Cascade limitation:** a cascaded `base.yaml` may declare `packages` and +> `post_install`, but **not** `apt_sources` (nor a package with `apt_source:`). +> Cascaded apt sources are not registered on the consumer, so `buildenv_setup` +> rejects them (fail-loud) instead of letting a later `apt-get install` fail. +> Keep any `apt_source` + its package in the consuming repo's local `build-env/`. + +## How CI uses it + +`.azure-pipelines/build-template.yml` runs, inside `container: sonic-slave-*`: + +```bash +PYTHONPATH=$(Build.SourcesDirectory)/ci python3 -m buildenv_setup \ + --repo-dir $(Build.SourcesDirectory) --scope build \ + --arch --debian-version --branch $(BUILD_BRANCH) +./build-env/build.sh +``` + +`buildenv_setup` installs the apt/pip packages, downloads + installs the upstream +libyang DEBs and sonic-yang wheels, and runs the redis / yang-models post-install +hooks. Then `build.sh` builds the package. + +## Local development + +```bash +cd build-env +DEBIAN_VERSION=bookworm docker compose run --rm build # build .debs +DEBIAN_VERSION=bookworm docker compose run --rm shell # poke around +``` + +The image bakes the dependency setup; your source is mounted live at +`/workspace`, so ordinary edits don't require an image rebuild (only changes under +`build-env/` do). Parity with CI covers **building and C++ unit tests**; the full +VS/DVS test suite needs CI-like infrastructure (KVM/privileged/nested-docker) and +is not part of local build parity. + +> Artifact download uses the Azure DevOps REST API. Public SONiC pipelines are +> readable anonymously; if your environment needs auth, set `AZURE_DEVOPS_EXT_PAT`, +> or pre-stage bundles and pass `--upstream-staged-dir`. diff --git a/build-env/build.sh b/build-env/build.sh new file mode 100755 index 000000000..973cab318 --- /dev/null +++ b/build-env/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Canonical build for sonic-swss-common. +# +# Single source of truth for "how to build this repo", used by BOTH CI +# (.azure-pipelines/build-template.yml) and local dev (build-env/compose.yaml). +# Must NOT depend on CI-only environment variables. +# +# The build environment (apt/pip deps + upstream libyang/yang artifacts) is set +# up beforehand by `buildenv_setup` (see build-env/README.md); this script only +# runs the actual package build. +set -ex + +# Run from the repo root regardless of where we were invoked. +cd "$(dirname "$0")/.." + +# Coverage-enabled build (matches build-template.yml for all arches today). +rm -f ../*.deb || true +./autogen.sh +DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' \ + DEB_CXXFLAGS_APPEND="-coverage -fprofile-abs-path" \ + DEB_LDFLAGS_APPEND="-coverage -fprofile-abs-path" \ + dpkg-buildpackage -Pnopython2 -us -uc -b -j"$(nproc)" + +# Collect the built .debs at the repo root (where CI publishes from). +mv ../*.deb . diff --git a/build-env/compose.yaml b/build-env/compose.yaml new file mode 100644 index 000000000..7c7cf3e24 --- /dev/null +++ b/build-env/compose.yaml @@ -0,0 +1,28 @@ +# Local-dev convenience for building sonic-swss-common in a CI-like container. +# +# Usage: +# cd build-env +# DEBIAN_VERSION=bookworm docker compose run --rm build # build the .debs +# DEBIAN_VERSION=bookworm docker compose run --rm shell # interactive shell +# +# The repo is mounted live at /workspace, so edits on the host are immediately +# visible in the container without rebuilding the image (only a change to +# build-env/ dependency declarations requires an image rebuild). + +services: + build: + build: + context: .. + dockerfile: build-env/Dockerfile + args: + DEBIAN_VERSION: ${DEBIAN_VERSION:-bookworm} + BUILD_BRANCH: ${BUILD_BRANCH:-master} + volumes: + - ..:/workspace + working_dir: /workspace + command: ["./build-env/build.sh"] + + shell: + extends: + service: build + command: ["/bin/bash"] diff --git a/build-env/configure-redis-for-tests.sh b/build-env/configure-redis-for-tests.sh new file mode 100755 index 000000000..1fb3ba13e --- /dev/null +++ b/build-env/configure-redis-for-tests.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Canonical redis-server configuration for the dataplane repos' tests, shared via +# the build-env cascade. sonic-swss-common owns this script; downstream consumers +# (e.g. sonic-sairedis) reference `configure-redis-for-tests.sh` from their own +# post_install and buildenv_setup resolves it from the cascaded sonic-swss-common +# bundle (see post_install.resolve_script) -- so the body lives in exactly one place. +# +# Reproduces exactly the redis config build-template.yml applied inline before the +# unit tests. `notify-keyspace-events AKE` is required by sonic-swss-common / +# sonic-swss and is harmless for sonic-sairedis (verified by sonic-sairedis CI). +set -ex + +sudo sed -i 's/notify-keyspace-events ""/notify-keyspace-events AKE/' /etc/redis/redis.conf +sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf +sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf +sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf +sudo service redis-server restart diff --git a/build-env/packages/base.yaml b/build-env/packages/base.yaml new file mode 100644 index 000000000..ca965cbda --- /dev/null +++ b/build-env/packages/base.yaml @@ -0,0 +1,63 @@ +# sonic-swss-common build dependencies (apt + pip). +# +# base.yaml CASCADES: repos downstream of sonic-swss-common (sonic-sairedis, +# sonic-swss) that build against libswsscommon inherit everything declared here, +# so it should list what is needed to build *and link against* this repo. +# +# Consumed by `buildenv_setup --scope build` (and, via the cascade, by the VS +# test host under `--scope test`). + +packages: + # --- C/C++ build dependencies (installed today by build-template.yml) ------ + - libhiredis-dev + # libnl: apt-installed from the distro (STOCK, not SONiC-patched). This keeps the + # libswsscommon .deb's libnl Depends on the stock version, which is satisfied in + # every build/test/docker environment without a SONiC-libnl companion step. These + # are the same four -dev packages master's build-template.yml apt-installs; the + # sonic-slave-* containers and the ubuntu-22.04 vmImage both provide them. + - libnl-3-dev + - libnl-genl-3-dev + - libnl-route-3-dev + - libnl-nf-3-dev + # swig: the sonic-slave-* containers ship 'swig'; Ubuntu 22.04 (jammy, the one + # non-container CI job) needs 'swig4.0'. host_os selects the right one. + - { name: swig, when: { host_os: { not: ubuntu-22.04 } } } + - { name: swig4.0, when: { host_os: ubuntu-22.04 } } + + # libyang Python (CFFI) bindings. common-lib ships the libyang3 C library DEBs + # (see upstream-artifacts.yaml); the Python bindings come from PyPI and are + # built against that system libyang3, so they must install AFTER the DEB + # (requires: libyang3) with build isolation disabled. Mirrors the + # `pip3 install --no-build-isolation libyang==3.3.0` step in build-template.yml. + - name: libyang==3.3.0 + type: pip + pip_args: [--no-build-isolation] + requires: [libyang3] + + # redis-server is required to run sonic-swss-common's unit tests during the + # Build stage, and by downstream VS test hosts. Kept in base.yaml (cascades) + # per the design; the test-config for it is applied via post_install below. + - redis-server + +post_install: + # Configure the system redis for tests. + # + # scopes: [test] ONLY (deliberately NOT build). swss-common's Build stage must + # NOT bring up the unixsocket redis at build-env setup time: the Rust + # `logger_init_without_redis` test (crates/swss-common/tests/logger_fallback.rs) + # asserts logger init fails when redis is unreachable, and it runs during + # `cargo test`. If the unixsocket redis is already up, that test fails. So the + # Build stage runs this SAME script INLINE (see .azure-pipelines/build-template.yml) + # AFTER cargo and before the C++/pytest tests. Here the entry applies only to the + # VS test host (--scope test) via the base.yaml cascade. + # configure-redis-for-tests.sh is the CANONICAL, shared redis test-config for the + # dataplane repos. It lives here (sonic-swss-common, the cascade root); downstream + # consumers (sonic-sairedis, sonic-swss) reuse the SAME script via the cascade + # rather than carrying their own -- buildenv_setup resolves a post_install source: + # from cascaded upstream build-env/ dirs when it isn't present locally. It sets + # `notify-keyspace-events AKE` (harmless for consumers that historically didn't set + # it; verified by their CI). No common/delta split. + - name: configure-redis-for-tests + source: configure-redis-for-tests.sh + requires: [redis-server] # run after redis-server is installed + scopes: [test] # VS test host only; Build stage runs it inline diff --git a/build-env/packages/tooling.yaml b/build-env/packages/tooling.yaml new file mode 100644 index 000000000..8ebf4f698 --- /dev/null +++ b/build-env/packages/tooling.yaml @@ -0,0 +1,53 @@ +# sonic-swss-common Build-stage-only tooling (does NOT cascade downstream). +# +# These are packages/config needed only by sonic-swss-common's own Build-stage +# unit tests and coverage, not by repos that build against libswsscommon. +# Consumed by `buildenv_setup --scope build`. + +packages: + # NOTE: Pympler / pytest are intentionally NOT installed here — the sonic-slave + # build containers already provide them (python3-pympler / python3-pytest), and + # the Ubuntu 22.04 job runs the Bazel test suite (no pytest). Re-add here only + # if a build host is found that lacks them. + + # --- Ubuntu 22.04 (jammy) build toolchain --------------------------------- + # sonic-swss-common has one CI job that runs directly on an Azure-hosted + # ubuntu-22.04 vmImage (not inside a sonic-slave-* container), so it must apt- + # install the toolchain the containers bake in. Gated to that host only + # (host_os: ubuntu-22.04); build-stage tooling, so it does NOT cascade. + # (gcc/build-essential come pre-installed on the ubuntu-22.04 vmImage.) + - { name: make, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: libtool, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: m4, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: autoconf, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: dh-exec, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: debhelper, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: cmake, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: pkg-config, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: nlohmann-json3-dev, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: libpython3-dev, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: libboost-dev, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: libboost-serialization-dev, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: uuid-dev, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: libzmq3-dev, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: redis-tools, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: libgtest-dev, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: libgmock-dev, type: apt, when: { host_os: ubuntu-22.04 } } + - { name: python3-pip, type: apt, when: { host_os: ubuntu-22.04 } } + # python3-cffi: needed so `pip install --no-build-isolation libyang` uses the + # system cffi on jammy (avoids a cffi version mismatch); baked into containers. + - { name: python3-cffi, type: apt, when: { host_os: ubuntu-22.04 } } + +post_install: + - name: mkdir-yang-models + script: | + # sudo-optional: CI runs the tool as non-root with sudo available, but the + # local-dev Dockerfile runs it as root with --no-sudo (and may lack a sudo + # binary). --no-sudo only strips the prefix from the tool's own apt/pip/dpkg + # commands, not inline hook scripts, so guard sudo here at runtime. + SUDO="" + if [ "$(id -u)" -ne 0 ] && command -v sudo >/dev/null 2>&1; then + SUDO=sudo + fi + $SUDO mkdir -p /usr/local/yang-models + scopes: [build] diff --git a/build-env/upstream-artifacts.yaml b/build-env/upstream-artifacts.yaml new file mode 100644 index 000000000..707ada1c6 --- /dev/null +++ b/build-env/upstream-artifacts.yaml @@ -0,0 +1,61 @@ +# Upstream build artifacts that sonic-swss-common needs at build time. +# +# Both entries are sonic-buildimage "leaf" artifacts (they do not ship a +# build-env/ of their own), hence cascade_optional: true. Consumed by +# `buildenv_setup`, which downloads each via the Azure DevOps REST API (or uses +# a --upstream-staged-dir// bundle when staged in the same pipeline run). +# +# Artifact-name convention: amd64 has no arch suffix; non-amd64 appends .{arch}. + +upstream: + # ------------------------------------------------------------------ # + # libyang3 DEBs from the common-libs build (latest on the branch). + # (sonic-swss-common migrated to libyang3 in sonic-net/sonic-swss-common#1206.) + # NOTE: libnl is deliberately NOT downloaded here. sonic-swss-common builds + # against the STOCK apt libnl (apt-installed via base.yaml) to keep the + # libswsscommon .deb's libnl Depends on the stock version (satisfied everywhere + # without downstream companion changes). Only libyang3 comes from common-libs. + # ------------------------------------------------------------------ # + - name: common-libs + project: build + pipeline: Azure.sonic-buildimage.common_libs + # No branch: pin -- resolves the latest common_libs run on ctx.branch + # (--branch $(BUILD_BRANCH)), matching today's runBranch: refs/heads/$(BUILD_BRANCH). + cascade_optional: true + apt_fix_broken: true # these DEBs have apt dependencies dpkg -i can't resolve + # (e.g. libyang-dev depends on libpcre2-dev). The + # sonic-slave-* containers bake those in, but the + # ubuntu-22.04 vmImage job may not, so fall back to + # `apt-get install -f` to pull them from apt. + artifact_name: + - { when: { arch: amd64 }, value: 'common-lib' } + - { when: { arch: { not: amd64 } }, value: 'common-lib.{arch}' } + debs: + # libyang3 C library + dev headers. (python3-libyang deb is intentionally + # NOT installed: the bookworm deb pins python3 >=3.11 so it won't install on + # the ubuntu-22.04 job's python 3.10, and swss-common uses the pip + # libyang==3.3.0 binding instead -- see base.yaml.) + - 'target/debs/{debian_version}/libyang3_*.deb' + - 'target/debs/{debian_version}/libyang-dev_3*.deb' + # NOTE: today's build-template.yml also lists `libpcre*.deb` for trixie, but + # common-lib does not currently publish it (the pattern is a no-op), so it is + # intentionally omitted. Re-add {path: ..., when: {debian_version: trixie}} if + # common-lib starts shipping libpcre. + + # ------------------------------------------------------------------ # + # sonic-yang python wheels from the sonic-buildimage vs build (pipeline 142). + # ------------------------------------------------------------------ # + - name: sonic-buildimage-vs + project: build + pipeline: '142' + # No branch: pin -- resolves the latest pipeline-142 run on ctx.branch + # (--branch $(BUILD_BRANCH)), matching today's runBranch: refs/heads/$(BUILD_BRANCH). + artifact_name: 'sonic-buildimage.vs' + cascade_optional: true + wheels: + # non-trixie: wheels under the matching debian_version directory + - { path: 'target/python-wheels/{debian_version}/sonic_yang_mgmt-1.0-py3-none-any.whl', when: { debian_version: { not: trixie } } } + - { path: 'target/python-wheels/{debian_version}/sonic_yang_models-1.0-py3-none-any.whl', when: { debian_version: { not: trixie } } } + # trixie: use the bookworm wheels (mirrors today's build-template.yml) + - { path: 'target/python-wheels/bookworm/sonic_yang_mgmt-1.0-py3-none-any.whl', when: { debian_version: trixie } } + - { path: 'target/python-wheels/bookworm/sonic_yang_models-1.0-py3-none-any.whl', when: { debian_version: trixie } } diff --git a/ci/.gitignore b/ci/.gitignore new file mode 100644 index 000000000..71a02977e --- /dev/null +++ b/ci/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.pyc +.pytest_cache/ +.coverage +htmlcov/ +.diff-coverage/ diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 000000000..30a9da1b4 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,60 @@ +# ci/ — shared build-environment tooling + +This directory hosts [`buildenv_setup`](./buildenv_setup/), the shared tool that +sets up the build/test environment for the SONiC dataplane repos +(sonic-swss-common, sonic-sairedis, sonic-swss). + +It lives in **sonic-swss-common** because that is the foundational repo at the +root of the dependency cascade — every consumer's CI already checks it out (or +clones it), so hosting the tool here adds no incremental clone cost. It is +versioned with sonic-swss-common's branches: consumers use the copy from the +branch they are building. + +* sonic-swss-common's own CI runs it straight from this checkout: + `PYTHONPATH=$(Build.SourcesDirectory)/ci python3 -m buildenv_setup ...` +* sonic-sairedis / sonic-swss clone sonic-swss-common at the matching branch and + run `PYTHONPATH=/tmp/sw-common/ci python3 -m buildenv_setup ...`. + +## What it does + +Given a repo's `build-env/` configuration, `buildenv_setup`: + +1. reads the declarative `packages/*.yaml` + `upstream-artifacts.yaml`, +2. resolves the (possibly cascaded) set of apt/pip packages and upstream DEBs/wheels, +3. installs them (apt → upstream DEBs → pip/wheels, so DEBs precede pip), and +4. runs the `post_install` configuration hooks. + +See `python3 -m buildenv_setup --help` for all flags, and +[`../build-env/README.md`](../build-env/README.md) for the config schema in use. + +## Layout + +``` +ci/ +├── buildenv_setup/ # the tool (multi-module Python package) +│ ├── cli.py # argparse entrypoint +│ ├── schema.py # YAML load + validation (fail-loud on unknown fields) +│ ├── predicates.py # when: evaluation + {var} substitution +│ ├── cascade.py # upstream-artifact resolution + recursive bundle walk +│ ├── azp_client.py # Azure DevOps REST artifact download +│ ├── installer.py # apt/pip/dpkg execution +│ ├── post_install.py # post_install selection + resolution +│ ├── planner.py # orchestration + --dry-run rendering +│ ├── topo.py # requires: topological sort +│ └── model.py # dataclasses for parsed config +└── tests/ # unit tests (run with pytest) +``` + +## Schema-evolution policy + +The `build-env/` schema is **additive-only** (fields are only added, never +removed/renamed/repurposed) and the tool **fails loud on unknown fields** rather +than silently ignoring them — an unknown field may encode a required setup step, +so dropping it silently would produce a subtly-broken environment. Land tool +support for a new field before any repo's `build-env/` uses it. + +## Tests + +```bash +cd ci && python3 -m pytest tests/ -q +``` diff --git a/ci/buildenv_setup/__init__.py b/ci/buildenv_setup/__init__.py new file mode 100644 index 000000000..d0240851d --- /dev/null +++ b/ci/buildenv_setup/__init__.py @@ -0,0 +1,14 @@ +"""buildenv_setup: shared build-environment setup for the SONiC dataplane repos. + +This package is the single source of truth for setting up the build/test +environment of sonic-swss-common, sonic-sairedis and sonic-swss. It reads the +declarative ``build-env/`` configuration of a repo, resolves the (possibly +cascaded) set of apt/pip packages and upstream-artifact DEBs/wheels, installs +them, and runs any post-install configuration. + +It is invoked as ``python3 -m buildenv_setup`` (see :mod:`buildenv_setup.cli`). +""" + +__version__ = "0.1.0" + +__all__ = ["__version__"] diff --git a/ci/buildenv_setup/__main__.py b/ci/buildenv_setup/__main__.py new file mode 100644 index 000000000..dbdd06617 --- /dev/null +++ b/ci/buildenv_setup/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/buildenv_setup/apt_sources.py b/ci/buildenv_setup/apt_sources.py new file mode 100644 index 000000000..5b42de538 --- /dev/null +++ b/ci/buildenv_setup/apt_sources.py @@ -0,0 +1,24 @@ +"""Register third-party APT repositories declared via ``apt_sources:``. + +Produces the shell commands to install the repo's signing key and ``.list`` +file. (Only exercised by test-host setup, e.g. the Microsoft dotnet feed; the +sonic-swss-common build path declares no apt_sources.) +""" + +from __future__ import annotations + +from typing import List + +from .model import AptSource + + +def register_commands(src: AptSource, use_sudo: bool = True) -> List[str]: + # Privilege control lives here (via use_sudo, threaded from the Executor) so the + # CLI's --no-sudo flag is honoured and runs in root/minimal containers without sudo. + sudo = "sudo " if use_sudo else "" + keyring = f"/usr/share/keyrings/{src.name}-archive-keyring.gpg" + list_file = f"/etc/apt/sources.list.d/{src.name}.list" + return [ + f"curl -fsSL {src.gpg_key_url} | gpg --dearmor | {sudo}tee {keyring} > /dev/null", + f"curl -fsSL {src.list_url} | {sudo}tee {list_file} > /dev/null", + ] diff --git a/ci/buildenv_setup/azp_client.py b/ci/buildenv_setup/azp_client.py new file mode 100644 index 000000000..66c257a50 --- /dev/null +++ b/ci/buildenv_setup/azp_client.py @@ -0,0 +1,232 @@ +"""Minimal Azure DevOps REST client for fetching pipeline-artifact bundles. + +Replaces the per-repo ``DownloadPipelineArtifact@2`` tasks. Given a resolved +:class:`ArtifactRef` it: resolves the pipeline definition id (by name or numeric +id), finds the build to use (a pinned run id, or the latest build on a branch +honouring ``result_filter``), gets the artifact download URL, and downloads + +extracts the zip, returning the artifact's content root. + +Auth (in order): ``SYSTEM_ACCESSTOKEN`` (Bearer, the in-pipeline token) then +``AZURE_DEVOPS_EXT_PAT`` (Basic); otherwise anonymous (works for public +projects). The org URL comes from ``--org-url`` or ``SYSTEM_COLLECTIONURI``. + +NOTE: bounded retries/timeouts are applied to every request; a fuller +retry/backoff policy across all network ops is an implementation-time item +(design doc F21). +""" + +from __future__ import annotations + +import base64 +import logging +import os +import stat +import tempfile +import time +import zipfile +from dataclasses import dataclass, field +from typing import List, Optional +from urllib.parse import quote + +import requests + +log = logging.getLogger(__name__) + +_API_VERSION = "7.0" +_TIMEOUT = 60 +_RETRIES = 3 + + +class AzpError(RuntimeError): + pass + + +@dataclass +class ArtifactRef: + project: str + pipeline: str # pipeline name or numeric definition id + artifact_name: str + branch: str + result_filter: List[str] = field(default_factory=lambda: ["succeeded"]) + run_id: Optional[int] = None # explicit pin overrides branch resolution + + +class AzpClient: + def __init__(self, org_url: Optional[str] = None, session: Optional[requests.Session] = None): + self.org_url = (org_url or os.environ.get("SYSTEM_COLLECTIONURI") or "").rstrip("/") + self.session = session or requests.Session() + self._auth = self._build_auth() + self._def_cache: dict = {} + + # -- auth / http ------------------------------------------------------- # + @staticmethod + def _build_auth() -> dict: + token = os.environ.get("SYSTEM_ACCESSTOKEN") + if token: + return {"Authorization": "Bearer " + token} + pat = os.environ.get("AZURE_DEVOPS_EXT_PAT") + if pat: + enc = base64.b64encode((":" + pat).encode()).decode() + return {"Authorization": "Basic " + enc} + log.warning("no SYSTEM_ACCESSTOKEN / AZURE_DEVOPS_EXT_PAT set; using anonymous access") + return {} + + def _require_org(self) -> str: + if not self.org_url: + raise AzpError( + "no Azure DevOps org URL (set --org-url or SYSTEM_COLLECTIONURI)" + ) + return self.org_url + + def _get(self, url: str, params: Optional[dict] = None, stream: bool = False): + last: Optional[Exception] = None + for attempt in range(1, _RETRIES + 1): + try: + resp = self.session.get( + url, params=params, headers=self._auth, stream=stream, timeout=_TIMEOUT + ) + if resp.status_code >= 500: + raise AzpError(f"server error {resp.status_code} for {url}") + resp.raise_for_status() + return resp + except (requests.RequestException, AzpError) as exc: # noqa: PERF203 + last = exc + if attempt < _RETRIES: + backoff = 2 ** attempt + log.warning("GET %s failed (attempt %d/%d): %s; retrying in %ds", + url, attempt, _RETRIES, exc, backoff) + time.sleep(backoff) + raise AzpError(f"GET {url} failed after {_RETRIES} attempts: {last}") + + # -- resolution -------------------------------------------------------- # + def resolve_definition_id(self, project: str, pipeline: str) -> int: + if str(pipeline).isdigit(): + return int(pipeline) + key = (project, pipeline) + if key in self._def_cache: + return self._def_cache[key] + url = f"{self._require_org()}/{project}/_apis/build/definitions" + resp = self._get(url, {"name": pipeline, "api-version": _API_VERSION}) + values = resp.json().get("value", []) + if not values: + raise AzpError(f"no pipeline definition named {pipeline!r} in project {project!r}") + def_id = values[0]["id"] + self._def_cache[key] = def_id + return def_id + + def resolve_build_id(self, ref: ArtifactRef) -> int: + if ref.run_id is not None: + return ref.run_id + def_id = self.resolve_definition_id(ref.project, ref.pipeline) + url = f"{self._require_org()}/{ref.project}/_apis/build/builds" + params = { + "definitions": def_id, + "branchName": f"refs/heads/{ref.branch}", + "statusFilter": "completed", + "resultFilter": ",".join(ref.result_filter), + "queryOrder": "finishTimeDescending", + "$top": 1, + "api-version": _API_VERSION, + } + resp = self._get(url, params) + values = resp.json().get("value", []) + if not values: + raise AzpError( + f"no completed build for pipeline {ref.pipeline!r} on branch " + f"{ref.branch!r} (results={ref.result_filter})" + ) + return values[0]["id"] + + def _artifact_download_url(self, project: str, build_id: int, artifact_name: str) -> str: + url = f"{self._require_org()}/{project}/_apis/build/builds/{build_id}/artifacts" + resp = self._get(url, {"artifactName": artifact_name, "api-version": _API_VERSION}) + download = resp.json().get("resource", {}).get("downloadUrl") + if not download: + raise AzpError( + f"artifact {artifact_name!r} not found in build {build_id} (project {project})" + ) + return download + + def _download(self, url: str, dest: str) -> None: + resp = self._get(url, stream=True) + with open(dest, "wb") as fh: + for chunk in resp.iter_content(chunk_size=1 << 20): + if chunk: + fh.write(chunk) + + @staticmethod + def _content_root(extract_dir: str, artifact_name: str) -> str: + # ADO zips wrap content in a top-level dir named after the artifact. + named = os.path.join(extract_dir, artifact_name) + if os.path.isdir(named): + return named + entries = os.listdir(extract_dir) + if len(entries) == 1 and os.path.isdir(os.path.join(extract_dir, entries[0])): + return os.path.join(extract_dir, entries[0]) + return extract_dir + + def fetch_artifact( + self, ref: ArtifactRef, dest_dir: str, subpaths: Optional[List[str]] = None + ) -> str: + """Download + extract the artifact; return its content-root directory. + + If ``subpaths`` is given (directory prefixes with no glob chars), only + those subtrees are downloaded via the artifact API's ``subPath`` filter + — essential for large artifacts (e.g. sonic-buildimage.vs) where we only + need a couple of wheels. Otherwise the whole artifact is downloaded. + """ + build_id = self.resolve_build_id(ref) + log.info("resolved %s -> build %d (branch %s)", ref.pipeline, build_id, ref.branch) + url = self._artifact_download_url(ref.project, build_id, ref.artifact_name) + os.makedirs(dest_dir, exist_ok=True) + extract_dir = os.path.join(dest_dir, ref.artifact_name + ".extracted") + os.makedirs(extract_dir, exist_ok=True) + if subpaths: + for sp in subpaths: + self._fetch_zip(url, extract_dir, subpath=sp) + else: + self._fetch_zip(url, extract_dir, subpath=None) + return self._content_root(extract_dir, ref.artifact_name) + + @staticmethod + def _safe_extractall(zf: zipfile.ZipFile, dest: str) -> None: + """Extract every member into ``dest``, rejecting anything that would escape + it. Artifacts come from Azure DevOps but are still untrusted input, so each + member is validated and extracted individually (never a bare + ``extractall()``): + + * absolute paths or ``../`` traversal (classic zip-slip) are rejected; + * symlink members are rejected outright. A symlink placed under ``dest`` + pointing outside it, followed by a write through that path, is a zip-slip + variant; we never materialise one. (CPython's ``zipfile`` happens not to + create symlinks, extracting them as regular files, but we do not rely on + that implementation detail.) + """ + dest_abs = os.path.abspath(dest) + for info in zf.infolist(): + target = os.path.abspath(os.path.join(dest_abs, info.filename)) + if target != dest_abs and not target.startswith(dest_abs + os.sep): + raise AzpError( + f"unsafe path in artifact zip (zip-slip): {info.filename!r}" + ) + if stat.S_ISLNK(info.external_attr >> 16): + raise AzpError( + f"symlink member in artifact zip (zip-slip): {info.filename!r}" + ) + zf.extract(info, dest_abs) + + def _fetch_zip(self, url: str, extract_dir: str, subpath: Optional[str]) -> None: + dl_url = url + if subpath: + sep = "&" if "?" in url else "?" + dl_url = f"{url}{sep}subPath=" + quote("/" + subpath.strip("/")) + log.info(" downloading subPath /%s", subpath.strip("/")) + fd, zip_path = tempfile.mkstemp(suffix=".zip", dir=extract_dir) + os.close(fd) + try: + self._download(dl_url, zip_path) + with zipfile.ZipFile(zip_path) as zf: + self._safe_extractall(zf, extract_dir) + finally: + if os.path.exists(zip_path): + os.remove(zip_path) diff --git a/ci/buildenv_setup/cascade.py b/ci/buildenv_setup/cascade.py new file mode 100644 index 000000000..50b54d0bb --- /dev/null +++ b/ci/buildenv_setup/cascade.py @@ -0,0 +1,265 @@ +"""Resolve ``upstream-artifacts.yaml`` into concrete DEB/wheel install actions. + +Two phases, so ``--dry-run`` can report without any network I/O: + +* :func:`resolve_upstream_file` — pure: apply ``when:``/scope filters and + ``{arch}``/``{debian_version}`` substitution, producing a list of + :class:`ResolvedUpstream` (an :class:`ArtifactRef` plus resolved DEB/wheel + glob patterns). No downloads. +* :func:`collect_bundles` — effectful: for each resolved upstream, obtain its + bundle (a ``--upstream-staged-dir`` subdir if present, else download via + :class:`~buildenv_setup.azp_client.AzpClient`), glob the patterns into real + files, and recurse into the bundle's own ``build-env/`` (cascade). Dedups by + upstream ``name`` with cycle detection. +""" + +from __future__ import annotations + +import glob +import logging +import os +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Set + +from .azp_client import ArtifactRef, AzpClient +from .model import Context, Upstream, UpstreamFile +from .predicates import evaluate, substitute + +log = logging.getLogger(__name__) + + +class CascadeError(RuntimeError): + pass + + +@dataclass +class ResolvedDeb: + pattern: str + dpkg_args: List[str] + apt_fix_broken: bool + + +@dataclass +class ResolvedUpstream: + name: str + ref: ArtifactRef + install_env: Dict[str, str] + debs: List[ResolvedDeb] + wheels: List[str] # resolved path globs + cascade_optional: bool + + +@dataclass +class InstalledArtifact: + """A concrete, fetched bundle with globbed files ready to install.""" + name: str + bundle_dir: str + deb_files: List[str] = field(default_factory=list) + wheel_files: List[str] = field(default_factory=list) + install_env: Dict[str, str] = field(default_factory=dict) + # per-deb-file dpkg options, keyed by absolute file path + deb_opts: Dict[str, tuple] = field(default_factory=dict) # path -> (dpkg_args, apt_fix_broken) + + +def _effective_scopes(entry_scopes: Optional[List[str]], upstream_scopes: List[str]) -> List[str]: + return entry_scopes if entry_scopes is not None else upstream_scopes + + +def _resolve_artifact_name(upstream: Upstream, ctx: Context) -> str: + raw = upstream.artifact_name + if raw is None: + raise CascadeError(f"upstream {upstream.name!r} has no artifact_name") + if isinstance(raw, str): + return substitute(raw, ctx) + if isinstance(raw, list): + for choice in raw: + if not isinstance(choice, dict) or "value" not in choice: + raise CascadeError( + f"upstream {upstream.name!r}: artifact_name list entries need a 'value'" + ) + if evaluate(choice.get("when"), ctx): + return substitute(choice["value"], ctx) + raise CascadeError( + f"upstream {upstream.name!r}: no artifact_name choice matched context" + ) + raise CascadeError(f"upstream {upstream.name!r}: artifact_name must be str or list") + + +def resolve_upstream_file(upfile: UpstreamFile, ctx: Context) -> List[ResolvedUpstream]: + resolved: List[ResolvedUpstream] = [] + for up in upfile.upstreams: + if not evaluate(up.when, ctx): + continue + if ctx.scope not in up.scopes: + continue + ref = ArtifactRef( + project=up.project, + pipeline=up.pipeline, + artifact_name=_resolve_artifact_name(up, ctx), + branch=up.branch or ctx.branch, + result_filter=up.result_filter, + run_id=up.run_id, + ) + debs: List[ResolvedDeb] = [] + for deb in up.debs: + if not evaluate(deb.when, ctx): + continue + if ctx.scope not in _effective_scopes(deb.scopes, up.scopes): + continue + debs.append( + ResolvedDeb( + pattern=substitute(deb.path, ctx), + dpkg_args=deb.dpkg_args or up.dpkg_args, + apt_fix_broken=deb.apt_fix_broken + if deb.apt_fix_broken is not None + else up.apt_fix_broken, + ) + ) + wheels: List[str] = [] + for wheel in up.wheels: + if not evaluate(wheel.when, ctx): + continue + if ctx.scope not in _effective_scopes(wheel.scopes, up.scopes): + continue + wheels.append(substitute(wheel.path, ctx)) + resolved.append( + ResolvedUpstream( + name=up.name, + ref=ref, + install_env=up.install_env, + debs=debs, + wheels=wheels, + cascade_optional=up.cascade_optional, + ) + ) + return resolved + + +def _glob_one(bundle_dir: str, pattern: str, what: str, upstream: str) -> List[str]: + matches = sorted(glob.glob(os.path.join(bundle_dir, pattern))) + if not matches: + # Fall back to a recursive search by basename: robust to differences in + # how the artifact zip is wrapped (e.g. /target/... vs target/...). + matches = sorted( + glob.glob(os.path.join(bundle_dir, "**", os.path.basename(pattern)), recursive=True) + ) + if not matches: + raise CascadeError( + f"upstream {upstream!r}: no {what} matched {pattern!r} under {bundle_dir}" + ) + return matches + + +def collect_bundles( + upfile: UpstreamFile, + ctx: Context, + *, + client: Optional[AzpClient], + work_dir: str, + staged_dir: Optional[str] = None, + required_staged: Optional[Set[str]] = None, + _seen: Optional[Set[str]] = None, + _used_staged: Optional[Set[str]] = None, +) -> List[InstalledArtifact]: + """Fetch/stage every resolved upstream, glob files, and recurse (cascade).""" + required_staged = required_staged or set() + seen = _seen if _seen is not None else set() + # Thread the used-staged accumulator explicitly through the recursion so a + # staged upstream referenced only via a NESTED bundle is still recorded as used + # (avoids a false "required staged upstream not used" error). + used_staged: Set[str] = _used_staged if _used_staged is not None else set() + + out: List[InstalledArtifact] = [] + for ru in resolve_upstream_file(upfile, ctx): + if ru.name in seen: + continue + seen.add(ru.name) + + bundle_dir = _obtain_bundle( + ru, client=client, work_dir=work_dir, staged_dir=staged_dir, + required_staged=required_staged, used_staged=used_staged, + ) + + art = InstalledArtifact(name=ru.name, bundle_dir=bundle_dir, install_env=ru.install_env) + for rdeb in ru.debs: + for f in _glob_one(bundle_dir, rdeb.pattern, "deb", ru.name): + art.deb_files.append(f) + art.deb_opts[f] = (rdeb.dpkg_args, rdeb.apt_fix_broken) + for wpat in ru.wheels: + art.wheel_files.extend(_glob_one(bundle_dir, wpat, "wheel", ru.name)) + out.append(art) + + # Cascade: recurse into the bundle's own build-env/ if present. + nested_up = os.path.join(bundle_dir, "build-env", "upstream-artifacts.yaml") + if os.path.isfile(nested_up): + from .schema import load_upstream_file + out.extend( + collect_bundles( + load_upstream_file(nested_up), ctx, + client=client, work_dir=work_dir, staged_dir=staged_dir, + required_staged=required_staged, _seen=seen, _used_staged=used_staged, + ) + ) + elif not ru.cascade_optional: + log.warning( + "upstream %r bundle has no build-env/ to cascade into; " + "treating as leaf (set cascade_optional: true to silence)", ru.name + ) + + # After the top-level pass, enforce required-staged overrides were all used. + if _seen is None: + unused = required_staged - used_staged + if unused: + raise CascadeError( + f"required staged upstream(s) {sorted(unused)} not found/used in " + f"--upstream-staged-dir {staged_dir!r}" + ) + return out + + +def _has_glob(text: str) -> bool: + return any(c in text for c in "*?[") + + +def _subpaths_for(ru: ResolvedUpstream) -> Optional[List[str]]: + """Directory prefixes (glob-free) of this upstream's deb/wheel patterns, so + only those subtrees are downloaded. Returns None if any pattern's directory + part contains a glob (fall back to whole-artifact download).""" + dirs = set() + for rdeb in ru.debs: + dirs.add(os.path.dirname(rdeb.pattern)) + for wpat in ru.wheels: + dirs.add(os.path.dirname(wpat)) + dirs.discard("") + if not dirs or any(_has_glob(d) for d in dirs): + return None + return sorted(dirs) + + +def _obtain_bundle( + ru: ResolvedUpstream, + *, + client: Optional[AzpClient], + work_dir: str, + staged_dir: Optional[str], + required_staged: Set[str], + used_staged: Set[str], +) -> str: + if staged_dir: + candidate = os.path.join(staged_dir, ru.name) + if os.path.isdir(candidate): + log.info("upstream %r: using staged bundle %s", ru.name, candidate) + used_staged.add(ru.name) + return candidate + if ru.name in required_staged: + raise CascadeError( + f"upstream {ru.name!r} is required-staged but not present in " + f"--upstream-staged-dir {staged_dir!r}" + ) + if client is None: + raise CascadeError( + f"upstream {ru.name!r} must be downloaded but no Azure DevOps client " + f"is available (provide --upstream-staged-dir or run in a pipeline)" + ) + log.info("upstream %r: downloading artifact %s", ru.name, ru.ref.artifact_name) + return client.fetch_artifact(ru.ref, work_dir, subpaths=_subpaths_for(ru)) diff --git a/ci/buildenv_setup/cli.py b/ci/buildenv_setup/cli.py new file mode 100644 index 000000000..4531f2007 --- /dev/null +++ b/ci/buildenv_setup/cli.py @@ -0,0 +1,92 @@ +"""Command-line interface for ``python3 -m buildenv_setup``. + +All configuration flows via CLI args (design doc §2.2/§2.4): explicit, +self-documenting, and easy to log/audit. +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from typing import List, Optional + +from . import __version__ +from .installer import Executor +from .model import Context +from . import planner + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="buildenv_setup", + description="Set up the build/test environment for a SONiC dataplane repo " + "from its declarative build-env/ configuration.", + ) + p.add_argument("--repo-dir", required=True, + help="path to the repo being set up (contains build-env/)") + p.add_argument("--scope", choices=["build", "test"], default="build", + help="which environment to set up (default: build)") + p.add_argument("--arch", default="amd64", + help="target architecture (amd64/armhf/arm64); default amd64") + p.add_argument("--debian-version", default="bookworm", + help="debian/ubuntu codename (bullseye/bookworm/trixie/...)") + p.add_argument("--host-os", default=None, + help="host OS identifier for host_os predicates " + "(default: -container)") + p.add_argument("--branch", default="master", + help="build branch, used to resolve upstream artifacts") + + p.add_argument("--upstream-staged-dir", default=None, + help="directory of same-run staged upstream bundles " + "(//); overrides download for those upstreams") + p.add_argument("--required-staged-upstream", action="append", default=[], + metavar="NAME", + help="fail if NAME is not found/used under --upstream-staged-dir " + "(repeatable)") + p.add_argument("--org-url", default=None, + help="Azure DevOps org URL (default: $SYSTEM_COLLECTIONURI)") + + p.add_argument("--dry-run", action="store_true", + help="print the resolved plan without downloading or installing") + p.add_argument("--no-sudo", action="store_true", + help="do not prefix commands with sudo (e.g. when already root)") + p.add_argument("-v", "--verbose", action="store_true", help="debug logging") + p.add_argument("--version", action="version", version=f"buildenv_setup {__version__}") + return p + + +def main(argv: Optional[List[str]] = None) -> int: + args = build_parser().parse_args(argv) + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s %(name)s: %(message)s", + ) + + host_os = args.host_os or f"{args.debian_version}-container" + ctx = Context( + arch=args.arch, + debian_version=args.debian_version, + host_os=host_os, + scope=args.scope, + branch=args.branch, + ) + executor = Executor(dry_run=args.dry_run, use_sudo=not args.no_sudo) + + try: + planner.run( + ctx, + repo_dir=args.repo_dir, + executor=executor, + staged_dir=args.upstream_staged_dir, + required_staged=set(args.required_staged_upstream), + org_url=args.org_url, + ) + except Exception as exc: # surface a clean error, non-zero exit + logging.getLogger("buildenv_setup").error("%s", exc) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/buildenv_setup/installer.py b/ci/buildenv_setup/installer.py new file mode 100644 index 000000000..2812a067c --- /dev/null +++ b/ci/buildenv_setup/installer.py @@ -0,0 +1,81 @@ +"""Low-level command execution: apt / pip / dpkg / shell primitives. + +All mutating actions go through :class:`Executor`, which honours ``--dry-run`` +(log the command, don't run it) and prefixes privileged commands with ``sudo``. +""" + +from __future__ import annotations + +import logging +import os +import subprocess +from typing import Dict, List, Optional + +log = logging.getLogger(__name__) + + +class Executor: + def __init__(self, dry_run: bool = False, use_sudo: bool = True): + self.dry_run = dry_run + self.sudo: List[str] = ["sudo"] if use_sudo else [] + + # -- primitives -------------------------------------------------------- # + def _exec(self, argv: List[str], env: Optional[Dict[str, str]] = None, + check: bool = True) -> None: + """Log then run a command (honouring --dry-run). Central choke point so + every execution is logged consistently and shares env handling.""" + log.info("+ %s", " ".join(argv)) + if self.dry_run: + return + full_env = {**os.environ, **(env or {})} + subprocess.run(argv, check=check, env=full_env) + + def run(self, argv: List[str], env: Optional[Dict[str, str]] = None) -> None: + self._exec(argv, env=env, check=True) + + def run_script(self, script: str, env: Optional[Dict[str, str]] = None) -> None: + # -e (exit on error) AND -o pipefail so a failure anywhere in a pipeline + # (e.g. `curl ... | gpg ... | tee ...`) fails the whole command instead of + # being masked by a successful final stage. + argv = ["bash", "-e", "-o", "pipefail", "-c", script] + log.info("+ bash -e -o pipefail -c <<'EOF'\n%s\nEOF", script.rstrip()) + if self.dry_run: + return + full_env = {**os.environ, **(env or {})} + subprocess.run(argv, check=True, env=full_env) + + # -- package managers -------------------------------------------------- # + def apt_update(self) -> None: + self.run(self.sudo + ["apt-get", "update"]) + + def apt_install(self, packages: List[str]) -> None: + if not packages: + return + self.run(self.sudo + ["apt-get", "install", "-y"] + packages) + + def pip_install(self, spec: str, pip_args: Optional[List[str]] = None) -> None: + self.run(self.sudo + ["pip3", "install"] + (pip_args or []) + [spec]) + + def dpkg_install( + self, + files: List[str], + dpkg_args: Optional[List[str]] = None, + apt_fix_broken: bool = False, + env: Optional[Dict[str, str]] = None, + ) -> None: + if not files: + return + prefix = list(self.sudo) + if env: + prefix += ["env"] + [f"{k}={v}" for k, v in env.items()] + argv = prefix + ["dpkg", "-i"] + (dpkg_args or []) + files + if apt_fix_broken and not self.dry_run: + try: + # Route through _exec so the command is logged consistently; catch + # the failure to run `apt-get install -f` as the dependency fixup. + self._exec(argv, check=True) + except subprocess.CalledProcessError: + log.warning("dpkg -i failed; running apt-get install -f to fix deps") + self.run(self.sudo + ["apt-get", "install", "-y", "-f"]) + else: + self.run(argv) diff --git a/ci/buildenv_setup/model.py b/ci/buildenv_setup/model.py new file mode 100644 index 000000000..fdff152e2 --- /dev/null +++ b/ci/buildenv_setup/model.py @@ -0,0 +1,111 @@ +"""Data model for parsed ``build-env/`` configuration. + +These dataclasses are intentionally close to the YAML schema (see +``build-env/README.md`` / the design doc). Parsing/validation lives in +:mod:`buildenv_setup.schema`; this module only holds the structures and the +build :class:`Context` used for predicate evaluation and template substitution. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass(frozen=True) +class Context: + """Runtime context supplied via CLI args; drives ``when:`` predicates and + ``{arch}`` / ``{debian_version}`` / ``{host_os}`` substitution.""" + + arch: str + debian_version: str + host_os: str + scope: str # "build" or "test" + branch: str + + def as_subst(self) -> Dict[str, str]: + return { + "arch": self.arch, + "debian_version": self.debian_version, + "host_os": self.host_os, + "branch": self.branch, + } + + +@dataclass +class Package: + name: str + type: str = "apt" # "apt" | "pip" + when: Optional[dict] = None + apt_source: Optional[str] = None + pip_args: List[str] = field(default_factory=list) + requires: List[str] = field(default_factory=list) + + +@dataclass +class AptSource: + name: str + list_url: str + gpg_key_url: str + when: Optional[dict] = None + + +@dataclass +class PostInstall: + name: str + source: Optional[str] = None # path relative to owning bundle's build-env/ + script: Optional[str] = None # inline snippet (mutually exclusive with source) + requires: List[str] = field(default_factory=list) + when: Optional[dict] = None + scopes: List[str] = field(default_factory=lambda: ["build"]) + # Resolved absolute path to the owning bundle's build-env/ dir; filled in by + # the loader so post_install source: paths resolve against the right repo. + owner_build_env: Optional[str] = None + + +@dataclass +class Deb: + path: str # glob, relative to artifact root + when: Optional[dict] = None + dpkg_args: List[str] = field(default_factory=list) + apt_fix_broken: Optional[bool] = None + scopes: Optional[List[str]] = None # None => inherit upstream scopes + + +@dataclass +class Wheel: + path: str # glob, relative to artifact root + when: Optional[dict] = None + scopes: Optional[List[str]] = None + + +@dataclass +class Upstream: + name: str + pipeline: str + project: str = "build" + # artifact_name is either a plain string or a list of {when, value} choices. + artifact_name: Any = None + branch: Optional[str] = None # None => use Context.branch + run_id: Optional[int] = None # pin a specific pipeline run (overrides branch resolution) + result_filter: List[str] = field(default_factory=lambda: ["succeeded"]) + cascade_optional: bool = False + when: Optional[dict] = None + scopes: List[str] = field(default_factory=lambda: ["build"]) + install_env: Dict[str, str] = field(default_factory=dict) + dpkg_args: List[str] = field(default_factory=list) + apt_fix_broken: bool = False + debs: List[Deb] = field(default_factory=list) + wheels: List[Wheel] = field(default_factory=list) + + +@dataclass +class PackagesFile: + apt_sources: List[AptSource] = field(default_factory=list) + packages: List[Package] = field(default_factory=list) + post_install: List[PostInstall] = field(default_factory=list) + + +@dataclass +class UpstreamFile: + upstreams: List[Upstream] = field(default_factory=list) diff --git a/ci/buildenv_setup/planner.py b/ci/buildenv_setup/planner.py new file mode 100644 index 000000000..45d1986af --- /dev/null +++ b/ci/buildenv_setup/planner.py @@ -0,0 +1,303 @@ +"""Orchestration: turn a repo's ``build-env/`` config into an ordered, executed +(or dry-run-printed) install plan. + +Order of operations (matches today's CI and the design doc; F6 DEB-before-pip): + +1. register any ``apt_sources`` referenced by selected apt packages, ``apt-get update`` +2. ``apt-get install`` the selected apt packages (batched, ``requires:``-ordered) +3. ``dpkg -i`` the upstream-artifact DEBs (cascaded), honouring install_env / dpkg_args / apt_fix_broken +4. ``pip install`` the selected pip packages and the upstream wheels +5. run the selected ``post_install`` scripts +""" + +from __future__ import annotations + +import logging +import os +import tempfile +from collections import OrderedDict +from typing import List, Optional, Tuple + +from .azp_client import AzpClient +from .cascade import collect_bundles, resolve_upstream_file +from .installer import Executor +from .model import Context, Package, PackagesFile, PostInstall +from .post_install import resolve_script, select as select_post_install +from .predicates import evaluate +from .schema import load_packages_file, load_upstream_file +from .topo import toposort + +log = logging.getLogger(__name__) + + +class PlannerError(Exception): + """Raised when a resolved plan cannot be executed safely.""" + + +_SCOPE_FILES = { + "build": ["base.yaml", "tooling.yaml"], + "test": ["test.yaml"], +} + + +def _load_local_packages(build_env: str, ctx: Context) -> List[PackagesFile]: + files: List[PackagesFile] = [] + for name in _SCOPE_FILES.get(ctx.scope, []): + path = os.path.join(build_env, "packages", name) + if os.path.isfile(path): + files.append(load_packages_file(path, build_env)) + else: + log.debug("no %s for scope %s (skipping)", name, ctx.scope) + return files + + +def _select_packages( + files: List[PackagesFile], cascaded: List[Package], ctx: Context +) -> Tuple[List[str], List[Tuple[str, Tuple[str, ...]]]]: + """Return (ordered apt names, ordered [(pip spec, pip_args)]) after when-filter, + dedup, and requires: topo-sort.""" + apt: "OrderedDict[str, Package]" = OrderedDict() + pip: "OrderedDict[str, Package]" = OrderedDict() + for pkg in cascaded + [p for f in files for p in f.packages]: + if not evaluate(pkg.when, ctx): + continue + bucket = apt if pkg.type == "apt" else pip + bucket.setdefault(pkg.name, pkg) + + def order(bucket: "OrderedDict[str, Package]") -> List[str]: + requires = {name: p.requires for name, p in bucket.items()} + return toposort(list(bucket), requires) + + apt_names = order(apt) + pip_names = order(pip) + pip_specs = [(name, tuple(pip[name].pip_args)) for name in pip_names] + return apt_names, pip_specs + + +def _select_apt_sources(files: List[PackagesFile], apt_names: List[str], ctx: Context): + referenced = set() + for f in files: + for p in f.packages: + if p.type == "apt" and p.apt_source and p.name in apt_names: + referenced.add(p.apt_source) + chosen = [] + seen = set() + for f in files: + for src in f.apt_sources: + if src.name in referenced and src.name not in seen and evaluate(src.when, ctx): + chosen.append(src) + seen.add(src.name) + return chosen + + +def _collect_cascaded_config(bundle_dirs: List[str], build_envs_seen: set, ctx: Context): + """From fetched upstream bundles, collect cascading base.yaml packages + + post_install (design: base.yaml cascades, tooling.yaml does not).""" + packages: List[Package] = [] + post: List[PostInstall] = [] + for bundle in bundle_dirs: + base = os.path.join(bundle, "build-env", "packages", "base.yaml") + if bundle in build_envs_seen or not os.path.isfile(base): + continue + build_envs_seen.add(bundle) + pf = load_packages_file(base, os.path.join(bundle, "build-env")) + # Fail loud on cascaded apt_sources. The cascade currently propagates a + # base.yaml's packages + post_install only; apt_sources are resolved from + # LOCAL files only (see _select_apt_sources). A cascaded package that + # references an apt_source — or a cascaded base.yaml that declares + # apt_sources — would be installed with its source never registered, so a + # later `apt-get install` fails confusingly. Reject it explicitly rather + # than silently breaking the base.yaml-cascades contract. Dormant today + # (no base.yaml declares apt_sources); revisit if cascaded apt_sources + # become a real requirement. + offending = [p.name for p in pf.packages if p.apt_source] + if pf.apt_sources or offending: + detail = ( + f"packages {offending} reference an apt_source" if offending + else f"declares apt_sources {[s.name for s in pf.apt_sources]}" + ) + raise PlannerError( + f"cascaded build-env '{base}' {detail}, but cascaded apt_sources " + "are not supported: the apt source would never be registered before " + "'apt-get install'. Move the apt_source + its package into the " + "consuming repo's local build-env/packages/, or add cascaded-" + "apt_source support to the planner." + ) + packages.extend(pf.packages) + post.extend(pf.post_install) + return packages, post + + +def _pip_batches(pip_specs: List[Tuple[str, Tuple[str, ...]]]): + """Turn the requires:-toposorted pip specs into ``pip3 install`` batches while + preserving that dependency order across batches. + + Consecutive no-extra-args specs are coalesced into a single install call (pip + resolves install order within one invocation, so grouping them is safe); a + spec carrying pip_args must run as its own call and is emitted in place. By + walking pip_specs in order and flushing the pending plain batch before each + args spec, a plain pip that ``requires:`` an args pip (or vice versa) is still + installed in dependency order — the earlier naive "all plain first, args after" + grouping discarded that ordering.""" + batches: List[Tuple[List[str], Tuple[str, ...]]] = [] + plain: List[str] = [] + for name, args in pip_specs: + if args: + if plain: + batches.append((plain, ())) + plain = [] + batches.append(([name], args)) + else: + plain.append(name) + if plain: + batches.append((plain, ())) + return batches + + +def _deb_install_groups(artifacts) -> List[dict]: + """Group every artifact's DEBs by install_env signature into single dpkg -i + calls (see run() step 3 for rationale). Returns groups ordered so that the + empty-install_env group (the usual library providers such as libnl/libyang3/ + libswsscommon) is installed before any special-install_env group (e.g. vpp). + Within a group, dpkg_args are unioned and apt_fix_broken is ORed.""" + deb_groups: "OrderedDict[tuple, dict]" = OrderedDict() + for art in artifacts: + env_sig = tuple(sorted((art.install_env or {}).items())) + for deb in art.deb_files: + dpkg_args, fix = art.deb_opts.get(deb, ([], False)) + g = deb_groups.get(env_sig) + if g is None: + g = {"files": [], "args": [], "fix": False, "env": dict(art.install_env or {})} + deb_groups[env_sig] = g + g["files"].append(deb) + for a in dpkg_args: + if a not in g["args"]: + g["args"].append(a) + g["fix"] = g["fix"] or fix + # Empty-install_env group first (the usual library providers, installed before + # any special-env group like vpp); all other groups keep their insertion order + # (sorted() is stable), so inter-DEB dependencies across special-env groups are + # not reordered. + return [g for sig, g in sorted(deb_groups.items(), key=lambda kv: 0 if not kv[0] else 1)] + + +def run( + ctx: Context, + repo_dir: str, + executor: Executor, + *, + staged_dir: Optional[str] = None, + required_staged: Optional[set] = None, + org_url: Optional[str] = None, + work_dir: Optional[str] = None, +) -> None: + build_env = os.path.join(repo_dir, "build-env") + if not os.path.isdir(build_env): + raise FileNotFoundError(f"no build-env/ directory under {repo_dir}") + + local_files = _load_local_packages(build_env, ctx) + up_path = os.path.join(build_env, "upstream-artifacts.yaml") + upfile = load_upstream_file(up_path) if os.path.isfile(up_path) else None + + # ----- dry run: report intent without any network I/O ------------------ # + if executor.dry_run: + apt_names, pip_specs = _select_packages(local_files, [], ctx) + sources = _select_apt_sources(local_files, apt_names, ctx) + resolved = resolve_upstream_file(upfile, ctx) if upfile else [] + post = select_post_install([p for f in local_files for p in f.post_install], ctx) + _render_dry_run(ctx, sources, apt_names, pip_specs, resolved, post) + return + + # ----- real execution -------------------------------------------------- # + work_dir = work_dir or tempfile.mkdtemp(prefix="buildenv-") + artifacts = [] + cascaded_pkgs: List[Package] = [] + cascaded_post: List[PostInstall] = [] + if upfile: + client = AzpClient(org_url) + artifacts = collect_bundles( + upfile, ctx, client=client, work_dir=work_dir, + staged_dir=staged_dir, required_staged=required_staged, + ) + cascaded_pkgs, cascaded_post = _collect_cascaded_config( + [a.bundle_dir for a in artifacts], set(), ctx + ) + + apt_names, pip_specs = _select_packages(local_files, cascaded_pkgs, ctx) + sources = _select_apt_sources(local_files, apt_names, ctx) + + # 1. apt sources + update + for src in sources: + from .apt_sources import register_commands + for cmd in register_commands(src, use_sudo=bool(executor.sudo)): + executor.run_script(cmd) + executor.apt_update() + + # 2. apt install + executor.apt_install(apt_names) + + # 3. upstream DEBs (dpkg -i), before pip (F6). Group DEBs across ALL artifacts + # by their install_env signature and install each group in ONE dpkg -i call, + # so inter-DEB dependencies resolve regardless of declaration/filename order + # — including cross-artifact deps, e.g. libswsscommon (sonic-swss-common) + # depends on libnl-nf-3-200 + libyang3 (common-libs). dpkg unpacks the whole + # set before configuring, so it orders configuration itself. Only a differing + # install_env forces a separate call (e.g. vpp needs VPP_INSTALL_SKIP_SYSCTL=1 + # during its maintainer scripts); dpkg_args are unioned and apt_fix_broken is + # ORed within a group. Empty-install_env groups (the usual library providers) + # install first, before any special-env group. + for g in _deb_install_groups(artifacts): + executor.dpkg_install(g["files"], dpkg_args=g["args"], + apt_fix_broken=g["fix"], env=g["env"]) + + # 4. pip packages + wheels + for names, args in _pip_batches(pip_specs): + executor.run(executor.sudo + ["pip3", "install"] + list(args) + names) + for art in artifacts: + for wheel in art.wheel_files: + executor.pip_install(wheel) + + # 5. post_install (cascaded upstream first, then local base, then tooling). + # search_dirs lets an entry's source: resolve from a cascaded upstream bundle + # when it isn't present locally, so a shared hook lives in one repo (the cascade + # root) and consumers reuse it without copying the script body. + ordered_post = cascaded_post + [p for f in local_files for p in f.post_install] + cascaded_build_envs = [os.path.join(a.bundle_dir, "build-env") for a in artifacts] + for entry in select_post_install(ordered_post, ctx): + script = resolve_script(entry, search_dirs=cascaded_build_envs) + log.info("post_install: %s", entry.name) + executor.run_script(script) + + +def _render_dry_run(ctx, sources, apt_names, pip_specs, resolved_upstreams, post): + print(f"# buildenv_setup dry-run (arch={ctx.arch} debian={ctx.debian_version} " + f"host_os={ctx.host_os} scope={ctx.scope} branch={ctx.branch})") + print("\n## apt_sources") + for s in sources: + print(f" - {s.name}: {s.list_url}") + if not sources: + print(" (none)") + print("\n## apt install") + print(" " + (" ".join(apt_names) if apt_names else "(none)")) + print("\n## pip install") + for name, args in pip_specs: + print(f" - {name}" + (f" [args: {' '.join(args)}]" if args else "")) + if not pip_specs: + print(" (none)") + print("\n## upstream artifacts (would download + install)") + for ru in resolved_upstreams: + print(f" - {ru.name}: pipeline={ru.ref.pipeline} project={ru.ref.project} " + f"artifact={ru.ref.artifact_name} branch={ru.ref.branch} " + f"results={ru.ref.result_filter}") + for d in ru.debs: + print(f" deb : {d.pattern}") + for w in ru.wheels: + print(f" whl : {w}") + if not resolved_upstreams: + print(" (none)") + print("\n## post_install") + for entry in post: + kind = "script" if entry.script is not None else f"source:{entry.source}" + print(f" - {entry.name} ({kind}, scopes={entry.scopes})") + if not post: + print(" (none)") diff --git a/ci/buildenv_setup/post_install.py b/ci/buildenv_setup/post_install.py new file mode 100644 index 000000000..2bfa9e378 --- /dev/null +++ b/ci/buildenv_setup/post_install.py @@ -0,0 +1,66 @@ +"""Select and resolve ``post_install:`` entries. + +Given the ordered list of :class:`~buildenv_setup.model.PostInstall` entries +(cascaded upstream entries first, then the repo's own base/tooling), apply the +current ``--scope`` and ``when:`` filters, dedup by ``name`` (first wins, i.e. +upstream cascade takes precedence), and resolve each entry's shell body. + +``source:`` paths resolve against the owning bundle's ``build-env/`` directory +(recorded on the entry at load time), so a script always travels with the YAML +that declares it. +""" + +from __future__ import annotations + +import os +from typing import List, Optional + +from .model import Context, PostInstall +from .predicates import evaluate + + +class PostInstallError(RuntimeError): + pass + + +def select(entries: List[PostInstall], ctx: Context) -> List[PostInstall]: + seen = set() + chosen: List[PostInstall] = [] + for entry in entries: + if entry.name in seen: + continue + if ctx.scope not in entry.scopes: + continue + if not evaluate(entry.when, ctx): + continue + seen.add(entry.name) + chosen.append(entry) + return chosen + + +def resolve_script(entry: PostInstall, search_dirs: Optional[List[str]] = None) -> str: + if entry.script is not None: + return entry.script + if not entry.source: + raise PostInstallError( + f"post_install {entry.name!r} has neither script nor source" + ) + # Candidate locations, in order: the entry's own owning build-env first (a repo's + # local script always wins), then any cascaded upstream build-env dirs (by + # basename). The fallback lets a consumer reuse a script provided by an upstream + # bundle just by referencing its filename, so a shared hook (e.g. + # configure-redis-for-tests.sh) lives in exactly ONE repo -- the cascade root -- + # and downstream repos don't duplicate its body. + candidates: List[str] = [] + if entry.owner_build_env: + candidates.append(os.path.join(entry.owner_build_env, entry.source)) + for d in search_dirs or []: + candidates.append(os.path.join(d, os.path.basename(entry.source))) + for path in candidates: + if os.path.isfile(path): + with open(path, "r", encoding="utf-8") as fh: + return fh.read() + raise PostInstallError( + f"post_install {entry.name!r}: source script {entry.source!r} not found " + f"(looked in: {', '.join(candidates) if candidates else '(no candidates)'})" + ) diff --git a/ci/buildenv_setup/predicates.py b/ci/buildenv_setup/predicates.py new file mode 100644 index 000000000..360bfb57b --- /dev/null +++ b/ci/buildenv_setup/predicates.py @@ -0,0 +1,63 @@ +"""``when:`` predicate evaluation and ``{var}`` template substitution. + +Predicate grammar (matches the design doc): + +* omitted / empty ``when`` -> always true +* ``{arch: amd64}`` -> scalar equality +* ``{arch: [amd64, armhf]}`` -> any-of (list membership) +* ``{arch: {not: amd64}}`` -> negation +* ``{arch: amd64, debian_version: trixie}`` -> AND across keys + +Supported keys: ``arch``, ``debian_version``, ``host_os``, ``scope``, ``branch``. +Unknown keys raise :class:`PredicateError` (fail-fast on typos). +""" + +from __future__ import annotations + +from typing import Any + +from .model import Context + +_KEYS = ("arch", "debian_version", "host_os", "scope", "branch") + + +class PredicateError(ValueError): + pass + + +def _match(matcher: Any, actual: str) -> bool: + if isinstance(matcher, dict): + if set(matcher) - {"not"}: + raise PredicateError(f"unsupported matcher keys: {sorted(matcher)}") + if "not" in matcher: + return not _match(matcher["not"], actual) + return True # empty dict matches anything + if isinstance(matcher, (list, tuple)): + return actual in [str(m) for m in matcher] + return actual == str(matcher) + + +def evaluate(when: Any, ctx: Context) -> bool: + """Return True if ``when`` predicate holds for ``ctx``.""" + if not when: + return True + if not isinstance(when, dict): + raise PredicateError(f"when: must be a mapping, got {type(when).__name__}") + for key, matcher in when.items(): + if key not in _KEYS: + raise PredicateError(f"unknown predicate key {key!r}; expected one of {_KEYS}") + actual = getattr(ctx, key) + if not _match(matcher, actual): + return False + return True + + +def substitute(text: str, ctx: Context) -> str: + """Replace ``{arch}`` / ``{debian_version}`` / ``{host_os}`` / ``{branch}``. + + Uses literal replacement (not str.format) so glob/braces elsewhere are safe. + """ + out = text + for key, val in ctx.as_subst().items(): + out = out.replace("{" + key + "}", val) + return out diff --git a/ci/buildenv_setup/schema.py b/ci/buildenv_setup/schema.py new file mode 100644 index 000000000..3e143cf56 --- /dev/null +++ b/ci/buildenv_setup/schema.py @@ -0,0 +1,233 @@ +"""Load and validate ``build-env/`` YAML into the :mod:`buildenv_setup.model`. + +Compatibility policy (design doc / F3): + +* **Additive-only** — fields are only ever added, never removed, renamed, or + repurposed. This guarantees a newer tool can always read older data. +* **Fail loud on unknown fields** — an unrecognised field is rejected with a + clear error rather than silently ignored. In this domain an unknown field may + encode a *required* setup step, so silently dropping it would produce a + subtly-broken environment (a hard-to-debug failure). Failing loud instead + surfaces tool/data version skew (and plain typos) at the introducing PR, and + enforces the correct ordering: tool support for a field must land before any + ``build-env/`` uses it. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +import yaml + +from .model import ( + AptSource, + Deb, + PackagesFile, + Package, + PostInstall, + Upstream, + UpstreamFile, + Wheel, +) + + +class SchemaError(ValueError): + pass + + +def _load_yaml(path: str) -> dict: + with open(path, "r", encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + if not isinstance(data, dict): + raise SchemaError(f"{path}: top-level YAML must be a mapping") + return data + + +def _reject_unknown(where: str, mapping: Dict[str, Any], known: set) -> None: + unknown = [k for k in mapping if k not in known] + if unknown: + raise SchemaError( + f"{where}: unknown field(s) {sorted(unknown)}; the buildenv_setup tool " + f"may need updating to support them (known fields: {sorted(known)})" + ) + + +def _as_entry(raw: Any) -> Dict[str, Any]: + """Normalise a list entry that may be a bare string or a mapping.""" + if isinstance(raw, str): + return {"name": raw} + if isinstance(raw, dict): + return dict(raw) + raise SchemaError(f"expected string or mapping, got {type(raw).__name__}: {raw!r}") + + +def _as_str_list(value: Any, where: str, field: str) -> List[str]: + """Validate a YAML list-of-strings field. Rejects a bare string (which + ``list()`` would silently explode into characters) and any non-list, so a + schema mistake fails loudly instead of producing a broken command.""" + if value is None: + return [] + if isinstance(value, (str, bytes)) or not isinstance(value, (list, tuple)): + raise SchemaError( + f"{where}: {field!r} must be a list, got {type(value).__name__}: {value!r}" + ) + return [str(v) for v in value] + + +def _as_str_map(value: Any, where: str, field: str) -> Dict[str, str]: + """Validate a YAML mapping-of-strings field (e.g. install_env).""" + if value is None: + return {} + if not isinstance(value, dict): + raise SchemaError( + f"{where}: {field!r} must be a mapping, got {type(value).__name__}: {value!r}" + ) + return {str(k): str(v) for k, v in value.items()} + + +# --------------------------------------------------------------------------- # +# packages/*.yaml +# --------------------------------------------------------------------------- # + +def _parse_apt_source(raw: dict, where: str) -> AptSource: + _reject_unknown(where, raw, {"name", "list_url", "gpg_key_url", "when"}) + for req in ("name", "list_url", "gpg_key_url"): + if req not in raw: + raise SchemaError(f"{where}: apt_sources entry missing {req!r}") + return AptSource( + name=raw["name"], + list_url=raw["list_url"], + gpg_key_url=raw["gpg_key_url"], + when=raw.get("when"), + ) + + +def _parse_package(raw: Any, where: str) -> Package: + entry = _as_entry(raw) + _reject_unknown(where, entry, {"name", "type", "when", "apt_source", "pip_args", "requires"}) + if "name" not in entry: + raise SchemaError(f"{where}: package entry missing 'name'") + ptype = entry.get("type", "apt") + if ptype not in ("apt", "pip"): + raise SchemaError(f"{where}: package {entry['name']!r} has invalid type {ptype!r}") + return Package( + name=entry["name"], + type=ptype, + when=entry.get("when"), + apt_source=entry.get("apt_source"), + pip_args=_as_str_list(entry.get("pip_args"), where, "pip_args"), + requires=_as_str_list(entry.get("requires"), where, "requires"), + ) + + +def _parse_post_install(raw: dict, where: str, owner_build_env: str) -> PostInstall: + _reject_unknown(where, raw, {"name", "source", "script", "requires", "when", "scopes"}) + if "name" not in raw: + raise SchemaError(f"{where}: post_install entry missing 'name'") + if bool(raw.get("source")) == bool(raw.get("script")): + raise SchemaError( + f"{where}: post_install {raw['name']!r} must set exactly one of source/script" + ) + return PostInstall( + name=raw["name"], + source=raw.get("source"), + script=raw.get("script"), + requires=_as_str_list(raw.get("requires"), where, "requires"), + when=raw.get("when"), + scopes=_as_str_list(raw.get("scopes"), where, "scopes") or ["build"], + owner_build_env=owner_build_env, + ) + + +def load_packages_file(path: str, owner_build_env: str) -> PackagesFile: + """Parse a packages/*.yaml file. ``owner_build_env`` is the build-env/ dir + the file belongs to (used to resolve post_install ``source:`` paths).""" + data = _load_yaml(path) + _reject_unknown(path, data, {"apt_sources", "packages", "post_install"}) + apt_sources = [ + _parse_apt_source(s, f"{path}:apt_sources") for s in (data.get("apt_sources") or []) + ] + packages = [_parse_package(p, f"{path}:packages") for p in (data.get("packages") or [])] + post_install = [ + _parse_post_install(p, f"{path}:post_install", owner_build_env) + for p in (data.get("post_install") or []) + ] + return PackagesFile(apt_sources=apt_sources, packages=packages, post_install=post_install) + + +# --------------------------------------------------------------------------- # +# upstream-artifacts.yaml +# --------------------------------------------------------------------------- # + +def _parse_deb(raw: Any, where: str) -> Deb: + entry = _as_entry(raw) + # bare string uses key 'name' from _as_entry; treat it as 'path' + if "path" not in entry and "name" in entry and len(entry) == 1: + entry = {"path": entry["name"]} + _reject_unknown(where, entry, {"path", "when", "dpkg_args", "apt_fix_broken", "scopes"}) + if "path" not in entry: + raise SchemaError(f"{where}: deb entry missing 'path'") + return Deb( + path=entry["path"], + when=entry.get("when"), + dpkg_args=_as_str_list(entry.get("dpkg_args"), where, "dpkg_args"), + apt_fix_broken=entry.get("apt_fix_broken"), + scopes=_as_str_list(entry["scopes"], where, "scopes") if entry.get("scopes") is not None else None, + ) + + +def _parse_wheel(raw: Any, where: str) -> Wheel: + entry = _as_entry(raw) + if "path" not in entry and "name" in entry and len(entry) == 1: + entry = {"path": entry["name"]} + _reject_unknown(where, entry, {"path", "when", "scopes"}) + if "path" not in entry: + raise SchemaError(f"{where}: wheel entry missing 'path'") + return Wheel( + path=entry["path"], + when=entry.get("when"), + scopes=_as_str_list(entry["scopes"], where, "scopes") if entry.get("scopes") is not None else None, + ) + + +def _parse_upstream(raw: dict, where: str) -> Upstream: + _reject_unknown( + where, + raw, + { + "name", "pipeline", "project", "artifact_name", "branch", "run_id", + "result_filter", "cascade_optional", "when", "scopes", "install_env", + "dpkg_args", "apt_fix_broken", "debs", "wheels", + }, + ) + for req in ("name", "pipeline"): + if req not in raw: + raise SchemaError(f"{where}: upstream entry missing {req!r}") + return Upstream( + name=raw["name"], + pipeline=str(raw["pipeline"]), + project=raw.get("project", "build"), + artifact_name=raw.get("artifact_name"), + branch=raw.get("branch"), + run_id=raw.get("run_id"), + result_filter=_as_str_list(raw.get("result_filter"), where, "result_filter") or ["succeeded"], + cascade_optional=bool(raw.get("cascade_optional", False)), + when=raw.get("when"), + scopes=_as_str_list(raw.get("scopes"), where, "scopes") or ["build"], + install_env=_as_str_map(raw.get("install_env"), where, "install_env"), + dpkg_args=_as_str_list(raw.get("dpkg_args"), where, "dpkg_args"), + apt_fix_broken=bool(raw.get("apt_fix_broken", False)), + debs=[_parse_deb(d, f"{where}:debs") for d in (raw.get("debs") or [])], + wheels=[_parse_wheel(w, f"{where}:wheels") for w in (raw.get("wheels") or [])], + ) + + +def load_upstream_file(path: str) -> UpstreamFile: + data = _load_yaml(path) + _reject_unknown(path, data, {"upstream"}) + upstreams = [_parse_upstream(u, f"{path}:upstream") for u in (data.get("upstream") or [])] + names = [u.name for u in upstreams] + dupes = {n for n in names if names.count(n) > 1} + if dupes: + raise SchemaError(f"{path}: duplicate upstream name(s): {sorted(dupes)}") + return UpstreamFile(upstreams=upstreams) diff --git a/ci/buildenv_setup/topo.py b/ci/buildenv_setup/topo.py new file mode 100644 index 000000000..054a0ebd8 --- /dev/null +++ b/ci/buildenv_setup/topo.py @@ -0,0 +1,48 @@ +"""Stable topological sort used to order install steps by ``requires:`` edges. + +Given a list of item names and a ``requires`` map (item -> list of names that +must come first), return the items in an order that respects the edges while +otherwise preserving the original (declaration) order. Raises :class:`CycleError` +on a cycle so misconfiguration fails fast. +""" + +from __future__ import annotations + +from typing import Dict, Iterable, List + + +class CycleError(ValueError): + pass + + +def toposort(items: Iterable[str], requires: Dict[str, Iterable[str]]) -> List[str]: + order = list(items) + index = {name: i for i, name in enumerate(order)} + + # Only consider edges between items we actually have (ignore requires that + # point at packages installed in an earlier group, e.g. an apt dep of a pip + # package handled by group ordering). + deps: Dict[str, List[str]] = { + name: [d for d in requires.get(name, []) if d in index] for name in order + } + + visited: Dict[str, int] = {} # 0 = visiting, 1 = done + result: List[str] = [] + + def visit(name: str, stack: List[str]) -> None: + state = visited.get(name) + if state == 1: + return + if state == 0: + cycle = " -> ".join(stack + [name]) + raise CycleError(f"dependency cycle: {cycle}") + visited[name] = 0 + # Visit dependencies in declaration order for determinism. + for dep in sorted(deps[name], key=lambda d: index[d]): + visit(dep, stack + [name]) + visited[name] = 1 + result.append(name) + + for name in order: + visit(name, []) + return result diff --git a/ci/tests/conftest.py b/ci/tests/conftest.py new file mode 100644 index 000000000..5de0d893c --- /dev/null +++ b/ci/tests/conftest.py @@ -0,0 +1,5 @@ +import os +import sys + +# Make the buildenv_setup package importable regardless of how pytest is invoked. +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) diff --git a/ci/tests/test_apt_sources.py b/ci/tests/test_apt_sources.py new file mode 100644 index 000000000..4ea58d85b --- /dev/null +++ b/ci/tests/test_apt_sources.py @@ -0,0 +1,27 @@ +"""apt_sources.register_commands -- privilege control honours use_sudo.""" +from buildenv_setup.apt_sources import register_commands +from buildenv_setup.model import AptSource + +SRC = AptSource(name="dotnet", list_url="https://x/prod.list", + gpg_key_url="https://x/microsoft.asc") + + +def test_register_commands_with_sudo(): + cmds = register_commands(SRC, use_sudo=True) + assert len(cmds) == 2 + assert "sudo tee /usr/share/keyrings/dotnet-archive-keyring.gpg" in cmds[0] + assert "sudo tee /etc/apt/sources.list.d/dotnet.list" in cmds[1] + assert "https://x/microsoft.asc" in cmds[0] + assert "https://x/prod.list" in cmds[1] + + +def test_register_commands_no_sudo(): + cmds = register_commands(SRC, use_sudo=False) + assert "sudo" not in cmds[0] + assert "sudo" not in cmds[1] + assert "| tee /usr/share/keyrings/dotnet-archive-keyring.gpg" in cmds[0] + + +def test_register_commands_defaults_to_sudo(): + cmds = register_commands(SRC) + assert "sudo tee" in cmds[0] diff --git a/ci/tests/test_azp_client.py b/ci/tests/test_azp_client.py new file mode 100644 index 000000000..019f9fb52 --- /dev/null +++ b/ci/tests/test_azp_client.py @@ -0,0 +1,232 @@ +"""AzpClient: Azure DevOps REST client. Uses a fake requests.Session so no network +I/O happens; exercises auth, definition/build/artifact resolution, retry/backoff, +zip download+extract, and the zip-slip guard.""" +import io +import os +import stat +import zipfile + +import pytest + +from buildenv_setup.azp_client import ArtifactRef, AzpClient, AzpError + + +class FakeResp: + def __init__(self, json_data=None, status=200, content=b""): + self._json = json_data or {} + self.status_code = status + self._content = content + + def json(self): + return self._json + + def raise_for_status(self): + if self.status_code >= 400: + import requests + raise requests.HTTPError(f"status {self.status_code}") + + def iter_content(self, chunk_size=1): + yield self._content + + +class FakeSession: + """Returns queued responses in order (or a callable for dynamic behaviour).""" + def __init__(self, responses): + self._responses = list(responses) + self.calls = [] + + def get(self, url, params=None, headers=None, stream=False, timeout=None): + self.calls.append((url, params)) + r = self._responses.pop(0) + if callable(r): + return r() + return r + + +def _client(responses, org="https://dev.azure.com/mssonic"): + return AzpClient(org_url=org, session=FakeSession(responses)) + + +# -- auth -------------------------------------------------------------------- # +def test_auth_bearer_from_system_accesstoken(monkeypatch): + monkeypatch.setenv("SYSTEM_ACCESSTOKEN", "tok") + monkeypatch.delenv("AZURE_DEVOPS_EXT_PAT", raising=False) + assert AzpClient(org_url="x")._auth == {"Authorization": "Bearer tok"} + + +def test_auth_basic_from_pat(monkeypatch): + monkeypatch.delenv("SYSTEM_ACCESSTOKEN", raising=False) + monkeypatch.setenv("AZURE_DEVOPS_EXT_PAT", "pat") + auth = AzpClient(org_url="x")._auth + assert auth["Authorization"].startswith("Basic ") + + +def test_auth_anonymous(monkeypatch): + monkeypatch.delenv("SYSTEM_ACCESSTOKEN", raising=False) + monkeypatch.delenv("AZURE_DEVOPS_EXT_PAT", raising=False) + assert AzpClient(org_url="x")._auth == {} + + +def test_require_org_missing(monkeypatch): + monkeypatch.delenv("SYSTEM_COLLECTIONURI", raising=False) + with pytest.raises(AzpError): + AzpClient(org_url=None)._require_org() + + +# -- _get retry/backoff ------------------------------------------------------ # +def test_get_retries_on_server_error(monkeypatch): + monkeypatch.setattr("buildenv_setup.azp_client.time.sleep", lambda s: None) # no real backoff + c = _client([FakeResp(status=500), FakeResp({"ok": 1}, status=200)]) + resp = c._get("http://x") + assert resp.json() == {"ok": 1} + + +def test_get_raises_after_retries(monkeypatch): + monkeypatch.setattr("buildenv_setup.azp_client.time.sleep", lambda s: None) + c = _client([FakeResp(status=500), FakeResp(status=500), FakeResp(status=500)]) + with pytest.raises(AzpError): + c._get("http://x") + + +# -- resolve_definition_id --------------------------------------------------- # +def test_resolve_definition_numeric(): + c = _client([]) + assert c.resolve_definition_id("build", "142") == 142 + + +def test_resolve_definition_by_name_and_cache(): + c = _client([FakeResp({"value": [{"id": 9}]})]) + assert c.resolve_definition_id("build", "Azure.sonic-swss-common") == 9 + # cached: second call makes no new request + assert c.resolve_definition_id("build", "Azure.sonic-swss-common") == 9 + assert len(c.session.calls) == 1 + + +def test_resolve_definition_not_found(): + c = _client([FakeResp({"value": []})]) + with pytest.raises(AzpError): + c.resolve_definition_id("build", "nope") + + +# -- resolve_build_id -------------------------------------------------------- # +def test_resolve_build_id_run_id_pin(): + c = _client([]) + ref = ArtifactRef("build", "142", "art", "master", run_id=555) + assert c.resolve_build_id(ref) == 555 + + +def test_resolve_build_id_latest_on_branch(): + c = _client([FakeResp({"value": [{"id": 1002}]})]) # numeric pipeline -> no def lookup + ref = ArtifactRef("build", "142", "art", "master") + assert c.resolve_build_id(ref) == 1002 + + +def test_resolve_build_id_none_found(): + c = _client([FakeResp({"value": []})]) + ref = ArtifactRef("build", "142", "art", "lawlee/ci-unify") + with pytest.raises(AzpError): + c.resolve_build_id(ref) + + +# -- artifact download url --------------------------------------------------- # +def test_artifact_download_url(): + c = _client([FakeResp({"resource": {"downloadUrl": "http://dl/zip"}})]) + assert c._artifact_download_url("build", 5, "art") == "http://dl/zip" + + +def test_artifact_download_url_missing(): + c = _client([FakeResp({"resource": {}})]) + with pytest.raises(AzpError): + c._artifact_download_url("build", 5, "art") + + +# -- _content_root ----------------------------------------------------------- # +def test_content_root_named_dir(tmp_path): + (tmp_path / "art").mkdir() + assert AzpClient._content_root(str(tmp_path), "art") == str(tmp_path / "art") + + +def test_content_root_single_dir(tmp_path): + (tmp_path / "wrapper").mkdir() + assert AzpClient._content_root(str(tmp_path), "art") == str(tmp_path / "wrapper") + + +def test_content_root_fallback(tmp_path): + (tmp_path / "a.deb").write_text("x") + (tmp_path / "b.deb").write_text("y") + assert AzpClient._content_root(str(tmp_path), "art") == str(tmp_path) + + +# -- zip-slip guard ---------------------------------------------------------- # +def _zip_bytes(members): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for name, data in members.items(): + zf.writestr(name, data) + return buf.getvalue() + + +def test_safe_extractall_ok(tmp_path): + zpath = tmp_path / "ok.zip" + zpath.write_bytes(_zip_bytes({"sub/file.txt": "hi"})) + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(str(zpath)) as zf: + AzpClient._safe_extractall(zf, str(dest)) + assert (dest / "sub" / "file.txt").read_text() == "hi" + + +def test_safe_extractall_rejects_zip_slip(tmp_path): + zpath = tmp_path / "evil.zip" + zpath.write_bytes(_zip_bytes({"../escape.txt": "pwn"})) + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(str(zpath)) as zf: + with pytest.raises(AzpError): + AzpClient._safe_extractall(zf, str(dest)) + assert not (tmp_path / "escape.txt").exists() + + +def test_safe_extractall_rejects_symlink_member(tmp_path): + # A symlink member pointing outside dest is a zip-slip variant (write a file + # through the symlinked path). Reject symlink members outright. + outside = tmp_path / "outside" + outside.mkdir() + zpath = tmp_path / "sym.zip" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + info = zipfile.ZipInfo("evil") + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + zf.writestr(info, str(outside)) # symlink target as content + zf.writestr("evil/payload", "PWNED") # write through the symlink + zpath.write_bytes(buf.getvalue()) + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(str(zpath)) as zf: + with pytest.raises(AzpError): + AzpClient._safe_extractall(zf, str(dest)) + assert not (outside / "payload").exists() + + +# -- fetch_artifact (end to end, mocked download) ---------------------------- # +def test_fetch_artifact_downloads_and_extracts(tmp_path, monkeypatch): + # resolve_build_id (numeric pipeline) + artifact url, then download writes a zip. + c = _client([ + FakeResp({"value": [{"id": 77}]}), # resolve_build_id + FakeResp({"resource": {"downloadUrl": "http://dl"}}), # artifact url + ]) + zip_payload = _zip_bytes({"art/target/x.deb": "deb"}) + + def fake_download(url, dest): + with open(dest, "wb") as fh: + fh.write(zip_payload) + + monkeypatch.setattr(c, "_download", fake_download) + ref = ArtifactRef("build", "142", "art", "master") + root = c.fetch_artifact(ref, str(tmp_path)) + assert os.path.isdir(root) + # content-root unwraps the single top-level dir; the deb is reachable underneath. + found = [] + for dp, _dn, fn in os.walk(str(tmp_path)): + found += [f for f in fn if f.endswith(".deb")] + assert "x.deb" in found diff --git a/ci/tests/test_cascade.py b/ci/tests/test_cascade.py new file mode 100644 index 000000000..3a01d19a1 --- /dev/null +++ b/ci/tests/test_cascade.py @@ -0,0 +1,201 @@ +import pytest + +from buildenv_setup.cascade import ( + CascadeError, + ResolvedDeb, + ResolvedUpstream, + _subpaths_for, + collect_bundles, + resolve_upstream_file, +) +from buildenv_setup.azp_client import ArtifactRef +from buildenv_setup.model import Context, Deb, Upstream, UpstreamFile + +BOOK_AMD = Context("amd64", "bookworm", "bookworm-container", "build", "master") +TRIXIE = Context("amd64", "trixie", "trixie-container", "build", "master") +ARMHF = Context("armhf", "bullseye", "bullseye-container", "build", "master") + + +def _common_libs(): + return Upstream( + name="common-libs", + pipeline="Azure.sonic-buildimage.common_libs", + cascade_optional=True, + artifact_name=[ + {"when": {"arch": "amd64"}, "value": "common-lib"}, + {"when": {"arch": {"not": "amd64"}}, "value": "common-lib.{arch}"}, + ], + debs=[ + Deb(path="target/debs/{debian_version}/libyang_1.0*.deb"), + Deb(path="target/debs/{debian_version}/libpcre*.deb", + when={"debian_version": "trixie"}), + ], + ) + + +def test_resolve_artifact_name_arch_suffix(): + uf = UpstreamFile([_common_libs()]) + amd = resolve_upstream_file(uf, BOOK_AMD)[0] + arm = resolve_upstream_file(uf, ARMHF)[0] + assert amd.ref.artifact_name == "common-lib" + assert arm.ref.artifact_name == "common-lib.armhf" + + +def test_resolve_deb_when_filter(): + uf = UpstreamFile([_common_libs()]) + book = resolve_upstream_file(uf, BOOK_AMD)[0] + trix = resolve_upstream_file(uf, TRIXIE)[0] + # libpcre only appears on trixie + book_debs = [d.pattern for d in book.debs] + trix_debs = [d.pattern for d in trix.debs] + assert "target/debs/bookworm/libpcre*.deb" not in book_debs + assert "target/debs/trixie/libpcre*.deb" in trix_debs + + +def test_scope_filter_skips_test_only_upstream(): + up = Upstream(name="test-only", pipeline="p", artifact_name="a", scopes=["test"]) + uf = UpstreamFile([up]) + assert resolve_upstream_file(uf, BOOK_AMD) == [] # build scope skips it + test_ctx = Context("amd64", "bookworm", "ubuntu-22.04", "test", "master") + assert len(resolve_upstream_file(uf, test_ctx)) == 1 + + +def test_run_id_passthrough(): + up = Upstream(name="x", pipeline="p", artifact_name="a", run_id=42, cascade_optional=True) + ru = resolve_upstream_file(UpstreamFile([up]), BOOK_AMD)[0] + assert ru.ref.run_id == 42 + + +def _ru(debs=(), wheels=()): + return ResolvedUpstream( + name="x", ref=ArtifactRef("build", "p", "a", "master"), install_env={}, + debs=[ResolvedDeb(p, [], False) for p in debs], wheels=list(wheels), + cascade_optional=True, + ) + + +def test_subpaths_derivation(): + ru = _ru(debs=["target/debs/bookworm/libyang3_*.deb"], + wheels=["target/python-wheels/bookworm/foo.whl"]) + assert _subpaths_for(ru) == ["target/debs/bookworm", "target/python-wheels/bookworm"] + + +def test_subpaths_none_when_dir_has_glob(): + # a glob in the directory portion means we can't use a subPath filter + ru = _ru(debs=["target/debs/*/libyang3_*.deb"]) + assert _subpaths_for(ru) is None + + +# --------------------------- staged bundles -------------------------------- # + +def test_collect_bundles_staged(tmp_path): + staged = tmp_path / "staged" + debdir = staged / "common-libs" / "target" / "debs" / "bookworm" + debdir.mkdir(parents=True) + (debdir / "libyang_1.0.deb").write_text("x") + + uf = UpstreamFile([Upstream( + name="common-libs", pipeline="p", cascade_optional=True, artifact_name="common-lib", + debs=[Deb(path="target/debs/{debian_version}/libyang_1.0*.deb")], + )]) + arts = collect_bundles(uf, BOOK_AMD, client=None, work_dir=str(tmp_path / "w"), + staged_dir=str(staged)) + assert len(arts) == 1 + assert arts[0].deb_files[0].endswith("libyang_1.0.deb") + + +def test_recursive_glob_fallback(tmp_path): + # deb is NOT at the pattern's path prefix; recursive-basename fallback should find it + staged = tmp_path / "staged" + (staged / "common-libs" / "unexpected").mkdir(parents=True) + (staged / "common-libs" / "unexpected" / "libyang_1.0.deb").write_text("x") + uf = UpstreamFile([Upstream( + name="common-libs", pipeline="p", cascade_optional=True, artifact_name="a", + debs=[Deb(path="target/debs/bookworm/libyang_1.0*.deb")], + )]) + arts = collect_bundles(uf, BOOK_AMD, client=None, work_dir=str(tmp_path / "w"), + staged_dir=str(staged)) + assert arts[0].deb_files[0].endswith("libyang_1.0.deb") + + +def test_required_staged_missing_raises(tmp_path): + uf = UpstreamFile([Upstream(name="x", pipeline="p", artifact_name="a", cascade_optional=True)]) + with pytest.raises(CascadeError): + collect_bundles(uf, BOOK_AMD, client=None, work_dir=str(tmp_path / "w"), + staged_dir=str(tmp_path / "staged"), required_staged={"x"}) + + +def test_missing_glob_raises(tmp_path): + staged = tmp_path / "staged" + (staged / "common-libs").mkdir(parents=True) + uf = UpstreamFile([Upstream( + name="common-libs", pipeline="p", cascade_optional=True, artifact_name="a", + debs=[Deb(path="target/debs/bookworm/does-not-exist_*.deb")], + )]) + with pytest.raises(CascadeError): + collect_bundles(uf, BOOK_AMD, client=None, work_dir=str(tmp_path / "w"), + staged_dir=str(staged)) + + +def test_cascade_recurses_into_nested_bundle(tmp_path): + # staged 'sairedis' bundle that itself declares an upstream 'sw-common' + staged = tmp_path / "staged" + sair = staged / "sairedis" + (sair).mkdir(parents=True) + (sair / "libsairedis_1.0.deb").write_text("x") + nested_be = sair / "build-env" + nested_be.mkdir() + (nested_be / "upstream-artifacts.yaml").write_text( + "upstream:\n" + " - name: sw-common\n" + " pipeline: p\n" + " cascade_optional: true\n" + " artifact_name: a\n" + " debs: [libswsscommon_1.0.deb]\n" + ) + swc = staged / "sw-common" + swc.mkdir() + (swc / "libswsscommon_1.0.deb").write_text("x") + + uf = UpstreamFile([Upstream( + name="sairedis", pipeline="p", cascade_optional=True, artifact_name="a", + debs=[Deb(path="libsairedis_1.0.deb")], + )]) + arts = collect_bundles(uf, BOOK_AMD, client=None, work_dir=str(tmp_path / "w"), + staged_dir=str(staged)) + names = {a.name for a in arts} + assert names == {"sairedis", "sw-common"} # cascade pulled in the nested upstream + + +def test_required_staged_used_only_via_nested_bundle(tmp_path): + # Regression: a staged upstream referenced ONLY through a NESTED bundle must be + # recorded as "used" so it doesn't trip the required-staged check. Before the + # used_staged threading fix, the recursion's used-set was not shared with the + # top-level call, so 'sw-common' (reached only via sairedis's cascade) was + # wrongly reported unused. + staged = tmp_path / "staged" + sair = staged / "sairedis" + sair.mkdir(parents=True) + (sair / "libsairedis_1.0.deb").write_text("x") + nested_be = sair / "build-env" + nested_be.mkdir() + (nested_be / "upstream-artifacts.yaml").write_text( + "upstream:\n" + " - name: sw-common\n" + " pipeline: p\n" + " cascade_optional: true\n" + " artifact_name: a\n" + " debs: [libswsscommon_1.0.deb]\n" + ) + swc = staged / "sw-common" + swc.mkdir() + (swc / "libswsscommon_1.0.deb").write_text("x") + + uf = UpstreamFile([Upstream( + name="sairedis", pipeline="p", cascade_optional=True, artifact_name="a", + debs=[Deb(path="libsairedis_1.0.deb")], + )]) + # Requiring 'sw-common' (only reachable via the nested cascade) must NOT raise. + arts = collect_bundles(uf, BOOK_AMD, client=None, work_dir=str(tmp_path / "w"), + staged_dir=str(staged), required_staged={"sw-common"}) + assert {a.name for a in arts} == {"sairedis", "sw-common"} diff --git a/ci/tests/test_cli.py b/ci/tests/test_cli.py new file mode 100644 index 000000000..f57da39d3 --- /dev/null +++ b/ci/tests/test_cli.py @@ -0,0 +1,53 @@ +"""CLI arg surface + main() dispatch. Uses --dry-run so main() runs planner without +touching the network or the system.""" +import os + +import pytest + +from buildenv_setup import cli + + +def _write(path, text): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + + +def test_build_parser_requires_repo_dir(): + with pytest.raises(SystemExit): + cli.build_parser().parse_args([]) + + +def test_build_parser_defaults(): + args = cli.build_parser().parse_args(["--repo-dir", "/x"]) + assert args.scope == "build" + assert args.arch == "amd64" + assert args.debian_version == "bookworm" + assert args.host_os is None + assert args.no_sudo is False + + +def test_main_dry_run_ok(tmp_path, capsys): + be = tmp_path / "build-env" + _write(str(be / "packages" / "base.yaml"), "packages:\n - libhiredis-dev\n") + rc = cli.main([ + "--repo-dir", str(tmp_path), "--scope", "build", + "--host-os", "ubuntu-22.04", "--dry-run", + ]) + assert rc == 0 + assert "buildenv_setup dry-run" in capsys.readouterr().out + + +def test_main_returns_1_on_error(tmp_path): + # No build-env/ -> planner raises -> main() catches and returns 1. + rc = cli.main(["--repo-dir", str(tmp_path / "nope"), "--dry-run"]) + assert rc == 1 + + +def test_main_host_os_defaults_to_container(tmp_path): + be = tmp_path / "build-env" + _write(str(be / "packages" / "base.yaml"), "packages:\n - libhiredis-dev\n") + # host_os default is -container; bookworm-container has no jammy + # gating, so this still succeeds in dry-run. + rc = cli.main(["--repo-dir", str(tmp_path), "--debian-version", "bookworm", "--dry-run"]) + assert rc == 0 diff --git a/ci/tests/test_installer.py b/ci/tests/test_installer.py new file mode 100644 index 000000000..ed2c6fd2a --- /dev/null +++ b/ci/tests/test_installer.py @@ -0,0 +1,129 @@ +"""Executor primitives + dpkg_install argv construction. + +Ensures per-upstream install_env (e.g. vpp's VPP_INSTALL_SKIP_SYSCTL=1) is applied +on BOTH the plain and the apt_fix_broken code paths, and that command execution +honours --dry-run / --no-sudo and pipefail. +""" +import subprocess + +import buildenv_setup.installer as installer + + +def _capture_subprocess(monkeypatch): + """Patch installer.subprocess.run to record the argv/kwargs it is called with.""" + calls = [] + monkeypatch.setattr( + installer.subprocess, "run", + lambda argv, **kw: calls.append((argv, kw)) or subprocess.CompletedProcess(argv, 0)) + return calls + + +# -- run / run_script -------------------------------------------------------- # +def test_run_executes_with_env(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False).run(["echo", "hi"], env={"K": "V"}) + argv, kw = calls[0] + assert argv == ["echo", "hi"] + assert kw["check"] is True + assert kw["env"]["K"] == "V" + + +def test_run_dry_run_is_noop(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=True).run(["echo", "hi"]) + assert calls == [] + + +def test_run_script_uses_pipefail(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False).run_script("curl x | tee y") + argv, _ = calls[0] + assert argv[:5] == ["bash", "-e", "-o", "pipefail", "-c"] + assert argv[5] == "curl x | tee y" + + +def test_run_script_dry_run_is_noop(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=True).run_script("echo hi") + assert calls == [] + + +# -- package managers -------------------------------------------------------- # +def test_apt_update(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False).apt_update() + assert calls[0][0] == ["sudo", "apt-get", "update"] + + +def test_apt_install_empty_is_noop(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False).apt_install([]) + assert calls == [] + + +def test_apt_install_no_sudo(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False, use_sudo=False).apt_install(["a", "b"]) + assert calls[0][0] == ["apt-get", "install", "-y", "a", "b"] + + +def test_pip_install_with_args(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False).pip_install("libyang==3.3.0", ["--no-build-isolation"]) + assert calls[0][0] == ["sudo", "pip3", "install", "--no-build-isolation", "libyang==3.3.0"] + + +# -- dpkg_install ------------------------------------------------------------ # +def test_dpkg_empty_is_noop(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False).dpkg_install([]) + assert calls == [] + + +def test_dpkg_plain_path_includes_env_prefix(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False).dpkg_install( + ["/d/vpp.deb"], env={"VPP_INSTALL_SKIP_SYSCTL": "1"}) + assert calls[0][0] == ["sudo", "env", "VPP_INSTALL_SKIP_SYSCTL=1", "dpkg", "-i", "/d/vpp.deb"] + + +def test_dpkg_fix_broken_path_includes_env_prefix(monkeypatch): + # apt_fix_broken routes through _exec (subprocess.run); the env prefix MUST still + # be in argv. + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False).dpkg_install( + ["/d/vpp.deb", "/d/libvppinfra.deb"], + apt_fix_broken=True, env={"VPP_INSTALL_SKIP_SYSCTL": "1"}) + argv = calls[0][0] + assert argv[:4] == ["sudo", "env", "VPP_INSTALL_SKIP_SYSCTL=1", "dpkg"] + assert "/d/vpp.deb" in argv and "/d/libvppinfra.deb" in argv + + +def test_dpkg_fix_broken_falls_back_to_apt_f(monkeypatch): + # First call (dpkg -i) raises CalledProcessError -> second call is apt-get -f. + seen = [] + + def fake_run(argv, **kw): + seen.append(argv) + if len(seen) == 1: + raise subprocess.CalledProcessError(1, argv) + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.setattr(installer.subprocess, "run", fake_run) + installer.Executor(dry_run=False).dpkg_install(["/d/a.deb"], apt_fix_broken=True) + assert seen[0][:3] == ["sudo", "dpkg", "-i"] + assert seen[1] == ["sudo", "apt-get", "install", "-y", "-f"] + + +def test_dpkg_no_env_no_prefix(monkeypatch): + calls = _capture_subprocess(monkeypatch) + installer.Executor(dry_run=False).dpkg_install(["/d/libnl.deb"]) + assert calls[0][0] == ["sudo", "dpkg", "-i", "/d/libnl.deb"] + + +def test_dpkg_dry_run_fix_broken_no_subprocess(monkeypatch): + calls = _capture_subprocess(monkeypatch) + # dry_run + apt_fix_broken -> falls through to run() (logs only), no subprocess. + installer.Executor(dry_run=True).dpkg_install( + ["/d/x.deb"], apt_fix_broken=True, env={"VPP_INSTALL_SKIP_SYSCTL": "1"}) + assert calls == [] diff --git a/ci/tests/test_planner.py b/ci/tests/test_planner.py new file mode 100644 index 000000000..b94dea6c6 --- /dev/null +++ b/ci/tests/test_planner.py @@ -0,0 +1,197 @@ +from buildenv_setup.model import Context, Package, PackagesFile +from buildenv_setup.planner import ( + _collect_cascaded_config, + _deb_install_groups, + _pip_batches, + _select_packages, + PlannerError, +) +from buildenv_setup.cascade import InstalledArtifact + +import textwrap + +import pytest + +CTX = Context("amd64", "bookworm", "bookworm-container", "build", "master") + + +def _pf(packages): + return PackagesFile(packages=packages) + + +def test_select_splits_apt_and_pip_and_dedups(): + files = [ + _pf([Package(name="libhiredis-dev"), Package(name="pytest", type="pip")]), + _pf([Package(name="libhiredis-dev"), Package(name="Pympler==0.8", type="pip")]), + ] + apt, pip = _select_packages(files, [], CTX) + assert apt == ["libhiredis-dev"] # deduped + assert [name for name, _ in pip] == ["pytest", "Pympler==0.8"] + + +def test_select_when_filter(): + files = [_pf([ + Package(name="always"), + Package(name="arm-only", when={"arch": {"not": "amd64"}}), + ])] + apt, _ = _select_packages(files, [], CTX) + assert apt == ["always"] + + +def test_select_requires_ordering(): + files = [_pf([ + Package(name="libyang", type="pip", requires=["python3-cffi"]), + Package(name="python3-cffi", type="pip"), + ])] + _, pip = _select_packages(files, [], CTX) + names = [name for name, _ in pip] + assert names.index("python3-cffi") < names.index("libyang") + + +def test_cascaded_packages_come_first(): + cascaded = [Package(name="from-upstream")] + files = [_pf([Package(name="local")])] + apt, _ = _select_packages(files, cascaded, CTX) + assert apt == ["from-upstream", "local"] + + +def test_pip_batches_groups_plain_and_splits_args(): + batches = _pip_batches([("a", ()), ("b", ()), ("libyang", ("--no-build-isolation",))]) + assert (["a", "b"], ()) in batches + assert (["libyang"], ("--no-build-isolation",)) in batches + + +def test_pip_batches_preserve_order_plain_requires_args(): + # requires:-toposort puts the args pip first because the plain pip depends on + # it; _pip_batches must keep that order (args pip installed before plain pip). + batches = _pip_batches([("argspip", ("--flag",)), ("plainpip", ())]) + assert batches == [(["argspip"], ("--flag",)), (["plainpip"], ())] + + +def test_pip_batches_preserve_order_args_requires_plain(): + # Mirror case: args pip depends on a plain pip -> toposort emits the plain pip + # first; the pending plain batch must be flushed before the args batch. + batches = _pip_batches([("plainpip", ()), ("argspip", ("--flag",))]) + assert batches == [(["plainpip"], ()), (["argspip"], ("--flag",))] + + +def test_pip_batches_interleaved_preserves_sequence(): + batches = _pip_batches([("a", ()), ("x", ("--f",)), ("b", ()), ("c", ())]) + assert batches == [(["a"], ()), (["x"], ("--f",)), (["b", "c"], ())] + + +def _write_cascaded_base(tmp_path, body: str) -> str: + build_env = tmp_path / "build-env" + (build_env / "packages").mkdir(parents=True) + (build_env / "packages" / "base.yaml").write_text(textwrap.dedent(body)) + return str(tmp_path) + + +def test_cascaded_package_with_apt_source_fails_loud(tmp_path): + bundle = _write_cascaded_base(tmp_path, """ + apt_sources: + - name: llvm + list_url: https://apt.llvm.org/x.list + gpg_key_url: https://apt.llvm.org/key.asc + packages: + - { name: clang-18, type: apt, apt_source: llvm } + """) + with pytest.raises(PlannerError) as ei: + _collect_cascaded_config([bundle], set(), CTX) + assert "clang-18" in str(ei.value) + assert "apt_source" in str(ei.value) + + +def test_cascaded_apt_sources_declaration_fails_loud(tmp_path): + # Even an apt_sources declaration with no referencing package is rejected: + # it signals reliance on unsupported cascaded-apt_source behavior. + bundle = _write_cascaded_base(tmp_path, """ + apt_sources: + - name: llvm + list_url: https://apt.llvm.org/x.list + gpg_key_url: https://apt.llvm.org/key.asc + packages: + - { name: build-essential, type: apt } + """) + with pytest.raises(PlannerError) as ei: + _collect_cascaded_config([bundle], set(), CTX) + assert "apt_sources" in str(ei.value) + + +def test_cascaded_base_without_apt_source_ok(tmp_path): + bundle = _write_cascaded_base(tmp_path, """ + packages: + - { name: libnl-3-dev, type: apt } + - { name: libyang, type: pip } + """) + pkgs, post = _collect_cascaded_config([bundle], set(), CTX) + assert [p.name for p in pkgs] == ["libnl-3-dev", "libyang"] + assert post == [] + + +def test_deb_groups_merge_plain_upstreams_into_one_call(): + # Cross-artifact dependency: libswsscommon (sonic-swss-common) depends on + # libnl/libyang3 (common-libs). Both have empty install_env, so they must be + # installed in a SINGLE dpkg -i call regardless of declaration order. + swss_common = InstalledArtifact( + name="sonic-swss-common", bundle_dir="/b/swss", + deb_files=["/b/swss/libswsscommon_1.0.0_amd64.deb"], + ) + common_libs = InstalledArtifact( + name="common-libs", bundle_dir="/b/cl", + deb_files=["/b/cl/libnl-nf.deb", "/b/cl/libyang3.deb"], + ) + groups = _deb_install_groups([swss_common, common_libs]) + assert len(groups) == 1 + assert set(groups[0]["files"]) == { + "/b/swss/libswsscommon_1.0.0_amd64.deb", "/b/cl/libnl-nf.deb", "/b/cl/libyang3.deb" + } + + +def test_deb_groups_split_by_install_env_plain_first(): + # vpp carries install_env (VPP_INSTALL_SKIP_SYSCTL) + apt_fix_broken, so it is + # a separate group installed AFTER the empty-install_env providers. + plain = InstalledArtifact( + name="common-libs", bundle_dir="/b/cl", deb_files=["/b/cl/libyang3.deb"], + ) + vpp = InstalledArtifact( + name="vpp", bundle_dir="/b/vpp", deb_files=["/b/vpp/vpp.deb", "/b/vpp/libvppinfra.deb"], + install_env={"VPP_INSTALL_SKIP_SYSCTL": "1"}, + deb_opts={"/b/vpp/vpp.deb": ([], True)}, + ) + groups = _deb_install_groups([vpp, plain]) # declared vpp-first on purpose + assert len(groups) == 2 + assert groups[0]["files"] == ["/b/cl/libyang3.deb"] # empty-env group first + assert groups[0]["env"] == {} + assert set(groups[1]["files"]) == {"/b/vpp/vpp.deb", "/b/vpp/libvppinfra.deb"} + assert groups[1]["env"] == {"VPP_INSTALL_SKIP_SYSCTL": "1"} + assert groups[1]["fix"] is True # apt_fix_broken ORed in + + +def test_deb_groups_union_dpkg_args_within_group(): + art = InstalledArtifact( + name="x", bundle_dir="/b/x", + deb_files=["/b/x/a.deb", "/b/x/b.deb"], + deb_opts={ + "/b/x/a.deb": (["--force-confask"], False), + "/b/x/b.deb": (["--force-confnew"], False), + }, + ) + groups = _deb_install_groups([art]) + assert len(groups) == 1 + assert groups[0]["args"] == ["--force-confask", "--force-confnew"] + + +def test_deb_groups_preserve_insertion_order_among_env_groups(): + # Two DIFFERENT non-empty install_env groups: their relative order must follow + # insertion order (not be reordered by env-signature length), so inter-DEB deps + # across special-env groups aren't broken. Empty-env group still goes first. + envA = InstalledArtifact(name="a", bundle_dir="/a", deb_files=["/a/a.deb"], + install_env={"A": "1"}) + envB = InstalledArtifact(name="b", bundle_dir="/b", deb_files=["/b/b.deb"], + install_env={"B": "1"}) + plain = InstalledArtifact(name="p", bundle_dir="/p", deb_files=["/p/p.deb"]) + groups = _deb_install_groups([envA, envB, plain]) # declared A, B, plain + assert groups[0]["files"] == ["/p/p.deb"] # empty-env first + assert groups[1]["env"] == {"A": "1"} # then insertion order A ... + assert groups[2]["env"] == {"B": "1"} # ... then B diff --git a/ci/tests/test_planner_run.py b/ci/tests/test_planner_run.py new file mode 100644 index 000000000..2a27f4a4f --- /dev/null +++ b/ci/tests/test_planner_run.py @@ -0,0 +1,128 @@ +"""planner.run() orchestration end-to-end, with a fake Executor and staged upstream +bundles (no network). Covers dry-run rendering, the real install sequence +(apt/dpkg/pip/post_install), apt_source selection, and the cascade collection.""" +import os + +import pytest + +from buildenv_setup.model import Context +from buildenv_setup import planner + + +class FakeExecutor: + def __init__(self, dry_run=False, use_sudo=True): + self.dry_run = dry_run + self.sudo = ["sudo"] if use_sudo else [] + self.calls = [] + + def run(self, argv, env=None): + self.calls.append(("run", list(argv), env)) + + def run_script(self, script, env=None): + self.calls.append(("script", script)) + + def apt_update(self): + self.calls.append(("apt_update",)) + + def apt_install(self, pkgs): + self.calls.append(("apt_install", list(pkgs))) + + def pip_install(self, spec, args=None): + self.calls.append(("pip", spec, list(args or []))) + + def dpkg_install(self, files, dpkg_args=None, apt_fix_broken=False, env=None): + self.calls.append(("dpkg", list(files), dict(env or {}))) + + +def _write(path, text): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + + +@pytest.fixture +def repo(tmp_path): + """A minimal repo build-env/ plus a staged upstream bundle.""" + be = tmp_path / "repo" / "build-env" + _write(str(be / "packages" / "base.yaml"), """ +packages: + - libhiredis-dev + - { name: redis-server } + - name: libyang==3.3.0 + type: pip + pip_args: [--no-build-isolation] + requires: [libyang3] +post_install: + - name: setup-redis + source: redis.sh + scopes: [build] +""") + _write(str(be / "packages" / "tooling.yaml"), """ +packages: + - { name: cmake, type: apt } +""") + _write(str(be / "upstream-artifacts.yaml"), """ +upstream: + - name: common-libs + pipeline: Azure.sonic-buildimage.common_libs + artifact_name: common-lib + cascade_optional: true + debs: + - 'target/debs/bookworm/libyang3_*.deb' +""") + _write(str(be / "redis.sh"), "echo configuring redis\n") + + # Staged bundle for the "common-libs" upstream with a matching deb. + staged = tmp_path / "staged" + _write(str(staged / "common-libs" / "target" / "debs" / "bookworm" / "libyang3_3.deb"), "x") + return {"repo_dir": str(tmp_path / "repo"), "staged": str(staged)} + + +CTX = Context("amd64", "bookworm", "bookworm-container", "build", "master") + + +def test_run_dry_run_prints_plan(repo, capsys): + ex = FakeExecutor(dry_run=True) + planner.run(CTX, repo["repo_dir"], ex, staged_dir=repo["staged"]) + out = capsys.readouterr().out + assert "buildenv_setup dry-run" in out + assert "libhiredis-dev" in out + assert "common-libs" in out + assert "setup-redis" in out + assert ex.calls == [] # dry-run executes nothing + + +def test_run_real_sequence(repo): + ex = FakeExecutor(dry_run=False) + planner.run(CTX, repo["repo_dir"], ex, staged_dir=repo["staged"]) + kinds = [c[0] for c in ex.calls] + assert "apt_update" in kinds + # apt install includes the base + tooling apt packages + apt = next(c for c in ex.calls if c[0] == "apt_install") + assert "libhiredis-dev" in apt[1] and "redis-server" in apt[1] and "cmake" in apt[1] + # the staged upstream deb is dpkg-installed + dpkg = [c for c in ex.calls if c[0] == "dpkg"] + assert any(any(f.endswith("libyang3_3.deb") for f in c[1]) for c in dpkg) + # pip install of libyang happened + assert any(c[0] == "run" and "libyang==3.3.0" in c[1] for c in ex.calls) \ + or any(c[0] == "pip" and "libyang==3.3.0" in c[1] for c in ex.calls) + # post_install script ran + assert any(c[0] == "script" and "configuring redis" in c[1] for c in ex.calls) + + +def test_run_no_build_env_raises(tmp_path): + with pytest.raises(FileNotFoundError): + planner.run(CTX, str(tmp_path / "empty"), FakeExecutor()) + + +def test_select_apt_sources_only_when_referenced(): + from buildenv_setup.model import AptSource, Package, PackagesFile + src = AptSource(name="dotnet", list_url="l", gpg_key_url="g") + files = [PackagesFile( + apt_sources=[src], + packages=[Package(name="dotnet-sdk", type="apt", apt_source="dotnet")], + )] + chosen = planner._select_apt_sources(files, ["dotnet-sdk"], CTX) + assert [s.name for s in chosen] == ["dotnet"] + # not referenced -> not chosen + assert planner._select_apt_sources(files, [], CTX) == [] diff --git a/ci/tests/test_post_install.py b/ci/tests/test_post_install.py new file mode 100644 index 000000000..8f2e7911c --- /dev/null +++ b/ci/tests/test_post_install.py @@ -0,0 +1,77 @@ +import os + +import pytest + +from buildenv_setup.model import Context, PostInstall +from buildenv_setup.post_install import PostInstallError, resolve_script, select + +CTX_BUILD = Context("amd64", "bookworm", "bookworm-container", "build", "master") +CTX_TEST = Context("amd64", "bookworm", "bookworm-container", "test", "master") + + +def _write(path, body): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(body) + + +def test_resolve_inline_script_wins(): + entry = PostInstall(name="x", script="echo hi") + assert resolve_script(entry) == "echo hi" + + +def test_resolve_source_from_owner(tmp_path): + owner = tmp_path / "sairedis" / "build-env" + _write(str(owner / "s.sh"), "local-body") + entry = PostInstall(name="x", source="s.sh", owner_build_env=str(owner)) + assert resolve_script(entry) == "local-body" + + +def test_resolve_source_falls_back_to_cascaded_upstream(tmp_path): + # Consumer declares the entry but ships NO local script; the body is provided by + # a cascaded upstream bundle's build-env/ and found by basename. + owner = tmp_path / "sairedis" / "build-env" # no s.sh here + upstream = tmp_path / "swss-common" / "build-env" + _write(str(upstream / "s.sh"), "shared-body") + entry = PostInstall(name="x", source="s.sh", owner_build_env=str(owner)) + assert resolve_script(entry, search_dirs=[str(upstream)]) == "shared-body" + + +def test_resolve_local_preferred_over_cascaded(tmp_path): + owner = tmp_path / "sairedis" / "build-env" + upstream = tmp_path / "swss-common" / "build-env" + _write(str(owner / "s.sh"), "local-body") + _write(str(upstream / "s.sh"), "shared-body") + entry = PostInstall(name="x", source="s.sh", owner_build_env=str(owner)) + assert resolve_script(entry, search_dirs=[str(upstream)]) == "local-body" + + +def test_resolve_missing_everywhere_raises(tmp_path): + entry = PostInstall(name="x", source="s.sh", + owner_build_env=str(tmp_path / "build-env")) + with pytest.raises(PostInstallError): + resolve_script(entry, search_dirs=[str(tmp_path / "other")]) + + +def test_select_scope_filter_lets_consumer_own_entry_win_at_build(): + # Mirrors the shared-redis dedup: upstream (sonic-swss-common) declares the hook + # test-scoped; the consumer (sonic-sairedis) re-declares it [build, test]. During + # a build-scope run the upstream entry is filtered out (so it is NOT marked seen), + # and the consumer's own entry -- which reuses the cascaded script -- is chosen. + upstream = PostInstall(name="configure-redis-for-tests", source="r.sh", + scopes=["test"], owner_build_env="/u/build-env") + consumer = PostInstall(name="configure-redis-for-tests", source="r.sh", + scopes=["build", "test"], owner_build_env="/c/build-env") + chosen = select([upstream, consumer], CTX_BUILD) # cascaded first, then local + assert [e.owner_build_env for e in chosen] == ["/c/build-env"] + + +def test_select_upstream_entry_wins_at_test_scope(): + # At test scope the upstream (cascaded-first) entry is chosen and marks the name + # seen, so the consumer's same-named entry is skipped -- no double run. + upstream = PostInstall(name="configure-redis-for-tests", source="r.sh", + scopes=["test"], owner_build_env="/u/build-env") + consumer = PostInstall(name="configure-redis-for-tests", source="r.sh", + scopes=["build", "test"], owner_build_env="/c/build-env") + chosen = select([upstream, consumer], CTX_TEST) + assert [e.owner_build_env for e in chosen] == ["/u/build-env"] diff --git a/ci/tests/test_predicates.py b/ci/tests/test_predicates.py new file mode 100644 index 000000000..6237eef48 --- /dev/null +++ b/ci/tests/test_predicates.py @@ -0,0 +1,43 @@ +import pytest + +from buildenv_setup.model import Context +from buildenv_setup.predicates import PredicateError, evaluate, substitute + +CTX = Context(arch="amd64", debian_version="bookworm", host_os="bookworm-container", + scope="build", branch="master") + + +def test_empty_when_is_true(): + assert evaluate(None, CTX) is True + assert evaluate({}, CTX) is True + + +def test_scalar_match(): + assert evaluate({"arch": "amd64"}, CTX) is True + assert evaluate({"arch": "armhf"}, CTX) is False + + +def test_list_match(): + assert evaluate({"arch": ["amd64", "armhf"]}, CTX) is True + assert evaluate({"arch": ["arm64", "armhf"]}, CTX) is False + + +def test_negation(): + assert evaluate({"arch": {"not": "armhf"}}, CTX) is True + assert evaluate({"arch": {"not": "amd64"}}, CTX) is False + + +def test_and_across_keys(): + assert evaluate({"arch": "amd64", "debian_version": "bookworm"}, CTX) is True + assert evaluate({"arch": "amd64", "debian_version": "trixie"}, CTX) is False + + +def test_unknown_key_raises(): + with pytest.raises(PredicateError): + evaluate({"distro": "debian"}, CTX) + + +def test_substitute(): + assert substitute("common-lib.{arch}", CTX) == "common-lib.amd64" + assert substitute("target/debs/{debian_version}/x.deb", CTX) == "target/debs/bookworm/x.deb" + assert substitute("no-vars", CTX) == "no-vars" diff --git a/ci/tests/test_schema.py b/ci/tests/test_schema.py new file mode 100644 index 000000000..f1d6c74b0 --- /dev/null +++ b/ci/tests/test_schema.py @@ -0,0 +1,271 @@ +import textwrap + +import pytest + +from buildenv_setup import schema + + +def _write(tmp_path, name, text): + p = tmp_path / name + p.write_text(textwrap.dedent(text)) + return str(p) + + +# --------------------------- packages/*.yaml -------------------------------- # + +def test_packages_bare_and_mapping(tmp_path): + path = _write(tmp_path, "base.yaml", """ + packages: + - libhiredis-dev + - {name: pytest, type: pip} + - {name: libnl-3-dev, when: {arch: {not: amd64}}} + """) + pf = schema.load_packages_file(path, str(tmp_path)) + assert [p.name for p in pf.packages] == ["libhiredis-dev", "pytest", "libnl-3-dev"] + assert pf.packages[0].type == "apt" + assert pf.packages[1].type == "pip" + assert pf.packages[2].when == {"arch": {"not": "amd64"}} + + +def test_post_install_source_xor_script(tmp_path): + path = _write(tmp_path, "base.yaml", """ + post_install: + - {name: bad, source: x.sh, script: "echo hi"} + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_post_install_records_owner(tmp_path): + path = _write(tmp_path, "base.yaml", """ + post_install: + - {name: redis, source: configure-redis.sh, requires: [redis-server], scopes: [build, test]} + """) + pf = schema.load_packages_file(path, "/some/build-env") + entry = pf.post_install[0] + assert entry.owner_build_env == "/some/build-env" + assert entry.scopes == ["build", "test"] + + +def test_invalid_pip_args_type_field(tmp_path): + path = _write(tmp_path, "base.yaml", """ + packages: + - {name: bogus, type: conda} + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +# ------------------- fail-loud on unknown fields (F3) ----------------------- # + +def test_unknown_package_field_raises(tmp_path): + path = _write(tmp_path, "base.yaml", """ + packages: + - {name: foo, typ: pip} + """) + with pytest.raises(schema.SchemaError) as exc: + schema.load_packages_file(path, str(tmp_path)) + assert "typ" in str(exc.value) + + +def test_unknown_toplevel_field_raises(tmp_path): + path = _write(tmp_path, "base.yaml", """ + packages: [libhiredis-dev] + postinstall: [] + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +# --------------------------- upstream-artifacts.yaml ------------------------ # + +def test_upstream_parse_and_run_id(tmp_path): + path = _write(tmp_path, "upstream-artifacts.yaml", """ + upstream: + - name: common-libs + pipeline: Azure.sonic-buildimage.common_libs + run_id: 926659 + cascade_optional: true + artifact_name: common-lib + debs: + - target/debs/{debian_version}/libyang_1.0*.deb + """) + uf = schema.load_upstream_file(path) + up = uf.upstreams[0] + assert up.run_id == 926659 + assert up.cascade_optional is True + assert up.debs[0].path == "target/debs/{debian_version}/libyang_1.0*.deb" + + +def test_duplicate_upstream_name_raises(tmp_path): + path = _write(tmp_path, "upstream-artifacts.yaml", """ + upstream: + - {name: dup, pipeline: p, artifact_name: a} + - {name: dup, pipeline: q, artifact_name: b} + """) + with pytest.raises(schema.SchemaError): + schema.load_upstream_file(path) + + +def test_upstream_missing_required_raises(tmp_path): + path = _write(tmp_path, "upstream-artifacts.yaml", """ + upstream: + - {name: x} + """) + with pytest.raises(schema.SchemaError): + schema.load_upstream_file(path) + + +# --------------------- list/map validation (fail-loud) ---------------------- # + +def test_package_pip_args_must_be_list(tmp_path): + path = _write(tmp_path, "base.yaml", """ + packages: + - {name: libyang, type: pip, pip_args: "--no-build-isolation"} + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_package_requires_must_be_list(tmp_path): + path = _write(tmp_path, "base.yaml", """ + packages: + - {name: libyang, type: pip, requires: libyang3} + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_post_install_scopes_must_be_list(tmp_path): + path = _write(tmp_path, "base.yaml", """ + post_install: + - {name: x, script: "echo hi", scopes: build} + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_upstream_deb_dpkg_args_must_be_list(tmp_path): + path = _write(tmp_path, "up.yaml", """ + upstream: + - name: u + pipeline: p + artifact_name: a + debs: + - {path: 'x_*.deb', dpkg_args: "--force-confnew"} + """) + with pytest.raises(schema.SchemaError): + schema.load_upstream_file(path) + + +def test_upstream_deb_scopes_must_be_list(tmp_path): + path = _write(tmp_path, "up.yaml", """ + upstream: + - name: u + pipeline: p + artifact_name: a + debs: + - {path: 'x_*.deb', scopes: test} + """) + with pytest.raises(schema.SchemaError): + schema.load_upstream_file(path) + + +def test_upstream_install_env_must_be_mapping(tmp_path): + path = _write(tmp_path, "up.yaml", """ + upstream: + - name: u + pipeline: p + artifact_name: a + install_env: [VPP_INSTALL_SKIP_SYSCTL=1] + """) + with pytest.raises(schema.SchemaError): + schema.load_upstream_file(path) + + +def test_upstream_install_env_mapping_ok(tmp_path): + path = _write(tmp_path, "up.yaml", """ + upstream: + - name: vpp + pipeline: p + artifact_name: a + install_env: {VPP_INSTALL_SKIP_SYSCTL: "1"} + """) + uf = schema.load_upstream_file(path) + assert uf.upstreams[0].install_env == {"VPP_INSTALL_SKIP_SYSCTL": "1"} + + +# ------------------------- missing-field errors ----------------------------- # + +def test_top_level_must_be_mapping(tmp_path): + path = _write(tmp_path, "base.yaml", "- just\n- a\n- list\n") + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_apt_source_missing_field(tmp_path): + path = _write(tmp_path, "base.yaml", """ + apt_sources: + - {name: dotnet, list_url: u} + packages: [] + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_package_missing_name(tmp_path): + path = _write(tmp_path, "base.yaml", """ + packages: + - {type: pip} + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_package_invalid_type(tmp_path): + path = _write(tmp_path, "base.yaml", """ + packages: + - {name: x, type: snap} + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_post_install_missing_name(tmp_path): + path = _write(tmp_path, "base.yaml", """ + post_install: + - {script: "echo hi"} + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_post_install_source_and_script_mutually_exclusive(tmp_path): + path = _write(tmp_path, "base.yaml", """ + post_install: + - {name: x, source: s.sh, script: "echo hi"} + """) + with pytest.raises(schema.SchemaError): + schema.load_packages_file(path, str(tmp_path)) + + +def test_upstream_missing_required_field(tmp_path): + path = _write(tmp_path, "up.yaml", """ + upstream: + - {name: u} + """) + with pytest.raises(schema.SchemaError): + schema.load_upstream_file(path) + + +def test_upstream_wheel_missing_path(tmp_path): + path = _write(tmp_path, "up.yaml", """ + upstream: + - name: u + pipeline: p + artifact_name: a + wheels: + - {when: {arch: amd64}} + """) + with pytest.raises(schema.SchemaError): + schema.load_upstream_file(path) diff --git a/ci/tests/test_topo.py b/ci/tests/test_topo.py new file mode 100644 index 000000000..bb8ab855f --- /dev/null +++ b/ci/tests/test_topo.py @@ -0,0 +1,23 @@ +import pytest + +from buildenv_setup.topo import CycleError, toposort + + +def test_preserves_order_without_deps(): + assert toposort(["a", "b", "c"], {}) == ["a", "b", "c"] + + +def test_requires_ordering(): + # c requires a; a must come before c + out = toposort(["c", "b", "a"], {"c": ["a"]}) + assert out.index("a") < out.index("c") + + +def test_requires_outside_items_ignored(): + # 'redis-server' isn't in the item set (installed elsewhere) -> ignored, no error + assert toposort(["x"], {"x": ["redis-server"]}) == ["x"] + + +def test_cycle_raises(): + with pytest.raises(CycleError): + toposort(["a", "b"], {"a": ["b"], "b": ["a"]})