From 64b2051a5fe79ff3b3661305e20de99ea8a2fb29 Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Mon, 6 Jul 2026 22:27:18 +0000 Subject: [PATCH 1/9] [ci]: use shared buildenv_setup for the build environment Replace the inline apt-get / DownloadPipelineArtifact / dpkg / redis / rsyslog steps in build-template.yml with a single invocation of the shared `buildenv_setup` tool (hosted in sonic-swss-common/ci/), driven by declarative config under build-env/: - build-env/packages/base.yaml core build+link deps (dbus/pcsc/hiredis/ swig/zmq/autoconf-archive, arm-only libnl, redis-server + configure-redis post_install) - build-env/packages/tooling.yaml docbook/aspell/rsyslog + sswsyncd/rsyslog - build-env/upstream-artifacts.yaml libswsscommon, common-libs (libnl+libyang3), vpp -- downloaded via the Azure DevOps REST API with the same run-selection semantics - build-env/configure-redis-for-tests.sh, build.sh, Dockerfile, compose.yaml build.sh is the single source of truth for the build command (autogen + dpkg-buildpackage -Psyncd,vs,nopython2, honouring ASAN), used by both CI and local dev (docker compose). The post-build steps (azsyslog rewrite, setcap, make check, gcov, syslog publish) stay inline in the template as they run after the build. Verified end-to-end in a sonic-slave-bookworm container against live Azure artifacts. TEMPORARY (PoC) pins, to be removed before merge: the sonic-swss-common clone and sonic-slave image are pinned to master / lawlee/ci-unify, and upstream artifacts are pinned to branch master (feature-branch artifacts don't exist). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Lawrence Lee --- .azure-pipelines/build-template.yml | 199 ++++--------------------- build-env/Dockerfile | 46 ++++++ build-env/README.md | 52 +++++++ build-env/build.sh | 30 ++++ build-env/compose.yaml | 33 ++++ build-env/configure-redis-for-tests.sh | 15 ++ build-env/packages/base.yaml | 51 +++++++ build-env/packages/tooling.yaml | 29 ++++ build-env/upstream-artifacts.yaml | 79 ++++++++++ 9 files changed, 360 insertions(+), 174 deletions(-) create mode 100644 build-env/Dockerfile create mode 100644 build-env/README.md create mode 100755 build-env/build.sh create mode 100644 build-env/compose.yaml create mode 100755 build-env/configure-redis-for-tests.sh create mode 100644 build-env/packages/base.yaml create mode 100644 build-env/packages/tooling.yaml create mode 100644 build-env/upstream-artifacts.yaml diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 1c5e620082..f0461f292b 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -59,189 +59,40 @@ jobs: vmImage: 'ubuntu-22.04' container: - image: sonicdev-microsoft.azurecr.io:443/sonic-slave-${{ parameters.debian_version }}:$(BUILD_BRANCH)-${{ parameters.arch }} + # TEMPORARY (PoC): pin the sonic-slave image tag to `master` instead of + # $(BUILD_BRANCH). Only master-tagged slave images are published; the + # lawlee/ci-unify feature branch has no image. Restore $(BUILD_BRANCH) before merge. + image: sonicdev-microsoft.azurecr.io:443/sonic-slave-${{ parameters.debian_version }}:master-${{ parameters.arch }} options: "--privileged" steps: - checkout: self clean: true submodules: true - - task: DownloadPipelineArtifact@2 - condition: eq('${{ parameters.arch }}', 'amd64') - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib - patterns: | - target/debs/${{ parameters.debian_version }}/libnl-3*.deb - target/debs/${{ parameters.debian_version }}/libnl-genl*.deb - target/debs/${{ parameters.debian_version }}/libnl-route*.deb - target/debs/${{ parameters.debian_version }}/libnl-nf*.deb - displayName: "Download libnl from common_libs" - - script: | - set -ex - sudo dpkg -i $(find ./download -name libnl-*_${{ parameters.arch }}.deb) - condition: eq('${{ parameters.arch }}', 'amd64') - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install sonic libnl" - script: | set -ex + # Bootstrap the tool's own runtime deps (buildenv_setup needs PyYAML + requests). sudo apt-get update - sudo apt-get install -qq -y \ - libdbus-glib-1-dev \ - libpcsclite-dev \ - docbook-to-man \ - docbook-utils \ - aspell-en \ - libhiredis-dev \ - swig \ - libzmq3-dev \ - autoconf-archive - - sudo apt-get install -y redis-server - sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf - sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf - sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf - sudo service redis-server start - sudo mkdir -m 755 /var/run/sswsyncd - - sudo apt-get install -y rsyslog - sudo rsyslogd - - cat /etc/apt/sources.list - dpkg --list | grep libnl - - displayName: "Install dependencies" - - script: | - set -ex - sudo apt-get update - sudo apt-get install -qq -y \ - libnl-3-dev \ - libnl-genl-3-dev \ - libnl-route-3-dev \ - libnl-nf-3-dev \ - libnl-cli-3-dev - - condition: ne('${{ parameters.arch }}', 'amd64') - displayName: "Install libnl for arm dependencies" - - task: DownloadPipelineArtifact@2 - # amd64 artifact name does not has arch suffix - condition: and(ne('${{ parameters.debian_version }}', 'trixie'), eq('${{ parameters.arch }}', 'amd64')) - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - displayName: "Download libyang from amd64 common lib" - - task: DownloadPipelineArtifact@2 - condition: and(ne('${{ parameters.debian_version }}', 'trixie'), ne('${{ parameters.arch }}', 'amd64')) - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib.${{ parameters.arch }} - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - - task: DownloadPipelineArtifact@2 - condition: and(eq('${{ parameters.debian_version }}', 'trixie'), eq('${{ parameters.arch }}', 'amd64')) - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - target/debs/${{ parameters.debian_version }}/libpcre3_*.deb - displayName: "Download libyang from amd64 common lib" - - task: DownloadPipelineArtifact@2 - condition: and(eq('${{ parameters.debian_version }}', 'trixie'), ne('${{ parameters.arch }}', 'amd64')) - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib.${{ parameters.arch }} - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - target/debs/${{ parameters.debian_version }}/libpcre3_*.deb - displayName: "Download libyang from common lib" - - script: | - set -ex - sudo dpkg -i $(find ./download -name *.deb) - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install libyang from common lib" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: sonic-net.sonic-platform-vpp - artifact: vpp-${{ parameters.debian_version }} - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - allowPartiallySucceededBuilds: true - path: $(Build.ArtifactStagingDirectory)/download - displayName: "Download sonic platform-vpp deb packages" - condition: eq('${{ parameters.arch }}', 'amd64') - - script: | - set -ex - sudo dpkg -i download/libvppinfra-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i download/libvppinfra_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-crypto-engines_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-dbg_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-plugin-core_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-plugin-devtools_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-plugin-dpdk_*_${{ parameters.arch }}.deb - sudo dpkg -i download/python3-vpp-api_*_${{ parameters.arch }}.deb - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install sonic platform-vpp packages" - condition: eq('${{ parameters.arch }}', 'amd64') - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: Azure.sonic-swss-common - artifact: ${{ parameters.swss_common_artifact_name }} - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - allowPartiallySucceededBuilds: true - path: $(Build.ArtifactStagingDirectory)/download - displayName: "Download sonic swss common deb packages" - - script: | - set -ex - sudo dpkg -i download/libswsscommon_1.0.0_${{ parameters.arch }}.deb - sudo dpkg -i download/libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb - rm -rf download - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install sonic swss Common" - - script: | - set -ex - rm ../*.deb || true - ./autogen.sh - extraflags='--enable-code-coverage' - if [ '${{ parameters.asan }}' == True ]; then - extraflags='--enable-asan' - fi - DEB_BUILD_OPTIONS=nocheck DEB_CONFIGURE_EXTRA_FLAGS=$extraflags dpkg-buildpackage -us -uc -b -Psyncd,vs,nopython2 -j$(nproc) - mv ../*.deb . + sudo apt-get install -qq -y python3-yaml python3-requests git + # buildenv_setup lives in sonic-swss-common/ci/; sonic-sairedis is a consumer, + # so clone sonic-swss-common to obtain the tool. + # TEMPORARY (PoC): pin to lawlee/ci-unify (the tool isn't on swss-common master + # yet). Once merged, clone $(BUILD_BRANCH) with a default-branch fallback. + git clone --depth 1 --branch lawlee/ci-unify \ + https://github.com/sonic-net/sonic-swss-common /tmp/sw-common + # Sets up apt/pip deps + downloads/installs libswsscommon, libnl, libyang3 and + # vpp DEBs (see build-env/upstream-artifacts.yaml), and runs the redis / + # sswsyncd / rsyslog post-install hooks. + PYTHONPATH=/tmp/sw-common/ci python3 -m buildenv_setup \ + --repo-dir $(Build.SourcesDirectory) \ + --scope build \ + --arch ${{ parameters.arch }} \ + --debian-version ${{ parameters.debian_version }} \ + --branch $(BUILD_BRANCH) + displayName: "Set up build environment (buildenv_setup)" + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - script: ASAN=${{ parameters.asan }} ./build-env/build.sh displayName: "Compile sonic sairedis with coverage enabled" - script: | set -ex diff --git a/build-env/Dockerfile b/build-env/Dockerfile new file mode 100644 index 0000000000..0cdfbd713f --- /dev/null +++ b/build-env/Dockerfile @@ -0,0 +1,46 @@ +# Local-dev image for building sonic-sairedis. +# +# CI does NOT use this file: CI runs inside `container: sonic-slave-*` and invokes +# buildenv_setup directly (see .azure-pipelines/build-template.yml). This exists +# so a developer can reproduce the CI build environment locally. +# +# sonic-sairedis is a CONSUMER of the shared buildenv_setup tool, which lives in +# sonic-swss-common/ci/. So (unlike sonic-swss-common) this image clones +# sonic-swss-common to obtain the tool. +# +# Layer ordering (design finding F7): clone the tool + copy ONLY the dependency +# declarations, run setup, and do NOT copy the source. build-env/compose.yaml +# mounts the working tree at /workspace at runtime, so editing source never +# invalidates the (heavy) dependency-setup layer below. + +ARG DEBIAN_VERSION=bookworm +FROM sonicdev-microsoft.azurecr.io:443/sonic-slave-${DEBIAN_VERSION}:latest + +ARG DEBIAN_VERSION=bookworm +# TEMPORARY (PoC): the buildenv_setup tool currently lives on sonic-swss-common's +# lawlee/ci-unify branch (not master yet). Pin the clone to that branch until it +# merges, after which this can track the matching build branch. +ARG SWSS_COMMON_REF=lawlee/ci-unify +ARG BUILD_BRANCH=master + +# Clone sonic-swss-common to obtain the shared buildenv_setup tool. +RUN git clone --depth 1 --branch ${SWSS_COMMON_REF} \ + https://github.com/sonic-net/sonic-swss-common /tmp/sw-common + +# Copy ONLY the dependency declarations so the heavy setup layer stays cached. +COPY build-env/ /workspace/build-env/ + +# Bootstrap the tool's own runtime deps, then set up the build environment. +# --org-url points at the public SONiC Azure DevOps org for artifact download +# (in CI this comes from $SYSTEM_COLLECTIONURI). Set AZURE_DEVOPS_EXT_PAT, or use +# --upstream-staged-dir, if your environment needs auth. +RUN apt-get update && apt-get install -y python3-yaml python3-requests \ + && PYTHONPATH=/tmp/sw-common/ci python3 -m buildenv_setup \ + --repo-dir /workspace \ + --scope build \ + --debian-version "${DEBIAN_VERSION}" \ + --branch "${BUILD_BRANCH}" \ + --org-url https://dev.azure.com/mssonic \ + --no-sudo + +WORKDIR /workspace diff --git a/build-env/README.md b/build-env/README.md new file mode 100644 index 0000000000..4075cfd0b0 --- /dev/null +++ b/build-env/README.md @@ -0,0 +1,52 @@ +# build-env/ + +Declarative build-environment configuration for sonic-sairedis, consumed by the +shared [`buildenv_setup`](https://github.com/sonic-net/sonic-swss-common/tree/master/ci) +tool (which lives in **sonic-swss-common**; sonic-sairedis is a consumer). This +is the single source of truth for "how to build this repo". + +## Contents + +| Path | Purpose | Cascades downstream? | +|------|---------|----------------------| +| `packages/base.yaml` | apt/pip packages needed to build **and link against** libsairedis (dbus/pcsc/hiredis/swig/zmq/autoconf-archive, arm libnl, redis + config) | **Yes** | +| `packages/tooling.yaml` | Build-stage-only tooling (docbook/aspell for man pages, rsyslog; sswsyncd dir) | No | +| `upstream-artifacts.yaml` | upstream DEBs: libswsscommon (sonic-swss-common), libnl+libyang3 (common-libs), vpp | Yes | +| `configure-redis-for-tests.sh` | redis test-config (no `notify-keyspace-events` — see F11), run via `post_install` | (travels with base.yaml) | +| `build.sh` | canonical build (`autogen` + `dpkg-buildpackage -Psyncd,vs,nopython2`), used by CI **and** local dev; `ASAN=true` for the asan build | — | +| `Dockerfile`, `compose.yaml` | local-dev image (CI does not use these) | — | + +## How CI uses it + +`.azure-pipelines/build-template.yml`, inside `container: sonic-slave-*`, clones +sonic-swss-common to get the tool and runs: + +```bash +PYTHONPATH=/tmp/sw-common/ci python3 -m buildenv_setup \ + --repo-dir $(Build.SourcesDirectory) --scope build \ + --arch --debian-version --branch $(BUILD_BRANCH) +ASAN= ./build-env/build.sh +``` + +`buildenv_setup` installs the apt/pip packages, downloads + installs the +libswsscommon / libnl / libyang3 / vpp DEBs (from `upstream-artifacts.yaml`), and +runs the redis / sswsyncd / rsyslog post-install hooks. Then `build.sh` builds +the package. + +## Local development + +```bash +cd build-env +DEBIAN_VERSION=bookworm docker compose run --rm build # build .debs +DEBIAN_VERSION=bookworm docker compose run --rm shell # poke around +``` + +The image bakes the dependency setup; your source is mounted live at +`/workspace`, so ordinary edits don't require an image rebuild. Parity with CI +covers **building and C++ unit tests** (`make check`); the full VS/DVS test suite +needs CI-like infrastructure (KVM/privileged/nested-docker). + +> **PoC note:** the tool currently lives on sonic-swss-common's `lawlee/ci-unify` +> branch, so the Dockerfile/CI clone is pinned there (`SWSS_COMMON_REF`), and the +> upstream artifacts + sonic-slave image are pinned to `master`. These TEMPORARY +> pins are removed once the change merges. diff --git a/build-env/build.sh b/build-env/build.sh new file mode 100755 index 0000000000..e2491057cc --- /dev/null +++ b/build-env/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Canonical build for sonic-sairedis. +# +# Single source of truth for "how to build this repo", used by BOTH CI +# (.azure-pipelines/build-template.yml) and local dev (build-env/compose.yaml). +# Must NOT depend on CI-only environment variables (only the optional ASAN flag). +# +# The build environment (apt/pip deps + upstream libnl/libyang/vpp/swss-common +# artifacts) is set up beforehand by `buildenv_setup` (see build-env/README.md); +# this script only runs the actual package build. +# +# ASAN=true ./build-env/build.sh # build with AddressSanitizer instead of gcov +set -ex + +# Run from the repo root regardless of where we were invoked. +cd "$(dirname "$0")/.." + +extraflags='--enable-code-coverage' +case "${ASAN:-false}" in + [Tt]rue|1|yes) extraflags='--enable-asan' ;; +esac + +rm -f ../*.deb || true +./autogen.sh +DEB_BUILD_OPTIONS=nocheck DEB_CONFIGURE_EXTRA_FLAGS="$extraflags" \ + dpkg-buildpackage -us -uc -b -Psyncd,vs,nopython2 -j"$(nproc)" + +# Collect the built .debs at the repo root (where CI publishes from). +mv ../*.deb . diff --git a/build-env/compose.yaml b/build-env/compose.yaml new file mode 100644 index 0000000000..f5c1754baf --- /dev/null +++ b/build-env/compose.yaml @@ -0,0 +1,33 @@ +# Local-dev convenience for building sonic-sairedis in a CI-like container. +# +# Usage: +# cd build-env +# DEBIAN_VERSION=bookworm docker compose run --rm build # build the .debs +# DEBIAN_VERSION=bookworm docker compose run --rm shell # interactive shell +# +# The repo is mounted live at /workspace, so edits on the host are immediately +# visible in the container without rebuilding the image (only a change to +# build-env/ dependency declarations requires an image rebuild). Set ASAN=true to +# build with AddressSanitizer instead of gcov. + +services: + build: + build: + context: .. + dockerfile: build-env/Dockerfile + args: + DEBIAN_VERSION: ${DEBIAN_VERSION:-bookworm} + BUILD_BRANCH: ${BUILD_BRANCH:-master} + SWSS_COMMON_REF: ${SWSS_COMMON_REF:-lawlee/ci-unify} + privileged: true + volumes: + - ..:/workspace + working_dir: /workspace + environment: + ASAN: ${ASAN:-false} + command: ["./build-env/build.sh"] + + shell: + extends: + service: build + command: ["/bin/bash"] diff --git a/build-env/configure-redis-for-tests.sh b/build-env/configure-redis-for-tests.sh new file mode 100755 index 0000000000..580d459934 --- /dev/null +++ b/build-env/configure-redis-for-tests.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# Configure the system redis-server for sonic-sairedis' tests. +# Applied via the `configure-redis-for-tests` post_install entry in +# build-env/packages/base.yaml. +# +# Reproduces exactly the redis config build-template.yml applied inline. +# NOTE (F11): sonic-sairedis does NOT set `notify-keyspace-events AKE` +# (unlike sonic-swss-common / sonic-swss) -- do not add it here. +set -ex + +sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf +sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf +sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf +sudo service redis-server start diff --git a/build-env/packages/base.yaml b/build-env/packages/base.yaml new file mode 100644 index 0000000000..4bd9f55797 --- /dev/null +++ b/build-env/packages/base.yaml @@ -0,0 +1,51 @@ +# sonic-sairedis build dependencies (apt + pip). +# +# base.yaml CASCADES: repos downstream of sonic-sairedis (sonic-swss) that build +# against libsairedis inherit everything declared here. +# +# Consumed by `buildenv_setup --scope build`. The buildenv_setup tool itself +# lives in sonic-swss-common/ci/ (cloned by the CI template); sonic-sairedis is a +# consumer. +# +# NOTE: once sonic-swss-common publishes its own build-env/ to the branch this +# build resolves, the sonic-swss-common cascade (see upstream-artifacts.yaml) +# will also contribute its base.yaml deps (libhiredis, swig, redis-server, ...). +# Those are re-declared here for now so the PoC is self-contained and correct +# regardless of cascade availability (apt install is idempotent). + +packages: + # --- core C/C++ build + link dependencies (from build-template.yml) -------- + - libdbus-glib-1-dev + - libpcsclite-dev + - libhiredis-dev + - swig + - libzmq3-dev + - autoconf-archive + + # libnl: on amd64 these come from the common-libs artifact (see + # upstream-artifacts.yaml); on arm they are apt-installed here (mirrors the + # "Install libnl for arm dependencies" step in build-template.yml). + - { name: libnl-3-dev, when: { arch: { not: amd64 } } } + - { name: libnl-genl-3-dev, when: { arch: { not: amd64 } } } + - { name: libnl-route-3-dev, when: { arch: { not: amd64 } } } + - { name: libnl-nf-3-dev, when: { arch: { not: amd64 } } } + - { name: libnl-cli-3-dev, when: { arch: { not: amd64 } } } + + # redis-server is needed to run sonic-sairedis' unit tests during the Build + # stage (and by downstream VS test hosts). Cascades. + - redis-server + +post_install: + # Configure the system redis for sonic-sairedis' tests. + # + # NOTE (design finding F11): sonic-sairedis' redis config does NOT set + # `notify-keyspace-events AKE` (unlike sonic-swss-common / sonic-swss). This is + # why a single shared cascading redis script can't be used naively: the common + # lines (unixsocket / unixsocketperm) belong in sonic-swss-common's cascading + # base.yaml, while the per-repo deltas stay local. For this PoC (sonic-swss-common + # build-env not yet on the resolved branch) sonic-sairedis carries its own + # complete-but-minimal redis config. + - name: configure-redis-for-tests + source: configure-redis-for-tests.sh + requires: [redis-server] + scopes: [build, test] diff --git a/build-env/packages/tooling.yaml b/build-env/packages/tooling.yaml new file mode 100644 index 0000000000..fc6dbff08c --- /dev/null +++ b/build-env/packages/tooling.yaml @@ -0,0 +1,29 @@ +# sonic-sairedis Build-stage-only tooling (does NOT cascade downstream). +# +# Packages/config needed by sonic-sairedis' own build + unit tests but not by +# repos that merely build against libsairedis. Consumed by +# `buildenv_setup --scope build`. + +packages: + # man-page / doc generation used during the sairedis build + - docbook-to-man + - docbook-utils + - aspell-en + # syslog used by the unit tests + - rsyslog + +post_install: + # syncd unit tests expect this runtime directory to exist. + - name: mkdir-sswsyncd + script: | + sudo mkdir -m 755 -p /var/run/sswsyncd + scopes: [build] + + # Start rsyslog so tests that log to syslog work. (The post-build + # "Update rsyslog.conf" step in build-template.yml swaps in azsyslog.conf and + # restarts rsyslogd; that stays in the template as it runs after the build.) + - name: start-rsyslog + script: | + sudo rsyslogd || true + requires: [rsyslog] + scopes: [build] diff --git a/build-env/upstream-artifacts.yaml b/build-env/upstream-artifacts.yaml new file mode 100644 index 0000000000..6b3c403526 --- /dev/null +++ b/build-env/upstream-artifacts.yaml @@ -0,0 +1,79 @@ +# Upstream build artifacts that sonic-sairedis needs at build time. +# +# Consumed by `buildenv_setup`, which downloads each via the Azure DevOps REST +# API (or uses a --upstream-staged-dir// bundle when staged in the same +# pipeline run, e.g. sonic-swss-common's own CI building sairedis downstream). +# +# Artifact-name convention: amd64 has no arch suffix; non-amd64 appends .{arch}. + +upstream: + # ------------------------------------------------------------------ # + # sonic-swss-common libraries that sonic-sairedis links against. + # + # cascade_optional: true -> if the resolved sonic-swss-common artifact ships a + # build-env/ (once sonic-swss-common# merges), buildenv_setup will cascade + # its base.yaml deps automatically; until then it is treated as a leaf and we + # just install the libswsscommon DEBs below. + # ------------------------------------------------------------------ # + - name: sonic-swss-common + project: build + pipeline: Azure.sonic-swss-common + branch: master # TEMPORARY (PoC): the lawlee/ci-unify pipeline run may + # not be present/stable; pin the linked libs to master. + # Remove before merge so it tracks BUILD_BRANCH. + cascade_optional: true + result_filter: [succeeded, partiallySucceeded] + artifact_name: + - { when: { arch: amd64 }, value: 'sonic-swss-common-{debian_version}' } + - { when: { arch: { not: amd64 } }, value: 'sonic-swss-common-{debian_version}.{arch}' } + debs: + - 'libswsscommon_1.0.0_{arch}.deb' + - 'libswsscommon-dev_1.0.0_{arch}.deb' + + # ------------------------------------------------------------------ # + # libnl (amd64 only) + libyang3 from the common-libs build. + # On arm, libnl is apt-installed instead (see packages/base.yaml). + # ------------------------------------------------------------------ # + - name: common-libs + project: build + pipeline: Azure.sonic-buildimage.common_libs + branch: master # TEMPORARY (PoC): common_libs has no lawlee/ci-unify + # branch. Remove before merge. + cascade_optional: true + artifact_name: + - { when: { arch: amd64 }, value: 'common-lib' } + - { when: { arch: { not: amd64 } }, value: 'common-lib.{arch}' } + debs: + # libnl DEBs: amd64 only + - { path: 'target/debs/{debian_version}/libnl-3*.deb', when: { arch: amd64 } } + - { path: 'target/debs/{debian_version}/libnl-genl*.deb', when: { arch: amd64 } } + - { path: 'target/debs/{debian_version}/libnl-route*.deb', when: { arch: amd64 } } + - { path: 'target/debs/{debian_version}/libnl-nf*.deb', when: { arch: amd64 } } + # libyang3: all arches + - 'target/debs/{debian_version}/libyang3_*.deb' + # libpcre3: trixie only + - { path: 'target/debs/{debian_version}/libpcre3_*.deb', when: { debian_version: trixie } } + + # ------------------------------------------------------------------ # + # sonic platform-vpp DEBs (amd64 only). Always built on master. + # ------------------------------------------------------------------ # + - name: vpp + project: build + pipeline: sonic-net.sonic-platform-vpp + branch: master # vpp is always pinned to master (unchanged from today). + when: { arch: amd64 } + cascade_optional: true + result_filter: [succeeded, partiallySucceeded] + apt_fix_broken: true # the 10 vpp DEBs are interdependent; fix up after dpkg -i + artifact_name: 'vpp-{debian_version}' + debs: + - 'libvppinfra-dev_*_{arch}.deb' + - 'libvppinfra_*_{arch}.deb' + - 'vpp_*_{arch}.deb' + - 'vpp-crypto-engines_*_{arch}.deb' + - 'vpp-dbg_*_{arch}.deb' + - 'vpp-dev_*_{arch}.deb' + - 'vpp-plugin-core_*_{arch}.deb' + - 'vpp-plugin-devtools_*_{arch}.deb' + - 'vpp-plugin-dpdk_*_{arch}.deb' + - 'python3-vpp-api_*_{arch}.deb' From 662f37592f822cd79c68e2c597dc497a04b423ba Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Mon, 6 Jul 2026 23:14:48 +0000 Subject: [PATCH 2/9] [ci]: thin common-libs + libnl from sairedis, rely on swss-common cascade Remove the common-libs upstream-artifacts entry AND the libnl apt packages from sonic-sairedis. Both libyang3 and the SONiC-patched libnl family (including libnl-cli) are now inherited transitively via the sonic-swss-common cascade: buildenv_setup downloads the sonic-swss-common artifact, reads its build-env/, and walks sonic-swss-common's upstream-artifacts.yaml, which declares common-libs -> libnl + libyang3 (all downloaded from the common-libs artifact, never apt). This replaces sonic-sairedis' old split of amd64-libnl-from-common-lib + arm-libnl-from-apt. Flip the sonic-swss-common upstream to cascade_optional: false so a missing build-env/ in the resolved artifact WARNS loudly. NOTE: depends on the resolved sonic-swss-common artifact shipping a build-env/, which only happens once sonic-swss-common is built from a branch that carries the tool. The sonic-swss-common upstream is still pinned to branch:master here, so this commit is intentionally incomplete and MUST NOT be pushed until that pin is lifted (to lawlee/ci-unify / BUILD_BRANCH) AND a green sonic-swss-common lawlee/ci-unify run has published an artifact with build-env/. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Lawrence Lee --- build-env/packages/base.yaml | 15 +++++------ build-env/upstream-artifacts.yaml | 45 +++++++++++-------------------- 2 files changed, 23 insertions(+), 37 deletions(-) diff --git a/build-env/packages/base.yaml b/build-env/packages/base.yaml index 4bd9f55797..414629fb78 100644 --- a/build-env/packages/base.yaml +++ b/build-env/packages/base.yaml @@ -22,14 +22,13 @@ packages: - libzmq3-dev - autoconf-archive - # libnl: on amd64 these come from the common-libs artifact (see - # upstream-artifacts.yaml); on arm they are apt-installed here (mirrors the - # "Install libnl for arm dependencies" step in build-template.yml). - - { name: libnl-3-dev, when: { arch: { not: amd64 } } } - - { name: libnl-genl-3-dev, when: { arch: { not: amd64 } } } - - { name: libnl-route-3-dev, when: { arch: { not: amd64 } } } - - { name: libnl-nf-3-dev, when: { arch: { not: amd64 } } } - - { name: libnl-cli-3-dev, when: { arch: { not: amd64 } } } + # libnl: NOT declared here at all. The SONiC-patched libnl family + # (libnl-{3,genl,route,nf,cli}-*) is inherited via the sonic-swss-common + # cascade -- sonic-swss-common's upstream-artifacts.yaml downloads the full + # libnl set (incl libnl-cli) from the common-libs artifact for all arches, and + # that cascades transitively to sonic-sairedis. Never apt-install libnl where + # common-libs provides it (see repo policy). This replaces sonic-sairedis' old + # split of amd64-libnl-from-common-lib + arm-libnl-from-apt. # redis-server is needed to run sonic-sairedis' unit tests during the Build # stage (and by downstream VS test hosts). Cascades. diff --git a/build-env/upstream-artifacts.yaml b/build-env/upstream-artifacts.yaml index 6b3c403526..6e9eb551b6 100644 --- a/build-env/upstream-artifacts.yaml +++ b/build-env/upstream-artifacts.yaml @@ -10,10 +10,21 @@ upstream: # ------------------------------------------------------------------ # # sonic-swss-common libraries that sonic-sairedis links against. # - # cascade_optional: true -> if the resolved sonic-swss-common artifact ships a - # build-env/ (once sonic-swss-common# merges), buildenv_setup will cascade - # its base.yaml deps automatically; until then it is treated as a leaf and we - # just install the libswsscommon DEBs below. + # This is sonic-sairedis' only cascading upstream (vpp below is a sairedis-only + # leaf). Note what is deliberately NOT declared here: + # * common-libs (libyang3) -- inherited TRANSITIVELY. buildenv_setup downloads + # the sonic-swss-common artifact, reads its build-env/, and walks + # sonic-swss-common's own upstream-artifacts.yaml, which declares common-libs + # -> libyang3 (the lib libswsscommon links against). See cascade.py. + # * libnl -- inherited from sonic-swss-common's base.yaml (apt libnl-*-dev, all + # arches), so it no longer needs a per-arch declaration here either. + # + # This REQUIRES the resolved sonic-swss-common artifact to actually ship a + # build-env/, i.e. sonic-swss-common must be built from a branch that already + # carries the buildenv_setup tool + build-env/ (see branch resolution below). + # cascade_optional: false so buildenv_setup WARNS loudly if the artifact has no + # build-env/ to cascade into -- otherwise the cascade silently no-ops and the + # libswsscommon dpkg -i fails downstream on the unmet libyang3 dependency. # ------------------------------------------------------------------ # - name: sonic-swss-common project: build @@ -21,7 +32,7 @@ upstream: branch: master # TEMPORARY (PoC): the lawlee/ci-unify pipeline run may # not be present/stable; pin the linked libs to master. # Remove before merge so it tracks BUILD_BRANCH. - cascade_optional: true + cascade_optional: false result_filter: [succeeded, partiallySucceeded] artifact_name: - { when: { arch: amd64 }, value: 'sonic-swss-common-{debian_version}' } @@ -30,30 +41,6 @@ upstream: - 'libswsscommon_1.0.0_{arch}.deb' - 'libswsscommon-dev_1.0.0_{arch}.deb' - # ------------------------------------------------------------------ # - # libnl (amd64 only) + libyang3 from the common-libs build. - # On arm, libnl is apt-installed instead (see packages/base.yaml). - # ------------------------------------------------------------------ # - - name: common-libs - project: build - pipeline: Azure.sonic-buildimage.common_libs - branch: master # TEMPORARY (PoC): common_libs has no lawlee/ci-unify - # branch. Remove before merge. - cascade_optional: true - artifact_name: - - { when: { arch: amd64 }, value: 'common-lib' } - - { when: { arch: { not: amd64 } }, value: 'common-lib.{arch}' } - debs: - # libnl DEBs: amd64 only - - { path: 'target/debs/{debian_version}/libnl-3*.deb', when: { arch: amd64 } } - - { path: 'target/debs/{debian_version}/libnl-genl*.deb', when: { arch: amd64 } } - - { path: 'target/debs/{debian_version}/libnl-route*.deb', when: { arch: amd64 } } - - { path: 'target/debs/{debian_version}/libnl-nf*.deb', when: { arch: amd64 } } - # libyang3: all arches - - 'target/debs/{debian_version}/libyang3_*.deb' - # libpcre3: trixie only - - { path: 'target/debs/{debian_version}/libpcre3_*.deb', when: { debian_version: trixie } } - # ------------------------------------------------------------------ # # sonic platform-vpp DEBs (amd64 only). Always built on master. # ------------------------------------------------------------------ # From ab03b534cc9d4fbe0178234b8534e84e6c3250aa Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Tue, 7 Jul 2026 00:37:58 +0000 Subject: [PATCH 3/9] [ci]: publish build-env/ in the artifact for downstream cascade Same fix as sonic-swss-common: .artifactignore was '**/* + !*.deb', publishing only debs. Un-ignore build-env/ so sonic-swss can cascade sonic-sairedis' dependency declarations via buildenv_setup once it is onboarded. (Not exercised in the current swss-common -> sairedis PoC, but keeps the repos consistent.) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Lawrence Lee --- .artifactignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.artifactignore b/.artifactignore index 1126a160d9..e21cd0e7b5 100644 --- a/.artifactignore +++ b/.artifactignore @@ -1,2 +1,7 @@ **/* !*.deb +# Also publish build-env/ so downstream consumers (sonic-swss) can cascade this +# repo's dependency declarations via buildenv_setup (reads build-env/ out of the +# downloaded artifact). Both the dir entry and its contents must be un-ignored. +!build-env +!build-env/** From 936905666dc11cd0805af7763daf4a5f2e0850ee Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Tue, 7 Jul 2026 00:49:21 +0000 Subject: [PATCH 4/9] [ci]: unpin sonic-swss-common upstream to track BUILD_BRANCH Remove the TEMPORARY 'branch: master' pin on the sonic-swss-common upstream so it resolves on ctx.branch (--branch $(BUILD_BRANCH)). On a lawlee/ci-unify branch build that is lawlee/ci-unify, whose sonic-swss-common artifact now ships build-env/ (commit 309a2bb) -- so the cascade activates and sonic-sairedis inherits the SONiC-patched libnl + libyang3 from common-libs transitively. NOTE (PoC): a PR build uses BUILD_BRANCH==master, whose artifact has no build-env/ yet, so the cascade would no-op there; validate this PoC via a lawlee/ci-unify branch build, not the PR build. common-libs/vpp pipelines have no lawlee/ci-unify branch, so those upstreams stay pinned to master. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Lawrence Lee --- build-env/upstream-artifacts.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/build-env/upstream-artifacts.yaml b/build-env/upstream-artifacts.yaml index 6e9eb551b6..1cea628895 100644 --- a/build-env/upstream-artifacts.yaml +++ b/build-env/upstream-artifacts.yaml @@ -12,12 +12,11 @@ upstream: # # This is sonic-sairedis' only cascading upstream (vpp below is a sairedis-only # leaf). Note what is deliberately NOT declared here: - # * common-libs (libyang3) -- inherited TRANSITIVELY. buildenv_setup downloads - # the sonic-swss-common artifact, reads its build-env/, and walks - # sonic-swss-common's own upstream-artifacts.yaml, which declares common-libs - # -> libyang3 (the lib libswsscommon links against). See cascade.py. - # * libnl -- inherited from sonic-swss-common's base.yaml (apt libnl-*-dev, all - # arches), so it no longer needs a per-arch declaration here either. + # * common-libs (SONiC-patched libnl + libyang3) -- inherited TRANSITIVELY. + # buildenv_setup downloads the sonic-swss-common artifact, reads its + # build-env/, and walks sonic-swss-common's own upstream-artifacts.yaml, + # which declares common-libs -> libnl + libyang3 (the libs libswsscommon + # links against -- all from the common-libs artifact, never apt). See cascade.py. # # This REQUIRES the resolved sonic-swss-common artifact to actually ship a # build-env/, i.e. sonic-swss-common must be built from a branch that already @@ -29,9 +28,11 @@ upstream: - name: sonic-swss-common project: build pipeline: Azure.sonic-swss-common - branch: master # TEMPORARY (PoC): the lawlee/ci-unify pipeline run may - # not be present/stable; pin the linked libs to master. - # Remove before merge so it tracks BUILD_BRANCH. + # No `branch:` pin -> resolves on ctx.branch (--branch $(BUILD_BRANCH)). On a + # lawlee/ci-unify BRANCH build BUILD_BRANCH=lawlee/ci-unify, whose + # sonic-swss-common artifact now ships build-env/ (so the cascade works). NOTE + # (PoC): a PR build uses BUILD_BRANCH==master, whose artifact has no + # build-env/ yet -> validate this PoC via a branch build, not the PR build. cascade_optional: false result_filter: [succeeded, partiallySucceeded] artifact_name: From 87de73d33b657128aeee0539285cac90c0dbc436 Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Tue, 7 Jul 2026 20:06:53 +0000 Subject: [PATCH 5/9] [ci]: TEMPORARY (PoC) pin downstream stages for lawlee/ci-unify branch build Get a manual sairedis lawlee/ci-unify branch build green: - Pin the sonic-slave container tag, sonic-swss checkout, and the common_libs / sonic-dash-api / sonic-buildimage-ubuntu22.04 runBranches to master in the BuildSwss/BuildDocker/Test templates (none of those exist on lawlee/ci-unify). - Keep the swss-common cascade artifacts on lawlee/ci-unify, but pin them EXPLICITLY (upstream-artifacts.yaml branch: + the 3 runBranches) rather than via BUILD_BRANCH: on a branch build BUILD_BRANCH = Build.SourceBranchName = 'ci-unify' (Azure returns only the last ref segment), which can't resolve the lawlee/ci-unify swss-common run. - Add the SONiC libnl runtime debs to the vstest host install (the jammy swss-common deb depends on the +b1sonicN libnl, absent from the host apt). All temporary; marked 'TEMPORARY (PoC)' with the value to restore before merge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Lawrence Lee --- .../build-docker-sonic-vs-template.yml | 13 +++++-- .azure-pipelines/build-swss-template.yml | 23 +++++++++--- .../test-docker-sonic-vs-template.yml | 36 ++++++++++++++++--- build-env/upstream-artifacts.yaml | 11 +++--- 4 files changed, 65 insertions(+), 18 deletions(-) diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml index 77cbad1ceb..864670c908 100644 --- a/.azure-pipelines/build-docker-sonic-vs-template.yml +++ b/.azure-pipelines/build-docker-sonic-vs-template.yml @@ -46,7 +46,10 @@ jobs: artifact: ${{ parameters.swss_common_artifact_name }} path: $(Build.ArtifactStagingDirectory)/download runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): resolve the swss-common cascade artifact from lawlee/ci-unify. + # BUILD_BRANCH = Build.SourceBranchName = "ci-unify" (last ref segment only), which + # can't find the lawlee/ci-unify run. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/lawlee/ci-unify' allowPartiallySucceededBuilds: true displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 @@ -66,7 +69,9 @@ jobs: pipeline: Azure.sonic-buildimage.common_libs artifact: common-lib runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; the common_libs pipeline has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download patterns: '**/target/debs/*/libnexthopgroup_*.deb' displayName: "Download sonic-buildimage libnexthopgroup package" @@ -95,7 +100,9 @@ jobs: ${{ else }}: artifact: sonic-dash-api.${{ parameters.arch }} runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; sonic-dash-api has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download patterns: | libdashapi*.deb diff --git a/.azure-pipelines/build-swss-template.yml b/.azure-pipelines/build-swss-template.yml index 0ea7e54866..f052c09bb2 100644 --- a/.azure-pipelines/build-swss-template.yml +++ b/.azure-pipelines/build-swss-template.yml @@ -45,7 +45,10 @@ jobs: vmImage: 'ubuntu-22.04' container: - image: sonicdev-microsoft.azurecr.io:443/sonic-slave-${{ parameters.debian_version }}:$(BUILD_BRANCH)-${{ parameters.arch }} + # TEMPORARY (PoC): pin the sonic-slave image tag to `master` instead of + # $(BUILD_BRANCH). Only master-tagged slave images are published; the + # lawlee/ci-unify feature branch has no image. Restore $(BUILD_BRANCH) before merge. + image: sonicdev-microsoft.azurecr.io:443/sonic-slave-${{ parameters.debian_version }}:master-${{ parameters.arch }} steps: - checkout: sonic-swss @@ -53,7 +56,10 @@ jobs: clean: true - script: | set -ex - git checkout $(BUILD_BRANCH) + # TEMPORARY (PoC): check out `master` instead of $(BUILD_BRANCH). sonic-swss + # has no lawlee/ci-unify branch on the sonic-net remote. Restore $(BUILD_BRANCH) + # before merge. + git checkout master git submodule update git status displayName: Set up sonic-swss branch @@ -63,7 +69,9 @@ jobs: project: build pipeline: Azure.sonic-buildimage.common_libs runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; the common_libs pipeline has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download artifact: common-lib patterns: | @@ -109,7 +117,10 @@ jobs: pipeline: Azure.sonic-swss-common artifact: ${{ parameters.swss_common_artifact_name }} runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): resolve the swss-common cascade artifact from lawlee/ci-unify. + # BUILD_BRANCH = Build.SourceBranchName = "ci-unify" (last ref segment only), which + # can't find the lawlee/ci-unify run. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/lawlee/ci-unify' allowPartiallySucceededBuilds: true path: $(Build.ArtifactStagingDirectory)/download patterns: | @@ -139,7 +150,9 @@ jobs: ${{ else }}: artifact: sonic-dash-api.${{ parameters.arch }} runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; sonic-dash-api has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download patterns: | libdashapi*.deb diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 1adf5d6cee..ded2f63104 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -40,7 +40,10 @@ jobs: - script: | set -ex cd sonic-swss - git checkout $(BUILD_BRANCH) + # TEMPORARY (PoC): check out `master` instead of $(BUILD_BRANCH). sonic-swss + # has no lawlee/ci-unify branch on the sonic-net remote. Restore $(BUILD_BRANCH) + # before merge. + git checkout master displayName: Set up sonic-swss branch - task: DownloadPipelineArtifact@2 inputs: @@ -55,7 +58,10 @@ jobs: artifact: sonic-swss-common.amd64.ubuntu22_04 path: $(Build.ArtifactStagingDirectory)/download runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): resolve the swss-common cascade artifact from lawlee/ci-unify. + # BUILD_BRANCH = Build.SourceBranchName = "ci-unify" (last ref segment only), which + # can't find the lawlee/ci-unify run. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/lawlee/ci-unify' allowPartiallySucceededBuilds: true displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 @@ -64,13 +70,19 @@ jobs: project: build pipeline: Azure.sonic-buildimage.common_libs runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; the common_libs pipeline has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download artifact: common-lib patterns: | + target/debs/${{ parameters.debian_version }}/libnl-3-200_*.deb + target/debs/${{ parameters.debian_version }}/libnl-genl-3-200_*.deb + target/debs/${{ parameters.debian_version }}/libnl-route-3-200_*.deb + target/debs/${{ parameters.debian_version }}/libnl-nf-3-200_*.deb target/debs/${{ parameters.debian_version }}/libyang3_*.deb target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb - displayName: "Download libyang from common lib" + displayName: "Download libnl + libyang from common lib" - task: DownloadPipelineArtifact@2 inputs: source: specific @@ -78,7 +90,10 @@ jobs: pipeline: sonic-net.sonic-buildimage-ubuntu22.04 artifact: sonic-buildimage.amd64.ubuntu22_04 runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; the sonic-buildimage-ubuntu22.04 + # pipeline has no lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) + # before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic buildimage ubuntu deb packages" @@ -105,6 +120,17 @@ jobs: sudo sonic-sairedis/.azure-pipelines/build_and_install_module.sh + # Install SONiC-patched libnl (3.7.0-0.2+b1sonicN) from common-lib BEFORE the + # swss-common deb. The jammy swss-common .deb (built by the ubuntu-22.04 job + # against common-lib libnl) depends on the +b1sonicN libnl, which the 22.04 + # host's apt does not have. Use apt so the bookworm libnl runtime debs' + # dependencies resolve on the jammy host (same as the libyang install below). + sudo apt-get install -y \ + $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libnl-3-200_*.deb \ + $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libnl-genl-3-200_*.deb \ + $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libnl-route-3-200_*.deb \ + $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libnl-nf-3-200_*.deb + # Install libyang packages from downloaded artifacts. Use apt rather # than `dpkg -i` so libyang-dev's libpcre2-dev dep is auto-resolved # from the host repos. diff --git a/build-env/upstream-artifacts.yaml b/build-env/upstream-artifacts.yaml index 1cea628895..19a0e272b1 100644 --- a/build-env/upstream-artifacts.yaml +++ b/build-env/upstream-artifacts.yaml @@ -28,11 +28,12 @@ upstream: - name: sonic-swss-common project: build pipeline: Azure.sonic-swss-common - # No `branch:` pin -> resolves on ctx.branch (--branch $(BUILD_BRANCH)). On a - # lawlee/ci-unify BRANCH build BUILD_BRANCH=lawlee/ci-unify, whose - # sonic-swss-common artifact now ships build-env/ (so the cascade works). NOTE - # (PoC): a PR build uses BUILD_BRANCH==master, whose artifact has no - # build-env/ yet -> validate this PoC via a branch build, not the PR build. + # TEMPORARY (PoC): pin the cascade source to the POC branch. On a branch build + # BUILD_BRANCH = Build.SourceBranchName = "ci-unify" (Azure returns only the last + # ref segment of refs/heads/lawlee/ci-unify), so ctx.branch can't resolve the + # lawlee/ci-unify swss-common run. Remove this line before merge to restore + # BUILD_BRANCH tracking (commit 93690566). + branch: lawlee/ci-unify cascade_optional: false result_filter: [succeeded, partiallySucceeded] artifact_name: From 591978b2cfe633a289db37e69b59e54b340020a0 Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Tue, 7 Jul 2026 23:21:06 +0000 Subject: [PATCH 6/9] [ci]: make build-template.yml reusable via a 'repo' parameter Add a 'repo' parameter (default: self) so a consumer pipeline can reuse this template to build sonic-sairedis by passing the sonic-sairedis repository resource. When overridden, the checkout uses path 's' so $(Build.SourcesDirectory) still points at the sairedis checkout and the rest of the template is unchanged. sonic-sairedis's own pipeline (repo: self) is unaffected. Enables sonic-swss-common's BuildSairedis stage to reuse this template + the buildenv_setup dependency cascade instead of maintaining its own copy (PoC). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Lawrence Lee --- .azure-pipelines/build-template.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index f0461f292b..9f296482cc 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -1,4 +1,7 @@ parameters: +- name: repo + type: string + default: self - name: arch type: string values: @@ -66,9 +69,15 @@ jobs: options: "--privileged" steps: - - checkout: self + # `repo` lets a consumer pipeline (e.g. sonic-swss-common) reuse this template to + # build sonic-sairedis. Default `self` preserves sairedis's own pipeline. When + # overridden with the sonic-sairedis repository resource, check out to `s` so + # $(Build.SourcesDirectory) still points at the sairedis checkout as usual. + - checkout: ${{ parameters.repo }} clean: true submodules: true + ${{ if ne(parameters.repo, 'self') }}: + path: s - script: | set -ex # Bootstrap the tool's own runtime deps (buildenv_setup needs PyYAML + requests). From b3ee915d6ed40cb1f5655a587aec57d44a23ece8 Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Wed, 8 Jul 2026 16:28:04 +0000 Subject: [PATCH 7/9] [ci]: branch-aware reuse + optional current-run swss-common staging Two additions to the reusable build-template.yml, both no-ops for sairedis's own pipeline (repo=self, stage_swss_common=false): - Branch-match the resource repo to $(BUILD_BRANCH) when reused (repo != self), so a consumer builds the matching sairedis branch. Gated off for self (a self PR build is on the merge ref and must not be switched); falls back to the checked-out ref when the branch doesn't exist here. - Add stage_swss_common: when set, download the consumer's current-run swss-common artifact to /staged/sonic-swss-common/ and pass --upstream-staged-dir + --required-staged-upstream so buildenv_setup uses THIS run's swss-common instead of the latest completed external run. Absent the staged dir, it falls back to download. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Lawrence Lee --- .azure-pipelines/build-template.yml | 36 ++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 9f296482cc..21969d2ea7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -43,6 +43,10 @@ parameters: type: boolean default: false +- name: stage_swss_common + type: boolean + default: false + - name: debian_version type: string @@ -54,6 +58,12 @@ jobs: DIFF_COVER_CHECK_THRESHOLD: 80 ${{ if eq(parameters.run_unit_test, true) }}: DIFF_COVER_ENABLE: 'true' + # When a consumer stages the current-run swss-common (stage_swss_common), + # require the cascade to use it rather than silently falling back to a download. + ${{ if eq(parameters.stage_swss_common, true) }}: + REQUIRE_STAGED: '--required-staged-upstream sonic-swss-common' + ${{ else }}: + REQUIRE_STAGED: '' pool: ${{ if ne(parameters.pool, 'default') }}: @@ -78,6 +88,24 @@ jobs: submodules: true ${{ if ne(parameters.repo, 'self') }}: path: s + - ${{ if ne(parameters.repo, 'self') }}: + - script: | + set -e + # Branch-match the reused resource repo to the source repo being built + # ($(BUILD_BRANCH)). Only for the reuse path (repo != self): sairedis's own + # checkout is already on the right ref, and a self PR build is on the merge + # ref which must NOT be switched. Fall back to the checked-out ref if that + # branch doesn't exist here (e.g. a PoC feature branch, or BUILD_BRANCH being + # the truncated last ref segment). + git checkout $(BUILD_BRANCH) 2>/dev/null && git submodule update || \ + echo "keeping checked-out ref $(git rev-parse --abbrev-ref HEAD)" + displayName: "Branch-match ${{ parameters.repo }} to $(BUILD_BRANCH)" + - ${{ if eq(parameters.stage_swss_common, true) }}: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: ${{ parameters.swss_common_artifact_name }} + path: $(Pipeline.Workspace)/staged/sonic-swss-common + displayName: "Stage current-run sonic-swss-common for the cascade" - script: | set -ex # Bootstrap the tool's own runtime deps (buildenv_setup needs PyYAML + requests). @@ -92,12 +120,18 @@ jobs: # Sets up apt/pip deps + downloads/installs libswsscommon, libnl, libyang3 and # vpp DEBs (see build-env/upstream-artifacts.yaml), and runs the redis / # sswsyncd / rsyslog post-install hooks. + # --upstream-staged-dir lets a consumer (sonic-swss-common) supply the + # current-run swss-common bundle at /sonic-swss-common/, so the cascade + # uses THIS run's swss-common instead of the latest completed external run. + # When that dir is absent (sairedis's own pipeline), buildenv_setup falls back + # to downloading; REQUIRE_STAGED enforces the override in the consumer case. PYTHONPATH=/tmp/sw-common/ci python3 -m buildenv_setup \ --repo-dir $(Build.SourcesDirectory) \ --scope build \ --arch ${{ parameters.arch }} \ --debian-version ${{ parameters.debian_version }} \ - --branch $(BUILD_BRANCH) + --branch $(BUILD_BRANCH) \ + --upstream-staged-dir $(Pipeline.Workspace)/staged $(REQUIRE_STAGED) displayName: "Set up build environment (buildenv_setup)" env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) From a1c653567bd2bf300f9c7932716a03a348b08fb3 Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Wed, 8 Jul 2026 22:33:33 +0000 Subject: [PATCH 8/9] [ci]: generalize staged_upstreams (list) + drop runtime branch-match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align the reusable build-template.yml with two design follow-ups from the PoC review: - Replace the single stage_swss_common bool with a staged_upstreams list param ({artifact, name} entries). A per-entry DownloadPipelineArtifact loop stages each to /staged//, and --required-staged-upstream flags are built from the same list (passed as JSON via convertToJson) so any expected-but-missing current-run bundle fails loudly. Lets any stage stage any set of upstreams (§6.6). - Drop the runtime 'git checkout $(BUILD_BRANCH)' branch-match (gated repo!=self): it contradicts the fail-closed design and is redundant -- the resource ref (resolved at compile time) already checks out the reused repo on the right branch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Lawrence Lee --- .azure-pipelines/build-template.yml | 52 ++++++++++++----------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 21969d2ea7..53bc4ce626 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -43,9 +43,12 @@ parameters: type: boolean default: false -- name: stage_swss_common - type: boolean - default: false +- name: staged_upstreams + # List of { artifact: , name: } + # to stage from the CURRENT run so the cascade uses them instead of the latest completed + # external run. Empty (a repo's own pipeline) => normal download. + type: object + default: [] - name: debian_version type: string @@ -58,12 +61,6 @@ jobs: DIFF_COVER_CHECK_THRESHOLD: 80 ${{ if eq(parameters.run_unit_test, true) }}: DIFF_COVER_ENABLE: 'true' - # When a consumer stages the current-run swss-common (stage_swss_common), - # require the cascade to use it rather than silently falling back to a download. - ${{ if eq(parameters.stage_swss_common, true) }}: - REQUIRE_STAGED: '--required-staged-upstream sonic-swss-common' - ${{ else }}: - REQUIRE_STAGED: '' pool: ${{ if ne(parameters.pool, 'default') }}: @@ -88,24 +85,14 @@ jobs: submodules: true ${{ if ne(parameters.repo, 'self') }}: path: s - - ${{ if ne(parameters.repo, 'self') }}: - - script: | - set -e - # Branch-match the reused resource repo to the source repo being built - # ($(BUILD_BRANCH)). Only for the reuse path (repo != self): sairedis's own - # checkout is already on the right ref, and a self PR build is on the merge - # ref which must NOT be switched. Fall back to the checked-out ref if that - # branch doesn't exist here (e.g. a PoC feature branch, or BUILD_BRANCH being - # the truncated last ref segment). - git checkout $(BUILD_BRANCH) 2>/dev/null && git submodule update || \ - echo "keeping checked-out ref $(git rev-parse --abbrev-ref HEAD)" - displayName: "Branch-match ${{ parameters.repo }} to $(BUILD_BRANCH)" - - ${{ if eq(parameters.stage_swss_common, true) }}: + # Branch selection for a reused resource repo comes from its `ref` (resolved at compile + # time), not a runtime git checkout — see the build-template reuse mechanics in the design. + - ${{ each up in parameters.staged_upstreams }}: - task: DownloadPipelineArtifact@2 inputs: - artifact: ${{ parameters.swss_common_artifact_name }} - path: $(Pipeline.Workspace)/staged/sonic-swss-common - displayName: "Stage current-run sonic-swss-common for the cascade" + artifact: ${{ up.artifact }} + path: $(Pipeline.Workspace)/staged/${{ up.name }} + displayName: "Stage current-run ${{ up.name }} for the cascade" - script: | set -ex # Bootstrap the tool's own runtime deps (buildenv_setup needs PyYAML + requests). @@ -120,21 +107,24 @@ jobs: # Sets up apt/pip deps + downloads/installs libswsscommon, libnl, libyang3 and # vpp DEBs (see build-env/upstream-artifacts.yaml), and runs the redis / # sswsyncd / rsyslog post-install hooks. - # --upstream-staged-dir lets a consumer (sonic-swss-common) supply the - # current-run swss-common bundle at /sonic-swss-common/, so the cascade - # uses THIS run's swss-common instead of the latest completed external run. - # When that dir is absent (sairedis's own pipeline), buildenv_setup falls back - # to downloading; REQUIRE_STAGED enforces the override in the consumer case. + # --upstream-staged-dir lets a consumer supply current-run upstream bundles at + # // (staged by the loop above from parameters.staged_upstreams), so the + # cascade uses THIS run's artifacts instead of the latest completed external run. + # When a bundle is absent (e.g. sairedis's own pipeline) buildenv_setup downloads it; + # --required-staged-upstream (built from the same list) makes a MISSING expected + # bundle fail loudly instead of silently regressing to a download. + REQ_ARGS=$(python3 -c "import os,json; print(' '.join('--required-staged-upstream '+u['name'] for u in json.loads(os.environ.get('STAGED_UPSTREAMS_JSON','[]'))))") PYTHONPATH=/tmp/sw-common/ci python3 -m buildenv_setup \ --repo-dir $(Build.SourcesDirectory) \ --scope build \ --arch ${{ parameters.arch }} \ --debian-version ${{ parameters.debian_version }} \ --branch $(BUILD_BRANCH) \ - --upstream-staged-dir $(Pipeline.Workspace)/staged $(REQUIRE_STAGED) + --upstream-staged-dir $(Pipeline.Workspace)/staged $REQ_ARGS displayName: "Set up build environment (buildenv_setup)" env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) + STAGED_UPSTREAMS_JSON: ${{ convertToJson(parameters.staged_upstreams) }} - script: ASAN=${{ parameters.asan }} ./build-env/build.sh displayName: "Compile sonic sairedis with coverage enabled" - script: | From a3e585c01ae392166fac962d1578075f9b6561fd Mon Sep 17 00:00:00 2001 From: Lawrence Lee Date: Thu, 9 Jul 2026 16:39:05 +0000 Subject: [PATCH 9/9] [PoC][ci]: set VPP_INSTALL_SKIP_SYSCTL for vpp; reuse shared redis script Two changes to sonic-sairedis' build-env config: 1. vpp upstream artifact: add install_env { VPP_INSTALL_SKIP_SYSCTL: 1 }. vpp's maintainer scripts call sysctl, which fails in the unprivileged CI container; this makes them skip it. buildenv_setup installs the vpp DEBs in their own dpkg -i group as 'sudo env VPP_INSTALL_SKIP_SYSCTL=1 dpkg -i ...', after the empty-env library providers. 2. redis dedup: delete sonic-sairedis' local configure-redis-for-tests.sh. The post_install entry now resolves the script from the cascaded sonic-swss-common bundle (single source of truth). This applies notify-keyspace-events AKE to sonic-sairedis too; verified harmless by this pipeline. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Lawrence Lee --- build-env/configure-redis-for-tests.sh | 15 --------------- build-env/packages/base.yaml | 23 ++++++++++++++--------- build-env/upstream-artifacts.yaml | 5 +++++ 3 files changed, 19 insertions(+), 24 deletions(-) delete mode 100755 build-env/configure-redis-for-tests.sh diff --git a/build-env/configure-redis-for-tests.sh b/build-env/configure-redis-for-tests.sh deleted file mode 100755 index 580d459934..0000000000 --- a/build-env/configure-redis-for-tests.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# -# Configure the system redis-server for sonic-sairedis' tests. -# Applied via the `configure-redis-for-tests` post_install entry in -# build-env/packages/base.yaml. -# -# Reproduces exactly the redis config build-template.yml applied inline. -# NOTE (F11): sonic-sairedis does NOT set `notify-keyspace-events AKE` -# (unlike sonic-swss-common / sonic-swss) -- do not add it here. -set -ex - -sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf -sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf -sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf -sudo service redis-server start diff --git a/build-env/packages/base.yaml b/build-env/packages/base.yaml index 414629fb78..125c03f496 100644 --- a/build-env/packages/base.yaml +++ b/build-env/packages/base.yaml @@ -35,16 +35,21 @@ packages: - redis-server post_install: - # Configure the system redis for sonic-sairedis' tests. + # Configure the system redis for sonic-sairedis' unit tests. These run during the + # Build stage (make check), so scopes: [build, test] -- unlike sonic-swss-common, + # whose Build stage runs the same script INLINE after cargo (its base.yaml entry is + # test-scoped only, to keep redis down during the Rust logger_init_without_redis test). # - # NOTE (design finding F11): sonic-sairedis' redis config does NOT set - # `notify-keyspace-events AKE` (unlike sonic-swss-common / sonic-swss). This is - # why a single shared cascading redis script can't be used naively: the common - # lines (unixsocket / unixsocketperm) belong in sonic-swss-common's cascading - # base.yaml, while the per-repo deltas stay local. For this PoC (sonic-swss-common - # build-env not yet on the resolved branch) sonic-sairedis carries its own - # complete-but-minimal redis config. + # DEDUP: the script body is NOT carried in sonic-sairedis. `configure-redis-for-tests.sh` + # is resolved from the cascaded sonic-swss-common bundle -- buildenv_setup falls back + # to upstream build-env/ dirs when a source file isn't present locally (see + # post_install.resolve_script) -- so the redis config lives in exactly ONE place: + # sonic-swss-common/build-env/configure-redis-for-tests.sh. + # + # That shared script sets `notify-keyspace-events AKE`, which sonic-sairedis' old + # standalone config did not. Reusing it applies AKE to sonic-sairedis too; the + # sonic-sairedis CI pipeline verifies this keyspace delta is harmless for its tests. - name: configure-redis-for-tests - source: configure-redis-for-tests.sh + source: configure-redis-for-tests.sh # resolved from the sonic-swss-common cascade requires: [redis-server] scopes: [build, test] diff --git a/build-env/upstream-artifacts.yaml b/build-env/upstream-artifacts.yaml index 19a0e272b1..39857b23e0 100644 --- a/build-env/upstream-artifacts.yaml +++ b/build-env/upstream-artifacts.yaml @@ -54,6 +54,11 @@ upstream: cascade_optional: true result_filter: [succeeded, partiallySucceeded] apt_fix_broken: true # the 10 vpp DEBs are interdependent; fix up after dpkg -i + # vpp's maintainer scripts call sysctl, which fails in the unprivileged CI + # container. VPP_INSTALL_SKIP_SYSCTL=1 makes them skip it. install_env puts the + # vpp DEBs in their own dpkg -i group (see planner._deb_install_groups), installed + # AFTER the empty-env library providers, as `sudo env VPP_INSTALL_SKIP_SYSCTL=1 dpkg -i ...`. + install_env: { VPP_INSTALL_SKIP_SYSCTL: "1" } artifact_name: 'vpp-{debian_version}' debs: - 'libvppinfra-dev_*_{arch}.deb'