[ci]: Add buildenv_setup and cut the Build stage over to it (CI unification, Phase 1)

.artifactignore

@@ -1,2 +1,9 @@
 **/*
 !*.deb
+# Also publish build-env/ so downstream consumers (sonic-sairedis, sonic-swss)
+# can cascade this repo's dependency declarations: buildenv_setup downloads this
+# artifact and reads build-env/upstream-artifacts.yaml + build-env/packages/*.yaml
+# from it. Both the directory and its contents must be un-ignored (the leading
+# **/* ignores the directory entry too).
+!build-env
+!build-env/**

.azure-pipelines/build-template.yml

@@ -57,129 +57,49 @@ jobs:
     clean: true
   - script: |
       set -ex
+      # Bootstrap the tool's own runtime deps (buildenv_setup needs PyYAML + requests).
       sudo apt-get update
-      sudo apt-get install -qq -y \
-        libhiredis-dev \
-        libnl-3-dev \
-        libnl-genl-3-dev \
-        libnl-route-3-dev \
-        libnl-nf-3-dev \
-        swig
-    displayName: "Install dependencies"
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      source: specific
-      project: build
-      pipeline: Azure.sonic-buildimage.common_libs
-      runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
-      ${{ if eq(parameters.arch, 'amd64') }}:
-        artifact: common-lib
-      ${{ else }}:
-        artifact: common-lib.${{ parameters.arch }}
-      patterns: |
-        target/debs/${{ parameters.debian_version }}/libyang3_*.deb
-        target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb
-        target/debs/${{ parameters.debian_version }}/python3-libyang*.deb
-    displayName: "Download libyang from ${{ parameters.arch }} common lib"
-    condition: ne('${{ parameters.debian_version }}', 'trixie')
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      source: specific
-      project: build
-      pipeline: Azure.sonic-buildimage.common_libs
-      runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
-      ${{ if eq(parameters.arch, 'amd64') }}:
-        artifact: common-lib
-      ${{ else }}:
-        artifact: common-lib.${{ parameters.arch }}
-      patterns: |
-        target/debs/${{ parameters.debian_version }}/libyang3_*.deb
-        target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb
-        target/debs/${{ parameters.debian_version }}/python3-libyang*.deb
-        target/debs/${{ parameters.debian_version }}/libpcre*.deb
-    displayName: "Download libyang from ${{ parameters.arch }} common lib"
-    condition: eq('${{ parameters.debian_version }}', 'trixie')
-  - script: |
-      set -ex
-      sudo dpkg -i $(find ./download -name *.deb)
-      # common-lib doesn't publish python3-libyang (LIBYANG3_PY3 isn't in
-      # slave.mk's lib-packages), so install the CFFI bindings from PyPI;
-      # they link against the libyang3 we just installed.
-      sudo pip3 install --no-build-isolation 'libyang==3.3.0'
-    workingDirectory: $(Build.ArtifactStagingDirectory)
-    displayName: "Install libyang from common lib"
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      source: specific
-      project: build
-      pipeline: 142
-      artifact: sonic-buildimage.vs
-      runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
-      patterns: |
-        target/python-wheels/${{ parameters.debian_version }}/sonic_yang_mgmt-1.0-py3-none-any.whl
-        target/python-wheels/${{ parameters.debian_version }}/sonic_yang_models-1.0-py3-none-any.whl
-    displayName: "Download yang wheel from latest sonic-buildimage build"
-    condition: ne('${{ parameters.debian_version }}', 'trixie')
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      source: specific
-      project: build
-      pipeline: 142
-      artifact: sonic-buildimage.vs
-      runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
-      patterns: |
-        target/python-wheels/bookworm/sonic_yang_mgmt-1.0-py3-none-any.whl
-        target/python-wheels/bookworm/sonic_yang_models-1.0-py3-none-any.whl
-    displayName: "Download yang wheel from latest sonic-buildimage build"
-    condition: eq('${{ parameters.debian_version }}', 'trixie')
-  - script: |
-      set -ex
-      sudo pip3 install ./download/target/python-wheels/${{ parameters.debian_version }}/sonic_yang_mgmt-1.0-py3-none-any.whl \
-                        ./download/target/python-wheels/${{ parameters.debian_version }}/sonic_yang_models-1.0-py3-none-any.whl
-    workingDirectory: $(Build.ArtifactStagingDirectory)
-    displayName: "Install yang wheel from common lib"
-    condition: ne('${{ parameters.debian_version }}', 'trixie')
-  - script: |
-      set -ex
-      sudo pip3 install ./download/target/python-wheels/bookworm/sonic_yang_mgmt-1.0-py3-none-any.whl \
-                        ./download/target/python-wheels/bookworm/sonic_yang_models-1.0-py3-none-any.whl
-    workingDirectory: $(Build.ArtifactStagingDirectory)
-    displayName: "Install yang wheel from common lib"
-    condition: eq('${{ parameters.debian_version }}', 'trixie')
-  - script: |
-      set -ex
-      rm ../*.deb || true
-      ./autogen.sh
-      DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' DEB_CXXFLAGS_APPEND="-coverage -fprofile-abs-path" DEB_LDFLAGS_APPEND="-coverage -fprofile-abs-path" dpkg-buildpackage -Pnopython2 -us -uc -b -j$(nproc)
-      mv ../*.deb .
+      sudo apt-get install -qq -y python3-yaml python3-requests
+      # buildenv_setup lives in this repo's ci/ (sonic-swss-common hosts the shared
+      # tool), so run it straight from the checkout -- no clone needed here.
+      # It installs apt/pip deps (libnl comes from stock apt, per base.yaml),
+      # downloads + installs the libyang3 DEBs and the sonic-yang wheels from
+      # common-lib (see upstream-artifacts.yaml), and runs the yang-models
+      # post-install hook. The redis test-config hook is test-scoped, so this Build
+      # job does NOT run it here; it is applied inline later, after cargo test.
+      PYTHONPATH=$(Build.SourcesDirectory)/ci python3 -m buildenv_setup \
+        --repo-dir $(Build.SourcesDirectory) \
+        --scope build \
+        --arch ${{ parameters.arch }} \
+        --debian-version ${{ parameters.debian_version }} \
+        --branch $(BUILD_BRANCH)
+    displayName: "Set up build environment (buildenv_setup)"
+    env:
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+  - script: ./build-env/build.sh
     displayName: "Compile sonic swss common with coverage enabled"
   - ${{ if eq(parameters.run_unit_test, true) }}:
     - script: |
         set -ex
-        sudo pip install Pympler==0.8 pytest
-        sudo apt-get install -y redis-server
         sudo dpkg -i libswsscommon_*.deb
         sudo dpkg -i libswsscommon-dev_*.deb
         sudo dpkg -i python3-swsscommon_*.deb
 
         ps aux
+        # Run the Rust tests BEFORE bringing up the unixsocket redis. redis-server is
+        # installed (base.yaml) and running on TCP only at this point — the SONiC
+        # database_config unixsocket is NOT up yet — so the Rust
+        # `logger_init_without_redis` test (crates/swss-common/tests/logger_fallback.rs)
+        # correctly sees redis as unreachable and passes. Redis-dependent Rust tests
+        # spawn their own ephemeral redis via swss-common-testing, so they are fine.
         cargo test           --workspace --all-features
         cargo test --release --workspace --all-features
 
-        sudo sed -i 's/notify-keyspace-events ""/notify-keyspace-events AKE/' /etc/redis/redis.conf
-        sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf
-        sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf
-        sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf
-        sudo service redis-server restart
-        sudo mkdir -p /usr/local/yang-models
+        # Bring up the unixsocket redis for the C++ / pytest tests below. This is the
+        # same shared script buildenv_setup runs on the VS test host (via the
+        # configure-redis-for-tests post_install, scope test); the Build stage runs it
+        # INLINE here because it must happen AFTER cargo, not at build-env setup time.
+        ./build-env/configure-redis-for-tests.sh
 
         ./tests/tests
         redis-cli FLUSHALL
@@ -189,8 +109,6 @@ jobs:
         make -C goext
         redis-cli FLUSHALL
         make -C goext check
-
-        rm -rf $(Build.ArtifactStagingDirectory)/download
       displayName: "Run swss common unit tests"
   - publish: $(System.DefaultWorkingDirectory)/
     artifact: ${{ parameters.artifact_name }}

.azure-pipelines/build-ubuntu-template.yml

@@ -0,0 +1,59 @@
+parameters:
+- name: debian_version
+  type: string
+  default: bookworm
+
+jobs:
+- job:
+  displayName: "amd64/ubuntu-22.04"
+  pool:
+    vmImage: 'ubuntu-22.04'
+
+  steps:
+  - script: |
+      set -ex
+      # Bootstrap the tool's own runtime deps (buildenv_setup needs PyYAML + requests).
+      sudo apt-get update
+      sudo apt-get install -qq -y python3-yaml python3-requests
+      # buildenv_setup lives in this repo's ci/ (sonic-swss-common hosts the shared
+      # tool), so run it straight from the checkout. --host-os ubuntu-22.04 selects
+      # the jammy package variants (swig4.0 + the vmImage build toolchain that the
+      # sonic-slave-* containers bake in); --debian-version selects which
+      # common-libs / sonic-buildimage artifacts to fetch (bookworm DEBs/wheels
+      # installed on jammy, as before). It apt/pip-installs deps (libnl comes from
+      # stock apt, per base.yaml), downloads + installs the libyang3 DEBs and the
+      # sonic-yang wheels from common-lib, and runs the yang-models post-install
+      # hook. The redis test-config hook is test-scoped, so it is not run by this
+      # Build job (which runs the Bazel test suite, not pytest).
+      PYTHONPATH=$(Build.SourcesDirectory)/ci python3 -m buildenv_setup \
+        --repo-dir $(Build.SourcesDirectory) \
+        --scope build \
+        --host-os ubuntu-22.04 \
+        --debian-version ${{ parameters.debian_version }} \
+        --branch $(BUILD_BRANCH)
+    displayName: "Set up build environment (buildenv_setup)"
+    env:
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+  - script: |
+      set -ex
+      # Special-case CLI-driven installs that don't fit the packages/*.yaml schema
+      # (see design doc): compile the gtest sources shipped by libgtest-dev, and
+      # install the bazelisk binary (provides `bazel`). libgtest-dev/libgmock-dev +
+      # cmake are apt-installed by buildenv_setup (packages/tooling.yaml, jammy).
+      cd /usr/src/gtest && sudo cmake . && sudo make
+      ARCH=$(dpkg --print-architecture)
+      sudo curl -fsSL -o /usr/local/bin/bazel \
+          https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-${ARCH}
+      sudo chmod 755 /usr/local/bin/bazel
+    displayName: "Compile gtest + install bazelisk"
+  - script: |
+      ./autogen.sh
+      dpkg-buildpackage -us -uc -Pnopython2 -b -j$(nproc) && cp ../*.deb .
+    displayName: "Compile sonic swss common"
+  - script: |
+      bazel build //...
+      bazel test //...
+    displayName: "Compile and test all Bazel targets"
+  - publish: $(System.DefaultWorkingDirectory)/
+    artifact: sonic-swss-common.amd64.ubuntu22_04
+    displayName: "Archive swss common debian packages"

azure-pipelines.yml

@@ -52,88 +52,9 @@ stages:
 - stage: Build
 
   jobs:
-  - job:
-    displayName: "amd64/ubuntu-22.04"
-    pool:
-      vmImage: 'ubuntu-22.04'
-
-    steps:
-    - script: |
-        sudo apt-get update
-        sudo apt-get install -y make libtool m4 autoconf dh-exec debhelper cmake pkg-config nlohmann-json3-dev \
-                         libhiredis-dev libnl-3-dev libnl-genl-3-dev libnl-route-3-dev libnl-nf-3-dev swig4.0 \
-                         libpython3-dev libboost-dev libboost-serialization-dev uuid-dev libzmq3-dev
-        sudo apt-get install -y sudo
-        sudo apt-get install -y redis-server redis-tools
-        sudo apt-get install -y python3-pip
-        sudo pip3 install pytest
-        sudo apt-get install -y python
-        sudo apt-get install cmake libgtest-dev libgmock-dev
-        cd /usr/src/gtest && sudo cmake . && sudo make
-        ARCH=$(dpkg --print-architecture)
-        set -x
-        sudo curl -fsSL -o /usr/local/bin/bazel \
-            https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-${ARCH}
-        sudo chmod 755 /usr/local/bin/bazel
-      displayName: "Install dependencies"
-    - task: DownloadPipelineArtifact@2
-      inputs:
-        source: specific
-        project: build
-        pipeline: Azure.sonic-buildimage.common_libs
-        runVersion: 'latestFromBranch'
-        runBranch: 'refs/heads/$(BUILD_BRANCH)'
-        path: $(Build.ArtifactStagingDirectory)/download
-        artifact: common-lib
-        patterns: |
-          target/debs/${{ parameters.debian_version }}/libyang3_*.deb
-          target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb
-      displayName: "Download yang deb from amd64 common lib"
-    - script: |
-        set -ex
-        sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang3_*.deb \
-                     $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb
-        # python3-libyang's bookworm .deb pins python3 (>= 3.11~, << 3.12) and
-        # won't install on Ubuntu 22.04 (python 3.10). Build the Python bindings
-        # from PyPI instead — they link against the libyang3 we just installed.
-        # Use --no-build-isolation with apt's python3-cffi to avoid jammy pip
-        # 22.0.2 picking up cffi 2.0.0 in its build env while Debian's older
-        # /usr/lib/python3/dist-packages/_cffi_backend.so still wins on sys.path
-        # (causing a "Version mismatch" Exception inside cffi at build time).
-        sudo apt-get install -y python3-cffi
-        sudo pip3 install --no-build-isolation 'libyang==3.3.0'
-      workingDirectory: $(Build.ArtifactStagingDirectory)
-      displayName: "Install yang deb from common lib"
-    - task: DownloadPipelineArtifact@2
-      inputs:
-        source: specific
-        project: build
-        pipeline: 142
-        artifact: sonic-buildimage.vs
-        runVersion: 'latestFromBranch'
-        runBranch: 'refs/heads/$(BUILD_BRANCH)'
-        path: $(Build.ArtifactStagingDirectory)/download
-        patterns: |
-          target/python-wheels/${{ parameters.debian_version }}/sonic_yang_mgmt-1.0-py3-none-any.whl
-          target/python-wheels/${{ parameters.debian_version }}/sonic_yang_models-1.0-py3-none-any.whl
-      displayName: "Download yang wheel from latest sonic-buildimage build"
-    - script: |
-        set -ex
-        sudo pip3 install ./download/target/python-wheels/${{ parameters.debian_version }}/sonic_yang_mgmt-1.0-py3-none-any.whl \
-                          ./download/target/python-wheels/${{ parameters.debian_version }}/sonic_yang_models-1.0-py3-none-any.whl
-      workingDirectory: $(Build.ArtifactStagingDirectory)
-      displayName: "Install yang wheel"
-    - script: |
-        ./autogen.sh
-        dpkg-buildpackage -us -uc -Pnopython2 -b -j$(nproc) && cp ../*.deb .
-      displayName: "Compile sonic swss common"
-    - script: |
-        bazel build //...
-        bazel test //...
-      displayName: "Compile and test all Bazel targets"
-    - publish: $(System.DefaultWorkingDirectory)/
-      artifact: sonic-swss-common.amd64.ubuntu22_04
-      displayName: "Archive swss common debian packages"
+  - template: .azure-pipelines/build-ubuntu-template.yml
+    parameters:
+      debian_version: ${{ parameters.debian_version }}
 
   - template: .azure-pipelines/build-template.yml
     parameters:
@@ -236,3 +157,27 @@ stages:
     parameters:
       log_artifact_name: log
       debian_version: ${{ parameters.debian_version }}
+
+- stage: BuildenvSetupUnitTests
+  # Runs the ci/buildenv_setup unit tests. Independent (dependsOn: []) so it runs in
+  # parallel and gates nothing else; its own result still contributes to the pipeline.
+  dependsOn: []
+  jobs:
+  - job:
+    displayName: "buildenv_setup unit tests"
+    pool:
+      vmImage: 'ubuntu-22.04'
+    steps:
+    - script: |
+        sudo pip3 install pytest pyyaml requests
+      displayName: "Install test dependencies"
+    - script: |
+        PYTHONPATH=. python3 -m pytest tests/ -v --junitxml=$(Build.ArtifactStagingDirectory)/buildenv_setup-tests.xml
+      workingDirectory: ci
+      displayName: "Run buildenv_setup unit tests"
+    - task: PublishTestResults@2
+      condition: succeededOrFailed()
+      inputs:
+        testResultsFormat: JUnit
+        testResultsFiles: '$(Build.ArtifactStagingDirectory)/buildenv_setup-tests.xml'
+        testRunTitle: 'buildenv_setup unit tests'

build-env/Dockerfile

@@ -0,0 +1,35 @@
+# Local-dev image for building sonic-swss-common.
+#
+# CI does NOT use this file: CI runs inside `container: sonic-slave-*` and invokes
+# buildenv_setup directly (see .azure-pipelines/build-template.yml). This exists
+# so a developer can reproduce the CI build environment locally.
+#
+# Layer ordering (design finding F7): copy ONLY the dependency declarations + the
+# buildenv_setup tool, run setup, and do NOT copy the source. build-env/compose.yaml
+# mounts the working tree at /workspace at runtime, so editing source never
+# invalidates the (heavy) dependency-setup layer below.
+
+ARG DEBIAN_VERSION=bookworm
+FROM sonicdev-microsoft.azurecr.io:443/sonic-slave-${DEBIAN_VERSION}:latest
+
+ARG DEBIAN_VERSION=bookworm
+ARG BUILD_BRANCH=master
+
+# The buildenv_setup tool (lives in this repo's ci/) + the dep declarations only.
+COPY ci/ /opt/buildenv/ci/
+COPY build-env/ /workspace/build-env/
+
+# Bootstrap the tool's own runtime deps, then set up the build environment.
+# --org-url points at the public SONiC Azure DevOps org for artifact download
+# (in CI this comes from $SYSTEM_COLLECTIONURI instead). Set AZURE_DEVOPS_EXT_PAT,
+# or use --upstream-staged-dir, if your environment needs auth.
+RUN apt-get update && apt-get install -y python3-yaml python3-requests \
+ && PYTHONPATH=/opt/buildenv/ci python3 -m buildenv_setup \
+      --repo-dir /workspace \
+      --scope build \
+      --debian-version "${DEBIAN_VERSION}" \
+      --branch "${BUILD_BRANCH}" \
+      --org-url https://dev.azure.com/mssonic \
+      --no-sudo
+
+WORKDIR /workspace