diff --git a/.artifactignore b/.artifactignore index 1126a160d9..e21cd0e7b5 100644 --- a/.artifactignore +++ b/.artifactignore @@ -1,2 +1,7 @@ **/* !*.deb +# Also publish build-env/ so downstream consumers (sonic-swss) can cascade this +# repo's dependency declarations via buildenv_setup (reads build-env/ out of the +# downloaded artifact). Both the dir entry and its contents must be un-ignored. +!build-env +!build-env/** diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml index 77cbad1ceb..864670c908 100644 --- a/.azure-pipelines/build-docker-sonic-vs-template.yml +++ b/.azure-pipelines/build-docker-sonic-vs-template.yml @@ -46,7 +46,10 @@ jobs: artifact: ${{ parameters.swss_common_artifact_name }} path: $(Build.ArtifactStagingDirectory)/download runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): resolve the swss-common cascade artifact from lawlee/ci-unify. + # BUILD_BRANCH = Build.SourceBranchName = "ci-unify" (last ref segment only), which + # can't find the lawlee/ci-unify run. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/lawlee/ci-unify' allowPartiallySucceededBuilds: true displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 @@ -66,7 +69,9 @@ jobs: pipeline: Azure.sonic-buildimage.common_libs artifact: common-lib runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; the common_libs pipeline has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download patterns: '**/target/debs/*/libnexthopgroup_*.deb' displayName: "Download sonic-buildimage libnexthopgroup package" @@ -95,7 +100,9 @@ jobs: ${{ else }}: artifact: sonic-dash-api.${{ parameters.arch }} runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; sonic-dash-api has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download patterns: | libdashapi*.deb diff --git a/.azure-pipelines/build-swss-template.yml b/.azure-pipelines/build-swss-template.yml index 0ea7e54866..f052c09bb2 100644 --- a/.azure-pipelines/build-swss-template.yml +++ b/.azure-pipelines/build-swss-template.yml @@ -45,7 +45,10 @@ jobs: vmImage: 'ubuntu-22.04' container: - image: sonicdev-microsoft.azurecr.io:443/sonic-slave-${{ parameters.debian_version }}:$(BUILD_BRANCH)-${{ parameters.arch }} + # TEMPORARY (PoC): pin the sonic-slave image tag to `master` instead of + # $(BUILD_BRANCH). Only master-tagged slave images are published; the + # lawlee/ci-unify feature branch has no image. Restore $(BUILD_BRANCH) before merge. + image: sonicdev-microsoft.azurecr.io:443/sonic-slave-${{ parameters.debian_version }}:master-${{ parameters.arch }} steps: - checkout: sonic-swss @@ -53,7 +56,10 @@ jobs: clean: true - script: | set -ex - git checkout $(BUILD_BRANCH) + # TEMPORARY (PoC): check out `master` instead of $(BUILD_BRANCH). sonic-swss + # has no lawlee/ci-unify branch on the sonic-net remote. Restore $(BUILD_BRANCH) + # before merge. + git checkout master git submodule update git status displayName: Set up sonic-swss branch @@ -63,7 +69,9 @@ jobs: project: build pipeline: Azure.sonic-buildimage.common_libs runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; the common_libs pipeline has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download artifact: common-lib patterns: | @@ -109,7 +117,10 @@ jobs: pipeline: Azure.sonic-swss-common artifact: ${{ parameters.swss_common_artifact_name }} runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): resolve the swss-common cascade artifact from lawlee/ci-unify. + # BUILD_BRANCH = Build.SourceBranchName = "ci-unify" (last ref segment only), which + # can't find the lawlee/ci-unify run. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/lawlee/ci-unify' allowPartiallySucceededBuilds: true path: $(Build.ArtifactStagingDirectory)/download patterns: | @@ -139,7 +150,9 @@ jobs: ${{ else }}: artifact: sonic-dash-api.${{ parameters.arch }} runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; sonic-dash-api has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download patterns: | libdashapi*.deb diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 1c5e620082..53bc4ce626 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -1,4 +1,7 @@ parameters: +- name: repo + type: string + default: self - name: arch type: string values: @@ -40,6 +43,13 @@ parameters: type: boolean default: false +- name: staged_upstreams + # List of { artifact: , name: } + # to stage from the CURRENT run so the cascade uses them instead of the latest completed + # external run. Empty (a repo's own pipeline) => normal download. + type: object + default: [] + - name: debian_version type: string @@ -59,189 +69,63 @@ jobs: vmImage: 'ubuntu-22.04' container: - image: sonicdev-microsoft.azurecr.io:443/sonic-slave-${{ parameters.debian_version }}:$(BUILD_BRANCH)-${{ parameters.arch }} + # TEMPORARY (PoC): pin the sonic-slave image tag to `master` instead of + # $(BUILD_BRANCH). Only master-tagged slave images are published; the + # lawlee/ci-unify feature branch has no image. Restore $(BUILD_BRANCH) before merge. + image: sonicdev-microsoft.azurecr.io:443/sonic-slave-${{ parameters.debian_version }}:master-${{ parameters.arch }} options: "--privileged" steps: - - checkout: self + # `repo` lets a consumer pipeline (e.g. sonic-swss-common) reuse this template to + # build sonic-sairedis. Default `self` preserves sairedis's own pipeline. When + # overridden with the sonic-sairedis repository resource, check out to `s` so + # $(Build.SourcesDirectory) still points at the sairedis checkout as usual. + - checkout: ${{ parameters.repo }} clean: true submodules: true - - task: DownloadPipelineArtifact@2 - condition: eq('${{ parameters.arch }}', 'amd64') - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib - patterns: | - target/debs/${{ parameters.debian_version }}/libnl-3*.deb - target/debs/${{ parameters.debian_version }}/libnl-genl*.deb - target/debs/${{ parameters.debian_version }}/libnl-route*.deb - target/debs/${{ parameters.debian_version }}/libnl-nf*.deb - displayName: "Download libnl from common_libs" - - script: | - set -ex - sudo dpkg -i $(find ./download -name libnl-*_${{ parameters.arch }}.deb) - condition: eq('${{ parameters.arch }}', 'amd64') - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install sonic libnl" - - script: | - set -ex - sudo apt-get update - sudo apt-get install -qq -y \ - libdbus-glib-1-dev \ - libpcsclite-dev \ - docbook-to-man \ - docbook-utils \ - aspell-en \ - libhiredis-dev \ - swig \ - libzmq3-dev \ - autoconf-archive - - sudo apt-get install -y redis-server - sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf - sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf - sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf - sudo service redis-server start - sudo mkdir -m 755 /var/run/sswsyncd - - sudo apt-get install -y rsyslog - sudo rsyslogd - - cat /etc/apt/sources.list - dpkg --list | grep libnl - - displayName: "Install dependencies" + ${{ if ne(parameters.repo, 'self') }}: + path: s + # Branch selection for a reused resource repo comes from its `ref` (resolved at compile + # time), not a runtime git checkout — see the build-template reuse mechanics in the design. + - ${{ each up in parameters.staged_upstreams }}: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: ${{ up.artifact }} + path: $(Pipeline.Workspace)/staged/${{ up.name }} + displayName: "Stage current-run ${{ up.name }} for the cascade" - script: | set -ex + # Bootstrap the tool's own runtime deps (buildenv_setup needs PyYAML + requests). sudo apt-get update - sudo apt-get install -qq -y \ - libnl-3-dev \ - libnl-genl-3-dev \ - libnl-route-3-dev \ - libnl-nf-3-dev \ - libnl-cli-3-dev - - condition: ne('${{ parameters.arch }}', 'amd64') - displayName: "Install libnl for arm dependencies" - - task: DownloadPipelineArtifact@2 - # amd64 artifact name does not has arch suffix - condition: and(ne('${{ parameters.debian_version }}', 'trixie'), eq('${{ parameters.arch }}', 'amd64')) - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - displayName: "Download libyang from amd64 common lib" - - task: DownloadPipelineArtifact@2 - condition: and(ne('${{ parameters.debian_version }}', 'trixie'), ne('${{ parameters.arch }}', 'amd64')) - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib.${{ parameters.arch }} - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - - task: DownloadPipelineArtifact@2 - condition: and(eq('${{ parameters.debian_version }}', 'trixie'), eq('${{ parameters.arch }}', 'amd64')) - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - target/debs/${{ parameters.debian_version }}/libpcre3_*.deb - displayName: "Download libyang from amd64 common lib" - - task: DownloadPipelineArtifact@2 - condition: and(eq('${{ parameters.debian_version }}', 'trixie'), ne('${{ parameters.arch }}', 'amd64')) - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib.${{ parameters.arch }} - patterns: | - target/debs/${{ parameters.debian_version }}/libyang3_*.deb - target/debs/${{ parameters.debian_version }}/libpcre3_*.deb - displayName: "Download libyang from common lib" - - script: | - set -ex - sudo dpkg -i $(find ./download -name *.deb) - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install libyang from common lib" - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: sonic-net.sonic-platform-vpp - artifact: vpp-${{ parameters.debian_version }} - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - allowPartiallySucceededBuilds: true - path: $(Build.ArtifactStagingDirectory)/download - displayName: "Download sonic platform-vpp deb packages" - condition: eq('${{ parameters.arch }}', 'amd64') - - script: | - set -ex - sudo dpkg -i download/libvppinfra-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i download/libvppinfra_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-crypto-engines_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-dbg_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-dev_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-plugin-core_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-plugin-devtools_*_${{ parameters.arch }}.deb - sudo dpkg -i download/vpp-plugin-dpdk_*_${{ parameters.arch }}.deb - sudo dpkg -i download/python3-vpp-api_*_${{ parameters.arch }}.deb - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install sonic platform-vpp packages" - condition: eq('${{ parameters.arch }}', 'amd64') - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: Azure.sonic-swss-common - artifact: ${{ parameters.swss_common_artifact_name }} - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' - allowPartiallySucceededBuilds: true - path: $(Build.ArtifactStagingDirectory)/download - displayName: "Download sonic swss common deb packages" - - script: | - set -ex - sudo dpkg -i download/libswsscommon_1.0.0_${{ parameters.arch }}.deb - sudo dpkg -i download/libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb - rm -rf download - workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install sonic swss Common" - - script: | - set -ex - rm ../*.deb || true - ./autogen.sh - extraflags='--enable-code-coverage' - if [ '${{ parameters.asan }}' == True ]; then - extraflags='--enable-asan' - fi - DEB_BUILD_OPTIONS=nocheck DEB_CONFIGURE_EXTRA_FLAGS=$extraflags dpkg-buildpackage -us -uc -b -Psyncd,vs,nopython2 -j$(nproc) - mv ../*.deb . + sudo apt-get install -qq -y python3-yaml python3-requests git + # buildenv_setup lives in sonic-swss-common/ci/; sonic-sairedis is a consumer, + # so clone sonic-swss-common to obtain the tool. + # TEMPORARY (PoC): pin to lawlee/ci-unify (the tool isn't on swss-common master + # yet). Once merged, clone $(BUILD_BRANCH) with a default-branch fallback. + git clone --depth 1 --branch lawlee/ci-unify \ + https://github.com/sonic-net/sonic-swss-common /tmp/sw-common + # Sets up apt/pip deps + downloads/installs libswsscommon, libnl, libyang3 and + # vpp DEBs (see build-env/upstream-artifacts.yaml), and runs the redis / + # sswsyncd / rsyslog post-install hooks. + # --upstream-staged-dir lets a consumer supply current-run upstream bundles at + # // (staged by the loop above from parameters.staged_upstreams), so the + # cascade uses THIS run's artifacts instead of the latest completed external run. + # When a bundle is absent (e.g. sairedis's own pipeline) buildenv_setup downloads it; + # --required-staged-upstream (built from the same list) makes a MISSING expected + # bundle fail loudly instead of silently regressing to a download. + REQ_ARGS=$(python3 -c "import os,json; print(' '.join('--required-staged-upstream '+u['name'] for u in json.loads(os.environ.get('STAGED_UPSTREAMS_JSON','[]'))))") + PYTHONPATH=/tmp/sw-common/ci python3 -m buildenv_setup \ + --repo-dir $(Build.SourcesDirectory) \ + --scope build \ + --arch ${{ parameters.arch }} \ + --debian-version ${{ parameters.debian_version }} \ + --branch $(BUILD_BRANCH) \ + --upstream-staged-dir $(Pipeline.Workspace)/staged $REQ_ARGS + displayName: "Set up build environment (buildenv_setup)" + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + STAGED_UPSTREAMS_JSON: ${{ convertToJson(parameters.staged_upstreams) }} + - script: ASAN=${{ parameters.asan }} ./build-env/build.sh displayName: "Compile sonic sairedis with coverage enabled" - script: | set -ex diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 1adf5d6cee..ded2f63104 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -40,7 +40,10 @@ jobs: - script: | set -ex cd sonic-swss - git checkout $(BUILD_BRANCH) + # TEMPORARY (PoC): check out `master` instead of $(BUILD_BRANCH). sonic-swss + # has no lawlee/ci-unify branch on the sonic-net remote. Restore $(BUILD_BRANCH) + # before merge. + git checkout master displayName: Set up sonic-swss branch - task: DownloadPipelineArtifact@2 inputs: @@ -55,7 +58,10 @@ jobs: artifact: sonic-swss-common.amd64.ubuntu22_04 path: $(Build.ArtifactStagingDirectory)/download runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): resolve the swss-common cascade artifact from lawlee/ci-unify. + # BUILD_BRANCH = Build.SourceBranchName = "ci-unify" (last ref segment only), which + # can't find the lawlee/ci-unify run. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/lawlee/ci-unify' allowPartiallySucceededBuilds: true displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 @@ -64,13 +70,19 @@ jobs: project: build pipeline: Azure.sonic-buildimage.common_libs runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; the common_libs pipeline has no + # lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download artifact: common-lib patterns: | + target/debs/${{ parameters.debian_version }}/libnl-3-200_*.deb + target/debs/${{ parameters.debian_version }}/libnl-genl-3-200_*.deb + target/debs/${{ parameters.debian_version }}/libnl-route-3-200_*.deb + target/debs/${{ parameters.debian_version }}/libnl-nf-3-200_*.deb target/debs/${{ parameters.debian_version }}/libyang3_*.deb target/debs/${{ parameters.debian_version }}/libyang-dev_3*.deb - displayName: "Download libyang from common lib" + displayName: "Download libnl + libyang from common lib" - task: DownloadPipelineArtifact@2 inputs: source: specific @@ -78,7 +90,10 @@ jobs: pipeline: sonic-net.sonic-buildimage-ubuntu22.04 artifact: sonic-buildimage.amd64.ubuntu22_04 runVersion: 'latestFromBranch' - runBranch: 'refs/heads/$(BUILD_BRANCH)' + # TEMPORARY (PoC): pin to refs/heads/master; the sonic-buildimage-ubuntu22.04 + # pipeline has no lawlee/ci-unify branch. Restore refs/heads/$(BUILD_BRANCH) + # before merge. + runBranch: 'refs/heads/master' path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic buildimage ubuntu deb packages" @@ -105,6 +120,17 @@ jobs: sudo sonic-sairedis/.azure-pipelines/build_and_install_module.sh + # Install SONiC-patched libnl (3.7.0-0.2+b1sonicN) from common-lib BEFORE the + # swss-common deb. The jammy swss-common .deb (built by the ubuntu-22.04 job + # against common-lib libnl) depends on the +b1sonicN libnl, which the 22.04 + # host's apt does not have. Use apt so the bookworm libnl runtime debs' + # dependencies resolve on the jammy host (same as the libyang install below). + sudo apt-get install -y \ + $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libnl-3-200_*.deb \ + $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libnl-genl-3-200_*.deb \ + $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libnl-route-3-200_*.deb \ + $(Build.ArtifactStagingDirectory)/download/target/debs/${{ parameters.debian_version }}/libnl-nf-3-200_*.deb + # Install libyang packages from downloaded artifacts. Use apt rather # than `dpkg -i` so libyang-dev's libpcre2-dev dep is auto-resolved # from the host repos. diff --git a/build-env/Dockerfile b/build-env/Dockerfile new file mode 100644 index 0000000000..0cdfbd713f --- /dev/null +++ b/build-env/Dockerfile @@ -0,0 +1,46 @@ +# Local-dev image for building sonic-sairedis. +# +# CI does NOT use this file: CI runs inside `container: sonic-slave-*` and invokes +# buildenv_setup directly (see .azure-pipelines/build-template.yml). This exists +# so a developer can reproduce the CI build environment locally. +# +# sonic-sairedis is a CONSUMER of the shared buildenv_setup tool, which lives in +# sonic-swss-common/ci/. So (unlike sonic-swss-common) this image clones +# sonic-swss-common to obtain the tool. +# +# Layer ordering (design finding F7): clone the tool + copy ONLY the dependency +# declarations, run setup, and do NOT copy the source. build-env/compose.yaml +# mounts the working tree at /workspace at runtime, so editing source never +# invalidates the (heavy) dependency-setup layer below. + +ARG DEBIAN_VERSION=bookworm +FROM sonicdev-microsoft.azurecr.io:443/sonic-slave-${DEBIAN_VERSION}:latest + +ARG DEBIAN_VERSION=bookworm +# TEMPORARY (PoC): the buildenv_setup tool currently lives on sonic-swss-common's +# lawlee/ci-unify branch (not master yet). Pin the clone to that branch until it +# merges, after which this can track the matching build branch. +ARG SWSS_COMMON_REF=lawlee/ci-unify +ARG BUILD_BRANCH=master + +# Clone sonic-swss-common to obtain the shared buildenv_setup tool. +RUN git clone --depth 1 --branch ${SWSS_COMMON_REF} \ + https://github.com/sonic-net/sonic-swss-common /tmp/sw-common + +# Copy ONLY the dependency declarations so the heavy setup layer stays cached. +COPY build-env/ /workspace/build-env/ + +# Bootstrap the tool's own runtime deps, then set up the build environment. +# --org-url points at the public SONiC Azure DevOps org for artifact download +# (in CI this comes from $SYSTEM_COLLECTIONURI). Set AZURE_DEVOPS_EXT_PAT, or use +# --upstream-staged-dir, if your environment needs auth. +RUN apt-get update && apt-get install -y python3-yaml python3-requests \ + && PYTHONPATH=/tmp/sw-common/ci python3 -m buildenv_setup \ + --repo-dir /workspace \ + --scope build \ + --debian-version "${DEBIAN_VERSION}" \ + --branch "${BUILD_BRANCH}" \ + --org-url https://dev.azure.com/mssonic \ + --no-sudo + +WORKDIR /workspace diff --git a/build-env/README.md b/build-env/README.md new file mode 100644 index 0000000000..4075cfd0b0 --- /dev/null +++ b/build-env/README.md @@ -0,0 +1,52 @@ +# build-env/ + +Declarative build-environment configuration for sonic-sairedis, consumed by the +shared [`buildenv_setup`](https://github.com/sonic-net/sonic-swss-common/tree/master/ci) +tool (which lives in **sonic-swss-common**; sonic-sairedis is a consumer). This +is the single source of truth for "how to build this repo". + +## Contents + +| Path | Purpose | Cascades downstream? | +|------|---------|----------------------| +| `packages/base.yaml` | apt/pip packages needed to build **and link against** libsairedis (dbus/pcsc/hiredis/swig/zmq/autoconf-archive, arm libnl, redis + config) | **Yes** | +| `packages/tooling.yaml` | Build-stage-only tooling (docbook/aspell for man pages, rsyslog; sswsyncd dir) | No | +| `upstream-artifacts.yaml` | upstream DEBs: libswsscommon (sonic-swss-common), libnl+libyang3 (common-libs), vpp | Yes | +| `configure-redis-for-tests.sh` | redis test-config (no `notify-keyspace-events` — see F11), run via `post_install` | (travels with base.yaml) | +| `build.sh` | canonical build (`autogen` + `dpkg-buildpackage -Psyncd,vs,nopython2`), used by CI **and** local dev; `ASAN=true` for the asan build | — | +| `Dockerfile`, `compose.yaml` | local-dev image (CI does not use these) | — | + +## How CI uses it + +`.azure-pipelines/build-template.yml`, inside `container: sonic-slave-*`, clones +sonic-swss-common to get the tool and runs: + +```bash +PYTHONPATH=/tmp/sw-common/ci python3 -m buildenv_setup \ + --repo-dir $(Build.SourcesDirectory) --scope build \ + --arch --debian-version --branch $(BUILD_BRANCH) +ASAN= ./build-env/build.sh +``` + +`buildenv_setup` installs the apt/pip packages, downloads + installs the +libswsscommon / libnl / libyang3 / vpp DEBs (from `upstream-artifacts.yaml`), and +runs the redis / sswsyncd / rsyslog post-install hooks. Then `build.sh` builds +the package. + +## Local development + +```bash +cd build-env +DEBIAN_VERSION=bookworm docker compose run --rm build # build .debs +DEBIAN_VERSION=bookworm docker compose run --rm shell # poke around +``` + +The image bakes the dependency setup; your source is mounted live at +`/workspace`, so ordinary edits don't require an image rebuild. Parity with CI +covers **building and C++ unit tests** (`make check`); the full VS/DVS test suite +needs CI-like infrastructure (KVM/privileged/nested-docker). + +> **PoC note:** the tool currently lives on sonic-swss-common's `lawlee/ci-unify` +> branch, so the Dockerfile/CI clone is pinned there (`SWSS_COMMON_REF`), and the +> upstream artifacts + sonic-slave image are pinned to `master`. These TEMPORARY +> pins are removed once the change merges. diff --git a/build-env/build.sh b/build-env/build.sh new file mode 100755 index 0000000000..e2491057cc --- /dev/null +++ b/build-env/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Canonical build for sonic-sairedis. +# +# Single source of truth for "how to build this repo", used by BOTH CI +# (.azure-pipelines/build-template.yml) and local dev (build-env/compose.yaml). +# Must NOT depend on CI-only environment variables (only the optional ASAN flag). +# +# The build environment (apt/pip deps + upstream libnl/libyang/vpp/swss-common +# artifacts) is set up beforehand by `buildenv_setup` (see build-env/README.md); +# this script only runs the actual package build. +# +# ASAN=true ./build-env/build.sh # build with AddressSanitizer instead of gcov +set -ex + +# Run from the repo root regardless of where we were invoked. +cd "$(dirname "$0")/.." + +extraflags='--enable-code-coverage' +case "${ASAN:-false}" in + [Tt]rue|1|yes) extraflags='--enable-asan' ;; +esac + +rm -f ../*.deb || true +./autogen.sh +DEB_BUILD_OPTIONS=nocheck DEB_CONFIGURE_EXTRA_FLAGS="$extraflags" \ + dpkg-buildpackage -us -uc -b -Psyncd,vs,nopython2 -j"$(nproc)" + +# Collect the built .debs at the repo root (where CI publishes from). +mv ../*.deb . diff --git a/build-env/compose.yaml b/build-env/compose.yaml new file mode 100644 index 0000000000..f5c1754baf --- /dev/null +++ b/build-env/compose.yaml @@ -0,0 +1,33 @@ +# Local-dev convenience for building sonic-sairedis in a CI-like container. +# +# Usage: +# cd build-env +# DEBIAN_VERSION=bookworm docker compose run --rm build # build the .debs +# DEBIAN_VERSION=bookworm docker compose run --rm shell # interactive shell +# +# The repo is mounted live at /workspace, so edits on the host are immediately +# visible in the container without rebuilding the image (only a change to +# build-env/ dependency declarations requires an image rebuild). Set ASAN=true to +# build with AddressSanitizer instead of gcov. + +services: + build: + build: + context: .. + dockerfile: build-env/Dockerfile + args: + DEBIAN_VERSION: ${DEBIAN_VERSION:-bookworm} + BUILD_BRANCH: ${BUILD_BRANCH:-master} + SWSS_COMMON_REF: ${SWSS_COMMON_REF:-lawlee/ci-unify} + privileged: true + volumes: + - ..:/workspace + working_dir: /workspace + environment: + ASAN: ${ASAN:-false} + command: ["./build-env/build.sh"] + + shell: + extends: + service: build + command: ["/bin/bash"] diff --git a/build-env/packages/base.yaml b/build-env/packages/base.yaml new file mode 100644 index 0000000000..125c03f496 --- /dev/null +++ b/build-env/packages/base.yaml @@ -0,0 +1,55 @@ +# sonic-sairedis build dependencies (apt + pip). +# +# base.yaml CASCADES: repos downstream of sonic-sairedis (sonic-swss) that build +# against libsairedis inherit everything declared here. +# +# Consumed by `buildenv_setup --scope build`. The buildenv_setup tool itself +# lives in sonic-swss-common/ci/ (cloned by the CI template); sonic-sairedis is a +# consumer. +# +# NOTE: once sonic-swss-common publishes its own build-env/ to the branch this +# build resolves, the sonic-swss-common cascade (see upstream-artifacts.yaml) +# will also contribute its base.yaml deps (libhiredis, swig, redis-server, ...). +# Those are re-declared here for now so the PoC is self-contained and correct +# regardless of cascade availability (apt install is idempotent). + +packages: + # --- core C/C++ build + link dependencies (from build-template.yml) -------- + - libdbus-glib-1-dev + - libpcsclite-dev + - libhiredis-dev + - swig + - libzmq3-dev + - autoconf-archive + + # libnl: NOT declared here at all. The SONiC-patched libnl family + # (libnl-{3,genl,route,nf,cli}-*) is inherited via the sonic-swss-common + # cascade -- sonic-swss-common's upstream-artifacts.yaml downloads the full + # libnl set (incl libnl-cli) from the common-libs artifact for all arches, and + # that cascades transitively to sonic-sairedis. Never apt-install libnl where + # common-libs provides it (see repo policy). This replaces sonic-sairedis' old + # split of amd64-libnl-from-common-lib + arm-libnl-from-apt. + + # redis-server is needed to run sonic-sairedis' unit tests during the Build + # stage (and by downstream VS test hosts). Cascades. + - redis-server + +post_install: + # Configure the system redis for sonic-sairedis' unit tests. These run during the + # Build stage (make check), so scopes: [build, test] -- unlike sonic-swss-common, + # whose Build stage runs the same script INLINE after cargo (its base.yaml entry is + # test-scoped only, to keep redis down during the Rust logger_init_without_redis test). + # + # DEDUP: the script body is NOT carried in sonic-sairedis. `configure-redis-for-tests.sh` + # is resolved from the cascaded sonic-swss-common bundle -- buildenv_setup falls back + # to upstream build-env/ dirs when a source file isn't present locally (see + # post_install.resolve_script) -- so the redis config lives in exactly ONE place: + # sonic-swss-common/build-env/configure-redis-for-tests.sh. + # + # That shared script sets `notify-keyspace-events AKE`, which sonic-sairedis' old + # standalone config did not. Reusing it applies AKE to sonic-sairedis too; the + # sonic-sairedis CI pipeline verifies this keyspace delta is harmless for its tests. + - name: configure-redis-for-tests + source: configure-redis-for-tests.sh # resolved from the sonic-swss-common cascade + requires: [redis-server] + scopes: [build, test] diff --git a/build-env/packages/tooling.yaml b/build-env/packages/tooling.yaml new file mode 100644 index 0000000000..fc6dbff08c --- /dev/null +++ b/build-env/packages/tooling.yaml @@ -0,0 +1,29 @@ +# sonic-sairedis Build-stage-only tooling (does NOT cascade downstream). +# +# Packages/config needed by sonic-sairedis' own build + unit tests but not by +# repos that merely build against libsairedis. Consumed by +# `buildenv_setup --scope build`. + +packages: + # man-page / doc generation used during the sairedis build + - docbook-to-man + - docbook-utils + - aspell-en + # syslog used by the unit tests + - rsyslog + +post_install: + # syncd unit tests expect this runtime directory to exist. + - name: mkdir-sswsyncd + script: | + sudo mkdir -m 755 -p /var/run/sswsyncd + scopes: [build] + + # Start rsyslog so tests that log to syslog work. (The post-build + # "Update rsyslog.conf" step in build-template.yml swaps in azsyslog.conf and + # restarts rsyslogd; that stays in the template as it runs after the build.) + - name: start-rsyslog + script: | + sudo rsyslogd || true + requires: [rsyslog] + scopes: [build] diff --git a/build-env/upstream-artifacts.yaml b/build-env/upstream-artifacts.yaml new file mode 100644 index 0000000000..39857b23e0 --- /dev/null +++ b/build-env/upstream-artifacts.yaml @@ -0,0 +1,73 @@ +# Upstream build artifacts that sonic-sairedis needs at build time. +# +# Consumed by `buildenv_setup`, which downloads each via the Azure DevOps REST +# API (or uses a --upstream-staged-dir// bundle when staged in the same +# pipeline run, e.g. sonic-swss-common's own CI building sairedis downstream). +# +# Artifact-name convention: amd64 has no arch suffix; non-amd64 appends .{arch}. + +upstream: + # ------------------------------------------------------------------ # + # sonic-swss-common libraries that sonic-sairedis links against. + # + # This is sonic-sairedis' only cascading upstream (vpp below is a sairedis-only + # leaf). Note what is deliberately NOT declared here: + # * common-libs (SONiC-patched libnl + libyang3) -- inherited TRANSITIVELY. + # buildenv_setup downloads the sonic-swss-common artifact, reads its + # build-env/, and walks sonic-swss-common's own upstream-artifacts.yaml, + # which declares common-libs -> libnl + libyang3 (the libs libswsscommon + # links against -- all from the common-libs artifact, never apt). See cascade.py. + # + # This REQUIRES the resolved sonic-swss-common artifact to actually ship a + # build-env/, i.e. sonic-swss-common must be built from a branch that already + # carries the buildenv_setup tool + build-env/ (see branch resolution below). + # cascade_optional: false so buildenv_setup WARNS loudly if the artifact has no + # build-env/ to cascade into -- otherwise the cascade silently no-ops and the + # libswsscommon dpkg -i fails downstream on the unmet libyang3 dependency. + # ------------------------------------------------------------------ # + - name: sonic-swss-common + project: build + pipeline: Azure.sonic-swss-common + # TEMPORARY (PoC): pin the cascade source to the POC branch. On a branch build + # BUILD_BRANCH = Build.SourceBranchName = "ci-unify" (Azure returns only the last + # ref segment of refs/heads/lawlee/ci-unify), so ctx.branch can't resolve the + # lawlee/ci-unify swss-common run. Remove this line before merge to restore + # BUILD_BRANCH tracking (commit 93690566). + branch: lawlee/ci-unify + cascade_optional: false + result_filter: [succeeded, partiallySucceeded] + artifact_name: + - { when: { arch: amd64 }, value: 'sonic-swss-common-{debian_version}' } + - { when: { arch: { not: amd64 } }, value: 'sonic-swss-common-{debian_version}.{arch}' } + debs: + - 'libswsscommon_1.0.0_{arch}.deb' + - 'libswsscommon-dev_1.0.0_{arch}.deb' + + # ------------------------------------------------------------------ # + # sonic platform-vpp DEBs (amd64 only). Always built on master. + # ------------------------------------------------------------------ # + - name: vpp + project: build + pipeline: sonic-net.sonic-platform-vpp + branch: master # vpp is always pinned to master (unchanged from today). + when: { arch: amd64 } + cascade_optional: true + result_filter: [succeeded, partiallySucceeded] + apt_fix_broken: true # the 10 vpp DEBs are interdependent; fix up after dpkg -i + # vpp's maintainer scripts call sysctl, which fails in the unprivileged CI + # container. VPP_INSTALL_SKIP_SYSCTL=1 makes them skip it. install_env puts the + # vpp DEBs in their own dpkg -i group (see planner._deb_install_groups), installed + # AFTER the empty-env library providers, as `sudo env VPP_INSTALL_SKIP_SYSCTL=1 dpkg -i ...`. + install_env: { VPP_INSTALL_SKIP_SYSCTL: "1" } + artifact_name: 'vpp-{debian_version}' + debs: + - 'libvppinfra-dev_*_{arch}.deb' + - 'libvppinfra_*_{arch}.deb' + - 'vpp_*_{arch}.deb' + - 'vpp-crypto-engines_*_{arch}.deb' + - 'vpp-dbg_*_{arch}.deb' + - 'vpp-dev_*_{arch}.deb' + - 'vpp-plugin-core_*_{arch}.deb' + - 'vpp-plugin-devtools_*_{arch}.deb' + - 'vpp-plugin-dpdk_*_{arch}.deb' + - 'python3-vpp-api_*_{arch}.deb'