From b0b813d7763563b08fc3d626bdb9c6c625765bc1 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 12:12:40 +1300 Subject: [PATCH 01/66] try w/ bak --- .github/workflows/docker.yml | 29 +++++++++++++++++------------ docker-bake.hcl | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 12 deletions(-) create mode 100644 docker-bake.hcl diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bdbe035..b2539c5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -52,15 +52,14 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push - uses: docker/build-push-action@v5 + - name: Build and push using Bake + uses: docker/bake-action@v6 with: - file: ./Dockerfile.cross - context: . - platforms: linux/${{ env.SHORT_ARCH }} + files: ./docker-bake.hcl push: true - tags: | - ${{ github.repository_owner }}/eleel:latest-${{ env.SHORT_ARCH }} + set: | + *.platform=linux/${{ env.SHORT_ARCH }} + *.tags=${{ github.repository_owner }}/eleel:latest-${{ env.SHORT_ARCH }} build-docker-multiarch: name: build-docker-eleel-multiarch @@ -74,8 +73,14 @@ jobs: run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Create and push multiarch manifests - run: | - docker buildx imagetools create -t ${{ github.repository_owner}}/eleel:latest \ - ${{ github.repository_owner}}/eleel:latest-arm64 \ - ${{ github.repository_owner}}/eleel:latest-amd64; + - name: Create multiarch manifest using Bake + uses: docker/bake-action@v6 + with: + files: ./docker-bake.hcl + push: true + targets: manifest + set: | + manifest.tags=${{ github.repository_owner}}/eleel:latest + manifest.images= | + ${{ github.repository_owner}}/eleel:latest-arm64 + ${{ github.repository_owner}}/eleel:latest-amd64 diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 0000000..13559a8 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,14 @@ +group "default" { + targets = ["binary"] +} + +target "binary" { + dockerfile = "Dockerfile.cross" + context = "." +} + +target "manifest" { + inherits = ["binary"] + platforms = ["linux/arm64", "linux/amd64"] + output = ["type=registry"] +} From 0a9e9276ab098abf28e1d6a900dca73c9d40ad55 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 12:18:11 +1300 Subject: [PATCH 02/66] branches --- .github/workflows/docker.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b2539c5..ce62c9b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,9 +1,11 @@ name: docker on: - push: - branches: - - main + push: + branches: + - main + - stable + - unstable concurrency: group: ${{ github.workflow }}-${{ github.ref }} From a8e72f6556b89bb41e56eacf1593c0ddb01bd983 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 12:23:39 +1300 Subject: [PATCH 03/66] Update DockerHub login method in workflow --- .github/workflows/docker.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ce62c9b..579355f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,9 +26,11 @@ jobs: - uses: actions/checkout@v4 - name: Update Rust run: rustup update stable - - name: Dockerhub login - run: | - echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Cross build binaries run: | cargo install cross From ab4ded2279502df8c2ac84aace41222772cbffa4 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 12:27:49 +1300 Subject: [PATCH 04/66] Add debug step to output Docker username in workflow --- .github/workflows/docker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 579355f..4797c96 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,6 +23,9 @@ jobs: matrix: cpu_arch: [aarch64, x86_64] steps: + - name: debug docker username + run: echo ${{ secrets.DOCKER_USERNAME }} + - uses: actions/checkout@v4 - name: Update Rust run: rustup update stable From a60b7ba95b7f137b6cb5495d74c0a7e2336b5be7 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 12:38:27 +1300 Subject: [PATCH 05/66] Update DockerHub credentials in workflow to use secrets --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4797c96..4e78c92 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,7 +12,7 @@ concurrency: cancel-in-progress: true env: - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} jobs: @@ -32,7 +32,7 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v3 with: - username: ${{ vars.DOCKERHUB_USERNAME }} + username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Cross build binaries run: | From fb47a5fa7770e834589330d78ce1bf85f31f82a7 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 12:41:21 +1300 Subject: [PATCH 06/66] Refactor DockerHub login to use environment variables for credentials --- .github/workflows/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4e78c92..4b43984 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,7 +24,7 @@ jobs: cpu_arch: [aarch64, x86_64] steps: - name: debug docker username - run: echo ${{ secrets.DOCKER_USERNAME }} + run: echo ${{ env.DOCKER_USERNAME }} - uses: actions/checkout@v4 - name: Update Rust @@ -32,8 +32,8 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} - name: Cross build binaries run: | cargo install cross From 7d1c6409871fd60a6abf88205482c8d924300b98 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 12:53:16 +1300 Subject: [PATCH 07/66] Enhance Docker workflow by adding fail-fast strategy and increasing timeout limit --- .github/workflows/docker.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4b43984..5192a0f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,8 +20,9 @@ jobs: name: build-docker-${{ matrix.cpu_arch }} runs-on: ubuntu-22.04 strategy: - matrix: - cpu_arch: [aarch64, x86_64] + fail-fast: false + matrix: + cpu_arch: [aarch64, x86_64] steps: - name: debug docker username run: echo ${{ env.DOCKER_USERNAME }} @@ -38,6 +39,7 @@ jobs: run: | cargo install cross make build-${{ matrix.cpu_arch }} + timeout-minutes: 100 - name: Make bin dir run: mkdir ./bin From 032168ad63d84577e54b98bbc10d7b3e53b4c8de Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 13:13:11 +1300 Subject: [PATCH 08/66] Fix Docker workflow by updating secret variable for Docker password and correcting binary move path --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5192a0f..630eb4d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,7 +12,7 @@ concurrency: cancel-in-progress: true env: - DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} jobs: @@ -45,7 +45,7 @@ jobs: run: mkdir ./bin - name: Move cross-built binary into Docker scope - run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/release/eleel ./bin + run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/release/eleel ./bin/eleel - name: Map aarch64 to arm64 short arch if: startsWith(matrix.cpu_arch, 'aarch64') From 8cce1d0035cd7792c642970cbceb91cbbb891656 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 13:23:10 +1300 Subject: [PATCH 09/66] Update debug step in Docker workflow to format Docker username output --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 630eb4d..789a78f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: cpu_arch: [aarch64, x86_64] steps: - name: debug docker username - run: echo ${{ env.DOCKER_USERNAME }} + run: echo ${{ env.DOCKER_USERNAME }} | sed 's/./& /g' - uses: actions/checkout@v4 - name: Update Rust From 2a86cb2528fe314cdccc4ab11e460dfb23e903a6 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 14:04:56 +1300 Subject: [PATCH 10/66] Add context to Docker build and cross build binaries step in workflow --- .github/workflows/docker.yml | 2 ++ docker-bake.hcl | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 789a78f..b6d05e0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -35,6 +35,7 @@ jobs: with: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} + - name: Cross build binaries run: | cargo install cross @@ -86,6 +87,7 @@ jobs: uses: docker/bake-action@v6 with: files: ./docker-bake.hcl + context: . push: true targets: manifest set: | diff --git a/docker-bake.hcl b/docker-bake.hcl index 13559a8..6c5b0cc 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,5 +1,6 @@ group "default" { targets = ["binary"] + context = "." } target "binary" { From 01b5d79d9ff8e3a15ea2522d93dbaab44d87b749 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 14:18:16 +1300 Subject: [PATCH 11/66] Add debugging steps to Docker workflow for current and bin directory contents --- .github/workflows/docker.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b6d05e0..83c5c9f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -35,7 +35,7 @@ jobs: with: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} - + - name: Cross build binaries run: | cargo install cross @@ -48,6 +48,12 @@ jobs: - name: Move cross-built binary into Docker scope run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/release/eleel ./bin/eleel + - name: Debug current directory current content + run: echo $(pwd) && ls -la + + - name: Debug bin directory content + run: ls -la ./bin + - name: Map aarch64 to arm64 short arch if: startsWith(matrix.cpu_arch, 'aarch64') run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV @@ -87,7 +93,6 @@ jobs: uses: docker/bake-action@v6 with: files: ./docker-bake.hcl - context: . push: true targets: manifest set: | From db2af1aa8257ae06f69b3f61c1630c2fa878c1b0 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 14:42:11 +1300 Subject: [PATCH 12/66] Update Docker workflow context and change COPY to ADD for binary in cross Dockerfile --- .github/workflows/docker.yml | 1 + Dockerfile.cross | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 83c5c9f..41bfe0f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -74,6 +74,7 @@ jobs: files: ./docker-bake.hcl push: true set: | + *.context=. *.platform=linux/${{ env.SHORT_ARCH }} *.tags=${{ github.repository_owner }}/eleel:latest-${{ env.SHORT_ARCH }} diff --git a/Dockerfile.cross b/Dockerfile.cross index 25adf89..fbbb0cc 100644 --- a/Dockerfile.cross +++ b/Dockerfile.cross @@ -7,4 +7,4 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY ./bin/eleel /usr/local/bin/eleel +ADD ./bin/eleel /usr/local/bin/eleel From 10d1fd0bcf3a91b377456ca1855b6883969c33ee Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 14:53:07 +1300 Subject: [PATCH 13/66] Add debug step to check for eleel binary in Docker build context --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 41bfe0f..e4cace6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -68,6 +68,10 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Debug Docker build context + run: tar -tf <(tar -cf - .) | grep bin/eleel || echo "eleel binary is missing in the context" + + - name: Build and push using Bake uses: docker/bake-action@v6 with: From 6dbd7cbc704a869ea5930ea6040409cb06799910 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 14:54:03 +1300 Subject: [PATCH 14/66] Comment out multiarch Docker build steps in workflow --- .github/workflows/docker.yml | 46 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e4cace6..ac54477 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -82,26 +82,26 @@ jobs: *.platform=linux/${{ env.SHORT_ARCH }} *.tags=${{ github.repository_owner }}/eleel:latest-${{ env.SHORT_ARCH }} - build-docker-multiarch: - name: build-docker-eleel-multiarch - runs-on: ubuntu-22.04 - needs: [build-docker-single-arch] - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Dockerhub login - run: | - echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - - name: Create multiarch manifest using Bake - uses: docker/bake-action@v6 - with: - files: ./docker-bake.hcl - push: true - targets: manifest - set: | - manifest.tags=${{ github.repository_owner}}/eleel:latest - manifest.images= | - ${{ github.repository_owner}}/eleel:latest-arm64 - ${{ github.repository_owner}}/eleel:latest-amd64 + # build-docker-multiarch: + # name: build-docker-eleel-multiarch + # runs-on: ubuntu-22.04 + # needs: [build-docker-single-arch] + # steps: + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Dockerhub login + # run: | + # echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + + # - name: Create multiarch manifest using Bake + # uses: docker/bake-action@v6 + # with: + # files: ./docker-bake.hcl + # push: true + # targets: manifest + # set: | + # manifest.tags=${{ github.repository_owner}}/eleel:latest + # manifest.images= | + # ${{ github.repository_owner}}/eleel:latest-arm64 + # ${{ github.repository_owner}}/eleel:latest-amd64 From c2088cca0432aa1f205acfedc7b67c9969087806 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 15:09:03 +1300 Subject: [PATCH 15/66] Update .dockerignore to exclude bin directory from being ignored --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 2f7896d..af487b3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ target/ +!bin/ From ecc371ffc9af0099e0f43ed3ff3fc24b09e42e1a Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 15:17:27 +1300 Subject: [PATCH 16/66] Remove .dockerignore file to simplify Docker build context --- .dockerignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index af487b3..0000000 --- a/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -target/ -!bin/ From 2fedbea7ae92935eef10143811ec26585d226b18 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 15:27:20 +1300 Subject: [PATCH 17/66] Add source context to Docker bake action in workflow --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ac54477..568a386 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -76,6 +76,7 @@ jobs: uses: docker/bake-action@v6 with: files: ./docker-bake.hcl + source: . push: true set: | *.context=. From e446bd4c6aa95693737b66fe0331899a6dda70b4 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 15:56:22 +1300 Subject: [PATCH 18/66] re add and remove debugs --- .dockerignore | 1 + .github/workflows/docker.yml | 55 ++++++++++++++++-------------------- docker-bake.hcl | 5 ++++ 3 files changed, 30 insertions(+), 31 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9f97022 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +target/ \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 568a386..c097931 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,6 +14,7 @@ concurrency: env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_BUILD_SUMMARY: true jobs: build-docker-single-arch: @@ -24,8 +25,6 @@ jobs: matrix: cpu_arch: [aarch64, x86_64] steps: - - name: debug docker username - run: echo ${{ env.DOCKER_USERNAME }} | sed 's/./& /g' - uses: actions/checkout@v4 - name: Update Rust @@ -48,12 +47,6 @@ jobs: - name: Move cross-built binary into Docker scope run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/release/eleel ./bin/eleel - - name: Debug current directory current content - run: echo $(pwd) && ls -la - - - name: Debug bin directory content - run: ls -la ./bin - - name: Map aarch64 to arm64 short arch if: startsWith(matrix.cpu_arch, 'aarch64') run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV @@ -83,26 +76,26 @@ jobs: *.platform=linux/${{ env.SHORT_ARCH }} *.tags=${{ github.repository_owner }}/eleel:latest-${{ env.SHORT_ARCH }} - # build-docker-multiarch: - # name: build-docker-eleel-multiarch - # runs-on: ubuntu-22.04 - # needs: [build-docker-single-arch] - # steps: - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - # - name: Dockerhub login - # run: | - # echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - # - name: Create multiarch manifest using Bake - # uses: docker/bake-action@v6 - # with: - # files: ./docker-bake.hcl - # push: true - # targets: manifest - # set: | - # manifest.tags=${{ github.repository_owner}}/eleel:latest - # manifest.images= | - # ${{ github.repository_owner}}/eleel:latest-arm64 - # ${{ github.repository_owner}}/eleel:latest-amd64 + build-docker-multiarch: + name: build-docker-eleel-multiarch + runs-on: ubuntu-22.04 + needs: [build-docker-single-arch] + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Dockerhub login + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + + - name: Create multiarch manifest using Bake + uses: docker/bake-action@v6 + with: + files: ./docker-bake.hcl + push: true + targets: manifest + set: | + manifest.tags=${{ github.repository_owner}}/eleel:latest + manifest.images= | + ${{ github.repository_owner}}/eleel:latest-arm64 + ${{ github.repository_owner}}/eleel:latest-amd64 diff --git a/docker-bake.hcl b/docker-bake.hcl index 6c5b0cc..f58a94c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -6,6 +6,11 @@ group "default" { target "binary" { dockerfile = "Dockerfile.cross" context = "." + description = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client. + +It is suitable for monitoring and analytics but should not be used to run validators and will cause you to miss block proposals if you attempt this. + +Eleel is written in Rust and makes use of components from Lighthouse." } target "manifest" { From e38ec1f4670d637e2ab4a8380650267fae05cbe4 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 16:09:00 +1300 Subject: [PATCH 19/66] Update Docker bake configuration to simplify description for Eleel multiplexer --- docker-bake.hcl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index f58a94c..a9a83fe 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -6,11 +6,8 @@ group "default" { target "binary" { dockerfile = "Dockerfile.cross" context = "." - description = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client. - -It is suitable for monitoring and analytics but should not be used to run validators and will cause you to miss block proposals if you attempt this. - -Eleel is written in Rust and makes use of components from Lighthouse." + name = "Eleel" + description = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client." } target "manifest" { From dd881ff800a0f56f94ebef9f4acde1a2364c47da Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 16:19:35 +1300 Subject: [PATCH 20/66] fix for "ERROR: ./docker-bake.hcl:6,8-16: Invalid name; name requires matrix" --- docker-bake.hcl | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index a9a83fe..17ab521 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -6,7 +6,6 @@ group "default" { target "binary" { dockerfile = "Dockerfile.cross" context = "." - name = "Eleel" description = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client." } From 44d10f288dc2258676f6f0a006995cf0a31a395c Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 16:36:39 +1300 Subject: [PATCH 21/66] Refactor Docker workflow by removing debug steps and updating tag formatto include branch name --- .github/workflows/docker.yml | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c097931..f7d1cc6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -61,10 +61,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Debug Docker build context - run: tar -tf <(tar -cf - .) | grep bin/eleel || echo "eleel binary is missing in the context" - - - name: Build and push using Bake uses: docker/bake-action@v6 with: @@ -74,28 +70,4 @@ jobs: set: | *.context=. *.platform=linux/${{ env.SHORT_ARCH }} - *.tags=${{ github.repository_owner }}/eleel:latest-${{ env.SHORT_ARCH }} - - build-docker-multiarch: - name: build-docker-eleel-multiarch - runs-on: ubuntu-22.04 - needs: [build-docker-single-arch] - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Dockerhub login - run: | - echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - - name: Create multiarch manifest using Bake - uses: docker/bake-action@v6 - with: - files: ./docker-bake.hcl - push: true - targets: manifest - set: | - manifest.tags=${{ github.repository_owner}}/eleel:latest - manifest.images= | - ${{ github.repository_owner}}/eleel:latest-arm64 - ${{ github.repository_owner}}/eleel:latest-amd64 + *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }}-${{ env.SHORT_ARCH }} From f871bd277c88205cef7d376f9fbaa9ae47728672 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 16:46:49 +1300 Subject: [PATCH 22/66] no need to specify arch --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f7d1cc6..4a12124 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -70,4 +70,4 @@ jobs: set: | *.context=. *.platform=linux/${{ env.SHORT_ARCH }} - *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }}-${{ env.SHORT_ARCH }} + *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }} From 4a83a05ce907eeb4be9f74adaccc30ae30f3baef Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 24 Mar 2025 16:59:33 +1300 Subject: [PATCH 23/66] Add description to Docker bake manifest for Eleel multiplexer --- docker-bake.hcl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 17ab521..1f95498 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -6,11 +6,12 @@ group "default" { target "binary" { dockerfile = "Dockerfile.cross" context = "." - description = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client." + } target "manifest" { inherits = ["binary"] platforms = ["linux/arm64", "linux/amd64"] output = ["type=registry"] + description = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client." } From 5d03e1e70e4cba5339d9cc9c708d468d0a363eda Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 10:20:44 +1300 Subject: [PATCH 24/66] try with var --- .github/workflows/docker.yml | 3 ++- docker-bake.hcl | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4a12124..9e6da47 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -70,4 +70,5 @@ jobs: set: | *.context=. *.platform=linux/${{ env.SHORT_ARCH }} - *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }} + ARCH=${{ env.SHORT_ARCH }} + *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }} diff --git a/docker-bake.hcl b/docker-bake.hcl index 1f95498..2f586c8 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,3 +1,7 @@ +variable "ARCH" { + default = "amd64" +} + group "default" { targets = ["binary"] context = "." @@ -11,7 +15,7 @@ target "binary" { target "manifest" { inherits = ["binary"] - platforms = ["linux/arm64", "linux/amd64"] + platforms = ["linux/${ARCH}"] output = ["type=registry"] description = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client." } From fc144dd5a72257e0ac1c2461847a2d58f4b3df41 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 10:33:46 +1300 Subject: [PATCH 25/66] try using env --- .github/workflows/docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9e6da47..15a8d8c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -70,5 +70,6 @@ jobs: set: | *.context=. *.platform=linux/${{ env.SHORT_ARCH }} - ARCH=${{ env.SHORT_ARCH }} *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }} + env: + ARCH: ${{ env.SHORT_ARCH }} From 5cec76bba66f06b2a326e1164b47299895d81672 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 11:04:24 +1300 Subject: [PATCH 26/66] Update Docker workflow to use x86_64 architecture only. test equals operator --- .github/workflows/docker.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 15a8d8c..ba195ad 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,7 +23,8 @@ jobs: strategy: fail-fast: false matrix: - cpu_arch: [aarch64, x86_64] + # cpu_arch: [aarch64, x86_64] + cpu_arch: [x86_64] steps: - uses: actions/checkout@v4 @@ -52,7 +53,7 @@ jobs: run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV - name: Map x86_64 to amd64 short arch - if: startsWith(matrix.cpu_arch, 'x86_64') + if: ${{ matrix.cpu_arch == 'x86_64' }} run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV - name: Install QEMU From ab96d3f1b21f7e2971b3a20b0def3c1a52db1992 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 11:15:16 +1300 Subject: [PATCH 27/66] Enable support for both aarch64 and x86_64 architectures in Docker workflow --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ba195ad..a9df9cb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,8 +23,7 @@ jobs: strategy: fail-fast: false matrix: - # cpu_arch: [aarch64, x86_64] - cpu_arch: [x86_64] + cpu_arch: [aarch64, x86_64] steps: - uses: actions/checkout@v4 From 0294a87f6980f518ed4d2e9bba691b280140031b Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 11:15:23 +1300 Subject: [PATCH 28/66] Add attestations for provenance and SBOM in Docker bake manifest --- docker-bake.hcl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-bake.hcl b/docker-bake.hcl index 2f586c8..a17ee98 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -5,6 +5,10 @@ variable "ARCH" { group "default" { targets = ["binary"] context = "." + attest = [ + "type=provenance,mode=max", + "type=sbom", + ] } target "binary" { From 7320683de26d9eeb1b8f31ae27129ff54456c086 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 11:34:51 +1300 Subject: [PATCH 29/66] try with platforms under "default" --- docker-bake.hcl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index a17ee98..8996a7e 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -4,6 +4,7 @@ variable "ARCH" { group "default" { targets = ["binary"] + platforms = ["linux/amd64", "linux/arm64"] context = "." attest = [ "type=provenance,mode=max", @@ -19,7 +20,7 @@ target "binary" { target "manifest" { inherits = ["binary"] - platforms = ["linux/${ARCH}"] + # platforms = ["linux/${ARCH}"] output = ["type=registry"] description = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client." } From dfaa7c8182682fcd5f160caac4216f346e025c85 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 11:39:53 +1300 Subject: [PATCH 30/66] Add GITHUB_REPO variable to Docker bake and update workflow environment --- .github/workflows/docker.yml | 2 ++ docker-bake.hcl | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a9df9cb..6f2fadd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -73,3 +73,5 @@ jobs: *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }} env: ARCH: ${{ env.SHORT_ARCH }} + GITHUB_REPO: ${{ github.repository_owner }}/${{ github.repository_name }} + DOCKER_BUILD_RECORD_UPLOAD: true diff --git a/docker-bake.hcl b/docker-bake.hcl index 8996a7e..c34ad6e 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -2,9 +2,16 @@ variable "ARCH" { default = "amd64" } +variable "GITHUB_REPO" { + default = "https://github.com/sigp/eleel" +} + group "default" { targets = ["binary"] platforms = ["linux/amd64", "linux/arm64"] + labels = { + "org.opencontainers.image.source" = "{GITHUB_REPO}" + } context = "." attest = [ "type=provenance,mode=max", From dc30639aa681e752e1b20fb3acf19d31aaa82e6e Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 12:04:04 +1300 Subject: [PATCH 31/66] without specifying platorms --- .github/workflows/docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6f2fadd..f48a078 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -69,7 +69,6 @@ jobs: push: true set: | *.context=. - *.platform=linux/${{ env.SHORT_ARCH }} *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }} env: ARCH: ${{ env.SHORT_ARCH }} From 207d21d9c9270880c019c981317ac4af5792adb5 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 13:46:23 +1300 Subject: [PATCH 32/66] try to push in a separate job --- .github/workflows/docker.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f48a078..232caff 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -66,11 +66,27 @@ jobs: with: files: ./docker-bake.hcl source: . - push: true + push: false set: | *.context=. - *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }} + *.tags=${{ github.repository_owner }}/${{ github.repository_name }}:${{ github.ref_name }} env: ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository_owner }}/${{ github.repository_name }} DOCKER_BUILD_RECORD_UPLOAD: true + + push-docker-image: + name: Push Docker Image + needs: build-docker-single-arch + runs-on: ubuntu-22.04 + steps: + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Push Docker Image + run: | + docker push ${{ github.repository_owner }}/${{ github.repository_name }}:${{ github.ref_name }} + \ No newline at end of file From 2a8c5818d39c7caaa86544dba32a1737e5f87730 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 14:02:16 +1300 Subject: [PATCH 33/66] Update Docker tags to use 'eleel' repository for consistency --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 232caff..62a6ece 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -69,7 +69,7 @@ jobs: push: false set: | *.context=. - *.tags=${{ github.repository_owner }}/${{ github.repository_name }}:${{ github.ref_name }} + *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }} env: ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository_owner }}/${{ github.repository_name }} From 6ee1685ef83b4a2cab18f9ba9e587ae54cb409f4 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 14:16:23 +1300 Subject: [PATCH 34/66] Refactor Docker workflow to use full repository name for tagging and pushing images --- .github/workflows/docker.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 62a6ece..3ef32bd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,6 +15,7 @@ env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_BUILD_SUMMARY: true + REPO_NAME: $GITHUB_REPOSITORY jobs: build-docker-single-arch: @@ -69,10 +70,10 @@ jobs: push: false set: | *.context=. - *.tags=${{ github.repository_owner }}/eleel:${{ github.ref_name }} + *.tags=${{ github.repository }}:${{ github.ref_name }} env: ARCH: ${{ env.SHORT_ARCH }} - GITHUB_REPO: ${{ github.repository_owner }}/${{ github.repository_name }} + GITHUB_REPO: ${{ github.repository }} DOCKER_BUILD_RECORD_UPLOAD: true push-docker-image: @@ -88,5 +89,5 @@ jobs: - name: Push Docker Image run: | - docker push ${{ github.repository_owner }}/${{ github.repository_name }}:${{ github.ref_name }} + docker push ${{ github.repository }}:${{ github.ref_name }} \ No newline at end of file From d116e12242600c46197b691b31633b03d480c7b8 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 14:33:18 +1300 Subject: [PATCH 35/66] Enhance Docker workflow by adding debug step to list and verify Docker images before pushing --- .github/workflows/docker.yml | 97 +++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3ef32bd..2859e59 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,10 +12,10 @@ concurrency: cancel-in-progress: true env: - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_BUILD_SUMMARY: true - REPO_NAME: $GITHUB_REPOSITORY + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_BUILD_SUMMARY: true + REPO_NAME: $GITHUB_REPOSITORY jobs: build-docker-single-arch: @@ -27,54 +27,54 @@ jobs: cpu_arch: [aarch64, x86_64] steps: - - uses: actions/checkout@v4 - - name: Update Rust - run: rustup update stable - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_PASSWORD }} - - - name: Cross build binaries - run: | - cargo install cross - make build-${{ matrix.cpu_arch }} - timeout-minutes: 100 + - uses: actions/checkout@v4 + - name: Update Rust + run: rustup update stable + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Cross build binaries + run: | + cargo install cross + make build-${{ matrix.cpu_arch }} + timeout-minutes: 100 - - name: Make bin dir - run: mkdir ./bin + - name: Make bin dir + run: mkdir ./bin - - name: Move cross-built binary into Docker scope - run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/release/eleel ./bin/eleel + - name: Move cross-built binary into Docker scope + run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/release/eleel ./bin/eleel - - name: Map aarch64 to arm64 short arch - if: startsWith(matrix.cpu_arch, 'aarch64') - run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV + - name: Map aarch64 to arm64 short arch + if: startsWith(matrix.cpu_arch, 'aarch64') + run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV - - name: Map x86_64 to amd64 short arch - if: ${{ matrix.cpu_arch == 'x86_64' }} - run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV + - name: Map x86_64 to amd64 short arch + if: ${{ matrix.cpu_arch == 'x86_64' }} + run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV - - name: Install QEMU - run: sudo apt-get update && sudo apt-get install -y qemu-user-static + - name: Install QEMU + run: sudo apt-get update && sudo apt-get install -y qemu-user-static - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Build and push using Bake - uses: docker/bake-action@v6 - with: - files: ./docker-bake.hcl - source: . - push: false - set: | - *.context=. - *.tags=${{ github.repository }}:${{ github.ref_name }} - env: - ARCH: ${{ env.SHORT_ARCH }} - GITHUB_REPO: ${{ github.repository }} - DOCKER_BUILD_RECORD_UPLOAD: true + - name: Build and push using Bake + uses: docker/bake-action@v6 + with: + files: ./docker-bake.hcl + source: . + push: false + set: | + *.context=. + *.tags=${{ github.repository }}:${{ github.ref_name }} + env: + ARCH: ${{ env.SHORT_ARCH }} + GITHUB_REPO: ${{ github.repository }} + DOCKER_BUILD_RECORD_UPLOAD: true push-docker-image: name: Push Docker Image @@ -87,6 +87,13 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Debug Docker Images + run: | + echo "Listing all Docker images:" + docker images + echo "Looking for the specific image:" + docker images | grep "${{ github.repository }}:${{ github.ref_name }}" + - name: Push Docker Image run: | docker push ${{ github.repository }}:${{ github.ref_name }} From 7d54cc85fc5997556b347f1dbcb5641be8c3ffbc Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 14:50:55 +1300 Subject: [PATCH 36/66] Add load option to Docker job to ensure image is loaded into local Docker daemon --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2859e59..76fd811 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -68,6 +68,7 @@ jobs: files: ./docker-bake.hcl source: . push: false + load: true # <--- Ensure the image is loaded into the local Docker daemon set: | *.context=. *.tags=${{ github.repository }}:${{ github.ref_name }} From a564a829a366bffde97e34f0c1b37d5022430978 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 16:58:25 +1300 Subject: [PATCH 37/66] w/o matrix - use make to build all platforms, and bake to build docker platforms --- .dockerignore | 2 +- .github/workflows/docker.yml | 44 +++++++----------------------------- Dockerfile.cross | 11 +++++---- Makefile | 2 ++ docker-bake.hcl | 26 +++++++++++++-------- 5 files changed, 33 insertions(+), 52 deletions(-) diff --git a/.dockerignore b/.dockerignore index 9f97022..c4c33f4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1 @@ -target/ \ No newline at end of file +!target/ \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 76fd811..25f8a1b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,8 +23,8 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false - matrix: - cpu_arch: [aarch64, x86_64] + # matrix: + # cpu_arch: [aarch64, x86_64] steps: - uses: actions/checkout@v4 @@ -42,23 +42,18 @@ jobs: make build-${{ matrix.cpu_arch }} timeout-minutes: 100 - - name: Make bin dir - run: mkdir ./bin + - - name: Move cross-built binary into Docker scope - run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/release/eleel ./bin/eleel + - - name: Map aarch64 to arm64 short arch - if: startsWith(matrix.cpu_arch, 'aarch64') - run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV - - name: Map x86_64 to amd64 short arch - if: ${{ matrix.cpu_arch == 'x86_64' }} - run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV - name: Install QEMU run: sudo apt-get update && sudo apt-get install -y qemu-user-static + - name: Build using Make + run: make build + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -67,7 +62,7 @@ jobs: with: files: ./docker-bake.hcl source: . - push: false + push: true load: true # <--- Ensure the image is loaded into the local Docker daemon set: | *.context=. @@ -76,26 +71,3 @@ jobs: ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository }} DOCKER_BUILD_RECORD_UPLOAD: true - - push-docker-image: - name: Push Docker Image - needs: build-docker-single-arch - runs-on: ubuntu-22.04 - steps: - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Debug Docker Images - run: | - echo "Listing all Docker images:" - docker images - echo "Looking for the specific image:" - docker images | grep "${{ github.repository }}:${{ github.ref_name }}" - - - name: Push Docker Image - run: | - docker push ${{ github.repository }}:${{ github.ref_name }} - \ No newline at end of file diff --git a/Dockerfile.cross b/Dockerfile.cross index fbbb0cc..932763c 100644 --- a/Dockerfile.cross +++ b/Dockerfile.cross @@ -2,9 +2,10 @@ # It assumes the eleel binary has already been # compiled for `$TARGETPLATFORM` and moved to `./bin`. FROM --platform=$TARGETPLATFORM ubuntu:24.04 +ARG TARGET_ARCH RUN apt-get update && apt-get install -y --no-install-recommends \ - libssl-dev \ - ca-certificates \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* -ADD ./bin/eleel /usr/local/bin/eleel + libssl-dev \ + ca-certificates \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +ADD ./target/$TARGET_ARCH/release/eleel /usr/local/bin/eleel diff --git a/Makefile b/Makefile index 71f8a1d..ea7d115 100644 --- a/Makefile +++ b/Makefile @@ -4,4 +4,6 @@ build-x86_64: build-aarch64: cross build --bin eleel --target aarch64-unknown-linux-gnu --profile release --locked +build: build-x86_64 build-aarch64 + .PHONY: build-x86_64 build-aarch64 diff --git a/docker-bake.hcl b/docker-bake.hcl index c34ad6e..0cc0ad8 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,14 +1,9 @@ -variable "ARCH" { - default = "amd64" -} - variable "GITHUB_REPO" { - default = "https://github.com/sigp/eleel" + default = "https://github.com/sigp/eleel" } group "default" { - targets = ["binary"] - platforms = ["linux/amd64", "linux/arm64"] + targets = ["binary-amd64", "binary-arm64"] labels = { "org.opencontainers.image.source" = "{GITHUB_REPO}" } @@ -19,15 +14,26 @@ group "default" { ] } -target "binary" { +target "binary-amd64" { + dockerfile = "Dockerfile.cross" + context = "." + platforms = ["linux/amd64"] + args = { + TARGET_ARCH = "x86_64-unknown-linux-gnu" + } +} + +target "binary-arm64" { dockerfile = "Dockerfile.cross" context = "." - + platforms = ["linux/arm64"] + args = { + TARGET_ARCH = "aarch64-unknown-linux-gnu" + } } target "manifest" { inherits = ["binary"] - # platforms = ["linux/${ARCH}"] output = ["type=registry"] description = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client." } From af7f2464be1cc4ceb21d17f8fb4eb671062d8456 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 16:59:57 +1300 Subject: [PATCH 38/66] Add 'make-and-bake' trigger to Docker workflow --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 25f8a1b..4ba865c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,6 +6,7 @@ on: - main - stable - unstable + - make-and-bake concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 80fbd0f04304b09ddd762f6d92bae88760c4b3bc Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 17:02:13 +1300 Subject: [PATCH 39/66] Simplify Docker build job by removing architecture-specific naming and redundant build step --- .github/workflows/docker.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4ba865c..e6c323a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,7 +20,7 @@ env: jobs: build-docker-single-arch: - name: build-docker-${{ matrix.cpu_arch }} + name: build-docker runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -40,7 +40,7 @@ jobs: - name: Cross build binaries run: | cargo install cross - make build-${{ matrix.cpu_arch }} + make build timeout-minutes: 100 @@ -52,9 +52,6 @@ jobs: - name: Install QEMU run: sudo apt-get update && sudo apt-get install -y qemu-user-static - - name: Build using Make - run: make build - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From c99007cc1e6e53810df81d3b3af9f0100e0bd23f Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 25 Mar 2025 17:24:03 +1300 Subject: [PATCH 40/66] try without attest entries from Docker bake configuration --- docker-bake.hcl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 0cc0ad8..c54f18c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -8,10 +8,7 @@ group "default" { "org.opencontainers.image.source" = "{GITHUB_REPO}" } context = "." - attest = [ - "type=provenance,mode=max", - "type=sbom", - ] + } target "binary-amd64" { From bb66109529b712ae5d89fc0a4aa87c8bc3a7bad4 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 07:40:18 +1300 Subject: [PATCH 41/66] Add multiarch manifest job and version extraction workflow to Docker build --- .github/workflows/docker.yml | 22 +++++++++++- .github/workflows/extract-version.yml | 49 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/extract-version.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e6c323a..fde24ba 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,7 +19,9 @@ env: REPO_NAME: $GITHUB_REPOSITORY jobs: - build-docker-single-arch: + extract-version: + uses: ./.github/workflows/extract-version.yml + build-docker: name: build-docker runs-on: ubuntu-22.04 strategy: @@ -69,3 +71,21 @@ jobs: ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository }} DOCKER_BUILD_RECORD_UPLOAD: true + docker-multiarch-manifest: + name: docker-multiarch-manifest + runs-on: ubuntu-24.04 + needs: [build-docker, extract-version] + env: + # We need to enable experimental docker features in order to use `docker manifest` + DOCKER_CLI_EXPERIMENTAL: enabled + VERSION: ${{ needs.extract-version.outputs.VERSION }} + VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} + steps: + - name: Dockerhub login + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + - name: Create and push multiarch manifest + run: | + docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX} \ + ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}-amd64; + ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}-arm64; \ No newline at end of file diff --git a/.github/workflows/extract-version.yml b/.github/workflows/extract-version.yml new file mode 100644 index 0000000..831a000 --- /dev/null +++ b/.github/workflows/extract-version.yml @@ -0,0 +1,49 @@ +name: Extract Version + +on: + workflow_call: + outputs: + VERSION: + description: "The extracted version (latest or vX.Y.Z)" + value: ${{ jobs.extract-version.outputs.VERSION || 'latest' }} + VERSION_SUFFIX: + description: "The version suffix (empty or -unstable)" + value: ${{ jobs.extract-version.outputs.VERSION_SUFFIX }} + +jobs: + extract-version: + runs-on: ubuntu-22.04 + outputs: + VERSION: ${{ env.VERSION }} + VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} + steps: + + + - name: Extract version (if stable) + if: github.event.ref == 'refs/heads/stable' + run: | + echo "VERSION=latest" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV + + - name: Extract version (if unstable) + if: github.event.ref == 'refs/heads/unstable' + run: | + echo "VERSION=latest" >> $GITHUB_ENV + echo "VERSION_SUFFIX=-unstable" >> $GITHUB_ENV + + - name: Extract version (if other branch) + if: github.event.ref != 'refs/heads/stable' && github.event.ref != 'refs/heads/unstable' && !startsWith(github.event.ref, 'refs/tags') + run: | + echo "VERSION=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV + + - name: Extract version (if tagged release) + if: startsWith(github.event.ref, 'refs/tags') + run: | + echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV + + - name: Debug - Print environment variables + run: | + echo "[debug] VERSION=${{ env.VERSION }}" + echo "[debug] VERSION_SUFFIX=${{ env.VERSION_SUFFIX }}" From 6c5d35d0a19ed20e6419364e7dba237f654cf558 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 08:00:03 +1300 Subject: [PATCH 42/66] Refactor Docker workflow to use environment variable for image name and improve DockerHub login step --- .github/workflows/docker.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fde24ba..9a7a872 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,6 +17,7 @@ env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_BUILD_SUMMARY: true REPO_NAME: $GITHUB_REPOSITORY + IMAGE_NAME: ${{ github.repository }}:${{ github.ref_name }} jobs: extract-version: @@ -66,7 +67,7 @@ jobs: load: true # <--- Ensure the image is loaded into the local Docker daemon set: | *.context=. - *.tags=${{ github.repository }}:${{ github.ref_name }} + *.tags=${{ env.IMAGE_NAME }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX}} env: ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository }} @@ -81,11 +82,12 @@ jobs: VERSION: ${{ needs.extract-version.outputs.VERSION }} VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} steps: - - name: Dockerhub login - run: | - echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} - name: Create and push multiarch manifest run: | - docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX} \ - ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}-amd64; - ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}-arm64; \ No newline at end of file + docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX} + \ No newline at end of file From 8b200683aa3f0816ac3e0f85d08fe19523956f36 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 08:34:52 +1300 Subject: [PATCH 43/66] Add debug step to output IMAGE_NAME in Docker workflow --- .github/workflows/docker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9a7a872..a192e81 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,6 +31,8 @@ jobs: # cpu_arch: [aarch64, x86_64] steps: + + - uses: actions/checkout@v4 - name: Update Rust run: rustup update stable @@ -39,6 +41,9 @@ jobs: with: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} + + - name: Debug IMAGE_NAME + run: echo "IMAGE_NAME=${{ env.IMAGE_NAME }}" - name: Cross build binaries run: | From bf6fc5a34d4373edcd89f419ad80a6dcf00cdad3 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 08:36:55 +1300 Subject: [PATCH 44/66] Remove versioning from Docker image tags in workflow --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a192e81..9ff6f83 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -72,7 +72,7 @@ jobs: load: true # <--- Ensure the image is loaded into the local Docker daemon set: | *.context=. - *.tags=${{ env.IMAGE_NAME }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX}} + *.tags=${{ env.IMAGE_NAME }} env: ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository }} From d42ac8d2fa9149fa8b27b7ea36ca24b541476d8b Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 09:16:31 +1300 Subject: [PATCH 45/66] Update Docker workflow to use environment variable for image name in multiarch manifest creation --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9ff6f83..55ea21d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -94,5 +94,5 @@ jobs: password: ${{ env.DOCKER_PASSWORD }} - name: Create and push multiarch manifest run: | - docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX} + docker buildx imagetools create -t ${{ env.IMAGE_NAME }} \ No newline at end of file From 226fa87e7ec91bf24c1bb2c556d3f474110599fb Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 09:43:07 +1300 Subject: [PATCH 46/66] Enhance multiarch manifest creation by adding --append and --debug options --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 55ea21d..6d1e4ea 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -94,5 +94,5 @@ jobs: password: ${{ env.DOCKER_PASSWORD }} - name: Create and push multiarch manifest run: | - docker buildx imagetools create -t ${{ env.IMAGE_NAME }} + docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug \ No newline at end of file From 41b479fca2b538b1aaa11e17bea2c8de31e7a9ce Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 10:39:57 +1300 Subject: [PATCH 47/66] Refactor Docker workflow to separate build and push jobs, and add version extraction step --- .github/workflows/docker.yml | 172 ++++++++++++++------------ .github/workflows/extract-version.yml | 49 ++++++++ 2 files changed, 140 insertions(+), 81 deletions(-) create mode 100644 .github/workflows/extract-version.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 76fd811..c4b21ac 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,86 +16,96 @@ env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_BUILD_SUMMARY: true REPO_NAME: $GITHUB_REPOSITORY + IMAGE_NAME: ${{ github.repository }}:${{ github.ref_name }} jobs: - build-docker-single-arch: - name: build-docker-${{ matrix.cpu_arch }} - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - cpu_arch: [aarch64, x86_64] - steps: - - - uses: actions/checkout@v4 - - name: Update Rust - run: rustup update stable - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_PASSWORD }} - - - name: Cross build binaries - run: | - cargo install cross - make build-${{ matrix.cpu_arch }} - timeout-minutes: 100 - - - name: Make bin dir - run: mkdir ./bin - - - name: Move cross-built binary into Docker scope - run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/release/eleel ./bin/eleel - - - name: Map aarch64 to arm64 short arch - if: startsWith(matrix.cpu_arch, 'aarch64') - run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV - - - name: Map x86_64 to amd64 short arch - if: ${{ matrix.cpu_arch == 'x86_64' }} - run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV - - - name: Install QEMU - run: sudo apt-get update && sudo apt-get install -y qemu-user-static - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push using Bake - uses: docker/bake-action@v6 - with: - files: ./docker-bake.hcl - source: . - push: false - load: true # <--- Ensure the image is loaded into the local Docker daemon - set: | - *.context=. - *.tags=${{ github.repository }}:${{ github.ref_name }} - env: - ARCH: ${{ env.SHORT_ARCH }} - GITHUB_REPO: ${{ github.repository }} - DOCKER_BUILD_RECORD_UPLOAD: true - - push-docker-image: - name: Push Docker Image - needs: build-docker-single-arch - runs-on: ubuntu-22.04 - steps: - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Debug Docker Images - run: | - echo "Listing all Docker images:" - docker images - echo "Looking for the specific image:" - docker images | grep "${{ github.repository }}:${{ github.ref_name }}" - - - name: Push Docker Image - run: | - docker push ${{ github.repository }}:${{ github.ref_name }} - \ No newline at end of file + extract-version: + uses: ./.github/workflows/extract-version.yml + build-docker: + name: build-docker-${{ matrix.cpu_arch }} + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + cpu_arch: [aarch64, x86_64] + steps: + + - uses: actions/checkout@v4 + - name: Update Rust + run: rustup update stable + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Cross build binaries + run: | + cargo install cross + make build-${{ matrix.cpu_arch }} + timeout-minutes: 100 + + - name: Make bin dir + run: mkdir ./bin + + - name: Move cross-built binary into Docker scope + run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/release/eleel ./bin/eleel + + - name: Map aarch64 to arm64 short arch + if: startsWith(matrix.cpu_arch, 'aarch64') + run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV + + - name: Map x86_64 to amd64 short arch + if: ${{ matrix.cpu_arch == 'x86_64' }} + run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV + + - name: Install QEMU + run: sudo apt-get update && sudo apt-get install -y qemu-user-static + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push using Bake + uses: docker/bake-action@v6 + with: + files: ./docker-bake.hcl + source: . + push: false + load: true # <--- Ensure the image is loaded into the local Docker daemon + set: | + *.context=. + *.tags=${{ env.IMAGE_NAME }} + env: + ARCH: ${{ env.SHORT_ARCH }} + GITHUB_REPO: ${{ github.repository }} + DOCKER_BUILD_RECORD_UPLOAD: true + + push-docker-image: + name: Push Docker Image + needs: build-docker + runs-on: ubuntu-22.04 + steps: + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Debug Docker Images + run: | + echo "Listing all Docker images:" + docker images + echo "Looking for the specific image:" + docker images | grep "${{ github.repository }}:${{ github.ref_name }}" + + - name: Create and push multiarch manifest + run: | + docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug + + - name: Push Docker Image + run: | + docker push ${{ github.repository }}:${{ github.ref_name }} + + + + \ No newline at end of file diff --git a/.github/workflows/extract-version.yml b/.github/workflows/extract-version.yml new file mode 100644 index 0000000..ca348cd --- /dev/null +++ b/.github/workflows/extract-version.yml @@ -0,0 +1,49 @@ +name: Extract Version + +on: + workflow_call: + outputs: + VERSION: + description: "The extracted version (latest or vX.Y.Z)" + value: ${{ jobs.extract-version.outputs.VERSION || 'latest' }} + VERSION_SUFFIX: + description: "The version suffix (empty or -unstable)" + value: ${{ jobs.extract-version.outputs.VERSION_SUFFIX }} + +jobs: + extract-version: + runs-on: ubuntu-22.04 + outputs: + VERSION: ${{ env.VERSION }} + VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} + steps: + + + - name: Extract version (if stable) + if: github.event.ref == 'refs/heads/stable' + run: | + echo "VERSION=latest" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV + + - name: Extract version (if unstable) + if: github.event.ref == 'refs/heads/unstable' + run: | + echo "VERSION=latest" >> $GITHUB_ENV + echo "VERSION_SUFFIX=-unstable" >> $GITHUB_ENV + + - name: Extract version (if other branch) + if: github.event.ref != 'refs/heads/stable' && github.event.ref != 'refs/heads/unstable' && !startsWith(github.event.ref, 'refs/tags') + run: | + echo "VERSION=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV + + - name: Extract version (if tagged release) + if: startsWith(github.event.ref, 'refs/tags') + run: | + echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV + + - name: Debug - Print environment variables + run: | + echo "[debug] VERSION=${{ env.VERSION }}" + echo "[debug] VERSION_SUFFIX=${{ env.VERSION_SUFFIX }}" \ No newline at end of file From 8944fa9d3e76113f48532e66d23eb45714bf1e79 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 10:50:08 +1300 Subject: [PATCH 48/66] Refactor Docker workflow by removing redundant push step and improving formatting --- .github/workflows/docker.yml | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c4b21ac..166f617 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -80,32 +80,12 @@ jobs: GITHUB_REPO: ${{ github.repository }} DOCKER_BUILD_RECORD_UPLOAD: true - push-docker-image: - name: Push Docker Image - needs: build-docker - runs-on: ubuntu-22.04 - steps: - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Debug Docker Images - run: | - echo "Listing all Docker images:" - docker images - echo "Looking for the specific image:" - docker images | grep "${{ github.repository }}:${{ github.ref_name }}" - - name: Create and push multiarch manifest run: | - docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug - + docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug + - name: Push Docker Image run: | - docker push ${{ github.repository }}:${{ github.ref_name }} + docker push ${{ github.repository }}:${{ github.ref_name }} - - - \ No newline at end of file + \ No newline at end of file From 232d54959cce449e12b12336c306eb865d238a27 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 11:12:23 +1300 Subject: [PATCH 49/66] Update Docker workflow to include architecture in image tags for better versioning --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 166f617..2cea854 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -74,7 +74,7 @@ jobs: load: true # <--- Ensure the image is loaded into the local Docker daemon set: | *.context=. - *.tags=${{ env.IMAGE_NAME }} + *.tags=${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} env: ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository }} @@ -82,7 +82,7 @@ jobs: - name: Create and push multiarch manifest run: | - docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug + docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }} ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} - name: Push Docker Image run: | From 9f66824d8b9f551b7309118fd3dfbeaa0fb5547a Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 11:28:33 +1300 Subject: [PATCH 50/66] Update Docker workflow to limit CPU architecture to aarch64 and add debug step for listing Docker images --- .github/workflows/docker.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2cea854..403e864 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,7 +27,8 @@ jobs: strategy: fail-fast: false matrix: - cpu_arch: [aarch64, x86_64] + # cpu_arch: [aarch64, x86_64] + cpu_arch: [aarch64] steps: - uses: actions/checkout@v4 @@ -80,6 +81,9 @@ jobs: GITHUB_REPO: ${{ github.repository }} DOCKER_BUILD_RECORD_UPLOAD: true + - name: Debug Docker Images + run: docker images + - name: Create and push multiarch manifest run: | docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }} ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} From 9644a0bb2538da7b6822f42111b4e21bbfa61ec5 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 11:42:02 +1300 Subject: [PATCH 51/66] try reversing images --- .github/workflows/docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 403e864..d6bf7c0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -86,7 +86,8 @@ jobs: - name: Create and push multiarch manifest run: | - docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }} ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} + docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} \ + ${{ env.IMAGE_NAME }} - name: Push Docker Image run: | From 72c3e3eb84a377d6314480e48c1e6afb6bd3becf Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 11:51:09 +1300 Subject: [PATCH 52/66] undo reverse and add push --- .github/workflows/docker.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d6bf7c0..590dfb5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -71,7 +71,7 @@ jobs: with: files: ./docker-bake.hcl source: . - push: false + push: true load: true # <--- Ensure the image is loaded into the local Docker daemon set: | *.context=. @@ -86,8 +86,7 @@ jobs: - name: Create and push multiarch manifest run: | - docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} \ - ${{ env.IMAGE_NAME }} + docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }} ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} - name: Push Docker Image run: | From 34535ee56d4c5e14f11fcda2e80ce1738e1e1252 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 12:04:36 +1300 Subject: [PATCH 53/66] re-re-reverse --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 590dfb5..59795d1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -86,7 +86,7 @@ jobs: - name: Create and push multiarch manifest run: | - docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }} ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} + docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} ${{ env.IMAGE_NAME }} - name: Push Docker Image run: | From 901f573dd969a9cb06bdc7fe64fbe0b514d99e36 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 12:27:29 +1300 Subject: [PATCH 54/66] Refactor Docker workflow to enhance multiarch manifest creation and improve image tagging format --- .github/workflows/docker.yml | 37 ++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 59795d1..1c1d922 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -75,21 +75,38 @@ jobs: load: true # <--- Ensure the image is loaded into the local Docker daemon set: | *.context=. - *.tags=${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} + *.tags=${{ env.IMAGE_NAME }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX}}-${{ env.SHORT_ARCH }} env: ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository }} DOCKER_BUILD_RECORD_UPLOAD: true - - name: Debug Docker Images - run: docker images + # - name: Debug Docker Images + # run: docker images - - name: Create and push multiarch manifest - run: | - docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} ${{ env.IMAGE_NAME }} + # - name: Create and push multiarch manifest + # run: | + # docker buildx imagetools create --append --debug -t ${{ env.IMAGE_NAME }}-${{ env.SHORT_ARCH }} ${{ env.IMAGE_NAME }} - - name: Push Docker Image - run: | - docker push ${{ github.repository }}:${{ github.ref_name }} + # - name: Push Docker Image + # run: | + # docker push ${{ github.repository }}:${{ github.ref_name }} - \ No newline at end of file + docker-multiarch-manifest: + name: docker-multiarch-manifest + runs-on: ubuntu-24.04 + needs: [build-docker, extract-version] + env: + # We need to enable experimental docker features in order to use `docker manifest` + DOCKER_CLI_EXPERIMENTAL: enabled + VERSION: ${{ needs.extract-version.outputs.VERSION }} + VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} + steps: + - name: Dockerhub login + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + - name: Create and push multiarch manifest + run: | + docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX} \ + ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}-amd64; +# ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}-arm64; \ No newline at end of file From 7f3d4a04fc96b9931ad14ab83bd009b1847051df Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 12:27:36 +1300 Subject: [PATCH 55/66] Set default values for VERSION and VERSION_SUFFIX in extract-version workflow --- .github/workflows/extract-version.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/extract-version.yml b/.github/workflows/extract-version.yml index ca348cd..8d520d5 100644 --- a/.github/workflows/extract-version.yml +++ b/.github/workflows/extract-version.yml @@ -14,8 +14,8 @@ jobs: extract-version: runs-on: ubuntu-22.04 outputs: - VERSION: ${{ env.VERSION }} - VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} + VERSION: ${{ env.VERSION || '0.0.0' }} + VERSION_SUFFIX: ${{ env.VERSION_SUFFIX || 'alpha' }} steps: From e76fa4e6e6080f918103d46cfb9946228b592351 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 12:39:24 +1300 Subject: [PATCH 56/66] Fix image tagging format in Docker workflow to include IMAGE_NAME prefix --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1c1d922..4c6ae09 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -75,7 +75,7 @@ jobs: load: true # <--- Ensure the image is loaded into the local Docker daemon set: | *.context=. - *.tags=${{ env.IMAGE_NAME }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX}}-${{ env.SHORT_ARCH }} + *.tags=${{ env.IMAGE_NAME }}-${{ env.VERSION }}${{ env.VERSION_SUFFIX}}-${{ env.SHORT_ARCH }} env: ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository }} From 10cf99a785f3acb9dfdb35ac5c1ac06d0168be0c Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 13:58:43 +1300 Subject: [PATCH 57/66] Update Docker workflow to create multiarch manifest with specific tags for arm64 and amd64 --- .github/workflows/docker.yml | 46 +++++++++++++++++++++++------------- docker-bake.hcl | 3 +++ 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6d1e4ea..eea6dd8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -74,25 +74,37 @@ jobs: *.context=. *.tags=${{ env.IMAGE_NAME }} env: - ARCH: ${{ env.SHORT_ARCH }} GITHUB_REPO: ${{ github.repository }} DOCKER_BUILD_RECORD_UPLOAD: true - docker-multiarch-manifest: - name: docker-multiarch-manifest - runs-on: ubuntu-24.04 - needs: [build-docker, extract-version] - env: - # We need to enable experimental docker features in order to use `docker manifest` - DOCKER_CLI_EXPERIMENTAL: enabled - VERSION: ${{ needs.extract-version.outputs.VERSION }} - VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} - steps: - - name: Login to DockerHub - uses: docker/login-action@v3 - with: + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_PASSWORD }} - - name: Create and push multiarch manifest - run: | - docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug + + - name: Create and push multiarch manifest + run: | + docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug \ + ${{ env.IMAGE_NAME }}-arm64 \ + ${{ env.IMAGE_NAME }}-amd64 + + # docker-multiarch-manifest: + # name: docker-multiarch-manifest + # runs-on: ubuntu-24.04 + # needs: [build-docker, extract-version] + # env: + # # We need to enable experimental docker features in order to use `docker manifest` + # DOCKER_CLI_EXPERIMENTAL: enabled + # VERSION: ${{ needs.extract-version.outputs.VERSION }} + # VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} + # steps: + # - name: Login to DockerHub + # uses: docker/login-action@v3 + # with: + # username: ${{ env.DOCKER_USERNAME }} + # password: ${{ env.DOCKER_PASSWORD }} + # - name: Create and push multiarch manifest + # run: | + # docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug \ No newline at end of file diff --git a/docker-bake.hcl b/docker-bake.hcl index c54f18c..64c3803 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -8,6 +8,7 @@ group "default" { "org.opencontainers.image.source" = "{GITHUB_REPO}" } context = "." + tags = ["magick93/eleel"] } @@ -15,6 +16,7 @@ target "binary-amd64" { dockerfile = "Dockerfile.cross" context = "." platforms = ["linux/amd64"] + tags = ["magick93/eleel-amd64"] args = { TARGET_ARCH = "x86_64-unknown-linux-gnu" } @@ -24,6 +26,7 @@ target "binary-arm64" { dockerfile = "Dockerfile.cross" context = "." platforms = ["linux/arm64"] + tags = ["magick93/eleel-arm64"] args = { TARGET_ARCH = "aarch64-unknown-linux-gnu" } From 0aaec360351bb15e437887d27e7aaa86c180cf80 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 14:19:28 +1300 Subject: [PATCH 58/66] Update multiarch manifest creation to use repository name for architecture-specific images --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eea6dd8..17cf00d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -86,8 +86,8 @@ jobs: - name: Create and push multiarch manifest run: | docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug \ - ${{ env.IMAGE_NAME }}-arm64 \ - ${{ env.IMAGE_NAME }}-amd64 + ${{ github.repository }}-arm64 \ + ${{ github.repository }}-amd64 # docker-multiarch-manifest: # name: docker-multiarch-manifest From 6e6b76f1d15c1f990be61a4f0bf1cf4482b5b9ab Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 15:09:14 +1300 Subject: [PATCH 59/66] Refactor Docker bake configuration to use repository name for tags and update image source labels --- docker-bake.hcl | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 64c3803..39b3b42 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,14 +1,13 @@ variable "GITHUB_REPO" { - default = "https://github.com/sigp/eleel" + default = "sigp/eleel" } group "default" { targets = ["binary-amd64", "binary-arm64"] - labels = { - "org.opencontainers.image.source" = "{GITHUB_REPO}" - } + context = "." - tags = ["magick93/eleel"] + + } @@ -16,7 +15,11 @@ target "binary-amd64" { dockerfile = "Dockerfile.cross" context = "." platforms = ["linux/amd64"] - tags = ["magick93/eleel-amd64"] + tags = ["${GITHUB_REPO}-amd64"] + labels = { + "org.opencontainers.image.source" = "https://github.com/{GITHUB_REPO}" + } + args = { TARGET_ARCH = "x86_64-unknown-linux-gnu" } @@ -26,7 +29,11 @@ target "binary-arm64" { dockerfile = "Dockerfile.cross" context = "." platforms = ["linux/arm64"] - tags = ["magick93/eleel-arm64"] + tags = ["${GITHUB_REPO}-arm64"] + labels = { + "org.opencontainers.image.source" = "https://github.com/${GITHUB_REPO}" + } + args = { TARGET_ARCH = "aarch64-unknown-linux-gnu" } From 4d11d5560d8bf53d361c96e6840bfa50dbfbf521 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 15:34:52 +1300 Subject: [PATCH 60/66] Add Docker image build instructions to README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index da29187..260d477 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,14 @@ Eleel is written in Rust and makes use of components from [Lighthouse][]. A binary will be installed to `~/.cargo/bin/eleel`. +### Build docker images + +### Using `bake` + +See https://docs.docker.com/build/building/multi-platform/. + +`docker buildx bake -f docker-bake.hcl` + ## Getting started Eleel needs to connect to a _real_ execution node, e.g. Geth/Nethermind/Besu. You From 4a85759a2486d070a82ea1eada8d432ef3768953 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 15:39:40 +1300 Subject: [PATCH 61/66] Add description variable to Docker bake configuration for clarity --- docker-bake.hcl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 39b3b42..81e8555 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -2,6 +2,10 @@ variable "GITHUB_REPO" { default = "sigp/eleel" } +variable "DESCRIPTION" { + default = "Eleel is a multiplexer for Ethereum execution clients. It allows multiple consensus clients to connect to a single execution client." +} + group "default" { targets = ["binary-amd64", "binary-arm64"] @@ -15,9 +19,10 @@ target "binary-amd64" { dockerfile = "Dockerfile.cross" context = "." platforms = ["linux/amd64"] + description = "${DESCRIPTION}" tags = ["${GITHUB_REPO}-amd64"] labels = { - "org.opencontainers.image.source" = "https://github.com/{GITHUB_REPO}" + "org.opencontainers.image.source" = "https://github.com/${GITHUB_REPO}" } args = { @@ -27,6 +32,7 @@ target "binary-amd64" { target "binary-arm64" { dockerfile = "Dockerfile.cross" + description = "${DESCRIPTION}" context = "." platforms = ["linux/arm64"] tags = ["${GITHUB_REPO}-arm64"] From 25f334a95af072a51e58048b43d4145e816db993 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 15:41:37 +1300 Subject: [PATCH 62/66] Add .qodo to .gitignore to exclude Qodo files from version control --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ba5a49f..90f7262 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target *.swp *.toml +.qodo From cad835e34aa660ef57de3881673cb24d19571900 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 15:46:41 +1300 Subject: [PATCH 63/66] fix fmt --- .github/workflows/docker.yml | 39 +++++------------------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7606023..c2e3e4a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,12 +27,8 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false - # matrix: - # cpu_arch: [aarch64, x86_64] steps: - - - uses: actions/checkout@v4 - name: Update Rust run: rustup update stable @@ -51,17 +47,11 @@ jobs: make build timeout-minutes: 100 - - - - - - - - name: Install QEMU - run: sudo apt-get update && sudo apt-get install -y qemu-user-static + - name: Install QEMU + run: sudo apt-get update && sudo apt-get install -y qemu-user-static - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Build and push using Bake uses: docker/bake-action@v6 @@ -69,7 +59,7 @@ jobs: files: ./docker-bake.hcl source: . push: true - load: true # <--- Ensure the image is loaded into the local Docker daemon + load: true set: | *.context=. *.tags=${{ env.IMAGE_NAME }} @@ -89,22 +79,3 @@ jobs: ${{ github.repository }}-arm64 \ ${{ github.repository }}-amd64 - # docker-multiarch-manifest: - # name: docker-multiarch-manifest - # runs-on: ubuntu-24.04 - # needs: [build-docker, extract-version] - # env: - # # We need to enable experimental docker features in order to use `docker manifest` - # DOCKER_CLI_EXPERIMENTAL: enabled - # VERSION: ${{ needs.extract-version.outputs.VERSION }} - # VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} - # steps: - # - name: Login to DockerHub - # uses: docker/login-action@v3 - # with: - # username: ${{ env.DOCKER_USERNAME }} - # password: ${{ env.DOCKER_PASSWORD }} - # - name: Create and push multiarch manifest - # run: | - # docker buildx imagetools create -t ${{ env.IMAGE_NAME }} --append --debug - \ No newline at end of file From efccd50376d397fd75830b9c3ae24fafcd6e08e8 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 16:07:44 +1300 Subject: [PATCH 64/66] Remove 'make-and-bake' branch from GitHub Actions workflow triggers --- .github/workflows/docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c2e3e4a..e4fc5b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,7 +6,6 @@ on: - main - stable - unstable - - make-and-bake concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 884413fa344e7a72b4760236292c1316807a9895 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 26 Mar 2025 16:16:36 +1300 Subject: [PATCH 65/66] Add tag triggers to Docker workflow for versioned releases --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e4fc5b7..8a25017 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,6 +6,8 @@ on: - main - stable - unstable + tags: + - v* concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 0a22bda209ca2445be668d374445563801ad8185 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Thu, 15 May 2025 11:27:55 +1200 Subject: [PATCH 66/66] Add Helm chart and tests for Eleel deployment - Create helm-tests.yml for CI workflow to run Helm unit tests - Add Chart.yaml for Eleel Helm chart configuration - Implement rendered.yaml for StatefulSet resource definition - Define _helpers.tpl for template functions and labels - Create statefulset.yaml for StatefulSet template - Add container_test.yaml to validate container configuration - Implement deployment_test.yaml to verify StatefulSet properties - Create secrets_test.yaml to ensure JWT secrets are mounted correctly - Add test-values.yaml for testing configurations - Update values.yaml with default configurations for Eleel --- .github/workflows/helm-tests.yml | 19 +++++++ charts/eleel/Chart.yaml | 7 +++ charts/eleel/rendered.yaml | 66 +++++++++++++++++++++++++ charts/eleel/templates/_helpers.tpl | 63 +++++++++++++++++++++++ charts/eleel/templates/statefulset.yaml | 64 ++++++++++++++++++++++++ charts/eleel/tests/container_test.yaml | 18 +++++++ charts/eleel/tests/deployment_test.yaml | 20 ++++++++ charts/eleel/tests/secrets_test.yaml | 20 ++++++++ charts/eleel/tests/test-values.yaml | 26 ++++++++++ charts/eleel/values.yaml | 48 ++++++++++++++++++ 10 files changed, 351 insertions(+) create mode 100644 .github/workflows/helm-tests.yml create mode 100644 charts/eleel/Chart.yaml create mode 100644 charts/eleel/rendered.yaml create mode 100644 charts/eleel/templates/_helpers.tpl create mode 100644 charts/eleel/templates/statefulset.yaml create mode 100644 charts/eleel/tests/container_test.yaml create mode 100644 charts/eleel/tests/deployment_test.yaml create mode 100644 charts/eleel/tests/secrets_test.yaml create mode 100644 charts/eleel/tests/test-values.yaml create mode 100644 charts/eleel/values.yaml diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml new file mode 100644 index 0000000..2d063cf --- /dev/null +++ b/.github/workflows/helm-tests.yml @@ -0,0 +1,19 @@ +name: Helm Chart Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install helm-unittest plugin + run: helm plugin install https://github.com/helm-unittest/helm-unittest.git + + - name: Run Helm unit tests + run: helm unittest charts/eleel -f "charts/eleel/tests/*_test.yaml" \ No newline at end of file diff --git a/charts/eleel/Chart.yaml b/charts/eleel/Chart.yaml new file mode 100644 index 0000000..35a5597 --- /dev/null +++ b/charts/eleel/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: eleel +description: A Helm chart for Kubernetes to deploy Eleel Ethereum execution engine multiplexer +type: application +version: 0.1.0 +appVersion: "1.0.0" +kubeVersion: ">=1.19.0-0" \ No newline at end of file diff --git a/charts/eleel/rendered.yaml b/charts/eleel/rendered.yaml new file mode 100644 index 0000000..54742c1 --- /dev/null +++ b/charts/eleel/rendered.yaml @@ -0,0 +1,66 @@ +--- +# Source: eleel/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: eleel-test + labels: + app.kubernetes.io/name: eleel + app.kubernetes.io/instance: eleel-test + app.kubernetes.io/managed-by: Helm + helm.sh/chart: eleel-0.1.0 + app.kubernetes.io/version: "1.0.0" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: eleel + app.kubernetes.io/instance: eleel-test + serviceName: eleel-test + template: + metadata: + labels: + app.kubernetes.io/name: eleel + app.kubernetes.io/instance: eleel-test + spec: + containers: + - name: eleel + image: "eleel:test" + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8552 + protocol: TCP + env: + - name: RUST_LOG + value: "eleel=debug" + - name: LISTEN_ADDRESS + value: "0.0.0.0" + - name: LISTEN_PORT + value: "8552" + - name: EE_URL + value: "http://test-engine:8551" + - name: NETWORK + value: "mainnet" + - name: EE_JWT_SECRET + value: "/test-secrets/ee.jwt" + - name: CONTROLLER_JWT_SECRET + value: "/test-secrets/controller.jwt" + - name: CLIENT_JWT_SECRETS + value: "/test-secrets/clients.toml" + volumeMounts: + - name: jwt-secrets + mountPath: /test-secrets + readOnly: true + livenessProbe: + httpGet: + path: /health + port: http + readinessProbe: + httpGet: + path: /health + port: http + volumes: + - name: jwt-secrets + secret: + secretName: "eleel-test-jwt-secrets" diff --git a/charts/eleel/templates/_helpers.tpl b/charts/eleel/templates/_helpers.tpl new file mode 100644 index 0000000..ed0b2fa --- /dev/null +++ b/charts/eleel/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "eleel.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "eleel.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "eleel.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "eleel.labels" -}} +helm.sh/chart: {{ include "eleel.chart" . }} +{{ include "eleel.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "eleel.selectorLabels" -}} +app.kubernetes.io/name: {{ include "eleel.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "eleel.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "eleel.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/eleel/templates/statefulset.yaml b/charts/eleel/templates/statefulset.yaml new file mode 100644 index 0000000..4d46892 --- /dev/null +++ b/charts/eleel/templates/statefulset.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "eleel.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "eleel.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + helm.sh/chart: {{ include "eleel.chart" . }} + app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "eleel.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + serviceName: {{ include "eleel.fullname" . }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "eleel.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + env: + - name: RUST_LOG + value: "eleel=debug" + - name: LISTEN_ADDRESS + value: "{{ .Values.config.listenAddress }}" + - name: LISTEN_PORT + value: "{{ .Values.config.listenPort }}" + - name: EE_URL + value: "{{ .Values.config.eeUrl }}" + - name: NETWORK + value: "{{ .Values.config.network }}" + - name: EE_JWT_SECRET + value: "{{ .Values.secrets.eeJwtSecret.mountPath }}/{{ .Values.secrets.eeJwtSecret.fileName }}" + - name: CONTROLLER_JWT_SECRET + value: "{{ .Values.secrets.controllerJwtSecret.mountPath }}/{{ .Values.secrets.controllerJwtSecret.fileName }}" + - name: CLIENT_JWT_SECRETS + value: "{{ .Values.secrets.clientJwtSecrets.mountPath }}/{{ .Values.secrets.clientJwtSecrets.fileName }}" + volumeMounts: + - name: jwt-secrets + mountPath: {{ .Values.secrets.eeJwtSecret.mountPath }} + readOnly: true + livenessProbe: + httpGet: + path: /health + port: http + readinessProbe: + httpGet: + path: /health + port: http + volumes: + - name: jwt-secrets + secret: + secretName: "{{ .Release.Name }}-jwt-secrets" \ No newline at end of file diff --git a/charts/eleel/tests/container_test.yaml b/charts/eleel/tests/container_test.yaml new file mode 100644 index 0000000..6022865 --- /dev/null +++ b/charts/eleel/tests/container_test.yaml @@ -0,0 +1,18 @@ +suite: test eleel container +templates: + - templates/statefulset.yaml +tests: + - it: should configure container correctly + asserts: + - equal: + path: spec.template.spec.containers[0].name + value: eleel + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 8552 + - equal: + path: spec.template.spec.containers[0].env[0].name + value: RUST_LOG + - equal: + path: spec.template.spec.containers[0].env[0].value + value: eleel=debug \ No newline at end of file diff --git a/charts/eleel/tests/deployment_test.yaml b/charts/eleel/tests/deployment_test.yaml new file mode 100644 index 0000000..e87152e --- /dev/null +++ b/charts/eleel/tests/deployment_test.yaml @@ -0,0 +1,20 @@ +suite: test statefulset +templates: + - statefulset.yaml +tests: + - it: should create statefulset with correct properties + asserts: + - isKind: + of: StatefulSet + - matchRegex: + path: metadata.name + pattern: .+-eleel$ + - matchRegex: + path: spec.serviceName + pattern: .+-eleel$ + - equal: + path: spec.replicas + value: 1 + - equal: + path: spec.template.spec.containers[0].name + value: eleel \ No newline at end of file diff --git a/charts/eleel/tests/secrets_test.yaml b/charts/eleel/tests/secrets_test.yaml new file mode 100644 index 0000000..9fe5962 --- /dev/null +++ b/charts/eleel/tests/secrets_test.yaml @@ -0,0 +1,20 @@ +suite: test secrets +templates: + - statefulset.yaml +values: + - test-values.yaml +tests: + - it: should mount jwt secrets + asserts: + - equal: + path: spec.template.spec.volumes[0].name + value: jwt-secrets + - matchRegex: + path: spec.template.spec.volumes[0].secret.secretName + pattern: .+-jwt-secrets$ + - equal: + path: spec.template.spec.containers[0].volumeMounts[0].mountPath + value: /test-secrets + - equal: + path: spec.template.spec.containers[0].volumeMounts[0].readOnly + value: true \ No newline at end of file diff --git a/charts/eleel/tests/test-values.yaml b/charts/eleel/tests/test-values.yaml new file mode 100644 index 0000000..d11ee75 --- /dev/null +++ b/charts/eleel/tests/test-values.yaml @@ -0,0 +1,26 @@ +replicaCount: 1 + +image: + repository: eleel + tag: test + pullPolicy: IfNotPresent + +service: + port: 8552 + +config: + listenAddress: "0.0.0.0" + listenPort: "8552" + eeUrl: "http://test-engine:8551" + network: "mainnet" + +secrets: + eeJwtSecret: + mountPath: "/test-secrets" + fileName: "ee.jwt" + controllerJwtSecret: + mountPath: "/test-secrets" + fileName: "controller.jwt" + clientJwtSecrets: + mountPath: "/test-secrets" + fileName: "clients.toml" \ No newline at end of file diff --git a/charts/eleel/values.yaml b/charts/eleel/values.yaml new file mode 100644 index 0000000..2f6bdc5 --- /dev/null +++ b/charts/eleel/values.yaml @@ -0,0 +1,48 @@ +# Default values for eleel. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: eleel + tag: latest + pullPolicy: IfNotPresent + +service: + port: 8552 + +config: + listenAddress: "127.0.0.1" + listenPort: "8552" + eeUrl: "http://localhost:8551" + network: "mainnet" + +secrets: + eeJwtSecret: + mountPath: "/secrets" + fileName: "ee.jwt" + controllerJwtSecret: + mountPath: "/secrets" + fileName: "controller.jwt" + clientJwtSecrets: + mountPath: "/secrets" + fileName: "clients.toml" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} \ No newline at end of file