From 840f7f612e4dffa9a6c6b29cd5a9b99e76af2999 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:41:24 -0400 Subject: [PATCH 01/12] fix(agents): a Buildx acknowledgement is owed only by an in-agent image build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The caller builds a recipe on the builder it selected, and an answer with no result kind built nothing — a vendor-image agent whose image is published elsewhere — so neither has an executor to acknowledge. Reading "no recipe" as "built in-agent" refused every render composing a vendor-image agent the moment the environment selected a builder (a registry mirror does): redis, vault, object-storage and clickhouse all answered BuildCapabilities and were still refused with "did not acknowledge requested Buildx builder". The check now reads the result kind it is about. Co-Authored-By: Claude Fable 5.1 --- agents/services/builder.go | 8 +++++++- agents/services/builder_cache_test.go | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/agents/services/builder.go b/agents/services/builder.go index 8c0f44f52..611351402 100644 --- a/agents/services/builder.go +++ b/agents/services/builder.go @@ -58,7 +58,13 @@ func (b *BuilderAgent) Build(ctx context.Context, req *builderv0.BuildRequest, o if err == nil && cache != nil && resp.GetState().GetState() == builderv0.BuildStatus_SUCCESS && resp.GetResult().GetDockerBuildPlan() == nil && resp.GetCacheContractVersion() != dockerhelpers.CacheContractVersion { return nil, fmt.Errorf("builder agent did not acknowledge build cache contract %s; upgrade the agent or remove cache options", dockerhelpers.CacheContractVersion) } - if selected != "" && err == nil && resp.GetState().GetState() == builderv0.BuildStatus_SUCCESS && resp.GetResult().GetDockerBuildPlan() == nil && resp.GetBuildxBuilder() != selected { + // Only an in-agent image build has an executor to acknowledge. A recipe + // (docker_build_plan) is built by the caller, on the builder it selected; + // an answer with no result kind built nothing — a vendor-image agent whose + // image is published elsewhere — and has nothing to acknowledge either. + // Reading "no recipe" as "built in-agent" refused every render composing + // such an agent the moment the environment selected a builder. + if selected != "" && err == nil && resp.GetState().GetState() == builderv0.BuildStatus_SUCCESS && resp.GetResult().GetDockerBuildResult() != nil && resp.GetBuildxBuilder() != selected { return nil, fmt.Errorf("builder agent did not acknowledge requested Buildx builder %q; upgrade the agent", selected) } if err == nil && req.GetExecution() != nil { diff --git a/agents/services/builder_cache_test.go b/agents/services/builder_cache_test.go index f5c5f3d23..b8d37c794 100644 --- a/agents/services/builder_cache_test.go +++ b/agents/services/builder_cache_test.go @@ -2,7 +2,6 @@ package services import ( "context" - "fmt" "net" "sync/atomic" "testing" @@ -87,12 +86,21 @@ func (s *buildxTestServer) Build(ctx context.Context, req *builderv0.BuildReques return resp, err } +// Only an in-agent build has an executor to acknowledge: a recipe is built by +// the caller on the builder it selected, and an answer that built nothing (a +// vendor-image agent) has nothing to acknowledge. Treating "no recipe" as +// "built in-agent" refused every render composing a vendor-image agent. func TestBuildxSelectionRequiresExecutorAcknowledgementOverGRPC(t *testing.T) { - for _, recipe := range []bool{false, true} { - t.Run(fmt.Sprintf("recipe=%t", recipe), func(t *testing.T) { + for name, result := range map[string]*builderv0.BuildResult{ + "in-agent build": {Kind: &builderv0.BuildResult_DockerBuildResult{DockerBuildResult: &builderv0.DockerBuildResult{}}}, + "recipe": {Kind: &builderv0.BuildResult_DockerBuildPlan{DockerBuildPlan: &builderv0.DockerBuildPlan{}}}, + "nothing built": nil, + } { + t.Run(name, func(t *testing.T) { + recipe := name != "in-agent build" wrapper := &BuilderWrapper{Base: &Base{loaded: true}} - if recipe { - wrapper.BuildResult = &builderv0.BuildResult{Kind: &builderv0.BuildResult_DockerBuildPlan{DockerBuildPlan: &builderv0.DockerBuildPlan{}}} + if result != nil { + wrapper.BuildResult = result } server := grpc.NewServer() builderv0.RegisterBuilderServer(server, &buildxTestServer{ From 61112df8c88abfbb848051a5df6b843f9f4338e4 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:42:31 -0400 Subject: [PATCH 02/12] test(agents): the acknowledgement cases answer an in-agent build Co-Authored-By: Claude Fable 5.1 --- agents/services/builder_cache_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/services/builder_cache_test.go b/agents/services/builder_cache_test.go index b8d37c794..149bc799b 100644 --- a/agents/services/builder_cache_test.go +++ b/agents/services/builder_cache_test.go @@ -152,7 +152,7 @@ func TestBuildxCapabilitiesCheckedBeforeBuildOverGRPC(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { implementation := &buildxTestServer{ - DefaultBuilder: NewDefaultBuilder(&BuilderWrapper{Base: &Base{loaded: true}}), + DefaultBuilder: NewDefaultBuilder(&BuilderWrapper{Base: &Base{loaded: true}, BuildResult: &builderv0.BuildResult{Kind: &builderv0.BuildResult_DockerBuildResult{DockerBuildResult: &builderv0.DockerBuildResult{}}}}), // an in-agent build: the one answer that owes an acknowledgement capabilities: &builderv0.BuildCapabilitiesResponse{BuildxSelection: tc.supported}, capabilityError: tc.probeError, acknowledgement: tc.ack, From 53c25492aa9bd75305018031756ffb160f3ea782 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:18:52 -0400 Subject: [PATCH 03/12] =?UTF-8?q?chore(scout):=20go=201.27.3=20=E2=80=94?= =?UTF-8?q?=20GO-2026-6809=20(crypto/x509)=20is=20fixed=20there,=20and=20t?= =?UTF-8?q?he=20Build=20job=20installs=20the=20toolchain=20go.mod=20names?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index cb90dfc38..dd72734c9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/codefly-dev/core -go 1.27.0 +go 1.27.3 require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.12-20260825204119-511051f7f437.2 From 41a1cddf9dfe6d9d8d762c97786b65b90d7158a1 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:33:58 -0400 Subject: [PATCH 04/12] chore(scout): the workflows that pin a Go toolchain pin 1.27.3, the version go.mod now requires Co-Authored-By: Claude Fable 5.1 --- .github/workflows/build-cache.yml | 4 ++-- .github/workflows/go.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index e162ad540..60fa951a3 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: "1.27.2" + go-version: "1.27.3" - name: Verify Buildx selection and cache arguments reach the docker CLI if: matrix.language == 'go' run: go test ./agents/helpers/docker -run '^TestBuildx|^TestBuildOmits|^TestBuildCarries' -count=1 -v -timeout=5m @@ -62,7 +62,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: "1.27.2" + go-version: "1.27.3" - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: registry-cache-${{ matrix.language }} diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f59d7bc67..fa71fac6e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -112,7 +112,7 @@ jobs: - name: set up uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: "1.27.2" + go-version: "1.27.3" - name: checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From 7588bf912579bdd877dae23540de6d7af7fdc4ea Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:02:57 -0400 Subject: [PATCH 05/12] diag: TEMPORARY sandbox cleanup diagnostics on the 26.04 runner (reverted before merge) --- .github/workflows/diag-sandbox.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/diag-sandbox.yml diff --git a/.github/workflows/diag-sandbox.yml b/.github/workflows/diag-sandbox.yml new file mode 100644 index 000000000..27956f915 --- /dev/null +++ b/.github/workflows/diag-sandbox.yml @@ -0,0 +1,28 @@ +# TEMPORARY diagnostic — removed before merge. TestLoadArtifactSandbox hangs in +# executor cleanup on the ubuntu-26.04 runner image and passes everywhere else. +name: diag-sandbox +on: + pull_request: +jobs: + diag: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: false + - name: environment + run: | + uname -a; cat /etc/os-release | head -2 + echo "pid1: $(cat /proc/1/comm)"; echo "apparmor userns restrict: $(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null)" + echo "ancestry:"; p=$$; while [ "$p" != 1 ] && [ -n "$p" ]; do echo " $(ps -o pid=,ppid=,comm= -p $p)"; p=$(ps -o ppid= -p $p | tr -d ' '); done + sudo apt-get install -y -qq bubblewrap >/dev/null; bwrap --version; mount | grep -E " /tmp | /proc " || true + - name: test with process snapshots + run: | + ( while true; do sleep 5; echo "--- ps $(date +%T)"; ps -eo pid,ppid,pgid,sid,stat,comm | grep -E "executor|bwrap|acquired|manager.test|codefly" | grep -v grep; done ) & + SNAP=$! + go test -tags sandbox_e2e -run 'TestLoadArtifactSandbox' -count=1 -v ./agents/manager 2>&1 | tail -80 + kill $SNAP || true + echo "--- leftovers"; ls -la /tmp/executor-* 2>/dev/null || true From 25df655c839591efbf7f3e7fd864e74da0cc7594 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:24:04 -0400 Subject: [PATCH 06/12] =?UTF-8?q?diag:=20TEMPORARY=20=E2=80=94=20snapshot?= =?UTF-8?q?=20loop=20survives=20empty=20greps;=20per-process=20state,=20na?= =?UTF-8?q?mespaces,=20creds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/diag-sandbox.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/diag-sandbox.yml b/.github/workflows/diag-sandbox.yml index 27956f915..cbf3e8bcd 100644 --- a/.github/workflows/diag-sandbox.yml +++ b/.github/workflows/diag-sandbox.yml @@ -20,9 +20,21 @@ jobs: echo "ancestry:"; p=$$; while [ "$p" != 1 ] && [ -n "$p" ]; do echo " $(ps -o pid=,ppid=,comm= -p $p)"; p=$(ps -o ppid= -p $p | tr -d ' '); done sudo apt-get install -y -qq bubblewrap >/dev/null; bwrap --version; mount | grep -E " /tmp | /proc " || true - name: test with process snapshots + shell: bash {0} run: | - ( while true; do sleep 5; echo "--- ps $(date +%T)"; ps -eo pid,ppid,pgid,sid,stat,comm | grep -E "executor|bwrap|acquired|manager.test|codefly" | grep -v grep; done ) & + snapshot() { + echo "--- ps $(date +%T)" + ps -eo pid,ppid,pgid,sid,stat,uid,comm | grep -E "executor|bwrap|acquired|manager.test" | grep -v grep || true + for pid in $(pgrep -x -f '.*acquired.*|bwrap|executor' 2>/dev/null; pgrep executor; pgrep acquired; pgrep bwrap); do + [ -r /proc/$pid/status ] || continue + echo " /proc/$pid: $(grep -E '^(Name|State|PPid|NSpid|Uid|CapEff|Seccomp):' /proc/$pid/status | tr '\n' ' ' | tr -s '\t ' ' ')" + pg=$(ps -o pgid= -p $pid | tr -d ' '); kill -0 -- -$pg 2>&1 && echo " kill -0 -$pg: ok" || echo " kill -0 -$pg: refused ($?)" + done + } + ( while true; do sleep 4; snapshot; done ) & SNAP=$! - go test -tags sandbox_e2e -run 'TestLoadArtifactSandbox' -count=1 -v ./agents/manager 2>&1 | tail -80 - kill $SNAP || true + go test -tags sandbox_e2e -run 'TestLoadArtifactSandbox' -count=1 -v ./agents/manager 2>&1 | grep -vE "^go: downloading" | tail -60 + kill $SNAP 2>/dev/null || true + echo "--- after"; snapshot echo "--- leftovers"; ls -la /tmp/executor-* 2>/dev/null || true + echo "--- dmesg tail"; sudo dmesg 2>/dev/null | tail -15 || true From 4029b6189a8af507d24323482a4b6518e30b7acf Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:46:34 -0400 Subject: [PATCH 07/12] =?UTF-8?q?diag:=20TEMPORARY=20=E2=80=94=20read=20wh?= =?UTF-8?q?at=20the=20inspector=20reads=20of=20the=20surviving=20executor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/diag-sandbox.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/diag-sandbox.yml b/.github/workflows/diag-sandbox.yml index cbf3e8bcd..34aee5a43 100644 --- a/.github/workflows/diag-sandbox.yml +++ b/.github/workflows/diag-sandbox.yml @@ -24,14 +24,20 @@ jobs: run: | snapshot() { echo "--- ps $(date +%T)" - ps -eo pid,ppid,pgid,sid,stat,uid,comm | grep -E "executor|bwrap|acquired|manager.test" | grep -v grep || true - for pid in $(pgrep -x -f '.*acquired.*|bwrap|executor' 2>/dev/null; pgrep executor; pgrep acquired; pgrep bwrap); do + ps -eo pid,ppid,pgid,sid,stat,uid,comm | grep -E "executor|bwrap|acquired|manager.test|\.test$" | grep -v grep || true + for pid in $(pgrep executor; pgrep acquired; pgrep bwrap); do [ -r /proc/$pid/status ] || continue - echo " /proc/$pid: $(grep -E '^(Name|State|PPid|NSpid|Uid|CapEff|Seccomp):' /proc/$pid/status | tr '\n' ' ' | tr -s '\t ' ' ')" + echo " /proc/$pid: $(grep -E '^(Name|State|PPid|NSpid|Uid|CapEff|NoNewPrivs|Seccomp):' /proc/$pid/status | tr '\n' ' ' | tr -s '\t ' ' ')" + echo " exe: $(readlink /proc/$pid/exe 2>&1)"; echo " stat: $(cut -c1-120 /proc/$pid/stat 2>&1)" + echo " environ: $(tr '\0' '\n' < /proc/$pid/environ 2>&1 | grep -cE '.' ) lines; $(tr '\0' '\n' < /proc/$pid/environ 2>&1 | grep -E 'CODEFLY' | cut -c1-40 | tr '\n' ' ')" + echo " ns: $(ls -l /proc/$pid/ns/ 2>&1 | awk '{print $9, $11}' | tr '\n' ' ')"; echo " self ns: $(ls -l /proc/self/ns/ | awk '{print $9, $11}' | tr '\n' ' ')" + echo " apparmor: $(cat /proc/$pid/attr/current 2>&1)"; echo " cmdline: $(tr '\0' ' ' < /proc/$pid/cmdline 2>&1 | cut -c1-200)" pg=$(ps -o pgid= -p $pid | tr -d ' '); kill -0 -- -$pg 2>&1 && echo " kill -0 -$pg: ok" || echo " kill -0 -$pg: refused ($?)" done } - ( while true; do sleep 4; snapshot; done ) & + echo "yama: $(cat /proc/sys/kernel/yama/ptrace_scope)"; echo "self apparmor: $(cat /proc/self/attr/current)"; mount | grep -E " /proc " || true + cat /etc/apparmor.d/bwrap* 2>/dev/null | grep -vE "^\s*#|^\s*$" | head -30 || true + ( for i in $(seq 1 12); do sleep 1; snapshot; done; while true; do sleep 5; snapshot; done ) & SNAP=$! go test -tags sandbox_e2e -run 'TestLoadArtifactSandbox' -count=1 -v ./agents/manager 2>&1 | grep -vE "^go: downloading" | tail -60 kill $SNAP 2>/dev/null || true From bf4508e2d41acae9168c6338583cd7a413d95df0 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:09:48 -0400 Subject: [PATCH 08/12] =?UTF-8?q?diag:=20TEMPORARY=20=E2=80=94=20forest=20?= =?UTF-8?q?snapshots=20on=20change,=20pgid=20records,=20manual=20escalatio?= =?UTF-8?q?n,=20apparmor=20audit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/diag-sandbox.yml | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/diag-sandbox.yml b/.github/workflows/diag-sandbox.yml index 34aee5a43..ac8a784cb 100644 --- a/.github/workflows/diag-sandbox.yml +++ b/.github/workflows/diag-sandbox.yml @@ -22,25 +22,17 @@ jobs: - name: test with process snapshots shell: bash {0} run: | + echo "bwrap: $(ls -l /usr/bin/bwrap) :: $(file -b /usr/bin/bwrap | cut -c1-60)" + echo "yama: $(cat /proc/sys/kernel/yama/ptrace_scope)" snapshot() { - echo "--- ps $(date +%T)" - ps -eo pid,ppid,pgid,sid,stat,uid,comm | grep -E "executor|bwrap|acquired|manager.test|\.test$" | grep -v grep || true - for pid in $(pgrep executor; pgrep acquired; pgrep bwrap); do - [ -r /proc/$pid/status ] || continue - echo " /proc/$pid: $(grep -E '^(Name|State|PPid|NSpid|Uid|CapEff|NoNewPrivs|Seccomp):' /proc/$pid/status | tr '\n' ' ' | tr -s '\t ' ' ')" - echo " exe: $(readlink /proc/$pid/exe 2>&1)"; echo " stat: $(cut -c1-120 /proc/$pid/stat 2>&1)" - echo " environ: $(tr '\0' '\n' < /proc/$pid/environ 2>&1 | grep -cE '.' ) lines; $(tr '\0' '\n' < /proc/$pid/environ 2>&1 | grep -E 'CODEFLY' | cut -c1-40 | tr '\n' ' ')" - echo " ns: $(ls -l /proc/$pid/ns/ 2>&1 | awk '{print $9, $11}' | tr '\n' ' ')"; echo " self ns: $(ls -l /proc/self/ns/ | awk '{print $9, $11}' | tr '\n' ' ')" - echo " apparmor: $(cat /proc/$pid/attr/current 2>&1)"; echo " cmdline: $(tr '\0' ' ' < /proc/$pid/cmdline 2>&1 | cut -c1-200)" - pg=$(ps -o pgid= -p $pid | tr -d ' '); kill -0 -- -$pg 2>&1 && echo " kill -0 -$pg: ok" || echo " kill -0 -$pg: refused ($?)" - done + T=$(ps -eo pid,ppid,pgid,sid,stat,comm,args --forest 2>/dev/null | grep -vE "ps -eo|grep|snapshot|sleep" | grep -E "manager.test|executor|bwrap|acquired|codefly|CODEFLY| sh |sh -c" || true) + if [ "$T" != "$LAST" ]; then echo "--- ps $(date +%T.%N | cut -c1-12)"; echo "$T" | cut -c1-200; LAST="$T"; fi } - echo "yama: $(cat /proc/sys/kernel/yama/ptrace_scope)"; echo "self apparmor: $(cat /proc/self/attr/current)"; mount | grep -E " /proc " || true - cat /etc/apparmor.d/bwrap* 2>/dev/null | grep -vE "^\s*#|^\s*$" | head -30 || true - ( for i in $(seq 1 12); do sleep 1; snapshot; done; while true; do sleep 5; snapshot; done ) & + ( LAST=""; while true; do snapshot; sleep 0.3; done ) & SNAP=$! - go test -tags sandbox_e2e -run 'TestLoadArtifactSandbox' -count=1 -v ./agents/manager 2>&1 | grep -vE "^go: downloading" | tail -60 + go test -tags sandbox_e2e -run 'TestLoadArtifactSandbox' -count=1 -v ./agents/manager 2>&1 | grep -vE "^go: downloading" | tail -40 kill $SNAP 2>/dev/null || true - echo "--- after"; snapshot - echo "--- leftovers"; ls -la /tmp/executor-* 2>/dev/null || true - echo "--- dmesg tail"; sudo dmesg 2>/dev/null | tail -15 || true + echo "--- pgid records"; find /tmp -name '*.pgid' 2>/dev/null | while read f; do echo "$f"; cat "$f" | cut -c1-400; echo; done + echo "--- survivors"; ps -eo pid,ppid,pgid,sid,stat,comm,args | grep -E "executor|bwrap" | grep -v grep || true + for pid in $(pgrep executor); do echo " SIGTERM $pid"; kill -TERM $pid; sleep 2; ps -o pid,stat,comm -p $pid || true; echo " SIGKILL $pid"; kill -KILL $pid; sleep 1; ps -o pid,stat,comm -p $pid || echo " gone"; done + echo "--- apparmor audit"; sudo dmesg 2>/dev/null | grep -iE "apparmor|DENIED" | grep -vE "systemd-detect|rsyslog|snap|docker|STATUS" | tail -20 || true From 70639217930896f941a12f2fb53c0a49675defb7 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:31:56 -0400 Subject: [PATCH 09/12] =?UTF-8?q?diag:=20TEMPORARY=20=E2=80=94=20bwrap=20p?= =?UTF-8?q?arent=20lifetime=20under=20strace,=20four=20argument=20sets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/diag-sandbox.yml | 32 +++++++++++++++++------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/diag-sandbox.yml b/.github/workflows/diag-sandbox.yml index ac8a784cb..1fdd468d5 100644 --- a/.github/workflows/diag-sandbox.yml +++ b/.github/workflows/diag-sandbox.yml @@ -19,20 +19,24 @@ jobs: echo "pid1: $(cat /proc/1/comm)"; echo "apparmor userns restrict: $(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null)" echo "ancestry:"; p=$$; while [ "$p" != 1 ] && [ -n "$p" ]; do echo " $(ps -o pid=,ppid=,comm= -p $p)"; p=$(ps -o ppid= -p $p | tr -d ' '); done sudo apt-get install -y -qq bubblewrap >/dev/null; bwrap --version; mount | grep -E " /tmp | /proc " || true - - name: test with process snapshots + - name: bwrap parent lifetime under strace shell: bash {0} run: | - echo "bwrap: $(ls -l /usr/bin/bwrap) :: $(file -b /usr/bin/bwrap | cut -c1-60)" - echo "yama: $(cat /proc/sys/kernel/yama/ptrace_scope)" - snapshot() { - T=$(ps -eo pid,ppid,pgid,sid,stat,comm,args --forest 2>/dev/null | grep -vE "ps -eo|grep|snapshot|sleep" | grep -E "manager.test|executor|bwrap|acquired|codefly|CODEFLY| sh |sh -c" || true) - if [ "$T" != "$LAST" ]; then echo "--- ps $(date +%T.%N | cut -c1-12)"; echo "$T" | cut -c1-200; LAST="$T"; fi + sudo apt-get install -y -qq strace >/dev/null 2>&1 || true + W=$(mktemp -d /tmp/bw-XXXX); mkdir -p $W/stage + run_case() { + NAME=$1; shift + echo "=== case $NAME: bwrap $*" + ( strace -f -tt -o $W/$NAME.trace -e trace=process,signal bwrap "$@" --ro-bind $W/stage $W/stage -- /bin/sh -c 'sleep 20' ) > $W/$NAME.out 2>&1 & + P=$!; sleep 1.5 + echo "--- forest after 1.5s (strace pid $P)"; ps -eo pid,ppid,pgid,sid,stat,comm --forest | grep -E "strace|bwrap|sleep| sh" | grep -vE "grep|ps -eo" | cut -c1-120 || true + BW=$(pgrep -x bwrap | head -1); echo "bwrap pid: ${BW:-none}" + echo "--- parent trace (first 40 process/signal events)"; grep -vE "SIGCHLD|rt_sigaction|rt_sigprocmask" $W/$NAME.trace | head -40 | cut -c1-200 + echo "--- stderr"; cat $W/$NAME.out | head -5 + echo "--- kill -TERM the sleep's ancestors group and check survivors"; for pid in $(pgrep -x bwrap); do kill -TERM $pid; done; sleep 1; ps -eo pid,ppid,pgid,sid,stat,comm | grep -E "bwrap|sleep" | grep -v grep || echo " none left" + pkill -KILL -x sleep 2>/dev/null || true; pkill -KILL -x bwrap 2>/dev/null || true; sleep 0.5 } - ( LAST=""; while true; do snapshot; sleep 0.3; done ) & - SNAP=$! - go test -tags sandbox_e2e -run 'TestLoadArtifactSandbox' -count=1 -v ./agents/manager 2>&1 | grep -vE "^go: downloading" | tail -40 - kill $SNAP 2>/dev/null || true - echo "--- pgid records"; find /tmp -name '*.pgid' 2>/dev/null | while read f; do echo "$f"; cat "$f" | cut -c1-400; echo; done - echo "--- survivors"; ps -eo pid,ppid,pgid,sid,stat,comm,args | grep -E "executor|bwrap" | grep -v grep || true - for pid in $(pgrep executor); do echo " SIGTERM $pid"; kill -TERM $pid; sleep 2; ps -o pid,stat,comm -p $pid || true; echo " SIGKILL $pid"; kill -KILL $pid; sleep 1; ps -o pid,stat,comm -p $pid || echo " gone"; done - echo "--- apparmor audit"; sudo dmesg 2>/dev/null | grep -iE "apparmor|DENIED" | grep -vE "systemd-detect|rsyslog|snap|docker|STATUS" | tail -20 || true + run_case default --die-with-parent --new-session --proc /proc --dev /dev --tmpfs /tmp --ro-bind-try /usr /usr --ro-bind-try /lib /lib --ro-bind-try /lib64 /lib64 --ro-bind-try /bin /bin --ro-bind-try /sbin /sbin --ro-bind-try /etc /etc --unshare-net + run_case nosession --die-with-parent --proc /proc --dev /dev --tmpfs /tmp --ro-bind-try /usr /usr --ro-bind-try /lib /lib --ro-bind-try /lib64 /lib64 --ro-bind-try /bin /bin --ro-bind-try /sbin /sbin --ro-bind-try /etc /etc --unshare-net + run_case unsharepid --die-with-parent --new-session --unshare-pid --proc /proc --dev /dev --tmpfs /tmp --ro-bind-try /usr /usr --ro-bind-try /lib /lib --ro-bind-try /lib64 /lib64 --ro-bind-try /bin /bin --ro-bind-try /sbin /sbin --ro-bind-try /etc /etc --unshare-net + run_case bare --proc /proc --dev /dev --ro-bind-try /usr /usr --ro-bind-try /lib /lib --ro-bind-try /lib64 /lib64 --ro-bind-try /bin /bin --ro-bind-try /etc /etc From 3b612f8275379a09b6dd6ee281d75782464faf45 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:51:49 -0400 Subject: [PATCH 10/12] fix(sandbox): the payload's lifetime is bound to a pid namespace, not to a parent-death signal bwrap --die-with-parent asks the kernel to kill the payload when the supervising bwrap dies. On Ubuntu 26.04 (kernel 7.0, bubblewrap 0.11.1 confined as bwrap//&unpriv_bwrap) that signal is not delivered: a payload outlived its terminated supervisor in its own session (--new-session), still holding the supervisor's stdio pipes, so the loader's reaper never returned and every executor cleanup ran into its 30s deadline (TestLoadArtifactSandbox, red on every branch since ubuntu-latest moved). --unshare-pid runs the payload under a pid-namespace init bubblewrap supervises; ending the supervisor ends the namespace and everything in it, on every platform. Co-Authored-By: Claude Fable 5.1 --- runners/sandbox/bwrap.go | 15 +++++++++++++-- runners/sandbox/sandbox_test.go | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/runners/sandbox/bwrap.go b/runners/sandbox/bwrap.go index 29e854899..8a9eb6508 100644 --- a/runners/sandbox/bwrap.go +++ b/runners/sandbox/bwrap.go @@ -12,6 +12,7 @@ import ( // bwrap \ // --die-with-parent \ // --new-session \ +// --unshare-pid \ // --proc /proc \ // --dev /dev \ // --tmpfs /tmp \ @@ -24,9 +25,18 @@ import ( // [--unshare-net] // -- // -// `--die-with-parent` ensures the bwrap child gets SIGTERM if the Go +// `--die-with-parent` asks for the bwrap child to be killed if the Go // parent dies abnormally. `--new-session` blocks TIOCSTI escapes from -// the sandboxed process onto the controlling tty. The system-image +// the sandboxed process onto the controlling tty. `--unshare-pid` is +// what actually bounds the payload's lifetime: with it the payload runs +// under a pid-namespace init bubblewrap supervises, so ending the +// supervisor ends the namespace and every process in it. Without it the +// bound rests on the parent-death signal alone, and that signal is not +// delivered on every platform: on Ubuntu 26.04 (kernel 7.0, bubblewrap +// 0.11.1 confined as bwrap//&unpriv_bwrap) a payload outlived its +// terminated supervisor, in its own session because of --new-session, +// holding the supervisor's stdio pipes, so the caller's reaper never +// returned and cleanup ran into its deadline. The system-image // binds (/usr, /lib, etc.) are required because most binaries are // dynamically linked to libraries rooted there; without them, even // `cat` fails to start in the sandbox. @@ -108,6 +118,7 @@ func (s *bwrapSandbox) buildArgs() []string { args := []string{ "--die-with-parent", "--new-session", + "--unshare-pid", "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", diff --git a/runners/sandbox/sandbox_test.go b/runners/sandbox/sandbox_test.go index c43929725..8e8484ff5 100644 --- a/runners/sandbox/sandbox_test.go +++ b/runners/sandbox/sandbox_test.go @@ -202,6 +202,21 @@ func TestBwrap_NetworkLoopback_UsesIsolatedNetwork(t *testing.T) { require.Equal(t, []string{"/tmp/codefly-agent", "--fixture"}, cmd.Args[len(cmd.Args)-2:]) } +func TestBwrapPayloadLifetimeIsBoundToAPidNamespace(t *testing.T) { + if runtime.GOOS != "linux" { + // bubblewrap is Linux-only; returning early per the no-skip rule. + return + } + sb, err := sandbox.New() + require.NoError(t, err) + cmd := exec.Command("/tmp/codefly-agent", "--fixture") + require.NoError(t, sb.Wrap(cmd)) + args := strings.Join(cmd.Args, " ") + require.Contains(t, args, "--unshare-pid", + "the payload must run under a pid-namespace init bubblewrap supervises: --die-with-parent alone is not honoured on every platform (Ubuntu 26.04 let a payload outlive its terminated supervisor)") + require.Contains(t, args, "--die-with-parent") +} + func TestSandbox_Wrap_RefusesDoubleWrap(t *testing.T) { sb, err := sandbox.New() if err != nil { From 06f1c4859a3302a643cde8e9c8f915140c6570b5 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:51:49 -0400 Subject: [PATCH 11/12] =?UTF-8?q?diag:=20TEMPORARY=20=E2=80=94=20run=20the?= =?UTF-8?q?=20sandbox=20test=20with=20the=20fix=20on=20the=2026.04=20runne?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/diag-sandbox.yml | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/.github/workflows/diag-sandbox.yml b/.github/workflows/diag-sandbox.yml index 1fdd468d5..272ed49a5 100644 --- a/.github/workflows/diag-sandbox.yml +++ b/.github/workflows/diag-sandbox.yml @@ -19,24 +19,8 @@ jobs: echo "pid1: $(cat /proc/1/comm)"; echo "apparmor userns restrict: $(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null)" echo "ancestry:"; p=$$; while [ "$p" != 1 ] && [ -n "$p" ]; do echo " $(ps -o pid=,ppid=,comm= -p $p)"; p=$(ps -o ppid= -p $p | tr -d ' '); done sudo apt-get install -y -qq bubblewrap >/dev/null; bwrap --version; mount | grep -E " /tmp | /proc " || true - - name: bwrap parent lifetime under strace - shell: bash {0} + - name: the sandbox test on this runner run: | - sudo apt-get install -y -qq strace >/dev/null 2>&1 || true - W=$(mktemp -d /tmp/bw-XXXX); mkdir -p $W/stage - run_case() { - NAME=$1; shift - echo "=== case $NAME: bwrap $*" - ( strace -f -tt -o $W/$NAME.trace -e trace=process,signal bwrap "$@" --ro-bind $W/stage $W/stage -- /bin/sh -c 'sleep 20' ) > $W/$NAME.out 2>&1 & - P=$!; sleep 1.5 - echo "--- forest after 1.5s (strace pid $P)"; ps -eo pid,ppid,pgid,sid,stat,comm --forest | grep -E "strace|bwrap|sleep| sh" | grep -vE "grep|ps -eo" | cut -c1-120 || true - BW=$(pgrep -x bwrap | head -1); echo "bwrap pid: ${BW:-none}" - echo "--- parent trace (first 40 process/signal events)"; grep -vE "SIGCHLD|rt_sigaction|rt_sigprocmask" $W/$NAME.trace | head -40 | cut -c1-200 - echo "--- stderr"; cat $W/$NAME.out | head -5 - echo "--- kill -TERM the sleep's ancestors group and check survivors"; for pid in $(pgrep -x bwrap); do kill -TERM $pid; done; sleep 1; ps -eo pid,ppid,pgid,sid,stat,comm | grep -E "bwrap|sleep" | grep -v grep || echo " none left" - pkill -KILL -x sleep 2>/dev/null || true; pkill -KILL -x bwrap 2>/dev/null || true; sleep 0.5 - } - run_case default --die-with-parent --new-session --proc /proc --dev /dev --tmpfs /tmp --ro-bind-try /usr /usr --ro-bind-try /lib /lib --ro-bind-try /lib64 /lib64 --ro-bind-try /bin /bin --ro-bind-try /sbin /sbin --ro-bind-try /etc /etc --unshare-net - run_case nosession --die-with-parent --proc /proc --dev /dev --tmpfs /tmp --ro-bind-try /usr /usr --ro-bind-try /lib /lib --ro-bind-try /lib64 /lib64 --ro-bind-try /bin /bin --ro-bind-try /sbin /sbin --ro-bind-try /etc /etc --unshare-net - run_case unsharepid --die-with-parent --new-session --unshare-pid --proc /proc --dev /dev --tmpfs /tmp --ro-bind-try /usr /usr --ro-bind-try /lib /lib --ro-bind-try /lib64 /lib64 --ro-bind-try /bin /bin --ro-bind-try /sbin /sbin --ro-bind-try /etc /etc --unshare-net - run_case bare --proc /proc --dev /dev --ro-bind-try /usr /usr --ro-bind-try /lib /lib --ro-bind-try /lib64 /lib64 --ro-bind-try /bin /bin --ro-bind-try /etc /etc + sudo apt-get install -y -qq bubblewrap >/dev/null + go test -tags sandbox_e2e -run 'TestLoadArtifactSandbox|TestBwrap' -count=1 -v ./agents/manager ./runners/sandbox 2>&1 | grep -vE "^go: downloading" | tail -30 + echo "--- survivors"; ps -eo pid,ppid,pgid,sid,stat,comm | grep -E "executor|bwrap" | grep -v grep || echo "none" From ec3e1e880b730d691af3307236b97e7c4dff7589 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:08:33 -0400 Subject: [PATCH 12/12] diag: remove the temporary sandbox diagnostics workflow --- .github/workflows/diag-sandbox.yml | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 .github/workflows/diag-sandbox.yml diff --git a/.github/workflows/diag-sandbox.yml b/.github/workflows/diag-sandbox.yml deleted file mode 100644 index 272ed49a5..000000000 --- a/.github/workflows/diag-sandbox.yml +++ /dev/null @@ -1,26 +0,0 @@ -# TEMPORARY diagnostic — removed before merge. TestLoadArtifactSandbox hangs in -# executor cleanup on the ubuntu-26.04 runner image and passes everywhere else. -name: diag-sandbox -on: - pull_request: -jobs: - diag: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache: false - - name: environment - run: | - uname -a; cat /etc/os-release | head -2 - echo "pid1: $(cat /proc/1/comm)"; echo "apparmor userns restrict: $(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null)" - echo "ancestry:"; p=$$; while [ "$p" != 1 ] && [ -n "$p" ]; do echo " $(ps -o pid=,ppid=,comm= -p $p)"; p=$(ps -o ppid= -p $p | tr -d ' '); done - sudo apt-get install -y -qq bubblewrap >/dev/null; bwrap --version; mount | grep -E " /tmp | /proc " || true - - name: the sandbox test on this runner - run: | - sudo apt-get install -y -qq bubblewrap >/dev/null - go test -tags sandbox_e2e -run 'TestLoadArtifactSandbox|TestBwrap' -count=1 -v ./agents/manager ./runners/sandbox 2>&1 | grep -vE "^go: downloading" | tail -30 - echo "--- survivors"; ps -eo pid,ppid,pgid,sid,stat,comm | grep -E "executor|bwrap" | grep -v grep || echo "none"