From 69d895f30083e0270f5b664180eb08723ec0c58a Mon Sep 17 00:00:00 2001
From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com>
Date: Wed, 11 Nov 2026 20:02:48 -0500
Subject: [PATCH 1/2] ci(eval): install ffmpeg for eval scaffolds that render
 frames

Four eval cases name ffmpeg as a host tool (three in explainer-video,
knowledge's vd-light-skips-vision-spot-check), and the hosted image has
none: run 42563549712 reported vd-light-skips-vision-spot-check NOT
COMPARABLE with "scaffold.sh: line 13: ffmpeg: command not found". Each
eval job (key, subscription, fallback) now installs the BtbN static
build pr-require-checks.yml's demo-video leg pins, with the same tag
and SHA-256, into /opt/tools/bin before the runner is isolated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VQb9Q2ceZLXhTR4d1vkTCe
---
 .github/workflows/eval-browser-dispatch.yml | 84 +++++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/.github/workflows/eval-browser-dispatch.yml b/.github/workflows/eval-browser-dispatch.yml
index 5b3a36abd59..fb1cc33942f 100644
--- a/.github/workflows/eval-browser-dispatch.yml
+++ b/.github/workflows/eval-browser-dispatch.yml
@@ -506,6 +506,34 @@ jobs:
           echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT"
           echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions."
 
+      # Eval cases whose scaffold renders frames (explainer-video, knowledge)
+      # name ffmpeg as a host tool, and the hosted image carries none. Same
+      # BtbN static build, tag and SHA-256 as pr-require-checks.yml's demo-video
+      # leg, into /opt/tools/bin, which every eval run's sandbox can read.
+      - name: Install ffmpeg for eval scaffolds
+        if: steps.stage.outputs.go == 'true'
+        timeout-minutes: 3
+        env:
+          FFMPEG_RELEASE: autobuild-2026-09-30-13-08
+          FFMPEG_BUILD: n8.1.3-9-g29e619e767
+          FFMPEG_SHA256: 97ce978979194b5cf7e06a5e68020dbdaa7a4f3294c5452b6a1bc347100dbb79
+        run: |
+          asset="ffmpeg-$FFMPEG_BUILD-linux64-gpl-8.1.tar.xz"
+          archive="$RUNNER_TEMP/$asset"
+          mkdir -p /opt/tools/bin
+          curl -fSL --no-progress-meter --retry 7 --retry-all-errors -o "$archive" \
+            "https://github.com/BtbN/FFmpeg-Builds/releases/download/$FFMPEG_RELEASE/$asset"
+          echo "$FFMPEG_SHA256  $archive" | sha256sum -c - ||
+            { echo "::error::$asset does not match FFMPEG_SHA256"; exit 1; }
+          tar -xJf "$archive" -C /opt/tools/bin --strip-components=2 --wildcards '*/bin/ffmpeg' '*/bin/ffprobe'
+          rm -f "$archive"
+          echo "/opt/tools/bin" >> "$GITHUB_PATH"
+          for tool in ffmpeg ffprobe; do
+            line=$("/opt/tools/bin/$tool" -version | sed -n 1p)
+            echo "$line"
+            [[ "$line" == *"$FFMPEG_BUILD"* ]] || { echo "::error::$tool reports '$line', not $FFMPEG_BUILD"; exit 1; }
+          done
+
       # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
       # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
       # and docker access for the rest of the job.
@@ -1337,6 +1365,34 @@ jobs:
           echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT"
           echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions."
 
+      # Eval cases whose scaffold renders frames (explainer-video, knowledge)
+      # name ffmpeg as a host tool, and the hosted image carries none. Same
+      # BtbN static build, tag and SHA-256 as pr-require-checks.yml's demo-video
+      # leg, into /opt/tools/bin, which every eval run's sandbox can read.
+      - name: Install ffmpeg for eval scaffolds
+        if: steps.stage.outputs.go == 'true'
+        timeout-minutes: 3
+        env:
+          FFMPEG_RELEASE: autobuild-2026-09-30-13-08
+          FFMPEG_BUILD: n8.1.3-9-g29e619e767
+          FFMPEG_SHA256: 97ce978979194b5cf7e06a5e68020dbdaa7a4f3294c5452b6a1bc347100dbb79
+        run: |
+          asset="ffmpeg-$FFMPEG_BUILD-linux64-gpl-8.1.tar.xz"
+          archive="$RUNNER_TEMP/$asset"
+          mkdir -p /opt/tools/bin
+          curl -fSL --no-progress-meter --retry 7 --retry-all-errors -o "$archive" \
+            "https://github.com/BtbN/FFmpeg-Builds/releases/download/$FFMPEG_RELEASE/$asset"
+          echo "$FFMPEG_SHA256  $archive" | sha256sum -c - ||
+            { echo "::error::$asset does not match FFMPEG_SHA256"; exit 1; }
+          tar -xJf "$archive" -C /opt/tools/bin --strip-components=2 --wildcards '*/bin/ffmpeg' '*/bin/ffprobe'
+          rm -f "$archive"
+          echo "/opt/tools/bin" >> "$GITHUB_PATH"
+          for tool in ffmpeg ffprobe; do
+            line=$("/opt/tools/bin/$tool" -version | sed -n 1p)
+            echo "$line"
+            [[ "$line" == *"$FFMPEG_BUILD"* ]] || { echo "::error::$tool reports '$line', not $FFMPEG_BUILD"; exit 1; }
+          done
+
       # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
       # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
       # and docker access for the rest of the job.
@@ -2121,6 +2177,34 @@ jobs:
           echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT"
           echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions."
 
+      # Eval cases whose scaffold renders frames (explainer-video, knowledge)
+      # name ffmpeg as a host tool, and the hosted image carries none. Same
+      # BtbN static build, tag and SHA-256 as pr-require-checks.yml's demo-video
+      # leg, into /opt/tools/bin, which every eval run's sandbox can read.
+      - name: Install ffmpeg for eval scaffolds
+        if: steps.stage.outputs.go == 'true'
+        timeout-minutes: 3
+        env:
+          FFMPEG_RELEASE: autobuild-2026-09-30-13-08
+          FFMPEG_BUILD: n8.1.3-9-g29e619e767
+          FFMPEG_SHA256: 97ce978979194b5cf7e06a5e68020dbdaa7a4f3294c5452b6a1bc347100dbb79
+        run: |
+          asset="ffmpeg-$FFMPEG_BUILD-linux64-gpl-8.1.tar.xz"
+          archive="$RUNNER_TEMP/$asset"
+          mkdir -p /opt/tools/bin
+          curl -fSL --no-progress-meter --retry 7 --retry-all-errors -o "$archive" \
+            "https://github.com/BtbN/FFmpeg-Builds/releases/download/$FFMPEG_RELEASE/$asset"
+          echo "$FFMPEG_SHA256  $archive" | sha256sum -c - ||
+            { echo "::error::$asset does not match FFMPEG_SHA256"; exit 1; }
+          tar -xJf "$archive" -C /opt/tools/bin --strip-components=2 --wildcards '*/bin/ffmpeg' '*/bin/ffprobe'
+          rm -f "$archive"
+          echo "/opt/tools/bin" >> "$GITHUB_PATH"
+          for tool in ffmpeg ffprobe; do
+            line=$("/opt/tools/bin/$tool" -version | sed -n 1p)
+            echo "$line"
+            [[ "$line" == *"$FFMPEG_BUILD"* ]] || { echo "::error::$tool reports '$line', not $FFMPEG_BUILD"; exit 1; }
+          done
+
       # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
       # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
       # and docker access for the rest of the job.

From aae9e6daf25d105fe50b99fca5207cc8dbd01cab Mon Sep 17 00:00:00 2001
From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com>
Date: Wed, 11 Nov 2026 20:53:03 -0500
Subject: [PATCH 2/2] ci(eval): install ffmpeg after isolation, only for
 plugins that need it

The step ran the downloaded ffmpeg while sudo still worked, which breaks
the job's rule that no third-party code runs before isolation. It now
runs after "Isolate the runner", beside the pinned shfmt step. It is
also gated on the staged plugin list holding explainer-video or
knowledge, so other evals do not download the archive.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VQb9Q2ceZLXhTR4d1vkTCe
---
 .github/workflows/eval-browser-dispatch.yml | 138 +++++++++++---------
 1 file changed, 75 insertions(+), 63 deletions(-)

diff --git a/.github/workflows/eval-browser-dispatch.yml b/.github/workflows/eval-browser-dispatch.yml
index fb1cc33942f..32aa33a622d 100644
--- a/.github/workflows/eval-browser-dispatch.yml
+++ b/.github/workflows/eval-browser-dispatch.yml
@@ -506,12 +506,35 @@ jobs:
           echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT"
           echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions."
 
+      # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
+      # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
+      # and docker access for the rest of the job.
+      - name: Isolate the runner
+        if: steps.stage.outputs.go == 'true'
+        uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository]
+        with:
+          install-subprocess-isolation: "true"
+
+      # Every binary this job runs comes from the locked root toolchain. Its
+      # install scripts run here, after sudo and docker access are gone.
+      - name: Install the locked root toolchain
+        timeout-minutes: 5
+        if: steps.stage.outputs.go == 'true'
+        run: |
+          cp .base/package.json .base/package-lock.json /opt/tools/
+          npm ci --prefix /opt/tools
+          echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH"
+
       # Eval cases whose scaffold renders frames (explainer-video, knowledge)
       # name ffmpeg as a host tool, and the hosted image carries none. Same
       # BtbN static build, tag and SHA-256 as pr-require-checks.yml's demo-video
-      # leg, into /opt/tools/bin, which every eval run's sandbox can read.
+      # leg, into /opt/tools/bin, which every eval run's sandbox can read. After
+      # isolation, like shfmt below, so the binary never runs with sudo.
       - name: Install ffmpeg for eval scaffolds
-        if: steps.stage.outputs.go == 'true'
+        if: >-
+          steps.stage.outputs.go == 'true' &&
+          (contains(steps.plugins.outputs.list, 'plugins/explainer-video') ||
+          contains(steps.plugins.outputs.list, 'plugins/knowledge'))
         timeout-minutes: 3
         env:
           FFMPEG_RELEASE: autobuild-2026-09-30-13-08
@@ -534,25 +557,6 @@ jobs:
             [[ "$line" == *"$FFMPEG_BUILD"* ]] || { echo "::error::$tool reports '$line', not $FFMPEG_BUILD"; exit 1; }
           done
 
-      # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
-      # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
-      # and docker access for the rest of the job.
-      - name: Isolate the runner
-        if: steps.stage.outputs.go == 'true'
-        uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository]
-        with:
-          install-subprocess-isolation: "true"
-
-      # Every binary this job runs comes from the locked root toolchain. Its
-      # install scripts run here, after sudo and docker access are gone.
-      - name: Install the locked root toolchain
-        timeout-minutes: 5
-        if: steps.stage.outputs.go == 'true'
-        run: |
-          cp .base/package.json .base/package-lock.json /opt/tools/
-          npm ci --prefix /opt/tools
-          echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH"
-
       # The guardrails hooks read every Bash command through shfmt and refuse the
       # call without it, so a guardrails eval needs it on PATH. Same v3.14.1 pin
       # and SHA-256 as pr-require-checks.yml. In /opt/tools, since each eval run
@@ -1365,12 +1369,35 @@ jobs:
           echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT"
           echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions."
 
+      # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
+      # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
+      # and docker access for the rest of the job.
+      - name: Isolate the runner
+        if: steps.stage.outputs.go == 'true'
+        uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository]
+        with:
+          install-subprocess-isolation: "true"
+
+      # Every binary this job runs comes from the locked root toolchain. Its
+      # install scripts run here, after sudo and docker access are gone.
+      - name: Install the locked root toolchain
+        timeout-minutes: 5
+        if: steps.stage.outputs.go == 'true'
+        run: |
+          cp .base/package.json .base/package-lock.json /opt/tools/
+          npm ci --prefix /opt/tools
+          echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH"
+
       # Eval cases whose scaffold renders frames (explainer-video, knowledge)
       # name ffmpeg as a host tool, and the hosted image carries none. Same
       # BtbN static build, tag and SHA-256 as pr-require-checks.yml's demo-video
-      # leg, into /opt/tools/bin, which every eval run's sandbox can read.
+      # leg, into /opt/tools/bin, which every eval run's sandbox can read. After
+      # isolation, like shfmt below, so the binary never runs with sudo.
       - name: Install ffmpeg for eval scaffolds
-        if: steps.stage.outputs.go == 'true'
+        if: >-
+          steps.stage.outputs.go == 'true' &&
+          (contains(steps.plugins.outputs.list, 'plugins/explainer-video') ||
+          contains(steps.plugins.outputs.list, 'plugins/knowledge'))
         timeout-minutes: 3
         env:
           FFMPEG_RELEASE: autobuild-2026-09-30-13-08
@@ -1393,25 +1420,6 @@ jobs:
             [[ "$line" == *"$FFMPEG_BUILD"* ]] || { echo "::error::$tool reports '$line', not $FFMPEG_BUILD"; exit 1; }
           done
 
-      # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
-      # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
-      # and docker access for the rest of the job.
-      - name: Isolate the runner
-        if: steps.stage.outputs.go == 'true'
-        uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository]
-        with:
-          install-subprocess-isolation: "true"
-
-      # Every binary this job runs comes from the locked root toolchain. Its
-      # install scripts run here, after sudo and docker access are gone.
-      - name: Install the locked root toolchain
-        timeout-minutes: 5
-        if: steps.stage.outputs.go == 'true'
-        run: |
-          cp .base/package.json .base/package-lock.json /opt/tools/
-          npm ci --prefix /opt/tools
-          echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH"
-
       # The guardrails hooks read every Bash command through shfmt and refuse the
       # call without it, so a guardrails eval needs it on PATH. Same v3.14.1 pin
       # and SHA-256 as pr-require-checks.yml. In /opt/tools, since each eval run
@@ -2177,12 +2185,35 @@ jobs:
           echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT"
           echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions."
 
+      # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
+      # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
+      # and docker access for the rest of the job.
+      - name: Isolate the runner
+        if: steps.stage.outputs.go == 'true'
+        uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository]
+        with:
+          install-subprocess-isolation: "true"
+
+      # Every binary this job runs comes from the locked root toolchain. Its
+      # install scripts run here, after sudo and docker access are gone.
+      - name: Install the locked root toolchain
+        timeout-minutes: 5
+        if: steps.stage.outputs.go == 'true'
+        run: |
+          cp .base/package.json .base/package-lock.json /opt/tools/
+          npm ci --prefix /opt/tools
+          echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH"
+
       # Eval cases whose scaffold renders frames (explainer-video, knowledge)
       # name ffmpeg as a host tool, and the hosted image carries none. Same
       # BtbN static build, tag and SHA-256 as pr-require-checks.yml's demo-video
-      # leg, into /opt/tools/bin, which every eval run's sandbox can read.
+      # leg, into /opt/tools/bin, which every eval run's sandbox can read. After
+      # isolation, like shfmt below, so the binary never runs with sudo.
       - name: Install ffmpeg for eval scaffolds
-        if: steps.stage.outputs.go == 'true'
+        if: >-
+          steps.stage.outputs.go == 'true' &&
+          (contains(steps.plugins.outputs.list, 'plugins/explainer-video') ||
+          contains(steps.plugins.outputs.list, 'plugins/knowledge'))
         timeout-minutes: 3
         env:
           FFMPEG_RELEASE: autobuild-2026-09-30-13-08
@@ -2205,25 +2236,6 @@ jobs:
             [[ "$line" == *"$FFMPEG_BUILD"* ]] || { echo "::error::$tool reports '$line', not $FFMPEG_BUILD"; exit 1; }
           done
 
-      # Installs the bubblewrap and socat the Bash sandbox needs, lifts the
-      # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo
-      # and docker access for the rest of the job.
-      - name: Isolate the runner
-        if: steps.stage.outputs.go == 'true'
-        uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository]
-        with:
-          install-subprocess-isolation: "true"
-
-      # Every binary this job runs comes from the locked root toolchain. Its
-      # install scripts run here, after sudo and docker access are gone.
-      - name: Install the locked root toolchain
-        timeout-minutes: 5
-        if: steps.stage.outputs.go == 'true'
-        run: |
-          cp .base/package.json .base/package-lock.json /opt/tools/
-          npm ci --prefix /opt/tools
-          echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH"
-
       # The guardrails hooks read every Bash command through shfmt and refuse the
       # call without it, so a guardrails eval needs it on PATH. Same v3.14.1 pin
       # and SHA-256 as pr-require-checks.yml. In /opt/tools, since each eval run