Fix worker images, code-upload git dependency, and CI/Cloud Agent Docker bootstrap

.cursor/environment.json

@@ -1,4 +1,4 @@
 {
-  "install": "docker compose build db sqs django",
+  "install": "bash cloud-agent/install.sh && docker compose build db sqs django",
   "start": "docker compose up -d db sqs django"
 }

AGENTS.md

@@ -59,7 +59,19 @@ docker run --rm -v nodejs_bc:/dest workspace-nodejs sh -c 'cp -a /code/bower_com
 
 ### Docker Setup (Cloud Agent VMs)
 
-Docker must be installed with `fuse-overlayfs` storage driver and `iptables-legacy`. The Docker daemon must use `"cgroup-parent": "system.slice"` in `/etc/docker/daemon.json` to avoid cgroup v2 threaded-mode errors. See the update script for the exact setup commands.
+Run the bootstrap script (also used by `.cursor/environment.json`):
+
+```bash
+bash cloud-agent/install.sh
+```
+
+On Ubuntu 24.04 (Noble) Cloud Agent images:
+
+- Install `docker-compose-v2` (not `docker-compose-plugin`, which is unavailable).
+- `iptables-legacy` alternatives may not exist; the bootstrap script skips that step when missing.
+- If `systemctl` cannot start Docker, the script falls back to launching `dockerd` directly.
+
+The daemon should use the `fuse-overlayfs` storage driver and `"cgroup-parent": "system.slice"` in `/etc/docker/daemon.json` to avoid cgroup v2 threaded-mode errors.
 
 ### Running Tests
 

apps/challenges/aws_utils.py

@@ -248,22 +248,37 @@ def update_evalai_worker_image_tag(image_url, commit_id):
     return f"{repository}:{commit_id}"
 
 
+def get_evalai_worker_image_tag(image_url=None, commit_id=None):
+    """
+    Resolve the ECR image tag for an EvalAI-managed worker image.
+    """
+    if commit_id:
+        return commit_id
+    if image_url and ":" in image_url:
+        return image_url.rsplit(":", 1)[1]
+    return "latest"
+
+
 def get_worker_image_for_challenge(challenge, commit_id=None):
     """
     Resolve the submission worker image for a challenge.
     """
+    python_version = normalize_worker_python_version(
+        getattr(challenge, "worker_python_version", None)
+    )
+
     if challenge.worker_image_url:
-        if commit_id and is_evalai_managed_submission_worker_image(
+        if is_evalai_managed_submission_worker_image(
             challenge.worker_image_url
         ):
-            return update_evalai_worker_image_tag(
+            image_tag = get_evalai_worker_image_tag(
                 challenge.worker_image_url, commit_id
             )
+            return get_evalai_submission_worker_ecr_image(
+                python_version, image_tag
+            )
         return challenge.worker_image_url
 
-    python_version = normalize_worker_python_version(
-        getattr(challenge, "worker_python_version", None)
-    )
     return get_evalai_submission_worker_ecr_image(python_version, commit_id)
 
 
@@ -1085,7 +1100,18 @@ def refresh_task_definition_for_challenge(
 
         task_def_arn = response["taskDefinition"]["taskDefinitionArn"]
         challenge.task_def_arn = task_def_arn
-        challenge.save()
+        update_fields = ["task_def_arn"]
+        resolved_worker_image = image_settings["WORKER_IMAGE"]
+        if (
+            challenge.worker_image_url
+            and is_evalai_managed_submission_worker_image(
+                challenge.worker_image_url
+            )
+        ):
+            if challenge.worker_image_url != resolved_worker_image:
+                challenge.worker_image_url = resolved_worker_image
+                update_fields.append("worker_image_url")
+        challenge.save(update_fields=update_fields)
 
         try:
             deregister_response = client.deregister_task_definition(

cloud-agent/install.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ensure_docker_compose_package() {
+  if apt-cache show docker-compose-plugin >/dev/null 2>&1; then
+    echo "docker-compose-plugin"
+    return 0
+  fi
+  if apt-cache show docker-compose-v2 >/dev/null 2>&1; then
+    echo "docker-compose-v2"
+    return 0
+  fi
+  echo "No docker compose package found in apt repositories." >&2
+  return 1
+}
+
+configure_iptables_legacy_if_available() {
+  if update-alternatives --list iptables 2>/dev/null | grep -q legacy; then
+    sudo update-alternatives --set iptables /usr/sbin/iptables-legacy
+  else
+    echo "Skipping iptables-legacy setup (not available on this image)."
+  fi
+
+  if update-alternatives --list ip6tables 2>/dev/null | grep -q legacy; then
+    sudo update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+  else
+    echo "Skipping ip6tables-legacy setup (not available on this image)."
+  fi
+}
+
+configure_docker_daemon() {
+  sudo mkdir -p /etc/docker
+  if [ ! -f /etc/docker/daemon.json ]; then
+    sudo tee /etc/docker/daemon.json >/dev/null <<'EOF'
+{
+  "storage-driver": "fuse-overlayfs",
+  "cgroup-parent": "system.slice"
+}
+EOF
+  fi
+}
+
+start_docker_daemon() {
+  local dockerd_log
+  dockerd_log="$(mktemp)"
+
+  if sudo docker info >/dev/null 2>&1; then
+    echo "Docker daemon is already running."
+    rm -f "${dockerd_log}"
+    return 0
+  fi
+
+  if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
+    if sudo systemctl enable --now docker 2>/dev/null; then
+      rm -f "${dockerd_log}"
+      return 0
+    fi
+    echo "systemctl could not start docker; falling back to dockerd."
+  fi
+
+  if ! pgrep -x dockerd >/dev/null 2>&1; then
+    echo "Starting dockerd in the background..."
+    sudo dockerd >"${dockerd_log}" 2>&1 &
+  fi
+
+  for _ in $(seq 1 30); do
+    if sudo docker info >/dev/null 2>&1; then
+      rm -f "${dockerd_log}"
+      return 0
+    fi
+    sleep 1
+  done
+
+  echo "Docker daemon failed to start. Recent dockerd logs:" >&2
+  tail -50 "${dockerd_log}" >&2 || true
+  rm -f "${dockerd_log}"
+  return 1
+}
+
+if command -v docker >/dev/null 2>&1; then
+  echo "Docker client is already installed."
+else
+  echo "Installing Docker for Cloud Agent VM..."
+  if ! compose_package="$(ensure_docker_compose_package)"; then
+    echo "Cannot install Docker without a compose package." >&2
+    exit 1
+  fi
+  sudo apt-get update
+  sudo apt-get install -y fuse-overlayfs iptables docker.io "${compose_package}"
+  configure_iptables_legacy_if_available
+  configure_docker_daemon
+  if getent group docker >/dev/null 2>&1; then
+    sudo usermod -aG docker "${USER}" || true
+  fi
+fi
+
+start_docker_daemon
+
+sudo docker --version
+sudo docker compose version

docker/dev/code-upload-worker/Dockerfile

@@ -35,8 +35,13 @@ ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1
 
 # Install runtime dependencies (apt-get handles multi-arch automatically)
+# Note: git is required for kubectl apply -k with github.com kustomize remote bases.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends unzip && \
+    apt-get install -y --no-install-recommends \
+    curl \
+    unzip \
+    git \
+    ca-certificates && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /var/cache/apt/*
 

docker/dev/worker_py3_7/Dockerfile

@@ -48,9 +48,12 @@ WORKDIR /code
 # Copy requirements first for better layer caching
 COPY requirements/ /code/requirements/
 
+# PyJWT>=2.10 requires Python 3.9+; pin a 3.7-compatible release for worker builds.
+RUN sed -i 's/^PyJWT==.*/PyJWT==2.8.0/' /code/requirements/common.txt
+
 # Install Python dependencies with BuildKit cache mount
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29 setuptools==57.5.0 && \
+    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29.36 setuptools==57.5.0 && \
     pip install --no-cache-dir --no-compile -r requirements/dev.txt && \
     pip install --no-cache-dir --no-compile -r requirements/worker_py3_7.txt && \
     # Clean up pip cache and Python bytecode (directories first, then files)

docker/dev/worker_py3_8/Dockerfile

@@ -66,9 +66,12 @@ WORKDIR /code
 # Copy requirements first for better layer caching
 COPY requirements/ /code/requirements/
 
+# PyJWT>=2.10 requires Python 3.9+; pin a 3.8-compatible release for worker builds.
+RUN sed -i 's/^PyJWT==.*/PyJWT==2.9.0/' /code/requirements/common.txt
+
 # Install Python dependencies with BuildKit cache mount
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29 setuptools==57.5.0 && \
+    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29.36 setuptools==57.5.0 && \
     pip install --no-cache-dir --no-compile -r requirements/dev.txt && \
     pip install --no-cache-dir --no-compile -r requirements/worker_py3_8.txt && \
     # Clean up pip cache and Python bytecode (directories first, then files)

docker/dev/worker_py3_9/Dockerfile

@@ -53,14 +53,18 @@ COPY requirements/ /code/requirements/
 # Install Python dependencies with BuildKit cache mount
 # Install prerequisite packages first (needed for compiling native extensions)
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29 setuptools==57.5.0 && \
+    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29.36 setuptools==57.5.0 && \
     pip install --no-cache-dir --no-compile -r requirements/dev.txt && \
-    pip install --no-cache-dir --no-compile -r requirements/worker_py3_9.txt && \
-    # Clean up pip cache and Python bytecode (directories first, then files)
-    rm -rf /root/.cache/pip && \
+    pip install --no-cache-dir --no-compile -r requirements/worker_py3_9.txt
+
+# Best-effort bytecode cleanup; keep separate so pip failures fail the build.
+RUN rm -rf /root/.cache/pip && \
     find /usr/local/lib/python3.9/site-packages -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true && \
     find /usr/local/lib/python3.9/site-packages \( -name "*.pyc" -o -name "*.pyo" \) -delete 2>/dev/null || true
 
+# Verify worker runtime imports required by submission_worker.py.
+RUN python -c "import boto3, botocore, django, requests, yaml"
+
 # Stage 2: Runtime stage - minimal dependencies only
 FROM python:3.9.21-slim-bullseye
 

docker/prod/code-upload-worker/Dockerfile

@@ -35,11 +35,13 @@ ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1
 
 # Install runtime dependencies (apt-get handles multi-arch automatically)
-# Note: curl and unzip are needed by install_dependencies.sh
+# Note: curl and unzip are needed by install_dependencies.sh; git is required
+# for kubectl apply -k with github.com kustomize remote bases.
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     curl \
     unzip \
+    git \
     ca-certificates && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /var/cache/apt/*

docker/prod/worker_py3_7/Dockerfile

@@ -64,9 +64,12 @@ WORKDIR /code
 # Copy requirements first for better layer caching
 COPY requirements/ /code/requirements/
 
+# PyJWT>=2.10 requires Python 3.9+; pin a 3.7-compatible release for worker builds.
+RUN sed -i 's/^PyJWT==.*/PyJWT==2.8.0/' /code/requirements/common.txt
+
 # Install Python dependencies with BuildKit cache mount
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29 numpy==1.18.1 && \
+    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29.36 numpy==1.18.1 && \
     pip install --no-cache-dir --no-compile -r requirements/prod.txt && \
     pip install --no-cache-dir --no-compile -r requirements/worker_py3_7.txt && \
     # Clean up pip cache and Python bytecode (directories first, then files)

docker/prod/worker_py3_8/Dockerfile

@@ -64,9 +64,12 @@ WORKDIR /code
 # Copy requirements first for better layer caching
 COPY requirements/ /code/requirements/
 
+# PyJWT>=2.10 requires Python 3.9+; pin a 3.8-compatible release for worker builds.
+RUN sed -i 's/^PyJWT==.*/PyJWT==2.9.0/' /code/requirements/common.txt
+
 # Install Python dependencies with BuildKit cache mount
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29 numpy==1.18.1 && \
+    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29.36 numpy==1.18.1 && \
     pip install --no-cache-dir --no-compile -r requirements/prod.txt && \
     pip install --no-cache-dir --no-compile -r requirements/worker_py3_8.txt && \
     # Clean up pip cache and Python bytecode (directories first, then files)

docker/prod/worker_py3_9/Dockerfile

@@ -70,14 +70,18 @@ COPY requirements/ /code/requirements/
 # Note: BuildKit cache mount persists outside container, but we clean internal cache for final image size
 # Install prerequisite packages first (needed for compiling native extensions)
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29 numpy==1.18.1 && \
+    pip install --no-cache-dir --no-compile -U cffi service_identity cython==0.29.36 numpy==1.18.1 && \
     pip install --no-cache-dir --no-compile -r requirements/prod.txt && \
-    pip install --no-cache-dir --no-compile -r requirements/worker_py3_9.txt && \
-    # Clean up pip cache and Python bytecode (directories first, then files)
-    rm -rf /root/.cache/pip && \
+    pip install --no-cache-dir --no-compile -r requirements/worker_py3_9.txt
+
+# Best-effort bytecode cleanup; keep separate so pip failures fail the build.
+RUN rm -rf /root/.cache/pip && \
     find /usr/local/lib/python3.9/site-packages -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true && \
     find /usr/local/lib/python3.9/site-packages \( -name "*.pyc" -o -name "*.pyo" \) -delete 2>/dev/null || true
 
+# Verify worker runtime imports required by submission_worker.py.
+RUN python -c "import boto3, botocore, django, requests, yaml"
+
 # Collect OpenCV runtime libraries to architecture-independent location
 # This allows multi-arch builds (amd64/arm64)
 RUN mkdir -p /runtime-libs && \

requirements/worker_py3_9.txt

@@ -4,6 +4,7 @@ networkx==3.2.1
 numpy==1.26.4
 pandas==2.2.3
 pycocotools==2.0.7
+PyYaml==5.4.1
 scikit-learn==1.4.2
 scipy==1.11.4
 tqdm==4.67.1