[2.0] Add DuckDB E2E query optimization task (#136)

* Document Frontier-CS 2.0 task authoring

* Add DuckDB E2E query optimization task

Author: joyemang33

2.0/problems/duckdb_e2e_query_optimization/config.yaml

@@ -0,0 +1,47 @@
+tag: systems
+runtime:
+  language: cpp
+  timeout_seconds: 10800
+  environment: "DuckDB source patch; TPC-H shell timing; experimental judge"
+  apt_packages:
+    - bash
+    - build-essential
+    - ca-certificates
+    - ccache
+    - cmake
+    - git
+    - ninja-build
+    - pkg-config
+    - python3
+  judge_apt_packages:
+    - bash
+    - build-essential
+    - ca-certificates
+    - ccache
+    - cmake
+    - git
+    - ninja-build
+    - pkg-config
+    - python3
+  docker:
+    # Experimental local images. Build them with
+    # 2.0/problems/duckdb_e2e_query_optimization/docker/build_images.sh before running a
+    # local Harbor trial.
+    image: frontiercs/duckdb-e2e-query-optimization-agent:experimental-v1.5.3
+    judge_image: frontiercs/duckdb-e2e-query-optimization-judge:experimental-v1.5.3
+environment:
+  cpus: 8
+  memory_mb: 16384
+  storage_mb: 32768
+  build_timeout_seconds: 7200
+evaluation:
+  scale_factor: 1
+  benchmark_repetitions: 3
+  build_timeout_seconds: 7200
+  query_timeout_seconds: 300
+  duckdb_memory_limit: "6GB"
+  duckdb_temp_limit: "2GB"
+  child_memory_mb: 12288
+submission:
+  kind: file
+  path: /app/solution.patch

2.0/problems/duckdb_e2e_query_optimization/docker/README.md

@@ -0,0 +1,54 @@
+# Experimental DuckDB Images
+
+This task needs DuckDB source trees in both the agent and judge containers.
+The standard Frontier-CS 2.0 adapter can select Docker base images, but it does
+not build problem-specific images by itself. Build these images before running
+a local Harbor trial:
+
+```bash
+bash 2.0/problems/duckdb_e2e_query_optimization/docker/build_images.sh
+```
+
+Defaults:
+
+```text
+DUCKDB_REF=v1.5.3
+DUCKDB_BUILD_JOBS=1
+AGENT_TAG=frontiercs/duckdb-e2e-query-optimization-agent:experimental-v1.5.3
+JUDGE_TAG=frontiercs/duckdb-e2e-query-optimization-judge:experimental-v1.5.3
+```
+
+The agent image contains:
+
+```text
+/app/duckdb
+```
+
+The judge image contains:
+
+```text
+/opt/duckdb-vanilla
+/opt/duckdb-clean
+```
+
+`/opt/duckdb-clean` is prebuilt with:
+
+```bash
+CMAKE_BUILD_PARALLEL_LEVEL=1 GEN=ninja DISABLE_UNITY=1 DISABLE_PARQUET=1 BUILD_JEMALLOC=0 BUILD_BENCHMARK=1 BUILD_EXTENSIONS='tpch' make
+```
+
+`DUCKDB_BUILD_JOBS` intentionally defaults to `1`; higher parallelism can make
+large DuckDB C++ objects exhaust Docker memory on local machines. Unity builds
+are disabled for the same reason. Jemalloc is disabled because DuckDB v1.5.3's
+non-unity build exposes a missing include in the jemalloc allocator wrapper;
+vanilla and patched binaries are built with the same allocator setting.
+
+The judge copies that prebuilt tree to `/opt/duckdb-vanilla` during image
+construction. At evaluation time the evaluator first resets tracked files in
+`/opt/duckdb-clean` back to `HEAD` and removes untracked files under `src`
+without deleting ignored build artifacts, applies the submitted patch, performs
+an incremental rebuild of the DuckDB shell and optional benchmark runner,
+checks correctness against `/opt/duckdb-vanilla`, and then times TPC-H queries
+through DuckDB's shell and TPC-H extension. Keeping build artifacts in the clean
+tree is intentional; full DuckDB rebuilds are too slow for iterative Harbor
+submissions.

2.0/problems/duckdb_e2e_query_optimization/docker/agent/Dockerfile

@@ -0,0 +1,26 @@
+FROM ubuntu:24.04
+
+ARG DUCKDB_REF=v1.5.3
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        bash \
+        build-essential \
+        ca-certificates \
+        ccache \
+        cmake \
+        git \
+        ninja-build \
+        pkg-config \
+        python3 \
+        python3-pip \
+        curl \
+        ripgrep && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN git clone --branch "${DUCKDB_REF}" --depth 1 https://github.com/duckdb/duckdb.git /app/duckdb && \
+    cd /app/duckdb && \
+    git submodule update --init --recursive
+
+WORKDIR /app

2.0/problems/duckdb_e2e_query_optimization/docker/build_images.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+TASK_DIR=$(cd "$SCRIPT_DIR/.." && pwd)
+
+DUCKDB_REF="${DUCKDB_REF:-v1.5.3}"
+DUCKDB_BUILD_JOBS="${DUCKDB_BUILD_JOBS:-1}"
+AGENT_TAG="${AGENT_TAG:-frontiercs/duckdb-e2e-query-optimization-agent:experimental-v1.5.3}"
+JUDGE_TAG="${JUDGE_TAG:-frontiercs/duckdb-e2e-query-optimization-judge:experimental-v1.5.3}"
+
+docker build \
+  --build-arg "DUCKDB_REF=$DUCKDB_REF" \
+  -t "$AGENT_TAG" \
+  "$TASK_DIR/docker/agent"
+
+docker build \
+  --build-arg "DUCKDB_REF=$DUCKDB_REF" \
+  --build-arg "DUCKDB_BUILD_JOBS=$DUCKDB_BUILD_JOBS" \
+  -t "$JUDGE_TAG" \
+  "$TASK_DIR/docker/judge"
+
+echo "Built:"
+echo "  $AGENT_TAG"
+echo "  $JUDGE_TAG"

2.0/problems/duckdb_e2e_query_optimization/docker/judge/Dockerfile

@@ -0,0 +1,30 @@
+# syntax=docker/dockerfile:1.7
+FROM ubuntu:24.04
+
+ARG DUCKDB_REF=v1.5.3
+ARG DUCKDB_BUILD_JOBS=1
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        bash \
+        build-essential \
+        ca-certificates \
+        ccache \
+        cmake \
+        git \
+        ninja-build \
+        pkg-config \
+        python3 && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN git clone --branch "${DUCKDB_REF}" --depth 1 https://github.com/duckdb/duckdb.git /opt/duckdb-clean && \
+    cd /opt/duckdb-clean && \
+    git submodule update --init --recursive
+
+RUN --mount=type=cache,target=/root/.cache/ccache \
+    cd /opt/duckdb-clean && \
+    GEN=ninja CMAKE_BUILD_PARALLEL_LEVEL="${DUCKDB_BUILD_JOBS}" DISABLE_UNITY=1 DISABLE_PARQUET=1 BUILD_JEMALLOC=0 BUILD_UNITTESTS=0 BUILD_BENCHMARK=1 BUILD_EXTENSIONS='tpch' make -j"${DUCKDB_BUILD_JOBS}" && \
+    cp -a /opt/duckdb-clean /opt/duckdb-vanilla
+
+WORKDIR /judge

2.0/problems/duckdb_e2e_query_optimization/docker/smoke_images.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+AGENT_TAG="${AGENT_TAG:-frontiercs/duckdb-e2e-query-optimization-agent:experimental-v1.5.3}"
+JUDGE_TAG="${JUDGE_TAG:-frontiercs/duckdb-e2e-query-optimization-judge:experimental-v1.5.3}"
+
+echo "[agent] checking $AGENT_TAG"
+docker run --rm "$AGENT_TAG" sh -lc '
+  test -d /app/duckdb/.git
+  git -C /app/duckdb rev-parse HEAD
+'
+
+echo "[judge] checking $JUDGE_TAG"
+docker run --rm "$JUDGE_TAG" sh -lc '
+  test -d /opt/duckdb-clean
+  test -d /opt/duckdb-vanilla
+  test -x /opt/duckdb-vanilla/build/release/duckdb
+  test -x /opt/duckdb-vanilla/build/release/benchmark/benchmark_runner
+  /opt/duckdb-vanilla/build/release/duckdb -c "LOAD tpch; SELECT count(*) FROM tpch_queries();"
+'

2.0/problems/duckdb_e2e_query_optimization/evaluate.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+if [[ $# -gt 0 ]]; then
+  exec python3 "$SCRIPT_DIR/evaluator.py" "$@"
+fi
+
+SOLUTION="/work/execution_env/solution_env/solution.patch"
+if [[ ! -f "$SOLUTION" ]]; then
+  echo "Error: Missing $SOLUTION" >&2
+  exit 1
+fi
+
+python3 "$SCRIPT_DIR/evaluator.py" "$SOLUTION"