Improve .build caching for faster builds. Part 2.

Dockerfile

@@ -1,4 +1,4 @@
-# Stage 1: Precompile TestRunner and Module Cache
+# Stage 1: Precompile TestRunner
 FROM swift:6.1.2 AS builder
 RUN swift --version
 
@@ -7,21 +7,19 @@ WORKDIR /TestRunner
 COPY src/TestRunner .
 RUN swift build --configuration release
 
-# Build WarmUp package
-# Build directory and final working paths should be equal for reuse of ModuleCache.
-WORKDIR /opt/test-runner
-COPY src/WarmUp .
-RUN swift build --build-tests
-
 # Stage 2: Prepare docker container image
 FROM swift:6.1.2
 RUN apt-get update && apt-get install -y jq
 
-WORKDIR /opt/test-runner/
+WORKDIR /opt/test-runner
+
+# Build TestEnvironment package
+# Build directory and final working paths should be equal for reuse of ModuleCache.
+COPY src/TestEnvironment .
+RUN swift build --build-tests
+
 COPY bin/run.sh bin/run-test.sh bin/
 COPY --from=builder /TestRunner/.build/release/TestRunner bin/
-COPY --from=builder /opt/test-runner/.build .build
-COPY --from=builder /opt/test-runner/Package.resolved Package.resolved
 
 ENV RUN_IN_DOCKER=TRUE
 

bin/benchmark-in-docker.sh

@@ -13,7 +13,7 @@
 # Example:
 # ./bin/benchmark-in-docker.sh
 
-set -eo pipefail
+set -euo pipefail
 
 die() { echo "$*" >&2; exit 1; }
 
@@ -26,14 +26,15 @@ required_tool docker
 required_tool hyperfine
 
 # Pre-build the Docker image
-if [[ -z "${SKIP_DOCKER_BUILD}" ]]; then  
+if [[ -z "${SKIP_DOCKER_BUILD:-}" ]]; then  
   docker build --rm -t exercism/swift-test-runner .
 else
-  echo "Skipping docker build because SKIP_DOCKER_BUILD is set."
+  printf "Skipping docker build because SKIP_DOCKER_BUILD is set.\n"
 fi
 
-CURRENT_PATH=${PWD}
+CURRENT_PATH="${PWD}"
 
+test_list=$(find tests -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | paste -sd "," -)
 hyperfine \
-    --parameter-list slug $(find tests -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | paste -sd ",") \
-    "SKIP_DOCKER_BUILD=true bin/run-in-docker.sh {slug} $CURRENT_PATH/tests/{slug} $CURRENT_PATH/tests/{slug}"
+    --parameter-list slug "${test_list}" \
+    "SKIP_DOCKER_BUILD=true bin/run-in-docker.sh {slug} ${CURRENT_PATH}/tests/{slug} ${CURRENT_PATH}/tests/{slug}"

bin/benchmark.sh

@@ -10,7 +10,7 @@
 # Example:
 # ./bin/benchmark.sh
 
-set -eo pipefail
+set -euo pipefail
 
 die() { echo "$*" >&2; exit 1; }
 
@@ -21,7 +21,8 @@ required_tool() {
 
 required_tool hyperfine
 
+test_list=$(find tests -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | paste -sd "," -)
 hyperfine \
-    --parameter-list slug $(find tests -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | paste -sd ",") \
+    --parameter-list slug "${test_list}" \
     --prepare 'git clean -xdfq tests/{slug}' \
-    'bin/run.sh {slug} tests/{slug} tests/{slug}'
+    './bin/run.sh {slug} tests/{slug} tests/{slug}'

bin/build-test-runner.sh

@@ -9,14 +9,16 @@
 # Example:
 # ./bin/build-test-runner.sh
 
+set -euo pipefail
+
 BIN_DIR="bin"
 BUILD_DIR="src/TestRunner"
 RELEASE_DIR=".build/release"
 
 # Build the test runner file
-cd "$BUILD_DIR"
+cd "${BUILD_DIR}"
 swift build --configuration release
 cd -
 
 # Copy generated file to bin dir
-cp "$BUILD_DIR/$RELEASE_DIR/TestRunner" "$BIN_DIR"/
+cp "${BUILD_DIR}/${RELEASE_DIR}/TestRunner" "${BIN_DIR}"/

bin/run-in-docker.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-set -e
 
 # Synopsis:
 # Run the test runner on a solution using the test runner Docker image.
@@ -18,8 +17,11 @@ set -e
 # ./bin/run-in-docker.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/
 
 # If any required arguments is missing, print the usage and exit
-if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
-    echo "usage: ./bin/run-in-docker.sh exercise-slug /absolute/path/to/solution/folder/ /absolute/path/to/output/directory/"
+
+set -euo pipefail
+
+if (( "$#" != 3 )); then
+    printf 'usage: %s exercise-slug /absolute/path/to/solution/folder/ /absolute/path/to/output/directory/\n' "$0"
     exit 1
 fi
 
@@ -31,10 +33,10 @@ OUTPUT_DIR="${3%/}"
 mkdir -p "${OUTPUT_DIR}"
 
 # Pre-build the Docker image
-if [[ -z "${SKIP_DOCKER_BUILD}" ]]; then  
+if [[ -z "${SKIP_DOCKER_BUILD:-}" ]]; then  
   docker build --rm -t exercism/swift-test-runner .
 else
-  echo "Skipping docker build because SKIP_DOCKER_BUILD is set."
+  printf "Skipping docker build because SKIP_DOCKER_BUILD is set.\n"
 fi
 
 # Run image passing the arguments
@@ -43,4 +45,4 @@ docker run \
     --mount type=bind,src="${INPUT_DIR}",dst=/solution \
     --mount type=bind,src="${OUTPUT_DIR}",dst=/output \
     --mount type=volume,dst=/tmp \
-    exercism/swift-test-runner $SLUG /solution/ /output/
+    exercism/swift-test-runner "${SLUG}" /solution/ /output/

bin/run-test.sh

@@ -14,12 +14,22 @@
 # Example:
 # ./bin/run-test.sh /absolute/path/to/tests/compile-error/
 
+set -euo pipefail
+
 # If any required arguments is missing, print the usage and exit
-if [ -z "$1" ]; then
-    echo "usage: ./bin/run-test.sh /absolute/path/to/test/folder/"
+if (( "$#" != 1 )); then
+    printf 'Usage: %s /absolute/path/to/test/folder/\n' "$0"
     exit 1
 fi
 
+sed_i() { 
+    if [[ "$(uname)" == "Darwin" ]]; then 
+        sed -i '' "$@"
+    else 
+        sed -i "$@"
+    fi 
+}
+
 test_dir="$1"
 
 test_dir_name=$(basename "${test_dir}")
@@ -30,11 +40,7 @@ expected_results_file_path="${test_dir_path}/expected_results.json"
 bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
 # Normalize the results file
-sed -i -e "s~\\\/~/~g" -e "s~${test_dir_path}~/solution~g" "${results_file_path}"
+sed_i -e "s~\\\/~/~g" -e "s~${test_dir_path}~/solution~g" "${results_file_path}"
 
-echo "${test_dir_name}: comparing results.json to expected_results.json"
+printf '%s: comparing results.json to expected_results.json\n' "${test_dir_name}"
 diff "${results_file_path}" "${expected_results_file_path}"
-
-if [ $? -ne 0 ]; then
-    echo "\"${test_dir_name}\" failed!"
-fi

bin/run-tests-in-docker.sh

@@ -20,7 +20,7 @@ exit_code=0
 # Iterate over all test directories
 work_dir=/opt/test-runner/
 for test_dir in tests/*; do
-    test_name=$(basename $test_dir)
+    test_name="${test_dir#*/}"
     dst_test_dir=${work_dir}/${test_name}
     docker run \
         --network none \
@@ -31,9 +31,10 @@ for test_dir in tests/*; do
         exercism/swift-test-runner \
         "${dst_test_dir}"
 
-    if [ $? -ne 0 ]; then
+    if (( "$?" != 0 )); then
+        printf 'Test "%s" failed!\n' "${test_name}" 
         exit_code=1
     fi
 done
 
-exit ${exit_code}
+exit "${exit_code}"

bin/run-tests.sh

@@ -19,12 +19,12 @@
 
 exit_code=0
 
-for test_dir in tests/*; do
-    bin/run-test.sh $test_dir
+for test_dir in ./tests/*; do
+    [[ -e "${test_dir}" ]] || continue
 
-    if [ $? -ne 0 ]; then
+    if bin/run-test.sh "${test_dir}"; then
         exit_code=1
     fi
 done
 
-exit ${exit_code}
+exit "${exit_code}"

bin/run.sh

@@ -15,34 +15,78 @@
 # Example:
 # ./bin/run.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/
 
+set -eu
+
 # If any required arguments is missing, print the usage and exit
-if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
-    echo "usage: ./bin/run.sh exercise-slug /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/"
+if (( "$#" != 3 )); then
+    printf 'Usage: %s exercise-slug /absolute/path/to/solution/ /absolute/path/to/output/\n' "$0"
     exit 1
 fi
 
 SLUG="$1"
 INPUT_DIR="${2%/}"
 OUTPUT_DIR="${3%/}"
 
-if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then  
+get_target_name() {
+    local section=$1
+    local path
+    path=$(jq -r --arg section "${section}" '.files.[$section][0]' "${CONFIG_FILE}")
+    path=${path%/*}
+    path=${path##*/}
+    echo "${path}"
+}
+
+if [[ "${RUN_IN_DOCKER:-}" == "TRUE" ]]; then
+    # Docker image contains a prebuilt package called TestEnvironment with a ready .build directory 
+    # to build solutions as fast as it possible. To minimize changes in the build graph,
+    # exercise resources are copied inside the TestEnvironment package as if they had always been there.
+
     WORKING_DIR="${PWD}"
-    cp -r "${INPUT_DIR}/." "${WORKING_DIR}"
+    CONFIG_FILE="${INPUT_DIR}/.meta/config.json"
+    SOURCE_TARGET_PATH="${WORKING_DIR}/Sources/TestEnvironment"
+    TEST_TARGET_PATH="${WORKING_DIR}/Tests/TestEnvironmentTests"
+
+    # 1. Replace source files with those of an exercise, saving TestEnvironement paths
+    rm -r -- "${SOURCE_TARGET_PATH}" "${TEST_TARGET_PATH}"
+
+    target_name=$(get_target_name solution)
+    test_target_name=$(get_target_name test)
+
+    # Copying everything from package in case of other targets.
+    cp -r -- "${INPUT_DIR}/Sources/." "${WORKING_DIR}/Sources"
+    cp -r -- "${INPUT_DIR}/Tests/." "${WORKING_DIR}/Tests"
+
+    mv -- "${WORKING_DIR}/Sources/${target_name}" "${SOURCE_TARGET_PATH}"
+    mv -- "${WORKING_DIR}/Tests/${test_target_name}" "${TEST_TARGET_PATH}"
+
+    # 2. Replace @testable import SomeModule with @testable import TestEnvironment
+    change_import="s/@testable import [^ ]\+/@testable import TestEnvironment/g"
+    sed -i -- "${change_import}" "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift
+
+    # 3. Copy Package.swift and rename main & test targets.
+    cp "${INPUT_DIR}/Package.swift" "${WORKING_DIR}"
+    change_target="s/\"${target_name}\"/\"TestEnvironment\"/g;"
+    change_target+="s/\"${test_target_name}\"/\"TestEnvironmentTests\"/g"
+    sed -i -- "${change_target}" "${WORKING_DIR}/Package.swift"
+
 else
     WORKING_DIR=${INPUT_DIR}
 fi
 
 junit_file="${WORKING_DIR}/results-swift-testing.xml"
-spec_file="${WORKING_DIR}/$(jq -r '.files.test[0]' ${WORKING_DIR}/.meta/config.json)"
+spec_file="${INPUT_DIR}/$(jq -r '.files.test[0]' "${INPUT_DIR}/.meta/config.json")"
 capture_file="${OUTPUT_DIR}/capture"
 results_file="${OUTPUT_DIR}/results.json"
 
 touch "${results_file}"
 
 export RUNALL=true
+
+# Builds and runs tests. The resulting artifacts are saved to $junit_file (yes, the file name will change).
 swift test \
     --package-path "${WORKING_DIR}" \
     --xunit-output "${WORKING_DIR}/results.xml" \
-    --skip-update &> "${capture_file}"
+    --skip-update &> "${capture_file}" || true
 
+# Convert test results into the Exercism test runner output format.
 ./bin/TestRunner "${spec_file}" "${junit_file}" "${capture_file}" "${results_file}" "${SLUG}"

changelog.md

@@ -1,3 +1,8 @@
+# 2.0.4
+
+- The WarmUp package has been renamed to TestEnvironment and is now used directly to build exercise solutions.
+- A more aggressive caching strategy is applied when source files are copied into the package.
+
 # 2.0.3
 
 - Add WarmUp package to improve exercise build time.

src/WarmUp/Package.swift → src/TestEnvironment/Package.swift

@@ -3,27 +3,27 @@
 import PackageDescription
 
 let package = Package(
-    name: "WarmUp",
+    name: "TestEnvironment",
     products: [
         .library(
-            name: "WarmUp",
-            targets: ["WarmUp"]
+            name: "TestEnvironment",
+            targets: ["TestEnvironment"]
         ),
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-numerics", from: "1.0.3")
     ],
     targets: [
         .target(
-            name: "WarmUp",
+            name: "TestEnvironment",
             dependencies: [
                 .product(name: "Numerics", package: "swift-numerics")
             ]
         ),
         .testTarget(
-            name: "WarmUpTests",
+            name: "TestEnvironmentTests",
             dependencies: [
-                "WarmUp",
+                "TestEnvironment",
                 .product(name: "Numerics", package: "swift-numerics")
             ]
         )

src/TestEnvironment/README.md

@@ -0,0 +1,36 @@
+# Swift build cache warm-up for faster builds inside docker
+
+## Overview
+
+This package is used in a production environment as a container for building and testing students' solutions.
+It is built and included in a Docker image with a pre-built .build directory, which minimizes changes in the build graph when applying different exercise solutions.
+Exercise resources are copied into the TestEnvironment package as if they had always been part of it.
+
+## The problem with slow build times
+
+Cold swift builds are slow.
+We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts.
+This package is used to build warm-up: it prebuilds the .build folder during docker image creation and is used as an environment for building and testing exercises.
+
+## Why cold builds are slow?
+
+When Swift compiles from a clean state, it spends a huge amount of time on resolving dependencies.
+Imports like Foundation, Numerics, Dispatch, etc. pull in a ton of underlying clang and swift modules like SwiftShims, SwiftGlibc, _Builtin_stddef, etc.
+
+Even if you don’t import them directly, Swift still needs to find, parse, and compile some of them into .build and specifically ModuleCache directory.
+
+## What does this package do?
+
+It simply imports all the common libraries that exercises usually rely on:
+
+```swift
+import Foundation
+import Numerics
+import Testing
+@testable import ModuleName
+```
+
+Then it is built during Docker image creation.
+When a student runs their solution, the code for a particular exercise is copied into the TestEnvironment package.
+Then, the package itself is rebuilt with minor changes, such as replacing code in Source and Tests files.
+See `bin/run.sh` for more details on which parts of an exercise are copied into docker image.

src/WarmUp/Tests/WarmUpTests/WarmUpTests.swift → src/TestEnvironment/Tests/TestEnvironmentTests/TestEnvironmentTests.swift

@@ -2,7 +2,7 @@ import Foundation
 import Numerics
 import Testing
 
-@testable import WarmUp
+@testable import TestEnvironment
 
 let RUNALL = Bool(ProcessInfo.processInfo.environment["RUNALL", default: "false"]) ?? false