diff --git a/Dockerfile b/Dockerfile index 6004f4c..c74c47f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Stage 1: Precompile TestRunner and Module Cache +# Stage 1: Precompile TestRunner FROM swift:6.1.2 AS builder RUN swift --version @@ -7,21 +7,19 @@ WORKDIR /TestRunner COPY src/TestRunner . RUN swift build --configuration release -# Build WarmUp package -# Build directory and final working paths should be equal for reuse of ModuleCache. -WORKDIR /opt/test-runner -COPY src/WarmUp . -RUN swift build --build-tests - # Stage 2: Prepare docker container image FROM swift:6.1.2 RUN apt-get update && apt-get install -y jq -WORKDIR /opt/test-runner/ +WORKDIR /opt/test-runner + +# Build TestEnvironment package +# Build directory and final working paths should be equal for reuse of ModuleCache. +COPY src/TestEnvironment . +RUN swift build --build-tests + COPY bin/run.sh bin/run-test.sh bin/ COPY --from=builder /TestRunner/.build/release/TestRunner bin/ -COPY --from=builder /opt/test-runner/.build .build -COPY --from=builder /opt/test-runner/Package.resolved Package.resolved ENV RUN_IN_DOCKER=TRUE diff --git a/bin/benchmark-in-docker.sh b/bin/benchmark-in-docker.sh index a980225..a0591e6 100755 --- a/bin/benchmark-in-docker.sh +++ b/bin/benchmark-in-docker.sh @@ -13,7 +13,7 @@ # Example: # ./bin/benchmark-in-docker.sh -set -eo pipefail +set -euo pipefail die() { echo "$*" >&2; exit 1; } @@ -26,14 +26,15 @@ required_tool docker required_tool hyperfine # Pre-build the Docker image -if [[ -z "${SKIP_DOCKER_BUILD}" ]]; then +if [[ -z "${SKIP_DOCKER_BUILD:-}" ]]; then docker build --rm -t exercism/swift-test-runner . else - echo "Skipping docker build because SKIP_DOCKER_BUILD is set." + printf "Skipping docker build because SKIP_DOCKER_BUILD is set.\n" fi -CURRENT_PATH=${PWD} +CURRENT_PATH="${PWD}" +test_list=$(find tests -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | paste -sd "," -) hyperfine \ - --parameter-list slug $(find tests -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | paste -sd ",") \ - "SKIP_DOCKER_BUILD=true bin/run-in-docker.sh {slug} $CURRENT_PATH/tests/{slug} $CURRENT_PATH/tests/{slug}" + --parameter-list slug "${test_list}" \ + "SKIP_DOCKER_BUILD=true bin/run-in-docker.sh {slug} ${CURRENT_PATH}/tests/{slug} ${CURRENT_PATH}/tests/{slug}" diff --git a/bin/benchmark.sh b/bin/benchmark.sh index d29b6ea..6c6e8f4 100755 --- a/bin/benchmark.sh +++ b/bin/benchmark.sh @@ -10,7 +10,7 @@ # Example: # ./bin/benchmark.sh -set -eo pipefail +set -euo pipefail die() { echo "$*" >&2; exit 1; } @@ -21,7 +21,8 @@ required_tool() { required_tool hyperfine +test_list=$(find tests -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | paste -sd "," -) hyperfine \ - --parameter-list slug $(find tests -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | paste -sd ",") \ + --parameter-list slug "${test_list}" \ --prepare 'git clean -xdfq tests/{slug}' \ - 'bin/run.sh {slug} tests/{slug} tests/{slug}' + './bin/run.sh {slug} tests/{slug} tests/{slug}' diff --git a/bin/build-test-runner.sh b/bin/build-test-runner.sh index 4b9c52f..49d4c0f 100755 --- a/bin/build-test-runner.sh +++ b/bin/build-test-runner.sh @@ -9,14 +9,16 @@ # Example: # ./bin/build-test-runner.sh +set -euo pipefail + BIN_DIR="bin" BUILD_DIR="src/TestRunner" RELEASE_DIR=".build/release" # Build the test runner file -cd "$BUILD_DIR" +cd "${BUILD_DIR}" swift build --configuration release cd - # Copy generated file to bin dir -cp "$BUILD_DIR/$RELEASE_DIR/TestRunner" "$BIN_DIR"/ +cp "${BUILD_DIR}/${RELEASE_DIR}/TestRunner" "${BIN_DIR}"/ diff --git a/bin/run-in-docker.sh b/bin/run-in-docker.sh index d37598b..02c45f4 100755 --- a/bin/run-in-docker.sh +++ b/bin/run-in-docker.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -set -e # Synopsis: # Run the test runner on a solution using the test runner Docker image. @@ -18,8 +17,11 @@ set -e # ./bin/run-in-docker.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/ # If any required arguments is missing, print the usage and exit -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "usage: ./bin/run-in-docker.sh exercise-slug /absolute/path/to/solution/folder/ /absolute/path/to/output/directory/" + +set -euo pipefail + +if (( "$#" != 3 )); then + printf 'usage: %s exercise-slug /absolute/path/to/solution/folder/ /absolute/path/to/output/directory/\n' "$0" exit 1 fi @@ -31,10 +33,10 @@ OUTPUT_DIR="${3%/}" mkdir -p "${OUTPUT_DIR}" # Pre-build the Docker image -if [[ -z "${SKIP_DOCKER_BUILD}" ]]; then +if [[ -z "${SKIP_DOCKER_BUILD:-}" ]]; then docker build --rm -t exercism/swift-test-runner . else - echo "Skipping docker build because SKIP_DOCKER_BUILD is set." + printf "Skipping docker build because SKIP_DOCKER_BUILD is set.\n" fi # Run image passing the arguments @@ -43,4 +45,4 @@ docker run \ --mount type=bind,src="${INPUT_DIR}",dst=/solution \ --mount type=bind,src="${OUTPUT_DIR}",dst=/output \ --mount type=volume,dst=/tmp \ - exercism/swift-test-runner $SLUG /solution/ /output/ + exercism/swift-test-runner "${SLUG}" /solution/ /output/ diff --git a/bin/run-test.sh b/bin/run-test.sh index 4e02e10..0f025ec 100755 --- a/bin/run-test.sh +++ b/bin/run-test.sh @@ -14,12 +14,22 @@ # Example: # ./bin/run-test.sh /absolute/path/to/tests/compile-error/ +set -euo pipefail + # If any required arguments is missing, print the usage and exit -if [ -z "$1" ]; then - echo "usage: ./bin/run-test.sh /absolute/path/to/test/folder/" +if (( "$#" != 1 )); then + printf 'Usage: %s /absolute/path/to/test/folder/\n' "$0" exit 1 fi +sed_i() { + if [[ "$(uname)" == "Darwin" ]]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + test_dir="$1" test_dir_name=$(basename "${test_dir}") @@ -30,11 +40,7 @@ expected_results_file_path="${test_dir_path}/expected_results.json" bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}" # Normalize the results file -sed -i -e "s~\\\/~/~g" -e "s~${test_dir_path}~/solution~g" "${results_file_path}" +sed_i -e "s~\\\/~/~g" -e "s~${test_dir_path}~/solution~g" "${results_file_path}" -echo "${test_dir_name}: comparing results.json to expected_results.json" +printf '%s: comparing results.json to expected_results.json\n' "${test_dir_name}" diff "${results_file_path}" "${expected_results_file_path}" - -if [ $? -ne 0 ]; then - echo "\"${test_dir_name}\" failed!" -fi \ No newline at end of file diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh index 6a4e527..e8b0227 100755 --- a/bin/run-tests-in-docker.sh +++ b/bin/run-tests-in-docker.sh @@ -20,7 +20,7 @@ exit_code=0 # Iterate over all test directories work_dir=/opt/test-runner/ for test_dir in tests/*; do - test_name=$(basename $test_dir) + test_name="${test_dir#*/}" dst_test_dir=${work_dir}/${test_name} docker run \ --network none \ @@ -31,9 +31,10 @@ for test_dir in tests/*; do exercism/swift-test-runner \ "${dst_test_dir}" - if [ $? -ne 0 ]; then + if (( "$?" != 0 )); then + printf 'Test "%s" failed!\n' "${test_name}" exit_code=1 fi done -exit ${exit_code} \ No newline at end of file +exit "${exit_code}" diff --git a/bin/run-tests.sh b/bin/run-tests.sh index 8e341d8..fc48c84 100755 --- a/bin/run-tests.sh +++ b/bin/run-tests.sh @@ -19,12 +19,12 @@ exit_code=0 -for test_dir in tests/*; do - bin/run-test.sh $test_dir +for test_dir in ./tests/*; do + [[ -e "${test_dir}" ]] || continue - if [ $? -ne 0 ]; then + if bin/run-test.sh "${test_dir}"; then exit_code=1 fi done -exit ${exit_code} \ No newline at end of file +exit "${exit_code}" diff --git a/bin/run.sh b/bin/run.sh index 45e53ba..84ef9c2 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -15,9 +15,11 @@ # Example: # ./bin/run.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/ +set -eu + # If any required arguments is missing, print the usage and exit -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "usage: ./bin/run.sh exercise-slug /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/" +if (( "$#" != 3 )); then + printf 'Usage: %s exercise-slug /absolute/path/to/solution/ /absolute/path/to/output/\n' "$0" exit 1 fi @@ -25,24 +27,66 @@ SLUG="$1" INPUT_DIR="${2%/}" OUTPUT_DIR="${3%/}" -if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then +get_target_name() { + local section=$1 + local path + path=$(jq -r --arg section "${section}" '.files.[$section][0]' "${CONFIG_FILE}") + path=${path%/*} + path=${path##*/} + echo "${path}" +} + +if [[ "${RUN_IN_DOCKER:-}" == "TRUE" ]]; then + # Docker image contains a prebuilt package called TestEnvironment with a ready .build directory + # to build solutions as fast as it possible. To minimize changes in the build graph, + # exercise resources are copied inside the TestEnvironment package as if they had always been there. + WORKING_DIR="${PWD}" - cp -r "${INPUT_DIR}/." "${WORKING_DIR}" + CONFIG_FILE="${INPUT_DIR}/.meta/config.json" + SOURCE_TARGET_PATH="${WORKING_DIR}/Sources/TestEnvironment" + TEST_TARGET_PATH="${WORKING_DIR}/Tests/TestEnvironmentTests" + + # 1. Replace source files with those of an exercise, saving TestEnvironement paths + rm -r -- "${SOURCE_TARGET_PATH}" "${TEST_TARGET_PATH}" + + target_name=$(get_target_name solution) + test_target_name=$(get_target_name test) + + # Copying everything from package in case of other targets. + cp -r -- "${INPUT_DIR}/Sources/." "${WORKING_DIR}/Sources" + cp -r -- "${INPUT_DIR}/Tests/." "${WORKING_DIR}/Tests" + + mv -- "${WORKING_DIR}/Sources/${target_name}" "${SOURCE_TARGET_PATH}" + mv -- "${WORKING_DIR}/Tests/${test_target_name}" "${TEST_TARGET_PATH}" + + # 2. Replace @testable import SomeModule with @testable import TestEnvironment + change_import="s/@testable import [^ ]\+/@testable import TestEnvironment/g" + sed -i -- "${change_import}" "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift + + # 3. Copy Package.swift and rename main & test targets. + cp "${INPUT_DIR}/Package.swift" "${WORKING_DIR}" + change_target="s/\"${target_name}\"/\"TestEnvironment\"/g;" + change_target+="s/\"${test_target_name}\"/\"TestEnvironmentTests\"/g" + sed -i -- "${change_target}" "${WORKING_DIR}/Package.swift" + else WORKING_DIR=${INPUT_DIR} fi junit_file="${WORKING_DIR}/results-swift-testing.xml" -spec_file="${WORKING_DIR}/$(jq -r '.files.test[0]' ${WORKING_DIR}/.meta/config.json)" +spec_file="${INPUT_DIR}/$(jq -r '.files.test[0]' "${INPUT_DIR}/.meta/config.json")" capture_file="${OUTPUT_DIR}/capture" results_file="${OUTPUT_DIR}/results.json" touch "${results_file}" export RUNALL=true + +# Builds and runs tests. The resulting artifacts are saved to $junit_file (yes, the file name will change). swift test \ --package-path "${WORKING_DIR}" \ --xunit-output "${WORKING_DIR}/results.xml" \ - --skip-update &> "${capture_file}" + --skip-update &> "${capture_file}" || true +# Convert test results into the Exercism test runner output format. ./bin/TestRunner "${spec_file}" "${junit_file}" "${capture_file}" "${results_file}" "${SLUG}" diff --git a/changelog.md b/changelog.md index 36cade6..1374441 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,8 @@ +# 2.0.4 + +- The WarmUp package has been renamed to TestEnvironment and is now used directly to build exercise solutions. +- A more aggressive caching strategy is applied when source files are copied into the package. + # 2.0.3 - Add WarmUp package to improve exercise build time. diff --git a/src/WarmUp/Package.swift b/src/TestEnvironment/Package.swift similarity index 71% rename from src/WarmUp/Package.swift rename to src/TestEnvironment/Package.swift index cd40e63..c4a69af 100644 --- a/src/WarmUp/Package.swift +++ b/src/TestEnvironment/Package.swift @@ -3,11 +3,11 @@ import PackageDescription let package = Package( - name: "WarmUp", + name: "TestEnvironment", products: [ .library( - name: "WarmUp", - targets: ["WarmUp"] + name: "TestEnvironment", + targets: ["TestEnvironment"] ), ], dependencies: [ @@ -15,15 +15,15 @@ let package = Package( ], targets: [ .target( - name: "WarmUp", + name: "TestEnvironment", dependencies: [ .product(name: "Numerics", package: "swift-numerics") ] ), .testTarget( - name: "WarmUpTests", + name: "TestEnvironmentTests", dependencies: [ - "WarmUp", + "TestEnvironment", .product(name: "Numerics", package: "swift-numerics") ] ) diff --git a/src/TestEnvironment/README.md b/src/TestEnvironment/README.md new file mode 100644 index 0000000..9b5a527 --- /dev/null +++ b/src/TestEnvironment/README.md @@ -0,0 +1,36 @@ +# Swift build cache warm-up for faster builds inside docker + +## Overview + +This package is used in a production environment as a container for building and testing students' solutions. +It is built and included in a Docker image with a pre-built .build directory, which minimizes changes in the build graph when applying different exercise solutions. +Exercise resources are copied into the TestEnvironment package as if they had always been part of it. + +## The problem with slow build times + +Cold swift builds are slow. +We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts. +This package is used to build warm-up: it prebuilds the .build folder during docker image creation and is used as an environment for building and testing exercises. + +## Why cold builds are slow? + +When Swift compiles from a clean state, it spends a huge amount of time on resolving dependencies. +Imports like Foundation, Numerics, Dispatch, etc. pull in a ton of underlying clang and swift modules like SwiftShims, SwiftGlibc, _Builtin_stddef, etc. + +Even if you don’t import them directly, Swift still needs to find, parse, and compile some of them into .build and specifically ModuleCache directory. + +## What does this package do? + +It simply imports all the common libraries that exercises usually rely on: + +```swift +import Foundation +import Numerics +import Testing +@testable import ModuleName +``` + +Then it is built during Docker image creation. +When a student runs their solution, the code for a particular exercise is copied into the TestEnvironment package. +Then, the package itself is rebuilt with minor changes, such as replacing code in Source and Tests files. +See `bin/run.sh` for more details on which parts of an exercise are copied into docker image. \ No newline at end of file diff --git a/src/WarmUp/Sources/Warmup/WarmUp.swift b/src/TestEnvironment/Sources/TestEnvironment/TestEnvironment.swift similarity index 100% rename from src/WarmUp/Sources/Warmup/WarmUp.swift rename to src/TestEnvironment/Sources/TestEnvironment/TestEnvironment.swift diff --git a/src/WarmUp/Tests/WarmUpTests/WarmUpTests.swift b/src/TestEnvironment/Tests/TestEnvironmentTests/TestEnvironmentTests.swift similarity index 88% rename from src/WarmUp/Tests/WarmUpTests/WarmUpTests.swift rename to src/TestEnvironment/Tests/TestEnvironmentTests/TestEnvironmentTests.swift index 6730c46..d6b0a22 100644 --- a/src/WarmUp/Tests/WarmUpTests/WarmUpTests.swift +++ b/src/TestEnvironment/Tests/TestEnvironmentTests/TestEnvironmentTests.swift @@ -2,7 +2,7 @@ import Foundation import Numerics import Testing -@testable import WarmUp +@testable import TestEnvironment let RUNALL = Bool(ProcessInfo.processInfo.environment["RUNALL", default: "false"]) ?? false diff --git a/src/TestRunner/Package.swift b/src/TestRunner/Package.swift index 83d6e5a..35a02f8 100644 --- a/src/TestRunner/Package.swift +++ b/src/TestRunner/Package.swift @@ -5,6 +5,7 @@ import PackageDescription let package = Package( name: "TestRunner", + platforms: [.macOS(.v10_15)], dependencies: [ // Dependencies declare other packages that this package depends on. // .package(url: /* package url */, from: "1.0.0"), diff --git a/src/TestRunner/Sources/TestRunner/main.swift b/src/TestRunner/Sources/TestRunner/main.swift index e431f35..162c574 100644 --- a/src/TestRunner/Sources/TestRunner/main.swift +++ b/src/TestRunner/Sources/TestRunner/main.swift @@ -1,5 +1,7 @@ import Foundation +#if os(Linux) import FoundationXML +#endif import SwiftParser import SwiftSyntax diff --git a/src/TestRunner/Sources/TestRunner/xmlParser.swift b/src/TestRunner/Sources/TestRunner/xmlParser.swift index 2f131b3..a6353c9 100644 --- a/src/TestRunner/Sources/TestRunner/xmlParser.swift +++ b/src/TestRunner/Sources/TestRunner/xmlParser.swift @@ -1,5 +1,7 @@ import Foundation +#if os(Linux) import FoundationXML +#endif class MyXMLParserDelegate: NSObject, XMLParserDelegate { var counter = 0 diff --git a/src/WarmUp/README.md b/src/WarmUp/README.md deleted file mode 100644 index 1e0e277..0000000 --- a/src/WarmUp/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Swift build cache warm-up for faster builds inside docker -## Overview - -Cold swift builds are slow. We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts. This package is used to build warm-up: it prebuilds the .build folder during docker image creation. - -## Why cold builds are slow? - -When Swift compiles from a clean state, it spends a huge amount of time on resolving dependencies. Imports like Foundation, Numerics, Dispatch, etc. pull in a ton of underlying clang and swift modules like SwiftShims, SwiftGlibc, _Builtin_stddef, etc. - -Even if you don’t import them directly, Swift still needs to find, parse, and compile some of them into .build and specifically ModuleCache directory. - -## What does this package do? - -It simply imports all the common libraries that exercises usually rely on: - -```swift -import Foundation -import Numerics -import _Concurrency -``` - -Then it gets built during Docker image creation. So when a student runs their solution later all the dependencies are already ready. \ No newline at end of file diff --git a/tests/compile-error/expected_results.json b/tests/compile-error/expected_results.json index d76639b..56de121 100644 --- a/tests/compile-error/expected_results.json +++ b/tests/compile-error/expected_results.json @@ -1,5 +1,5 @@ { - "message" : "/opt/test-runner/Sources/CompileError/CompileError.swift:2:12: error: unexpected non-void return value in void function1 | func sum(_ x: Int, _ y: Int) {2 | return x + y | |- error: unexpected non-void return value in void function | `- note: did you mean to add a return type?3 | }4 | error: fatalError", + "message" : "/opt/test-runner/Sources/TestEnvironment/CompileError.swift:2:12: error: unexpected non-void return value in void function1 | func sum(_ x: Int, _ y: Int) {2 | return x + y | |- error: unexpected non-void return value in void function | `- note: did you mean to add a return type?3 | }4 | error: fatalError", "status" : "error", "tests" : [ diff --git a/tests/tests-using-library/Sources/UsingLibrary/UsingLibrary.swift b/tests/tests-using-library/Sources/UsingLibrary/UsingLibrary.swift index 372a590..7f076ec 100644 --- a/tests/tests-using-library/Sources/UsingLibrary/UsingLibrary.swift +++ b/tests/tests-using-library/Sources/UsingLibrary/UsingLibrary.swift @@ -1,3 +1,6 @@ +import Foundation +import Numerics + enum TestError: Error { case testError(String) } diff --git a/tests/tests-using-library/Tests/UsingLibraryTests/UsingLibraryTests.swift b/tests/tests-using-library/Tests/UsingLibraryTests/UsingLibraryTests.swift index 33c8b38..781fa53 100644 --- a/tests/tests-using-library/Tests/UsingLibraryTests/UsingLibraryTests.swift +++ b/tests/tests-using-library/Tests/UsingLibraryTests/UsingLibraryTests.swift @@ -1,3 +1,4 @@ +import Foundation import Testing import Numerics