From 7c3222bbf0953fa194130c2e8398a3c041356448 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Tue, 5 Aug 2025 01:47:56 +0300 Subject: [PATCH 01/27] Fix mistake with env RUN_IN_DOCKER --- bin/benchmark-in-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/benchmark-in-docker.sh b/bin/benchmark-in-docker.sh index a980225..820662e 100755 --- a/bin/benchmark-in-docker.sh +++ b/bin/benchmark-in-docker.sh @@ -36,4 +36,4 @@ CURRENT_PATH=${PWD} hyperfine \ --parameter-list slug $(find tests -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | paste -sd ",") \ - "SKIP_DOCKER_BUILD=true bin/run-in-docker.sh {slug} $CURRENT_PATH/tests/{slug} $CURRENT_PATH/tests/{slug}" + "bin/run-in-docker.sh {slug} $CURRENT_PATH/tests/{slug} $CURRENT_PATH/tests/{slug}" From d39e28b3baa4a077a4a4b92e77994baf4474aa9d Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sun, 10 Aug 2025 18:38:53 +0300 Subject: [PATCH 02/27] PoC: could be faster --- Dockerfile | 20 +++++++++---------- bin/run.sh | 19 +++++++++++++++++- .../Sources/{Warmup => WarmUp}/WarmUp.swift | 0 tests/compile-error/expected_results.json | 2 +- 4 files changed, 28 insertions(+), 13 deletions(-) rename src/WarmUp/Sources/{Warmup => WarmUp}/WarmUp.swift (100%) diff --git a/Dockerfile b/Dockerfile index 6004f4c..aadbc4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,25 @@ -# Stage 1: Precompile TestRunner and Module Cache +# Stage 1: Precompile TestRunner FROM swift:6.1.2 AS builder RUN swift --version # Build TestRunner executable WORKDIR /TestRunner COPY src/TestRunner . -RUN swift build --configuration release +RUN swift build --configuration debug + +# Stage 2: Prepare docker container image +FROM swift:6.1.2 +RUN apt-get update && apt-get install -y jq + +WORKDIR /opt/test-runner # Build WarmUp package # Build directory and final working paths should be equal for reuse of ModuleCache. -WORKDIR /opt/test-runner COPY src/WarmUp . RUN swift build --build-tests -# Stage 2: Prepare docker container image -FROM swift:6.1.2 -RUN apt-get update && apt-get install -y jq - -WORKDIR /opt/test-runner/ COPY bin/run.sh bin/run-test.sh bin/ -COPY --from=builder /TestRunner/.build/release/TestRunner bin/ -COPY --from=builder /opt/test-runner/.build .build -COPY --from=builder /opt/test-runner/Package.resolved Package.resolved +COPY --from=builder /TestRunner/.build/debug/TestRunner bin/ ENV RUN_IN_DOCKER=TRUE diff --git a/bin/run.sh b/bin/run.sh index 45e53ba..947311c 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -27,7 +27,24 @@ OUTPUT_DIR="${3%/}" if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then WORKING_DIR="${PWD}" - cp -r "${INPUT_DIR}/." "${WORKING_DIR}" + + # 1. Modify Package.swift to add new dependencies + # 2. Copy source and destination files + cp -r "${INPUT_DIR}/.meta" "${WORKING_DIR}" + + find "${WORKING_DIR}/Sources/WarmUp" -type f -delete + find "${WORKING_DIR}/Tests/WarmUpTests" -type f -delete + + find "${INPUT_DIR}/Sources" -name '*.swift' -exec cp {} "${WORKING_DIR}/Sources/WarmUp/" \; + find "${INPUT_DIR}/Tests" -name '*.swift' -exec cp {} "${WORKING_DIR}/Tests/WarmUpTests/" \; + + ls -al "${WORKING_DIR}/Tests/WarmUpTests/" + + filename=$(jq -r '.files.test[0] | split("/") | last' ${INPUT_DIR}/.meta/config.json) + destination_path="/Tests/WarmUpTests/${filename}" + jq --arg fname "$destination_path" '.files.test[0] = $fname' ${INPUT_DIR}/.meta/config.json > tmp.json && mv tmp.json ${WORKING_DIR}/.meta/config.json + + sed -i 's/@testable import [^ ]\+/@testable import WarmUp/g' "${WORKING_DIR}/Tests/WarmUpTests"/*.swift else WORKING_DIR=${INPUT_DIR} fi diff --git a/src/WarmUp/Sources/Warmup/WarmUp.swift b/src/WarmUp/Sources/WarmUp/WarmUp.swift similarity index 100% rename from src/WarmUp/Sources/Warmup/WarmUp.swift rename to src/WarmUp/Sources/WarmUp/WarmUp.swift diff --git a/tests/compile-error/expected_results.json b/tests/compile-error/expected_results.json index d76639b..fcef595 100644 --- a/tests/compile-error/expected_results.json +++ b/tests/compile-error/expected_results.json @@ -1,5 +1,5 @@ { - "message" : "/opt/test-runner/Sources/CompileError/CompileError.swift:2:12: error: unexpected non-void return value in void function1 | func sum(_ x: Int, _ y: Int) {2 | return x + y | |- error: unexpected non-void return value in void function | `- note: did you mean to add a return type?3 | }4 | error: fatalError", + "message" : "/opt/test-runner/Sources/WarmUp/CompileError.swift:2:12: error: unexpected non-void return value in void function1 | func sum(_ x: Int, _ y: Int) {2 | return x + y | |- error: unexpected non-void return value in void function | `- note: did you mean to add a return type?3 | }4 | error: fatalError", "status" : "error", "tests" : [ From 6a45499c4c02677ae059a65d24b149e70d7cfe47 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sun, 10 Aug 2025 21:06:11 +0300 Subject: [PATCH 03/27] Review fixes --- bin/run.sh | 29 +++++++++++++++++------------ src/WarmUp/README.md | 9 ++++++--- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 947311c..3720ff5 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -26,31 +26,36 @@ INPUT_DIR="${2%/}" OUTPUT_DIR="${3%/}" if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then - WORKING_DIR="${PWD}" + # Docker image contains a prebuilt package called WarmUp with a ready .build directory. + # To minimize changes in the build graph, exercise resources are copied inside the WarmUp package + # as if they had always been there. - # 1. Modify Package.swift to add new dependencies - # 2. Copy source and destination files - cp -r "${INPUT_DIR}/.meta" "${WORKING_DIR}" + WORKING_DIR="${PWD}" + desc_file="${WORKING_DIR}/package_desc.json" + swift package dump-package --package-path ${INPUT_DIR} > "$desc_file" + # 1. Replace source files with those of an exercise find "${WORKING_DIR}/Sources/WarmUp" -type f -delete find "${WORKING_DIR}/Tests/WarmUpTests" -type f -delete + target_name=$(jq -r '.targets[] | select(.type=="regular") | .name' "$desc_file" | head -1) + test_target_name=$(jq -r '.targets[] | select(.type=="test") | .name' "$desc_file" | head -1) - find "${INPUT_DIR}/Sources" -name '*.swift' -exec cp {} "${WORKING_DIR}/Sources/WarmUp/" \; - find "${INPUT_DIR}/Tests" -name '*.swift' -exec cp {} "${WORKING_DIR}/Tests/WarmUpTests/" \; + cp -rf "${INPUT_DIR}/Sources/${target_name}/." "${WORKING_DIR}/Sources/WarmUp/" + cp -rf "${INPUT_DIR}/Tests/${test_target_name}/." "${WORKING_DIR}/Tests/WarmUpTests/" - ls -al "${WORKING_DIR}/Tests/WarmUpTests/" + # 2. Replace @testable import SomeModule with @testable import WarmUp + sed -i 's/@testable import [^ ]\+/@testable import WarmUp/g' "${WORKING_DIR}/Tests/WarmUpTests"/*.swift - filename=$(jq -r '.files.test[0] | split("/") | last' ${INPUT_DIR}/.meta/config.json) - destination_path="/Tests/WarmUpTests/${filename}" - jq --arg fname "$destination_path" '.files.test[0] = $fname' ${INPUT_DIR}/.meta/config.json > tmp.json && mv tmp.json ${WORKING_DIR}/.meta/config.json + # 3. Copy and modify Package.swift + cp "${INPUT_DIR}/Package.swift" "${WORKING_DIR}" + sed -i "s/${target_name}/WarmUp/g" "${WORKING_DIR}/Package.swift" - sed -i 's/@testable import [^ ]\+/@testable import WarmUp/g' "${WORKING_DIR}/Tests/WarmUpTests"/*.swift else WORKING_DIR=${INPUT_DIR} fi junit_file="${WORKING_DIR}/results-swift-testing.xml" -spec_file="${WORKING_DIR}/$(jq -r '.files.test[0]' ${WORKING_DIR}/.meta/config.json)" +spec_file="${INPUT_DIR}/$(jq -r '.files.test[0]' ${INPUT_DIR}/.meta/config.json)" capture_file="${OUTPUT_DIR}/capture" results_file="${OUTPUT_DIR}/results.json" diff --git a/src/WarmUp/README.md b/src/WarmUp/README.md index 1e0e277..5e82089 100644 --- a/src/WarmUp/README.md +++ b/src/WarmUp/README.md @@ -1,7 +1,7 @@ # Swift build cache warm-up for faster builds inside docker ## Overview -Cold swift builds are slow. We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts. This package is used to build warm-up: it prebuilds the .build folder during docker image creation. +Cold swift builds are slow. We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts. This package is used to build warm-up: it prebuilds the .build folder during docker image creation and is used as an environment for building exercises. ## Why cold builds are slow? @@ -16,7 +16,10 @@ It simply imports all the common libraries that exercises usually rely on: ```swift import Foundation import Numerics -import _Concurrency +import Testing +@testable import ModuleName ``` -Then it gets built during Docker image creation. So when a student runs their solution later all the dependencies are already ready. \ No newline at end of file +Then it gets built during Docker image creation. When a student runs their solution later the code for a particular exercise is copied into WarmUp package and the package itself is rebuilt with minor difference, like you've just replace code in files and rebuilt the same package. + +Then it is built during Docker image creation. Later, when a student runs their solution, the code for a particular exercise is copied into the WarmUp package. Theb, the package itself is rebuilt with minor changes, such as replacing code in Source and Tests files. See `bin/run.sh` for more details on which parts of an exercise are copied into docker image. \ No newline at end of file From 59b2b8bee47ef9e8726b98edb8a8c937f10dac1d Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sat, 13 Sep 2025 17:40:31 +0300 Subject: [PATCH 04/27] Add --enable-experimental-prebuilts flag --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index aadbc4f..9196ba7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ RUN swift --version # Build TestRunner executable WORKDIR /TestRunner COPY src/TestRunner . -RUN swift build --configuration debug +RUN swift build --configuration release --enable-experimental-prebuilts # Stage 2: Prepare docker container image FROM swift:6.1.2 @@ -19,7 +19,7 @@ COPY src/WarmUp . RUN swift build --build-tests COPY bin/run.sh bin/run-test.sh bin/ -COPY --from=builder /TestRunner/.build/debug/TestRunner bin/ +COPY --from=builder /TestRunner/.build/release/TestRunner bin/ ENV RUN_IN_DOCKER=TRUE From 9f57f809006d44a269a1e557bb416324193a9041 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sat, 13 Sep 2025 17:41:11 +0300 Subject: [PATCH 05/27] Fix typo in warmup readme --- src/WarmUp/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WarmUp/README.md b/src/WarmUp/README.md index 5e82089..9bd03d8 100644 --- a/src/WarmUp/README.md +++ b/src/WarmUp/README.md @@ -1,7 +1,7 @@ # Swift build cache warm-up for faster builds inside docker ## Overview -Cold swift builds are slow. We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts. This package is used to build warm-up: it prebuilds the .build folder during docker image creation and is used as an environment for building exercises. +Cold swift builds are slow. We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts. This package is used to build warm-up: it prebuilds the .build folder during docker image creation and is used as an environment for building and testing exercises. ## Why cold builds are slow? From e2a2de83d3809dea3823bcf7c56b4f3e51f82288 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sat, 13 Sep 2025 18:02:32 +0300 Subject: [PATCH 06/27] Remove unwanted changes --- bin/benchmark-in-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/benchmark-in-docker.sh b/bin/benchmark-in-docker.sh index 820662e..a980225 100755 --- a/bin/benchmark-in-docker.sh +++ b/bin/benchmark-in-docker.sh @@ -36,4 +36,4 @@ CURRENT_PATH=${PWD} hyperfine \ --parameter-list slug $(find tests -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | paste -sd ",") \ - "bin/run-in-docker.sh {slug} $CURRENT_PATH/tests/{slug} $CURRENT_PATH/tests/{slug}" + "SKIP_DOCKER_BUILD=true bin/run-in-docker.sh {slug} $CURRENT_PATH/tests/{slug} $CURRENT_PATH/tests/{slug}" From 898c9ad69cb20e43ea057bdec7d90594473eb616 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sat, 13 Sep 2025 18:04:03 +0300 Subject: [PATCH 07/27] Add change log --- changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.md b/changelog.md index 36cade6..1374441 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,8 @@ +# 2.0.4 + +- The WarmUp package has been renamed to TestEnvironment and is now used directly to build exercise solutions. +- A more aggressive caching strategy is applied when source files are copied into the package. + # 2.0.3 - Add WarmUp package to improve exercise build time. From 4152f5b4b4399c2f1751cb4d6348edaf2cee54bd Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sat, 13 Sep 2025 18:05:06 +0300 Subject: [PATCH 08/27] review fixes. Rename WarmUp to TestEnvironment to reduce misleading paths in errors. --- Dockerfile | 4 ++-- bin/run.sh | 24 +++++++++---------- src/{WarmUp => TestEnvironment}/Package.swift | 12 +++++----- src/{WarmUp => TestEnvironment}/README.md | 8 +++++-- .../Sources/WarmUp/TestEnvironment.swift} | 0 .../WarmUpTests/TestEnvironmentTests.swift} | 2 +- tests/compile-error/expected_results.json | 2 +- 7 files changed, 28 insertions(+), 24 deletions(-) rename src/{WarmUp => TestEnvironment}/Package.swift (71%) rename src/{WarmUp => TestEnvironment}/README.md (60%) rename src/{WarmUp/Sources/WarmUp/WarmUp.swift => TestEnvironment/Sources/WarmUp/TestEnvironment.swift} (100%) rename src/{WarmUp/Tests/WarmUpTests/WarmUpTests.swift => TestEnvironment/Tests/WarmUpTests/TestEnvironmentTests.swift} (88%) diff --git a/Dockerfile b/Dockerfile index 9196ba7..52f5023 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,9 +13,9 @@ RUN apt-get update && apt-get install -y jq WORKDIR /opt/test-runner -# Build WarmUp package +# Build TestEnvironment package # Build directory and final working paths should be equal for reuse of ModuleCache. -COPY src/WarmUp . +COPY src/TestEnvironment . RUN swift build --build-tests COPY bin/run.sh bin/run-test.sh bin/ diff --git a/bin/run.sh b/bin/run.sh index 3720ff5..35f7b34 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -25,30 +25,30 @@ SLUG="$1" INPUT_DIR="${2%/}" OUTPUT_DIR="${3%/}" -if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then - # Docker image contains a prebuilt package called WarmUp with a ready .build directory. - # To minimize changes in the build graph, exercise resources are copied inside the WarmUp package - # as if they had always been there. +if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then + # Docker image contains a prebuilt package called TestEnvironment with a ready .build directory + # to build solutions as fast as it possible. To minimize changes in the build graph, + # exercise resources are copied inside the TestEnvironment package as if they had always been there. WORKING_DIR="${PWD}" desc_file="${WORKING_DIR}/package_desc.json" swift package dump-package --package-path ${INPUT_DIR} > "$desc_file" # 1. Replace source files with those of an exercise - find "${WORKING_DIR}/Sources/WarmUp" -type f -delete - find "${WORKING_DIR}/Tests/WarmUpTests" -type f -delete + find "${WORKING_DIR}/Sources/TestEnvironment" -type f -delete target_name=$(jq -r '.targets[] | select(.type=="regular") | .name' "$desc_file" | head -1) - test_target_name=$(jq -r '.targets[] | select(.type=="test") | .name' "$desc_file" | head -1) + cp -rf "${INPUT_DIR}/Sources/${target_name}/." "${WORKING_DIR}/Sources/TestEnvironment/" - cp -rf "${INPUT_DIR}/Sources/${target_name}/." "${WORKING_DIR}/Sources/WarmUp/" - cp -rf "${INPUT_DIR}/Tests/${test_target_name}/." "${WORKING_DIR}/Tests/WarmUpTests/" + find "${WORKING_DIR}/Tests/TestEnvironmentTests" -type f -delete + test_target_name=$(jq -r '.targets[] | select(.type=="test") | .name' "$desc_file" | head -1) + cp -rf "${INPUT_DIR}/Tests/${test_target_name}/." "${WORKING_DIR}/Tests/TestEnvironmentTests/" - # 2. Replace @testable import SomeModule with @testable import WarmUp - sed -i 's/@testable import [^ ]\+/@testable import WarmUp/g' "${WORKING_DIR}/Tests/WarmUpTests"/*.swift + # 2. Replace @testable import SomeModule with @testable import TestEnvironment + sed -i 's/@testable import [^ ]\+/@testable import TestEnvironment/g' "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift # 3. Copy and modify Package.swift cp "${INPUT_DIR}/Package.swift" "${WORKING_DIR}" - sed -i "s/${target_name}/WarmUp/g" "${WORKING_DIR}/Package.swift" + sed -i "s/${target_name}/TestEnvironment/g" "${WORKING_DIR}/Package.swift" else WORKING_DIR=${INPUT_DIR} diff --git a/src/WarmUp/Package.swift b/src/TestEnvironment/Package.swift similarity index 71% rename from src/WarmUp/Package.swift rename to src/TestEnvironment/Package.swift index cd40e63..c4a69af 100644 --- a/src/WarmUp/Package.swift +++ b/src/TestEnvironment/Package.swift @@ -3,11 +3,11 @@ import PackageDescription let package = Package( - name: "WarmUp", + name: "TestEnvironment", products: [ .library( - name: "WarmUp", - targets: ["WarmUp"] + name: "TestEnvironment", + targets: ["TestEnvironment"] ), ], dependencies: [ @@ -15,15 +15,15 @@ let package = Package( ], targets: [ .target( - name: "WarmUp", + name: "TestEnvironment", dependencies: [ .product(name: "Numerics", package: "swift-numerics") ] ), .testTarget( - name: "WarmUpTests", + name: "TestEnvironmentTests", dependencies: [ - "WarmUp", + "TestEnvironment", .product(name: "Numerics", package: "swift-numerics") ] ) diff --git a/src/WarmUp/README.md b/src/TestEnvironment/README.md similarity index 60% rename from src/WarmUp/README.md rename to src/TestEnvironment/README.md index 9bd03d8..7f8b4c9 100644 --- a/src/WarmUp/README.md +++ b/src/TestEnvironment/README.md @@ -1,6 +1,10 @@ # Swift build cache warm-up for faster builds inside docker ## Overview +This package is used in a production environment as a container for building and testing students’ solutions. It is built and included in a Docker image with a pre-built .build directory, which minimizes changes in the build graph when applying different exercise solutions. Exercise resources are copied into the TestEnvironment package as if they had always been part of it. + +## The problem with slow build times + Cold swift builds are slow. We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts. This package is used to build warm-up: it prebuilds the .build folder during docker image creation and is used as an environment for building and testing exercises. ## Why cold builds are slow? @@ -20,6 +24,6 @@ import Testing @testable import ModuleName ``` -Then it gets built during Docker image creation. When a student runs their solution later the code for a particular exercise is copied into WarmUp package and the package itself is rebuilt with minor difference, like you've just replace code in files and rebuilt the same package. +Then it gets built during Docker image creation. When a student runs their solution later the code for a particular exercise is copied into TestEnvironment package and the package itself is rebuilt with minor difference, like you've just replace code in files and rebuilt the same package. -Then it is built during Docker image creation. Later, when a student runs their solution, the code for a particular exercise is copied into the WarmUp package. Theb, the package itself is rebuilt with minor changes, such as replacing code in Source and Tests files. See `bin/run.sh` for more details on which parts of an exercise are copied into docker image. \ No newline at end of file +Then it is built during Docker image creation. Later, when a student runs their solution, the code for a particular exercise is copied into the TestEnvironment package. Then, the package itself is rebuilt with minor changes, such as replacing code in Source and Tests files. See `bin/run.sh` for more details on which parts of an exercise are copied into docker image. \ No newline at end of file diff --git a/src/WarmUp/Sources/WarmUp/WarmUp.swift b/src/TestEnvironment/Sources/WarmUp/TestEnvironment.swift similarity index 100% rename from src/WarmUp/Sources/WarmUp/WarmUp.swift rename to src/TestEnvironment/Sources/WarmUp/TestEnvironment.swift diff --git a/src/WarmUp/Tests/WarmUpTests/WarmUpTests.swift b/src/TestEnvironment/Tests/WarmUpTests/TestEnvironmentTests.swift similarity index 88% rename from src/WarmUp/Tests/WarmUpTests/WarmUpTests.swift rename to src/TestEnvironment/Tests/WarmUpTests/TestEnvironmentTests.swift index 6730c46..d6b0a22 100644 --- a/src/WarmUp/Tests/WarmUpTests/WarmUpTests.swift +++ b/src/TestEnvironment/Tests/WarmUpTests/TestEnvironmentTests.swift @@ -2,7 +2,7 @@ import Foundation import Numerics import Testing -@testable import WarmUp +@testable import TestEnvironment let RUNALL = Bool(ProcessInfo.processInfo.environment["RUNALL", default: "false"]) ?? false diff --git a/tests/compile-error/expected_results.json b/tests/compile-error/expected_results.json index fcef595..56de121 100644 --- a/tests/compile-error/expected_results.json +++ b/tests/compile-error/expected_results.json @@ -1,5 +1,5 @@ { - "message" : "/opt/test-runner/Sources/WarmUp/CompileError.swift:2:12: error: unexpected non-void return value in void function1 | func sum(_ x: Int, _ y: Int) {2 | return x + y | |- error: unexpected non-void return value in void function | `- note: did you mean to add a return type?3 | }4 | error: fatalError", + "message" : "/opt/test-runner/Sources/TestEnvironment/CompileError.swift:2:12: error: unexpected non-void return value in void function1 | func sum(_ x: Int, _ y: Int) {2 | return x + y | |- error: unexpected non-void return value in void function | `- note: did you mean to add a return type?3 | }4 | error: fatalError", "status" : "error", "tests" : [ From c58793eb2189350101e48a9dd5c720593f216fed Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sat, 13 Sep 2025 18:12:46 +0300 Subject: [PATCH 09/27] Add imports to make build harder --- .../Sources/UsingLibrary/UsingLibrary.swift | 3 +++ .../Tests/UsingLibraryTests/UsingLibraryTests.swift | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/tests-using-library/Sources/UsingLibrary/UsingLibrary.swift b/tests/tests-using-library/Sources/UsingLibrary/UsingLibrary.swift index 372a590..7f076ec 100644 --- a/tests/tests-using-library/Sources/UsingLibrary/UsingLibrary.swift +++ b/tests/tests-using-library/Sources/UsingLibrary/UsingLibrary.swift @@ -1,3 +1,6 @@ +import Foundation +import Numerics + enum TestError: Error { case testError(String) } diff --git a/tests/tests-using-library/Tests/UsingLibraryTests/UsingLibraryTests.swift b/tests/tests-using-library/Tests/UsingLibraryTests/UsingLibraryTests.swift index 33c8b38..781fa53 100644 --- a/tests/tests-using-library/Tests/UsingLibraryTests/UsingLibraryTests.swift +++ b/tests/tests-using-library/Tests/UsingLibraryTests/UsingLibraryTests.swift @@ -1,3 +1,4 @@ +import Foundation import Testing import Numerics From dd8582ad861fa9ecf7ebfd07f789e93350fec0b4 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sat, 13 Sep 2025 18:39:45 +0300 Subject: [PATCH 10/27] Remove --enable-experimental-prebuilts to buid-test-runner --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 52f5023..c74c47f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ RUN swift --version # Build TestRunner executable WORKDIR /TestRunner COPY src/TestRunner . -RUN swift build --configuration release --enable-experimental-prebuilts +RUN swift build --configuration release # Stage 2: Prepare docker container image FROM swift:6.1.2 From 02c9192cd032202ea693778f7316c3385946f36d Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sat, 13 Sep 2025 19:57:37 +0300 Subject: [PATCH 11/27] Rename --- .../Sources/{WarmUp => TestEnvironment}/TestEnvironment.swift | 0 .../TestEnvironmentTests.swift | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/TestEnvironment/Sources/{WarmUp => TestEnvironment}/TestEnvironment.swift (100%) rename src/TestEnvironment/Tests/{WarmUpTests => TestEnvironmentTests}/TestEnvironmentTests.swift (100%) diff --git a/src/TestEnvironment/Sources/WarmUp/TestEnvironment.swift b/src/TestEnvironment/Sources/TestEnvironment/TestEnvironment.swift similarity index 100% rename from src/TestEnvironment/Sources/WarmUp/TestEnvironment.swift rename to src/TestEnvironment/Sources/TestEnvironment/TestEnvironment.swift diff --git a/src/TestEnvironment/Tests/WarmUpTests/TestEnvironmentTests.swift b/src/TestEnvironment/Tests/TestEnvironmentTests/TestEnvironmentTests.swift similarity index 100% rename from src/TestEnvironment/Tests/WarmUpTests/TestEnvironmentTests.swift rename to src/TestEnvironment/Tests/TestEnvironmentTests/TestEnvironmentTests.swift From 8e409d4feb5d5f5d8d6d6734701e42bfdf30a0e2 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sun, 14 Sep 2025 00:39:08 +0300 Subject: [PATCH 12/27] Fixes --- bin/run.sh | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 35f7b34..1a1e4ba 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -31,22 +31,27 @@ if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then # exercise resources are copied inside the TestEnvironment package as if they had always been there. WORKING_DIR="${PWD}" - desc_file="${WORKING_DIR}/package_desc.json" - swift package dump-package --package-path ${INPUT_DIR} > "$desc_file" + CONFIG_FILE="${INPUT_DIR}/.meta/config.json" + SOURCE_TARGET_PATH="${WORKING_DIR}/Sources/TestEnvironment" + TEST_TARGET_PATH="${WORKING_DIR}/Tests/TestEnvironmentTests" - # 1. Replace source files with those of an exercise - find "${WORKING_DIR}/Sources/TestEnvironment" -type f -delete - target_name=$(jq -r '.targets[] | select(.type=="regular") | .name' "$desc_file" | head -1) - cp -rf "${INPUT_DIR}/Sources/${target_name}/." "${WORKING_DIR}/Sources/TestEnvironment/" + # 1. Replace source files with those of an exercise, saving TestEnvironement paths + rm -Rf $SOURCE_TARGET_PATH $TEST_TARGET_PATH - find "${WORKING_DIR}/Tests/TestEnvironmentTests" -type f -delete - test_target_name=$(jq -r '.targets[] | select(.type=="test") | .name' "$desc_file" | head -1) - cp -rf "${INPUT_DIR}/Tests/${test_target_name}/." "${WORKING_DIR}/Tests/TestEnvironmentTests/" + target_name=$(jq -r '.files.solution[0]' $CONFIG_FILE | xargs dirname | xargs basename) + test_target_name=$(jq -r '.files.test[0]' $CONFIG_FILE | xargs dirname | xargs basename) + + # Copying everything from package in case of other targets. + cp -rf "${INPUT_DIR}/Sources/." "${WORKING_DIR}/Sources" + cp -rf "${INPUT_DIR}/Tests/." "${WORKING_DIR}/Tests" + + mv "${WORKING_DIR}/Sources/${target_name}" $SOURCE_TARGET_PATH + mv "${WORKING_DIR}/Tests/${test_target_name}" $TEST_TARGET_PATH # 2. Replace @testable import SomeModule with @testable import TestEnvironment sed -i 's/@testable import [^ ]\+/@testable import TestEnvironment/g' "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift - # 3. Copy and modify Package.swift + # 3. Copy Package.swift and rename main & test targets. cp "${INPUT_DIR}/Package.swift" "${WORKING_DIR}" sed -i "s/${target_name}/TestEnvironment/g" "${WORKING_DIR}/Package.swift" From 9aeeb67d9e142d08e35e656de73267ae7b5ca1d4 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sun, 14 Sep 2025 01:13:54 +0300 Subject: [PATCH 13/27] Rename only exact match --- bin/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/run.sh b/bin/run.sh index 1a1e4ba..a055583 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -53,7 +53,8 @@ if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then # 3. Copy Package.swift and rename main & test targets. cp "${INPUT_DIR}/Package.swift" "${WORKING_DIR}" - sed -i "s/${target_name}/TestEnvironment/g" "${WORKING_DIR}/Package.swift" + sed -i "s/\"${target_name}\"/\"TestEnvironment\"/g" "${WORKING_DIR}/Package.swift" + sed -i "s/\"${test_target_name}\"/\"TestEnvironmentTests\"/g" "${WORKING_DIR}/Package.swift" else WORKING_DIR=${INPUT_DIR} From 874dc29bd2bf479fa95351ba106de4f9beca5da2 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sun, 14 Sep 2025 14:11:50 +0300 Subject: [PATCH 14/27] Review fixes: readme --- src/TestEnvironment/README.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/TestEnvironment/README.md b/src/TestEnvironment/README.md index 7f8b4c9..edfe452 100644 --- a/src/TestEnvironment/README.md +++ b/src/TestEnvironment/README.md @@ -1,15 +1,21 @@ # Swift build cache warm-up for faster builds inside docker + ## Overview -This package is used in a production environment as a container for building and testing students’ solutions. It is built and included in a Docker image with a pre-built .build directory, which minimizes changes in the build graph when applying different exercise solutions. Exercise resources are copied into the TestEnvironment package as if they had always been part of it. +This package is used in a production environment as a container for building and testing students’ solutions. +It is built and included in a Docker image with a pre-built .build directory, which minimizes changes in the build graph when applying different exercise solutions. +Exercise resources are copied into the TestEnvironment package as if they had always been part of it. ## The problem with slow build times -Cold swift builds are slow. We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts. This package is used to build warm-up: it prebuilds the .build folder during docker image creation and is used as an environment for building and testing exercises. +Cold swift builds are slow. +We need fast feedback — to respond quickly when someone runs an exercise and not hit Docker timeouts. +This package is used to build warm-up: it prebuilds the .build folder during docker image creation and is used as an environment for building and testing exercises. ## Why cold builds are slow? -When Swift compiles from a clean state, it spends a huge amount of time on resolving dependencies. Imports like Foundation, Numerics, Dispatch, etc. pull in a ton of underlying clang and swift modules like SwiftShims, SwiftGlibc, _Builtin_stddef, etc. +When Swift compiles from a clean state, it spends a huge amount of time on resolving dependencies. +Imports like Foundation, Numerics, Dispatch, etc. pull in a ton of underlying clang and swift modules like SwiftShims, SwiftGlibc, _Builtin_stddef, etc. Even if you don’t import them directly, Swift still needs to find, parse, and compile some of them into .build and specifically ModuleCache directory. @@ -24,6 +30,7 @@ import Testing @testable import ModuleName ``` -Then it gets built during Docker image creation. When a student runs their solution later the code for a particular exercise is copied into TestEnvironment package and the package itself is rebuilt with minor difference, like you've just replace code in files and rebuilt the same package. - -Then it is built during Docker image creation. Later, when a student runs their solution, the code for a particular exercise is copied into the TestEnvironment package. Then, the package itself is rebuilt with minor changes, such as replacing code in Source and Tests files. See `bin/run.sh` for more details on which parts of an exercise are copied into docker image. \ No newline at end of file +Then it is built during Docker image creation. +When a student runs their solution, the code for a particular exercise is copied into the TestEnvironment package. +Then, the package itself is rebuilt with minor changes, such as replacing code in Source and Tests files. +See `bin/run.sh` for more details on which parts of an exercise are copied into docker image. \ No newline at end of file From c449eb3a43ac849c806f4b12c4254f3286180ce9 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Sun, 14 Sep 2025 14:15:54 +0300 Subject: [PATCH 15/27] Review fixes: run.sh fixes --- bin/run.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index a055583..aed718f 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -30,23 +30,32 @@ if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then # to build solutions as fast as it possible. To minimize changes in the build graph, # exercise resources are copied inside the TestEnvironment package as if they had always been there. + get_target_name() { + local section=$1 + local path + path=$(jq -r ".files.${section}[0]" "$CONFIG_FILE") + path=${path%/*} + path=${path##*/} + echo "$path" + } + WORKING_DIR="${PWD}" CONFIG_FILE="${INPUT_DIR}/.meta/config.json" SOURCE_TARGET_PATH="${WORKING_DIR}/Sources/TestEnvironment" TEST_TARGET_PATH="${WORKING_DIR}/Tests/TestEnvironmentTests" # 1. Replace source files with those of an exercise, saving TestEnvironement paths - rm -Rf $SOURCE_TARGET_PATH $TEST_TARGET_PATH + rm -rf "${SOURCE_TARGET_PATH}" "${TEST_TARGET_PATH}" - target_name=$(jq -r '.files.solution[0]' $CONFIG_FILE | xargs dirname | xargs basename) - test_target_name=$(jq -r '.files.test[0]' $CONFIG_FILE | xargs dirname | xargs basename) + target_name=$(get_target_name solution) + test_target_name=$(get_target_name test) # Copying everything from package in case of other targets. cp -rf "${INPUT_DIR}/Sources/." "${WORKING_DIR}/Sources" cp -rf "${INPUT_DIR}/Tests/." "${WORKING_DIR}/Tests" - mv "${WORKING_DIR}/Sources/${target_name}" $SOURCE_TARGET_PATH - mv "${WORKING_DIR}/Tests/${test_target_name}" $TEST_TARGET_PATH + mv "${WORKING_DIR}/Sources/${target_name}" "${SOURCE_TARGET_PATH}" + mv "${WORKING_DIR}/Tests/${test_target_name}" "${TEST_TARGET_PATH}" # 2. Replace @testable import SomeModule with @testable import TestEnvironment sed -i 's/@testable import [^ ]\+/@testable import TestEnvironment/g' "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift @@ -61,7 +70,7 @@ else fi junit_file="${WORKING_DIR}/results-swift-testing.xml" -spec_file="${INPUT_DIR}/$(jq -r '.files.test[0]' ${INPUT_DIR}/.meta/config.json)" +spec_file="${INPUT_DIR}/$(jq -r '.files.test[0]' "${INPUT_DIR}/.meta/config.json")" capture_file="${OUTPUT_DIR}/capture" results_file="${OUTPUT_DIR}/results.json" From b4b05ab8abf41826480665ae05cadc2ec89a6f83 Mon Sep 17 00:00:00 2001 From: Sencudra Date: Mon, 15 Sep 2025 17:14:20 +0300 Subject: [PATCH 16/27] Apply suggestions from code review #1 Co-authored-by: Isaac Good --- bin/run.sh | 4 ++-- src/TestEnvironment/README.md | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index aed718f..8ea1564 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -33,10 +33,10 @@ if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then get_target_name() { local section=$1 local path - path=$(jq -r ".files.${section}[0]" "$CONFIG_FILE") + path=$(jq -r section "${section}" '.files.[$section][0]' "${CONFIG_FILE}") path=${path%/*} path=${path##*/} - echo "$path" + echo "${path}" } WORKING_DIR="${PWD}" diff --git a/src/TestEnvironment/README.md b/src/TestEnvironment/README.md index edfe452..d929376 100644 --- a/src/TestEnvironment/README.md +++ b/src/TestEnvironment/README.md @@ -1,8 +1,9 @@ # Swift build cache warm-up for faster builds inside docker + ## Overview -This package is used in a production environment as a container for building and testing students’ solutions. +This package is used in a production environment as a container for building and testing students' solutions. It is built and included in a Docker image with a pre-built .build directory, which minimizes changes in the build graph when applying different exercise solutions. Exercise resources are copied into the TestEnvironment package as if they had always been part of it. From 17cdbb1da6259968e41e40836b460cde0dcd19b9 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Mon, 15 Sep 2025 17:20:09 +0300 Subject: [PATCH 17/27] Fix jq command and remove "force" flag from commands. --- bin/run.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 8ea1564..e52a55f 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -15,6 +15,8 @@ # Example: # ./bin/run.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/ +set -euo pipefail + # If any required arguments is missing, print the usage and exit if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then echo "usage: ./bin/run.sh exercise-slug /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/" @@ -33,7 +35,7 @@ if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then get_target_name() { local section=$1 local path - path=$(jq -r section "${section}" '.files.[$section][0]' "${CONFIG_FILE}") + path=$(jq -r --arg section "${section}" '.files.[$section][0]' "${CONFIG_FILE}") path=${path%/*} path=${path##*/} echo "${path}" @@ -45,14 +47,14 @@ if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then TEST_TARGET_PATH="${WORKING_DIR}/Tests/TestEnvironmentTests" # 1. Replace source files with those of an exercise, saving TestEnvironement paths - rm -rf "${SOURCE_TARGET_PATH}" "${TEST_TARGET_PATH}" + rm -r "${SOURCE_TARGET_PATH}" "${TEST_TARGET_PATH}" target_name=$(get_target_name solution) test_target_name=$(get_target_name test) # Copying everything from package in case of other targets. - cp -rf "${INPUT_DIR}/Sources/." "${WORKING_DIR}/Sources" - cp -rf "${INPUT_DIR}/Tests/." "${WORKING_DIR}/Tests" + cp -r "${INPUT_DIR}/Sources/." "${WORKING_DIR}/Sources" + cp -r "${INPUT_DIR}/Tests/." "${WORKING_DIR}/Tests" mv "${WORKING_DIR}/Sources/${target_name}" "${SOURCE_TARGET_PATH}" mv "${WORKING_DIR}/Tests/${test_target_name}" "${TEST_TARGET_PATH}" From 3f3862123f39e883a1652dab71f592700bc5fb96 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Tue, 16 Sep 2025 16:46:27 +0300 Subject: [PATCH 18/27] Review fixes. Fixing error code propagation. --- bin/run-test.sh | 8 +++----- bin/run-tests-in-docker.sh | 1 + bin/run.sh | 20 ++++++++++++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/bin/run-test.sh b/bin/run-test.sh index 4e02e10..0cdc67c 100755 --- a/bin/run-test.sh +++ b/bin/run-test.sh @@ -14,6 +14,8 @@ # Example: # ./bin/run-test.sh /absolute/path/to/tests/compile-error/ +set -eu + # If any required arguments is missing, print the usage and exit if [ -z "$1" ]; then echo "usage: ./bin/run-test.sh /absolute/path/to/test/folder/" @@ -33,8 +35,4 @@ bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}" sed -i -e "s~\\\/~/~g" -e "s~${test_dir_path}~/solution~g" "${results_file_path}" echo "${test_dir_name}: comparing results.json to expected_results.json" -diff "${results_file_path}" "${expected_results_file_path}" - -if [ $? -ne 0 ]; then - echo "\"${test_dir_name}\" failed!" -fi \ No newline at end of file +diff "${results_file_path}" "${expected_results_file_path}" \ No newline at end of file diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh index 6a4e527..4b6027e 100755 --- a/bin/run-tests-in-docker.sh +++ b/bin/run-tests-in-docker.sh @@ -32,6 +32,7 @@ for test_dir in tests/*; do "${dst_test_dir}" if [ $? -ne 0 ]; then + echo "Test \"${test_name}\" failed!" exit_code=1 fi done diff --git a/bin/run.sh b/bin/run.sh index e52a55f..0c97939 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -15,10 +15,10 @@ # Example: # ./bin/run.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/ -set -euo pipefail +set -eu # If any required arguments is missing, print the usage and exit -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then +if [ "$#" -ne 3 ]; then echo "usage: ./bin/run.sh exercise-slug /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/" exit 1 fi @@ -60,13 +60,16 @@ if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then mv "${WORKING_DIR}/Tests/${test_target_name}" "${TEST_TARGET_PATH}" # 2. Replace @testable import SomeModule with @testable import TestEnvironment - sed -i 's/@testable import [^ ]\+/@testable import TestEnvironment/g' "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift + sed -i.bak 's/@testable import [^ ]\+/@testable import TestEnvironment/g' \ + "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift \ + && rm -f "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift.bak # 3. Copy Package.swift and rename main & test targets. cp "${INPUT_DIR}/Package.swift" "${WORKING_DIR}" - sed -i "s/\"${target_name}\"/\"TestEnvironment\"/g" "${WORKING_DIR}/Package.swift" - sed -i "s/\"${test_target_name}\"/\"TestEnvironmentTests\"/g" "${WORKING_DIR}/Package.swift" - + sed -i.bak \ + -e "s/\"${target_name}\"/\"TestEnvironment\"/g" \ + -e "s/\"${test_target_name}\"/\"TestEnvironmentTests\"/g" \ + "${WORKING_DIR}/Package.swift" && rm -f "${WORKING_DIR}/Package.swift.bak" else WORKING_DIR=${INPUT_DIR} fi @@ -79,9 +82,14 @@ results_file="${OUTPUT_DIR}/results.json" touch "${results_file}" export RUNALL=true + +set +e # Disable for swift test + swift test \ --package-path "${WORKING_DIR}" \ --xunit-output "${WORKING_DIR}/results.xml" \ --skip-update &> "${capture_file}" +set -e # Re-enable + ./bin/TestRunner "${spec_file}" "${junit_file}" "${capture_file}" "${results_file}" "${SLUG}" From af046024258b8081d3b06b452defc441630bab8b Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Wed, 17 Sep 2025 13:25:41 +0300 Subject: [PATCH 19/27] Review fixes --- bin/run-tests-in-docker.sh | 2 +- bin/run.sh | 38 ++++++++++++++++------------------- src/TestEnvironment/README.md | 1 - 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh index 4b6027e..c5e109a 100755 --- a/bin/run-tests-in-docker.sh +++ b/bin/run-tests-in-docker.sh @@ -32,7 +32,7 @@ for test_dir in tests/*; do "${dst_test_dir}" if [ $? -ne 0 ]; then - echo "Test \"${test_name}\" failed!" + printf 'Test "%s" failed!\n' "$test_name" exit_code=1 fi done diff --git a/bin/run.sh b/bin/run.sh index 0c97939..e716544 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -27,20 +27,19 @@ SLUG="$1" INPUT_DIR="${2%/}" OUTPUT_DIR="${3%/}" +get_target_name() { + local section=$1 + local path=$(jq -r --arg section "${section}" '.files.[$section][0]' "${CONFIG_FILE}") + path=${path%/*} + path=${path##*/} + echo "${path}" +} + if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then # Docker image contains a prebuilt package called TestEnvironment with a ready .build directory # to build solutions as fast as it possible. To minimize changes in the build graph, # exercise resources are copied inside the TestEnvironment package as if they had always been there. - get_target_name() { - local section=$1 - local path - path=$(jq -r --arg section "${section}" '.files.[$section][0]' "${CONFIG_FILE}") - path=${path%/*} - path=${path##*/} - echo "${path}" - } - WORKING_DIR="${PWD}" CONFIG_FILE="${INPUT_DIR}/.meta/config.json" SOURCE_TARGET_PATH="${WORKING_DIR}/Sources/TestEnvironment" @@ -60,16 +59,17 @@ if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then mv "${WORKING_DIR}/Tests/${test_target_name}" "${TEST_TARGET_PATH}" # 2. Replace @testable import SomeModule with @testable import TestEnvironment - sed -i.bak 's/@testable import [^ ]\+/@testable import TestEnvironment/g' \ - "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift \ - && rm -f "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift.bak + change_import='s/@testable import [^ ]\+/@testable import TestEnvironment/g' + for file in "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift; do + sed -i "${change_import}" "${file}" + done # 3. Copy Package.swift and rename main & test targets. cp "${INPUT_DIR}/Package.swift" "${WORKING_DIR}" - sed -i.bak \ - -e "s/\"${target_name}\"/\"TestEnvironment\"/g" \ - -e "s/\"${test_target_name}\"/\"TestEnvironmentTests\"/g" \ - "${WORKING_DIR}/Package.swift" && rm -f "${WORKING_DIR}/Package.swift.bak" + change_target="s/\"${target_name}\"/\"TestEnvironment\"/g; " + change_target+="s/\"${test_target_name}\"/\"TestEnvironmentTests\"/g" + sed -i "${change_target}" "${WORKING_DIR}/Package.swift" + else WORKING_DIR=${INPUT_DIR} fi @@ -83,13 +83,9 @@ touch "${results_file}" export RUNALL=true -set +e # Disable for swift test - swift test \ --package-path "${WORKING_DIR}" \ --xunit-output "${WORKING_DIR}/results.xml" \ - --skip-update &> "${capture_file}" - -set -e # Re-enable + --skip-update &> "${capture_file}" || true ./bin/TestRunner "${spec_file}" "${junit_file}" "${capture_file}" "${results_file}" "${SLUG}" diff --git a/src/TestEnvironment/README.md b/src/TestEnvironment/README.md index d929376..9b5a527 100644 --- a/src/TestEnvironment/README.md +++ b/src/TestEnvironment/README.md @@ -1,6 +1,5 @@ # Swift build cache warm-up for faster builds inside docker - ## Overview This package is used in a production environment as a container for building and testing students' solutions. From 58da54475c8b8c3453d093a4bf67c2eeaed002f4 Mon Sep 17 00:00:00 2001 From: Sencudra Date: Wed, 17 Sep 2025 17:12:30 +0300 Subject: [PATCH 20/27] Apply suggestions from code review Co-authored-by: Isaac Good --- bin/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run.sh b/bin/run.sh index e716544..efd184e 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -18,7 +18,7 @@ set -eu # If any required arguments is missing, print the usage and exit -if [ "$#" -ne 3 ]; then +if (( "$#" != 3 )); then echo "usage: ./bin/run.sh exercise-slug /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/" exit 1 fi From 5a851fcd9738d6ee2629293a6414904030bd1e10 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Wed, 17 Sep 2025 17:17:47 +0300 Subject: [PATCH 21/27] Revert some inaccurate changes. --- bin/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/run.sh b/bin/run.sh index efd184e..eb9de31 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -29,7 +29,8 @@ OUTPUT_DIR="${3%/}" get_target_name() { local section=$1 - local path=$(jq -r --arg section "${section}" '.files.[$section][0]' "${CONFIG_FILE}") + local path + path=$(jq -r --arg section "${section}" '.files.[$section][0]' "${CONFIG_FILE}") path=${path%/*} path=${path##*/} echo "${path}" From ceaf4879ca7aa52e281b3a25db838e1a03ab3455 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Thu, 25 Sep 2025 16:21:33 +0300 Subject: [PATCH 22/27] Update minimal version as required for swift-syntax. --- src/TestRunner/Package.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/src/TestRunner/Package.swift b/src/TestRunner/Package.swift index 83d6e5a..35a02f8 100644 --- a/src/TestRunner/Package.swift +++ b/src/TestRunner/Package.swift @@ -5,6 +5,7 @@ import PackageDescription let package = Package( name: "TestRunner", + platforms: [.macOS(.v10_15)], dependencies: [ // Dependencies declare other packages that this package depends on. // .package(url: /* package url */, from: "1.0.0"), From 1cc42845cdf16a44221aee90875edd3fa34f6369 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Thu, 25 Sep 2025 16:22:35 +0300 Subject: [PATCH 23/27] Add directives for linux --- src/TestRunner/Sources/TestRunner/main.swift | 2 ++ src/TestRunner/Sources/TestRunner/xmlParser.swift | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/TestRunner/Sources/TestRunner/main.swift b/src/TestRunner/Sources/TestRunner/main.swift index e431f35..162c574 100644 --- a/src/TestRunner/Sources/TestRunner/main.swift +++ b/src/TestRunner/Sources/TestRunner/main.swift @@ -1,5 +1,7 @@ import Foundation +#if os(Linux) import FoundationXML +#endif import SwiftParser import SwiftSyntax diff --git a/src/TestRunner/Sources/TestRunner/xmlParser.swift b/src/TestRunner/Sources/TestRunner/xmlParser.swift index 2f131b3..a6353c9 100644 --- a/src/TestRunner/Sources/TestRunner/xmlParser.swift +++ b/src/TestRunner/Sources/TestRunner/xmlParser.swift @@ -1,5 +1,7 @@ import Foundation +#if os(Linux) import FoundationXML +#endif class MyXMLParserDelegate: NSObject, XMLParserDelegate { var counter = 0 From 18fc1fe96467a72042b932c072dba581098eed5e Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Thu, 25 Sep 2025 16:46:51 +0300 Subject: [PATCH 24/27] Review fixes. --- bin/benchmark-in-docker.sh | 13 +++++++------ bin/benchmark.sh | 7 ++++--- bin/build-test-runner.sh | 6 ++++-- bin/run-in-docker.sh | 14 ++++++++------ bin/run-test.sh | 20 ++++++++++++++------ bin/run-tests-in-docker.sh | 6 +++--- bin/run-tests.sh | 8 ++++---- bin/run.sh | 26 +++++++++++++------------- 8 files changed, 57 insertions(+), 43 deletions(-) diff --git a/bin/benchmark-in-docker.sh b/bin/benchmark-in-docker.sh index a980225..a0591e6 100755 --- a/bin/benchmark-in-docker.sh +++ b/bin/benchmark-in-docker.sh @@ -13,7 +13,7 @@ # Example: # ./bin/benchmark-in-docker.sh -set -eo pipefail +set -euo pipefail die() { echo "$*" >&2; exit 1; } @@ -26,14 +26,15 @@ required_tool docker required_tool hyperfine # Pre-build the Docker image -if [[ -z "${SKIP_DOCKER_BUILD}" ]]; then +if [[ -z "${SKIP_DOCKER_BUILD:-}" ]]; then docker build --rm -t exercism/swift-test-runner . else - echo "Skipping docker build because SKIP_DOCKER_BUILD is set." + printf "Skipping docker build because SKIP_DOCKER_BUILD is set.\n" fi -CURRENT_PATH=${PWD} +CURRENT_PATH="${PWD}" +test_list=$(find tests -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | paste -sd "," -) hyperfine \ - --parameter-list slug $(find tests -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | paste -sd ",") \ - "SKIP_DOCKER_BUILD=true bin/run-in-docker.sh {slug} $CURRENT_PATH/tests/{slug} $CURRENT_PATH/tests/{slug}" + --parameter-list slug "${test_list}" \ + "SKIP_DOCKER_BUILD=true bin/run-in-docker.sh {slug} ${CURRENT_PATH}/tests/{slug} ${CURRENT_PATH}/tests/{slug}" diff --git a/bin/benchmark.sh b/bin/benchmark.sh index d29b6ea..6c6e8f4 100755 --- a/bin/benchmark.sh +++ b/bin/benchmark.sh @@ -10,7 +10,7 @@ # Example: # ./bin/benchmark.sh -set -eo pipefail +set -euo pipefail die() { echo "$*" >&2; exit 1; } @@ -21,7 +21,8 @@ required_tool() { required_tool hyperfine +test_list=$(find tests -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | paste -sd "," -) hyperfine \ - --parameter-list slug $(find tests -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | paste -sd ",") \ + --parameter-list slug "${test_list}" \ --prepare 'git clean -xdfq tests/{slug}' \ - 'bin/run.sh {slug} tests/{slug} tests/{slug}' + './bin/run.sh {slug} tests/{slug} tests/{slug}' diff --git a/bin/build-test-runner.sh b/bin/build-test-runner.sh index 4b9c52f..49d4c0f 100755 --- a/bin/build-test-runner.sh +++ b/bin/build-test-runner.sh @@ -9,14 +9,16 @@ # Example: # ./bin/build-test-runner.sh +set -euo pipefail + BIN_DIR="bin" BUILD_DIR="src/TestRunner" RELEASE_DIR=".build/release" # Build the test runner file -cd "$BUILD_DIR" +cd "${BUILD_DIR}" swift build --configuration release cd - # Copy generated file to bin dir -cp "$BUILD_DIR/$RELEASE_DIR/TestRunner" "$BIN_DIR"/ +cp "${BUILD_DIR}/${RELEASE_DIR}/TestRunner" "${BIN_DIR}"/ diff --git a/bin/run-in-docker.sh b/bin/run-in-docker.sh index d37598b..02c45f4 100755 --- a/bin/run-in-docker.sh +++ b/bin/run-in-docker.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -set -e # Synopsis: # Run the test runner on a solution using the test runner Docker image. @@ -18,8 +17,11 @@ set -e # ./bin/run-in-docker.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/ # If any required arguments is missing, print the usage and exit -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "usage: ./bin/run-in-docker.sh exercise-slug /absolute/path/to/solution/folder/ /absolute/path/to/output/directory/" + +set -euo pipefail + +if (( "$#" != 3 )); then + printf 'usage: %s exercise-slug /absolute/path/to/solution/folder/ /absolute/path/to/output/directory/\n' "$0" exit 1 fi @@ -31,10 +33,10 @@ OUTPUT_DIR="${3%/}" mkdir -p "${OUTPUT_DIR}" # Pre-build the Docker image -if [[ -z "${SKIP_DOCKER_BUILD}" ]]; then +if [[ -z "${SKIP_DOCKER_BUILD:-}" ]]; then docker build --rm -t exercism/swift-test-runner . else - echo "Skipping docker build because SKIP_DOCKER_BUILD is set." + printf "Skipping docker build because SKIP_DOCKER_BUILD is set.\n" fi # Run image passing the arguments @@ -43,4 +45,4 @@ docker run \ --mount type=bind,src="${INPUT_DIR}",dst=/solution \ --mount type=bind,src="${OUTPUT_DIR}",dst=/output \ --mount type=volume,dst=/tmp \ - exercism/swift-test-runner $SLUG /solution/ /output/ + exercism/swift-test-runner "${SLUG}" /solution/ /output/ diff --git a/bin/run-test.sh b/bin/run-test.sh index 0cdc67c..8e500f6 100755 --- a/bin/run-test.sh +++ b/bin/run-test.sh @@ -14,14 +14,22 @@ # Example: # ./bin/run-test.sh /absolute/path/to/tests/compile-error/ -set -eu +set -euo pipefail # If any required arguments is missing, print the usage and exit -if [ -z "$1" ]; then - echo "usage: ./bin/run-test.sh /absolute/path/to/test/folder/" +if (( "$#" != 1 )); then + printf 'usage: %s /absolute/path/to/test/folder/\n' "$0" exit 1 fi +sed_i() { + if [[ "$(uname)" == "Darwin" ]]; then + sed -i '' "$@"; + else + sed -i "$@"; + fi +} + test_dir="$1" test_dir_name=$(basename "${test_dir}") @@ -32,7 +40,7 @@ expected_results_file_path="${test_dir_path}/expected_results.json" bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}" # Normalize the results file -sed -i -e "s~\\\/~/~g" -e "s~${test_dir_path}~/solution~g" "${results_file_path}" +sed_i -e "s~\\\/~/~g" -e "s~${test_dir_path}~/solution~g" "${results_file_path}" -echo "${test_dir_name}: comparing results.json to expected_results.json" -diff "${results_file_path}" "${expected_results_file_path}" \ No newline at end of file +printf '%s: comparing results.json to expected_results.json' "${test_dir_name}" +diff "${results_file_path}" "${expected_results_file_path}" diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh index c5e109a..284d31a 100755 --- a/bin/run-tests-in-docker.sh +++ b/bin/run-tests-in-docker.sh @@ -20,7 +20,7 @@ exit_code=0 # Iterate over all test directories work_dir=/opt/test-runner/ for test_dir in tests/*; do - test_name=$(basename $test_dir) + test_name=$(basename "${test_dir}") dst_test_dir=${work_dir}/${test_name} docker run \ --network none \ @@ -32,9 +32,9 @@ for test_dir in tests/*; do "${dst_test_dir}" if [ $? -ne 0 ]; then - printf 'Test "%s" failed!\n' "$test_name" + printf 'Test "%s" failed!\n' "${test_name}" exit_code=1 fi done -exit ${exit_code} \ No newline at end of file +exit "${exit_code}" diff --git a/bin/run-tests.sh b/bin/run-tests.sh index 8e341d8..cc6df24 100755 --- a/bin/run-tests.sh +++ b/bin/run-tests.sh @@ -19,12 +19,12 @@ exit_code=0 -for test_dir in tests/*; do - bin/run-test.sh $test_dir +for test_dir in ./tests/*; do + [ -e "${test_dir}" ] || continue - if [ $? -ne 0 ]; then + if bin/run-test.sh "${test_dir}"; then exit_code=1 fi done -exit ${exit_code} \ No newline at end of file +exit "${exit_code}" diff --git a/bin/run.sh b/bin/run.sh index eb9de31..42e53fa 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -19,7 +19,7 @@ set -eu # If any required arguments is missing, print the usage and exit if (( "$#" != 3 )); then - echo "usage: ./bin/run.sh exercise-slug /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/" + printf 'usage: %s exercise-slug /absolute/path/to/solution/ /absolute/path/to/output/\n' "$0" exit 1 fi @@ -36,7 +36,7 @@ get_target_name() { echo "${path}" } -if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then +if [[ "${RUN_IN_DOCKER:-}" == "TRUE" ]]; then # Docker image contains a prebuilt package called TestEnvironment with a ready .build directory # to build solutions as fast as it possible. To minimize changes in the build graph, # exercise resources are copied inside the TestEnvironment package as if they had always been there. @@ -47,29 +47,27 @@ if [[ "${RUN_IN_DOCKER}" == "TRUE" ]]; then TEST_TARGET_PATH="${WORKING_DIR}/Tests/TestEnvironmentTests" # 1. Replace source files with those of an exercise, saving TestEnvironement paths - rm -r "${SOURCE_TARGET_PATH}" "${TEST_TARGET_PATH}" + rm -r -- "${SOURCE_TARGET_PATH}" "${TEST_TARGET_PATH}" target_name=$(get_target_name solution) test_target_name=$(get_target_name test) # Copying everything from package in case of other targets. - cp -r "${INPUT_DIR}/Sources/." "${WORKING_DIR}/Sources" - cp -r "${INPUT_DIR}/Tests/." "${WORKING_DIR}/Tests" + cp -r -- "${INPUT_DIR}/Sources/." "${WORKING_DIR}/Sources" + cp -r -- "${INPUT_DIR}/Tests/." "${WORKING_DIR}/Tests" - mv "${WORKING_DIR}/Sources/${target_name}" "${SOURCE_TARGET_PATH}" - mv "${WORKING_DIR}/Tests/${test_target_name}" "${TEST_TARGET_PATH}" + mv -- "${WORKING_DIR}/Sources/${target_name}" "${SOURCE_TARGET_PATH}" + mv -- "${WORKING_DIR}/Tests/${test_target_name}" "${TEST_TARGET_PATH}" # 2. Replace @testable import SomeModule with @testable import TestEnvironment - change_import='s/@testable import [^ ]\+/@testable import TestEnvironment/g' - for file in "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift; do - sed -i "${change_import}" "${file}" - done + change_import="s/@testable import [^ ]\+/@testable import TestEnvironment/g" + sed -i -- "${change_import}" "${WORKING_DIR}/Tests/TestEnvironmentTests"/*.swift # 3. Copy Package.swift and rename main & test targets. cp "${INPUT_DIR}/Package.swift" "${WORKING_DIR}" - change_target="s/\"${target_name}\"/\"TestEnvironment\"/g; " + change_target="s/\"${target_name}\"/\"TestEnvironment\"/g;" change_target+="s/\"${test_target_name}\"/\"TestEnvironmentTests\"/g" - sed -i "${change_target}" "${WORKING_DIR}/Package.swift" + sed -i -- "${change_target}" "${WORKING_DIR}/Package.swift" else WORKING_DIR=${INPUT_DIR} @@ -84,9 +82,11 @@ touch "${results_file}" export RUNALL=true +# Builds and runs tests. The resulting artifacts are saved to $junit_file (yes, the file name will change). swift test \ --package-path "${WORKING_DIR}" \ --xunit-output "${WORKING_DIR}/results.xml" \ --skip-update &> "${capture_file}" || true +# Convert test results into the Exercism test runner output format. ./bin/TestRunner "${spec_file}" "${junit_file}" "${capture_file}" "${results_file}" "${SLUG}" From 46037c601a8fba9bfc5fc4352f33bcc2fb3ac843 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Thu, 25 Sep 2025 17:04:00 +0300 Subject: [PATCH 25/27] Add missing \n --- bin/run-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-test.sh b/bin/run-test.sh index 8e500f6..baa3394 100755 --- a/bin/run-test.sh +++ b/bin/run-test.sh @@ -42,5 +42,5 @@ bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}" # Normalize the results file sed_i -e "s~\\\/~/~g" -e "s~${test_dir_path}~/solution~g" "${results_file_path}" -printf '%s: comparing results.json to expected_results.json' "${test_dir_name}" +printf '%s: comparing results.json to expected_results.json\n' "${test_dir_name}" diff "${results_file_path}" "${expected_results_file_path}" From daaf2426e1da927e0f810d75d4d0efefbf0be0bb Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Thu, 25 Sep 2025 17:17:51 +0300 Subject: [PATCH 26/27] Some more review fixes --- bin/run-tests-in-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh index 284d31a..59693c4 100755 --- a/bin/run-tests-in-docker.sh +++ b/bin/run-tests-in-docker.sh @@ -31,7 +31,7 @@ for test_dir in tests/*; do exercism/swift-test-runner \ "${dst_test_dir}" - if [ $? -ne 0 ]; then + if (( "$?" != 0 )); then printf 'Test "%s" failed!\n' "${test_name}" exit_code=1 fi From 528df4297e1937a711270e6a76ed28c7c630bcf4 Mon Sep 17 00:00:00 2001 From: Vladislav Tarasevich Date: Thu, 25 Sep 2025 19:17:16 +0300 Subject: [PATCH 27/27] Review fixes --- bin/run-test.sh | 6 +++--- bin/run-tests-in-docker.sh | 2 +- bin/run-tests.sh | 2 +- bin/run.sh | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/run-test.sh b/bin/run-test.sh index baa3394..0f025ec 100755 --- a/bin/run-test.sh +++ b/bin/run-test.sh @@ -18,15 +18,15 @@ set -euo pipefail # If any required arguments is missing, print the usage and exit if (( "$#" != 1 )); then - printf 'usage: %s /absolute/path/to/test/folder/\n' "$0" + printf 'Usage: %s /absolute/path/to/test/folder/\n' "$0" exit 1 fi sed_i() { if [[ "$(uname)" == "Darwin" ]]; then - sed -i '' "$@"; + sed -i '' "$@" else - sed -i "$@"; + sed -i "$@" fi } diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh index 59693c4..e8b0227 100755 --- a/bin/run-tests-in-docker.sh +++ b/bin/run-tests-in-docker.sh @@ -20,7 +20,7 @@ exit_code=0 # Iterate over all test directories work_dir=/opt/test-runner/ for test_dir in tests/*; do - test_name=$(basename "${test_dir}") + test_name="${test_dir#*/}" dst_test_dir=${work_dir}/${test_name} docker run \ --network none \ diff --git a/bin/run-tests.sh b/bin/run-tests.sh index cc6df24..fc48c84 100755 --- a/bin/run-tests.sh +++ b/bin/run-tests.sh @@ -20,7 +20,7 @@ exit_code=0 for test_dir in ./tests/*; do - [ -e "${test_dir}" ] || continue + [[ -e "${test_dir}" ]] || continue if bin/run-test.sh "${test_dir}"; then exit_code=1 diff --git a/bin/run.sh b/bin/run.sh index 42e53fa..84ef9c2 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -19,7 +19,7 @@ set -eu # If any required arguments is missing, print the usage and exit if (( "$#" != 3 )); then - printf 'usage: %s exercise-slug /absolute/path/to/solution/ /absolute/path/to/output/\n' "$0" + printf 'Usage: %s exercise-slug /absolute/path/to/solution/ /absolute/path/to/output/\n' "$0" exit 1 fi