Guard test timing and quarantine rules in CI

.github/workflows/ci.yml

@@ -71,6 +71,18 @@ jobs:
             echo "code=false" >> "$GITHUB_OUTPUT"
           fi
 
+  guards:
+    needs: changes
+    if: needs.changes.outputs.code == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - name: Test the test timing guard
+        run: Tools/tests/check-test-timing-test.sh
+      - name: Check test timing and quarantine rules
+        run: Tools/ci/check-test-timing.sh
+
   unit-tests:
     needs: changes
     if: needs.changes.outputs.code == 'true'
@@ -140,7 +152,7 @@ jobs:
   # Keep `always()` rather than `!cancelled()`: a cancelled run must fail this
   # check, because a skipped required check counts as passing.
   test:
-    needs: [changes, unit-tests, release-package, app-debug, app-release]
+    needs: [changes, guards, unit-tests, release-package, app-debug, app-release]
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 5

Tools/ci/check-test-timing.sh

@@ -0,0 +1,184 @@
+#!/bin/bash
+# Count real-time operations without invoking Swift or touching the desktop.
+set -euo pipefail
+trap 'exit 1' ERR
+export LC_ALL=C
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+cd "$script_dir/../.."
+baseline=Tools/ci/test-timing-baseline.txt
+update=0
+allow_increase=0
+for arg in "$@"; do
+    case "$arg" in
+        --update-baseline) update=1 ;;
+        --allow-increase) allow_increase=1 ;;
+        *) echo "Usage: $0 [--update-baseline [--allow-increase]]" >&2; exit 1 ;;
+    esac
+done
+if [[ "$allow_increase" == 1 && "$update" == 0 ]]; then
+    echo "--allow-increase requires --update-baseline." >&2
+    exit 1
+fi
+work_dir="$(mktemp -d)"
+trap 'rm -rf "$work_dir"' EXIT
+# A missing baseline starts with zero allowances; it never grants new ones.
+if [[ -f "$baseline" ]]; then
+    cp "$baseline" "$work_dir/old"
+else
+    : > "$work_dir/old"
+fi
+
+find Tests -type f -name '*.swift' -exec awk '
+BEGIN {
+    ws = "[[:space:]]*"
+    boundary = "(^|[^[:alnum:]_])"
+    dot = ws "[.]" ws
+    patterns["Task.sleep"] = boundary "Task(" ws "<[^>]+>)?" dot "sleep" ws "\\("
+    patterns["usleep"] = boundary "usleep" ws "\\("
+    patterns["Thread.sleep"] = boundary "Thread" dot "sleep" ws "\\("
+    patterns["sleep"] = "((^|[^[:alnum:]_.])|" boundary "(Darwin|Glibc)" dot ")sleep" ws "\\("
+    patterns["Date()"] = boundary "Date" ws "\\(" ws "\\)"
+    patterns["Date.now"] = boundary "Date" dot "now([^[:alnum:]_]|$)"
+    patterns["CFAbsoluteTimeGetCurrent"] = boundary "CFAbsoluteTimeGetCurrent" ws "\\("
+    patterns["systemUptime"] = boundary "ProcessInfo" dot "processInfo" dot "systemUptime([^[:alnum:]_]|$)"
+    patterns["DispatchTime.now"] = boundary "DispatchTime" dot "now" ws "\\("
+    patterns["ContinuousClock"] = boundary "ContinuousClock(" dot "now([^[:alnum:]_]|$)|(" dot "init)?" ws "\\()"
+    patterns["SuspendingClock"] = boundary "SuspendingClock(" dot "now([^[:alnum:]_]|$)|(" dot "init)?" ws "\\()"
+    disabled = "[.]" ws "disabled" ws "\\("
+    quarantine = "^[[:space:]]*\"quarantine: #[1-9][0-9]*[[:space:]]+[^\"[:space:]][^\"]*\"[[:space:]]*\\)"
+    # A condition gate, like `.enabled(if:)`, is not a quarantine.
+    conditional = "^[[:space:]]*if[[:space:]]*:"
+}
+# Mask comments and string contents while retaining offsets and newlines.
+# This is a spelling guard, not a Swift parser (aliases are outside its scope).
+function scrub(line,    i, c, pair, triple, masked, visible, n, rest, token, chunk, spaces, size, j) {
+    masked = visible = ""
+    n = length(line)
+    for (i = 1; i <= n; i++) {
+        # Copy whole runs between delimiters instead of visiting every byte.
+        if (!escaped) {
+            rest = substr(line, i)
+            token = (block ? "/\\*|\\*/" : (quoted ? "[\"\\\\]" : "[\"/]"))
+            size = (match(rest, token) ? RSTART - 1 : length(rest))
+            if (size) {
+                chunk = substr(rest, 1, size); spaces = chunk
+                gsub(/./, " ", spaces)
+                masked = masked (block || quoted ? spaces : chunk)
+                visible = visible (block ? spaces : chunk)
+                i += size
+                if (i > n) break
+            }
+        }
+        c = substr(line, i, 1)
+        pair = substr(line, i, 2)
+        triple = substr(line, i, 3)
+        if (block) {
+            if (pair == "/*") { block++; i++; masked = masked "  "; visible = visible "  " }
+            else if (pair == "*/") { block--; i++; masked = masked "  "; visible = visible "  " }
+            else { masked = masked " "; visible = visible " " }
+        } else if (quoted) {
+            visible = visible c
+            masked = masked " "
+            if (escaped) escaped = 0
+            else if (c == "\\" && hashes == "") escaped = 1
+            else if (quoted == 3 && triple == "\"\"\"" && substr(line, i + 3, length(hashes)) == hashes) {
+                visible = visible "\"\""; masked = masked "  "; i += 2; quoted = 0
+            } else if (quoted == 1 && c == "\"" && substr(line, i + 1, length(hashes)) == hashes) quoted = 0
+        } else if (pair == "//") {
+            if (substr(line, i) ~ /\/\/ real-time:[[:space:]]*[^[:space:]]/) annotated[FNR] = 1
+            spaces = substr(line, i); gsub(/./, " ", spaces)
+            masked = masked spaces; visible = visible spaces
+            break
+        } else if (pair == "/*") {
+            block = 1; i++; masked = masked "  "; visible = visible "  "
+        } else if (c == "\"") {
+            hashes = ""
+            for (j = i - 1; j > 0 && substr(line, j, 1) == "#"; j--) hashes = hashes "#"
+            quoted = (triple == "\"\"\"" ? 3 : 1)
+            visible = visible (quoted == 3 ? "\"\"\"" : "\"")
+            masked = masked (quoted == 3 ? "   " : " ")
+            if (quoted == 3) i += 2
+        } else { masked = masked c; visible = visible c }
+    }
+    source = source masked "\n"
+    original = original visible "\n"
+}
+function line_at(offset,    prefix) {
+    prefix = substr(source, 1, offset - 1)
+    return gsub(/\n/, "", prefix) + 1
+}
+function scan(    key, rest, offset, start, size, line, argument, declaration) {
+    for (key in patterns) {
+        rest = source; offset = 0
+        while (match(rest, patterns[key])) {
+            start = offset + RSTART; size = RLENGTH
+            # The boundary may include the previous newline or indentation.
+            while (substr(source, start, 1) ~ /[[:space:]]/) start++
+            line = line_at(start)
+            declaration = (key == "sleep" && substr(source, 1, start - 1) ~ /(^|[^[:alnum:]_])func[[:space:]]*$/)
+            if (!annotated[line] && !declaration) printf "%s\t%s\t%d\n", file, key, line
+            offset += RSTART + size - 1
+            rest = substr(source, offset + 1)
+        }
+    }
+    rest = source; offset = 0
+    while (match(rest, disabled)) {
+        start = offset + RSTART; size = RLENGTH
+        argument = substr(original, offset + RSTART + size)
+        if (argument !~ quarantine && argument !~ conditional) printf "%s\tquarantine\t%d\n", file, line_at(start)
+        offset += RSTART + size - 1
+        rest = substr(source, offset + 1)
+    }
+}
+FNR == 1 {
+    if (NR > 1) scan()
+    file = FILENAME; source = original = ""; block = quoted = escaped = 0
+    for (line in annotated) delete annotated[line]
+}
+{ scrub($0) }
+END { scan() }
+' {} + > "$work_dir/hits"
+
+awk -F '\t' '
+    { counts[$1 FS $2]++ }
+    END { for (key in counts) print key FS counts[key] }
+' "$work_dir/hits" | sort > "$work_dir/current"
+
+if ! awk -F '\t' -v update="$update" -v allow="$allow_increase" '
+FILENAME == ARGV[1] {
+    if ($0 ~ /^#/ || $0 == "") next
+    if (NF != 3 || $3 !~ /^[1-9][0-9]*$/ || ($1 FS $2) in old) {
+        print "Invalid timing baseline row " FNR ": " $0; failed = 1
+    }
+    old[$1 FS $2] = $3
+    next
+}
+FILENAME == ARGV[2] { current[$1 FS $2] = $3; next }
+{
+    key = $1 FS $2
+    if (current[key] > old[key] && !(update && allow)) {
+        if (!reported[key]++) {
+            printf "%s:%s: %s count %d exceeds baseline %d. ", $1, $3, $2, current[key], old[key]
+            if ($2 == "quarantine") print "Use .disabled(\"quarantine: #<number> <reason>\")."
+            else print "Use the injected clock or add // real-time: <reason> on the call line."
+        }
+        failed = 1
+    }
+}
+END {
+    for (key in old) if (current[key] + 0 < old[key]) {
+        split(key, parts, FS)
+        printf "%s:1: hint: %s count fell from %d to %d; run Tools/ci/check-test-timing.sh --update-baseline to lower the baseline.\n", parts[1], parts[2], old[key], current[key]
+    }
+    if (failed && update) print "Baseline unchanged. Refusing increases without --allow-increase (or fix the invalid baseline)."
+    exit (failed ? 1 : 0)
+}' "$work_dir/old" "$work_dir/current" "$work_dir/hits"; then
+    exit 1
+fi
+
+if [[ "$update" == 1 ]]; then
+    { printf '# File\tPattern\tCount (unannotated calls; quarantine counts invalid disabled traits)\n'; cat "$work_dir/current"; } > "$baseline"
+    echo "Updated $baseline."
+else
+    echo "Test timing and quarantine guard passed."
+fi

Tools/ci/test-timing-baseline.txt

@@ -0,0 +1,28 @@
+# File	Pattern	Count (unannotated calls; quarantine counts invalid disabled traits)
+Tests/BetterTileAppTests/BentoBalanceModelTests.swift	Task.sleep	2
+Tests/BetterTileAppTests/ConfigurationModelTests.swift	Task.sleep	1
+Tests/BetterTileAppTests/DockSpaceModelTests.swift	Task.sleep	2
+Tests/BetterTileAppTests/LayoutWheelModelTests.swift	Task.sleep	91
+Tests/BetterTileAppTests/LayoutWheelModelTests.swift	Thread.sleep	1
+Tests/BetterTileAppTests/LayoutWheelModelTests.swift	systemUptime	3
+Tests/BetterTileAppTests/SettingsPreviewTests.swift	Task.sleep	1
+Tests/BetterTileAppTests/TabbedArrivalModelTests.swift	Task.sleep	53
+Tests/BetterTileAppTests/TabbedModelTests.swift	Task.sleep	50
+Tests/BetterTileAppTests/TabbedPaneSizeTests.swift	Task.sleep	7
+Tests/BetterTileAppTests/TabbedSnapModelTests.swift	Task.sleep	13
+Tests/BetterTileAppTests/TabbedSteppedModelTests.swift	Task.sleep	35
+Tests/BetterTileAppTests/TabbedWorkAreaModelTests.swift	Task.sleep	4
+Tests/BetterTileMacOSTests/DividerDragOverlayTests.swift	Task.sleep	8
+Tests/BetterTileMacOSTests/DividerLensTests.swift	Task.sleep	2
+Tests/BetterTileMacOSTests/DividerOverlayControllerTests.swift	Task.sleep	1
+Tests/BetterTileMacOSTests/DragSnapControllerTests.swift	ContinuousClock	4
+Tests/BetterTileMacOSTests/DragSnapControllerTests.swift	Task.sleep	4
+Tests/BetterTileMacOSTests/DragSnapControllerTests.swift	systemUptime	3
+Tests/BetterTileMacOSTests/FakeWindowSystem.swift	ContinuousClock	2
+Tests/BetterTileMacOSTests/InteractionBenchmarkTests.swift	systemUptime	1
+Tests/BetterTileMacOSTests/OverlayGlassTests.swift	Task.sleep	1
+Tests/BetterTileMacOSTests/TabbedCoordinatorTests.swift	ContinuousClock	2
+Tests/BetterTileMacOSTests/TabbedCoordinatorTests.swift	systemUptime	4
+Tests/BetterTileMacOSTests/TabbedDragTests.swift	Task.sleep	1
+Tests/BetterTileMacOSTests/WindowCoordinatorTests.swift	Task.sleep	2
+Tests/BetterTileMacOSTests/WindowCoordinatorTests.swift	systemUptime	2

Tools/tests/check-test-timing-test.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+# Exercise the guard against source fixtures; never compile or run Swift.
+set -euo pipefail
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+work_dir="$(mktemp -d)"
+trap 'rm -rf "$work_dir"' EXIT
+mkdir -p "$work_dir/Tools/ci" "$work_dir/Tests/Nested"
+cp "$script_dir/../ci/check-test-timing.sh" "$work_dir/Tools/ci/"
+guard="$work_dir/Tools/ci/check-test-timing.sh"
+fixture="$work_dir/Tests/Nested/Example.swift"
+baseline="$work_dir/Tools/ci/test-timing-baseline.txt"
+checks=0
+expect_pass() {
+    if ! "$guard" "$@" > "$work_dir/output" 2>&1; then
+        cat "$work_dir/output" >&2
+        echo "Expected guard to pass: $*" >&2
+        exit 1
+    fi
+    checks=$((checks + 1))
+}
+expect_fail() {
+    if "$guard" "$@" > "$work_dir/output" 2>&1; then
+        echo "Expected guard to fail: $*" >&2
+        exit 1
+    fi
+    checks=$((checks + 1))
+}
+require_output() {
+    grep -F "$1" "$work_dir/output" > /dev/null || {
+        cat "$work_dir/output" >&2
+        echo "Missing diagnostic: $1" >&2
+        exit 1
+    }
+}
+
+cat > "$fixture" <<'SWIFT'
+let fixedDate = Date(timeIntervalSince1970: 0)
+let instant: ContinuousClock.Instant
+try await clock.sleep(for: .seconds(1))
+func sleep(for duration: Duration) {}
+@Test(.enabled(if: environmentAllowsTest)) func example() {}
+// Task.sleep(for: .seconds(1))
+/* Nested /* Task.sleep(for: .seconds(1)) */ Date() */
+let description = "Task.sleep and Date() are examples, not calls"
+let raw = #""Date()" and Task.sleep(for: .seconds(1))"#
+let multiline = """
+Date() and Task.sleep(for: .seconds(1))
+"""
+SWIFT
+expect_pass
+expect_pass --update-baseline
+
+echo 'try await Task.sleep(for: .seconds(1))' >> "$fixture"
+expect_fail
+require_output 'Tests/Nested/Example.swift:13: Task.sleep count 1 exceeds baseline 0.'
+require_output 'Use the injected clock or add // real-time: <reason>'
+cp "$baseline" "$work_dir/before"
+expect_fail --update-baseline
+require_output 'Refusing increases without --allow-increase'
+cmp "$baseline" "$work_dir/before"
+
+cat > "$fixture" <<'SWIFT'
+try await Task.sleep(for: .seconds(1)) // real-time: measures a system timer
+SWIFT
+expect_pass
+echo 'try await Task.sleep(for: .seconds(1)) // real-time:   ' > "$fixture"
+expect_fail
+echo 'try await Task.sleep(for: .seconds(1)); let text = "// real-time: not a comment"' > "$fixture"
+expect_fail
+
+echo 'try await Task.sleep(for: .seconds(1))' > "$fixture"
+expect_pass --update-baseline --allow-increase
+expect_pass
+# Unrelated edits must not consume an allowance or change its location.
+printf '\n\ntry await Task.sleep(for: .seconds(1))\n' > "$fixture"
+expect_pass
+echo 'try await Task.sleep(for: .seconds(1)); try await Task.sleep(for: .seconds(1))' > "$fixture"
+expect_fail
+require_output 'count 2 exceeds baseline 1'
+: > "$fixture"
+expect_pass
+require_output 'hint: Task.sleep count fell from 1 to 0'
+expect_pass --update-baseline
+if grep -F 'Task.sleep' "$baseline"; then
+    echo 'Removed calls must disappear from the baseline.' >&2
+    exit 1
+fi
+
+# Every supported spelling is rejected without an allowance.
+while IFS= read -r call; do
+    printf '%s\n' "$call" > "$fixture"
+    expect_fail
+done <<'SWIFT'
+try await Task.sleep(for: .seconds(1))
+try await Task<Never, Never>.sleep(nanoseconds: 1)
+usleep(1)
+Thread.sleep(forTimeInterval: 1)
+sleep(1)
+Darwin.sleep(1)
+Glibc.sleep(1)
+let date = Date()
+let date = Date.now
+let now = CFAbsoluteTimeGetCurrent()
+let now = ProcessInfo.processInfo.systemUptime
+let now = DispatchTime.now()
+let now = ContinuousClock.now
+let clock = ContinuousClock()
+let clock = ContinuousClock.init()
+let now = SuspendingClock.now
+let clock = SuspendingClock()
+let clock = SuspendingClock.init()
+SWIFT
+cat > "$fixture" <<'SWIFT'
+try await Task
+    .sleep(
+        for: .seconds(1))
+SWIFT
+expect_fail
+require_output 'Tests/Nested/Example.swift:1:'
+
+echo '@Test(.disabled("temporarily flaky")) func example() {}' > "$fixture"
+expect_fail
+require_output 'Use .disabled("quarantine: #<number> <reason>").'
+echo '@Test(.disabled("quarantine: #42 ")) func example() {}' > "$fixture"
+expect_fail
+echo '@Test(.disabled("quarantine: #42 tracks a system timer race")) func example() {}' > "$fixture"
+expect_pass
+echo '@Test(.disabled(if: isRunningOnCI)) func example() {}' > "$fixture"
+expect_pass
+cat > "$fixture" <<'SWIFT'
+@Test(.disabled(
+    "quarantine: #42 tracks a system timer race"
+)) func example() {}
+SWIFT
+expect_pass
+# Timing annotations cannot bypass quarantine rules.
+echo '@Test(.disabled()) // real-time: system integration' > "$fixture"
+expect_fail
+expect_pass --update-baseline --allow-increase
+expect_pass
+echo '@Test(.enabled(if: environmentAllowsTest)) func example() {}' > "$fixture"
+expect_pass --update-baseline
+require_output 'hint: quarantine count fell from 1 to 0'
+expect_fail --allow-increase
+expect_fail --unknown
+printf 'malformed\n' > "$baseline"
+expect_fail
+require_output 'Invalid timing baseline row'
+
+echo "check-test-timing tests passed ($checks cases)"