Skip to content

Add explicit flaky test annotation. - #197

Merged
marshall-mcmullen merged 3 commits into
mainfrom
flaky
Jun 29, 2026
Merged

Add explicit flaky test annotation.#197
marshall-mcmullen merged 3 commits into
mainfrom
flaky

Conversation

@marshall-mcmullen

Copy link
Copy Markdown
Collaborator

Opening PR on behalf of Raymond Huang to add explicit Flaky test annotation

Comment thread share/etest/runners.sh
Comment thread share/etest/runners.sh Outdated
@marshall-mcmullen

Copy link
Copy Markdown
Collaborator Author
diff --git a/tests/etest/flaky/flaky.etest b/tests/etest/flaky/flaky.etest
index ee71743..5c0cbf4 100644
--- a/tests/etest/flaky/flaky.etest
+++ b/tests/etest/flaky/flaky.etest
@@ -100,6 +100,21 @@ write_fixture_normal_fail()
 EOF
 }

+write_fixture_flaky_skip()
+{
+    # FLAKY_ test that requests a skip on its (non-final) first attempt. The die after skip_if must never run --
+    # a skip has to short-circuit the retry loop. This exercises the skip-code (77) propagation path in
+    # share/etest/runners.sh.
+    sed 's/^ *| //' > "$1" <<'EOF'
+        | #!/usr/bin/env bash
+        | FLAKY_ETEST_skips_on_first_attempt()
+        | {
+        |     $(skip_if "true")
+        |     die "should never reach here -- skip must short-circuit before the retry"
+        | }
+EOF
+}
+
 #-----------------------------------------------------------------------------------------------------------------------
 # Layer A: Integration tests via nested etest subprocess
 #-----------------------------------------------------------------------------------------------------------------------
@@ -185,6 +200,30 @@ ETEST_flaky_passes_first_time()
     rm -f "${counter}" "${fixture}"
 }

+# A FLAKY_ test that requests a skip on its first attempt must be reported SKIPPED -- the skip code (77) must
+# short-circuit the retry loop instead of being treated as a failure to retry.
+ETEST_flaky_skip_reported_skipped()
+{
+    local fixture
+    fixture=$(mktemp --suffix=.etest)
+    write_fixture_flaky_skip "${fixture}"
+
+    etestmsg "Running nested etest with flaky_skip fixture"
+    local output rc=0
+    output=$(run_nested_etest --verbose "${fixture}") || rc=$?
+
+    etestmsg "Checking exit code is 0 (skip is not a failure)"
+    assert_eq 0 "${rc}" "Expected exit code 0 but got ${rc}"
+
+    etestmsg "Checking output reports SKIPPED"
+    assert_match "${output}" "SKIPPED"
+
+    etestmsg "Checking output does NOT report FAILED"
+    assert_not_match "${output}" "FAILED"
+
+    rm -f "${fixture}"
+}
+
 # A normal (non-flaky) ETEST_ that fails should NOT be retried -- it should fail immediately.
 # This is a regression guard ensuring the retry logic only applies to FLAKY_ tests.
 ETEST_normal_test_not_retried()

@marshall-mcmullen
marshall-mcmullen merged commit 95ac250 into main Jun 29, 2026
11 checks passed
@marshall-mcmullen
marshall-mcmullen deleted the flaky branch June 29, 2026 17:33
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

1 participant