Put test output into dedicated statedir

doc/etest.md

@@ -28,6 +28,53 @@ is run on every commit against a massive [matrix](compatibility.md) of Linux Dis
 
 See [etest usage](binaries/etest.md)
 
+## Directory Structure
+
+When etest runs, it creates a working directory structure to isolate tests and capture output.
+Understanding this structure is helpful when debugging test failures or writing tests that create files.
+
+```text
+workdir/                              # e.g., /tmp/etest-XXXXX (--logdir controls this)
+├── suite.etest/                      # One directory per test suite
+│   ├── ETEST_foo/                    # testdir - test runs here (cd to this dir)
+│   │   └── <files created by test>   # Any files the test creates in CWD
+│   ├── ETEST_foo.state/              # statedir - etest internal state (sibling)
+│   │   ├── output.log                # Captured stdout/stderr from the test
+│   │   └── tmp/                      # TMPDIR for the test
+│   ├── ETEST_bar/
+│   ├── ETEST_bar.state/
+│   └── ...
+├── another-suite.etest/
+│   └── ...
+└── logs/
+    └── jobs/                         # Parallel worker management
+        ├── 0/                        # Job 0
+        │   ├── output.log            # Aggregated worker output
+        │   ├── worker.pid
+        │   └── done
+        ├── 1/
+        └── ...
+```
+
+### Key Directories
+
+- **testdir** (`${workdir}/${suite}.etest/${testfunc}/`): The working directory where each test runs.
+  Tests start with `cd` to this directory, so any files created with relative paths land here.
+
+- **statedir** (`${testdir}.state/`): A sibling directory where etest stores its internal files.
+  This separation prevents tests from accidentally overwriting etest's `output.log` when they create
+  files in their working directory.
+
+- **jobdir** (`${workdir}/logs/jobs/`): Contains per-job state for parallel test execution.
+  Each job gets a numbered subdirectory with worker output and status files.
+
+### Environment Variables
+
+Tests have access to these variables:
+
+- `TEST_DIR_OUTPUT`: Path to the test's working directory (same as testdir)
+- `TMPDIR`: Points to `${statedir}/tmp/` for temporary files
+
 ## Asserts
 
 ebash and etest provide an extremely rich interface for testing code via various `assert` functions. If an assertion

share/etest/results.sh

@@ -107,14 +107,16 @@ create_options_json()
     mv "${ETEST_OPTIONS}.tmp" "${ETEST_OPTIONS}"
 }
 
-# Extract test output from the per-test output file.
-# Each test writes its output to ${workdir}/${suite}.etest/${name}/output.log
+# Extract test output from the per-test state directory.
+# Test state (output.log, tmp/) lives in ${testdir}.state to prevent tests from
+# accidentally overwriting etest files when they create files in their working directory.
 __extract_test_output()
 {
     local suite=$1
     local name=$2
 
-    local test_output="${workdir}/${suite}.etest/${name}/output.log"
+    # statedir is always ${testdir}.state (sibling directory pattern)
+    local test_output="${workdir}/${suite}.etest/${name}.state/output.log"
     [[ -f "${test_output}" ]] || return 0
 
     # Strip ANSI codes and NUL bytes (which cause bash warnings in command substitution)

share/etest/runners.sh

@@ -21,6 +21,11 @@ run_single_test()
                           a function inside that file." \
         "testname       | Command to execute to run the test")
 
+    # statedir holds etest internal state (output.log, tmp/) as a sibling of testdir.
+    # This prevents tests from accidentally overwriting etest files when they create
+    # files in their working directory.
+    local statedir="${testdir}.state"
+
     local rc=0
 
     # Record start time of the test and at the end of the test we'll update the total duration for the test. This will
@@ -94,11 +99,14 @@ run_single_test()
         # of the test fails, as if etest weren't running it inside a try/catch
         __EBASH_INSIDE_TRY=0
 
-        # Create our temporary workspace in the directory specified by the caller
+        # Create test workspace and etest internal state directory.
+        # statedir holds etest's files (output.log, tmp/) separate from the test's workspace
+        # to prevent tests from accidentally overwriting them.
         efreshdir "${testdir}"
-        exec &> "${testdir}/output.log"
-        mkdir "${testdir}/tmp"
-        TMPDIR="$(readlink -m ${testdir}/tmp)"
+        efreshdir "${statedir}"
+        exec &> "${statedir}/output.log"
+        mkdir "${statedir}/tmp"
+        TMPDIR="$(readlink -m ${statedir}/tmp)"
         export TMPDIR
 
         # Move test process into cgroup. Skip cgroup_create since global_setup already created ETEST_CGROUP.
@@ -170,9 +178,9 @@ run_single_test()
     }
 
     # Include the test's output in the job's output so verbose mode can display it.
-    # The test redirected its output to testdir/output.log (line 99), so we include it here.
-    if [[ -f "${testdir}/output.log" ]]; then
-        cat "${testdir}/output.log"
+    # The test redirected its output to statedir/output.log (line 104), so we include it here.
+    if [[ -f "${statedir}/output.log" ]]; then
+        cat "${statedir}/output.log"
     fi
 
     edebug "Finished $(lval testname display_testname rc)"
@@ -702,21 +710,23 @@ __run_all_tests_parallel()
                         TEST_SUITES+=( "${suite}" )
                     fi
 
-                    # Determine testdir for the crashed test
-                    local testdir
+                    # Determine testdir and statedir for the crashed test
+                    # statedir is always ${testdir}.state (sibling directory pattern)
+                    local testdir statedir
                     if [[ "${testfunc}" == ETEST_* ]]; then
                         testdir="${workdir}/${suite}.etest/${testfunc}"
                     else
                         testdir="${workdir}/$(basename "${testfunc}")"
                     fi
+                    statedir="${testdir}.state"
 
                     # Append crash info to the per-test output.log (for __extract_test_output)
                     {
                         echo ""
                         echo "[WORKER CRASHED - exit code ${wrc}]"
                         echo ""
                         echo "${testfunc} FAILED."
-                    } >> "${testdir}/output.log"
+                    } >> "${statedir}/output.log"
 
                     # Append crash info to ETEST_LOG
                     {

tests/etest/results/summary.etest

@@ -150,8 +150,9 @@ ETEST_results_failure_log_format()
     DURATION="${SECONDS}"
 
     # Create a mock per-test output.log file that __extract_test_output will read
-    mkdir -p "${workdir}/mysuite.etest/ETEST_failing_test"
-    cat > "${workdir}/mysuite.etest/ETEST_failing_test/output.log" <<'EOF'
+    # statedir is ${testdir}.state (sibling directory pattern)
+    mkdir -p "${workdir}/mysuite.etest/ETEST_failing_test.state"
+    cat > "${workdir}/mysuite.etest/ETEST_failing_test.state/output.log" <<'EOF'
 [2024-01-01T00:00:00Z] Running command="ETEST_failing_test" attempt="0"
 Some test output here
 Error: something went wrong
@@ -181,15 +182,16 @@ ETEST_results_failure_log_multiple_suites()
     DURATION="${SECONDS}"
 
     # Create mock per-test output.log files that __extract_test_output will read
-    mkdir -p "${workdir}/suite1.etest/ETEST_fail1"
-    cat > "${workdir}/suite1.etest/ETEST_fail1/output.log" <<'EOF'
+    # statedir is ${testdir}.state (sibling directory pattern)
+    mkdir -p "${workdir}/suite1.etest/ETEST_fail1.state"
+    cat > "${workdir}/suite1.etest/ETEST_fail1.state/output.log" <<'EOF'
 [2024-01-01T00:00:00Z] Running command="ETEST_fail1" attempt="0"
 Suite1 failure output
 [2024-01-01T00:00:01Z] tests/suite1.etest:ETEST_fail1 FAILED.
 EOF
 
-    mkdir -p "${workdir}/suite2.etest/ETEST_fail2"
-    cat > "${workdir}/suite2.etest/ETEST_fail2/output.log" <<'EOF'
+    mkdir -p "${workdir}/suite2.etest/ETEST_fail2.state"
+    cat > "${workdir}/suite2.etest/ETEST_fail2.state/output.log" <<'EOF'
 [2024-01-01T00:00:02Z] Running command="ETEST_fail2" attempt="0"
 Suite2 failure output
 [2024-01-01T00:00:03Z] tests/suite2.etest:ETEST_fail2 FAILED.

tests/nodejs_install.etest

@@ -170,8 +170,8 @@ ETEST_install_node_linux()
     local test_prefix_dir="${TEST_DIR_OUTPUT}/${test_prefix}"
 
     local tarball_folder="node-v${lts_version}-linux-x64"
-    local tarball="${TEST_DIR_OUTPUT}/tmp/${tarball_folder}.tar.gz"
-    local tarball_loc="${TEST_DIR_OUTPUT}/tmp/${tarball_folder}"
+    local tarball="${TMPDIR}/${tarball_folder}.tar.gz"
+    local tarball_loc="${TMPDIR}/${tarball_folder}"
     local test_url="https://nodejs.org/download/release/latest-${lts_codename}/node-v${lts_version}-linux-x64.tar.gz"
 
     # Create simulated extracted folder
@@ -188,7 +188,7 @@ ETEST_install_node_linux()
     done
 
     # Create the dummy tar.gz that efetch should download
-    pushd "${TEST_DIR_OUTPUT}/tmp"
+    pushd "${TMPDIR}"
     tar cvf - "${tarball_folder}" | gzip > "${tarball}"
     popd