From 5ebb5e5067ef03ca2b6c446bf63afafa4f9fbd8c Mon Sep 17 00:00:00 2001 From: Naviabheeman <45412531+Naviabheeman@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:43:43 +0530 Subject: [PATCH 1/2] =?UTF-8?q?-=20Collapsed=20feature-pruning/feature-dbc?= =?UTF-8?q?rash=20into=20one=20heavy-tests=20job=20with=20strategy.matrix?= =?UTF-8?q?=20=E2=80=94=20build=20pipeline=20lives=20once,=20matrix-driven?= =?UTF-8?q?=20test=5Fargs/name=20differentiate=20the=20two.=20-=20Added=20?= =?UTF-8?q?step=20to=20Print=20full=20combined=20logs=20on=20failure:=20ru?= =?UTF-8?q?ns=20combine=5Flogs.py=20directly=20over=20every=20test=20tmpdi?= =?UTF-8?q?r=20and=20writes=20the=20full,=20untruncated=20output=20(all=20?= =?UTF-8?q?nodes=20+=20test=20framework=20log)=20to=20=20=E2=80=94=20works?= =?UTF-8?q?=20even=20for=20a=20Timeout=20result,=20which=20test=5Frunner.p?= =?UTF-8?q?y=20itself=20never=20surfaces=20to=20the=20console.=20-=20Retai?= =?UTF-8?q?ned=20the=20dead-timeout=20removal,=20chmod=20cleanup,=20ulimit?= =?UTF-8?q?=20fix,=20and=20dangling-process=20kill=20from=20the=20earlier?= =?UTF-8?q?=20split.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/weekly-heavy-tests.yml | 94 +++++++++++++++++++----- 1 file changed, 76 insertions(+), 18 deletions(-) diff --git a/.github/workflows/weekly-heavy-tests.yml b/.github/workflows/weekly-heavy-tests.yml index 8b480ce46..787abe056 100644 --- a/.github/workflows/weekly-heavy-tests.yml +++ b/.github/workflows/weekly-heavy-tests.yml @@ -7,6 +7,10 @@ name: Weekly Heavy Test # Runs tests that are too slow for the daily CI matrix (feature_pruning: 1.5–4 h, 4 GB; # feature_dbcrash: needs far more than the daily 1800s per-test budget for its # loosest dbcrashratio/dbcache tier to reliably crash at least once). +# Each test runs in its own job (feature-pruning, feature-dbcrash) so a slow +# run of one doesn't eat into the other's share of the 6h GitHub-hosted-runner +# execution ceiling -- combining them previously caused a TIMEOUT-retry of +# feature_pruning to blow through that ceiling mid-job. # Scheduled weekly; can also be triggered manually or after a successful daily build. on: schedule: @@ -34,12 +38,21 @@ env: jobs: heavy-tests: - name: Pruning + DB-crash + USDT tests (Linux x86_64 RelWithDebInfo) + strategy: + fail-fast: false + matrix: + config: + - name: feature_pruning + test_args: 'feature_pruning.py' + - name: feature_dbcrash + test_args: 'feature_dbcrash.py "feature_dbcrash.py --scheme SCHNORR"' + name: ${{ matrix.config.name }} test (Linux x86_64 RelWithDebInfo) runs-on: ubuntu-latest - # feature_pruning uses up to 4 GB disk and takes 1.5–4 hours; feature_dbcrash - # runs alongside it in the same --heavy invocation and needs its own headroom - # to reach coverage on its loosest dbcrashratio/dbcache tier; allow 10 hours total. - timeout-minutes: 600 + # No timeout-minutes set: GitHub-hosted runners hard-cap job execution at + # 6 hours regardless of any value configured here, so anything above that + # is dead configuration. Each test now runs in its own matrix job (not + # sharing a job, and without a TIMEOUT-retry from another test eating + # into the budget), comfortably fitting within that 6h ceiling. env: BASE_ROOT_DIR: ${{ github.workspace }} CCACHE_DIR: ${{ github.workspace }}/ccache @@ -99,12 +112,9 @@ jobs: run: | CORE_DUMP_DIR="${{ github.workspace }}/core_dumps" mkdir -p "$CORE_DUMP_DIR" - chmod 777 "$CORE_DUMP_DIR" sudo sysctl -w kernel.core_pattern="$CORE_DUMP_DIR/core.%e.%p" - ulimit -c unlimited TEST_DIR="${{ github.workspace }}/tapyrus_test" mkdir -p "$TEST_DIR" - chmod -R 777 "$TEST_DIR" echo "TAPYRUS_TEST_DIR=$TEST_DIR" >> $GITHUB_ENV - name: Configure and build @@ -141,27 +151,32 @@ jobs: cmake --build "$BASE_BUILD_DIR" --parallel -j$(nproc) --target all - - name: Run heavy test suite (feature_pruning + feature_dbcrash) + - name: Run ${{ matrix.config.name }} test run: | source $HOME/venv/bin/activate cd "$BASE_ROOT_DIR" export PATH="$BASE_BUILD_DIR/bin:$PATH" - echo "Disk space before heavy tests:" + # ulimit is per-process and does not carry over from the "Enable + # core dumps" step (each `run:` block is a separate shell) -- it + # must be set here, in the shell that actually execs the test + # binaries and their children (test_runner.py -> tapyrusd). + ulimit -c unlimited + echo "Disk space before test:" df -h . - echo "---------------- Start heavy test suite ----------------" + echo "---------------- Start ${{ matrix.config.name }} test ----------------" if python3 test/functional/test_runner.py \ --tmpdir="$TAPYRUS_TEST_DIR" \ -j 1 \ --combinedlogslen=4000 \ --failfast \ --test-timeout=18000 \ - --heavy; then - echo "heavy test suite passed" + ${{ matrix.config.test_args }}; then + echo "${{ matrix.config.name }} test passed" else - echo "heavy test suite FAILED" + echo "${{ matrix.config.name }} test FAILED" exit 1 fi - echo "Disk space after heavy tests:" + echo "Disk space after test:" df -h . # TODO: Uncomment this section once USDT tests are ready to run in CI. @@ -190,19 +205,62 @@ jobs: # exit 1 # fi + - name: Print full combined logs (on failure) + if: failure() + run: | + # test_runner.py only ever prints the last 4000 lines of the + # combined per-node logs to the console, and only for a genuine + # "Failed" result -- a "Timeout" result (test_runner.py's other + # failure branch) prints nothing at all on its first attempt, only + # on a retry that then fails too. Bypass both limitations by + # combining the full, untruncated logs (test framework + every + # node's debug.log) directly and writing them to the job summary. + shopt -s nullglob + TESTDIRS=("$TAPYRUS_TEST_DIR"/test_runner_*/*/) + if [ ${#TESTDIRS[@]} -eq 0 ]; then + echo "No test directories found under $TAPYRUS_TEST_DIR" + exit 0 + fi + echo "## Full Combined Logs — ${{ matrix.config.name }}" >> "$GITHUB_STEP_SUMMARY" + for testdir in "${TESTDIRS[@]}"; do + testdir="${testdir%/}" + echo "Combining logs for $testdir" + { + echo "" + echo "### $(basename "$testdir")" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + python3 "$BASE_ROOT_DIR/test/functional/combine_logs.py" "$testdir" >> "$GITHUB_STEP_SUMMARY" 2>&1 + echo '```' >> "$GITHUB_STEP_SUMMARY" + done + - name: Save Ccache cache if: github.ref == 'refs/heads/master' uses: actions/cache/save@v4 continue-on-error: true with: path: ${{ env.CCACHE_DIR }} - key: daily-linux-x86_64-RelWithDebInfo-true-true-true-ccache-${{ github.run_id }} + key: daily-linux-x86_64-RelWithDebInfo-true-true-true-ccache-${{ github.run_id }}-${{ matrix.config.name }} + + - name: Stop dangling test processes + if: always() + run: | + # CI_FAILFAST_TEST_LEAVE_DANGLING=1 intentionally skips node.stop() + # on a failed/timed-out test so datadirs/logs survive for + # inspection, but a still-running tapyrusd keeps rotating its + # wallet's BDB log files -- racing the artifact zip below and + # producing sporadic ENOENT errors during zip creation. The + # process itself isn't needed (there's no live-attach step after + # the job ends), only the on-disk data is, so kill it here to + # make the directory quiescent before archiving. + pkill -9 -f tapyrusd || true + sleep 2 - name: Upload test results if: always() uses: actions/upload-artifact@v4 with: - name: heavy-test-results-${{ github.run_id }} + name: heavy-test-results-${{ matrix.config.name }}-${{ github.run_id }} path: ${{ github.workspace }}/tapyrus_test retention-days: 7 if-no-files-found: ignore @@ -220,7 +278,7 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: heavy-core-dumps-${{ github.run_id }} + name: heavy-core-dumps-${{ matrix.config.name }}-${{ github.run_id }} path: ${{ github.workspace }}/core_dumps/ retention-days: 7 From 55d07e3e2c6b14ff5a50a3b2cbf92d274a9af3a8 Mon Sep 17 00:00:00 2001 From: Naviabheeman <45412531+Naviabheeman@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:16:18 +0530 Subject: [PATCH 2/2] Review feedback - add cancelled to condition for failure, change code dump debugging --- .github/workflows/weekly-heavy-tests.yml | 80 ++++++++++++++++++++---- test/functional/test_runner.py | 8 ++- 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/.github/workflows/weekly-heavy-tests.yml b/.github/workflows/weekly-heavy-tests.yml index 787abe056..d3ba2749c 100644 --- a/.github/workflows/weekly-heavy-tests.yml +++ b/.github/workflows/weekly-heavy-tests.yml @@ -25,7 +25,7 @@ on: required: false concurrency: - group: ${{ github.workflow }}-heavy + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: @@ -206,7 +206,10 @@ jobs: # fi - name: Print full combined logs (on failure) - if: failure() + # A job hitting the 6h GitHub-hosted-runner ceiling ends with + # conclusion "cancelled", not "failure" -- exactly the case this step + # exists for, so it must run on both. + if: failure() || cancelled() run: | # test_runner.py only ever prints the last 4000 lines of the # combined per-node logs to the console, and only for a genuine @@ -214,26 +217,41 @@ jobs: # failure branch) prints nothing at all on its first attempt, only # on a retry that then fails too. Bypass both limitations by # combining the full, untruncated logs (test framework + every - # node's debug.log) directly and writing them to the job summary. + # node's debug.log) directly. $GITHUB_STEP_SUMMARY silently drops + # anything past its 1 MB per-step cap, so the full output goes to a + # file uploaded as an artifact, with only a tail in the summary. shopt -s nullglob TESTDIRS=("$TAPYRUS_TEST_DIR"/test_runner_*/*/) if [ ${#TESTDIRS[@]} -eq 0 ]; then echo "No test directories found under $TAPYRUS_TEST_DIR" exit 0 fi + LOGDIR="${{ github.workspace }}/combined_logs" + mkdir -p "$LOGDIR" echo "## Full Combined Logs — ${{ matrix.config.name }}" >> "$GITHUB_STEP_SUMMARY" for testdir in "${TESTDIRS[@]}"; do testdir="${testdir%/}" echo "Combining logs for $testdir" + LOGFILE="$LOGDIR/$(basename "$testdir").log" + python3 "$BASE_ROOT_DIR/test/functional/combine_logs.py" "$testdir" > "$LOGFILE" 2>&1 { echo "" - echo "### $(basename "$testdir")" + echo "### $(basename "$testdir") (tail; full log in the heavy-combined-logs artifact)" + echo '```' + tail -c 950000 "$LOGFILE" echo '```' } >> "$GITHUB_STEP_SUMMARY" - python3 "$BASE_ROOT_DIR/test/functional/combine_logs.py" "$testdir" >> "$GITHUB_STEP_SUMMARY" 2>&1 - echo '```' >> "$GITHUB_STEP_SUMMARY" done + - name: Upload combined logs (on failure) + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: heavy-combined-logs-${{ matrix.config.name }}-${{ github.run_id }} + path: ${{ github.workspace }}/combined_logs/ + retention-days: 7 + if-no-files-found: ignore + - name: Save Ccache cache if: github.ref == 'refs/heads/master' uses: actions/cache/save@v4 @@ -266,16 +284,54 @@ jobs: if-no-files-found: ignore - name: Debug core dump (on failure) - if: failure() + if: failure() || cancelled() run: | - COREFILE=$(ls -t ${{ github.workspace }}/core_dumps/core.* 2>/dev/null | head -n 1) - if [ -f "$COREFILE" ]; then - echo "Found core dump: $COREFILE" - gdb -batch -ex "bt" -ex "quit" "$BASE_BUILD_DIR/bin/tapyrusd" "$COREFILE" || true + # Scan the directory configured via core_pattern in "Enable core + # dumps and set test dir" directly -- coredumpctl (systemd-coredump's + # own index) never sees these dumps, since core_pattern was pointed + # at a plain file path instead of the systemd-coredump pipe. + CORE_DUMP_DIR="${{ github.workspace }}/core_dumps" + shopt -s nullglob + COREFILES=("$CORE_DUMP_DIR"/core.*) + if [ ${#COREFILES[@]} -eq 0 ]; then + echo "No core dumps found in $CORE_DUMP_DIR" + exit 0 fi + { + echo "## Core Dump Backtraces — ${{ matrix.config.name }}" + } >> "$GITHUB_STEP_SUMMARY" + + for COREFILE in "${COREFILES[@]}"; do + echo "Found core dump: $COREFILE" + + # `file` embeds the crashing binary's path, but it may be relative + # to wherever it was invoked from (e.g. "./tapyrusd" run from + # $BASE_BUILD_DIR/bin) rather than this step's cwd ($GITHUB_WORKSPACE). + BINARY_RAW=$(file "$COREFILE" | awk -F"'" '{print $2}') + if [ -x "$BINARY_RAW" ]; then + BINARY="$BINARY_RAW" + elif [ -x "$BASE_BUILD_DIR/bin/$(basename "$BINARY_RAW" 2>/dev/null)" ]; then + BINARY="$BASE_BUILD_DIR/bin/$(basename "$BINARY_RAW")" + else + echo "Could not detect crashed binary, using tapyrusd as default" + BINARY="$BASE_BUILD_DIR/bin/tapyrusd" + fi + echo "Using binary: $BINARY" + + { + echo "" + echo "### $(basename "$COREFILE") ($BINARY)" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + gdb -batch -ex "bt full" -ex "quit" "$BINARY" "$COREFILE" 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + + echo '```' >> "$GITHUB_STEP_SUMMARY" + done + - name: Upload core dumps (on failure) - if: failure() + if: failure() || cancelled() uses: actions/upload-artifact@v4 with: name: heavy-core-dumps-${{ matrix.config.name }}-${{ github.run_id }} diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index a1528a625..9c75705f1 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -250,9 +250,11 @@ ] # Tests that are too slow/resource-intensive for daily CI (90 min – 4 hours, 4 GB disk). -# Run via the weekly-heavy-tests.yml workflow (weekly). Tests added here are exercised -# ONLY by that weekly workflow (--heavy below) — they never run in the daily base/extended -# suites, so don't add a test here expecting daily coverage. +# weekly-heavy-tests.yml's matrix now lists each of these directly as its own +# job (matrix.config.test_args) instead of running this list wholesale, so +# this list itself no longer drives what runs weekly -- it's kept alive only +# by the completeness check below (test_runner.py:678). Keep it in lockstep +# with that matrix whenever a heavy test is added or removed. HEAVY_SCRIPTS = [ 'feature_pruning.py', 'feature_dbcrash.py',