fix: harden miner recovery, pool validation, and launchers

.github/workflows/ci.yml

@@ -80,6 +80,9 @@ jobs:
             hiveos/h-run.sh \
             hiveos/h-stats.sh
 
+      - name: Run shell launcher regressions
+        run: bash tests/start_c29_launcher_test.sh
+
   windows:
     name: Windows CPU and launcher checks
     runs-on: windows-latest
@@ -133,6 +136,10 @@ jobs:
             throw 'HiveOS manifest must use LF line endings.'
           }
 
+      - name: Run PowerShell launcher regressions
+        shell: powershell
+        run: .\tests\start_c29_launcher_test.ps1
+
       - name: Run GPU recall verifier fixtures
         shell: powershell
         run: python tests\tari_c29_gpu_recall.py --self-test

README.md

@@ -184,9 +184,40 @@ TARI_LOGIN_SEPARATOR=/ \
 together: a pool defines both the endpoint and the login format it accepts.
 Passing `--login-separator` after the starter command works as well.
 
+The wallet is checked before the first connection. Whitespace, control
+characters, and inputs larger than the longest supported Tari text encoding are
+rejected outright. A login with one of the two Base58 address lengths that uses
+a `0`, `O`, `I`, or `l` emits a typo warning before mining starts. It is not
+rejected solely for that warning, because pools exist that expect a username
+rather than an address.
+
 The pool connection is plain TCP. Do not use a sensitive password for
 `--pass`; the default `x` is sufficient for LuckyPool.
 
+### Exit Codes
+
+A miner worker keeps running through anything it can recover from, including a
+dropped connection and a pool outage. It exits non-zero only for a condition
+that needs a restart or an operator, so a rig supervisor can act on the code:
+
+| Code | Meaning |
+|------|---------|
+| 0 | Clean shutdown, or `--max-runtime-sec` elapsed |
+| 1 | Startup failure: sockets unavailable, no such CUDA device, or not enough VRAM for one solver |
+| 2 | Invalid command line |
+| 3 | A GPU solution failed host verification; the GPU or its tuning is suspect |
+| 4 | The pool rejected the login repeatedly; check wallet, worker, password, and separator |
+| 5 | Solver failure: a CUDA error, or three consecutive graphs with no surviving edges |
+| 6 | The pool accepted the connection but never sent a job |
+| 7 | The pool repeatedly sent invalid protocol data |
+
+The starters propagate these. When a GPU worker exits non-zero, the starter
+stops the remaining workers and exits with that same code, rather than carrying
+on with its healthy GPUs — otherwise HiveOS never sees the failure. A starter
+that fails before any worker runs uses its own codes: 2 for a missing wallet, 3
+when `nvidia-smi` finds no GPU, 4 when `TARI_DEVICES` matches none, 5 for a
+missing backend binary, and 130 for Ctrl+C.
+
 ## Test The Solver
 
 The standalone solver checks GPU results with an independent CPU verifier.

mean_c29.cu

@@ -1077,25 +1077,42 @@ struct solver_ctx {
         // print_log(" (%x, %x)", soledges[j].x, soledges[j].y);
       }
       // print_log("\n");
-      outSols.resize(outSols.size() + PROOFSIZE);
-      checkCudaErrors(cudaMemcpyToSymbol(recoveredges, soledges, sizeof(soledges)));
+      // Recovery fills this slot. On failure it is removed again, so a caller
+      // never sees a half-written proof of zeros that would then be reported as
+      // a verification failure.
+      const size_t solbase = outSols.size();
+      outSols.resize(solbase + PROOFSIZE);
+      cudaError_t rc = cudaMemcpyToSymbol(recoveredges, soledges, sizeof(soledges));
 #if RECOVERY_SMALL_OUTPUT
-      checkCudaErrors(cudaMemset(recoverIndexes, 0, PROOFSIZE * sizeof(u32)));
-      Recovery<<<trimmer.tp.recover.blocks, trimmer.tp.recover.tpb>>>(keys, (ulonglong4*)trimmer.bufferA, (int *)recoverIndexes);
-      checkCudaErrors(cudaGetLastError());
-      checkCudaErrors(cudaMemcpy(&outSols[outSols.size()-PROOFSIZE], recoverIndexes,
-                                 PROOFSIZE * sizeof(u32), cudaMemcpyDeviceToHost));
+      if (rc == cudaSuccess)
+        rc = cudaMemset(recoverIndexes, 0, PROOFSIZE * sizeof(u32));
+      if (rc == cudaSuccess) {
+        Recovery<<<trimmer.tp.recover.blocks, trimmer.tp.recover.tpb>>>(keys, (ulonglong4*)trimmer.bufferA, (int *)recoverIndexes);
+        rc = cudaGetLastError();
+      }
+      if (rc == cudaSuccess)
+        rc = cudaMemcpy(&outSols[solbase], recoverIndexes,
+                        PROOFSIZE * sizeof(u32), cudaMemcpyDeviceToHost);
 #else
-      checkCudaErrors(cudaMemset(trimmer.indexesE[1], 0, trimmer.indexesSize));
-      Recovery<<<trimmer.tp.recover.blocks, trimmer.tp.recover.tpb>>>(keys, (ulonglong4*)trimmer.bufferA, (int *)trimmer.indexesE[1]);
-      checkCudaErrors(cudaGetLastError());
-      checkCudaErrors(cudaMemcpy(&outSols[outSols.size()-PROOFSIZE], trimmer.indexesE[1],
-                                 PROOFSIZE * sizeof(u32), cudaMemcpyDeviceToHost));
+      if (rc == cudaSuccess)
+        rc = cudaMemset(trimmer.indexesE[1], 0, trimmer.indexesSize);
+      if (rc == cudaSuccess) {
+        Recovery<<<trimmer.tp.recover.blocks, trimmer.tp.recover.tpb>>>(keys, (ulonglong4*)trimmer.bufferA, (int *)trimmer.indexesE[1]);
+        rc = cudaGetLastError();
+      }
+      if (rc == cudaSuccess)
+        rc = cudaMemcpy(&outSols[solbase], trimmer.indexesE[1],
+                        PROOFSIZE * sizeof(u32), cudaMemcpyDeviceToHost);
 #endif
       // Recovery uses the calling thread's default stream. Synchronizing that
       // stream preserves overlap with trims running in other host threads.
-      checkCudaErrors(cudaStreamSynchronize(0));
-      qsort(&outSols[outSols.size()-PROOFSIZE], PROOFSIZE, sizeof(u32), cg.nonce_cmp);
+      if (rc == cudaSuccess)
+        rc = cudaStreamSynchronize(0);
+      if (rc != cudaSuccess) {
+        outSols.resize(solbase);
+        return gpuAssert(rc, __FILE__, __LINE__);
+      }
+      qsort(&outSols[solbase], PROOFSIZE, sizeof(u32), cg.nonce_cmp);
     }
     return 0;
   }

start-c29.ps1

@@ -121,6 +121,7 @@ if (-not [string]::IsNullOrEmpty($logDir) -and -not (Test-Path -LiteralPath $log
 
 $workers = @()
 $workerFailed = $false
+$workerExitCode = 0
 try {
     foreach ($item in $plan) {
         $startArgs = @{
@@ -151,19 +152,35 @@ try {
     }
     Write-Host 'Press Ctrl+C to stop all GPU workers.'
 
+    # A worker exits non-zero only for something a restart must clear: a
+    # repeatedly rejected login (4), a failed solver (5), an unresponsive pool
+    # (6), or invalid pool protocol data (7). Stop the survivors and surface that
+    # code, so a rig supervisor sees the failure instead of a launcher still
+    # babysitting its healthy GPUs.
     while ($true) {
-        $alive = @($workers | Where-Object { -not $_.HasExited })
-        if ($alive.Count -eq 0) { break }
-        Start-Sleep -Seconds 2
-    }
-    Write-Host 'All GPU workers have exited.'
-    foreach ($worker in $workers) {
-        $worker.WaitForExit()
-        $exitCode = $worker.ExitCode
-        if ($exitCode -ne 0) {
-            Write-Host "ERROR: Miner worker PID $($worker.Id) exited with code $exitCode."
+        $exited = @($workers | Where-Object { $_.HasExited })
+        $failed = @($exited |
+            Where-Object { $_.ExitCode -ne 0 } |
+            Sort-Object ExitTime, Id |
+            Select-Object -First 1)
+        if ($failed.Count -gt 0) {
+            $worker = $failed[0]
+            Write-Host "ERROR: Miner worker PID $($worker.Id) exited with code $($worker.ExitCode)."
+            $workerExitCode = $worker.ExitCode
             $workerFailed = $true
         }
+        if ($workerFailed) {
+            $alive = @($workers | Where-Object { -not $_.HasExited })
+            if ($alive.Count -gt 0) {
+                Write-Host "Stopping $($alive.Count) remaining GPU worker(s)."
+            }
+            break
+        }
+        if ($exited.Count -eq $workers.Count) {
+            Write-Host 'All GPU workers have exited.'
+            break
+        }
+        Start-Sleep -Seconds 2
     }
 }
 catch {
@@ -179,6 +196,9 @@ finally {
     }
 }
 
+if ($workerFailed) {
+    if ($workerExitCode -ne 0) { exit $workerExitCode }
+    exit 1
+}
 if ($missing -gt 0) { exit 5 }
-if ($workerFailed) { exit 1 }
 exit 0

start-c29.sh

@@ -121,17 +121,72 @@ if [[ "${TARI_DRY_RUN:-0}" == "1" ]]; then
     exit 0
 fi
 
+kill_workers() {
+    if ((${#pids[@]} > 0)); then
+        local all_pids=("${pids[@]}")
+        local survivors=("${pids[@]}")
+        local pass pid
+        kill "${all_pids[@]}" 2>/dev/null || true
+        # A wedged miner must not hold the launcher (and therefore the rig
+        # supervisor) forever. Give TERM two seconds, then force the survivors.
+        for ((pass = 0; pass < 20 && ${#survivors[@]} > 0; pass++)); do
+            sleep 0.1
+            local remaining=()
+            for pid in "${survivors[@]}"; do
+                kill -0 "$pid" 2>/dev/null && remaining+=("$pid")
+            done
+            survivors=("${remaining[@]}")
+        done
+        if ((${#survivors[@]} > 0)); then
+            kill -KILL "${survivors[@]}" 2>/dev/null || true
+        fi
+        wait "${all_pids[@]}" 2>/dev/null || true
+        pids=()
+    fi
+}
+
 stop_workers() {
     trap - INT TERM
-    kill "${pids[@]}" 2>/dev/null || true
-    wait "${pids[@]}" 2>/dev/null || true
+    kill_workers
     exit 130
 }
 trap stop_workers INT TERM
 
 status=0
-((missing == 0)) || status=1
-for pid in "${pids[@]}"; do
-    wait "$pid" || status=1
+((missing == 0)) || status=5
+
+# A miner worker exits non-zero when it hits something only a restart can clear:
+# a repeatedly rejected login (4), a failed solver (5), an unresponsive pool
+# (6), or invalid pool protocol data (7). Stop the surviving workers and exit
+# with that code, so a rig supervisor sees the failure instead of a launcher
+# that keeps running its healthy GPUs.
+worker_failure=0
+while ((${#pids[@]} > 0)); do
+    remaining=()
+    for pid in "${pids[@]}"; do
+        if kill -0 "$pid" 2>/dev/null; then
+            remaining+=("$pid")
+            continue
+        fi
+        worker_status=0
+        wait "$pid" || worker_status=$?
+        if ((worker_status != 0)) && ((worker_failure == 0)); then
+            echo "ERROR: GPU worker $pid exited with code $worker_status." >&2
+            worker_failure="$worker_status"
+        fi
+    done
+    pids=(${remaining[@]+"${remaining[@]}"})
+    ((worker_failure == 0)) || break
+    ((${#pids[@]} > 0)) || break
+    sleep 1
 done
+
+if ((worker_failure != 0)); then
+    if ((${#pids[@]} > 0)); then
+        echo "Stopping ${#pids[@]} remaining GPU worker(s)." >&2
+        kill_workers
+    fi
+    exit "$worker_failure"
+fi
+
 exit "$status"