chore: stop re-broadcasting permanently-failed market settlements

extensions/tn_settlement/internal/engine_ops.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -261,6 +262,17 @@ func (e *EngineOperations) BroadcastSettleMarketWithRetry(
 			"tx_hash", hash.String(),
 			"is_nonce_error", isNonceError(err),
 			"error", err)
+
+		// A permanent (deterministic) failure recurs identically on every retry,
+		// so stop now instead of burning more nonces and committing more failed
+		// txs to blocks. Returning the wrapped sentinel lets the scheduler
+		// quarantine the market and flag it for manual intervention.
+		if isPermanentSettleError(err) {
+			e.logger.Error("settle_market permanently failed; not retrying (needs manual intervention)",
+				"query_id", queryID,
+				"error", err)
+			return fmt.Errorf("%w (query_id=%d): %w", ErrPermanentSettleFailure, queryID, err)
+		}
 	}
 
 	return fmt.Errorf("settle_market failed after %d retries: %w", maxRetries, lastErr)
@@ -476,6 +488,66 @@ func isNonceError(err error) bool {
 	return strings.Contains(msg, "nonce") || strings.Contains(msg, "invalid nonce")
 }
 
+// ErrPermanentSettleFailure marks a settle_market failure that will recur
+// identically on every retry. A market's attestation is signed and immutable, so
+// a deterministic parse/decode failure of it can never succeed by re-broadcasting
+// — the retry only burns nonces and commits another failed tx to a block. The
+// scheduler detects this with errors.Is and quarantines the market for manual
+// intervention (re-attestation or admin_force_settle_market) instead of retrying.
+var ErrPermanentSettleFailure = errors.New("permanent settle_market failure")
+
+// permanentSettleErrorSignatures are lowercased substrings that identify a
+// settle_market revert caused by an unparseable or malformed attestation. These
+// come from tn_utils.parse_attestation_boolean / parseBinaryActionResult
+// (extensions/tn_utils/precompiles.go) and are deterministic on-chain action
+// errors: the signed attestation cannot change, so the same parse fails forever.
+// The list is deliberately narrow — only errors that are provably permanent — so
+// a transient or unfamiliar failure keeps the existing retry behavior rather than
+// being mistakenly quarantined.
+var permanentSettleErrorSignatures = []string{
+	// Binary-action payload wrong width, e.g. an empty 128-byte result on a binary
+	// market whose data landed after the attestation was captured (the market-368
+	// case): "binary action result must be 32 bytes (abi-encoded bool), got 128".
+	"binary action result must be",
+	"abi-encoded bool",
+	"failed to decode boolean abi result",
+	"expected 1 value from boolean decode",
+	"decoded value is not boolean",
+	// Numeric-action payload (action_id 1-5): parse_attestation_boolean routes a
+	// numeric-settled market to parseNumericActionResult, which fails
+	// deterministically on an empty or malformed immutable payload — the same
+	// late-arriving-data failure mode as market 368, for numeric markets.
+	"result payload contains no values",
+	"failed to decode abi result payload",
+	"expected 2 arrays (timestamps, values)",
+	"values must be []*big.int",
+	// Malformed / empty canonical result — an immutable attestation that can never
+	// be parsed.
+	"invalid result_canonical",
+	"result_canonical cannot be empty",
+	"unsupported action_id",
+}
+
+// isPermanentSettleError reports whether a settle_market broadcast error is a
+// deterministic, non-recoverable failure that must not be retried. Nonce and
+// network/broadcast errors mean the tx never executed deterministically, so they
+// are explicitly excluded and left to the retry path.
+func isPermanentSettleError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if isNonceError(err) {
+		return false
+	}
+	msg := strings.ToLower(err.Error())
+	for _, sig := range permanentSettleErrorSignatures {
+		if strings.Contains(msg, sig) {
+			return true
+		}
+	}
+	return false
+}
+
 // RequestAttestationForMarket broadcasts a request_attestation transaction for a market
 // with retry logic (exponential backoff for transient errors like nonce conflicts)
 func (e *EngineOperations) RequestAttestationForMarket(

extensions/tn_settlement/internal/engine_ops_test.go

@@ -693,3 +693,69 @@ func (m *mockReadDBForQueryComponents) Execute(ctx context.Context, stmt string,
 func (m *mockReadDBForQueryComponents) BeginTx(ctx context.Context) (sql.Tx, error) {
 	return nil, fmt.Errorf("mock read handle does not support transactions")
 }
+
+// =============================================================================
+// Test: Permanent-failure classification (infinite-retry fix)
+// =============================================================================
+
+func TestIsPermanentSettleError(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil", nil, false},
+		{"market-368 empty binary payload", errors.New("transaction failed with code 65535: binary action result must be 32 bytes (abi-encoded bool), got 128"), true},
+		{"wrapped malformed canonical", fmt.Errorf("settle: %w", errors.New("invalid result_canonical: too short for version")), true},
+		{"empty canonical", errors.New("result_canonical cannot be empty"), true},
+		{"boolean decode failure", errors.New("failed to decode boolean ABI result: bad payload"), true},
+		{"numeric market empty payload (368 analog)", errors.New("transaction failed with code 65535: result payload contains no values"), true},
+		{"numeric market malformed payload", errors.New("transaction failed with code 65535: failed to decode ABI result payload: bad"), true},
+		{"numeric market wrong array count", errors.New("expected 2 arrays (timestamps, values), got 1"), true},
+		{"nonce conflict is transient", errors.New("invalid nonce: expected 5 got 4"), false},
+		{"network/broadcast is transient", errors.New("broadcast tx: connection refused"), false},
+		{"unrelated action revert stays transient (conservative)", errors.New("transaction failed with code 1: insufficient collateral"), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.want, isPermanentSettleError(tc.err))
+		})
+	}
+}
+
+// TestBroadcastSettleMarketWithRetry_PermanentFailureStopsImmediately asserts a
+// permanent (deterministic) settle_market failure is attempted exactly once —
+// never retried — and returns the ErrPermanentSettleFailure sentinel so the
+// scheduler can quarantine the market. This is the nonce-burn / block-spam fix.
+func TestBroadcastSettleMarketWithRetry_PermanentFailureStopsImmediately(t *testing.T) {
+	accounts := &mockAccounts{}
+	broadcaster := &mockBroadcaster{
+		failUntil: 10, // always return the failed result below
+		successResult: &ktypes.TxResult{
+			Code: 65535,
+			Log:  "ERROR: binary action result must be 32 bytes (abi-encoded bool), got 128",
+		},
+	}
+
+	priv, _, err := crypto.GenerateSecp256k1Key(nil)
+	require.NoError(t, err)
+	signer := auth.GetUserSigner(priv)
+
+	ops := &EngineOperations{
+		logger:   log.New(),
+		accounts: accounts,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	err = ops.BroadcastSettleMarketWithRetry(
+		ctx, "test-chain", signer, broadcaster.broadcast,
+		368, // queryID
+		3,   // maxRetries — must be ignored for a permanent failure
+	)
+
+	require.Error(t, err)
+	require.ErrorIs(t, err, ErrPermanentSettleFailure)
+	require.Equal(t, 1, broadcaster.attempts, "a permanent failure must be attempted exactly once (no retries)")
+}

extensions/tn_settlement/scheduler/constants.go

@@ -16,3 +16,13 @@ const RetryBackoffInitial = 2 * time.Second
 
 // RetryBackoffMax is the maximum backoff duration for retries
 const RetryBackoffMax = 30 * time.Second
+
+// PermanentFailureReprobeCooldown is how long a market whose settlement failed
+// with a permanent (deterministic) error is quarantined before the scheduler
+// re-probes it once. A permanent failure — e.g. a malformed, immutable
+// attestation — recurs identically on every attempt, so re-broadcasting it each
+// poll only burns nonces and commits failed txs to blocks. Quarantining bounds
+// that to at most one failed tx per cooldown per stuck market, while still
+// auto-recovering a market an operator repairs (re-attestation): after the
+// cooldown, one probe is allowed through, and a success clears the quarantine.
+const PermanentFailureReprobeCooldown = 1 * time.Hour