Send the faded order hashes that caused a circuit breaker block in the blocking notification

lib/constants.ts

@@ -19,6 +19,7 @@ export const DYNAMO_TABLE_KEY = {
   FADE_WINDOW_START: 'fadeWindowStart',
   CONSECUTIVE_BLOCKS: 'consecutiveBlocks',
   CONSECUTIVE_CLEAN_RUNS: 'consecutiveCleanRuns',
+  FADED_ORDER_HASHES: 'fadedOrderHashes',
 };
 
 export const POST_ORDER_ERROR_REASON = {

lib/cron/fade-rate-v2.ts

@@ -66,12 +66,20 @@
   // fillers sit at the cap constantly.) Surfaced as an aggregate metric so per-address
   // volume outgrowing the window shows on the dashboard instead of as a stuck recovery.
   saturatedAddresses: number;
+  // Hashes of the faded orders behind each rate cohort, so a block entry can carry the
+  // reason for the block (surfaced to the filler in the blocking notification).
+  windowFadedOrderHashes?: string[];
+  duringBlockFadedOrderHashes?: string[];
 };
 export type FillerFadeStatsMap = Record<string, FillerFadeStats>;
 export type FillerTimestamps = Map<string, Omit<TimestampRepoRow, 'hash'>>;
 
 export const BASE_BLOCK_SECS = 60 * 15; // 15 minutes
 
+/** Bound on the faded order hashes persisted per block entry (most recent kept), so
+    repeatedly extended blocks can't grow the Dynamo item without limit. */
+export const MAX_FADED_ORDER_HASHES = 50;
+
 // Laplace (additive) smoothing applied to each filler's fade rate so a few fades on a
 // small sample don't trip the breaker. Equivalent to seeding every filler with ALPHA
 // pretend-fades and BETA pretend-clean-fills. Prior mean = ALPHA/(ALPHA+BETA) = 1/20 = 5%.
@@ -317,6 +325,12 @@
         fadeWindowStart: extendedBlockUntil, // clean slate resumes when the extended block ends
         consecutiveBlocks,
         consecutiveCleanRuns: 0, // faded while blocked: recovery streak restarts
+        // the extended block was caused by the previously stored fades plus the
+        // in-flight fades that extended it
+        fadedOrderHashes: mergeFadedOrderHashes(
+          fillerTimestamp.fadedOrderHashes,
+          stats.duringBlockFadedOrderHashes ?? []
+        ),
       });
     } else if (isCurrentlyBlocked) {
       // Blocked with the in-flight cohort under threshold - keep existing block and floor,
@@ -330,6 +344,8 @@
         fadeWindowStart: fillerTimestamp.fadeWindowStart,
         consecutiveBlocks,
         consecutiveCleanRuns: newFades > 0 ? 0 : previousCleanRuns,
+        // carry forward the fades that caused the still-active block
+        fadedOrderHashes: fillerTimestamp.fadedOrderHashes,
       });
     } else if (fadeRate > FADE_RATE_BLOCK_THRESHOLD || duringBlockRate > FADE_RATE_BLOCK_THRESHOLD) {
       // duringBlockRate covers in-flight fades that landed near the end of a block that
@@ -353,6 +369,11 @@
         fadeWindowStart: blockUntilTimestamp, // clean slate resumes when the block ends
         consecutiveBlocks,
         consecutiveCleanRuns: 0, // blocked: recovery streak restarts
+        // the faded orders from whichever cohort(s) exceeded the threshold
+        fadedOrderHashes: mergeFadedOrderHashes(undefined, [
+          ...(fadeRate > FADE_RATE_BLOCK_THRESHOLD ? stats.windowFadedOrderHashes ?? [] : []),
+          ...(duringBlockRate > FADE_RATE_BLOCK_THRESHOLD ? stats.duringBlockFadedOrderHashes ?? [] : []),
+        ]),
       });
     } else {
       // Under threshold: not blocked. Reset blockUntilTimestamp to unblocked. Escalation
@@ -423,6 +444,14 @@
   return [...effectiveBlockUntil.values()].filter((blockUntil) => blockUntil > now).length;
 }
 
+/* combine the faded order hashes already stored on a block entry with the new window's,
+   deduped and capped to the most recent MAX_FADED_ORDER_HASHES.
+   Returns undefined when there are none, so no attribute is written to the entry. */
+function mergeFadedOrderHashes(existing: string[] | undefined, incoming: string[]): string[] | undefined {
+  const merged = Array.from(new Set([...(existing ?? []), ...incoming]));
+  return merged.length > 0 ? merged.slice(-MAX_FADED_ORDER_HASHES) : undefined;
+}
+
 /* Laplace-smoothed fade rate: pretend we've already seen LAPLACE_ALPHA fades and
    LAPLACE_BETA clean fills, so small samples are pulled toward the prior mean instead
    of swinging to 0% or 100%. */
@@ -486,6 +515,8 @@
       newFades: number;
       chronicFades: number;
       chronicTotal: number;
+      windowFadedOrderHashes: string[];
+      blockFadedOrderHashes: string[];
     }
   > = {};
   // per-address row count and oldest deadline, to detect addresses whose latest-N window has
@@ -514,6 +545,8 @@
         newFades: 0,
         chronicFades: 0,
         chronicTotal: 0,
+        windowFadedOrderHashes: [],
+        blockFadedOrderHashes: [],
       };
     }
     const windowStart = fillerTimestamp?.fadeWindowStart ?? UNBLOCKED_BLOCK_UNTIL_TIMESTAMP;
@@ -542,12 +575,18 @@
       // Rate window: orders completed after the filler's last block ended (clean slate).
       tallies[fillerHash].windowTotal += 1;
       tallies[fillerHash].windowFades += row.faded;
+      if (row.faded && row.orderHash) {
+        tallies[fillerHash].windowFadedOrderHashes.push(row.orderHash);
+      }
     } else if (row.deadline > lastExaminedTimestamp) {
       // During-block cohort: completed since the last cron run but on/before the block end,
       // i.e. in flight during the block. Never overlaps the rate window (deadline <= floor),
       // so during-block orders stay excluded from the post-block clean slate.
       tallies[fillerHash].blockTotal += 1;
       tallies[fillerHash].blockFades += row.faded;
+      if (row.faded && row.orderHash) {
+        tallies[fillerHash].blockFadedOrderHashes.push(row.orderHash);
+      }
     }
   });
 
@@ -559,7 +598,7 @@
   addressRowCounts.forEach((count, addr) => {
     const oldestDeadline = addressOldestDeadline.get(addr) ?? 0;
     if (count >= ORDERS_PER_FILLER_LIMIT && oldestDeadline > now - STREAK_FINALITY_LAG_SECS) {
       const hash = addressHash.get(addr)!;
       saturatedByHash[hash] = (saturatedByHash[hash] ?? 0) + 1;
       log?.info({ addr, hash, count, oldestDeadline }, 'filler address window has outrun the streak finality horizon');
     }
@@ -575,6 +614,8 @@
       chronicRate: t.chronicTotal > 0 ? t.chronicFades / t.chronicTotal : 0,
       chronicTotal: t.chronicTotal,
       saturatedAddresses: saturatedByHash[hash] ?? 0,
+      windowFadedOrderHashes: t.windowFadedOrderHashes,
+      duringBlockFadedOrderHashes: t.blockFadedOrderHashes,
     };
   });
   log?.info({ tallies, stats }, 'fade stats by filler');

lib/providers/circuit-breaker/dynamo.ts

@@ -56,16 +56,17 @@
       if (fillerTimestamps.size) {
         this.log.info({ fillerTimestamps: [...fillerTimestamps.entries()] }, `Circuit breaker config used`);
         const enabledEndpoints = endpoints.filter((e) => {
           return !(fillerTimestamps.has(e.endpoint) && fillerTimestamps.get(e.endpoint)!.blockUntilTimestamp > now);
         });
         const disabledEndpoints = endpoints
           .filter((e) => {
             return fillerTimestamps.has(e.endpoint) && fillerTimestamps.get(e.endpoint)!.blockUntilTimestamp > now;
           })
           .map((e) => {
             return {
               webhook: e,
               blockUntil: fillerTimestamps.get(e.endpoint)!.blockUntilTimestamp,
+              fadedOrderHashes: fillerTimestamps.get(e.endpoint)!.fadedOrderHashes,
             };
           });
 

lib/providers/circuit-breaker/index.ts

@@ -5,6 +5,9 @@ export interface EndpointStatuses {
   disabled: {
     webhook: WebhookConfiguration;
     blockUntil: number;
+    // hashes of the faded orders that caused the block;
+    // absent on block entries written before hashes were persisted
+    fadedOrderHashes?: string[];
   }[];
 }
 

lib/providers/circuit-breaker/mock.ts

@@ -24,6 +24,7 @@ export class MockV2CircuitBreakerConfigurationProvider implements CircuitBreaker
           return {
             webhook: e,
             blockUntil: fillerTimestamps.get(e.endpoint)!.blockUntilTimestamp,
+            fadedOrderHashes: fillerTimestamps.get(e.endpoint)!.fadedOrderHashes,
           };
         });
 

lib/quoters/WebhookQuoter.ts

@@ -110,7 +110,7 @@ export class WebhookQuoter implements Quoter {
 
     // should not await and block
     if (!isPermissionedToken) {
-      Promise.allSettled(disabledEndpoints.map((e) => this.notifyBlock(e))).then((results) => {
+      Promise.allSettled(disabledEndpoints.map((e) => this.notifyBlock(e, request))).then((results) => {
         this.log.info({ requestId: request.requestId, results }, 'Notified disabled endpoints');
       });
     }
@@ -424,7 +424,10 @@ export class WebhookQuoter implements Quoter {
     }
   }
 
-  private async notifyBlock(status: { webhook: WebhookConfiguration; blockUntil: number }): Promise<void> {
+  private async notifyBlock(
+    status: { webhook: WebhookConfiguration; blockUntil: number; fadedOrderHashes?: string[] },
+    request: QuoteRequest
+  ): Promise<void> {
     const axiosConfig = {
       timeout: NOTIFICATION_TIMEOUT_MS,
       ...(!!status.webhook.headers && { headers: status.webhook.headers }),
@@ -434,6 +437,11 @@ export class WebhookQuoter implements Quoter {
         status.webhook.endpoint,
         {
           blockUntilTimestamp: status.blockUntil,
+          // The faded order(s) that caused this block, so fillers know exactly which
+          // orders triggered the circuit breaker. Omitted for block entries written
+          // before the faded order hashes were persisted alongside the block.
+          ...(!!status.fadedOrderHashes?.length && { orderHashes: status.fadedOrderHashes }),
+          ...(request.quoteId && { quoteId: request.quoteId }),
         },
         axiosConfig
       )

lib/repositories/base.ts

@@ -38,6 +38,10 @@ export type TimestampRepoRow = {
   // while unblocked and escalated. consecutiveBlocks decays one level per
   // CLEAN_RUNS_PER_DECAY of these; any new fade resets the streak, idle runs freeze it.
   consecutiveCleanRuns: number;
+  // hashes of the faded orders that caused the current block; absent on rows written
+  // before this field existed. Deliberately optional on writes: omitting it on the
+  // full-item put is how an expired block's hashes are cleared.
+  fadedOrderHashes?: string[];
 };
 
 // Rows round-trip as native numbers now (number-typed attributes read via a wrapNumbers:false

lib/repositories/fades-repository.ts

@@ -19,6 +19,7 @@ export type V2FadesRowType = {
   faded: number;
   postTimestamp: number;
   deadline: number; // When the order outcome was finalized
+  orderHash: string;
 };
 
 /**
@@ -51,10 +52,10 @@ export class V2FadesRepository extends BaseRedshiftRepository {
     const stmtId = await this.executeStatement(V2_FADE_RATE_SQL, V2FadesRepository.log, { waitTimeMs: 2_000 });
     const response = await this.client.send(new GetStatementResultCommand({ Id: stmtId }));
     /* result should be in the following format
-        | rfqFiller    |   postTimestamp  |   deadline   |   faded  |
-        |---- bar ------|---- 12222222 ---|--- 12222282 -|---- 0 ---|
-        |---- foo ------|---- 12345679 ---|--- 12345739 -|---- 1 ---|
-        |---- foo ------|---- 12345678 ---|--- 12345738 -|---- 0 ---|
+        | rfqFiller    |   postTimestamp  |   deadline   |   faded  |   orderHash  |
+        |---- bar ------|---- 12222222 ---|--- 12222282 -|---- 0 ---|---- 0xbar ---|
+        |---- foo ------|---- 12345679 ---|--- 12345739 -|---- 1 ---|---- 0xfoo ---|
+        |---- foo ------|---- 12345678 ---|--- 12345738 -|---- 0 ---|---- 0xbaz ---|
       */
     const result = response.Records;
     if (!result) {
@@ -68,6 +69,7 @@ export class V2FadesRepository extends BaseRedshiftRepository {
         postTimestamp: parseInt(row[1].stringValue as string),
         deadline: parseInt(row[2].stringValue as string),
         faded: Number(row[3].longValue as number),
+        orderHash: row[4].stringValue as string,
       };
       return formattedRow;
     });
@@ -91,7 +93,7 @@ WITH latestOrdersV2 AS (
   LIMIT ${FADE_QUERY_ROW_LIMIT}
 )
 SELECT
-    latestOrdersV2.chainid as chainId, latestOrdersV2.ordertype as orderType, latestOrdersV2.filler as rfqFiller, latestOrdersV2.startTime as decayStartTime, latestOrdersV2.quoteid, archivedorders.filler as actualFiller, latestOrdersV2.createdat as postTimestamp, latestOrdersV2.deadline as deadline, archivedorders.txhash as txHash, archivedOrders.fillTimestamp as fillTimestamp, archivedorders.fillTimeBlocks as fillTimeBlocks, archivedOrders.tokenIn as tokenIn, archivedOrders.tokenOut as tokenOut,
+    latestOrdersV2.chainid as chainId, latestOrdersV2.ordertype as orderType, latestOrdersV2.filler as rfqFiller, latestOrdersV2.startTime as decayStartTime, latestOrdersV2.quoteid, latestOrdersV2.orderhash as orderHash, archivedorders.filler as actualFiller, latestOrdersV2.createdat as postTimestamp, latestOrdersV2.deadline as deadline, archivedorders.txhash as txHash, archivedOrders.fillTimestamp as fillTimestamp, archivedorders.fillTimeBlocks as fillTimeBlocks, archivedOrders.tokenIn as tokenIn, archivedOrders.tokenOut as tokenOut,
     CASE
       WHEN latestOrdersV2.inputstartamount = latestOrdersV2.inputendamount THEN 'EXACT_INPUT'
       ELSE 'EXACT_OUTPUT'
@@ -111,10 +113,11 @@ LIMIT ${FADE_QUERY_ROW_LIMIT}
 `;
 
 const V2_FADE_RATE_SQL = `
-SELECT 
+SELECT
     rfqFiller,
     postTimestamp,
     deadline,
+    -- the parser in getFades() indexes columns by position; keep this ordering in sync
     CASE
       -- Never filled (any order type) => fade.
       WHEN fillTimestamp IS NULL THEN 1
@@ -127,7 +130,8 @@ SELECT
       -- Dutch_V2 (time-based decay): filled after decay start => fade.
       WHEN orderType = '${OrderType.Dutch_V2}' AND decayStartTime < fillTimestamp THEN 1
       ELSE 0
-    END AS faded
+    END AS faded,
+    orderHash
 FROM latestRfqsV2
 WHERE LOWER(tokenIn) NOT IN (${PERMISSIONED_TOKENS.map((token) => `'${token.address.toLowerCase()}'`).join(',')})
 AND LOWER(tokenOut) NOT IN (${PERMISSIONED_TOKENS.map((token) => `'${token.address.toLowerCase()}'`).join(',')})

lib/repositories/timestamp-repository.ts

@@ -50,6 +50,7 @@ export class TimestampRepository implements BaseTimestampRepository {
         [`${DYNAMO_TABLE_KEY.FADE_WINDOW_START}`]: { type: 'number' },
         [`${DYNAMO_TABLE_KEY.CONSECUTIVE_BLOCKS}`]: { type: 'number' },
         [`${DYNAMO_TABLE_KEY.CONSECUTIVE_CLEAN_RUNS}`]: { type: 'number' },
+        [`${DYNAMO_TABLE_KEY.FADED_ORDER_HASHES}`]: { type: 'list' },
       },
       table: table,
       autoExecute: true,
@@ -74,6 +75,7 @@ export class TimestampRepository implements BaseTimestampRepository {
           [`${DYNAMO_TABLE_KEY.FADE_WINDOW_START}`]: row.fadeWindowStart,
           [`${DYNAMO_TABLE_KEY.CONSECUTIVE_BLOCKS}`]: row.consecutiveBlocks,
           [`${DYNAMO_TABLE_KEY.CONSECUTIVE_CLEAN_RUNS}`]: row.consecutiveCleanRuns,
+          ...(row.fadedOrderHashes && { [`${DYNAMO_TABLE_KEY.FADED_ORDER_HASHES}`]: row.fadedOrderHashes }),
         });
       }),
       {
@@ -96,6 +98,8 @@ export class TimestampRepository implements BaseTimestampRepository {
       fadeWindowStart: Item?.fadeWindowStart ?? UNBLOCKED_BLOCK_UNTIL_TIMESTAMP,
       consecutiveBlocks: Item?.consecutiveBlocks ?? 0,
       consecutiveCleanRuns: Item?.consecutiveCleanRuns ?? 0,
+      // absent on rows written before this attribute existed
+      fadedOrderHashes: Item?.fadedOrderHashes,
     };
   }
 
@@ -119,6 +123,7 @@ export class TimestampRepository implements BaseTimestampRepository {
         fadeWindowStart: row.fadeWindowStart ?? UNBLOCKED_BLOCK_UNTIL_TIMESTAMP,
         consecutiveBlocks: row.consecutiveBlocks ?? 0,
         consecutiveCleanRuns: row.consecutiveCleanRuns ?? 0,
+        fadedOrderHashes: row.fadedOrderHashes,
       };
     });
   }
@@ -133,6 +138,7 @@ export class TimestampRepository implements BaseTimestampRepository {
         fadeWindowStart: row.fadeWindowStart,
         consecutiveBlocks: row.consecutiveBlocks,
         consecutiveCleanRuns: row.consecutiveCleanRuns,
+        fadedOrderHashes: row.fadedOrderHashes,
       });
     });
     return res;