fix(api): fail closed when getPayload delivery mutex hits Redis errors

services/api/service.go

@@ -1812,24 +1812,23 @@ func (api *RelayAPI) innerHandleGetPayload(w http.ResponseWriter, req *http.Requ
 	// Check whether getPayload has already been called -- TODO: do we need to allow multiple submissions of one blinded block?
 	err = api.redis.CheckAndSetLastSlotAndHashDelivered(uint64(slot), blockHash.String())
 	log = log.WithField("timestampAfterAlreadyDeliveredCheck", time.Now().UTC().UnixMilli())
-	if err != nil {
+	if status, msg, abort := getPayloadDeliveryCheckAbort(err); abort {
 		if errors.Is(err, datastore.ErrAnotherPayloadAlreadyDeliveredForSlot) {
 			// BAD VALIDATOR, 2x GETPAYLOAD FOR DIFFERENT PAYLOADS
 			log.Warn("validator called getPayload twice for different payload hashes")
-			api.RespondError(w, http.StatusBadRequest, "another payload for this slot was already delivered")
-			return
 		} else if errors.Is(err, datastore.ErrPastSlotAlreadyDelivered) {
 			// BAD VALIDATOR, 2x GETPAYLOAD FOR PAST SLOT
 			log.Warn("validator called getPayload for past slot")
-			api.RespondError(w, http.StatusBadRequest, "payload for this slot was already delivered")
-			return
 		} else if errors.Is(err, redis.TxFailedErr) {
 			// BAD VALIDATOR, 2x GETPAYLOAD + RACE
 			log.Warn("validator called getPayload twice (race)")
-			api.RespondError(w, http.StatusBadRequest, "payload for this slot was already delivered (race)")
-			return
+		} else {
+			// Unexpected Redis/Watch failure: previously logged and continued into
+			// payload delivery without a successful last-slot/hash record.
+			log.WithError(err).Error("redis.CheckAndSetLastSlotAndHashDelivered failed")
 		}
-		log.WithError(err).Error("redis.CheckAndSetLastSlotAndHashDelivered failed")
+		api.RespondError(w, status, msg)
+		return
 	}
 
 	// Handle early/late requests
@@ -3306,3 +3305,22 @@ func (api *RelayAPI) processValidatorRegistrationsSSZ(regs []*builderApiV1.Signe
 
 	return newRegistrations, nil, nil
 }
+
+// getPayloadDeliveryCheckAbort maps CheckAndSetLastSlotAndHashDelivered errors
+// to an HTTP response. Unexpected Redis failures must abort delivery (fail closed)
+// so getPayload cannot proceed without a successful last-slot/hash record.
+func getPayloadDeliveryCheckAbort(err error) (status int, msg string, abort bool) {
+	if err == nil {
+		return 0, "", false
+	}
+	if errors.Is(err, datastore.ErrAnotherPayloadAlreadyDeliveredForSlot) {
+		return http.StatusBadRequest, "another payload for this slot was already delivered", true
+	}
+	if errors.Is(err, datastore.ErrPastSlotAlreadyDelivered) {
+		return http.StatusBadRequest, "payload for this slot was already delivered", true
+	}
+	if errors.Is(err, redis.TxFailedErr) {
+		return http.StatusBadRequest, "payload for this slot was already delivered (race)", true
+	}
+	return http.StatusInternalServerError, "failed to check payload delivery status", true
+}

services/api/utils_test.go

@@ -1,15 +1,48 @@
 package api
 
 import (
+	"fmt"
 	"net/http"
 	"strconv"
 	"testing"
 	"time"
 
 	"github.com/flashbots/mev-boost-relay/common"
+	"github.com/flashbots/mev-boost-relay/datastore"
+	"github.com/redis/go-redis/v9"
 	"github.com/stretchr/testify/require"
 )
 
+func TestGetPayloadDeliveryCheckAbort(t *testing.T) {
+	t.Parallel()
+
+	status, msg, abort := getPayloadDeliveryCheckAbort(nil)
+	require.False(t, abort)
+	require.Equal(t, 0, status)
+	require.Empty(t, msg)
+
+	status, msg, abort = getPayloadDeliveryCheckAbort(datastore.ErrAnotherPayloadAlreadyDeliveredForSlot)
+	require.True(t, abort)
+	require.Equal(t, http.StatusBadRequest, status)
+	require.Contains(t, msg, "another payload")
+
+	status, msg, abort = getPayloadDeliveryCheckAbort(datastore.ErrPastSlotAlreadyDelivered)
+	require.True(t, abort)
+	require.Equal(t, http.StatusBadRequest, status)
+	require.Contains(t, msg, "already delivered")
+
+	status, msg, abort = getPayloadDeliveryCheckAbort(redis.TxFailedErr)
+	require.True(t, abort)
+	require.Equal(t, http.StatusBadRequest, status)
+	require.Contains(t, msg, "race")
+
+	// Unexpected Redis/Watch errors must fail closed (previously logged and continued).
+	status, msg, abort = getPayloadDeliveryCheckAbort(fmt.Errorf("redis: connection refused"))
+	require.True(t, abort)
+	require.Equal(t, http.StatusInternalServerError, status)
+	require.Equal(t, "failed to check payload delivery status", msg)
+}
+
 func TestGetHeaderContentType(t *testing.T) {
 	for _, tc := range []struct {
 		header   http.Header