From fc7ceb412687c07554f4bc2529d89223efc1294f Mon Sep 17 00:00:00 2001 From: owen Date: Tue, 30 Jun 2026 08:27:37 -0400 Subject: [PATCH] Retry Bungie Cloudflare blocks via queue backoff instead of dropping messages. Use one quick in-process CF retry (~2s with jitter) and two transient retries per request, then republish failed crawl messages indefinitely on the delayed exchange (30m cap) so workers are not blocked during long Cloudflare outages. Co-authored-by: Cursor --- .../queue-workers/activity_history.go | 2 +- lib/messaging/queue-workers/character_fill.go | 2 +- lib/messaging/queue-workers/clan_crawl.go | 2 +- lib/messaging/queue-workers/player_crawl.go | 2 +- lib/utils/network/retry.go | 15 +++++++------- lib/web/bungie/client.go | 20 ++++++++----------- 6 files changed, 19 insertions(+), 24 deletions(-) diff --git a/lib/messaging/queue-workers/activity_history.go b/lib/messaging/queue-workers/activity_history.go index b4b5eda..760a196 100644 --- a/lib/messaging/queue-workers/activity_history.go +++ b/lib/messaging/queue-workers/activity_history.go @@ -25,7 +25,7 @@ func ActivityHistoryTopic() processing.Topic { ScaleUpPercent: 0.2, ScaleDownPercent: 0.1, BungieSystemDeps: []string{"Destiny2", "Activities", "D2Profiles"}, - MaxRetryCount: 3, + MaxRetryCount: 0, RetryDelay: processing.ExponentialRetryDelay(time.Second), }, processActivityHistory) } diff --git a/lib/messaging/queue-workers/character_fill.go b/lib/messaging/queue-workers/character_fill.go index aa8c37e..74b4184 100644 --- a/lib/messaging/queue-workers/character_fill.go +++ b/lib/messaging/queue-workers/character_fill.go @@ -25,7 +25,7 @@ func CharacterFillTopic() processing.Topic { ScaleUpPercent: 0.2, ScaleDownPercent: 0.1, BungieSystemDeps: []string{"Destiny2", "D2Characters"}, - MaxRetryCount: 4, // Character data is useful but not critical + MaxRetryCount: 0, RetryDelay: processing.ExponentialRetryDelay(5 * time.Minute), }, processCharacterFill) } diff --git a/lib/messaging/queue-workers/clan_crawl.go b/lib/messaging/queue-workers/clan_crawl.go index 53705ab..22c84b7 100644 --- a/lib/messaging/queue-workers/clan_crawl.go +++ b/lib/messaging/queue-workers/clan_crawl.go @@ -25,7 +25,7 @@ func ClanCrawlTopic() processing.Topic { ScaleUpPercent: 0.2, ScaleDownPercent: 0.1, BungieSystemDeps: []string{"Groups", "Clans", "Destiny2"}, - MaxRetryCount: 5, + MaxRetryCount: 0, RetryDelay: processing.ExponentialRetryDelay(time.Second), }, processClanCrawl) } diff --git a/lib/messaging/queue-workers/player_crawl.go b/lib/messaging/queue-workers/player_crawl.go index 7047f81..bf538ef 100644 --- a/lib/messaging/queue-workers/player_crawl.go +++ b/lib/messaging/queue-workers/player_crawl.go @@ -32,7 +32,7 @@ func PlayerCrawlTopic() processing.Topic { ConsecutiveChecksDown: 3, // More conservative for scale-down ScaleCooldown: 30 * time.Second, // Shorter cooldown for faster scaling BungieSystemDeps: []string{"Destiny2", "D2Profiles", "Activities"}, - MaxRetryCount: 5, // Reduced from 12 to prevent exponential retry amplification + MaxRetryCount: 0, // Unlimited queue backoff (30m cap); CF blocks retry via delayed exchange RetryDelay: processing.ExponentialRetryDelay(5 * time.Minute), }, processPlayerCrawl) } diff --git a/lib/utils/network/retry.go b/lib/utils/network/retry.go index a83f1db..ca61501 100644 --- a/lib/utils/network/retry.go +++ b/lib/utils/network/retry.go @@ -30,16 +30,15 @@ func TransientNetworkErrorRetryConfig() retry.RetryConfig { } } -// Uses more attempts and longer delays to handle Cloudflare's rate limiting and blocking pages -// Only retries if the error is a Cloudflare error -// Params: logger: logger to use for logging, loggingFields: fields to add to the logging +// CloudflareRetryConfig performs one quick retry after ~2s for Cloudflare challenge pages. +// Longer outages are handled by Hermes queue backoff. func CloudflareRetryConfig(logger logging.Logger, loggingFields map[string]any) retry.RetryConfig { return retry.RetryConfig{ - MaxAttempts: 3, - InitialDelay: 1 * time.Second, - MaxDelay: 10 * time.Second, - Multiplier: 4, // Back off fast - Jitter: 0.2, // 20% jitter for better distribution of retries + MaxAttempts: 1, + InitialDelay: 2 * time.Second, + MaxDelay: 2 * time.Second, + Multiplier: 1.0, + Jitter: 0.2, OnRetry: func(attempt int, err error) { fields := map[string]any{ logging.ATTEMPTS: attempt, diff --git a/lib/web/bungie/client.go b/lib/web/bungie/client.go index 1c7dd11..5a9a76f 100644 --- a/lib/web/bungie/client.go +++ b/lib/web/bungie/client.go @@ -59,8 +59,8 @@ func (e *BungieResponseParseError) Error() string { return fmt.Sprintf("%s: unexpected bungie %d %s response: '%s'", e.Operation, e.StatusCode, e.ContentType, e.Title) } -// isKnownHTMLErrorPage checks if the error is a known HTML error page that should not be sent to Sentry -// These are expected/recoverable conditions that are handled by retry logic +// isKnownHTMLErrorPage checks if the error is a known HTML error page that should not be sent to Sentry. +// Cloudflare pages get one quick in-process retry; longer blocks use Hermes queue backoff. func (e *BungieResponseParseError) isKnownHTMLErrorPage() bool { if e.ContentType == "" || e.Title == "" { return false @@ -97,12 +97,9 @@ func (e *BungieResponseParseError) isKnownHTMLErrorPage() bool { } func get[T any](ctx context.Context, c *BungieClient, url netUrl.URL, operation string, params map[string]any) (BungieHttpResult[T], error) { - // Wraps the get in 2 layers of retry: - // Inner layer retries timeout and connection errors (excludes BungieError instances) - // Outer layer retries Cloudflare errors + // Outer: one ~2s Cloudflare retry. Inner: up to two quick transient retries. Queue handles the rest. return retry.WithRetryForResult(ctx, network.CloudflareRetryConfig(clientLogger, params), func(attempt int) (BungieHttpResult[T], error) { if attempt > 1 { - // add a query parameter to the url to indicate the retry attempt queryValues := url.Query() queryValues.Add("retry", fmt.Sprintf("%d", attempt)) url.RawQuery = queryValues.Encode() @@ -289,17 +286,16 @@ func IsTransientError(bungieErrorCode int, httpStatusCode int) bool { return true } -// BungieRetryConfig retries transient network errors for Bungie API calls -// It specifically excludes BungieError instances (application-level errors) from retries -// such as timeout, connection errors, and server errors (5xx) +// BungieRetryConfig retries transient network errors for Bungie API calls (timeout, connection, 5xx). +// At most two quick retries; Cloudflare and longer outages are handled by Hermes queue backoff. func BungieRetryConfig() retry.RetryConfig { transientRetryConfig := network.TransientNetworkErrorRetryConfig() return retry.RetryConfig{ - MaxAttempts: 3, + MaxAttempts: 2, InitialDelay: 50 * time.Millisecond, - MaxDelay: 5 * time.Second, + MaxDelay: 2 * time.Second, Multiplier: 2.0, - Jitter: 0.1, // 10% jitter + Jitter: 0.1, OnRetry: nil, ShouldRetry: func(err error) bool { // Check if this is a Bungie error (application-level error, not a network error)