From c6c9d4946c253b176639bc696f3c8e739d66aaf4 Mon Sep 17 00:00:00 2001 From: Chris Czub Date: Tue, 21 Jul 2026 22:12:22 -0400 Subject: [PATCH 1/3] Fix retry loop in main(): bounded retries, backoff, error classification The retries counter increment was commented out, so the retry guard could never fire and any persistent error spun the loop forever with zero delay, restarting the full mailbox scan from page one each time. - Re-enable the retry counter and return an anyhow::Error after MAX_RETRIES attempts instead of panicking, for a clean nonzero exit. - Classify errors before retrying: only transient failures are retried (SQLite SQLITE_BUSY/SQLITE_LOCKED including extended codes, sqlx pool timeouts, and Gmail API 429/5xx responses); auth failures, other 4xx, IO errors, etc. fail fast on first occurrence. - Space retries with exponential backoff (1s doubling, capped at 32s) plus up to 1s of jitter via tokio::time::sleep. - Have work() resume from the last fully processed page's next_page_token on retry instead of restarting from page one. Closes #3 --- src/main.rs | 117 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/src/main.rs b/src/main.rs index 5983189..09df3ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use std::str::FromStr; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use futures::TryStreamExt; use google_gmail1::api::Message; @@ -65,55 +66,105 @@ async fn main() -> anyhow::Result<()> { auth, ); - // Some kind of exponential backpressure on a worker would be nicer - let retries = 0; + // Retry transient failures with exponential backoff; fail fast on everything else. + let mut retries = 0; + let mut resume_token: Option = None; loop { - // TODO: lol handle these better, i keep getting deadlocks but wanna just churn some emails - // retries += 1; - if retries > 3 { - panic!("Too many retries"); - } - - let res = work(&pool, &mut hub).await; - if res.is_ok() { - break; + match work(&pool, &mut hub, &mut resume_token).await { + Ok(()) => break, + Err(err) if !is_transient(&err) => { + return Err(err.context("giving up: error is not retryable")); + } + Err(err) => { + retries += 1; + if retries > MAX_RETRIES { + return Err(err.context(format!("giving up after {MAX_RETRIES} retries"))); + } + let delay = backoff_delay(retries); + println!( + "Transient error (attempt {retries}/{MAX_RETRIES}), retrying in {delay:?}: {err:?}" + ); + tokio::time::sleep(delay).await; + } } - - println!("Error encountered, retrying: {:?}", res); } Ok(()) } -async fn work(pool: &Pool, hub: &mut GmailHub) -> anyhow::Result<()> { - // Fetch 500 messages at a time... - let result = hub - .users() - .messages_list("me") - .max_results(500) - .include_spam_trash(false) - .doit() - .await?; +const MAX_RETRIES: u32 = 3; - let mut next_page_token = result.1.next_page_token; +/// Exponential backoff (1s, 2s, 4s, ... capped at 32s) plus up to 1s of jitter. +fn backoff_delay(attempt: u32) -> Duration { + let base = Duration::from_secs(1 << attempt.saturating_sub(1).min(5)); + let jitter_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| u64::from(d.subsec_millis())) + .unwrap_or(0); + base + Duration::from_millis(jitter_ms) +} - parse_messages(pool, result.1.messages.unwrap_or_default(), hub).await?; +/// Only transient failures are worth retrying: SQLite busy/locked (the deadlocks +/// that motivated the retry loop) and Gmail API rate limits / server errors. +/// Auth failures, other 4xx responses, IO errors, etc. fail fast. +fn is_transient(err: &anyhow::Error) -> bool { + for cause in err.chain() { + if let Some(sqlx_err) = cause.downcast_ref::() { + return match sqlx_err { + sqlx::Error::Database(db_err) => { + // SQLITE_BUSY (5) and SQLITE_LOCKED (6); extended result codes + // (e.g. SQLITE_BUSY_SNAPSHOT = 517) keep the primary code in + // the low byte. + db_err + .code() + .and_then(|code| code.parse::().ok()) + .is_some_and(|code| matches!(code & 0xff, 5 | 6)) + } + // Timed out waiting for a pool connection while the DB is busy. + sqlx::Error::PoolTimedOut => true, + _ => false, + }; + } + if let Some(gmail_err) = cause.downcast_ref::() { + return match gmail_err { + google_gmail1::Error::Failure(response) => { + let status = response.status(); + status.as_u16() == 429 || status.is_server_error() + } + _ => false, + }; + } + } + false +} - while let Some(token) = next_page_token { - let result = hub +async fn work( + pool: &Pool, + hub: &mut GmailHub, + resume_token: &mut Option, +) -> anyhow::Result<()> { + // Fetch 500 messages at a time, starting from the last page we fully + // processed (so a retry doesn't re-list the whole mailbox from page one). + loop { + let mut call = hub .users() .messages_list("me") .max_results(500) - .include_spam_trash(false) - .page_token(&token) - .doit() - .await?; + .include_spam_trash(false); + if let Some(token) = resume_token.as_deref() { + call = call.page_token(token); + } + let result = call.doit().await?; - next_page_token = result.1.next_page_token; + let next_page_token = result.1.next_page_token; parse_messages(pool, result.1.messages.unwrap_or_default(), hub).await?; - } - Ok(()) + // Only advance the resume point once the page has been fully processed. + *resume_token = next_page_token; + if resume_token.is_none() { + return Ok(()); + } + } } async fn parse_messages( From 114cf404ced117ca6f701ee3a9443856b149e50d Mon Sep 17 00:00:00 2001 From: Chris Czub Date: Tue, 21 Jul 2026 22:37:10 -0400 Subject: [PATCH 2/3] Fix transient-error classification and retry budget in main() Two review findings on the retry loop: 1. Gmail 429/5xx responses were classified as non-retryable. With google-gmail1 7.0.0 / google-apis-common 8.0.0, the generated doit() code parses any non-success HTTP response body as JSON first and returns Error::BadRequest(value) when that succeeds. Gmail's 429 rate-limit and 5xx errors carry Google's standard JSON error envelope, so they never reached the Error::Failure arm and fell into is_transient's catch-all `false`, aborting the run with "giving up: error is not retryable". is_transient now also matches Error::BadRequest and reads the numeric `error.code` field from the envelope (429 or 5xx => transient); the Failure arm remains for non-JSON responses such as an HTML 502 from an intermediary. 2. The retry counter was never reset, so MAX_RETRIES bounded *lifetime* transient errors instead of consecutive ones: on a long scan, the 4th intermittent SQLITE_BUSY or Gmail 5xx would kill the run even if thousands of pages succeeded between failures. The counter now resets whenever the failing attempt advanced the resume token (i.e. made progress) since the previous attempt. Also adds unit tests for is_transient's Gmail error classification and removes the unused `tokio::task` import that failed `cargo clippy --all-targets -- -D warnings`. --- src/main.rs | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/main.rs b/src/main.rs index 09df3ab..38d211a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,12 +70,21 @@ async fn main() -> anyhow::Result<()> { let mut retries = 0; let mut resume_token: Option = None; loop { + let token_before_attempt = resume_token.clone(); match work(&pool, &mut hub, &mut resume_token).await { Ok(()) => break, Err(err) if !is_transient(&err) => { return Err(err.context("giving up: error is not retryable")); } Err(err) => { + // If this attempt advanced the resume point (i.e. fully processed + // at least one page) before failing, the previous errors were + // intermittent, not persistent — start the budget over so + // MAX_RETRIES bounds *consecutive* failures, not failures + // accumulated over the whole run. + if resume_token != token_before_attempt { + retries = 0; + } retries += 1; if retries > MAX_RETRIES { return Err(err.context(format!("giving up after {MAX_RETRIES} retries"))); @@ -127,6 +136,15 @@ fn is_transient(err: &anyhow::Error) -> bool { } if let Some(gmail_err) = cause.downcast_ref::() { return match gmail_err { + // Non-success HTTP responses whose body parses as Google's JSON + // error envelope (the normal case for Gmail 429s and 5xxs) are + // surfaced as `BadRequest(value)`, not `Failure`; the HTTP + // status is the numeric `error.code` field in the envelope. + google_gmail1::Error::BadRequest(value) => value + .pointer("/error/code") + .and_then(serde_json::Value::as_u64) + .is_some_and(|code| code == 429 || (500..=599).contains(&code)), + // Non-JSON error responses (e.g. an HTML 502 from a proxy). google_gmail1::Error::Failure(response) => { let status = response.status(); status.as_u16() == 429 || status.is_server_error() @@ -356,3 +374,46 @@ fn get_sender(message: &Message) -> anyhow::Result { .expect("expected sender for mail") .to_string()) } + +#[cfg(test)] +mod tests { + use super::is_transient; + use serde_json::json; + + fn envelope(code: u64) -> anyhow::Error { + // Google's standard JSON error envelope, as parsed into + // `google_gmail1::Error::BadRequest` by the generated doit() code. + anyhow::Error::new(google_gmail1::Error::BadRequest(json!({ + "error": { "code": code, "message": "boom", "status": "UNKNOWN" } + }))) + } + + #[test] + fn json_429_and_5xx_are_transient() { + assert!(is_transient(&envelope(429))); + assert!(is_transient(&envelope(500))); + assert!(is_transient(&envelope(503))); + } + + #[test] + fn json_4xx_other_than_429_is_not_transient() { + assert!(!is_transient(&envelope(400))); + assert!(!is_transient(&envelope(401))); + assert!(!is_transient(&envelope(403))); + assert!(!is_transient(&envelope(404))); + } + + #[test] + fn bad_request_without_error_code_is_not_transient() { + let err = anyhow::Error::new(google_gmail1::Error::BadRequest(json!({ + "message": "no envelope here" + }))); + assert!(!is_transient(&err)); + } + + #[test] + fn transient_cause_is_found_through_context_chain() { + let err = envelope(429).context("while listing messages"); + assert!(is_transient(&err)); + } +} From d54193c85e85df1003b1ffe24141647b67c908dd Mon Sep 17 00:00:00 2001 From: Chris Czub Date: Tue, 21 Jul 2026 23:38:03 -0400 Subject: [PATCH 3/3] Treat Gmail 403 rate-limit errors as transient Gmail delivers per-user rate limiting as HTTP 403 as well as 429: the API error guide documents 403 rateLimitExceeded and 403 userRateLimitExceeded (usageLimits domain) with the instruction to retry with exponential backoff. is_transient() only accepted 429/5xx from the JSON envelope, so a burst of messages_get calls that tripped the 250-quota-units/sec per-user limit surfaced a 403 BadRequest, classified it as fatal, and aborted the whole mailbox scan -- exactly the condition the retry machinery was built to survive. Now the BadRequest arm also treats 403 as transient when error.errors[*].reason is rateLimitExceeded/userRateLimitExceeded or error.status is RESOURCE_EXHAUSTED, while other 403s (e.g. insufficientPermissions, dailyLimitExceeded) still fail fast. Adds unit tests for both the retryable and fatal 403 variants. --- src/main.rs | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 38d211a..f5e8ada 100644 --- a/src/main.rs +++ b/src/main.rs @@ -140,10 +140,22 @@ fn is_transient(err: &anyhow::Error) -> bool { // error envelope (the normal case for Gmail 429s and 5xxs) are // surfaced as `BadRequest(value)`, not `Failure`; the HTTP // status is the numeric `error.code` field in the envelope. - google_gmail1::Error::BadRequest(value) => value - .pointer("/error/code") - .and_then(serde_json::Value::as_u64) - .is_some_and(|code| code == 429 || (500..=599).contains(&code)), + google_gmail1::Error::BadRequest(value) => { + match value + .pointer("/error/code") + .and_then(serde_json::Value::as_u64) + { + Some(code) if code == 429 || (500..=599).contains(&code) => true, + // Gmail delivers per-user rate limiting as 403 too + // (usageLimits domain, reason `rateLimitExceeded` / + // `userRateLimitExceeded`); Google's error guide says to + // retry those with backoff. Other 403s (e.g. + // `insufficientPermissions`, `dailyLimitExceeded`) are + // genuinely fatal and fail fast. + Some(403) => is_rate_limit_envelope(value), + _ => false, + } + } // Non-JSON error responses (e.g. an HTML 502 from a proxy). google_gmail1::Error::Failure(response) => { let status = response.status(); @@ -156,6 +168,28 @@ fn is_transient(err: &anyhow::Error) -> bool { false } +/// True when a Google JSON error envelope describes a retryable rate-limit +/// condition: `error.errors[*].reason` of `rateLimitExceeded` / +/// `userRateLimitExceeded`, or `error.status` of `RESOURCE_EXHAUSTED`. +fn is_rate_limit_envelope(value: &serde_json::Value) -> bool { + let reason_is_rate_limit = value + .pointer("/error/errors") + .and_then(serde_json::Value::as_array) + .is_some_and(|errors| { + errors.iter().any(|e| { + matches!( + e.get("reason").and_then(serde_json::Value::as_str), + Some("rateLimitExceeded" | "userRateLimitExceeded") + ) + }) + }); + reason_is_rate_limit + || value + .pointer("/error/status") + .and_then(serde_json::Value::as_str) + == Some("RESOURCE_EXHAUSTED") +} + async fn work( pool: &Pool, hub: &mut GmailHub, @@ -403,6 +437,39 @@ mod tests { assert!(!is_transient(&envelope(404))); } + fn envelope_403(reason: &str) -> anyhow::Error { + anyhow::Error::new(google_gmail1::Error::BadRequest(json!({ + "error": { + "code": 403, + "message": "boom", + "errors": [ + { "domain": "usageLimits", "reason": reason, "message": "boom" } + ] + } + }))) + } + + #[test] + fn json_403_rate_limit_reasons_are_transient() { + assert!(is_transient(&envelope_403("rateLimitExceeded"))); + assert!(is_transient(&envelope_403("userRateLimitExceeded"))); + } + + #[test] + fn json_403_resource_exhausted_status_is_transient() { + let err = anyhow::Error::new(google_gmail1::Error::BadRequest(json!({ + "error": { "code": 403, "message": "boom", "status": "RESOURCE_EXHAUSTED" } + }))); + assert!(is_transient(&err)); + } + + #[test] + fn json_403_non_rate_limit_reasons_are_not_transient() { + assert!(!is_transient(&envelope_403("insufficientPermissions"))); + assert!(!is_transient(&envelope_403("dailyLimitExceeded"))); + assert!(!is_transient(&envelope_403("domainPolicy"))); + } + #[test] fn bad_request_without_error_code_is_not_transient() { let err = anyhow::Error::new(google_gmail1::Error::BadRequest(json!({