diff --git a/src-tauri/src/buzzheavier.rs b/src-tauri/src/buzzheavier.rs index de22eb8..f9e6b2d 100644 --- a/src-tauri/src/buzzheavier.rs +++ b/src-tauri/src/buzzheavier.rs @@ -1,5 +1,5 @@ use crate::download_resolver::{ - Provider, ResolveError, ResolvedDownload, ResolvedFile, network_error, + Provider, ResolveError, ResolvedDownload, ResolvedFile, network_error, retry_after, }; use regex::Regex; use reqwest::{ @@ -82,9 +82,10 @@ async fn fetch(client: &Client, url: &str, headers: HeaderMap) -> Result Err("The Buzzheavier file was not found".into()), - StatusCode::TOO_MANY_REQUESTS => { - Err("Buzzheavier rate limit reached, try again later".into()) - } + StatusCode::TOO_MANY_REQUESTS => Err(ResolveError::RateLimited { + message: "Buzzheavier rate limit reached, try again later".into(), + retry_after: retry_after(response.headers()), + }), status if status.is_success() => Ok(response), status => Err(ResolveError::Http(status)), }; @@ -93,7 +94,10 @@ async fn fetch(client: &Client, url: &str, headers: HeaderMap) -> Result Result { diff --git a/src-tauri/src/download_queue.rs b/src-tauri/src/download_queue.rs index 135977f..40b388c 100644 --- a/src-tauri/src/download_queue.rs +++ b/src-tauri/src/download_queue.rs @@ -1257,6 +1257,11 @@ enum TransferError { Network(String), Server(String), Link(String), + /// The host asked to wait before the next attempt on this link. + RateLimited { + message: String, + delay: Duration, + }, } impl std::fmt::Display for TransferError { @@ -1266,7 +1271,8 @@ impl std::fmt::Display for TransferError { Self::Local(message) | Self::Network(message) | Self::Server(message) - | Self::Link(message) => formatter.write_str(message), + | Self::Link(message) + | Self::RateLimited { message, .. } => formatter.write_str(message), } } } @@ -1278,6 +1284,10 @@ impl From for TransferError { ResolveError::Network(_) => Self::Network(message), ResolveError::Http(status) if status.is_server_error() => Self::Server(message), ResolveError::Http(_) | ResolveError::Link(_) => Self::Link(message), + ResolveError::RateLimited { retry_after, .. } => Self::RateLimited { + message, + delay: crate::download_selection::rate_limit_delay(retry_after), + }, } } } @@ -1349,6 +1359,7 @@ async fn transfer>( let mut current = job.clone(); let mut failure = "No supported download links are available".to_owned(); while let Some(uri) = plan.current().map(str::to_owned) { + let mut retry_delay = Duration::from_millis(250); if *stopped.borrow() { return Err(TransferError::Stopped); } @@ -1395,7 +1406,12 @@ async fn transfer>( Err(error @ TransferError::Network(_)) if !online().await => { return Err(error); } - Err(error) => failure = error.to_string(), + Err(error) => { + if let TransferError::RateLimited { delay, .. } = &error { + retry_delay = *delay; + } + failure = error.to_string(); + } } plan.fail(); save_mirror_state(database.clone(), ¤t, &plan, false).await?; @@ -1403,7 +1419,7 @@ async fn transfer>( tokio::select! { biased; _ = stopped.changed() => return Err(TransferError::Stopped), - () = tokio::time::sleep(Duration::from_millis(250)) => {}, + () = tokio::time::sleep(retry_delay) => {}, } } } diff --git a/src-tauri/src/download_resolver.rs b/src-tauri/src/download_resolver.rs index 641845b..7b66082 100644 --- a/src-tauri/src/download_resolver.rs +++ b/src-tauri/src/download_resolver.rs @@ -44,6 +44,24 @@ pub(crate) enum ResolveError { Network(String), Http(StatusCode), Link(String), + /// The host asks to try again later; `retry_after` is its stated wait, if any. + RateLimited { + message: String, + retry_after: Option, + }, +} + +impl ResolveError { + /// Failures that say nothing lasting about the link, so retrying later can succeed. + pub(crate) fn is_transient(&self) -> bool { + match self { + Self::Network(_) | Self::RateLimited { .. } => true, + Self::Http(status) => { + status.is_server_error() || matches!(status.as_u16(), 408 | 425 | 429) + } + Self::Link(_) => false, + } + } } impl std::fmt::Display for ResolveError { @@ -51,7 +69,7 @@ impl std::fmt::Display for ResolveError { match self { Self::Network(message) => write!(formatter, "Network: {message}"), Self::Http(status) => write!(formatter, "HTTP {}", status.as_u16()), - Self::Link(message) => formatter.write_str(message), + Self::Link(message) | Self::RateLimited { message, .. } => formatter.write_str(message), } } } @@ -74,6 +92,18 @@ impl From for String { } } +/// Seconds form of `Retry-After`; the date form is ignored. +pub(crate) fn retry_after(headers: &HeaderMap) -> Option { + headers + .get(reqwest::header::RETRY_AFTER)? + .to_str() + .ok()? + .trim() + .parse() + .ok() + .map(Duration::from_secs) +} + pub(crate) fn network_error(error: reqwest::Error) -> ResolveError { ResolveError::Network(error.without_url().to_string()) } @@ -179,6 +209,12 @@ async fn follow_redirects( url = next; continue; } + if response.status() == StatusCode::TOO_MANY_REQUESTS { + return Err(ResolveError::RateLimited { + message: "The download host is limiting requests, try again later".into(), + retry_after: retry_after(response.headers()), + }); + } if !response.status().is_success() { return Err(ResolveError::Http(response.status())); } @@ -410,9 +446,7 @@ async fn resolve_gofile(client: &Client, uri: &str) -> Result) -> Duration { + retry_after + .unwrap_or(Duration::from_secs(30)) + .clamp(Duration::from_secs(1), Duration::from_secs(300)) +} + pub(crate) fn native_cdn(uri: &str) -> bool { reqwest::Url::parse(uri) .is_ok_and(|url| url.scheme() == "https" && url.host_str() == Some("cdn.taxphobia.top")) @@ -63,7 +71,7 @@ pub(crate) async fn inspect(client: &reqwest::Client, uri: &str) -> Result Result { @@ -140,6 +148,23 @@ async fn inspect_file(client: &reqwest::Client, uri: &str) -> Result, secret: &str) -> Result { Ok(headers) } -async fn response_bytes(mut response: reqwest::Response) -> Result, String> { +async fn response_bytes(mut response: reqwest::Response) -> Result, GofileError> { let mut bytes = Vec::new(); - while let Some(chunk) = response - .chunk() - .await - .map_err(|e| format!("GoFile network: {}", e.without_url()))? - { + while let Some(chunk) = response.chunk().await.map_err(network_error)? { if bytes.len() + chunk.len() > 2 * 1024 * 1024 { return Err("GoFile response exceeds the size limit".into()); } @@ -249,17 +245,47 @@ async fn response_bytes(mut response: reqwest::Response) -> Result, Stri } Ok(bytes) } -/// Lets the caller renew a rejected guest token without matching message text. +/// Lets the caller renew a rejected guest token and tell network failures from +/// refusals without matching message text. enum GofileError { Unauthorized, + Network(String), + RateLimited(Option), Other(String), } +fn network_error(error: reqwest::Error) -> GofileError { + GofileError::Network(format!("GoFile network: {}", error.without_url())) +} + impl std::fmt::Display for GofileError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Unauthorized => formatter.write_str("GoFile authorization rejected"), - Self::Other(message) => formatter.write_str(message), + Self::RateLimited(_) => formatter.write_str("GoFile rate limit reached; retry later"), + Self::Network(message) | Self::Other(message) => formatter.write_str(message), + } + } +} + +impl From for GofileError { + fn from(error: ResolveError) -> Self { + match error { + ResolveError::Network(message) => Self::Network(message), + other => Self::Other(other.to_string()), + } + } +} + +impl From for ResolveError { + fn from(error: GofileError) -> Self { + match error { + GofileError::Network(message) => Self::Network(message), + GofileError::RateLimited(retry_after) => Self::RateLimited { + message: GofileError::RateLimited(retry_after).to_string(), + retry_after, + }, + other => Self::Link(other.to_string()), } } } @@ -295,8 +321,9 @@ async fn request( .timeout(Duration::from_secs(15)) .send() .await - .map_err(|e| format!("GoFile network: {}", e.without_url()))?; + .map_err(network_error)?; let status = response.status(); + let retry_after = crate::download_resolver::retry_after(response.headers()); let bytes = response_bytes(response).await?; let reply = serde_json::from_slice::(&bytes); let rate_limited = reply.as_ref().is_ok_and(|r| r.status == "error-rateLimit"); @@ -310,7 +337,7 @@ async fn request( return Err(GofileError::Unauthorized); } if rate_limited || status.as_u16() == 429 { - return Err("GoFile rate limit reached; retry later".into()); + return Err(GofileError::RateLimited(retry_after)); } let reply = reply.map_err(|_| { format!( @@ -348,7 +375,7 @@ fn script_url(html: &str) -> Option { && url.password().is_none()) .then(|| url.to_string()) } -async fn website_text(client: &Client, url: &str) -> Result { +async fn website_text(client: &Client, url: &str) -> Result { let mut headers = HeaderMap::new(); headers.insert("user-agent", HeaderValue::from_static(USER_AGENT)); headers.insert("referer", HeaderValue::from_static("https://gofile.io/")); @@ -362,7 +389,7 @@ async fn website_text(client: &Client, url: &str) -> Result { String::from_utf8(response_bytes(response).await?) .map_err(|_| "GoFile returned invalid text".into()) } -async fn secret(client: &Client) -> Result { +async fn secret(client: &Client) -> Result { let mut urls = Vec::new(); if let Ok(html) = website_text(client, "https://gofile.io/").await && let Some(url) = script_url(&html) @@ -376,6 +403,7 @@ async fn secret(client: &Client) -> Result { urls.dedup(); // Keep the first failure: later candidates are fallback paths that usually just 404. let mut first_error = None; + let mut unreachable = false; for url in urls { let script = match website_text(client, &url).await { Ok(script) if script.trim_start().starts_with('<') => { @@ -384,7 +412,8 @@ async fn secret(client: &Client) -> Result { } Ok(script) => script, Err(error) => { - first_error.get_or_insert(error); + unreachable |= matches!(error, GofileError::Network(_)); + first_error.get_or_insert(error.to_string()); continue; } }; @@ -401,9 +430,12 @@ async fn secret(client: &Client) -> Result { } let reason = first_error.unwrap_or_else(|| "no script URL was available".to_owned()); crate::application_log::failure("gofile_website_token", &reason); - Err(format!( - "Cannot load the current GoFile website-token generator: {reason}" - )) + let message = format!("Cannot load the current GoFile website-token generator: {reason}"); + Err(if unreachable { + GofileError::Network(message) + } else { + GofileError::Other(message) + }) } async fn load_token() -> Option { let path = CACHE_PATH.get()?; @@ -451,7 +483,7 @@ async fn persist_token(token: &str) { crate::application_log::failure("gofile_guest_cache", &error); } } -async fn create_token(client: &Client, secret: &str) -> Result { +async fn create_token(client: &Client, secret: &str) -> Result { let reply = request( client, Method::POST, @@ -568,7 +600,7 @@ async fn official( client: &Client, id: &str, password: Option<&str>, -) -> Result { +) -> Result { let mut state = STATE .get_or_init(|| AsyncMutex::new(Session::default())) .lock() @@ -622,13 +654,13 @@ pub(crate) async fn resolve( client: &Client, id: &str, password: Option<&str>, -) -> Result { +) -> Result { if !valid_id(id) { return Err("Invalid GoFile content ID".into()); } let file = tokio::time::timeout(Duration::from_secs(120), official(client, id, password)) .await - .map_err(|_| "GoFile resolution timed out".to_owned())??; + .map_err(|_| ResolveError::Network("GoFile resolution timed out".to_owned()))??; Ok(ResolvedDownload { provider: Provider::Gofile, files: vec![file], @@ -639,6 +671,21 @@ pub(crate) async fn resolve( mod tests { use super::*; + #[test] + fn network_failures_stay_network_errors() { + let offline = ResolveError::from(GofileError::Network("GoFile network: offline".into())); + assert!(matches!(offline, ResolveError::Network(_))); + let refused = ResolveError::from(GofileError::Other("GoFile denied".into())); + assert!(matches!(refused, ResolveError::Link(_))); + let limited = ResolveError::from(GofileError::RateLimited(Some(Duration::from_secs(9)))); + assert!(matches!( + limited, + ResolveError::RateLimited { retry_after: Some(wait), .. } if wait == Duration::from_secs(9) + )); + let propagated = GofileError::from(ResolveError::Network("reset".into())); + assert!(matches!(propagated, GofileError::Network(_))); + } + #[test] fn extracts_secret_by_running_the_website_generator() { let script = include_str!("../../scripts/fixtures/gofile-wt.js"); diff --git a/src-tauri/src/link_check.rs b/src-tauri/src/link_check.rs index c0b58c2..7e092bd 100644 --- a/src-tauri/src/link_check.rs +++ b/src-tauri/src/link_check.rs @@ -1,4 +1,5 @@ use crate::database::{Database, database_error}; +use crate::download_resolver::ResolveError; use rusqlite::{OptionalExtension, params}; use std::{collections::BTreeMap, sync::Arc}; @@ -129,7 +130,8 @@ fn check_size(expected: Option, result: Result) -> Result, @@ -162,18 +164,20 @@ pub(crate) async fn check_links( }; let client = client.clone(); tasks.spawn(async move { - let result = crate::download_selection::inspect(&client, &uri) - .await - .map_err(|error| error.to_string()); - (uri, expected, result) + let outcome = crate::download_selection::inspect(&client, &uri).await; + (uri, expected, outcome) }); } let Some(joined) = tasks.join_next().await else { break; }; - let (uri, expected, result) = + let (uri, expected, outcome) = joined.map_err(|error| format!("Link check task failed: {error}"))?; - fresh.insert(uri.clone(), result.clone()); + let cacheable = !outcome.as_ref().is_err_and(ResolveError::is_transient); + let result = outcome.map_err(|error| error.to_string()); + if cacheable { + fresh.insert(uri.clone(), result.clone()); + } results.insert(uri, check_size(expected, result)); } if !fresh.is_empty() { @@ -195,6 +199,24 @@ mod tests { (directory, database) } + #[test] + fn transient_failures_are_not_cacheable() { + use reqwest::StatusCode; + let transient = [ + ResolveError::Network("offline".into()), + ResolveError::Http(StatusCode::TOO_MANY_REQUESTS), + ResolveError::Http(StatusCode::BAD_GATEWAY), + ResolveError::Http(StatusCode::REQUEST_TIMEOUT), + ]; + assert!(transient.iter().all(ResolveError::is_transient)); + let permanent = [ + ResolveError::Http(StatusCode::NOT_FOUND), + ResolveError::Http(StatusCode::FORBIDDEN), + ResolveError::Link("The download link returned a web page".into()), + ]; + assert!(!permanent.iter().any(ResolveError::is_transient)); + } + #[test] fn freshness_expires_after_a_day_and_rejects_future_timestamps() { assert!(is_fresh(100, 100)); diff --git a/src-tauri/src/one_fichier.rs b/src-tauri/src/one_fichier.rs index 732f9dd..d236c84 100644 --- a/src-tauri/src/one_fichier.rs +++ b/src-tauri/src/one_fichier.rs @@ -6,7 +6,7 @@ use reqwest::{ Client, StatusCode, Url, header::{CONTENT_TYPE, HeaderMap, HeaderValue, REFERER}, }; -use std::path::Path; +use std::{path::Path, time::Duration}; const FILE_PAGE: &str = "https://1fichier.com/"; const LINK_CHECKER: &str = "https://1fichier.com/check_links.pl"; @@ -103,7 +103,7 @@ fn is_download_host(link: &str) -> bool { /// The form answers with a one-time link on a download server, or with a notice /// when the guest has to wait for the next free download. -fn parse_download_page(html: &str) -> Result { +fn parse_download_page(html: &str) -> Result { let anchor = Regex::new(r"(?is)]*>").map_err(|error| error.to_string())?; let href = Regex::new(r#"(?i)\bhref="([^"]+)""#).map_err(|error| error.to_string())?; let class = Regex::new(r#"(?i)\bclass="([^"]*)""#).map_err(|error| error.to_string())?; @@ -123,22 +123,40 @@ fn parse_download_page(html: &str) -> Result { Err(refusal(html)?) } -fn refusal(html: &str) -> Result { +fn refusal(html: &str) -> Result { let notice = Regex::new(r"(?is)notice-alert.*?").map_err(|error| error.to_string())?; let tags = Regex::new(r"<[^>]*>").map_err(|error| error.to_string())?; let wait = Regex::new(r"(?i)\bwait\s+([0-9]{1,4})\s+(second|minute|hour)") .map_err(|error| error.to_string())?; let Some(notice) = notice.find(html) else { - return Ok("1fichier did not offer a download link".into()); + return Ok(ResolveError::Link( + "1fichier did not offer a download link".into(), + )); }; let text = tags.replace_all(notice.as_str(), " "); Ok(match wait.captures(&text) { Some(wait) => { let (count, unit) = (&wait[1], wait[2].to_lowercase()); let plural = if count == "1" { "" } else { "s" }; - format!("1fichier requires a wait of {count} {unit}{plural} between free downloads") + let seconds = match unit.as_str() { + "hour" => 3600, + "minute" => 60, + _ => 1, + }; + ResolveError::RateLimited { + message: format!( + "1fichier requires a wait of {count} {unit}{plural} between free downloads" + ), + retry_after: count + .parse::() + .ok() + .map(|count| Duration::from_secs(count * seconds)), + } } - None => "1fichier is limiting free downloads right now, try again later".into(), + None => ResolveError::RateLimited { + message: "1fichier is limiting free downloads right now, try again later".into(), + retry_after: None, + }, }) } @@ -249,7 +267,7 @@ mod tests { r#"Start"#, ] { assert_eq!( - parse_download_page(html).unwrap_err(), + parse_download_page(html).unwrap_err().to_string(), "1fichier did not offer a download link", "{html}" ); @@ -263,19 +281,32 @@ mod tests {
Without subscription, you must wait between downloads.
You must wait 60 minutes... "#; + let waiting = parse_download_page(html).unwrap_err(); assert_eq!( - parse_download_page(html).unwrap_err(), + waiting.to_string(), "1fichier requires a wait of 60 minutes between free downloads" ); + assert!(matches!( + waiting, + ResolveError::RateLimited { retry_after: Some(wait), .. } if wait == Duration::from_secs(3600) + )); let single = html.replace("60 minutes", "1 minute"); assert_eq!( - parse_download_page(&single).unwrap_err(), + parse_download_page(&single).unwrap_err().to_string(), "1fichier requires a wait of 1 minute between free downloads" ); let unknown = html.replace("You must wait 60 minutes...", "Try later."); + let unknown = parse_download_page(&unknown).unwrap_err(); assert_eq!( - parse_download_page(&unknown).unwrap_err(), + unknown.to_string(), "1fichier is limiting free downloads right now, try again later" ); + assert!(matches!( + unknown, + ResolveError::RateLimited { + retry_after: None, + .. + } + )); } }