Don't panic when somebody else's version index is down

src/stage4/src/bloody_maven.rs

@@ -2,10 +2,12 @@ use std::collections::HashSet;
 use std::future::Future;
 use std::pin::Pin;
 
+use log::warn;
 use serde::{Deserialize, Serialize};
 use serde_xml_rs::from_str;
 
 use crate::executor::{Download, GgVersion};
+use crate::fetch::fetch_text;
 use crate::target::{Arch, Os, Variant};
 
 #[derive(Serialize, Deserialize)]
@@ -45,10 +47,17 @@ pub fn get_download_urls_from_maven<'a>(group: &'a str, artifact: &'a str) -> Pi
     Box::pin(async move {
         let root_url = format!("https://repo1.maven.org/maven2/org/{group}/{artifact}");
         let metadata_url = format!("{root_url}/maven-metadata.xml");
-        let body = reqwest::get(metadata_url.clone()).await
-            .expect("Unable to connect to archive.apache.org").text().await
-            .expect("Unable to download maven metadata xml");
-        let root: Metadata = from_str(body.as_str()).expect("XML was not well-formatted");
+        let body = match fetch_text(&metadata_url).await {
+            Some(body) => body,
+            None => return vec![],
+        };
+        let root: Metadata = match from_str(body.as_str()) {
+            Ok(root) => root,
+            Err(e) => {
+                warn!("{metadata_url} did not answer with the maven metadata we expected: {e}");
+                return vec![];
+            }
+        };
 
         root.versioning.versions.version.into_iter().map(|ver| {
             let mut tags = HashSet::new();

src/stage4/src/checker.rs

@@ -107,11 +107,13 @@ fn matches_requested_tool(executor: &dyn Executor, tool_name: &str, requested_na
         || executor.get_executor_cmd().cmd == tool_name
 }
 
+/// `Err` is "could not read the version list", not "nothing newer" - calling a tool up
+/// to date because the index was down tells someone their old build is the current one.
 async fn check_tool_update(
     meta: GgMeta,
     path: std::path::PathBuf,
     input: &AppInput,
-) -> Option<UpdateInfo> {
+) -> Result<Option<UpdateInfo>, String> {
     info!(
         "Checking tool update for cmd: {:?} with version: {:?}",
         meta.cmd.cmd, meta.cmd.version
@@ -129,6 +131,12 @@ async fn check_tool_update(
             executor.get_name(),
             meta.cmd.cmd
         );
+        if urls.is_empty() {
+            return Err(format!(
+                "{}: could not read the version list",
+                registry_name(&*executor)
+            ));
+        }
         let urls_matches = executor.get_url_matches(&urls, input);
         info!(
             "Got {} url matches for {}",
@@ -155,7 +163,7 @@ async fn check_tool_update(
 
             let version_selector = meta.cmd.to_version_selector();
 
-            return Some(UpdateInfo {
+            return Ok(Some(UpdateInfo {
                 tool_name: registry_name(&*executor),
                 version_selector,
                 current_version: current_version.map(|v| v.to_string()),
@@ -164,10 +172,10 @@ async fn check_tool_update(
                 is_major_update,
                 path,
                 executor,
-            });
+            }));
         }
     }
-    None
+    Ok(None)
 }
 
 fn should_include_update(update_info: &UpdateInfo, allow_major: bool) -> bool {
@@ -208,28 +216,29 @@ pub async fn check_or_update_all_including_gg(
     should_update: bool,
     allow_major: bool,
     force: bool,
-) {
+) -> ExitCode {
     if should_update {
         updater::perform_update(gg_version, force).await;
     } else {
         updater::check_gg_update(gg_version).await;
     }
     println!();
 
-    check_or_update_all(input, should_update, allow_major, force).await;
+    check_or_update_all(input, should_update, allow_major, force).await
 }
 
 pub async fn check_or_update_all(
     input: &AppInput,
     should_update: bool,
     allow_major: bool,
     force: bool,
-) {
+) -> ExitCode {
+    let mut update_failed = false;
     let metas = get_all_tool_metas().await;
 
     if metas.is_empty() {
         println!("No cached tools found.");
-        return;
+        return ExitCode::from(0);
     }
 
     println!("Checking for updates...");
@@ -276,9 +285,11 @@ pub async fn check_or_update_all(
                 let spinner_key = path.to_string_lossy().to_string();
                 let result = check_tool_update(meta, path, input).await;
 
-                if result.is_some() {
-                    if let Some(pb) = tool_spinners.get(&spinner_key) {
-                        pb.finish_with_message("done");
+                if let Some(pb) = tool_spinners.get(&spinner_key) {
+                    match &result {
+                        Ok(Some(_)) => pb.finish_with_message("done"),
+                        Ok(None) => pb.finish_and_clear(),
+                        Err(_) => pb.finish_with_message("could not check"),
                     }
                 }
 
@@ -287,7 +298,15 @@ pub async fn check_or_update_all(
         })
         .collect();
 
-    let update_infos: Vec<UpdateInfo> = join_all(check_tasks).await.into_iter().flatten().collect();
+    let mut check_failures: Vec<String> = Vec::new();
+    let mut update_infos: Vec<UpdateInfo> = Vec::new();
+    for result in join_all(check_tasks).await {
+        match result {
+            Ok(Some(info)) => update_infos.push(info),
+            Ok(None) => {}
+            Err(reason) => check_failures.push(reason),
+        }
+    }
 
     m.clear().unwrap();
 
@@ -337,9 +356,20 @@ pub async fn check_or_update_all(
         }
     }
 
+    if !check_failures.is_empty() {
+        eprintln!();
+        for reason in &check_failures {
+            eprintln!("Could not check for updates - {reason}");
+        }
+    }
+
     if filtered_updates.is_empty() {
-        println!("\nAll tools are up to date!");
-        return;
+        if check_failures.is_empty() {
+            println!("\nAll tools are up to date!");
+            return ExitCode::from(0);
+        }
+        println!("\nEverything we could check is up to date.");
+        return ExitCode::from(1);
     }
 
     if !should_update {
@@ -359,16 +389,30 @@ pub async fn check_or_update_all(
             if let Some(parent) = info.path.parent() {
                 if fs::remove_dir_all(parent).is_ok() {
                     let pb = create_barus();
-                    let _ = prep(&*info.executor, input, &pb).await;
-                    println!("Successfully updated {}", info.tool_name);
+                    // Cache dir is already gone, so swallowing this loses the tool
+                    match prep(&*info.executor, input, &pb).await {
+                        Ok(_) => println!("Successfully updated {}", info.tool_name),
+                        Err(e) => {
+                            eprintln!("Failed to update {}: {}", info.tool_name, e);
+                            update_failed = true;
+                        }
+                    }
                 } else {
-                    println!("Unable to update {}", info.tool_name);
+                    eprintln!("Unable to update {}", info.tool_name);
+                    update_failed = true;
                 }
             } else {
-                println!("Unable to update {}", info.tool_name);
+                eprintln!("Unable to update {}", info.tool_name);
+                update_failed = true;
             }
         }
     }
+
+    if update_failed {
+        ExitCode::from(1)
+    } else {
+        ExitCode::from(0)
+    }
 }
 
 pub async fn check_or_update_tool(
@@ -453,7 +497,16 @@ pub async fn check_or_update_tool(
     let mut update_failed = false;
 
     for (meta, path) in matching_metas {
-        if let Some(info) = check_tool_update(meta, path, input).await {
+        let checked = match check_tool_update(meta, path, input).await {
+            Ok(checked) => checked,
+            Err(reason) => {
+                // Keep going - another cached copy of the same tool may check fine
+                eprintln!("Could not check for updates - {reason}");
+                update_failed = true;
+                continue;
+            }
+        };
+        if let Some(info) = checked {
             let display_name = if info.version_selector.is_empty() {
                 info.tool_name.clone()
             } else {

src/stage4/src/executor.rs

@@ -433,7 +433,12 @@ pub async fn prep(
         for reason in crate::github_utils::take_github_errors() {
             eprintln!("{reason}");
         }
-        panic!("Did not find any download URL!");
+        // Someone else's site being down is not a bug in gg (#272). Any warning from
+        // fetch.rs is right above this with the real reason.
+        return Err(format!(
+            "Did not find any download URL for {}. The version list may be unreachable - run with -v for details.",
+            executor.get_name()
+        ));
     }
 
     let urls_match = get_url_matches(&urls, input, executor);
@@ -820,6 +825,58 @@ mod tests {
     use super::*;
     use crate::github_utils::{detect_arch_from_name, detect_os_from_name};
 
+    /// An executor whose index is down: no URLs, no explanation.
+    struct NoUrls {
+        cmd: ExecutorCmd,
+    }
+
+    impl Executor for NoUrls {
+        fn get_executor_cmd(&self) -> &ExecutorCmd {
+            &self.cmd
+        }
+        fn get_download_urls<'a>(
+            &'a self,
+            _input: &'a AppInput,
+        ) -> Pin<Box<dyn Future<Output = Vec<Download>> + 'a>> {
+            Box::pin(async { vec![] })
+        }
+        fn get_bins(&self, _input: &AppInput) -> Vec<BinPattern> {
+            vec![BinPattern::Exact("nope".to_string())]
+        }
+        fn get_name(&self) -> &str {
+            "nope"
+        }
+    }
+
+    // Used to be panic!("Did not find any download URL!") - that is #272
+    #[tokio::test]
+    async fn test_prep_errors_instead_of_panicking_when_there_are_no_urls() {
+        let executor = NoUrls {
+            cmd: ExecutorCmd {
+                cmd: "nope".to_string(),
+                version: None,
+                distribution: None,
+                include_tags: Default::default(),
+                exclude_tags: Default::default(),
+                gems: None,
+            },
+        };
+        let input = AppInput {
+            target: Target {
+                arch: Arch::X86_64,
+                os: Os::Linux,
+                variant: None,
+            },
+            app_args: vec![],
+        };
+        let result = prep(&executor, &input, &ProgressBar::hidden()).await;
+        let message = result.err().expect("empty url list must not be Ok");
+        assert!(
+            message.contains("nope"),
+            "the tool should be named: {message}"
+        );
+    }
+
     #[test]
     fn test_find_jar_file_skips_companions_and_is_deterministic() {
         // The real jar wins over the -javadoc/-sources companions read_dir might

src/stage4/src/executors/go.rs

@@ -4,6 +4,7 @@ use std::future::Future;
 use std::pin::Pin;
 
 use crate::executor::{AppInput, BinPattern, Download, Executor, ExecutorCmd, GgVersion};
+use crate::fetch::fetch_text;
 use crate::target::Arch::{Arm64, X86_64};
 use crate::target::Os::{Linux, Mac, Windows};
 use crate::target::Variant::Any;
@@ -67,12 +68,10 @@ impl Executor for Go {
     ) -> Pin<Box<dyn Future<Output = Vec<Download>> + 'a>> {
         Box::pin(async move {
             // let mut downloads: Vec<Download> = vec!();
-            let body = reqwest::get("https://go.dev/dl/")
-                .await
-                .expect("Unable to connect to go.dev")
-                .text()
-                .await
-                .expect("Unable to download gradle list of versions");
+            let body = match fetch_text("https://go.dev/dl/").await {
+                Some(body) => body,
+                None => return vec![],
+            };
 
             let document = Html::parse_document(body.as_str());
             let downloads: Vec<Download> = document

src/stage4/src/executors/gradle.rs

@@ -9,6 +9,7 @@ use sha256::try_digest;
 
 use crate::executor::{java_deps, AppInput, BinPattern, Download, ExecutorCmd, ExecutorDep};
 use crate::executors::gradle_properties::GradleAndWrapperProperties;
+use crate::fetch::fetch_text;
 use crate::target::Variant;
 use crate::{target, Executor};
 
@@ -53,12 +54,10 @@ impl Executor for Gradle {
                 }
             }
 
-            let body = reqwest::get("https://gradle.org/releases")
-                .await
-                .expect("Unable to connect to services.gradle.org")
-                .text()
-                .await
-                .expect("Unable to download gradle list of versions");
+            let body = match fetch_text("https://gradle.org/releases").await {
+                Some(body) => body,
+                None => return vec![],
+            };
 
             let document = Html::parse_document(body.as_str());
             document

src/stage4/src/executors/java_distributions.rs

@@ -2,11 +2,11 @@ use std::collections::HashSet;
 use std::future::Future;
 use std::pin::Pin;
 
-use log::debug;
 use serde::Deserialize;
 use serde::Serialize;
 
 use crate::executor::{Download, GgVersion};
+use crate::fetch::fetch_json;
 use crate::target::{Arch, Os, Target, Variant};
 
 type DistributionHandler = fn(&Target) -> Pin<Box<dyn Future<Output = Vec<Download>> + Send>>;
@@ -91,24 +91,9 @@ fn get_azul_downloads(target: &Target) -> Pin<Box<dyn Future<Output = Vec<Downlo
     Box::pin(async move {
         // Azul backs the default now, not just an explicit -azul, so a bad day at
         // admin-ajax.php (it likes answering with HTML) must not panic the whole run
-        let bundles: Vec<AzulBundle> = match reqwest::get("https://www.azul.com/wp-admin/admin-ajax.php?action=bundles&endpoint=community&use_stage=false&include_fields=java_version,release_status,abi,arch,bundle_type,cpu_gen,ext,features,hw_bitness,javafx,latest,os,support_term").await {
-            Ok(response) => match response.text().await {
-                Ok(text) => match serde_json::from_str(text.as_str()) {
-                    Ok(bundles) => bundles,
-                    Err(e) => {
-                        debug!("Azul returned something that was not bundle JSON: {e}");
-                        return vec![];
-                    }
-                },
-                Err(e) => {
-                    debug!("Could not read the Azul response: {e}");
-                    return vec![];
-                }
-            },
-            Err(e) => {
-                debug!("Could not reach Azul: {e}");
-                return vec![];
-            }
+        let bundles: Vec<AzulBundle> = match fetch_json("https://www.azul.com/wp-admin/admin-ajax.php?action=bundles&endpoint=community&use_stage=false&include_fields=java_version,release_status,abi,arch,bundle_type,cpu_gen,ext,features,hw_bitness,javafx,latest,os,support_term").await {
+            Some(bundles) => bundles,
+            None => return vec![],
         };
 
         bundles