Implement retry logic for curl command in WireGuard interface probe

config_manager.py

@@ -2,6 +2,9 @@
 
 import subprocess
 import logging
+import time
+
+WG_RESTART_DELAY = 3
 
 
 def new_config(interface):
@@ -28,3 +31,5 @@ def new_config(interface):
         check=False,
     )
     logging.info("Restarted wireguard service")
+    # give the interface/handshake time to come up before it gets probed
+    time.sleep(WG_RESTART_DELAY)

test_wireguard.py

@@ -1,6 +1,7 @@
 """Tests for WireGuard status checks and Prometheus metrics."""
 
 import socket
+import subprocess
 import unittest
 from unittest.mock import patch
 from urllib.request import urlopen
@@ -24,6 +25,31 @@ def test_interface_probe_uses_requested_interface(self):
         self.assertEqual(command[0], "curl")
         self.assertIn("wg-test", command)
 
+    def test_interface_probe_retries_after_transient_failure(self):
+        """A single failed curl attempt should not fail the probe if a later attempt succeeds."""
+        curl_result = type("Result", (), {"stdout": '{"mullvad_exit_ip": true}'})()
+        error = subprocess.CalledProcessError(28, ["curl"], stderr="timed out")
+        with patch(
+            "wireguard.subprocess.run", side_effect=[error, curl_result]
+        ) as run:
+            with patch("wireguard.time.sleep") as sleep:
+                self.assertTrue(wireguard.test_interface("wg-test"))
+
+        self.assertEqual(run.call_count, 2)
+        sleep.assert_called_once_with(wireguard.CURL_RETRY_DELAY)
+
+    def test_interface_probe_fails_after_exhausting_retries(self):
+        """The probe should give up and return False once all retries fail."""
+        error = subprocess.CalledProcessError(28, ["curl"], stderr="timed out")
+        with patch(
+            "wireguard.subprocess.run", side_effect=[error] * wireguard.CURL_RETRIES
+        ) as run:
+            with patch("wireguard.time.sleep") as sleep:
+                self.assertFalse(wireguard.test_interface("wg-test"))
+
+        self.assertEqual(run.call_count, wireguard.CURL_RETRIES)
+        self.assertEqual(sleep.call_count, wireguard.CURL_RETRIES - 1)
+
     def test_metrics_endpoint_exposes_cached_status(self):
         """The HTTP endpoint should expose the latest cached gauge value."""
         with socket.socket() as sock:

wireguard.py

@@ -13,6 +13,8 @@
 
 WIREGUARD_INTERFACE = "exit"
 FASTD_SERVICE = "ffsh"
+CURL_RETRIES = 3
+CURL_RETRY_DELAY = 2
 wireguard_up = Gauge(
     "wireguard_up", "Whether the WireGuard connection is up", ["interface"]
 )
@@ -41,17 +43,43 @@ def test_interface(interface_name):
     curl_cmd = [
         "curl",
         "--connect-timeout",
-        "10",
+        "15",
         "--interface",
         interface_name,
         "https://am.i.mullvad.net/json",
     ]
-    try:
-        result = subprocess.run(curl_cmd, capture_output=True, text=True, check=True)
-        data = json.loads(result.stdout)
-    except (subprocess.CalledProcessError, json.JSONDecodeError) as e:
-        logging.error("Curl could not connect to Mullvad or json was not valid")
-        logging.error(e)
+    data = None
+    for attempt in range(1, CURL_RETRIES + 1):
+        try:
+            result = subprocess.run(
+                curl_cmd, capture_output=True, text=True, check=True
+            )
+            data = json.loads(result.stdout)
+            break
+        except subprocess.CalledProcessError as e:
+            logging.error(
+                "Curl could not connect to Mullvad (attempt %d/%d), "
+                "returncode=%s, stderr=%s",
+                attempt,
+                CURL_RETRIES,
+                e.returncode,
+                e.stderr,
+            )
+        except json.JSONDecodeError as e:
+            logging.error(
+                "Curl returned invalid json (attempt %d/%d): %s, stdout=%s",
+                attempt,
+                CURL_RETRIES,
+                e,
+                result.stdout,
+            )
+        if attempt < CURL_RETRIES:
+            time.sleep(CURL_RETRY_DELAY)
+    if data is None:
+        logging.error(
+            "Curl could not connect to Mullvad or json was not valid after %d attempts",
+            CURL_RETRIES,
+        )
         return False
     try:
         connected = data["mullvad_exit_ip"] is True