diff --git a/config_manager.py b/config_manager.py index b786119..9938cfd 100644 --- a/config_manager.py +++ b/config_manager.py @@ -2,6 +2,9 @@ import subprocess import logging +import time + +WG_RESTART_DELAY = 3 def new_config(interface): @@ -28,3 +31,5 @@ def new_config(interface): check=False, ) logging.info("Restarted wireguard service") + # give the interface/handshake time to come up before it gets probed + time.sleep(WG_RESTART_DELAY) diff --git a/test_wireguard.py b/test_wireguard.py index 1766904..e2dc7a6 100644 --- a/test_wireguard.py +++ b/test_wireguard.py @@ -1,6 +1,7 @@ """Tests for WireGuard status checks and Prometheus metrics.""" import socket +import subprocess import unittest from unittest.mock import patch from urllib.request import urlopen @@ -24,6 +25,31 @@ def test_interface_probe_uses_requested_interface(self): self.assertEqual(command[0], "curl") self.assertIn("wg-test", command) + def test_interface_probe_retries_after_transient_failure(self): + """A single failed curl attempt should not fail the probe if a later attempt succeeds.""" + curl_result = type("Result", (), {"stdout": '{"mullvad_exit_ip": true}'})() + error = subprocess.CalledProcessError(28, ["curl"], stderr="timed out") + with patch( + "wireguard.subprocess.run", side_effect=[error, curl_result] + ) as run: + with patch("wireguard.time.sleep") as sleep: + self.assertTrue(wireguard.test_interface("wg-test")) + + self.assertEqual(run.call_count, 2) + sleep.assert_called_once_with(wireguard.CURL_RETRY_DELAY) + + def test_interface_probe_fails_after_exhausting_retries(self): + """The probe should give up and return False once all retries fail.""" + error = subprocess.CalledProcessError(28, ["curl"], stderr="timed out") + with patch( + "wireguard.subprocess.run", side_effect=[error] * wireguard.CURL_RETRIES + ) as run: + with patch("wireguard.time.sleep") as sleep: + self.assertFalse(wireguard.test_interface("wg-test")) + + self.assertEqual(run.call_count, wireguard.CURL_RETRIES) + self.assertEqual(sleep.call_count, wireguard.CURL_RETRIES - 1) + def test_metrics_endpoint_exposes_cached_status(self): """The HTTP endpoint should expose the latest cached gauge value.""" with socket.socket() as sock: diff --git a/wireguard.py b/wireguard.py index 36b10ae..bc7a77e 100644 --- a/wireguard.py +++ b/wireguard.py @@ -13,6 +13,8 @@ WIREGUARD_INTERFACE = "exit" FASTD_SERVICE = "ffsh" +CURL_RETRIES = 3 +CURL_RETRY_DELAY = 2 wireguard_up = Gauge( "wireguard_up", "Whether the WireGuard connection is up", ["interface"] ) @@ -41,17 +43,43 @@ def test_interface(interface_name): curl_cmd = [ "curl", "--connect-timeout", - "10", + "15", "--interface", interface_name, "https://am.i.mullvad.net/json", ] - try: - result = subprocess.run(curl_cmd, capture_output=True, text=True, check=True) - data = json.loads(result.stdout) - except (subprocess.CalledProcessError, json.JSONDecodeError) as e: - logging.error("Curl could not connect to Mullvad or json was not valid") - logging.error(e) + data = None + for attempt in range(1, CURL_RETRIES + 1): + try: + result = subprocess.run( + curl_cmd, capture_output=True, text=True, check=True + ) + data = json.loads(result.stdout) + break + except subprocess.CalledProcessError as e: + logging.error( + "Curl could not connect to Mullvad (attempt %d/%d), " + "returncode=%s, stderr=%s", + attempt, + CURL_RETRIES, + e.returncode, + e.stderr, + ) + except json.JSONDecodeError as e: + logging.error( + "Curl returned invalid json (attempt %d/%d): %s, stdout=%s", + attempt, + CURL_RETRIES, + e, + result.stdout, + ) + if attempt < CURL_RETRIES: + time.sleep(CURL_RETRY_DELAY) + if data is None: + logging.error( + "Curl could not connect to Mullvad or json was not valid after %d attempts", + CURL_RETRIES, + ) return False try: connected = data["mullvad_exit_ip"] is True