diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..d19d09d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "ffshmon", + "image": "mcr.microsoft.com/devcontainers/python:3.13-trixie", + "postCreateCommand": "python -m pip install --requirement requirements.txt", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ], + "settings": { + "python.analysis.autoSearchPaths": true, + "python.analysis.typeCheckingMode": "basic" + } + } + } +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..1417a1e --- /dev/null +++ b/README.md @@ -0,0 +1,129 @@ +# ffshmon + +`ffshmon` monitors a WireGuard connection and attempts to recover it when the VPN check fails. It checks the configured FastD service, tests the WireGuard interface through Mullvad, regenerates the WireGuard configuration once after a failure, and alerts the NOC if recovery fails. + +It also provides a Prometheus endpoint with the latest up/down status. + +## Requirements + +- Linux with `systemd` and `systemctl` +- Python 3.10 or newer +- `curl` +- A WireGuard interface named `exit` by default +- A FastD service named `fastd@ffsh.service` by default +- `/opt/wg-conf-gen/wg-conf-gen.py` for automatic configuration recovery +- SMTP access to the configured mail host for failure alerts + +## Installation + +Create a virtual environment and install the Python dependencies: + +```bash +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt +``` + +## One-shot check + +Run the existing scheduled check with mail credentials and a log file: + +```bash +.venv/bin/python wireguard.py check \ + --user noc@example.org \ + --password 'mail-password' \ + --log /var/log/ffshmon.log +``` + +The command exits after one health cycle. If the FastD service is down, the connection probe is skipped and the status is considered down. If the probe fails, `ffshmon` regenerates the WireGuard configuration and retries once. A second failure stops FastD and WireGuard and sends an email alert. + +## Prometheus endpoint + +Start the long-running monitor with: + +```bash +.venv/bin/python wireguard.py serve \ + --user noc@example.org \ + --password 'mail-password' \ + --log /var/log/ffshmon.log +``` + +By default, the process: + +- Runs an immediate health check, then repeats every 60 seconds. +- Listens on `127.0.0.1:8000`. +- Exposes the latest completed result at `/metrics`. +- Does not run a new health check when Prometheus scrapes the endpoint. + +Example request: + +```bash +curl http://127.0.0.1:8000/metrics +``` + +The relevant metric is: + +```text +wireguard_up{interface="exit"} 1.0 +``` + +A value of `1` means the latest check succeeded. A value of `0` means the FastD service or WireGuard connectivity check is down. + +The listener and polling interval can be changed with `--host`, `--port`, and `--interval`: + +```bash +.venv/bin/python wireguard.py serve \ + --user noc@example.org \ + --password 'mail-password' \ + --log /var/log/ffshmon.log \ + --host 127.0.0.1 \ + --port 8000 \ + --interval 60 +``` + +## Prometheus configuration + +Add a scrape job for the host running `ffshmon`: + +```yaml +scrape_configs: + - job_name: ffshmon + static_configs: + - targets: ["127.0.0.1:8000"] +``` + +If Prometheus runs on another host, bind `serve` to an appropriate reachable address and protect the endpoint with firewall rules or a reverse proxy. The endpoint has no built-in authentication. + +## Running as a service + +Run `serve` as a supervised systemd service so the endpoint remains available. A minimal unit could look like this: + +```ini +[Unit] +Description=WireGuard connectivity monitor +After=network-online.target + +[Service] +Type=simple +WorkingDirectory=/opt/ffshmon +ExecStart=/opt/ffshmon/.venv/bin/python /opt/ffshmon/wireguard.py serve --user noc@example.org --password mail-password --log /var/log/ffshmon.log +Restart=on-failure + +[Install] +WantedBy=multi-user.target +``` + +Avoid storing real credentials directly in a world-readable unit file. Use a protected environment file or another systemd credential mechanism in production. + +## Development + +Run the focused tests with: + +```bash +.venv/bin/python -m unittest -v test_wireguard.py +``` + +Run Pylint with: + +```bash +.venv/bin/python -m pylint wireguard.py test_wireguard.py +``` diff --git a/requirements.txt b/requirements.txt index f80ea3b..9020c7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests click +prometheus-client diff --git a/test_wireguard.py b/test_wireguard.py new file mode 100644 index 0000000..159983d --- /dev/null +++ b/test_wireguard.py @@ -0,0 +1,41 @@ +"""Tests for WireGuard status checks and Prometheus metrics.""" + +import socket +import unittest +from unittest.mock import patch +from urllib.request import urlopen + +from prometheus_client import start_http_server + +import wireguard + + +class WireguardTests(unittest.TestCase): + """Test the public monitoring behavior without system services.""" + + def test_interface_probe_uses_requested_interface(self): + """The connectivity probe should use the requested interface.""" + curl_result = type("Result", (), {"stdout": '{"mullvad_exit_ip": true}'})() + with patch("wireguard.subprocess.run", return_value=curl_result) as run: + self.assertTrue(wireguard.test_interface("wg-test")) + + command = run.call_args.args[0] + self.assertIn("wg-test", command) + + def test_metrics_endpoint_exposes_cached_status(self): + """The HTTP endpoint should expose the latest cached gauge value.""" + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + + wireguard.wireguard_up.labels(interface="exit").set(1) + start_http_server(port, addr="127.0.0.1") + + with urlopen(f"http://127.0.0.1:{port}/metrics") as response: + metrics = response.read().decode("utf-8") + + self.assertIn('wireguard_up{interface="exit"} 1.0', metrics) + + +if __name__ == "__main__": + unittest.main() diff --git a/wireguard.py b/wireguard.py index 2c215d9..90b7440 100644 --- a/wireguard.py +++ b/wireguard.py @@ -1,17 +1,30 @@ -import subprocess +"""Monitor WireGuard connectivity and expose its status to Prometheus.""" + import json import logging +import subprocess +import time import click +from prometheus_client import Gauge, start_http_server from config_manager import new_config from hard_stop import stop_fastd, stop_wg from inform_admin import send_mail +WIREGUARD_INTERFACE = "exit" +FASTD_SERVICE = "ffsh" +wireguard_up = Gauge( + "wireguard_up", "Whether the WireGuard connection is up", ["interface"] +) + + def is_service_running(service_name): + """Return whether the configured FastD service is running.""" result = subprocess.run( ["systemctl", "show", "-p", "SubState", f"fastd@{service_name}.service"], capture_output=True, text=True, + check=False, ) return result.stdout.strip() == "SubState=running" @@ -23,7 +36,7 @@ def test_interface(interface_name): "--connect-timeout", "10", "--interface", - "exit", + interface_name, "https://am.i.mullvad.net/json", ] try: @@ -34,20 +47,22 @@ def test_interface(interface_name): logging.error(e) return False try: - if data["mullvad_exit_ip"] is True: - logging.info("Everything ok.") - return True - else: - # something went wrong, Mullvad says we are not connected to Mullvad - logging.error("Mullvad says we are not connected to Mullvad") - return False + connected = data["mullvad_exit_ip"] is True except KeyError: # something went wrong the json did not contain mullvad_exit_ip logging.error("mullvad_exit_ip was not in the json") return False + if connected: + logging.info("Everything ok.") + return True + + logging.error("Mullvad says we are not connected to Mullvad") + return False + def verify(interface_name, fastd_name, mail_config): + """Check the connection and attempt recovery once if it is down.""" result = test_interface(interface_name) if result is False: @@ -61,56 +76,97 @@ def verify(interface_name, fastd_name, mail_config): stop_wg(interface_name) send_mail( mail_config, - "VPN connection did not work, new VPN config did not help.\nFastd and wireguard stopped.", + "VPN connection did not work, new VPN config did not help.\n" + "Fastd and wireguard stopped.", ) + return result -# Cli group, could add more commands in the future -@click.group() -def cli(): - pass +def run_check( + mail_config, interface_name=WIREGUARD_INTERFACE, fastd_name=FASTD_SERVICE +): + """Run one health cycle and return the resulting up/down state.""" + if is_service_running(service_name=fastd_name): + return verify( + interface_name=interface_name, + fastd_name=fastd_name, + mail_config=mail_config, + ) + logging.info("Fastd service is down, not checking connection") + return False -@cli.command() -@click.option("--user", help="Mail address", required=True) -@click.option("--password", help="Password for Mail Address", required=True) -@click.option("--log", help="Path to log file", required=True) -def check(user, password, log): - """Check Status of wireguard interface""" - # Create log file if it does not exist +def configure_logging(log): + """Create the log file if needed and configure application logging.""" try: - with open(log, "x"): - # This part will only execute if the file is created successfully + with open(log, "x", encoding="utf-8"): pass except FileExistsError: pass - # Logging Config - # LogLevel DEBUG, INFO, WARNING, ERROR - log_level = logging.INFO - log_format = "%(asctime)s %(levelname)-8s %(message)s" - date_format = "%Y-%m-%d %H:%M:%S" logging.basicConfig( - format=log_format, - datefmt=date_format, + format="%(asctime)s %(levelname)-8s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", filename=log, encoding="utf-8", - level=log_level, + level=logging.INFO, ) - # Mail Config - config = { + +def build_mail_config(user, password): + """Build the mail settings used by the recovery alert.""" + return { "target": "noc@freifunk-suedholstein.de", "host": "mail.freifunk-suedholstein.de", "port": "465", "user": user, "password": password, } - if is_service_running(service_name="ffsh"): - verify(interface_name="exit", fastd_name="ffsh", mail_config=config) - else: - logging.info("Fastd service is down, not checking connection") + + +@click.group() +def cli(): + """WireGuard monitoring commands.""" + + +@cli.command() +@click.option("--user", help="Mail address", required=True) +@click.option("--password", help="Password for Mail Address", required=True) +@click.option("--log", help="Path to log file", required=True) +def check(user, password, log): + """Check the status of the WireGuard interface once.""" + configure_logging(log) + run_check(build_mail_config(user, password)) + + +@cli.command() +@click.option("--user", help="Mail address", required=True) +@click.option("--password", help="Password for Mail Address", required=True) +@click.option("--log", help="Path to log file", required=True) +@click.option("--interval", type=float, default=60.0, show_default=True) +@click.option("--host", default="127.0.0.1", show_default=True) +@click.option("--port", type=int, default=8000, show_default=True) +@click.pass_context +def serve(context): + """Run checks and expose the latest WireGuard status as Prometheus metrics.""" + user = context.params["user"] + password = context.params["password"] + log = context.params["log"] + interval = context.params["interval"] + host = context.params["host"] + port = context.params["port"] + configure_logging(log) + start_http_server(port, addr=host) + config = build_mail_config(user, password) + try: + while True: + wireguard_up.labels(interface=WIREGUARD_INTERFACE).set( + 1 if run_check(config) else 0 + ) + time.sleep(interval) + except KeyboardInterrupt: + logging.info("Stopping WireGuard metrics server") if __name__ == "__main__":