From d401cffe8b0c1c246f27594b98121d5ccee95e0b Mon Sep 17 00:00:00 2001 From: Johnathan Brennan Date: Sat, 14 Feb 2026 11:14:50 -0500 Subject: [PATCH 1/3] Alert Integrations --- src/critic/alerts.py | 78 +++++++++++++++ src/critic/libs/mailgun.py | 36 +++++++ src/critic/libs/slack.py | 38 +++++++ src/critic/libs/uptime.py | 33 ++++-- tests/critic_tests/test_alerts.py | 161 ++++++++++++++++++++++++++++++ 5 files changed, 339 insertions(+), 7 deletions(-) create mode 100644 src/critic/alerts.py create mode 100644 src/critic/libs/mailgun.py create mode 100644 src/critic/libs/slack.py create mode 100644 tests/critic_tests/test_alerts.py diff --git a/src/critic/alerts.py b/src/critic/alerts.py new file mode 100644 index 0000000..b9b39c8 --- /dev/null +++ b/src/critic/alerts.py @@ -0,0 +1,78 @@ +import logging + +from critic.libs.mailgun import send_email +from critic.libs.slack import post_message, post_webhook +from critic.models import MonitorState, UptimeMonitorModel + + +log = logging.getLogger(__name__) + + +def _monitor_url(monitor: UptimeMonitorModel) -> str: + return str(monitor.url) + + +def _monitor_label(monitor: UptimeMonitorModel) -> str: + return f'{monitor.project_id}/{monitor.slug}' + + +def _send_slack(monitor: UptimeMonitorModel, text: str) -> None: + for dest in monitor.alert_slack_channels: + try: + # If it looks like a webhook, use webhook mode otherwise treat as channel id/name. + if dest.startswith(('http://', 'https://')): + post_webhook(dest, text) + else: + post_message(dest, text) + except Exception as e: + log.exception(f'Failed to send Slack alert to {dest}: {e}') + + +def _send_email(monitor: UptimeMonitorModel, subject: str, text: str) -> None: + for email in monitor.alert_emails: + try: + send_email(email, subject, text) + except Exception as e: + log.exception(f'Failed to send email alert to {email}: {e}') + + +def maybe_send_alerts( + *, + monitor: UptimeMonitorModel, + prev_state: MonitorState, + prev_consecutive_fails: int, +) -> None: + # Decide whether to send alerts based on state transitions and fail thresholds. + if monitor.state == MonitorState.paused: + return + + label = _monitor_label(monitor) + url = _monitor_url(monitor) + + # Recovery + if prev_state == MonitorState.down and monitor.state == MonitorState.up: + subject = f'CRITIC RECOVERY: {label}' + text = f'Recovered: {label}\nURL: {url}' + log.info(f'Sending recovery alert for {label}') + _send_slack(monitor, text) + _send_email(monitor, subject, text) + return + + # Down alert + if ( + monitor.state == MonitorState.down + and monitor.consecutive_fails >= monitor.failures_before_alerting + ): + crossed_threshold = prev_consecutive_fails < monitor.failures_before_alerting + became_down = prev_state != MonitorState.down + if crossed_threshold or became_down: + subject = f'CRITIC DOWN: {label}' + text = ( + f'Down: {label}\n' + f'URL: {url}\n' + f'Consecutive fails: {monitor.consecutive_fails} ' + f'(threshold: {monitor.failures_before_alerting})' + ) + log.info(f'Sending down alert for {label}') + _send_slack(monitor, text) + _send_email(monitor, subject, text) diff --git a/src/critic/libs/mailgun.py b/src/critic/libs/mailgun.py new file mode 100644 index 0000000..c9c6ddb --- /dev/null +++ b/src/critic/libs/mailgun.py @@ -0,0 +1,36 @@ +import os + +import httpx + + +class MailgunError(Exception): + pass + + +def send_email(to_email: str, subject: str, text: str) -> None: + # Send an email via Mailgun HTTP API. + api_key = os.environ.get('MAILGUN_API_KEY') + domain = os.environ.get('MAILGUN_DOMAIN') + mail_from = os.environ.get('MAILGUN_FROM') + + if not api_key: + raise MailgunError('Missing MAILGUN_API_KEY') + if not domain: + raise MailgunError('Missing MAILGUN_DOMAIN') + if not mail_from: + raise MailgunError('Missing MAILGUN_FROM') + if not to_email: + raise MailgunError('Missing recipient email') + + resp = httpx.post( + f'https://api.mailgun.net/v3/{domain}/messages', + auth=('api', api_key), + data={ + 'from': mail_from, + 'to': to_email, + 'subject': subject, + 'text': text, + }, + timeout=10, + ) + resp.raise_for_status() diff --git a/src/critic/libs/slack.py b/src/critic/libs/slack.py new file mode 100644 index 0000000..00cb525 --- /dev/null +++ b/src/critic/libs/slack.py @@ -0,0 +1,38 @@ +import os + +import httpx + + +class SlackError(Exception): + pass + + +def post_webhook(webhook_url: str, text: str) -> None: + # Send a Slack message via Incoming Webhook URL. + if not webhook_url: + raise SlackError('Missing webhook_url') + + resp = httpx.post( + webhook_url, + json={'text': text}, + timeout=10, + ) + resp.raise_for_status() + + +def post_message(channel: str, text: str) -> None: + # Send a Slack message using chat.postMessage (requires SLACK_BOT_TOKEN). + token = os.environ.get('SLACK_BOT_TOKEN') + if not token: + raise SlackError('Missing SLACK_BOT_TOKEN') + + resp = httpx.post( + 'https://slack.com/api/chat.postMessage', + headers={'Authorization': f'Bearer {token}'}, + json={'channel': channel, 'text': text}, + timeout=10, + ) + resp.raise_for_status() + data = resp.json() + if not data.get('ok'): + raise SlackError(f'Slack API error: {data.get("error")}') diff --git a/src/critic/libs/uptime.py b/src/critic/libs/uptime.py index 2696347..f4256f5 100644 --- a/src/critic/libs/uptime.py +++ b/src/critic/libs/uptime.py @@ -5,6 +5,7 @@ import httpx +from critic.alerts import maybe_send_alerts from critic.libs.dt import round_minute from critic.models import MonitorState, UptimeLogModel, UptimeMonitorModel from critic.tables import UptimeLogTable, UptimeMonitorTable @@ -90,8 +91,12 @@ def make_req(self) -> tuple[httpx.Response | None, float]: latency = response.elapsed.total_seconds() * 1000 if response else (finished - start) return response, latency - def alert(self): - """TODO: alert self.monitor.alert_slack_channels and self.monitor.alert_emails.""" + def alert(self, prev_state: MonitorState, prev_consecutive_fails: int): + maybe_send_alerts( + monitor=self.monitor, + prev_state=prev_state, + prev_consecutive_fails=prev_consecutive_fails, + ) def check_resp(self, response: httpx.Response | None) -> tuple[MonitorState, int, list[str]]: """Checks the response and returns the new state and consecutive fails and list of error @@ -115,12 +120,15 @@ def check_resp(self, response: httpx.Response | None) -> tuple[MonitorState, int error_messages.append('Connection Timeout') consecutive_fails = 0 if state == MonitorState.up else self.monitor.consecutive_fails + 1 - if consecutive_fails >= self.monitor.failures_before_alerting: - self.alert() + return state, consecutive_fails, error_messages def put_log( - self, state: MonitorState, status_code: int, latency: float, error_message: str | None + self, + state: MonitorState, + status_code: int, + latency: float, + error_messages: list[str] | None, ): """ Puts a log for the check. This method should only be called once per monitor check. @@ -133,7 +141,7 @@ def put_log( status=state, resp_code=status_code, latency_secs=latency, - error_message=error_message, + error_message=error_messages, ) UptimeLogTable.put(uptime_log) self._put_log = True @@ -152,6 +160,10 @@ def run(self): self.update_monitor() return + # Capture previous values for alert logic + prev_state = self.monitor.state + prev_consecutive_fails = self.monitor.consecutive_fails + # Make the request resp, latency = self.make_req() @@ -163,9 +175,16 @@ def run(self): # Save a log if updated: + # Keep the in-memory monitor consistent for alert formatting/logic + self.monitor.state = state + self.monitor.consecutive_fails = consecutive_fails + + # Alerts (only if update succeeded to prevent duplicates on race conditions) + self.alert(prev_state, prev_consecutive_fails) + self.put_log( state, resp.status_code if resp else 0, latency, - error_messages if error_messages else None, + error_messages, ) diff --git a/tests/critic_tests/test_alerts.py b/tests/critic_tests/test_alerts.py new file mode 100644 index 0000000..9e5b40b --- /dev/null +++ b/tests/critic_tests/test_alerts.py @@ -0,0 +1,161 @@ +from unittest import mock + +from critic.alerts import maybe_send_alerts +from critic.libs.testing import UptimeMonitorFactory +from critic.libs.uptime import UptimeCheck +from critic.models import MonitorState + + +class TestAlerts: + # These tests are validating the alert decision logic. + # Actual slack and email delivery is mocked until we decide how we want to set that up. + + @mock.patch('critic.alerts._send_email') + @mock.patch('critic.alerts._send_slack') + def test_no_alert_below_threshold(self, m_slack, m_email): + # Below Threshold -> Do Not Alert + monitor = UptimeMonitorFactory.put( + state=MonitorState.down, + consecutive_fails=1, + failures_before_alerting=2, + alert_slack_channels=['C123'], + alert_emails=['a@example.com'], + ) + + maybe_send_alerts( + monitor=monitor, + prev_state=MonitorState.up, + prev_consecutive_fails=0, + ) + + m_slack.assert_not_called() + m_email.assert_not_called() + + @mock.patch('critic.alerts._send_email') + @mock.patch('critic.alerts._send_slack') + def test_alert_on_threshold_cross(self, m_slack, m_email): + # Cross Threshold -> Alert Once + monitor = UptimeMonitorFactory.put( + state=MonitorState.down, + consecutive_fails=2, + failures_before_alerting=2, + alert_slack_channels=['C123'], + alert_emails=['a@example.com'], + ) + + maybe_send_alerts( + monitor=monitor, + prev_state=MonitorState.up, + prev_consecutive_fails=1, + ) + + m_slack.assert_called_once() + m_email.assert_called_once() + + @mock.patch('critic.alerts._send_email') + @mock.patch('critic.alerts._send_slack') + def test_recovery_alert(self, m_slack, m_email): + # Down state to up state -> Send Recovery Alert + monitor = UptimeMonitorFactory.put( + state=MonitorState.up, + consecutive_fails=0, + failures_before_alerting=2, + alert_slack_channels=['C123'], + alert_emails=['a@example.com'], + ) + + maybe_send_alerts( + monitor=monitor, + prev_state=MonitorState.down, + prev_consecutive_fails=3, + ) + + m_slack.assert_called_once() + m_email.assert_called_once() + + @mock.patch('critic.alerts._send_email') + @mock.patch('critic.alerts._send_slack') + def test_paused_no_alert(self, m_slack, m_email): + # Paused Monitors -> No Alerts + monitor = UptimeMonitorFactory.put( + state=MonitorState.paused, + consecutive_fails=999, + failures_before_alerting=1, + alert_slack_channels=['C123'], + alert_emails=['a@example.com'], + ) + + maybe_send_alerts( + monitor=monitor, + prev_state=MonitorState.down, + prev_consecutive_fails=999, + ) + + m_slack.assert_not_called() + m_email.assert_not_called() + + @mock.patch('critic.alerts._send_email') + @mock.patch('critic.alerts._send_slack') + def test_no_repeat_alert_when_already_over_threshold(self, m_slack, m_email): + # Spam Prevention -> No Duplicate Alerts unless state change + monitor = UptimeMonitorFactory.put( + state=MonitorState.down, + consecutive_fails=5, + failures_before_alerting=2, + alert_slack_channels=['C123'], + alert_emails=['a@example.com'], + ) + + maybe_send_alerts( + monitor=monitor, + prev_state=MonitorState.down, + prev_consecutive_fails=4, + ) + + m_slack.assert_not_called() + m_email.assert_not_called() + + +class TestUptimeAlertHook: + # These tests validate where alerting happens in the UptimeCheck.run() flow. + # The key invariant is race safety. Only the winning conditional update should alert + log. + + @mock.patch('critic.libs.uptime.UptimeLogTable.put') + @mock.patch('critic.libs.uptime.UptimeMonitorTable.update') + @mock.patch('critic.libs.uptime.UptimeCheck.make_req') + @mock.patch('critic.libs.uptime.UptimeCheck.alert') + def test_no_alert_if_update_fails(self, m_alert, m_make_req, m_update, m_put_log): + # If update fails -> No alert + No Log + monitor = UptimeMonitorFactory.put( + state=MonitorState.up, + consecutive_fails=1, + failures_before_alerting=2, + ) + + m_make_req.return_value = (None, 0.1) # failing check -> down + m_update.return_value = False # simulate race condition + + UptimeCheck(monitor.project_id, monitor.slug).run() + + m_alert.assert_not_called() + m_put_log.assert_not_called() + + @mock.patch('critic.libs.uptime.UptimeLogTable.put') + @mock.patch('critic.libs.uptime.UptimeMonitorTable.update') + @mock.patch('critic.libs.uptime.UptimeCheck.make_req') + @mock.patch('critic.libs.uptime.UptimeCheck.alert') + def test_alert_only_after_successful_update(self, m_alert, m_make_req, m_update, m_put_log): + # If update succeeds -> Alert + Write Log + monitor = UptimeMonitorFactory.put( + state=MonitorState.up, + consecutive_fails=1, + failures_before_alerting=2, + ) + + m_make_req.return_value = (None, 0.1) + m_update.return_value = True + + UptimeCheck(monitor.project_id, monitor.slug).run() + + m_alert.assert_called_once() + m_put_log.assert_called_once() From fe3520fd955a8c82289ef69b265b3e82dde91b8f Mon Sep 17 00:00:00 2001 From: Caleb Syring Date: Tue, 3 Mar 2026 09:43:36 -0500 Subject: [PATCH 2/3] Fix tests and add alert secrets --- mu-qa.toml | 4 ++++ src/critic/libs/uptime.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mu-qa.toml b/mu-qa.toml index 8688948..700c7e7 100644 --- a/mu-qa.toml +++ b/mu-qa.toml @@ -7,6 +7,10 @@ policy-arns = [ [deployed-env] CRITIC_NAMESPACE = 'qa' +MAILGUN_API_KEY = 'op://critic/mailgun/password' +MAILGUN_DOMAIN = 'mg.level12.io' +MAILGUN_FROM = 'critic@level12.io' +SLACK_BOT_TOKEN = 'op://critic/slack_bot_token/password' [event-rules.run-due-checks] action='run_due_checks' diff --git a/src/critic/libs/uptime.py b/src/critic/libs/uptime.py index f4256f5..35424cd 100644 --- a/src/critic/libs/uptime.py +++ b/src/critic/libs/uptime.py @@ -128,7 +128,7 @@ def put_log( state: MonitorState, status_code: int, latency: float, - error_messages: list[str] | None, + error_messages: list[str], ): """ Puts a log for the check. This method should only be called once per monitor check. @@ -141,7 +141,7 @@ def put_log( status=state, resp_code=status_code, latency_secs=latency, - error_message=error_messages, + error_message=error_messages if error_messages else None, ) UptimeLogTable.put(uptime_log) self._put_log = True From 95c62f49bab61bb23c35f0f8dde176cde16824b1 Mon Sep 17 00:00:00 2001 From: Caleb Syring Date: Tue, 3 Mar 2026 10:53:42 -0500 Subject: [PATCH 3/3] Tweak demo code --- src/critic/cli.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/critic/cli.py b/src/critic/cli.py index ce61fc0..419aa3d 100644 --- a/src/critic/cli.py +++ b/src/critic/cli.py @@ -3,7 +3,8 @@ import click -from critic.models import UptimeMonitorModel +from critic.libs.assertions import Assertion +from critic.models import MonitorState, UptimeMonitorModel from critic.tables import ProjectTable, UptimeMonitorTable @@ -41,6 +42,11 @@ def put_fake_monitors(project_id: UUID, count: int): project_id=project_id, slug=str(i), url='https://google.com', + failures_before_alerting=1, + alert_slack_channels=['C09D3TDEB9B'], + alert_emails=['critic@level12.io'], + assertions=[Assertion(assertion_string='status_code == 301')], + state=MonitorState.down, ) UptimeMonitorTable.put(monitor) click.echo(f' Put monitor: {project_id}/{i}')