feat: add native incident.io Alertmanager receiver

CHANGELOG.md

@@ -22,6 +22,7 @@
 * [ENHANCEMENT] Validation: Add an optional `reason` field to `limited_queries` rules, aligning them with `blocked_queries`. When set, the reason is included in the client-facing error and the query-frontend's `"query limited"` log line. #16407
 * [ENHANCEMENT] Compactor: Add the experimental `-compactor.scheduler-client.enable-ring-based-cleanup` option, which when disabled stops a scheduler-mode compactor from running the ring-based background blocks cleaner. #16457
 * [ENHANCEMENT] Block-builder-scheduler: Add `cortex_blockbuilder_scheduler_end_offset_probe_failed_total`, counting failures to list a cluster's end offsets, and `cortex_blockbuilder_scheduler_startup_jobs_skipped_total`, counting observed jobs that startup recovery could not import. #16134
+* [FEATURE] Alertmanager: Add incident.io as a supported integration.
 * [FEATURE] Querier: Add experimental per-tenant limit `-querier.max-blocks-per-store-request` to cap the number of blocks a single store-gateway request may reference. Disabled by default. #16292
 * [FEATURE] Validation: Add optional `id`, `note`, `created_by`, `created_at`, and `expires_at` fields to `blocked_queries` and `limited_queries` rules, for tooling to attach ownership/context metadata to a rule. For rules with `expires_at` set, the earliest `expires_at` per tenant and `id` (rules without an `id` are grouped together) is exported as the `cortex_blocked_query_rule_expires_at`/`cortex_limited_query_rule_expires_at` metrics, so an alert can fire on stale rules; this is informational only and never affects enforcement. The query-frontend's `"query blocked"` log line now also includes the matched rule's `id` and whether it is expired, and rate-limited queries are now logged with a new `"query limited"` line carrying the same fields. #16395
 * [BUGFIX] Query-frontend: Wait for the querier ring to be populated during startup, up to 30 seconds, before reporting the query-frontend as ready. Previously a query-frontend could become ready before it had seen any querier in the ring and fail every query it received until the ring was populated. Only applies when remote execution is enabled, and can be disabled with the experimental `-query-frontend.wait-for-querier-ring-on-startup=false`. #16333

pkg/alertmanager/alertmanager.go

@@ -37,6 +37,7 @@ import (
 	"github.com/prometheus/alertmanager/notify"
 	discord "github.com/prometheus/alertmanager/notify/discord"
 	email "github.com/prometheus/alertmanager/notify/email"
+	incidentio "github.com/prometheus/alertmanager/notify/incidentio"
 	msteams "github.com/prometheus/alertmanager/notify/msteams"
 	msteamsv2 "github.com/prometheus/alertmanager/notify/msteamsv2"
 	opsgenie "github.com/prometheus/alertmanager/notify/opsgenie"
@@ -644,6 +645,11 @@ func buildReceiverIntegrations(nc config.Receiver, tmpl *template.Template, fire
 			return msteamsv2.New(c, tmpl, l, httpOps...)
 		})
 	}
+	for i, c := range nc.IncidentioConfigs {
+		add("incidentio", i, c, func(l *slog.Logger) (notify.Notifier, error) {
+			return incidentio.New(c, tmpl, l, httpOps...)
+		})
+	}
 	// If we add support for more integrations, we need to add them to validation as well. See validation.allowedIntegrationNames field.
 	if len(errs) > 0 {
 		return nil, errors.Join(errs...)

pkg/alertmanager/api.go

@@ -20,6 +20,7 @@ import (
 	"github.com/grafana/dskit/tenant"
 	"github.com/prometheus/alertmanager/config"
 	discord "github.com/prometheus/alertmanager/notify/discord"
+	incidentio "github.com/prometheus/alertmanager/notify/incidentio"
 	msteams "github.com/prometheus/alertmanager/notify/msteams"
 	webhook "github.com/prometheus/alertmanager/notify/webhook"
 	"github.com/prometheus/alertmanager/template"
@@ -47,20 +48,22 @@ const (
 )
 
 var (
-	errPasswordFileNotAllowed            = errors.New("setting smtp_auth_password_file, password_file, bearer_token_file, auth_password_file or credentials_file is not allowed")
-	errOAuth2SecretFileNotAllowed        = errors.New("setting OAuth2 client_secret_file is not allowed")
-	errProxyURLNotAllowed                = errors.New("setting proxy_url is not allowed")
-	errProxyFromEnvironmentURLNotAllowed = errors.New("setting proxy_from_environment is not allowed")
-	errTLSConfigNotAllowed               = errors.New("setting TLS ca_file, cert_file, key_file, ca, cert or key is not allowed")
-	errSlackAPIURLFileNotAllowed         = errors.New("setting Slack api_url_file or global slack_api_url_file is not allowed")
-	errVictorOpsAPIKeyFileNotAllowed     = errors.New("setting VictorOps api_key_file or global victorops_api_key_file is not allowed")
-	errOpsGenieAPIKeyFileFileNotAllowed  = errors.New("setting OpsGenie api_key_file or global opsgenie_api_key_file is not allowed")
-	errPagerDutyServiceKeyFileNotAllowed = errors.New("setting PagerDuty service_key_file is not allowed")
-	errPagerDutyRoutingKeyFileNotAllowed = errors.New("setting PagerDuty routing_key_file is not allowed")
-	errPushoverUserKeyFileNotAllowed     = errors.New("setting Pushover user_key_file is not allowed")
-	errPushoverTokenFileNotAllowed       = errors.New("setting Pushover token_file is not allowed")
-	errTelegramBotTokenFileNotAllowed    = errors.New("setting Telegram bot_token_file is not allowed")
-	errWebhookURLFileNotAllowed          = errors.New("setting Webhook url_file is not allowed")
+	errPasswordFileNotAllowed                   = errors.New("setting smtp_auth_password_file, password_file, bearer_token_file, auth_password_file or credentials_file is not allowed")
+	errOAuth2SecretFileNotAllowed               = errors.New("setting OAuth2 client_secret_file is not allowed")
+	errProxyURLNotAllowed                       = errors.New("setting proxy_url is not allowed")
+	errProxyFromEnvironmentURLNotAllowed        = errors.New("setting proxy_from_environment is not allowed")
+	errTLSConfigNotAllowed                      = errors.New("setting TLS ca_file, cert_file, key_file, ca, cert or key is not allowed")
+	errSlackAPIURLFileNotAllowed                = errors.New("setting Slack api_url_file or global slack_api_url_file is not allowed")
+	errVictorOpsAPIKeyFileNotAllowed            = errors.New("setting VictorOps api_key_file or global victorops_api_key_file is not allowed")
+	errOpsGenieAPIKeyFileFileNotAllowed         = errors.New("setting OpsGenie api_key_file or global opsgenie_api_key_file is not allowed")
+	errPagerDutyServiceKeyFileNotAllowed        = errors.New("setting PagerDuty service_key_file is not allowed")
+	errPagerDutyRoutingKeyFileNotAllowed        = errors.New("setting PagerDuty routing_key_file is not allowed")
+	errPushoverUserKeyFileNotAllowed            = errors.New("setting Pushover user_key_file is not allowed")
+	errPushoverTokenFileNotAllowed              = errors.New("setting Pushover token_file is not allowed")
+	errTelegramBotTokenFileNotAllowed           = errors.New("setting Telegram bot_token_file is not allowed")
+	errWebhookURLFileNotAllowed                 = errors.New("setting Webhook url_file is not allowed")
+	errIncidentioURLFileNotAllowed              = errors.New("setting incident.io url_file is not allowed")
+	errIncidentioAlertSourceTokenFileNotAllowed = errors.New("setting incident.io alert_source_token_file is not allowed")
 )
 
 // UserConfig is used to communicate a users alertmanager configs
@@ -414,6 +417,11 @@ func validateAlertmanagerConfig(cfg interface{}) error {
 		if err := validateWebhookConfig(v.Interface().(webhook.WebhookConfig)); err != nil {
 			return err
 		}
+
+	case reflect.TypeOf(incidentio.IncidentioConfig{}):
+		if err := validateIncidentioConfig(v.Interface().(incidentio.IncidentioConfig)); err != nil {
+			return err
+		}
 	}
 
 	// If the input config is a struct, recursively iterate on all fields.
@@ -621,3 +629,15 @@ func validateWebhookConfig(cfg webhook.WebhookConfig) error {
 	}
 	return nil
 }
+
+// validateIncidentioConfig validates the incident.io config and returns an error if it
+// contains settings not allowed by Mimir.
+func validateIncidentioConfig(cfg incidentio.IncidentioConfig) error {
+	if cfg.URLFile != "" {
+		return errIncidentioURLFileNotAllowed
+	}
+	if cfg.AlertSourceTokenFile != "" {
+		return errIncidentioAlertSourceTokenFileNotAllowed
+	}
+	return nil
+}

pkg/alertmanager/api_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/featurecontrol"
 	discord "github.com/prometheus/alertmanager/notify/discord"
+	incidentio "github.com/prometheus/alertmanager/notify/incidentio"
 	msteams "github.com/prometheus/alertmanager/notify/msteams"
 	webhook "github.com/prometheus/alertmanager/notify/webhook"
 	"github.com/prometheus/client_golang/prometheus"
@@ -732,6 +733,50 @@ alertmanager_config: |
 `,
 			err: fmt.Errorf("error validating Alertmanager config: %w", errWebhookURLFileNotAllowed),
 		},
+		{
+			name: "should pass if incident.io config uses inline token",
+			cfg: `
+alertmanager_config: |
+  receivers:
+    - name: default-receiver
+      incidentio_configs:
+        - url: https://api.incident.io/v2/alert_events/alertmanager/01EXAMPLE
+          alert_source_token: token
+
+  route:
+    receiver: 'default-receiver'
+`,
+		},
+		{
+			name: "should return error if incident.io url_file is set",
+			cfg: `
+alertmanager_config: |
+  receivers:
+    - name: default-receiver
+      incidentio_configs:
+        - url_file: /secrets
+          alert_source_token: token
+
+  route:
+    receiver: 'default-receiver'
+`,
+			err: fmt.Errorf("error validating Alertmanager config: %w", errIncidentioURLFileNotAllowed),
+		},
+		{
+			name: "should return error if incident.io alert_source_token_file is set",
+			cfg: `
+alertmanager_config: |
+  receivers:
+    - name: default-receiver
+      incidentio_configs:
+        - url: https://api.incident.io/v2/alert_events/alertmanager/01EXAMPLE
+          alert_source_token_file: /secrets
+
+  route:
+    receiver: 'default-receiver'
+`,
+			err: fmt.Errorf("error validating Alertmanager config: %w", errIncidentioAlertSourceTokenFileNotAllowed),
+		},
 		{
 			name: "should return error if template is wrong",
 			cfg: `
@@ -1205,6 +1250,30 @@ func TestValidateAlertmanagerConfig(t *testing.T) {
 			},
 			expected: errWebhookURLFileNotAllowed,
 		},
+		"*IncidentioConfig.URLFile": {
+			input: &incidentio.IncidentioConfig{
+				URLFile: "/file",
+			},
+			expected: errIncidentioURLFileNotAllowed,
+		},
+		"IncidentioConfig.URLFile": {
+			input: incidentio.IncidentioConfig{
+				URLFile: "/file",
+			},
+			expected: errIncidentioURLFileNotAllowed,
+		},
+		"*IncidentioConfig.AlertSourceTokenFile": {
+			input: &incidentio.IncidentioConfig{
+				AlertSourceTokenFile: "/file",
+			},
+			expected: errIncidentioAlertSourceTokenFileNotAllowed,
+		},
+		"IncidentioConfig.AlertSourceTokenFile": {
+			input: incidentio.IncidentioConfig{
+				AlertSourceTokenFile: "/file",
+			},
+			expected: errIncidentioAlertSourceTokenFileNotAllowed,
+		},
 		"struct containing *HTTPClientConfig as direct child": {
 			input: config.GlobalConfig{
 				HTTPConfig: &commoncfg.HTTPClientConfig{

pkg/util/validation/notifications_limit_flag.go

@@ -21,7 +21,7 @@ func validateIntegrationLimit(k string, _ float64) error {
 
 // allowedIntegrationNames is a list of all the integrations that can be rate limited.
 var allowedIntegrationNames = []string{
-	"webhook", "email", "pagerduty", "opsgenie", "wechat", "slack", "victorops", "pushover", "sns", "webex", "telegram", "discord", "msteams", "msteamsv2",
+	"webhook", "email", "pagerduty", "opsgenie", "wechat", "slack", "victorops", "pushover", "sns", "webex", "telegram", "discord", "msteams", "msteamsv2", "incidentio",
 }
 
 // NotificationRateLimitMap returns a map that can be used as a flag for setting notification rate limits.