Skip to content

Add cluster label to SLM metrics #1239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
locker95 wants to merge 1 commit into
base: master
Choose a base branch
from
+48 −29
Open

Add cluster label to SLM metrics #1239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion collector/collector_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,5 @@ func (w wrapCollector) Collect(ch chan<- prometheus.Metric) {
type mockUpdateContext struct{}

func (m *mockUpdateContext) GetClusterInfo(_ context.Context) (cluster.Info, error) {
return cluster.Info{}, nil
return cluster.Info{ClusterName: "test-cluster"}, nil
}
45 changes: 32 additions & 13 deletions collector/slm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package collector

import (
"context"
"fmt"
"log/slog"
"net/http"
"net/url"
Expand All @@ -28,69 +29,69 @@ var (
slmRetentionRunsTotal = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "retention_runs_total"),
"Total retention runs",
nil, nil,
[]string{"cluster"}, nil,
)
slmRetentionFailedTotal = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "retention_failed_total"),
"Total failed retention runs",
nil, nil,
[]string{"cluster"}, nil,
)
slmRetentionTimedOutTotal = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "retention_timed_out_total"),
"Total timed out retention runs",
nil, nil,
[]string{"cluster"}, nil,
)
slmRetentionDeletionTimeSeconds = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "retention_deletion_time_seconds"),
"Retention run deletion time",
nil, nil,
[]string{"cluster"}, nil,
)
slmTotalSnapshotsTaken = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "total_snapshots_taken_total"),
"Total snapshots taken",
nil, nil,
[]string{"cluster"}, nil,
)
slmTotalSnapshotsFailed = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "total_snapshots_failed_total"),
"Total snapshots failed",
nil, nil,
[]string{"cluster"}, nil,
)
slmTotalSnapshotsDeleted = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "total_snapshots_deleted_total"),
"Total snapshots deleted",
nil, nil,
[]string{"cluster"}, nil,
)
slmTotalSnapshotsDeleteFailed = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "total_snapshot_deletion_failures_total"),
"Total snapshot deletion failures",
nil, nil,
[]string{"cluster"}, nil,
)

slmOperationMode = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "operation_mode"),
"Operating status of SLM",
[]string{"operation_mode"}, nil,
[]string{"operation_mode", "cluster"}, nil,
)

slmSnapshotsTaken = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "snapshots_taken_total"),
"Total snapshots taken",
[]string{"policy"}, nil,
[]string{"policy", "cluster"}, nil,
)
slmSnapshotsFailed = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "snapshots_failed_total"),
"Total snapshots failed",
[]string{"policy"}, nil,
[]string{"policy", "cluster"}, nil,
)
slmSnapshotsDeleted = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "snapshots_deleted_total"),
"Total snapshots deleted",
[]string{"policy"}, nil,
[]string{"policy", "cluster"}, nil,
)
slmSnapshotsDeletionFailure = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "slm_stats", "snapshot_deletion_failures_total"),
"Total snapshot deletion failures",
[]string{"policy"}, nil,
[]string{"policy", "cluster"}, nil,
)
)

Expand Down Expand Up @@ -143,6 +144,11 @@ type SLMStatusResponse struct {
}

func (s *SLM) Update(ctx context.Context, uc UpdateContext, ch chan<- prometheus.Metric) error {
clusterInfo, err := uc.GetClusterInfo(ctx)
if err != nil {
return fmt.Errorf("failed to get cluster info: %v", err)
}

u := s.u.ResolveReference(&url.URL{Path: "/_slm/status"})
var slmStatusResp SLMStatusResponse

Expand All @@ -167,50 +173,59 @@ func (s *SLM) Update(ctx context.Context, uc UpdateContext, ch chan<- prometheus
prometheus.GaugeValue,
value,
status,
clusterInfo.ClusterName,
)
}

ch <- prometheus.MustNewConstMetric(
slmRetentionRunsTotal,
prometheus.CounterValue,
float64(slmStatsResp.RetentionRuns),
clusterInfo.ClusterName,
)

ch <- prometheus.MustNewConstMetric(
slmRetentionFailedTotal,
prometheus.CounterValue,
float64(slmStatsResp.RetentionFailed),
clusterInfo.ClusterName,
)

ch <- prometheus.MustNewConstMetric(
slmRetentionTimedOutTotal,
prometheus.CounterValue,
float64(slmStatsResp.RetentionTimedOut),
clusterInfo.ClusterName,
)
ch <- prometheus.MustNewConstMetric(
slmRetentionDeletionTimeSeconds,
prometheus.GaugeValue,
float64(slmStatsResp.RetentionDeletionTimeMillis)/1000,
clusterInfo.ClusterName,
)
ch <- prometheus.MustNewConstMetric(
slmTotalSnapshotsTaken,
prometheus.CounterValue,
float64(slmStatsResp.TotalSnapshotsTaken),
clusterInfo.ClusterName,
)
ch <- prometheus.MustNewConstMetric(
slmTotalSnapshotsFailed,
prometheus.CounterValue,
float64(slmStatsResp.TotalSnapshotsFailed),
clusterInfo.ClusterName,
)
ch <- prometheus.MustNewConstMetric(
slmTotalSnapshotsDeleted,
prometheus.CounterValue,
float64(slmStatsResp.TotalSnapshotsDeleted),
clusterInfo.ClusterName,
)
ch <- prometheus.MustNewConstMetric(
slmTotalSnapshotsDeleteFailed,
prometheus.CounterValue,
float64(slmStatsResp.TotalSnapshotDeletionFailures),
clusterInfo.ClusterName,
)

for _, policy := range slmStatsResp.PolicyStats {
Expand All @@ -219,24 +234,28 @@ func (s *SLM) Update(ctx context.Context, uc UpdateContext, ch chan<- prometheus
prometheus.CounterValue,
float64(policy.SnapshotsTaken),
policy.Policy,
clusterInfo.ClusterName,
)
ch <- prometheus.MustNewConstMetric(
slmSnapshotsFailed,
prometheus.CounterValue,
float64(policy.SnapshotsFailed),
policy.Policy,
clusterInfo.ClusterName,
)
ch <- prometheus.MustNewConstMetric(
slmSnapshotsDeleted,
prometheus.CounterValue,
float64(policy.SnapshotsDeleted),
policy.Policy,
clusterInfo.ClusterName,
)
ch <- prometheus.MustNewConstMetric(
slmSnapshotsDeletionFailure,
prometheus.CounterValue,
float64(policy.SnapshotDeletionFailures),
policy.Policy,
clusterInfo.ClusterName,
)
}

Expand Down
30 changes: 15 additions & 15 deletions collector/slm_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,45 +45,45 @@ func TestSLM(t *testing.T) {
file: "7.15.0.json",
want: `# HELP elasticsearch_slm_stats_operation_mode Operating status of SLM
# TYPE elasticsearch_slm_stats_operation_mode gauge
elasticsearch_slm_stats_operation_mode{operation_mode="RUNNING"} 0
elasticsearch_slm_stats_operation_mode{operation_mode="STOPPED"} 0
elasticsearch_slm_stats_operation_mode{operation_mode="STOPPING"} 0
elasticsearch_slm_stats_operation_mode{cluster="test-cluster",operation_mode="RUNNING"} 0
elasticsearch_slm_stats_operation_mode{cluster="test-cluster",operation_mode="STOPPED"} 0
elasticsearch_slm_stats_operation_mode{cluster="test-cluster",operation_mode="STOPPING"} 0
# HELP elasticsearch_slm_stats_retention_deletion_time_seconds Retention run deletion time
# TYPE elasticsearch_slm_stats_retention_deletion_time_seconds gauge
elasticsearch_slm_stats_retention_deletion_time_seconds 72.491
elasticsearch_slm_stats_retention_deletion_time_seconds{cluster="test-cluster"} 72.491
# HELP elasticsearch_slm_stats_retention_failed_total Total failed retention runs
# TYPE elasticsearch_slm_stats_retention_failed_total counter
elasticsearch_slm_stats_retention_failed_total 0
elasticsearch_slm_stats_retention_failed_total{cluster="test-cluster"} 0
# HELP elasticsearch_slm_stats_retention_runs_total Total retention runs
# TYPE elasticsearch_slm_stats_retention_runs_total counter
elasticsearch_slm_stats_retention_runs_total 9
elasticsearch_slm_stats_retention_runs_total{cluster="test-cluster"} 9
# HELP elasticsearch_slm_stats_retention_timed_out_total Total timed out retention runs
# TYPE elasticsearch_slm_stats_retention_timed_out_total counter
elasticsearch_slm_stats_retention_timed_out_total 0
elasticsearch_slm_stats_retention_timed_out_total{cluster="test-cluster"} 0
# HELP elasticsearch_slm_stats_snapshot_deletion_failures_total Total snapshot deletion failures
# TYPE elasticsearch_slm_stats_snapshot_deletion_failures_total counter
elasticsearch_slm_stats_snapshot_deletion_failures_total{policy="everything"} 0
elasticsearch_slm_stats_snapshot_deletion_failures_total{cluster="test-cluster",policy="everything"} 0
# HELP elasticsearch_slm_stats_snapshots_deleted_total Total snapshots deleted
# TYPE elasticsearch_slm_stats_snapshots_deleted_total counter
elasticsearch_slm_stats_snapshots_deleted_total{policy="everything"} 20
elasticsearch_slm_stats_snapshots_deleted_total{cluster="test-cluster",policy="everything"} 20
# HELP elasticsearch_slm_stats_snapshots_failed_total Total snapshots failed
# TYPE elasticsearch_slm_stats_snapshots_failed_total counter
elasticsearch_slm_stats_snapshots_failed_total{policy="everything"} 2
elasticsearch_slm_stats_snapshots_failed_total{cluster="test-cluster",policy="everything"} 2
# HELP elasticsearch_slm_stats_snapshots_taken_total Total snapshots taken
# TYPE elasticsearch_slm_stats_snapshots_taken_total counter
elasticsearch_slm_stats_snapshots_taken_total{policy="everything"} 50
elasticsearch_slm_stats_snapshots_taken_total{cluster="test-cluster",policy="everything"} 50
# HELP elasticsearch_slm_stats_total_snapshot_deletion_failures_total Total snapshot deletion failures
# TYPE elasticsearch_slm_stats_total_snapshot_deletion_failures_total counter
elasticsearch_slm_stats_total_snapshot_deletion_failures_total 0
elasticsearch_slm_stats_total_snapshot_deletion_failures_total{cluster="test-cluster"} 0
# HELP elasticsearch_slm_stats_total_snapshots_deleted_total Total snapshots deleted
# TYPE elasticsearch_slm_stats_total_snapshots_deleted_total counter
elasticsearch_slm_stats_total_snapshots_deleted_total 20
elasticsearch_slm_stats_total_snapshots_deleted_total{cluster="test-cluster"} 20
# HELP elasticsearch_slm_stats_total_snapshots_failed_total Total snapshots failed
# TYPE elasticsearch_slm_stats_total_snapshots_failed_total counter
elasticsearch_slm_stats_total_snapshots_failed_total 2
elasticsearch_slm_stats_total_snapshots_failed_total{cluster="test-cluster"} 2
# HELP elasticsearch_slm_stats_total_snapshots_taken_total Total snapshots taken
# TYPE elasticsearch_slm_stats_total_snapshots_taken_total counter
elasticsearch_slm_stats_total_snapshots_taken_total 103
elasticsearch_slm_stats_total_snapshots_taken_total{cluster="test-cluster"} 103
`,
},
}
Expand Down