Make Seed operation metrics StateSet

receiver/gardenerreceiver/README.md

@@ -89,7 +89,8 @@ unique identifier for a shoot is `gardener.shoot.uid`.
 | Seeds          | `garden.seed.capacity`                    | Capacity reported by the seed (e.g. shoots).                                                                                  |
 | Seeds          | `garden.seed.usage`                       | Allocatable resources reported by the seed.                                                                                   |
 | Seeds          | `garden.seed.condition`                   | One data point per seed condition with value `1`; condition type/status/reason are attributes.                                |
-| Seeds          | `garden.seed.operation`                   | Current operation type/state of the seed (with `gardener.operation.progress`).                                                |
+| Seeds          | `garden.seed.operation_states`            | One data point per supported operation type; current operation is `1`, all other operation types are `0`.                     |
+| Seeds          | `garden.seed.operation_progress_percent`  | Progress of the current operation in percent; non-current operation types are emitted with `0`.                               |
 | Projects       | `garden.project.info`                     | Static project metadata.                                                                                                      |
 | Projects       | `garden.users`                            | Total project member count grouped by user kind.                                                                              |
 | ManagedSeeds   | `garden.managed_seed.info`                | Static managed seed metadata.                                                                                                 |

receiver/gardenerreceiver/operation_test.go

@@ -0,0 +1,53 @@
+// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package gardenerreceiver
+
+import (
+	"testing"
+
+	corev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/pdata/pmetric"
+)
+
+var expectedOperationTypes = []string{
+	string(corev1beta1.LastOperationTypeCreate),
+	string(corev1beta1.LastOperationTypeReconcile),
+	string(corev1beta1.LastOperationTypeDelete),
+	string(corev1beta1.LastOperationTypeMigrate),
+	string(corev1beta1.LastOperationTypeRestore),
+	string(corev1beta1.LastOperationTypeLiveMigrate),
+}
+
+func requireOperationTypes(t *testing.T, dataPoints pmetric.NumberDataPointSlice) {
+	t.Helper()
+
+	require.Equal(t, len(expectedOperationTypes), dataPoints.Len(), "unexpected operation data point count")
+
+	operationTypes := make([]string, 0, dataPoints.Len())
+	for i := 0; i < dataPoints.Len(); i++ {
+		opType, ok := dataPoints.At(i).Attributes().Get("gardener.operation.type")
+		require.Truef(t, ok, "missing gardener.operation.type attribute on data point %d", i)
+		operationTypes = append(operationTypes, opType.Str())
+	}
+
+	require.ElementsMatch(t, expectedOperationTypes, operationTypes, "unexpected operation types")
+}
+
+func requireReconcileOperationDataPoint(t *testing.T, dataPoints pmetric.NumberDataPointSlice) pmetric.NumberDataPoint {
+	t.Helper()
+
+	for i := 0; i < dataPoints.Len(); i++ {
+		dp := dataPoints.At(i)
+		opType, ok := dp.Attributes().Get("gardener.operation.type")
+		require.Truef(t, ok, "missing gardener.operation.type attribute on data point %d", i)
+		if opType.Str() == string(corev1beta1.LastOperationTypeReconcile) {
+			return dp
+		}
+	}
+
+	t.Fatalf("missing operation data point for %q", corev1beta1.LastOperationTypeReconcile)
+	return pmetric.NumberDataPoint{}
+}

receiver/gardenerreceiver/seed.go

@@ -143,25 +143,51 @@ func (r *gardenerReceiver) collectSeedOperationStates(sm *pmetric.ScopeMetrics,
 		return
 	}
 
-	metric := sm.Metrics().AppendEmpty()
-	metric.SetName("garden.seed.operation")
-	metric.SetDescription("Operation state of a Seed. Available operations: 'Create'|'Reconcile'|'Delete'|'Restore'|'Migrate'.")
-	metric.SetUnit("")
-
-	gauge := metric.SetEmptyGauge()
+	statesMetric := sm.Metrics().AppendEmpty()
+	statesMetric.SetName("garden.seed.operation_states")
+	statesMetric.SetDescription("Operation state of a Seed. Available operations: 'Create'|'Reconcile'|'Delete'|'Migrate'|'Restore'|'LiveMigrate'.")
+	statesMetric.SetUnit("")
+	statesGauge := statesMetric.SetEmptyGauge()
+
+	progressMetric := sm.Metrics().AppendEmpty()
+	progressMetric.SetName("garden.seed.operation_progress_percent")
+	progressMetric.SetDescription("Operation progress of a Seed in percent.")
+	progressMetric.SetUnit("%")
+	progressGauge := progressMetric.SetEmptyGauge()
+
+	allOperationTypes := []corev1beta1.LastOperationType{
+		corev1beta1.LastOperationTypeCreate,
+		corev1beta1.LastOperationTypeReconcile,
+		corev1beta1.LastOperationTypeDelete,
+		corev1beta1.LastOperationTypeMigrate,
+		corev1beta1.LastOperationTypeRestore,
+		corev1beta1.LastOperationTypeLiveMigrate,
+	}
 
 	for _, seedListItem := range seedList {
 		seed := seedListItem.(*corev1beta1.Seed)
-		if seed.Status.LastOperation == nil {
-			continue
+
+		for _, opType := range allOperationTypes {
+			statesDp := statesGauge.DataPoints().AppendEmpty()
+			statesDp.SetTimestamp(now)
+			statesDp.Attributes().PutStr("gardener.seed.name", seed.Name)
+			statesDp.Attributes().PutStr("gardener.operation.type", string(opType))
+
+			progressDp := progressGauge.DataPoints().AppendEmpty()
+			progressDp.SetTimestamp(now)
+			progressDp.Attributes().PutStr("gardener.seed.name", seed.Name)
+			progressDp.Attributes().PutStr("gardener.operation.type", string(opType))
+
+			if seed.Status.LastOperation != nil && seed.Status.LastOperation.Type == opType {
+				statesDp.Attributes().PutStr("gardener.operation.state", string(seed.Status.LastOperation.State))
+				statesDp.SetIntValue(1)
+				progressDp.SetIntValue(int64(seed.Status.LastOperation.Progress))
+			} else {
+				statesDp.Attributes().PutStr("gardener.operation.state", "")
+				statesDp.SetIntValue(0)
+				progressDp.SetIntValue(0)
+			}
 		}
-		dp := gauge.DataPoints().AppendEmpty()
-		dp.SetTimestamp(now)
-		dp.SetIntValue(1)
-		dp.Attributes().PutStr("gardener.seed.name", seed.Name)
-		dp.Attributes().PutStr("gardener.operation.type", string(seed.Status.LastOperation.Type))
-		dp.Attributes().PutStr("gardener.operation.state", string(seed.Status.LastOperation.State))
-		dp.Attributes().PutInt("gardener.operation.progress", int64(seed.Status.LastOperation.Progress))
 	}
 }
 

receiver/gardenerreceiver/seed_test.go

@@ -58,26 +58,60 @@ func TestCollectSeedOperationStates(t *testing.T) {
 	sm := gardenerReceiver.initScopeMetrics(&md)
 	gardenerReceiver.collectSeedOperationStates(&sm, nowTimestamp())
 
-	require.Equal(t, 1, md.MetricCount())
-	require.Equal(t, 1, md.DataPointCount())
+	// collectSeedOperationStates emits 2 metrics (operation_states + operation_progress_percent),
+	// each with 6 data points (one per operation type: Create, Reconcile, Delete, Migrate, Restore, LiveMigrate).
+	require.Equal(t, 2, md.MetricCount())
+	require.Equal(t, 12, md.DataPointCount())
 
-	m := md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0)
-	require.Equal(t, "garden.seed.operation", m.Name())
+	scopeMetrics := md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics()
+	statesMetric := scopeMetrics.At(0)
+	require.Equal(t, "garden.seed.operation_states", statesMetric.Name())
 
-	dp := m.Gauge().DataPoints().At(0)
-	require.Equal(t, int64(1), dp.IntValue())
+	statesDataPoints := statesMetric.Gauge().DataPoints()
+	requireOperationTypes(t, statesDataPoints)
+	reconcileDp := requireReconcileOperationDataPoint(t, statesDataPoints)
 
-	opType, ok := dp.Attributes().Get("gardener.operation.type")
-	require.True(t, ok)
-	require.Equal(t, "Reconcile", opType.Str())
+	name, ok := reconcileDp.Attributes().Get("gardener.seed.name")
+	require.True(t, ok, "missing name attribute")
+	require.Equal(t, "test-seed", name.Str())
 
-	opState, ok := dp.Attributes().Get("gardener.operation.state")
+	opState, ok := reconcileDp.Attributes().Get("gardener.operation.state")
 	require.True(t, ok)
 	require.Equal(t, "Succeeded", opState.Str())
 
-	progress, ok := dp.Attributes().Get("gardener.operation.progress")
-	require.True(t, ok)
-	require.Equal(t, int64(100), progress.Int())
+	require.Equal(t, int64(1), reconcileDp.IntValue(), "active operation should have value 1")
+
+	// Every non-active operation type should have value 0 and an empty state.
+	for i := 0; i < statesDataPoints.Len(); i++ {
+		dp := statesDataPoints.At(i)
+		opType, ok := dp.Attributes().Get("gardener.operation.type")
+		require.True(t, ok, "missing operation type")
+		if opType.Str() == "Reconcile" {
+			continue
+		}
+		require.Equal(t, int64(0), dp.IntValue(), "inactive operation should have value 0")
+		state, ok := dp.Attributes().Get("gardener.operation.state")
+		require.True(t, ok, "missing operation state")
+		require.Empty(t, state.Str(), "inactive operation should have empty state")
+	}
+
+	// Verify progress metric contains the right progress for the Reconcile operation.
+	progressMetric := scopeMetrics.At(1)
+	require.Equal(t, "garden.seed.operation_progress_percent", progressMetric.Name())
+	progressDataPoints := progressMetric.Gauge().DataPoints()
+	requireOperationTypes(t, progressDataPoints)
+	reconcileProgressDp := requireReconcileOperationDataPoint(t, progressDataPoints)
+	require.Equal(t, int64(100), reconcileProgressDp.IntValue(), "unexpected progress value")
+
+	for i := 0; i < progressDataPoints.Len(); i++ {
+		dp := progressDataPoints.At(i)
+		opType, ok := dp.Attributes().Get("gardener.operation.type")
+		require.True(t, ok, "missing operation type")
+		if opType.Str() == "Reconcile" {
+			continue
+		}
+		require.Equal(t, int64(0), dp.IntValue(), "inactive operation should have progress 0")
+	}
 }
 
 func TestCollectSeedOperationStates_NoLastOperation(t *testing.T) {
@@ -105,9 +139,33 @@ func TestCollectSeedOperationStates_NoLastOperation(t *testing.T) {
 	sm := gardenerReceiver.initScopeMetrics(&md)
 	gardenerReceiver.collectSeedOperationStates(&sm, nowTimestamp())
 
-	// No last operation: metric is emitted but with no data points
-	require.Equal(t, 1, md.MetricCount())
-	require.Equal(t, 0, md.DataPointCount())
+	// No last operation: both metrics are still emitted densely, one data point
+	// per operation type, all with value 0 and an empty state.
+	require.Equal(t, 2, md.MetricCount())
+	require.Equal(t, 12, md.DataPointCount())
+
+	scopeMetrics := md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics()
+	statesMetric := scopeMetrics.At(0)
+	require.Equal(t, "garden.seed.operation_states", statesMetric.Name())
+
+	statesDataPoints := statesMetric.Gauge().DataPoints()
+	requireOperationTypes(t, statesDataPoints)
+	for i := 0; i < statesDataPoints.Len(); i++ {
+		dp := statesDataPoints.At(i)
+		require.Equal(t, int64(0), dp.IntValue(), "no active operation should have value 0")
+		state, ok := dp.Attributes().Get("gardener.operation.state")
+		require.True(t, ok, "missing operation state")
+		require.Empty(t, state.Str(), "no active operation should have empty state")
+	}
+
+	progressMetric := scopeMetrics.At(1)
+	require.Equal(t, "garden.seed.operation_progress_percent", progressMetric.Name())
+	progressDataPoints := progressMetric.Gauge().DataPoints()
+	requireOperationTypes(t, progressDataPoints)
+	for i := 0; i < progressDataPoints.Len(); i++ {
+		dp := progressDataPoints.At(i)
+		require.Equal(t, int64(0), dp.IntValue(), "no active operation should have progress 0")
+	}
 }
 
 func TestEmitSeeds_Empty(t *testing.T) {

receiver/gardenerreceiver/shoot.go

@@ -272,7 +272,7 @@ func (r *gardenerReceiver) collectShootOperationStates(sm *pmetric.ScopeMetrics,
 
 	statesMetric := sm.Metrics().AppendEmpty()
 	statesMetric.SetName("garden.shoot.operation_states")
-	statesMetric.SetDescription("Operation state of a Shoot. Available operations: 'Create'|'Reconcile'|'Delete'|'Migrate'|'Restore'.")
+	statesMetric.SetDescription("Operation state of a Shoot. Available operations: 'Create'|'Reconcile'|'Delete'|'Migrate'|'Restore'|'LiveMigrate'.")
 	statesMetric.SetUnit("")
 	statesGauge := statesMetric.SetEmptyGauge()
 
@@ -288,6 +288,7 @@ func (r *gardenerReceiver) collectShootOperationStates(sm *pmetric.ScopeMetrics,
 		corev1beta1.LastOperationTypeDelete,
 		corev1beta1.LastOperationTypeMigrate,
 		corev1beta1.LastOperationTypeRestore,
+		corev1beta1.LastOperationTypeLiveMigrate,
 	}
 
 	for _, item := range shootList {

receiver/gardenerreceiver/shoot_test.go

@@ -217,26 +217,18 @@ func TestEmitShootOperations(t *testing.T) {
 	gardenerReceiver.collectShootOperationStates(&sm, nowTimestamp())
 
 	// collectShootOperationStates emits 2 metrics (operation_states + operation_progress_percent),
-	// each with 5 data points (one per operation type: Create, Reconcile, Delete, Migrate, Restore).
+	// each with 6 data points (one per operation type: Create, Reconcile, Delete, Migrate, Restore, LiveMigrate).
 	require.Equal(t, 0, consumer.DataPointCount(), "unexpected data points")
 	require.Equal(t, 2, md.MetricCount(), "unexpected metric count")
-	require.Equal(t, 10, md.DataPointCount(), "unexpected data point count")
+	require.Equal(t, 12, md.DataPointCount(), "unexpected data point count")
 
 	scopeMetrics := md.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics()
 	statesMetric := scopeMetrics.At(0)
 	require.Equal(t, "garden.shoot.operation_states", statesMetric.Name(), "unexpected metric name")
 
-	// Find the data point for the active Reconcile operation.
-	var reconcileDp pmetric.NumberDataPoint
-	for i := 0; i < statesMetric.Gauge().DataPoints().Len(); i++ {
-		dp := statesMetric.Gauge().DataPoints().At(i)
-		opType, _ := dp.Attributes().Get("gardener.operation.type")
-		if opType.Str() == "Reconcile" {
-			reconcileDp = dp
-			break
-		}
-	}
-	require.NotNil(t, reconcileDp, "missing Reconcile data point")
+	statesDataPoints := statesMetric.Gauge().DataPoints()
+	requireOperationTypes(t, statesDataPoints)
+	reconcileDp := requireReconcileOperationDataPoint(t, statesDataPoints)
 
 	attributes := reconcileDp.Attributes()
 
@@ -261,17 +253,35 @@ func TestEmitShootOperations(t *testing.T) {
 	require.Equal(t, "shoot-uid-123", uid.Str(), "unexpected uid attribute")
 
 	require.Equal(t, int64(1), reconcileDp.IntValue(), "active operation should have value 1")
+	for i := 0; i < statesDataPoints.Len(); i++ {
+		dp := statesDataPoints.At(i)
+		opType, ok := dp.Attributes().Get("gardener.operation.type")
+		require.True(t, ok, "missing operation type")
+		if opType.Str() == "Reconcile" {
+			continue
+		}
+		require.Equal(t, int64(0), dp.IntValue(), "inactive operation should have value 0")
+		state, ok := dp.Attributes().Get("gardener.operation.state")
+		require.True(t, ok, "missing operation state")
+		require.Empty(t, state.Str(), "inactive operation should have empty state")
+	}
 
 	// Verify progress metric contains the right progress for the Reconcile operation.
 	progressMetric := scopeMetrics.At(1)
 	require.Equal(t, "garden.shoot.operation_progress_percent", progressMetric.Name(), "unexpected progress metric name")
-	for i := 0; i < progressMetric.Gauge().DataPoints().Len(); i++ {
-		dp := progressMetric.Gauge().DataPoints().At(i)
-		opType, _ := dp.Attributes().Get("gardener.operation.type")
+	progressDataPoints := progressMetric.Gauge().DataPoints()
+	requireOperationTypes(t, progressDataPoints)
+	reconcileProgressDp := requireReconcileOperationDataPoint(t, progressDataPoints)
+	require.Equal(t, int64(100), reconcileProgressDp.IntValue(), "unexpected progress value")
+
+	for i := 0; i < progressDataPoints.Len(); i++ {
+		dp := progressDataPoints.At(i)
+		opType, ok := dp.Attributes().Get("gardener.operation.type")
+		require.True(t, ok, "missing operation type")
 		if opType.Str() == "Reconcile" {
-			require.Equal(t, int64(100), dp.IntValue(), "unexpected progress value")
-			break
+			continue
 		}
+		require.Equal(t, int64(0), dp.IntValue(), "inactive operation should have progress 0")
 	}
 }