diff --git a/changelog/sophoah-fix-iostat-invalid-device-metric-name.md b/changelog/sophoah-fix-iostat-invalid-device-metric-name.md new file mode 100644 index 00000000000..7d92542f4f0 --- /dev/null +++ b/changelog/sophoah-fix-iostat-invalid-device-metric-name.md @@ -0,0 +1,3 @@ +### Fixed +- Fix Prometheus scrape failures caused by `iostat` occasionally parsing a numeric string as a device name; `parseStream` now skips rows whose device name parses as a float +- Sanitize `iostat` device names via the shared `metricsutil.CanonicalizeMetricName` helper in `RegisterAndPopulateMetrics` instead of only replacing hyphens, so any invalid Prometheus metric-name character is handled diff --git a/util/iostat/iostat.go b/util/iostat/iostat.go index 7ac4ccba415..d49ea46a42c 100644 --- a/util/iostat/iostat.go +++ b/util/iostat/iostat.go @@ -14,6 +14,8 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + + "github.com/offchainlabs/nitro/util/metricsutil" ) func RegisterAndPopulateMetrics(ctx context.Context, spawnInterval, maxDeviceCount int) { @@ -33,8 +35,7 @@ func RegisterAndPopulateMetrics(ctx context.Context, spawnInterval, maxDeviceCou if _, ok := deviceMetrics[stat.DeviceName]; !ok { // Register metrics for a maximum of maxDeviceCount (fail safe in case iostat command returns incorrect names indefinitely) if len(deviceMetrics) < maxDeviceCount { - // Replace hyphens with underscores to avoid metric name issues - sanitizedDeviceName := strings.ReplaceAll(stat.DeviceName, "-", "_") + sanitizedDeviceName := metricsutil.CanonicalizeMetricName(stat.DeviceName) baseMetricName := fmt.Sprintf("iostat/%s/", sanitizedDeviceName) deviceMetrics[stat.DeviceName] = make(map[string]*metrics.GaugeFloat64) deviceMetrics[stat.DeviceName]["readspersecond"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"readspersecond", nil) @@ -121,6 +122,10 @@ func parseStream(r io.Reader, receiver chan<- DeviceStats) { if stat.DeviceName == "" { continue } + if _, err := strconv.ParseFloat(stat.DeviceName, 64); err == nil { + log.Warn("iostat returned a numeric device name, skipping implausible row", "deviceName", stat.DeviceName) + continue + } receiver <- stat } if scanner.Err() != nil { diff --git a/util/iostat/iostat_test.go b/util/iostat/iostat_test.go index 873cad19635..124f2bc48b6 100644 --- a/util/iostat/iostat_test.go +++ b/util/iostat/iostat_test.go @@ -54,6 +54,14 @@ func TestParseStream(t *testing.T) { Await: 6.75, }}, }, + { + name: "numeric device name is dropped", + input: ` + Device r/s w/s await + 99.99 1.25 2.50 3.75 + `, + want: nil, + }, } for _, tt := range tests { diff --git a/util/metricsutil/metricsutil_test.go b/util/metricsutil/metricsutil_test.go new file mode 100644 index 00000000000..6cd25fcded1 --- /dev/null +++ b/util/metricsutil/metricsutil_test.go @@ -0,0 +1,29 @@ +// Copyright 2021-2026, Offchain Labs, Inc. +// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE.md + +package metricsutil + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCanonicalizeMetricName(t *testing.T) { + tests := []struct { + name string + metric string + want string + }{ + {name: "hyphenated LVM device vg0-swap", metric: "vg0-swap", want: "vg0_swap"}, + {name: "hyphenated LVM device vg0-root", metric: "vg0-root", want: "vg0_root"}, + {name: "already valid device name nvme0n1", metric: "nvme0n1", want: "nvme0n1"}, + {name: "already valid device name loop0", metric: "loop0", want: "loop0"}, + {name: "dotted numeric token", metric: "99.99", want: "99_99"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, CanonicalizeMetricName(tt.metric)) + }) + } +}