Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changelog/sophoah-fix-iostat-invalid-device-metric-name.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed
- Fix Prometheus scrape failures caused by `iostat` occasionally parsing a numeric string as a device name; `parseStream` now skips rows whose device name parses as a float
- Sanitize `iostat` device names via the shared `metricsutil.CanonicalizeMetricName` helper in `RegisterAndPopulateMetrics` instead of only replacing hyphens, so any invalid Prometheus metric-name character is handled
9 changes: 7 additions & 2 deletions util/iostat/iostat.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (

"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"

"github.com/offchainlabs/nitro/util/metricsutil"
)

func RegisterAndPopulateMetrics(ctx context.Context, spawnInterval, maxDeviceCount int) {
Expand All @@ -33,8 +35,7 @@ func RegisterAndPopulateMetrics(ctx context.Context, spawnInterval, maxDeviceCou
if _, ok := deviceMetrics[stat.DeviceName]; !ok {
// Register metrics for a maximum of maxDeviceCount (fail safe in case iostat command returns incorrect names indefinitely)
if len(deviceMetrics) < maxDeviceCount {
// Replace hyphens with underscores to avoid metric name issues
sanitizedDeviceName := strings.ReplaceAll(stat.DeviceName, "-", "_")
sanitizedDeviceName := metricsutil.CanonicalizeMetricName(stat.DeviceName)
baseMetricName := fmt.Sprintf("iostat/%s/", sanitizedDeviceName)
deviceMetrics[stat.DeviceName] = make(map[string]*metrics.GaugeFloat64)
deviceMetrics[stat.DeviceName]["readspersecond"] = metrics.NewRegisteredGaugeFloat64(baseMetricName+"readspersecond", nil)
Expand Down Expand Up @@ -121,6 +122,10 @@ func parseStream(r io.Reader, receiver chan<- DeviceStats) {
if stat.DeviceName == "" {
continue
}
if _, err := strconv.ParseFloat(stat.DeviceName, 64); err == nil {
log.Warn("iostat returned a numeric device name, skipping implausible row", "deviceName", stat.DeviceName)
continue
}
receiver <- stat
}
if scanner.Err() != nil {
Expand Down
8 changes: 8 additions & 0 deletions util/iostat/iostat_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ func TestParseStream(t *testing.T) {
Await: 6.75,
}},
},
{
name: "numeric device name is dropped",
input: `
Device r/s w/s await
99.99 1.25 2.50 3.75
`,
want: nil,
},
}

for _, tt := range tests {
Expand Down
29 changes: 29 additions & 0 deletions util/metricsutil/metricsutil_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Copyright 2021-2026, Offchain Labs, Inc.
// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE.md

package metricsutil

import (
"testing"

"github.com/stretchr/testify/require"
)

func TestCanonicalizeMetricName(t *testing.T) {
tests := []struct {
name string
metric string
want string
}{
{name: "hyphenated LVM device vg0-swap", metric: "vg0-swap", want: "vg0_swap"},
{name: "hyphenated LVM device vg0-root", metric: "vg0-root", want: "vg0_root"},
{name: "already valid device name nvme0n1", metric: "nvme0n1", want: "nvme0n1"},
{name: "already valid device name loop0", metric: "loop0", want: "loop0"},
{name: "dotted numeric token", metric: "99.99", want: "99_99"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
require.Equal(t, tt.want, CanonicalizeMetricName(tt.metric))
})
}
}