From 47c03bbc7effa32c95218a26bad3e4e11f670ba1 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Thu, 22 Jan 2026 09:32:04 +0100 Subject: [PATCH] OCPBUGS-66152: split grpc requests slow into read/write alerts Signed-off-by: Thomas Jungblut --- jsonnet/custom.libsonnet | 29 +- jsonnet/jsonnetfile.lock.json | 8 +- .../jsonnet-libs/xtd/array.libsonnet | 14 + .../jsonnet-libs/xtd/docs/README.md | 1 + .../github.com/jsonnet-libs/xtd/docs/array.md | 10 + .../jsonnet-libs/xtd/docs/string.md | 16 ++ .../github.com/jsonnet-libs/xtd/docs/units.md | 61 +++++ .../jsonnet-libs/xtd/main.libsonnet | 1 + .../jsonnet-libs/xtd/string.libsonnet | 26 ++ .../jsonnet-libs/xtd/test/array_test.jsonnet | 30 +++ .../jsonnet-libs/xtd/test/string_test.jsonnet | 33 +++ .../jsonnet-libs/xtd/test/units_test.jsonnet | 248 ++++++++++++++++++ .../jsonnet-libs/xtd/units.libsonnet | 93 +++++++ ...00_90_etcd-operator_03_prometheusrule.yaml | 27 +- 14 files changed, 569 insertions(+), 28 deletions(-) create mode 100644 jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/units.md create mode 100644 jsonnet/vendor/github.com/jsonnet-libs/xtd/test/string_test.jsonnet create mode 100644 jsonnet/vendor/github.com/jsonnet-libs/xtd/test/units_test.jsonnet create mode 100644 jsonnet/vendor/github.com/jsonnet-libs/xtd/units.libsonnet diff --git a/jsonnet/custom.libsonnet b/jsonnet/custom.libsonnet index 5cd4cb1d75..0378995436 100644 --- a/jsonnet/custom.libsonnet +++ b/jsonnet/custom.libsonnet @@ -41,22 +41,27 @@ }, }, { - alert: 'etcdGRPCRequestsSlow', - expr: ||| - histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method!="Defragment", grpc_type="unary"}[10m])) without(grpc_type)) - > on () group_left (type) - bottomk(1, - 1.5 * group by (type) (cluster_infrastructure_provider{type="Azure"}) - or - 1 * group by (type) (cluster_infrastructure_provider)) - |||, - 'for': '30m', + alert: 'etcdGRPCReadRequestsSlow', + expr: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method="Range", grpc_type="unary"}[10m])) without(grpc_type)) > 3', + 'for': '10m', + labels: { + severity: 'critical', + }, + annotations: { + description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC read requests is {{ $value }}s on etcd instance {{ $labels.instance }}.', + summary: 'etcd grpc read requests are slow', + }, + }, + { + alert: 'etcdGRPCWriteRequestsSlow', + expr: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method="Txn", grpc_type="unary"}[10m])) without(grpc_type)) > 5', + 'for': '10m', labels: { severity: 'critical', }, annotations: { - description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.', - summary: 'etcd grpc requests are slow', + description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC write requests is {{ $value }}s on etcd instance {{ $labels.instance }}.', + summary: 'etcd grpc write requests are slow', }, }, { diff --git a/jsonnet/jsonnetfile.lock.json b/jsonnet/jsonnetfile.lock.json index 3ec8200a61..36ce0ca370 100644 --- a/jsonnet/jsonnetfile.lock.json +++ b/jsonnet/jsonnetfile.lock.json @@ -8,7 +8,7 @@ "subdir": "contrib/mixin" } }, - "version": "e4d6a05f8f1ec972384e24a83c420f707a6644f2", + "version": "9a2b574b70e69eb1f47dfc896b2f63245f1873f6", "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po=" }, { @@ -18,7 +18,7 @@ "subdir": "gen/grafonnet-v10.0.0" } }, - "version": "42d098fae987f25f08480e203ca6ddc548c6efbf", + "version": "7380c9c64fb973f34c3ec46265621a2b0dee0058", "sum": "xdcrJPJlpkq4+5LpGwN4tPAuheNNLXZjE6tDcyvFjr0=" }, { @@ -38,8 +38,8 @@ "subdir": "" } }, - "version": "4eee017d21cb63a303925d1dcd9fc5c496809b46", - "sum": "Kh0GbIycNmJPzk6IOMXn1BbtLNyaiiimclYk7+mvsns=" + "version": "4d7f8cb24d613430799f9d56809cc6964f35cea9", + "sum": "hOrwkOx34tOXqoDVnwuI/Uf/dr9HFFSPWpDPOvnEGrk=" } ], "legacyImports": false diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/array.libsonnet b/jsonnet/vendor/github.com/jsonnet-libs/xtd/array.libsonnet index b43c0f0ea0..22a5059092 100644 --- a/jsonnet/vendor/github.com/jsonnet-libs/xtd/array.libsonnet +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/array.libsonnet @@ -52,4 +52,18 @@ local d = import 'github.com/jsonnet-libs/docsonnet/doc-util/main.libsonnet'; for i in std.range(0, std.length(arr) - 1) if filter_func(i, arr[i]) ], + + '#chunkArray':: d.fn( + ||| + `chunkArray` chunks an array into smaller arrays of the given max size. + |||, + [ + d.arg('arr', d.T.array), + d.arg('maxSize', d.T.number), + ] + ), + chunkArray(arr, maxSize): [ + arr[i * maxSize:std.min((i + 1) * maxSize, std.length(arr))] + for i in std.range(0, std.ceil(std.length(arr) / maxSize) - 1) + ], } diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/README.md b/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/README.md index 61d9c39e60..b5c05d359c 100644 --- a/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/README.md +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/README.md @@ -23,4 +23,5 @@ in the future, but also provides a place for less general, yet useful utilities. * [jsonpath](jsonpath.md) * [number](number.md) * [string](string.md) +* [units](units.md) * [url](url.md) \ No newline at end of file diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/array.md b/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/array.md index cc538d867f..93642d8e80 100644 --- a/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/array.md +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/array.md @@ -12,11 +12,21 @@ local array = import "github.com/jsonnet-libs/xtd/array.libsonnet" ## Index +* [`fn chunkArray(arr, maxSize)`](#fn-chunkarray) * [`fn filterMapWithIndex(filter_func, map_func, arr)`](#fn-filtermapwithindex) * [`fn slice(indexable, index, end='null', step=1)`](#fn-slice) ## Fields +### fn chunkArray + +```ts +chunkArray(arr, maxSize) +``` + +`chunkArray` chunks an array into smaller arrays of the given max size. + + ### fn filterMapWithIndex ```ts diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/string.md b/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/string.md index 62873e57f9..2642e1b7d1 100644 --- a/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/string.md +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/string.md @@ -13,6 +13,7 @@ local string = import "github.com/jsonnet-libs/xtd/string.libsonnet" ## Index * [`fn splitEscape(str, c, escape='\\')`](#fn-splitescape) +* [`fn strReplaceMulti(str, replacements)`](#fn-strreplacemulti) ## Fields @@ -24,3 +25,18 @@ splitEscape(str, c, escape='\\') `split` works the same as `std.split` but with support for escaping the dividing string `c`. + + +### fn strReplaceMulti + +```ts +strReplaceMulti(str, replacements) +``` + +`strReplaceMulti` replaces multiple substrings in a string. + +Example: +```jsonnet +strReplaceMulti('hello world', [['hello', 'goodbye'], ['world', 'universe']]) +// 'goodbye universe' +``` diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/units.md b/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/units.md new file mode 100644 index 0000000000..a3ae7627eb --- /dev/null +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/docs/units.md @@ -0,0 +1,61 @@ +--- +permalink: /units/ +--- + +# units + +```jsonnet +local units = import "github.com/jsonnet-libs/xtd/units.libsonnet" +``` + +`units` implements helper functions for converting units. + +## Index + +* [`fn formatDuration(seconds)`](#fn-formatduration) +* [`fn parseDuration(duration)`](#fn-parseduration) +* [`fn parseKubernetesCPU(input)`](#fn-parsekubernetescpu) +* [`fn siToBytes(str)`](#fn-sitobytes) + +## Fields + +### fn formatDuration + +```ts +formatDuration(seconds) +``` + +`formatDuration` formats a number of seconds into a human-readable duration string. +Returns the duration in the smallest appropriate unit (s, m, h, or combined formats like "4m30s"). + + +### fn parseDuration + +```ts +parseDuration(duration) +``` + +`parseDuration` parses a duration string and returns the number of seconds. +Handles milliseconds (ms), seconds (s), minutes (m), hours (h), and combined formats like "4m30s" or "1h30m". + + +### fn parseKubernetesCPU + +```ts +parseKubernetesCPU(input) +``` + +`parseKubernetesCPU` parses a Kubernetes CPU string/number into a number of cores. + The function assumes the input is in a correct Kubernetes format, i.e., an integer, a float, + a string representation of an integer or a float, or a string containing a number ending with 'm' + representing a number of millicores. + + +### fn siToBytes + +```ts +siToBytes(str) +``` + +`siToBytes` converts Kubernetes byte units to bytes. +Only works for limited set of SI prefixes: Ki, Mi, Gi, Ti. diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/main.libsonnet b/jsonnet/vendor/github.com/jsonnet-libs/xtd/main.libsonnet index 5bda29773f..3edfdc384d 100644 --- a/jsonnet/vendor/github.com/jsonnet-libs/xtd/main.libsonnet +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/main.libsonnet @@ -21,5 +21,6 @@ local d = import 'github.com/jsonnet-libs/docsonnet/doc-util/main.libsonnet'; jsonpath: (import './jsonpath.libsonnet'), number: (import './number.libsonnet'), string: (import './string.libsonnet'), + units: (import './units.libsonnet'), url: (import './url.libsonnet'), } diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/string.libsonnet b/jsonnet/vendor/github.com/jsonnet-libs/xtd/string.libsonnet index 665598189f..5dccf13126 100644 --- a/jsonnet/vendor/github.com/jsonnet-libs/xtd/string.libsonnet +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/string.libsonnet @@ -32,4 +32,30 @@ local d = import 'github.com/jsonnet-libs/docsonnet/doc-util/main.libsonnet'; c, ) ), + + '#strReplaceMulti':: d.fn( + ||| + `strReplaceMulti` replaces multiple substrings in a string. + + Example: + ```jsonnet + strReplaceMulti('hello world', [['hello', 'goodbye'], ['world', 'universe']]) + // 'goodbye universe' + ``` + |||, + [ + d.arg('str', d.T.string), + d.arg('replacements', d.T.array), + ] + ), + strReplaceMulti(str, replacements): + assert std.isString(str) : 'str must be a string'; + assert std.isArray(replacements) : 'replacements must be an array'; + assert std.all([std.isArray(r) && std.length(r) == 2 && std.isString(r[0]) && std.isString(r[1]) for r in replacements]) : 'replacements must be an array of arrays of strings'; + std.foldl( + function(acc, replacement) + std.strReplace(acc, replacement[0], replacement[1]), + replacements, + str, + ), } diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/array_test.jsonnet b/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/array_test.jsonnet index 65aef8a210..4c9a36e0ff 100644 --- a/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/array_test.jsonnet +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/array_test.jsonnet @@ -2,6 +2,7 @@ local array = import '../array.libsonnet'; local test = import 'github.com/jsonnet-libs/testonnet/main.libsonnet'; local arr = std.range(0, 10); +local mixedArr = ['a', 1, 'b', 2, 'c', 3, 4, 5, 6, 7, 8, 9, 'd', 'e', 'f', 'g', 'h', 'i', 'j']; test.new(std.thisFile) @@ -81,3 +82,32 @@ test.new(std.thisFile) expected=[0, 2, 4], ) ) + ++ test.case.new( + name='chunkArray', + test=test.expect.eq( + actual=array.chunkArray(mixedArr, maxSize=3), + expected=[['a', 1, 'b'], [2, 'c', 3], [4, 5, 6], [7, 8, 9], ['d', 'e', 'f'], ['g', 'h', 'i'], ['j']], + ) +) ++ test.case.new( + name='chunkArray - maxSize is 2', + test=test.expect.eq( + actual=array.chunkArray(mixedArr, maxSize=2), + expected=[['a', 1], ['b', 2], ['c', 3], [4, 5], [6, 7], [8, 9], ['d', 'e'], ['f', 'g'], ['h', 'i'], ['j']], + ) +) ++ test.case.new( + name='chunkArray - maxSize is larger than array length', + test=test.expect.eq( + actual=array.chunkArray(mixedArr, maxSize=100), + expected=[mixedArr], + ) +) ++ test.case.new( + name='chunkArray - maxSize is 1', + test=test.expect.eq( + actual=array.chunkArray(mixedArr, maxSize=1), + expected=[['a'], [1], ['b'], [2], ['c'], [3], [4], [5], [6], [7], [8], [9], ['d'], ['e'], ['f'], ['g'], ['h'], ['i'], ['j']], + ) +) diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/string_test.jsonnet b/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/string_test.jsonnet new file mode 100644 index 0000000000..4f375eb1d2 --- /dev/null +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/string_test.jsonnet @@ -0,0 +1,33 @@ +local string = import '../string.libsonnet'; +local test = import 'github.com/jsonnet-libs/testonnet/main.libsonnet'; + +test.new(std.thisFile) + ++ test.case.new( + name='strReplaceMulti', + test=test.expect.eq( + actual=string.strReplaceMulti('hello world', [['hello', 'goodbye'], ['world', 'universe']]), + expected='goodbye universe', + ) +) ++ test.case.new( + name='strReplaceMulti - chained', + test=test.expect.eq( + actual=string.strReplaceMulti('hello world', [['hello', 'goodbye'], ['goodbye', 'hi again']]), + expected='hi again world', + ) +) ++ test.case.new( + name='strReplaceMulti - not found', + test=test.expect.eq( + actual=string.strReplaceMulti('hello world', [['first', 'second'], ['third', 'fourth']]), + expected='hello world', + ) +) ++ test.case.new( + name='strReplaceMulti - empty replacements', + test=test.expect.eq( + actual=string.strReplaceMulti('hello world', []), + expected='hello world', + ) +) diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/units_test.jsonnet b/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/units_test.jsonnet new file mode 100644 index 0000000000..d2f4459d95 --- /dev/null +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/test/units_test.jsonnet @@ -0,0 +1,248 @@ +local units = import '../units.libsonnet'; +local test = import 'github.com/jsonnet-libs/testonnet/main.libsonnet'; + +test.new(std.thisFile) +// parseCPU(10) = parseCPU("10") = 10 +// parseCPU(4.5) = parse("4.5") = 4.5 +// parseCPU("3000m") = 3000 / 1000 +// parseCPU("3580m") = 3580 / 1000 +// parseCPU("3980.7m") = 3980.7 / 1000 +// parseCPU(0.5) = parse("0.5") = parse("500m") = 0.5 ++ test.case.new( + name='parseCPU - integer', + test=test.expect.eq( + actual=units.parseKubernetesCPU(10), + expected=10, + ) +) ++ test.case.new( + name='parseCPU - string integer', + test=test.expect.eq( + actual=units.parseKubernetesCPU('10'), + expected=10, + ) +) ++ test.case.new( + name='parseCPU - float', + test=test.expect.eq( + actual=units.parseKubernetesCPU(4.5), + expected=4.5, + ) +) ++ test.case.new( + name='parseCPU - string float', + test=test.expect.eq( + actual=units.parseKubernetesCPU('4.5'), + expected=4.5, + ) +) ++ test.case.new( + name='parseCPU - string millicores', + test=test.expect.eq( + actual=units.parseKubernetesCPU('3000m'), + expected=3, + ) +) ++ test.case.new( + name='parseCPU - string millicores', + test=test.expect.eq( + actual=units.parseKubernetesCPU('3580m'), + expected=3.58, + ) +) ++ test.case.new( + name='parseCPU - string millicores', + test=test.expect.eq( + actual=units.parseKubernetesCPU('3980.7m'), + expected=3980.7 / 1000, // Use a division to avoid floating point precision issues + ) +) ++ test.case.new( + name='parseCPU - fraction', + test=test.expect.eq( + actual=units.parseKubernetesCPU(0.5), + expected=0.5, + ) +) ++ test.case.new( + name='parseCPU - string fraction', + test=test.expect.eq( + actual=units.parseKubernetesCPU('0.5'), + expected=0.5, + ) +) ++ test.case.new( + name='parseCPU - fraction millicores', + test=test.expect.eq( + actual=units.parseKubernetesCPU('500m'), + expected=0.5, + ) +) + +// siToBytes tests ++ test.case.new( + name='siToBytes - plain number', + test=test.expect.eq( + actual=units.siToBytes('1024'), + expected=1024, + ) +) ++ test.case.new( + name='siToBytes - Ki', + test=test.expect.eq( + actual=units.siToBytes('1Ki'), + expected=1024, + ) +) ++ test.case.new( + name='siToBytes - Mi', + test=test.expect.eq( + actual=units.siToBytes('1Mi'), + expected=1048576, + ) +) ++ test.case.new( + name='siToBytes - Gi', + test=test.expect.eq( + actual=units.siToBytes('2Gi'), + expected=2147483648, + ) +) ++ test.case.new( + name='siToBytes - Ti', + test=test.expect.eq( + actual=units.siToBytes('1Ti'), + expected=1099511627776, + ) +) ++ test.case.new( + name='siToBytes - fractional Ki', + test=test.expect.eq( + actual=units.siToBytes('1.5Ki'), + expected=1536, + ) +) + +// parseDuration tests ++ test.case.new( + name='parseDuration - milliseconds', + test=test.expect.eq( + actual=units.parseDuration('500ms'), + expected=0.5, + ) +) ++ test.case.new( + name='parseDuration - seconds', + test=test.expect.eq( + actual=units.parseDuration('30s'), + expected=30, + ) +) ++ test.case.new( + name='parseDuration - minutes', + test=test.expect.eq( + actual=units.parseDuration('5m'), + expected=300, + ) +) ++ test.case.new( + name='parseDuration - hours', + test=test.expect.eq( + actual=units.parseDuration('2h'), + expected=7200, + ) +) ++ test.case.new( + name='parseDuration - combined minutes and seconds', + test=test.expect.eq( + actual=units.parseDuration('4m30s'), + expected=270, + ) +) ++ test.case.new( + name='parseDuration - combined hours and fractional seconds', + test=test.expect.eq( + actual=units.parseDuration('1h30.5s'), + expected=3600 + 30.5, + ) +) ++ test.case.new( + name='parseDuration - combined hours and minutes', + test=test.expect.eq( + actual=units.parseDuration('1h30m'), + expected=5400, + ) +) ++ test.case.new( + name='parseDuration - combined hours and milliseconds', + test=test.expect.eq( + actual=units.parseDuration('1h500ms'), + expected=3600 + 0.5, + ) +) + +// formatDuration tests ++ test.case.new( + name='formatDuration - milliseconds', + test=test.expect.eq( + actual=units.formatDuration(0.5), + expected='500ms', + ) +) ++ test.case.new( + name='formatDuration - seconds', + test=test.expect.eq( + actual=units.formatDuration(45), + expected='45s', + ) +) ++ test.case.new( + name='formatDuration - minutes', + test=test.expect.eq( + actual=units.formatDuration(300), + expected='5m', + ) +) ++ test.case.new( + name='formatDuration - hours', + test=test.expect.eq( + actual=units.formatDuration(7200), + expected='2h', + ) +) ++ test.case.new( + name='formatDuration - combined minutes and seconds', + test=test.expect.eq( + actual=units.formatDuration(270), + expected='4m30s', + ) +) ++ test.case.new( + name='formatDuration - combined hours and minutes', + test=test.expect.eq( + actual=units.formatDuration(5400), + expected='1h30m', + ) +) ++ test.case.new( + name='formatDuration - combined hours and minutes and seconds', + test=test.expect.eq( + actual=units.formatDuration(5400 + 30), + expected='1h30m30s', + ) +) ++ test.case.new( + name='formatDuration - combined hours and minutes and seconds and milliseconds', + test=test.expect.eq( + actual=units.formatDuration(5400 + 30 + 0.5), + expected='1h30m30s500ms', + ) +) + ++ test.case.new( + name='formatDuration - combined hours and milliseconds', + test=test.expect.eq( + actual=units.formatDuration(3600 + 0.5), + expected='1h500ms', + ) +) diff --git a/jsonnet/vendor/github.com/jsonnet-libs/xtd/units.libsonnet b/jsonnet/vendor/github.com/jsonnet-libs/xtd/units.libsonnet new file mode 100644 index 0000000000..9d3bdcab28 --- /dev/null +++ b/jsonnet/vendor/github.com/jsonnet-libs/xtd/units.libsonnet @@ -0,0 +1,93 @@ +local d = import 'github.com/jsonnet-libs/docsonnet/doc-util/main.libsonnet'; + +{ + '#': d.pkg( + name='units', + url='github.com/jsonnet-libs/xtd/units.libsonnet', + help='`units` implements helper functions for converting units.', + ), + + '#parseKubernetesCPU':: d.fn( + ||| + `parseKubernetesCPU` parses a Kubernetes CPU string/number into a number of cores. + The function assumes the input is in a correct Kubernetes format, i.e., an integer, a float, + a string representation of an integer or a float, or a string containing a number ending with 'm' + representing a number of millicores. + |||, + [ + d.arg('input', d.T.any), + ] + ), + parseKubernetesCPU(input): + if std.isString(input) && std.endsWith(input, 'm') then std.parseJson(std.rstripChars(input, 'm')) / 1000 + else if std.isString(input) then std.parseJson(input) + else if std.isNumber(input) then input + else 0, + + '#siToBytes':: d.fn( + ||| + `siToBytes` converts Kubernetes byte units to bytes. + Only works for limited set of SI prefixes: Ki, Mi, Gi, Ti. + |||, + [ + d.arg('str', d.T.string), + ] + ), + siToBytes(str): + local siToBytesDecimal(str) = + if std.endsWith(str, 'Ki') then + std.parseJson(std.rstripChars(str, 'Ki')) * std.pow(2, 10) + else if std.endsWith(str, 'Mi') then + std.parseJson(std.rstripChars(str, 'Mi')) * std.pow(2, 20) + else if std.endsWith(str, 'Gi') then + std.parseJson(std.rstripChars(str, 'Gi')) * std.pow(2, 30) + else if std.endsWith(str, 'Ti') then + std.parseJson(std.rstripChars(str, 'Ti')) * std.pow(2, 40) + else + std.parseJson(str); + std.floor(siToBytesDecimal(str)), + + '#parseDuration':: d.fn( + ||| + `parseDuration` parses a duration string and returns the number of seconds. + Handles milliseconds (ms), seconds (s), minutes (m), hours (h), and combined formats like "4m30s" or "1h30m". + |||, + [ + d.arg('duration', d.T.string), + ] + ), + parseDuration(duration): + local replaced = std.strReplace(duration, 'ms', 'n'); // replace ms with n to avoid confusion with minutes + + local splitH = std.split(replaced, 'h'); + local hours = if std.length(splitH) > 1 then std.parseJson(splitH[0]) else 0; + local withoutHours = if std.length(splitH) > 1 then splitH[1] else replaced; + + local splitM = std.split(withoutHours, 'm'); + local minutes = if std.length(splitM) > 1 then std.parseJson(splitM[0]) else 0; + local withoutMinutes = if std.length(splitM) > 1 then splitM[1] else withoutHours; + + local splitS = std.split(withoutMinutes, 's'); + local seconds = if std.length(splitS) > 1 then std.parseJson(splitS[0]) else 0; + local withoutSeconds = if std.length(splitS) > 1 then splitS[1] else withoutMinutes; + + local splitN = std.split(withoutSeconds, 'n'); + local milliseconds = if std.length(splitN) > 1 then std.parseJson(splitN[0]) else 0; + + hours * 3600 + minutes * 60 + seconds + milliseconds / 1000, + + '#formatDuration':: d.fn( + ||| + `formatDuration` formats a number of seconds into a human-readable duration string. + Returns the duration in the smallest appropriate unit (s, m, h, or combined formats like "4m30s"). + |||, + [ + d.arg('seconds', d.T.number), + ] + ), + formatDuration(seconds): + (if std.floor(seconds / 3600) > 0 then '%dh' % (seconds / 3600) else '') + + (if std.floor((seconds % 3600) / 60) > 0 then '%dm' % ((seconds % 3600) / 60) else '') + + (if (seconds % 60) > 1 then '%ds' % (seconds % 60) else '') + + (if seconds % 1 > 0 then '%dms' % ((seconds % 1) * 1000) else ''), +} diff --git a/manifests/0000_90_etcd-operator_03_prometheusrule.yaml b/manifests/0000_90_etcd-operator_03_prometheusrule.yaml index 1db3073044..5156f7b2ae 100644 --- a/manifests/0000_90_etcd-operator_03_prometheusrule.yaml +++ b/manifests/0000_90_etcd-operator_03_prometheusrule.yaml @@ -109,19 +109,22 @@ spec: for: 10m labels: severity: critical - - alert: etcdGRPCRequestsSlow + - alert: etcdGRPCReadRequestsSlow annotations: - description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.' - runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdGRPCRequestsSlow.md - summary: etcd grpc requests are slow - expr: | - histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method!="Defragment", grpc_type="unary"}[10m])) without(grpc_type)) - > on () group_left (type) - bottomk(1, - 1.5 * group by (type) (cluster_infrastructure_provider{type="Azure"}) - or - 1 * group by (type) (cluster_infrastructure_provider)) - for: 30m + description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC read requests is {{ $value }}s on etcd instance {{ $labels.instance }}.' + runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdGRPCReadRequestsSlow.md + summary: etcd grpc read requests are slow + expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method="Range", grpc_type="unary"}[10m])) without(grpc_type)) > 3 + for: 10m + labels: + severity: critical + - alert: etcdGRPCWriteRequestsSlow + annotations: + description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC write requests is {{ $value }}s on etcd instance {{ $labels.instance }}.' + runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdGRPCWriteRequestsSlow.md + summary: etcd grpc write requests are slow + expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd", grpc_method="Txn", grpc_type="unary"}[10m])) without(grpc_type)) > 5 + for: 10m labels: severity: critical - alert: etcdHighCommitDurations