diff --git a/beam/BEAM-ErlangVM-DeepDive.json b/beam/BEAM-ErlangVM-DeepDive.json new file mode 100644 index 00000000..ce0434be --- /dev/null +++ b/beam/BEAM-ErlangVM-DeepDive.json @@ -0,0 +1,9291 @@ +{ + "description": "Per-subsystem breakdown of BEAM runtime internals: process/port/atom saturation detail, memory segments, ETS, MSACC scheduler microstate, GC, I/O, memory allocator fragmentation, Erlang distribution link health, and Mnesia. Generic for any Elixir/Erlang project using prometheus_ex.", + "image": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTgiIHZpZXdCb3g9IjAgMCAxOCAxOCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0Ljk0OTQgMTMuOTQyQzE2LjIzMTggMTIuNDI1OCAxNy4zMjY4IDkuNzAyMiAxNi4xOTU2IDYuNTc0ODdDMTUuNjQ0MyA1LjA1MjQ1IDE1LjAyMTkgNC4yMDI0OSAxNC4yOTY5IDMuNjYyNTJDMTMuODU1NyAzLjMzMzc5IDEyLjA5MzMgMi41MDYzMyA5Ljc1OTY1IDIuODY3NTZDOC4wNTM0OSAzLjEzMjU1IDUuNzc0ODcgNC4yMDg3NCA0LjI5MzY5IDUuOTU5OUMyLjg1NzUyIDcuNjYxMDYgMS43NDg4MyA5LjAwNDc0IDEuNjk3NTggMTAuMzA5N0MxLjYzMTMzIDExLjk4ODMgMi44OTYyNyAxMy40MzA4IDMuMDUwMDEgMTMuNjY0NUMzLjMyMzc0IDE0LjA3OTUgNS4xOTExNSAxNi40NTE4IDguNjk5NzEgMTYuNTczMUMxMS43OTcgMTYuNjc5MyAxMy44MTQ0IDE1LjI4NDQgMTQuOTQ5NCAxMy45NDJaIiBmaWxsPSIjNDAzRDNFIi8+CjxwYXRoIGQ9Ik00LjU1MzYzIDIuNzM3NDdDMi45Mzc0NiAzLjg5MTE2IDEuMTIxMzEgNi4yNTEwMyAxLjQ0NzU0IDkuNTYwODZDMS42MDYyOCAxMS4xNzIgMi4wMDI1MSAxMi4xNDk1IDIuNTcxMjMgMTIuODUwN0MyLjkxNzQ2IDEzLjI3ODIgNC40MTk4OCAxNC41NDkzIDYuNzczNTEgMTQuNzM2OEM5LjE0NTg4IDE0LjkyNTYgMTAuOTQ5NSAxNC4zOTQ0IDEyLjgzMzIgMTMuMDg0NEMxNi42NjE3IDEwLjQyMDggMTYuMDk4IDYuMzkzNTMgMTUuOTM0MyA1LjkyNDhDMTUuNzcwNSA1LjQ1NjA3IDE0LjU0NDQgMi42OTYyMiAxMS4xNzMzIDEuNzE1MDJDOC4xOTg0NCAwLjg1MDA2OCA1Ljk4MzU1IDEuNzE1MDIgNC41NTM2MyAyLjczNzQ3WiIgZmlsbD0iIzVFNjM2NyIvPgo8cGF0aCBkPSJNNy4zOTM1MyAyLjk2MTA5QzUuNjE3MzcgMi44OTczNCAzLjkxOTk2IDQuMjg4NTIgMy43NTYyMiA2LjAwNTkzQzMuNTkyNDggNy43MjIwOSA0LjY1NDkyIDkuMDI5NTIgNi4zMDk4MyA5LjI5NTc2QzcuOTY0NzUgOS41NjA3NCA5Ljg3ODM5IDguNTU1OCAxMC4yNjM0IDYuNDUwOTFDMTAuNjYwOSA0LjI4MjI3IDkuMDg5NjkgMy4wMjIzNCA3LjM5MzUzIDIuOTYxMDlaIiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBkPSJNNy45NDIxNyA1LjkwMTE1QzcuOTQyMTcgNS45MDExNSA4LjM2OTY1IDUuODEyNCA4LjQ1NDY1IDUuMTgyNDRDOC41MzgzOSA0LjU2MjQ3IDguMjMwOTEgNC4wMzM3NSA3LjUxMzQ1IDMuODQzNzZDNi43MzM0OSAzLjYzNzUyIDYuMjA0NzcgNC4wNjYyNSA2LjA2NzI3IDQuNTE3NDdDNS44NzYwMyA1LjE0NDk0IDYuMTU4NTIgNS40NDM2NyA2LjE1ODUyIDUuNDQzNjdDNi4xNTg1MiA1LjQ0MzY3IDUuMzkzNTYgNS42Mjc0MSA1LjMzMjMxIDYuNTI5ODdDNS4yNzQ4MSA3LjM4MTA3IDUuODU2MDMgNy44Mzg1NSA2LjQzOTc1IDcuOTc4NTRDNy4xNjA5NiA4LjE1MjI4IDcuOTc4NDIgNy45NTQ3OSA4LjE3ODQxIDcuMDM0ODRDOC4zNDQ2NSA2LjI3NzM4IDcuOTQyMTcgNS45MDExNSA3Ljk0MjE3IDUuOTAxMTVaIiBmaWxsPSIjMzAzMDMwIi8+CjxwYXRoIGQ9Ik02LjczOTgzIDQuNzUzNjJDNi42NzEwOSA1LjAxMjM1IDYuODA4NTggNS4yNjIzNCA3LjA3ODU3IDUuMzMxMDlDNy4zNjk4IDUuNDA0ODMgNy42MzQ3OSA1LjMwODU5IDcuNzA2MDMgNS4wMTExQzcuNzY4NTMgNC43NDczNyA3LjY0MzU0IDQuNTE0ODggNy4zMzYwNSA0LjQzOTg4QzcuMDgzNTcgNC4zNzczOSA2LjgxNDgzIDQuNDcxMTMgNi43Mzk4MyA0Ljc1MzYyWiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTYuOTU5NzggNi4wMzk3NEM2LjYzMjMgNS45Mzg0OSA2LjE5OTgyIDYuMDY0NzMgNi4xMzEwNyA2LjUwNDcxQzYuMDYyMzMgNi45NDQ2OSA2LjMyNjA2IDcuMTY5NjggNi42NzEwNCA3LjIzMjE3QzcuMDE2MDMgNy4yOTQ2NyA3LjM0MjI2IDcuMTEzNDMgNy40MDYwMSA2Ljc2MDk1QzcuNDY4NSA2LjQwOTcyIDcuMjg2MDEgNi4xMzk3MyA2Ljk1OTc4IDYuMDM5NzRaIiBmaWxsPSJ3aGl0ZSIvPgo8L3N2Zz4K", + "layout": [ + { + "h": 1, + "i": "r1", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 0 + }, + { + "h": 7, + "i": "39eb4c4f-5129-4a3c-92fd-8786274b1159", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 1 + }, + { + "h": 7, + "i": "w2", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 1 + }, + { + "h": 7, + "i": "w4", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 1 + }, + { + "h": 7, + "i": "w5", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 8 + }, + { + "h": 7, + "i": "w1", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 8 + }, + { + "h": 7, + "i": "w3", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 8 + }, + { + "h": 1, + "i": "r2", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 29 + }, + { + "h": 7, + "i": "w6", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 72 + }, + { + "h": 7, + "i": "w7", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 72 + }, + { + "h": 7, + "i": "w8", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 72 + }, + { + "h": 7, + "i": "w9", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 79 + }, + { + "h": 7, + "i": "w10", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 79 + }, + { + "h": 7, + "i": "w11", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 79 + }, + { + "h": 1, + "i": "r3", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 86 + }, + { + "h": 7, + "i": "w14", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 143 + }, + { + "h": 7, + "i": "w12", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 143 + }, + { + "h": 7, + "i": "w13", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 143 + }, + { + "h": 7, + "i": "w16", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 150 + }, + { + "h": 1, + "i": "r4", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 227 + }, + { + "h": 1, + "i": "r5", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 228 + }, + { + "h": 1, + "i": "r6", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 250 + }, + { + "h": 1, + "i": "r7", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 265 + }, + { + "h": 1, + "i": "r8", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 273 + }, + { + "h": 1, + "i": "r9", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 316 + }, + { + "h": 1, + "i": "r10", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 352 + } + ], + "name": "BEAM / Erlang VM - Deep Dive", + "panelMap": { + "r1": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "39eb4c4f-5129-4a3c-92fd-8786274b1159", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 1 + }, + { + "h": 7, + "i": "w2", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 1 + }, + { + "h": 7, + "i": "w4", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 1 + }, + { + "h": 7, + "i": "w5", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 8 + }, + { + "h": 7, + "i": "w1", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 8 + }, + { + "h": 7, + "i": "w3", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 8 + } + ] + }, + "r10": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w51", + "moved": false, + "static": false, + "w": 3, + "x": 0, + "y": 353 + }, + { + "h": 7, + "i": "w52", + "moved": false, + "static": false, + "w": 3, + "x": 3, + "y": 353 + }, + { + "h": 7, + "i": "w53", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 353 + }, + { + "h": 7, + "i": "w54", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 360 + }, + { + "h": 7, + "i": "w55", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 360 + }, + { + "h": 7, + "i": "w56", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 367 + }, + { + "h": 7, + "i": "w57", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 367 + }, + { + "h": 7, + "i": "w58", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 367 + } + ] + }, + "r2": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "w6", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 72 + }, + { + "h": 7, + "i": "w7", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 72 + }, + { + "h": 7, + "i": "w8", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 72 + }, + { + "h": 7, + "i": "w9", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 79 + }, + { + "h": 7, + "i": "w10", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 79 + }, + { + "h": 7, + "i": "w11", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 79 + } + ] + }, + "r3": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "w14", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 143 + }, + { + "h": 7, + "i": "w12", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 143 + }, + { + "h": 7, + "i": "w13", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 143 + }, + { + "h": 7, + "i": "w16", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 150 + } + ] + }, + "r4": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w17", + "moved": false, + "static": false, + "w": 3, + "x": 0, + "y": 298 + }, + { + "h": 7, + "i": "w18", + "moved": false, + "static": false, + "w": 3, + "x": 3, + "y": 298 + }, + { + "h": 7, + "i": "w19", + "moved": false, + "static": false, + "w": 3, + "x": 6, + "y": 298 + }, + { + "h": 7, + "i": "w20", + "moved": false, + "static": false, + "w": 3, + "x": 9, + "y": 298 + } + ] + }, + "r5": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w21", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 229 + }, + { + "h": 7, + "i": "w22", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 229 + }, + { + "h": 7, + "i": "w23", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 229 + }, + { + "h": 7, + "i": "w24", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 236 + }, + { + "h": 7, + "i": "w25", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 236 + }, + { + "h": 7, + "i": "w26", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 236 + }, + { + "h": 7, + "i": "w27", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 243 + } + ] + }, + "r6": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w28", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 251 + }, + { + "h": 7, + "i": "w29", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 251 + }, + { + "h": 7, + "i": "w30", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 251 + }, + { + "h": 7, + "i": "w31", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 258 + }, + { + "h": 7, + "i": "w32", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 258 + } + ] + }, + "r7": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w33", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 266 + }, + { + "h": 7, + "i": "w34", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 266 + } + ] + }, + "r8": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w35", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 274 + }, + { + "h": 7, + "i": "w36", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 274 + }, + { + "h": 7, + "i": "w37", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 281 + }, + { + "h": 7, + "i": "w38", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 281 + }, + { + "h": 7, + "i": "w39", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 281 + } + ] + }, + "r9": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w40", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 317 + }, + { + "h": 7, + "i": "w41", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 317 + }, + { + "h": 7, + "i": "w42", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 324 + }, + { + "h": 7, + "i": "w43", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 324 + }, + { + "h": 7, + "i": "w44", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 331 + }, + { + "h": 7, + "i": "w45", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 331 + }, + { + "h": 7, + "i": "w46", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 338 + }, + { + "h": 7, + "i": "w47", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 338 + }, + { + "h": 7, + "i": "w48", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 345 + }, + { + "h": 7, + "i": "w49", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 345 + }, + { + "h": 7, + "i": "w50", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 345 + } + ] + } + }, + "tags": [ + "beam", + "erlang", + "elixir", + "debug" + ], + "title": "BEAM / Erlang VM - Deep Dive", + "uploadedGrafana": false, + "variables": { + "environment": { + "allSelected": true, + "description": "Deployment environment filter", + "dynamicVariablesAttribute": "deployment.environment", + "dynamicVariablesSource": "Metrics", + "id": "var-environment", + "multiSelect": true, + "name": "environment", + "order": 0, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + }, + "k8s_cluster": { + "allSelected": true, + "description": "Kubernetes cluster name filter", + "dynamicVariablesAttribute": "k8s.cluster.name", + "dynamicVariablesSource": "Metrics", + "id": "var-k8s-cluster", + "multiSelect": true, + "name": "k8s_cluster", + "order": 1, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + }, + "k8s_pod": { + "allSelected": true, + "description": "Pod name filter — select specific pods to scope all panels to one application", + "dynamicVariablesAttribute": "k8s.pod.name", + "dynamicVariablesSource": "Metrics", + "id": "var-k8s-pod", + "multiSelect": true, + "name": "k8s_pod", + "order": 2, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + } + }, + "version": "v5", + "widgets": [ + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r1", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Process, Port & Atom Saturation" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Live processes (solid) vs hard limit (dashed). Gap = headroom. Shrinking gap = leak or load spike.", + "id": "w1", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Process Count", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_process_limit", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Process Limit", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "d745aedc-ca2a-480e-b538-2782ea4fbbb9", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Process Count vs Limit", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": ">85% is critical — BEAM refuses to spawn new processes at the limit.", + "fillMode": "none", + "fillSpans": false, + "id": "w2", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_process_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "d9b4c6b7-a492-42d0-a64d-55a3ec119ab2", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Process Utilization %", + "yAxisUnit": "percent" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "Ports represent open sockets, files, and NIF handles. Exhaustion causes connection failures.", + "fillMode": "none", + "fillSpans": false, + "id": "w3", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_port_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": { + "expression": "" + }, + "legend": "Port Count", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_port_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": false, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": { + "expression": "" + }, + "legend": "Port Limit", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "ea4de080-95d9-46d5-bd1a-51c637c13eee", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Port Count vs Limit", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "Port exhaustion does not crash the node but causes {error, system_limit} on new connections.", + "fillMode": "none", + "fillSpans": false, + "id": "w4", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_port_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_port_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "a2163961-8c39-4672-8e29-996dbb26665a", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Port Utilization %", + "yAxisUnit": "percent" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Atom table is global and never shrinks. Monotonic growth without restart = atom leak (dynamic atom creation).", + "id": "w5", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_count", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Atom Count", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_limit", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Atom Limit", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "cdb7c126-a228-4297-888c-4d5dfc65a4cc", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Atom Count vs Limit", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r2", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Memory Detail" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Memory allocated to all Erlang process stacks and heaps. Growing without a matching process count increase = per-process memory leak.", + "id": "w6", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_processes_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Process Heap", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "dafc56fc-e40c-4ed7-ae40-3ab34b1464ec", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Process Heap Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "BEAM VM system memory: ETS, code, stack, and VM overhead. Excludes process heap.", + "id": "w7", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_system_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "System", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c04b416e-405d-4773-84fd-a6cfbdd5072d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "System Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Memory consumed by the atom table. Correlated with atom count.", + "id": "w8", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_atom_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Atom Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "93a93556-2769-4387-8302-6119c7b5ff40", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Atom Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Off-heap reference-counted binaries. Steady monotonic growth between GCs = binary leak (processes holding large binary references).", + "id": "w9", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "beam_memory_binary_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Binary Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "6d8ca821-9676-4b7c-bfc8-56afaca9d9ef", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Binary Memory (off-heap)", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total memory used by all ETS tables. Correlates with table count and row density.", + "id": "w10", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "beam_memory_ets_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "ETS Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "2184b14f-5e55-4a8f-af90-3451a7583b4b", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "ETS Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Memory occupied by loaded BEAM modules. Significant spikes after hot upgrades or repeated code:load calls.", + "id": "w11", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "beam_memory_code_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Code Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "80db8391-3a5f-47b4-9ada-912df39e90d4", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Code Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r3", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "ETS Tables" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total open ETS tables. Steady growth without a corresponding application load increase = table leak (missing :ets.delete).", + "id": "w12", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_ets_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "ETS Tables", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "9397af8e-b8a7-4cd9-90c0-0bddb580126d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "ETS Table Count", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Hard ceiling for ETS tables (default 8192). Reference line — compare against ETS Count.", + "id": "w13", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_ets_limit", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "ETS Limit", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c0295751-2141-489e-8cdb-977960827e95", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "ETS Table Limit", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "ETS tables used vs limit. >80% warrants investigation into table ownership and lifecycle.", + "fillMode": "none", + "fillSpans": false, + "id": "w14", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_ets_tables", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_ets_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "9b50b63a-c493-427c-bdec-ba3129d7dac1", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "ETS Utilization %", + "yAxisUnit": "percent" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Disk-backed ETS tables. Normally 0 in most apps. Unexpected growth = open disk tables not being closed.", + "id": "w16", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_dets_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "DETS Tables", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "4d1ad25d-fe40-474f-982f-10e8f188388a", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "DETS (Disk ETS) Table Count", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r4", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "MSACC Scheduler Microstate" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Fraction of scheduler time each MSACC state consumes. 'aux' + 'check_io' + 'emulator' should dominate; high 'sleep' = idle schedulers; high 'gc' = GC pressure.", + "id": "w17", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_aux_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "aux", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_check_io_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "check_io", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_emulator_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "emulator", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_gc_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "D", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "gc", + "orderBy": [], + "queryName": "D", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "1c540b59-0c75-4c3d-ae9b-2d49a6f2ea22", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MSACC: aux / check_io / emulator / gc", + "yAxisUnit": "s" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Scheduler time in port / nif / bif / send. Elevated nif = heavy NIF calls blocking schedulers; elevated send = message-passing saturation.", + "id": "w18", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_port_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "port", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_nif_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "nif", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_bif_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "bif", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_send_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "D", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "send", + "orderBy": [], + "queryName": "D", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "5dad1b84-abbb-4efb-98d3-3c23e3f97fb6", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MSACC: port / nif / bif / send", + "yAxisUnit": "s" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Scheduler time asleep. High sleep = under-utilised schedulers (normal during low load) or blocked waiting on I/O.", + "id": "w19", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_sleep_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "sleep", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c77bc2c8-af64-47ec-bc27-0f12bdb7b00c", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MSACC: sleep", + "yAxisUnit": "s" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Scheduler time in alloc / other / timers. Elevated alloc = heavy allocation pressure; elevated timers = many active timer references.", + "id": "w20", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_alloc_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "alloc", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_other_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "other", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_timers_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "timers", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "e25c0a42-7f8d-4048-b5f1-e78c17b61c3c", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MSACC: alloc / other / timers", + "yAxisUnit": "s" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r5", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "GC & I/O" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "GC runs per second. Short spikes are normal; sustained high rates with rising heap = object churn.", + "id": "w21", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_number_of_gcs", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "GC Count Rate", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "0a8cc42d-7082-4ee3-b35e-7a62f9a0e32b", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Count Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Bytes reclaimed per second by GC. High reclaim with high GC count = frequent short-lived allocation churn.", + "id": "w22", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_bytes_reclaimed", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Bytes Reclaimed/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "5e52a97e-1c5c-4b7b-a1c5-17d26640dbd5", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Bytes Reclaimed Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Words moved across generations per GC cycle. Large word counts = major GC activity; correlate with pauses.", + "id": "w23", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_words_reclaimed", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Words Reclaimed/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "4db37a6d-e10e-40ba-b9a8-6ea32c9c78e5", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Words Reclaimed Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Raw bytes received by all ports (sockets, pipes). Sustained high read throughput is normal; sudden drops = upstream pressure.", + "id": "w24", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_bytes_received_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Bytes In/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "3b4de0fd-f6d1-42be-9f36-2e91b7bde0d0", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "I/O Bytes In Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Raw bytes sent by all ports. Sustained high write throughput is normal; compare with in-rate for asymmetry.", + "id": "w25", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_bytes_output_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Bytes Out/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "5de6b8d3-a5bd-4b1f-8ee5-fd08b64aa9a7", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "I/O Bytes Out Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total context switches across all schedulers per second. Sudden step-changes correlate with process spawning bursts or supervisor restarts.", + "id": "w26", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_context_switches", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Context Switches/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "b2da73e4-0a0e-4ead-8a03-e1c10b4fa218", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Context Switches Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total reductions (function calls + tail calls) per second. Steady baseline; spikes indicate CPU-intensive work or hot loops.", + "id": "w27", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_reductions_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Reductions/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "aa98a5ca-e975-48e4-9ce8-c6ea2f524e79", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Reductions Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r6", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Memory Allocator Fragmentation" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Bytes allocated by the BEAM allocator vs bytes actually used by live data. Gap = fragmentation overhead.", + "id": "w28", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_pool_blocks_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} pool_blocks_size", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "6bd1e624-1f66-4dc5-88c9-b0893b6c0bde", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Allocator MBCS Pool Block Size", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Number of multi-block carriers in the pool. Growing without load growth = fragmented carrier accumulation.", + "id": "w29", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_pool_blocks", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} pool_blocks", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "a9a19bb3-5069-441b-aece-5a3199b48720", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Allocator MBCS Pool Block Count", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Single-block carriers created by the allocator. SBC growth = large allocations exceeding MBC threshold; consider +MBcst tuning.", + "id": "w30", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_sbcs_blocks_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} sbc_blocks_size", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c44f2174-b40b-4b00-ba97-b3fe55f9a1a1", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Allocator SBC Block Size", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Fragmentation ratio = (allocated − used) / allocated. Values >40% sustained = fragmented memory; consider allocator flags or node restart.", + "id": "w31", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_block_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_carriers_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "expression": "(B - A) / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "id": "4e48e5b5-6e2f-489e-a2e7-72c76b90ae74", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 100, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MBC Fragmentation %", + "yAxisUnit": "percent" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Allocated carrier size (total reserved) vs block size (live data). Wide gap = wasted carrier memory.", + "id": "w32", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_block_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} block", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_carriers_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} carrier", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "b39cc3d5-6acb-4a8e-883d-84be6c7f6462", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MBC Block vs Carrier Size", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r7", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Erlang Distribution Link" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Distribution bytes sent per second to remote nodes. Sudden drop = node disconnect or network partition.", + "id": "w33", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_send_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Bytes Sent/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "d96a4d31-80fe-415c-b2c7-a4fc1c8e86e6", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Bytes Sent Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Distribution bytes received per second from remote nodes. Spikes = large remote message deliveries or data transfers.", + "id": "w34", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_recv_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Bytes Recv/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "4e22b2db-83b8-4ad2-8a38-ef81f89c4eb0", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Bytes Received Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r8", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Distribution Messages" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Messages sent across distribution links per second. High rate + growing queues = dist buffer saturation risk.", + "id": "w35", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_send_messages_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Messages Sent/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "f8f4c226-0c52-4d8f-8eae-f5c41a8e5e3d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Messages Sent Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Messages received over distribution links per second. Asymmetry with sent rate = one-way traffic imbalance.", + "id": "w36", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_recv_messages_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Messages Recv/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c17ca374-f79d-4eac-9770-4dbf19da4c37", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Messages Received Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Pending bytes in the distribution send buffer. Sustained non-zero values = congested distribution link.", + "id": "w37", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_send_pending_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Pending Send Bytes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "8e73f8ea-c6a9-4bef-bd86-adde23c8e4ca", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Send Pending", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Packets sent across distribution links. Low-frequency, high-byte packets = large message transfers.", + "id": "w38", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_send_packets_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Packets Sent/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "1e4d4d9f-b8a0-4b12-af6f-13c4cd9e0f4d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Packets Sent Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Packets received over distribution links. Compare with sent rate and bytes for flow analysis.", + "id": "w39", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_recv_packets_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Packets Recv/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c01c7098-49c3-4dd4-834e-e7ed9f97e8a0", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Packets Received Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r9", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Mnesia" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Active Mnesia transactions. Sustained high counts = long-running transactions blocking table locks.", + "id": "w40", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_held_locks", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Held Locks", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "d36ff9fa-7e51-492e-aa81-61f5a7c5f7e3", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Held Locks", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Processes waiting to acquire Mnesia locks. Non-zero = lock contention; high values = potential deadlock or slow transactions.", + "id": "w41", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_lock_queue", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Lock Queue", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "2ec3b61e-2065-4eef-a6d3-dc9219ef6a05", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Lock Queue Depth", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia transaction restarts due to lock conflicts. Elevated restart rate = high write contention; consider table sharding or optimistic locking.", + "id": "w42", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_restarts", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Transaction Restarts/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "e7f3c17a-59e4-4e54-9e22-71b6d4c4d25a", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Transaction Restart Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia transactions aborted per second. Aborts can be caused by conflicts, node failures, or application errors.", + "id": "w43", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_aborts", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Transaction Aborts/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "bab96b77-1e6e-4fbb-a7e7-61f88a5cd07f", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Transaction Abort Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia transactions committed per second. Baseline for throughput; compare with abort and restart rates for health ratio.", + "id": "w44", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_commits", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Transaction Commits/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "addd1a97-d7dc-42e8-9424-2cc26789d04d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Transaction Commit Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia log entries written per second. Disk-based tables only. High log rate + low commit rate = replay overhead.", + "id": "w45", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_logged_transactions", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Logged Transactions/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "deea3f37-9c5b-4d99-82b4-95e285c1d47f", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Logged Transactions Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Failed Mnesia writes. Any non-zero value = data integrity issue or schema mismatch.", + "id": "w46", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_failed_transactions", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Failed Transactions/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "66cc4fb5-cd2a-4f42-aba6-a6c0e6c15e95", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Failed Transaction Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables currently open. Steady count = stable schema; sudden drop = node partition or table deletion.", + "id": "w47", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_memory_usage_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Memory Usage", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "0e88bf72-e297-424b-b4a4-3aef8bd5e5e5", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Memory Usage", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables with unsynchronised data across nodes. Non-zero = split-brain risk. Should return to zero after healing.", + "id": "w48", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_unsynchronised_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Unsynchronised Tables", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "43c0c24d-1ecf-4af7-83e4-b8c4d7f3a3e3", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Unsynchronised Tables", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Subscribed Mnesia event listeners. Unexpected drop = subscriber process crashed.", + "id": "w49", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_subscriptions", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Subscriptions", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "0d9dc6bf-9b77-4b5a-a5bf-8975c38fe9d1", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Subscriptions", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Number of connected Mnesia cluster nodes. Drop below expected count = node departure or partition.", + "id": "w50", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_db_nodes_running", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Running DB Nodes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "bb0d8b63-fa29-4f70-a6a6-7f64fc46dca9", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Running DB Nodes", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r10", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Mnesia Extended" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables registered in the schema. Sudden change = schema migration or table deletion.", + "id": "w51", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_db_nodes_stopped", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Stopped DB Nodes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "8b57d1ef-ef46-4c5f-b8ef-73f52f3499e4", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Stopped DB Nodes", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Table copies held by this node. Reduction = replica departure or table deletion.", + "id": "w52", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_held_locks", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Held Locks (detail)", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "b4fad1ac-d9ba-4668-a81e-26a36c2bbf74", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Held Locks (detail)", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia event subscribers. Each type of activity subscription appears as a separate line.", + "id": "w53", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_subscriptions", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "type--string--tag", + "isColumn": false, + "key": "type", + "type": "tag" + } + ], + "legend": "{{type}} subscriptions", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "28e2e9c4-c5aa-472e-850e-ee85e8e22b49", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Subscriptions by Type", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia commit rate vs abort rate vs restart rate. High abort-to-commit ratio = significant write contention.", + "id": "w54", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_commits", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Commits/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_aborts", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Aborts/s", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_restarts", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Restarts/s", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "1f5e5c8e-fbd2-4b9c-802e-b73b8ba56f19", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Transaction Health Overview", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Memory consumed by Mnesia data vs process heap vs system. Helps identify if Mnesia dominates memory.", + "id": "w55", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_memory_usage_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Mnesia Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_processes_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Process Heap", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_system_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "System", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "7f15e2c5-07ab-49b8-a1ee-0d8edfd06fc5", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia vs VM Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables as currently reported. Any drops = partition or node failure.", + "id": "w56", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_db_nodes_running", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Running Nodes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_mnesia_db_nodes_stopped", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Stopped Nodes", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "e1a7b8c9-d2e3-4f56-a789-b0c1d2e3f456", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Node Status", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables with unsynchronised copies (split-brain indicator) vs held locks (contention indicator) over time.", + "id": "w57", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_unsynchronised_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Unsync Tables", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_mnesia_held_locks", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Held Locks", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "a2b3c4d5-e6f7-4890-a1b2-c3d4e5f60123", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Unsync Tables vs Held Locks", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Active Mnesia transaction coordinators (dirty vs transactional). Dominance of dirty writes = performance but no isolation.", + "id": "w58", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_coordinators", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Transaction Coordinators", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "f1e2d3c4-b5a6-4789-9012-3456789abcde", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Active Transaction Coordinators", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "% of atom table used. Atom table never shrinks — runaway growth = leak.", + "fillMode": "none", + "fillSpans": false, + "id": "39eb4c4f-5129-4a3c-92fd-8786274b1159", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": { + "expression": "" + }, + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": { + "expression": "" + }, + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "d9b4c6b7-a492-42d0-a64d-55a3ec119ab2", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Atom Utilization %", + "yAxisUnit": "percent" + } + ] +} diff --git a/beam/BEAM-ErlangVM-Overview.json b/beam/BEAM-ErlangVM-Overview.json new file mode 100644 index 00000000..9f6b0522 --- /dev/null +++ b/beam/BEAM-ErlangVM-Overview.json @@ -0,0 +1,2981 @@ +{ + "description": "Key BEAM runtime health indicators. First stop for on-call triage — process/port saturation, memory pressure, scheduler backlog, GC activity. Generic for any Elixir/Erlang project using prometheus_ex.", + "layout": [ + { + "h": 3, + "i": "w1", + "moved": false, + "static": false, + "w": 2, + "x": 0, + "y": 0 + }, + { + "h": 3, + "i": "w2", + "moved": false, + "static": false, + "w": 2, + "x": 2, + "y": 0 + }, + { + "h": 3, + "i": "w3", + "moved": false, + "static": false, + "w": 2, + "x": 4, + "y": 0 + }, + { + "h": 3, + "i": "w4", + "moved": false, + "static": false, + "w": 2, + "x": 6, + "y": 0 + }, + { + "h": 3, + "i": "w5", + "moved": false, + "static": false, + "w": 2, + "x": 8, + "y": 0 + }, + { + "h": 3, + "i": "w6", + "moved": false, + "static": false, + "w": 2, + "x": 10, + "y": 0 + }, + { + "h": 1, + "i": "w7", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 3 + }, + { + "h": 7, + "i": "w8", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 4 + }, + { + "h": 7, + "i": "w9", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 4 + }, + { + "h": 7, + "i": "w10", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 4 + }, + { + "h": 1, + "i": "w11", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 11 + }, + { + "h": 7, + "i": "w12", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 47 + }, + { + "h": 7, + "i": "w13", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 47 + }, + { + "h": 7, + "i": "d0d18ace-cca9-4d5a-8440-86be0469022d", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 54 + }, + { + "h": 7, + "i": "w14", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 54 + }, + { + "h": 1, + "i": "w15", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 61 + } + ], + "panelMap": { + "w11": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "w12", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 47 + }, + { + "h": 7, + "i": "w13", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 47 + }, + { + "h": 7, + "i": "d0d18ace-cca9-4d5a-8440-86be0469022d", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 54 + }, + { + "h": 7, + "i": "w14", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 54 + } + ] + }, + "w15": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w16", + "moved": false, + "static": false, + "w": 3, + "x": 0, + "y": 209 + }, + { + "h": 7, + "i": "w17", + "moved": false, + "static": false, + "w": 3, + "x": 3, + "y": 209 + }, + { + "h": 7, + "i": "w18", + "moved": false, + "static": false, + "w": 3, + "x": 6, + "y": 209 + }, + { + "h": 7, + "i": "w19", + "moved": false, + "static": false, + "w": 3, + "x": 9, + "y": 209 + } + ] + }, + "w7": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "w8", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 4 + }, + { + "h": 7, + "i": "w9", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 4 + }, + { + "h": 7, + "i": "w10", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 4 + } + ] + } + }, + "tags": [ + "beam", + "erlang", + "elixir", + "overview" + ], + "title": "BEAM / Erlang VM - Overview", + "variables": { + "environment": { + "allSelected": true, + "description": "Deployment environment filter", + "dynamicVariablesAttribute": "deployment.environment", + "dynamicVariablesSource": "Metrics", + "id": "var-environment", + "multiSelect": true, + "name": "environment", + "order": 0, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + }, + "k8s_cluster": { + "allSelected": true, + "description": "Kubernetes cluster name filter", + "dynamicVariablesAttribute": "k8s.cluster.name", + "dynamicVariablesSource": "Metrics", + "id": "var-k8s-cluster", + "multiSelect": true, + "name": "k8s_cluster", + "order": 1, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + }, + "k8s_pod": { + "allSelected": true, + "description": "Pod name filter — select specific pods to scope all panels to one application", + "dynamicVariablesAttribute": "k8s.pod.name", + "dynamicVariablesSource": "Metrics", + "haveCustomValuesSelected": false, + "id": "var-k8s-pod", + "multiSelect": true, + "name": "k8s_pod", + "order": 2, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + } + }, + "version": "v5", + "widgets": [ + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "% of process limit in use. >85% is critical — node will crash at 100%.", + "fillMode": "none", + "fillSpans": false, + "id": "w1", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_process_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "c9912465-de59-4645-b89d-8c6625222c29", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 100, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [ + { + "index": "0", + "thresholdColor": "#FFA500", + "thresholdFormat": "Background", + "thresholdLabel": "Warning", + "thresholdOperator": ">", + "thresholdValue": 70 + }, + { + "index": "1", + "keyIndex": 1, + "thresholdColor": "#FF0000", + "thresholdFormat": "Background", + "thresholdLabel": "Critical", + "thresholdOperator": ">", + "thresholdValue": 85 + } + ], + "timePreferance": "GLOBAL_TIME", + "title": "Process Utilization", + "yAxisUnit": "percent" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "% of port limit in use. Ports = sockets, files, NIFs.", + "fillMode": "none", + "fillSpans": false, + "id": "w2", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_port_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_port_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "4e912e40-34ac-43e1-abec-0059a1d54bd2", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 100, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [ + { + "index": "0", + "thresholdColor": "#FFA500", + "thresholdFormat": "Background", + "thresholdLabel": "Warning", + "thresholdOperator": ">", + "thresholdValue": 70 + }, + { + "index": "1", + "keyIndex": 1, + "thresholdColor": "#FF0000", + "thresholdFormat": "Background", + "thresholdLabel": "Critical", + "thresholdOperator": ">", + "thresholdValue": 85 + } + ], + "timePreferance": "GLOBAL_TIME", + "title": "Port Utilization", + "yAxisUnit": "percent" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "% of atom table used. Atom table never shrinks — runaway growth = leak.", + "fillMode": "none", + "fillSpans": false, + "id": "w3", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "42aacb5f-0752-4d92-a370-412be4a007ec", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 100, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [ + { + "index": "0", + "thresholdColor": "#FFA500", + "thresholdFormat": "Background", + "thresholdLabel": "Warning", + "thresholdOperator": ">", + "thresholdValue": 70 + }, + { + "index": "1", + "keyIndex": 1, + "thresholdColor": "#FF0000", + "thresholdFormat": "Background", + "thresholdLabel": "Critical", + "thresholdOperator": ">", + "thresholdValue": 85 + } + ], + "timePreferance": "GLOBAL_TIME", + "title": "Atom Utilization", + "yAxisUnit": "percent" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Number of ETS tables currently open.", + "id": "w4", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_ets_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "31a03bbe-951d-4804-b963-6467fbf4cfa3", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "ETS Tables", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "CPU scheduler threads available to the BEAM.", + "id": "w5", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_schedulers_online", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "1818e786-b85d-486e-a178-2e74f3787812", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Schedulers Online", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total BEAM memory allocation (processes + system).", + "id": "w6", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "034d759b-9272-4b5d-acbc-a8ecbcca65da", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Total Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "w7", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Memory" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "w8", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Total", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "a9d086d8-f2fe-471e-acf9-4e98882a5639", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Total Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Stacked composition of BEAM memory — watch for any segment growing unbounded.", + "id": "w9", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_processes_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Processes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_system_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "System", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_atom_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Atom", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "beam_memory_binary_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "D", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Binary", + "orderBy": [], + "queryName": "D", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "beam_memory_ets_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "E", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "ETS", + "orderBy": [], + "queryName": "E", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "beam_memory_code_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "F", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Code", + "orderBy": [], + "queryName": "F", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "deeddc90-b07a-46cd-839d-f21ac488d28d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Memory Breakdown", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Steady monotonic growth indicates a binary leak (unreferenced binaries not GC'd).", + "id": "w10", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "beam_memory_binary_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Binary", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "718c7582-29a1-4453-b16e-8949bb23f0ff", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Binary Memory (off-heap)", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "w11", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Schedulers & Run Queues" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Normal scheduler run queue. Sustained > 0 = CPU saturation.", + "id": "w12", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_run_queues_length", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Run Queue", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "51a166ed-a243-48ac-8470-bda62852a44d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Run Queue Length", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Dirty CPU = long-running NIF pressure. Dirty IO = file/socket I/O pressure.", + "id": "w13", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_dirty_cpu_run_queue_length", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dirty CPU", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_dirty_io_run_queue_length", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dirty IO", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "9b84336d-4a49-4741-8415-f0234a989666", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Dirty CPU / IO Run Queues", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total Erlang processes. Runaway growth = process leak (missing termination or gen_server loop).", + "id": "w14", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Processes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "da910f90-465d-4865-b6cb-02a7e33de9d9", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Live Processes", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "w15", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "GC & Reductions" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Garbage collections per second. Spikes indicate sudden allocation pressure.", + "id": "w16", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_number_of_gcs", + "spaceAggregation": "sum", + "temporality": "cumulative", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "GCs/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "42d2abba-afb3-437b-bcd4-2ca0584f1126", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Rate", + "yAxisUnit": "reqps" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Bytes reclaimed per second by the garbage collector.", + "id": "w17", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_bytes_reclaimed", + "spaceAggregation": "sum", + "temporality": "cumulative", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Bytes/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "f3225155-7370-4922-b824-8c259bdcf563", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Bytes Reclaimed", + "yAxisUnit": "binBps" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Scheduler context switches per second. High rate = many short-lived processes.", + "id": "w18", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_context_switches", + "spaceAggregation": "sum", + "temporality": "cumulative", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Switches/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "7c57ba19-7d01-4843-b418-72abab7d6201", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Context Switch Rate", + "yAxisUnit": "reqps" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "BEAM reductions per second — proxy for total CPU work across all processes.", + "id": "w19", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_reductions_total", + "spaceAggregation": "sum", + "temporality": "cumulative", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Reductions/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "6a213747-fccf-44b6-b393-abae8e3a856e", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Reductions Rate", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": ">85% is critical — BEAM refuses to spawn new processes at the limit.", + "fillMode": "none", + "fillSpans": false, + "id": "d0d18ace-cca9-4d5a-8440-86be0469022d", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "reduceTo": "avg", + "spaceAggregation": "max", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "k8s.pod.name--string--tag", + "isColumn": false, + "isJSON": false, + "key": "k8s.pod.name", + "type": "tag" + } + ], + "having": { + "expression": "" + }, + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_process_limit", + "reduceTo": "avg", + "spaceAggregation": "max", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "k8s.pod.name--string--tag", + "isColumn": false, + "isJSON": false, + "key": "k8s.pod.name", + "type": "tag" + } + ], + "having": { + "expression": "" + }, + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "disabled": false, + "expression": "A / B * 100", + "legend": "{{k8s.pod.name}}", + "queryName": "F1" + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "6fb58024-22cd-42d0-9865-f89351f3fbbe", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Process Utilization %", + "yAxisUnit": "%" + } + ] +} diff --git a/beam/README.md b/beam/README.md new file mode 100644 index 00000000..f54d4ba0 --- /dev/null +++ b/beam/README.md @@ -0,0 +1,61 @@ +# BEAM / Erlang VM Dashboards + +Two dashboards for monitoring the BEAM Erlang/Elixir runtime. Both are generic — they work with any BEAM project (Elixir or Erlang) that exposes metrics via [`prometheus_ex`](https://hex.pm/packages/prometheus_ex). Import each JSON file directly into SigNoz. + +**Template variables (shared):** `environment` · `k8s_cluster` · `k8s_pod` — all multi-select, scoped to metrics dimensions. + +--- + +## BEAM / Erlang VM - Overview + +**File:** `BEAM-ErlangVM-Overview.json` +**Tags:** `beam` `erlang` `elixir` `overview` + +First stop for on-call triage. Shows the key runtime health indicators at a glance — saturation ratios, memory pressure, scheduler backlog, and GC activity. + +| Section | Panels | +|-----------------------------|---------------------------------------------------------------------------------------------------------------------| +| **Stats ** | Process Utilization %, Port Utilization %, Atom Utilization %, ETS Tables, Schedulers Online, Total Memory | +| **Memory** | Total Memory, Memory Breakdown (stacked: Processes / System / Atom / Binary / ETS / Code), Binary Memory (off-heap) | +| **Schedulers & Run Queues** | Run Queue Length, Dirty CPU / IO Run Queues, Process Utilization % (per pod), Live Processes | +| **GC & Reductions** | GC Rate, GC Bytes Reclaimed, Context Switch Rate, Reductions Rate | + +Stat panels include warning (>70%) and critical (>85%) thresholds. Process/Port/Atom utilization exceeding 85% indicates imminent node exhaustion. + +--- + +## BEAM / Erlang VM - Deep Dive + +**File:** `BEAM-ErlangVM-DeepDive.json` +**Tags:** `beam` `erlang` `elixir` `debug` + +Per-subsystem breakdown for root-cause investigation. Use this after the Overview flags an anomaly. Covers memory allocator internals, MSACC scheduler microstate accounting, Erlang distribution link health, and Mnesia. + +| Section | Panels | +|---------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Process, Port & Atom Saturation** | Process Count vs Limit, Process Utilization %, Port Count vs Limit, Port Utilization %, Atom Count vs Limit, Atom Utilization % | +| **Memory Detail** | Process Heap Memory, System Memory, Atom Memory, Binary Memory (off-heap), ETS Memory, Code Memory | +| **ETS Tables** | ETS Table Count, ETS Table Limit, ETS Utilization %, DETS (Disk ETS) Table Count | +| **MSACC Scheduler Microstate** | aux / check_io / emulator / gc, port / nif / bif / send, sleep, alloc / other / timers | +| **GC & I/O** | GC Count Rate, GC Bytes Reclaimed Rate, GC Words Reclaimed Rate, I/O Bytes In/Out Rate, Context Switches Rate, Reductions Rate | +| **Memory Allocator Fragmentation** | MBCS Pool Block Size & Count, SBC Block Size, MBC Fragmentation %, MBC Block vs Carrier Size | +| **Erlang Distribution Link** | Distribution Bytes Sent/Received Rate, Distribution Send Pending | +| **Distribution Messages** | Distribution Messages Sent/Received Rate, Distribution Packets Sent/Received Rate | +| **Mnesia** | Held Locks, Lock Queue Depth, Tx Restart/Abort/Commit/Logged/Failed Rates, Memory Usage, Unsynchronised Tables, Subscriptions, Running/Stopped DB Nodes | +| **Mnesia Extended** | Held Locks (detail), Subscriptions by Type, Transaction Health Overview, Mnesia vs VM Memory, Node Status, Unsync Tables vs Held Locks, Active Transaction Coordinators | + +### Key signals to watch + +- **MSACC `sleep` high** → schedulers are idle; check I/O wait or under-provisioned concurrency. +- **MSACC `nif` high** → long-running NIFs blocking schedulers. +- **Binary Memory monotonically growing** → binary leak (processes holding large off-heap binary references). +- **MBC Fragmentation % rising** → memory allocator fragmentation; consider allocator tuning or restart. +- **Distribution Send Pending > 0 sustained** → backpressure on an Erlang cluster link. +- **Mnesia Unsynchronised Tables > 0** → partitioned or lagging replica; check node connectivity. + +--- + +## Prerequisites + +- Metrics exported via [`prometheus_ex`](https://hex.pm/packages/prometheus_ex) (or compatible exporter). +- Metrics must be ingested into SigNoz with the resource attributes `deployment.environment`, `k8s.cluster.name`, and `k8s.pod.name` populated for the template variable filters to work.