From b46da4f57172e591e5eb9777e4986fe4bf3524d4 Mon Sep 17 00:00:00 2001 From: Sebastian YEPES Date: Tue, 16 Jun 2026 14:16:44 +0200 Subject: [PATCH 1/2] feat: add BEAM-ErlangVM Dashboards --- beam/BEAM-ErlangVM-DeepDive.json | 1 + beam/BEAM-ErlangVM-Overview.json | 1 + beam/README.md | 61 ++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 beam/BEAM-ErlangVM-DeepDive.json create mode 100644 beam/BEAM-ErlangVM-Overview.json create mode 100644 beam/README.md diff --git a/beam/BEAM-ErlangVM-DeepDive.json b/beam/BEAM-ErlangVM-DeepDive.json new file mode 100644 index 00000000..65a8fc54 --- /dev/null +++ b/beam/BEAM-ErlangVM-DeepDive.json @@ -0,0 +1 @@ +{"description":"Per-subsystem breakdown of BEAM runtime internals: process/port/atom saturation detail, memory segments, ETS, MSACC scheduler microstate, GC, I/O, memory allocator fragmentation, Erlang distribution link health, and Mnesia. Generic for any Elixir/Erlang project using prometheus_ex.","name":"BEAM / Erlang VM - Deep Dive","tags":["beam","erlang","elixir","debug"],"image":"data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTgiIHZpZXdCb3g9IjAgMCAxOCAxOCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0Ljk0OTQgMTMuOTQyQzE2LjIzMTggMTIuNDI1OCAxNy4zMjY4IDkuNzAyMiAxNi4xOTU2IDYuNTc0ODdDMTUuNjQ0MyA1LjA1MjQ1IDE1LjAyMTkgNC4yMDI0OSAxNC4yOTY5IDMuNjYyNTJDMTMuODU1NyAzLjMzMzc5IDEyLjA5MzMgMi41MDYzMyA5Ljc1OTY1IDIuODY3NTZDOC4wNTM0OSAzLjEzMjU1IDUuNzc0ODcgNC4yMDg3NCA0LjI5MzY5IDUuOTU5OUMyLjg1NzUyIDcuNjYxMDYgMS43NDg4MyA5LjAwNDc0IDEuNjk3NTggMTAuMzA5N0MxLjYzMTMzIDExLjk4ODMgMi44OTYyNyAxMy40MzA4IDMuMDUwMDEgMTMuNjY0NUMzLjMyMzc0IDE0LjA3OTUgNS4xOTExNSAxNi40NTE4IDguNjk5NzEgMTYuNTczMUMxMS43OTcgMTYuNjc5MyAxMy44MTQ0IDE1LjI4NDQgMTQuOTQ5NCAxMy45NDJaIiBmaWxsPSIjNDAzRDNFIi8+CjxwYXRoIGQ9Ik00LjU1MzYzIDIuNzM3NDdDMi45Mzc0NiAzLjg5MTE2IDEuMTIxMzEgNi4yNTEwMyAxLjQ0NzU0IDkuNTYwODZDMS42MDYyOCAxMS4xNzIgMi4wMDI1MSAxMi4xNDk1IDIuNTcxMjMgMTIuODUwN0MyLjkxNzQ2IDEzLjI3ODIgNC40MTk4OCAxNC41NDkzIDYuNzczNTEgMTQuNzM2OEM5LjE0NTg4IDE0LjkyNTYgMTAuOTQ5NSAxNC4zOTQ0IDEyLjgzMzIgMTMuMDg0NEMxNi42NjE3IDEwLjQyMDggMTYuMDk4IDYuMzkzNTMgMTUuOTM0MyA1LjkyNDhDMTUuNzcwNSA1LjQ1NjA3IDE0LjU0NDQgMi42OTYyMiAxMS4xNzMzIDEuNzE1MDJDOC4xOTg0NCAwLjg1MDA2OCA1Ljk4MzU1IDEuNzE1MDIgNC41NTM2MyAyLjczNzQ3WiIgZmlsbD0iIzVFNjM2NyIvPgo8cGF0aCBkPSJNNy4zOTM1MyAyLjk2MTA5QzUuNjE3MzcgMi44OTczNCAzLjkxOTk2IDQuMjg4NTIgMy43NTYyMiA2LjAwNTkzQzMuNTkyNDggNy43MjIwOSA0LjY1NDkyIDkuMDI5NTIgNi4zMDk4MyA5LjI5NTc2QzcuOTY0NzUgOS41NjA3NCA5Ljg3ODM5IDguNTU1OCAxMC4yNjM0IDYuNDUwOTFDMTAuNjYwOSA0LjI4MjI3IDkuMDg5NjkgMy4wMjIzNCA3LjM5MzUzIDIuOTYxMDlaIiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBkPSJNNy45NDIxNyA1LjkwMTE1QzcuOTQyMTcgNS45MDExNSA4LjM2OTY1IDUuODEyNCA4LjQ1NDY1IDUuMTgyNDRDOC41MzgzOSA0LjU2MjQ3IDguMjMwOTEgNC4wMzM3NSA3LjUxMzQ1IDMuODQzNzZDNi43MzM0OSAzLjYzNzUyIDYuMjA0NzcgNC4wNjYyNSA2LjA2NzI3IDQuNTE3NDdDNS44NzYwMyA1LjE0NDk0IDYuMTU4NTIgNS40NDM2NyA2LjE1ODUyIDUuNDQzNjdDNi4xNTg1MiA1LjQ0MzY3IDUuMzkzNTYgNS42Mjc0MSA1LjMzMjMxIDYuNTI5ODdDNS4yNzQ4MSA3LjM4MTA3IDUuODU2MDMgNy44Mzg1NSA2LjQzOTc1IDcuOTc4NTRDNy4xNjA5NiA4LjE1MjI4IDcuOTc4NDIgNy45NTQ3OSA4LjE3ODQxIDcuMDM0ODRDOC4zNDQ2NSA2LjI3NzM4IDcuOTQyMTcgNS45MDExNSA3Ljk0MjE3IDUuOTAxMTVaIiBmaWxsPSIjMzAzMDMwIi8+CjxwYXRoIGQ9Ik02LjczOTgzIDQuNzUzNjJDNi42NzEwOSA1LjAxMjM1IDYuODA4NTggNS4yNjIzNCA3LjA3ODU3IDUuMzMxMDlDNy4zNjk4IDUuNDA0ODMgNy42MzQ3OSA1LjMwODU5IDcuNzA2MDMgNS4wMTExQzcuNzY4NTMgNC43NDczNyA3LjY0MzU0IDQuNTE0ODggNy4zMzYwNSA0LjQzOTg4QzcuMDgzNTcgNC4zNzczOSA2LjgxNDgzIDQuNDcxMTMgNi43Mzk4MyA0Ljc1MzYyWiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTYuOTU5NzggNi4wMzk3NEM2LjYzMjMgNS45Mzg0OSA2LjE5OTgyIDYuMDY0NzMgNi4xMzEwNyA2LjUwNDcxQzYuMDYyMzMgNi45NDQ2OSA2LjMyNjA2IDcuMTY5NjggNi42NzEwNCA3LjIzMjE3QzcuMDE2MDMgNy4yOTQ2NyA3LjM0MjI2IDcuMTEzNDMgNy40MDYwMSA2Ljc2MDk1QzcuNDY4NSA2LjQwOTcyIDcuMjg2MDEgNi4xMzk3MyA2Ljk1OTc4IDYuMDM5NzRaIiBmaWxsPSJ3aGl0ZSIvPgo8L3N2Zz4K","variables":{"environment":{"allSelected":true,"description":"Deployment environment filter","dynamicVariablesAttribute":"deployment.environment","dynamicVariablesSource":"Metrics","id":"var-environment","multiSelect":true,"name":"environment","order":0,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"},"k8s_cluster":{"allSelected":true,"description":"Kubernetes cluster name filter","dynamicVariablesAttribute":"k8s.cluster.name","dynamicVariablesSource":"Metrics","id":"var-k8s-cluster","multiSelect":true,"name":"k8s_cluster","order":1,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"},"k8s_pod":{"allSelected":true,"description":"Pod name filter — select specific pods to scope all panels to one application","dynamicVariablesAttribute":"k8s.pod.name","dynamicVariablesSource":"Metrics","id":"var-k8s-pod","multiSelect":true,"name":"k8s_pod","order":2,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"}},"widgets":[{"contextLinks":{"linksData":[]},"description":"","id":"r1","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Process, Port & Atom Saturation"},{"contextLinks":{"linksData":[]},"description":"Live processes (solid) vs hard limit (dashed). Gap = headroom. Shrinking gap = leak or load spike.","id":"w1","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Process Count","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_process_limit","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Process Limit","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"d745aedc-ca2a-480e-b538-2782ea4fbbb9","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Process Count vs Limit","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":">85% is critical — BEAM refuses to spawn new processes at the limit.","fillMode":"none","fillSpans":false,"id":"w2","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_process_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"d9b4c6b7-a492-42d0-a64d-55a3ec119ab2","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Process Utilization %","yAxisUnit":"percent"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"Ports represent open sockets, files, and NIF handles. Exhaustion causes connection failures.","fillMode":"none","fillSpans":false,"id":"w3","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_port_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":false,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":{"expression":""},"legend":"Port Count","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_port_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":false,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":{"expression":""},"legend":"Port Limit","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"ea4de080-95d9-46d5-bd1a-51c637c13eee","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Port Count vs Limit","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"Port exhaustion does not crash the node but causes {error, system_limit} on new connections.","fillMode":"none","fillSpans":false,"id":"w4","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_port_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_port_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"a2163961-8c39-4672-8e29-996dbb26665a","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Port Utilization %","yAxisUnit":"percent"},{"contextLinks":{"linksData":[]},"description":"Atom table is global and never shrinks. Monotonic growth without restart = atom leak (dynamic atom creation).","id":"w5","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_atom_count","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Atom Count","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_atom_limit","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Atom Limit","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"cdb7c126-a228-4297-888c-4d5dfc65a4cc","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Atom Count vs Limit","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r2","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Memory Detail"},{"contextLinks":{"linksData":[]},"description":"Memory allocated to all Erlang process stacks and heaps. Growing without a matching process count increase = per-process memory leak.","id":"w6","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_processes_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Process Heap","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"dafc56fc-e40c-4ed7-ae40-3ab34b1464ec","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Process Heap Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"BEAM VM system memory: ETS, code, stack, and VM overhead. Excludes process heap.","id":"w7","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_system_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"System","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c04b416e-405d-4773-84fd-a6cfbdd5072d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"System Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Memory consumed by the atom table. Correlated with atom count.","id":"w8","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_atom_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Atom Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"93a93556-2769-4387-8302-6119c7b5ff40","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Atom Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Off-heap reference-counted binaries. Steady monotonic growth between GCs = binary leak (processes holding large binary references).","id":"w9","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"beam_memory_binary_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Binary Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"6d8ca821-9676-4b7c-bfc8-56afaca9d9ef","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Binary Memory (off-heap)","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Total memory used by all ETS tables. Correlates with table count and row density.","id":"w10","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"beam_memory_ets_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"ETS Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"2184b14f-5e55-4a8f-af90-3451a7583b4b","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"ETS Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Memory occupied by loaded BEAM modules. Significant spikes after hot upgrades or repeated code:load calls.","id":"w11","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"beam_memory_code_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Code Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"80db8391-3a5f-47b4-9ada-912df39e90d4","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Code Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"r3","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"ETS Tables"},{"contextLinks":{"linksData":[]},"description":"Total open ETS tables. Steady growth without a corresponding application load increase = table leak (missing :ets.delete).","id":"w12","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_ets_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"ETS Tables","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"9397af8e-b8a7-4cd9-90c0-0bddb580126d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"ETS Table Count","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Hard ceiling for ETS tables (default 8192). Reference line — compare against ETS Count.","id":"w13","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_ets_limit","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"ETS Limit","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c0295751-2141-489e-8cdb-977960827e95","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"ETS Table Limit","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"ETS tables used vs limit. >80% warrants investigation into table ownership and lifecycle.","fillMode":"none","fillSpans":false,"id":"w14","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_ets_tables","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_ets_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"9b50b63a-c493-427c-bdec-ba3129d7dac1","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"ETS Utilization %","yAxisUnit":"percent"},{"contextLinks":{"linksData":[]},"description":"Disk-backed ETS tables. Normally 0 in most apps. Unexpected growth = open disk tables not being closed.","id":"w16","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_dets_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"DETS Tables","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"4d1ad25d-fe40-474f-982f-10e8f188388a","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"DETS (Disk ETS) Table Count","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r4","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"MSACC Scheduler Microstate"},{"contextLinks":{"linksData":[]},"description":"Fraction of scheduler time each MSACC state consumes. 'aux' + 'check_io' + 'emulator' should dominate; high 'sleep' = idle schedulers; high 'gc' = GC pressure.","id":"w17","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_msacc_aux_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"aux","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_check_io_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"check_io","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_emulator_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"emulator","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_gc_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"D","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"gc","orderBy":[],"queryName":"D","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"1c540b59-0c75-4c3d-ae9b-2d49a6f2ea22","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MSACC: aux / check_io / emulator / gc","yAxisUnit":"s"},{"contextLinks":{"linksData":[]},"description":"Scheduler time in port / nif / bif / send. Elevated nif = heavy NIF calls blocking schedulers; elevated send = message-passing saturation.","id":"w18","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_msacc_port_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"port","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_nif_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"nif","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_bif_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"bif","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_send_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"D","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"send","orderBy":[],"queryName":"D","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"5dad1b84-abbb-4efb-98d3-3c23e3f97fb6","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MSACC: port / nif / bif / send","yAxisUnit":"s"},{"contextLinks":{"linksData":[]},"description":"Scheduler time asleep. High sleep = under-utilised schedulers (normal during low load) or blocked waiting on I/O.","id":"w19","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_msacc_sleep_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"sleep","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c77bc2c8-af64-47ec-bc27-0f12bdb7b00c","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MSACC: sleep","yAxisUnit":"s"},{"contextLinks":{"linksData":[]},"description":"Scheduler time in alloc / other / timers. Elevated alloc = heavy allocation pressure; elevated timers = many active timer references.","id":"w20","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_msacc_alloc_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"alloc","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_other_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"other","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_timers_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"timers","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"e25c0a42-7f8d-4048-b5f1-e78c17b61c3c","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MSACC: alloc / other / timers","yAxisUnit":"s"},{"contextLinks":{"linksData":[]},"description":"","id":"r5","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"GC & I/O"},{"contextLinks":{"linksData":[]},"description":"GC runs per second. Short spikes are normal; sustained high rates with rising heap = object churn.","id":"w21","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_number_of_gcs","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"GC Count Rate","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"0a8cc42d-7082-4ee3-b35e-7a62f9a0e32b","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Count Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Bytes reclaimed per second by GC. High reclaim with high GC count = frequent short-lived allocation churn.","id":"w22","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_bytes_reclaimed","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Bytes Reclaimed/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"5e52a97e-1c5c-4b7b-a1c5-17d26640dbd5","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Bytes Reclaimed Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Words moved across generations per GC cycle. Large word counts = major GC activity; correlate with pauses.","id":"w23","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_words_reclaimed","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Words Reclaimed/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"4db37a6d-e10e-40ba-b9a8-6ea32c9c78e5","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Words Reclaimed Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Raw bytes received by all ports (sockets, pipes). Sustained high read throughput is normal; sudden drops = upstream pressure.","id":"w24","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_bytes_received_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Bytes In/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"3b4de0fd-f6d1-42be-9f36-2e91b7bde0d0","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"I/O Bytes In Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Raw bytes sent by all ports. Sustained high write throughput is normal; compare with in-rate for asymmetry.","id":"w25","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_bytes_output_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Bytes Out/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"5de6b8d3-a5bd-4b1f-8ee5-fd08b64aa9a7","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"I/O Bytes Out Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Total context switches across all schedulers per second. Sudden step-changes correlate with process spawning bursts or supervisor restarts.","id":"w26","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_context_switches","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Context Switches/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"b2da73e4-0a0e-4ead-8a03-e1c10b4fa218","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Context Switches Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Total reductions (function calls + tail calls) per second. Steady baseline; spikes indicate CPU-intensive work or hot loops.","id":"w27","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_reductions_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Reductions/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"aa98a5ca-e975-48e4-9ce8-c6ea2f524e79","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Reductions Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r6","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Memory Allocator Fragmentation"},{"contextLinks":{"linksData":[]},"description":"Bytes allocated by the BEAM allocator vs bytes actually used by live data. Gap = fragmentation overhead.","id":"w28","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_pool_blocks_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} pool_blocks_size","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"6bd1e624-1f66-4dc5-88c9-b0893b6c0bde","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Allocator MBCS Pool Block Size","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Number of multi-block carriers in the pool. Growing without load growth = fragmented carrier accumulation.","id":"w29","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_pool_blocks","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} pool_blocks","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"a9a19bb3-5069-441b-aece-5a3199b48720","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Allocator MBCS Pool Block Count","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Single-block carriers created by the allocator. SBC growth = large allocations exceeding MBC threshold; consider +MBcst tuning.","id":"w30","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_sbcs_blocks_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} sbc_blocks_size","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c44f2174-b40b-4b00-ba97-b3fe55f9a1a1","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Allocator SBC Block Size","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Fragmentation ratio = (allocated − used) / allocated. Values >40% sustained = fragmented memory; consider allocator flags or node restart.","id":"w31","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_block_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_carriers_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","expression":"(B - A) / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"id":"4e48e5b5-6e2f-489e-a2e7-72c76b90ae74","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":100,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MBC Fragmentation %","yAxisUnit":"percent"},{"contextLinks":{"linksData":[]},"description":"Allocated carrier size (total reserved) vs block size (live data). Wide gap = wasted carrier memory.","id":"w32","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_block_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} block","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_carriers_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} carrier","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"b39cc3d5-6acb-4a8e-883d-84be6c7f6462","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MBC Block vs Carrier Size","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"r7","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Erlang Distribution Link"},{"contextLinks":{"linksData":[]},"description":"Distribution bytes sent per second to remote nodes. Sudden drop = node disconnect or network partition.","id":"w33","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_send_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Bytes Sent/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"d96a4d31-80fe-415c-b2c7-a4fc1c8e86e6","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Bytes Sent Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Distribution bytes received per second from remote nodes. Spikes = large remote message deliveries or data transfers.","id":"w34","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_recv_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Bytes Recv/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"4e22b2db-83b8-4ad2-8a38-ef81f89c4eb0","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Bytes Received Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"r8","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Distribution Messages"},{"contextLinks":{"linksData":[]},"description":"Messages sent across distribution links per second. High rate + growing queues = dist buffer saturation risk.","id":"w35","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_send_messages_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Messages Sent/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"f8f4c226-0c52-4d8f-8eae-f5c41a8e5e3d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Messages Sent Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Messages received over distribution links per second. Asymmetry with sent rate = one-way traffic imbalance.","id":"w36","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_recv_messages_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Messages Recv/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c17ca374-f79d-4eac-9770-4dbf19da4c37","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Messages Received Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Pending bytes in the distribution send buffer. Sustained non-zero values = congested distribution link.","id":"w37","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_send_pending_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Pending Send Bytes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"8e73f8ea-c6a9-4bef-bd86-adde23c8e4ca","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Send Pending","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Packets sent across distribution links. Low-frequency, high-byte packets = large message transfers.","id":"w38","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_send_packets_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Packets Sent/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"1e4d4d9f-b8a0-4b12-af6f-13c4cd9e0f4d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Packets Sent Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Packets received over distribution links. Compare with sent rate and bytes for flow analysis.","id":"w39","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_recv_packets_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Packets Recv/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c01c7098-49c3-4dd4-834e-e7ed9f97e8a0","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Packets Received Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r9","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Mnesia"},{"contextLinks":{"linksData":[]},"description":"Active Mnesia transactions. Sustained high counts = long-running transactions blocking table locks.","id":"w40","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_held_locks","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Held Locks","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"d36ff9fa-7e51-492e-aa81-61f5a7c5f7e3","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Held Locks","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Processes waiting to acquire Mnesia locks. Non-zero = lock contention; high values = potential deadlock or slow transactions.","id":"w41","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_lock_queue","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Lock Queue","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"2ec3b61e-2065-4eef-a6d3-dc9219ef6a05","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Lock Queue Depth","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia transaction restarts due to lock conflicts. Elevated restart rate = high write contention; consider table sharding or optimistic locking.","id":"w42","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_restarts","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Transaction Restarts/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"e7f3c17a-59e4-4e54-9e22-71b6d4c4d25a","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Transaction Restart Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia transactions aborted per second. Aborts can be caused by conflicts, node failures, or application errors.","id":"w43","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_aborts","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Transaction Aborts/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"bab96b77-1e6e-4fbb-a7e7-61f88a5cd07f","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Transaction Abort Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia transactions committed per second. Baseline for throughput; compare with abort and restart rates for health ratio.","id":"w44","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_commits","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Transaction Commits/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"addd1a97-d7dc-42e8-9424-2cc26789d04d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Transaction Commit Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia log entries written per second. Disk-based tables only. High log rate + low commit rate = replay overhead.","id":"w45","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_logged_transactions","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Logged Transactions/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"deea3f37-9c5b-4d99-82b4-95e285c1d47f","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Logged Transactions Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Failed Mnesia writes. Any non-zero value = data integrity issue or schema mismatch.","id":"w46","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_failed_transactions","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Failed Transactions/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"66cc4fb5-cd2a-4f42-aba6-a6c0e6c15e95","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Failed Transaction Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables currently open. Steady count = stable schema; sudden drop = node partition or table deletion.","id":"w47","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_memory_usage_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Memory Usage","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"0e88bf72-e297-424b-b4a4-3aef8bd5e5e5","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Memory Usage","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables with unsynchronised data across nodes. Non-zero = split-brain risk. Should return to zero after healing.","id":"w48","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_unsynchronised_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Unsynchronised Tables","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"43c0c24d-1ecf-4af7-83e4-b8c4d7f3a3e3","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Unsynchronised Tables","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Subscribed Mnesia event listeners. Unexpected drop = subscriber process crashed.","id":"w49","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_subscriptions","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Subscriptions","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"0d9dc6bf-9b77-4b5a-a5bf-8975c38fe9d1","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Subscriptions","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Number of connected Mnesia cluster nodes. Drop below expected count = node departure or partition.","id":"w50","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_db_nodes_running","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Running DB Nodes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"bb0d8b63-fa29-4f70-a6a6-7f64fc46dca9","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Running DB Nodes","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r10","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Mnesia Extended"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables registered in the schema. Sudden change = schema migration or table deletion.","id":"w51","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_db_nodes_stopped","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Stopped DB Nodes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"8b57d1ef-ef46-4c5f-b8ef-73f52f3499e4","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Stopped DB Nodes","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Table copies held by this node. Reduction = replica departure or table deletion.","id":"w52","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_held_locks","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Held Locks (detail)","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"b4fad1ac-d9ba-4668-a81e-26a36c2bbf74","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Held Locks (detail)","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia event subscribers. Each type of activity subscription appears as a separate line.","id":"w53","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_subscriptions","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"type--string--tag","isColumn":false,"key":"type","type":"tag"}],"legend":"{{type}} subscriptions","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"28e2e9c4-c5aa-472e-850e-ee85e8e22b49","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Subscriptions by Type","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia commit rate vs abort rate vs restart rate. High abort-to-commit ratio = significant write contention.","id":"w54","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_commits","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Commits/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_mnesia_transaction_aborts","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Aborts/s","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_mnesia_transaction_restarts","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Restarts/s","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"1f5e5c8e-fbd2-4b9c-802e-b73b8ba56f19","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Transaction Health Overview","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Memory consumed by Mnesia data vs process heap vs system. Helps identify if Mnesia dominates memory.","id":"w55","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_memory_usage_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Mnesia Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_memory_processes_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Process Heap","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_memory_system_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"System","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"7f15e2c5-07ab-49b8-a1ee-0d8edfd06fc5","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia vs VM Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables as currently reported. Any drops = partition or node failure.","id":"w56","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_db_nodes_running","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Running Nodes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_mnesia_db_nodes_stopped","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Stopped Nodes","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"e1a7b8c9-d2e3-4f56-a789-b0c1d2e3f456","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Node Status","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables with unsynchronised copies (split-brain indicator) vs held locks (contention indicator) over time.","id":"w57","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_unsynchronised_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Unsync Tables","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_mnesia_held_locks","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Held Locks","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"a2b3c4d5-e6f7-4890-a1b2-c3d4e5f60123","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Unsync Tables vs Held Locks","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Active Mnesia transaction coordinators (dirty vs transactional). Dominance of dirty writes = performance but no isolation.","id":"w58","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_coordinators","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Transaction Coordinators","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"f1e2d3c4-b5a6-4789-9012-3456789abcde","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Active Transaction Coordinators","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"% of atom table used. Atom table never shrinks — runaway growth = leak.","fillMode":"none","fillSpans":false,"id":"39eb4c4f-5129-4a3c-92fd-8786274b1159","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_atom_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":{"expression":""},"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_atom_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":{"expression":""},"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"d9b4c6b7-a492-42d0-a64d-55a3ec119ab2","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Atom Utilization %","yAxisUnit":"percent"}],"layout":[{"h":1,"i":"r1","moved":false,"static":false,"w":12,"x":0,"y":0},{"h":7,"i":"39eb4c4f-5129-4a3c-92fd-8786274b1159","moved":false,"static":false,"w":4,"x":0,"y":1},{"h":7,"i":"w2","moved":false,"static":false,"w":4,"x":4,"y":1},{"h":7,"i":"w4","moved":false,"static":false,"w":4,"x":8,"y":1},{"h":7,"i":"w5","moved":false,"static":false,"w":4,"x":0,"y":8},{"h":7,"i":"w1","moved":false,"static":false,"w":4,"x":4,"y":8},{"h":7,"i":"w3","moved":false,"static":false,"w":4,"x":8,"y":8},{"h":1,"i":"r2","moved":false,"static":false,"w":12,"x":0,"y":15},{"h":7,"i":"w6","moved":false,"static":false,"w":4,"x":0,"y":16},{"h":7,"i":"w7","moved":false,"static":false,"w":4,"x":4,"y":16},{"h":7,"i":"w8","moved":false,"static":false,"w":4,"x":8,"y":16},{"h":7,"i":"w9","moved":false,"static":false,"w":4,"x":0,"y":23},{"h":7,"i":"w10","moved":false,"static":false,"w":4,"x":4,"y":23},{"h":7,"i":"w11","moved":false,"static":false,"w":4,"x":8,"y":23},{"h":1,"i":"r3","moved":false,"static":false,"w":12,"x":0,"y":30},{"h":7,"i":"w14","moved":false,"static":false,"w":4,"x":0,"y":31},{"h":7,"i":"w12","moved":false,"static":false,"w":4,"x":4,"y":31},{"h":7,"i":"w13","moved":false,"static":false,"w":4,"x":8,"y":31},{"h":7,"i":"w16","moved":false,"static":false,"w":12,"x":0,"y":38},{"h":1,"i":"r4","moved":false,"static":false,"w":12,"x":0,"y":45},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":0,"y":46},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":3,"y":46},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":6,"y":46},{"h":7,"i":"w20","moved":false,"static":false,"w":3,"x":9,"y":46},{"h":1,"i":"r5","moved":false,"static":false,"w":12,"x":0,"y":53},{"h":7,"i":"w21","moved":false,"static":false,"w":4,"x":0,"y":54},{"h":7,"i":"w22","moved":false,"static":false,"w":4,"x":4,"y":54},{"h":7,"i":"w23","moved":false,"static":false,"w":4,"x":8,"y":54},{"h":7,"i":"w24","moved":false,"static":false,"w":4,"x":0,"y":61},{"h":7,"i":"w25","moved":false,"static":false,"w":4,"x":4,"y":61},{"h":7,"i":"w26","moved":false,"static":false,"w":4,"x":8,"y":61},{"h":7,"i":"w27","moved":false,"static":false,"w":12,"x":0,"y":68},{"h":1,"i":"r6","moved":false,"static":false,"w":12,"x":0,"y":75},{"h":7,"i":"w28","moved":false,"static":false,"w":4,"x":0,"y":76},{"h":7,"i":"w29","moved":false,"static":false,"w":4,"x":4,"y":76},{"h":7,"i":"w30","moved":false,"static":false,"w":4,"x":8,"y":76},{"h":7,"i":"w31","moved":false,"static":false,"w":6,"x":0,"y":83},{"h":7,"i":"w32","moved":false,"static":false,"w":6,"x":6,"y":83},{"h":1,"i":"r7","moved":false,"static":false,"w":12,"x":0,"y":90},{"h":7,"i":"w33","moved":false,"static":false,"w":6,"x":0,"y":91},{"h":7,"i":"w34","moved":false,"static":false,"w":6,"x":6,"y":91},{"h":1,"i":"r8","moved":false,"static":false,"w":12,"x":0,"y":98},{"h":7,"i":"w35","moved":false,"static":false,"w":6,"x":0,"y":99},{"h":7,"i":"w36","moved":false,"static":false,"w":6,"x":6,"y":99},{"h":7,"i":"w37","moved":false,"static":false,"w":4,"x":0,"y":106},{"h":7,"i":"w38","moved":false,"static":false,"w":4,"x":4,"y":106},{"h":7,"i":"w39","moved":false,"static":false,"w":4,"x":8,"y":106},{"h":1,"i":"r9","moved":false,"static":false,"w":12,"x":0,"y":113},{"h":7,"i":"w40","moved":false,"static":false,"w":6,"x":0,"y":114},{"h":7,"i":"w41","moved":false,"static":false,"w":6,"x":6,"y":114},{"h":7,"i":"w42","moved":false,"static":false,"w":6,"x":0,"y":121},{"h":7,"i":"w43","moved":false,"static":false,"w":6,"x":6,"y":121},{"h":7,"i":"w44","moved":false,"static":false,"w":6,"x":0,"y":128},{"h":7,"i":"w45","moved":false,"static":false,"w":6,"x":6,"y":128},{"h":7,"i":"w46","moved":false,"static":false,"w":6,"x":0,"y":135},{"h":7,"i":"w47","moved":false,"static":false,"w":6,"x":6,"y":135},{"h":7,"i":"w48","moved":false,"static":false,"w":4,"x":0,"y":142},{"h":7,"i":"w49","moved":false,"static":false,"w":4,"x":4,"y":142},{"h":7,"i":"w50","moved":false,"static":false,"w":4,"x":8,"y":142},{"h":1,"i":"r10","moved":false,"static":false,"w":12,"x":0,"y":149},{"h":7,"i":"w51","moved":false,"static":false,"w":3,"x":0,"y":150},{"h":7,"i":"w52","moved":false,"static":false,"w":3,"x":3,"y":150},{"h":7,"i":"w53","moved":false,"static":false,"w":6,"x":6,"y":150},{"h":7,"i":"w54","moved":false,"static":false,"w":6,"x":0,"y":157},{"h":7,"i":"w55","moved":false,"static":false,"w":6,"x":6,"y":157},{"h":7,"i":"w56","moved":false,"static":false,"w":4,"x":0,"y":164},{"h":7,"i":"w57","moved":false,"static":false,"w":4,"x":4,"y":164},{"h":7,"i":"w58","moved":false,"static":false,"w":4,"x":8,"y":164}],"panelMap":{"r1":{"collapsed":false,"widgets":[{"h":7,"i":"w1","moved":false,"static":false,"w":4,"x":0,"y":1},{"h":7,"i":"w2","moved":false,"static":false,"w":4,"x":4,"y":1},{"h":7,"i":"w3","moved":false,"static":false,"w":4,"x":8,"y":1},{"h":7,"i":"w4","moved":false,"static":false,"w":6,"x":0,"y":8},{"h":7,"i":"w5","moved":false,"static":false,"w":6,"x":6,"y":8},{"h":1,"i":"r2","moved":false,"static":false,"w":12,"x":0,"y":15},{"h":7,"i":"w6","moved":false,"static":false,"w":4,"x":0,"y":16},{"h":7,"i":"w7","moved":false,"static":false,"w":4,"x":4,"y":16},{"h":7,"i":"w8","moved":false,"static":false,"w":4,"x":8,"y":16},{"h":7,"i":"w9","moved":false,"static":false,"w":4,"x":0,"y":23},{"h":7,"i":"w10","moved":false,"static":false,"w":4,"x":4,"y":23},{"h":7,"i":"w11","moved":false,"static":false,"w":4,"x":8,"y":23},{"h":1,"i":"r3","moved":false,"static":false,"w":12,"x":0,"y":30},{"h":7,"i":"w12","moved":false,"static":false,"w":3,"x":0,"y":31},{"h":7,"i":"w13","moved":false,"static":false,"w":3,"x":3,"y":31},{"h":7,"i":"w14","moved":false,"static":false,"w":3,"x":6,"y":31},{"h":7,"i":"w15","moved":false,"static":false,"w":3,"x":9,"y":31},{"h":7,"i":"w16","moved":false,"static":false,"w":12,"x":0,"y":38},{"h":1,"i":"r4","moved":false,"static":false,"w":12,"x":0,"y":45},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":0,"y":46},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":3,"y":46},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":6,"y":46},{"h":7,"i":"w20","moved":false,"static":false,"w":3,"x":9,"y":46},{"h":1,"i":"r5","moved":false,"static":false,"w":12,"x":0,"y":53},{"h":7,"i":"w21","moved":false,"static":false,"w":4,"x":0,"y":54},{"h":7,"i":"w22","moved":false,"static":false,"w":4,"x":4,"y":54},{"h":7,"i":"w23","moved":false,"static":false,"w":4,"x":8,"y":54},{"h":7,"i":"w24","moved":false,"static":false,"w":4,"x":0,"y":61},{"h":7,"i":"w25","moved":false,"static":false,"w":4,"x":4,"y":61},{"h":7,"i":"w26","moved":false,"static":false,"w":4,"x":8,"y":61},{"h":7,"i":"w27","moved":false,"static":false,"w":12,"x":0,"y":68},{"h":1,"i":"r6","moved":false,"static":false,"w":12,"x":0,"y":75},{"h":7,"i":"w28","moved":false,"static":false,"w":4,"x":0,"y":76},{"h":7,"i":"w29","moved":false,"static":false,"w":4,"x":4,"y":76},{"h":7,"i":"w30","moved":false,"static":false,"w":4,"x":8,"y":76},{"h":7,"i":"w31","moved":false,"static":false,"w":6,"x":0,"y":83},{"h":7,"i":"w32","moved":false,"static":false,"w":6,"x":6,"y":83},{"h":1,"i":"r7","moved":false,"static":false,"w":12,"x":0,"y":90},{"h":7,"i":"w33","moved":false,"static":false,"w":6,"x":0,"y":91},{"h":7,"i":"w34","moved":false,"static":false,"w":6,"x":6,"y":91},{"h":1,"i":"r8","moved":false,"static":false,"w":12,"x":0,"y":98},{"h":7,"i":"w35","moved":false,"static":false,"w":6,"x":0,"y":99},{"h":7,"i":"w36","moved":false,"static":false,"w":6,"x":6,"y":99},{"h":7,"i":"w37","moved":false,"static":false,"w":4,"x":0,"y":106},{"h":7,"i":"w38","moved":false,"static":false,"w":4,"x":4,"y":106},{"h":7,"i":"w39","moved":false,"static":false,"w":4,"x":8,"y":106},{"h":1,"i":"r9","moved":false,"static":false,"w":12,"x":0,"y":113},{"h":7,"i":"w40","moved":false,"static":false,"w":6,"x":0,"y":114},{"h":7,"i":"w41","moved":false,"static":false,"w":6,"x":6,"y":114},{"h":7,"i":"w42","moved":false,"static":false,"w":6,"x":0,"y":121},{"h":7,"i":"w43","moved":false,"static":false,"w":6,"x":6,"y":121},{"h":7,"i":"w44","moved":false,"static":false,"w":6,"x":0,"y":128},{"h":7,"i":"w45","moved":false,"static":false,"w":6,"x":6,"y":128},{"h":7,"i":"w46","moved":false,"static":false,"w":6,"x":0,"y":135},{"h":7,"i":"w47","moved":false,"static":false,"w":6,"x":6,"y":135},{"h":7,"i":"w48","moved":false,"static":false,"w":4,"x":0,"y":142},{"h":7,"i":"w49","moved":false,"static":false,"w":4,"x":4,"y":142},{"h":7,"i":"w50","moved":false,"static":false,"w":4,"x":8,"y":142},{"h":1,"i":"r10","moved":false,"static":false,"w":12,"x":0,"y":149},{"h":7,"i":"w51","moved":false,"static":false,"w":3,"x":0,"y":150},{"h":7,"i":"w52","moved":false,"static":false,"w":3,"x":3,"y":150},{"h":7,"i":"w53","moved":false,"static":false,"w":6,"x":6,"y":150},{"h":7,"i":"w54","moved":false,"static":false,"w":6,"x":0,"y":157},{"h":7,"i":"w55","moved":false,"static":false,"w":6,"x":6,"y":157},{"h":7,"i":"w56","moved":false,"static":false,"w":4,"x":0,"y":164},{"h":7,"i":"w57","moved":false,"static":false,"w":4,"x":4,"y":164},{"h":7,"i":"w58","moved":false,"static":false,"w":4,"x":8,"y":164},{"h":7,"i":"3776cf0d-5c63-468a-9a99-d2dbc81431df","moved":false,"static":false,"w":4,"x":0,"y":171}]}},"version":"v5","title":"BEAM / Erlang VM - Deep Dive"} diff --git a/beam/BEAM-ErlangVM-Overview.json b/beam/BEAM-ErlangVM-Overview.json new file mode 100644 index 00000000..0561f951 --- /dev/null +++ b/beam/BEAM-ErlangVM-Overview.json @@ -0,0 +1 @@ +{"description":"Key BEAM runtime health indicators. First stop for on-call triage — process/port saturation, memory pressure, scheduler backlog, GC activity. Generic for any Elixir/Erlang project using prometheus_ex.","name":"BEAM / Erlang VM - Overview","tags":["beam","erlang","elixir","overview"],"image":"data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTgiIHZpZXdCb3g9IjAgMCAxOCAxOCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0Ljk0OTQgMTMuOTQyQzE2LjIzMTggMTIuNDI1OCAxNy4zMjY4IDkuNzAyMiAxNi4xOTU2IDYuNTc0ODdDMTUuNjQ0MyA1LjA1MjQ1IDE1LjAyMTkgNC4yMDI0OSAxNC4yOTY5IDMuNjYyNTJDMTMuODU1NyAzLjMzMzc5IDEyLjA5MzMgMi41MDYzMyA5Ljc1OTY1IDIuODY3NTZDOC4wNTM0OSAzLjEzMjU1IDUuNzc0ODcgNC4yMDg3NCA0LjI5MzY5IDUuOTU5OUMyLjg1NzUyIDcuNjYxMDYgMS43NDg4MyA5LjAwNDc0IDEuNjk3NTggMTAuMzA5N0MxLjYzMTMzIDExLjk4ODMgMi44OTYyNyAxMy40MzA4IDMuMDUwMDEgMTMuNjY0NUMzLjMyMzc0IDE0LjA3OTUgNS4xOTExNSAxNi40NTE4IDguNjk5NzEgMTYuNTczMUMxMS43OTcgMTYuNjc5MyAxMy44MTQ0IDE1LjI4NDQgMTQuOTQ5NCAxMy45NDJaIiBmaWxsPSIjNDAzRDNFIi8+CjxwYXRoIGQ9Ik00LjU1MzYzIDIuNzM3NDdDMi45Mzc0NiAzLjg5MTE2IDEuMTIxMzEgNi4yNTEwMyAxLjQ0NzU0IDkuNTYwODZDMS42MDYyOCAxMS4xNzIgMi4wMDI1MSAxMi4xNDk1IDIuNTcxMjMgMTIuODUwN0MyLjkxNzQ2IDEzLjI3ODIgNC40MTk4OCAxNC41NDkzIDYuNzczNTEgMTQuNzM2OEM5LjE0NTg4IDE0LjkyNTYgMTAuOTQ5NSAxNC4zOTQ0IDEyLjgzMzIgMTMuMDg0NEMxNi42NjE3IDEwLjQyMDggMTYuMDk4IDYuMzkzNTMgMTUuOTM0MyA1LjkyNDhDMTUuNzcwNSA1LjQ1NjA3IDE0LjU0NDQgMi42OTYyMiAxMS4xNzMzIDEuNzE1MDJDOC4xOTg0NCAwLjg1MDA2OCA1Ljk4MzU1IDEuNzE1MDIgNC41NTM2MyAyLjczNzQ3WiIgZmlsbD0iIzVFNjM2NyIvPgo8cGF0aCBkPSJNNy4zOTM1MyAyLjk2MTA5QzUuNjE3MzcgMi44OTczNCAzLjkxOTk2IDQuMjg4NTIgMy43NTYyMiA2LjAwNTkzQzMuNTkyNDggNy43MjIwOSA0LjY1NDkyIDkuMDI5NTIgNi4zMDk4MyA5LjI5NTc2QzcuOTY0NzUgOS41NjA3NCA5Ljg3ODM5IDguNTU1OCAxMC4yNjM0IDYuNDUwOTFDMTAuNjYwOSA0LjI4MjI3IDkuMDg5NjkgMy4wMjIzNCA3LjM5MzUzIDIuOTYxMDlaIiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBkPSJNNy45NDIxNyA1LjkwMTE1QzcuOTQyMTcgNS45MDExNSA4LjM2OTY1IDUuODEyNCA4LjQ1NDY1IDUuMTgyNDRDOC41MzgzOSA0LjU2MjQ3IDguMjMwOTEgNC4wMzM3NSA3LjUxMzQ1IDMuODQzNzZDNi43MzM0OSAzLjYzNzUyIDYuMjA0NzcgNC4wNjYyNSA2LjA2NzI3IDQuNTE3NDdDNS44NzYwMyA1LjE0NDk0IDYuMTU4NTIgNS40NDM2NyA2LjE1ODUyIDUuNDQzNjdDNi4xNTg1MiA1LjQ0MzY3IDUuMzkzNTYgNS42Mjc0MSA1LjMzMjMxIDYuNTI5ODdDNS4yNzQ4MSA3LjM4MTA3IDUuODU2MDMgNy44Mzg1NSA2LjQzOTc1IDcuOTc4NTRDNy4xNjA5NiA4LjE1MjI4IDcuOTc4NDIgNy45NTQ3OSA4LjE3ODQxIDcuMDM0ODRDOC4zNDQ2NSA2LjI3NzM4IDcuOTQyMTcgNS45MDExNSA3Ljk0MjE3IDUuOTAxMTVaIiBmaWxsPSIjMzAzMDMwIi8+CjxwYXRoIGQ9Ik02LjczOTgzIDQuNzUzNjJDNi42NzEwOSA1LjAxMjM1IDYuODA4NTggNS4yNjIzNCA3LjA3ODU3IDUuMzMxMDlDNy4zNjk4IDUuNDA0ODMgNy42MzQ3OSA1LjMwODU5IDcuNzA2MDMgNS4wMTExQzcuNzY4NTMgNC43NDczNyA3LjY0MzU0IDQuNTE0ODggNy4zMzYwNSA0LjQzOTg4QzcuMDgzNTcgNC4zNzczOSA2LjgxNDgzIDQuNDcxMTMgNi43Mzk4MyA0Ljc1MzYyWiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTYuOTU5NzggNi4wMzk3NEM2LjYzMjMgNS45Mzg0OSA2LjE5OTgyIDYuMDY0NzMgNi4xMzEwNyA2LjUwNDcxQzYuMDYyMzMgNi45NDQ2OSA2LjMyNjA2IDcuMTY5NjggNi42NzEwNCA3LjIzMjE3QzcuMDE2MDMgNy4yOTQ2NyA3LjM0MjI2IDcuMTEzNDMgNy40MDYwMSA2Ljc2MDk1QzcuNDY4NSA2LjQwOTcyIDcuMjg2MDEgNi4xMzk3MyA2Ljk1OTc4IDYuMDM5NzRaIiBmaWxsPSJ3aGl0ZSIvPgo8L3N2Zz4K","variables":{"environment":{"allSelected":true,"description":"Deployment environment filter","dynamicVariablesAttribute":"deployment.environment","dynamicVariablesSource":"Metrics","id":"var-environment","multiSelect":true,"name":"environment","order":0,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"},"k8s_cluster":{"allSelected":true,"description":"Kubernetes cluster name filter","dynamicVariablesAttribute":"k8s.cluster.name","dynamicVariablesSource":"Metrics","id":"var-k8s-cluster","multiSelect":true,"name":"k8s_cluster","order":1,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"},"k8s_pod":{"allSelected":true,"description":"Pod name filter — select specific pods to scope all panels to one application","dynamicVariablesAttribute":"k8s.pod.name","dynamicVariablesSource":"Metrics","haveCustomValuesSelected":false,"id":"var-k8s-pod","multiSelect":true,"name":"k8s_pod","order":2,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"}},"widgets":[{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"% of process limit in use. >85% is critical — node will crash at 100%.","fillMode":"none","fillSpans":false,"id":"w1","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_process_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"c9912465-de59-4645-b89d-8c6625222c29","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":100,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[{"index":"0","thresholdColor":"#FFA500","thresholdFormat":"Background","thresholdLabel":"Warning","thresholdOperator":">","thresholdValue":70},{"index":"1","keyIndex":1,"thresholdColor":"#FF0000","thresholdFormat":"Background","thresholdLabel":"Critical","thresholdOperator":">","thresholdValue":85}],"timePreferance":"GLOBAL_TIME","title":"Process Utilization","yAxisUnit":"percent"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"% of port limit in use. Ports = sockets, files, NIFs.","fillMode":"none","fillSpans":false,"id":"w2","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_port_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_port_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"4e912e40-34ac-43e1-abec-0059a1d54bd2","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":100,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[{"index":"0","thresholdColor":"#FFA500","thresholdFormat":"Background","thresholdLabel":"Warning","thresholdOperator":">","thresholdValue":70},{"index":"1","keyIndex":1,"thresholdColor":"#FF0000","thresholdFormat":"Background","thresholdLabel":"Critical","thresholdOperator":">","thresholdValue":85}],"timePreferance":"GLOBAL_TIME","title":"Port Utilization","yAxisUnit":"percent"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"% of atom table used. Atom table never shrinks — runaway growth = leak.","fillMode":"none","fillSpans":false,"id":"w3","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_atom_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_atom_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"42aacb5f-0752-4d92-a370-412be4a007ec","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":100,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[{"index":"0","thresholdColor":"#FFA500","thresholdFormat":"Background","thresholdLabel":"Warning","thresholdOperator":">","thresholdValue":70},{"index":"1","keyIndex":1,"thresholdColor":"#FF0000","thresholdFormat":"Background","thresholdLabel":"Critical","thresholdOperator":">","thresholdValue":85}],"timePreferance":"GLOBAL_TIME","title":"Atom Utilization","yAxisUnit":"percent"},{"contextLinks":{"linksData":[]},"description":"Number of ETS tables currently open.","id":"w4","nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_ets_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"31a03bbe-951d-4804-b963-6467fbf4cfa3","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"ETS Tables","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"CPU scheduler threads available to the BEAM.","id":"w5","nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_schedulers_online","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"1818e786-b85d-486e-a178-2e74f3787812","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Schedulers Online","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Total BEAM memory allocation (processes + system).","id":"w6","nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"034d759b-9272-4b5d-acbc-a8ecbcca65da","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Total Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"w7","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Memory"},{"contextLinks":{"linksData":[]},"description":"","id":"w8","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Total","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"a9d086d8-f2fe-471e-acf9-4e98882a5639","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Total Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Stacked composition of BEAM memory — watch for any segment growing unbounded.","id":"w9","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_processes_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Processes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_memory_system_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"System","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_memory_atom_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Atom","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"beam_memory_binary_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"D","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Binary","orderBy":[],"queryName":"D","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"beam_memory_ets_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"E","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"ETS","orderBy":[],"queryName":"E","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"beam_memory_code_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"F","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Code","orderBy":[],"queryName":"F","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"deeddc90-b07a-46cd-839d-f21ac488d28d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Memory Breakdown","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Steady monotonic growth indicates a binary leak (unreferenced binaries not GC'd).","id":"w10","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"beam_memory_binary_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Binary","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"718c7582-29a1-4453-b16e-8949bb23f0ff","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Binary Memory (off-heap)","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"w11","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Schedulers & Run Queues"},{"contextLinks":{"linksData":[]},"description":"Normal scheduler run queue. Sustained > 0 = CPU saturation.","id":"w12","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_run_queues_length","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Run Queue","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"51a166ed-a243-48ac-8470-bda62852a44d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Run Queue Length","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Dirty CPU = long-running NIF pressure. Dirty IO = file/socket I/O pressure.","id":"w13","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_dirty_cpu_run_queue_length","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dirty CPU","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_statistics_dirty_io_run_queue_length","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dirty IO","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"9b84336d-4a49-4741-8415-f0234a989666","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Dirty CPU / IO Run Queues","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Total Erlang processes. Runaway growth = process leak (missing termination or gen_server loop).","id":"w14","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Processes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"da910f90-465d-4865-b6cb-02a7e33de9d9","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Live Processes","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"w15","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"GC & Reductions"},{"contextLinks":{"linksData":[]},"description":"Garbage collections per second. Spikes indicate sudden allocation pressure.","id":"w16","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_number_of_gcs","spaceAggregation":"sum","temporality":"cumulative","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"GCs/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"42d2abba-afb3-437b-bcd4-2ca0584f1126","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Rate","yAxisUnit":"reqps"},{"contextLinks":{"linksData":[]},"description":"Bytes reclaimed per second by the garbage collector.","id":"w17","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_bytes_reclaimed","spaceAggregation":"sum","temporality":"cumulative","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Bytes/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"f3225155-7370-4922-b824-8c259bdcf563","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Bytes Reclaimed","yAxisUnit":"binBps"},{"contextLinks":{"linksData":[]},"description":"Scheduler context switches per second. High rate = many short-lived processes.","id":"w18","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_context_switches","spaceAggregation":"sum","temporality":"cumulative","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Switches/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"7c57ba19-7d01-4843-b418-72abab7d6201","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Context Switch Rate","yAxisUnit":"reqps"},{"contextLinks":{"linksData":[]},"description":"BEAM reductions per second — proxy for total CPU work across all processes.","id":"w19","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_reductions_total","spaceAggregation":"sum","temporality":"cumulative","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Reductions/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"6a213747-fccf-44b6-b393-abae8e3a856e","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Reductions Rate","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":">85% is critical — BEAM refuses to spawn new processes at the limit.","fillMode":"none","fillSpans":false,"id":"d0d18ace-cca9-4d5a-8440-86be0469022d","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","reduceTo":"avg","spaceAggregation":"max","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[{"dataType":"string","id":"k8s.pod.name--string--tag","isColumn":false,"isJSON":false,"key":"k8s.pod.name","type":"tag"}],"having":{"expression":""},"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_process_limit","reduceTo":"avg","spaceAggregation":"max","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[{"dataType":"string","id":"k8s.pod.name--string--tag","isColumn":false,"isJSON":false,"key":"k8s.pod.name","type":"tag"}],"having":{"expression":""},"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"disabled":false,"expression":"A / B * 100","legend":"{{k8s.pod.name}}","queryName":"F1"}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"6fb58024-22cd-42d0-9865-f89351f3fbbe","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Process Utilization %","yAxisUnit":"%"}],"layout":[{"h":3,"i":"w1","moved":false,"static":false,"w":2,"x":0,"y":0},{"h":3,"i":"w2","moved":false,"static":false,"w":2,"x":2,"y":0},{"h":3,"i":"w3","moved":false,"static":false,"w":2,"x":4,"y":0},{"h":3,"i":"w4","moved":false,"static":false,"w":2,"x":6,"y":0},{"h":3,"i":"w5","moved":false,"static":false,"w":2,"x":8,"y":0},{"h":3,"i":"w6","moved":false,"static":false,"w":2,"x":10,"y":0},{"h":1,"i":"w7","moved":false,"static":false,"w":12,"x":0,"y":3},{"h":7,"i":"w8","moved":false,"static":false,"w":4,"x":0,"y":4},{"h":7,"i":"w9","moved":false,"static":false,"w":4,"x":4,"y":4},{"h":7,"i":"w10","moved":false,"static":false,"w":4,"x":8,"y":4},{"h":1,"i":"w11","moved":false,"static":false,"w":12,"x":0,"y":11},{"h":7,"i":"w12","moved":false,"static":false,"w":6,"x":0,"y":12},{"h":7,"i":"w13","moved":false,"static":false,"w":6,"x":6,"y":12},{"h":7,"i":"d0d18ace-cca9-4d5a-8440-86be0469022d","moved":false,"static":false,"w":6,"x":0,"y":19},{"h":7,"i":"w14","moved":false,"static":false,"w":6,"x":6,"y":19},{"h":1,"i":"w15","moved":false,"static":false,"w":12,"x":0,"y":26},{"h":7,"i":"w16","moved":false,"static":false,"w":3,"x":0,"y":27},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":3,"y":27},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":6,"y":27},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":9,"y":27}],"panelMap":{"w11":{"collapsed":false,"widgets":[{"h":7,"i":"w12","moved":false,"static":false,"w":6,"x":0,"y":12},{"h":7,"i":"w13","moved":false,"static":false,"w":6,"x":6,"y":12},{"h":7,"i":"w14","moved":false,"static":false,"w":12,"x":0,"y":19},{"h":1,"i":"w15","moved":false,"static":false,"w":12,"x":0,"y":26},{"h":7,"i":"w16","moved":false,"static":false,"w":3,"x":0,"y":27},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":3,"y":27},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":6,"y":27},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":9,"y":27}]},"w7":{"collapsed":false,"widgets":[{"h":7,"i":"w8","moved":false,"static":false,"w":4,"x":0,"y":4},{"h":7,"i":"w9","moved":false,"static":false,"w":4,"x":4,"y":4},{"h":7,"i":"w10","moved":false,"static":false,"w":4,"x":8,"y":4},{"h":1,"i":"w11","moved":false,"static":false,"w":12,"x":0,"y":11},{"h":7,"i":"w12","moved":false,"static":false,"w":6,"x":0,"y":12},{"h":7,"i":"w13","moved":false,"static":false,"w":6,"x":6,"y":12},{"h":7,"i":"w14","moved":false,"static":false,"w":12,"x":0,"y":19},{"h":1,"i":"w15","moved":false,"static":false,"w":12,"x":0,"y":26},{"h":7,"i":"w16","moved":false,"static":false,"w":3,"x":0,"y":27},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":3,"y":27},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":6,"y":27},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":9,"y":27}]}},"version":"v5","title":"BEAM / Erlang VM - Overview"} diff --git a/beam/README.md b/beam/README.md new file mode 100644 index 00000000..f54d4ba0 --- /dev/null +++ b/beam/README.md @@ -0,0 +1,61 @@ +# BEAM / Erlang VM Dashboards + +Two dashboards for monitoring the BEAM Erlang/Elixir runtime. Both are generic — they work with any BEAM project (Elixir or Erlang) that exposes metrics via [`prometheus_ex`](https://hex.pm/packages/prometheus_ex). Import each JSON file directly into SigNoz. + +**Template variables (shared):** `environment` · `k8s_cluster` · `k8s_pod` — all multi-select, scoped to metrics dimensions. + +--- + +## BEAM / Erlang VM - Overview + +**File:** `BEAM-ErlangVM-Overview.json` +**Tags:** `beam` `erlang` `elixir` `overview` + +First stop for on-call triage. Shows the key runtime health indicators at a glance — saturation ratios, memory pressure, scheduler backlog, and GC activity. + +| Section | Panels | +|-----------------------------|---------------------------------------------------------------------------------------------------------------------| +| **Stats ** | Process Utilization %, Port Utilization %, Atom Utilization %, ETS Tables, Schedulers Online, Total Memory | +| **Memory** | Total Memory, Memory Breakdown (stacked: Processes / System / Atom / Binary / ETS / Code), Binary Memory (off-heap) | +| **Schedulers & Run Queues** | Run Queue Length, Dirty CPU / IO Run Queues, Process Utilization % (per pod), Live Processes | +| **GC & Reductions** | GC Rate, GC Bytes Reclaimed, Context Switch Rate, Reductions Rate | + +Stat panels include warning (>70%) and critical (>85%) thresholds. Process/Port/Atom utilization exceeding 85% indicates imminent node exhaustion. + +--- + +## BEAM / Erlang VM - Deep Dive + +**File:** `BEAM-ErlangVM-DeepDive.json` +**Tags:** `beam` `erlang` `elixir` `debug` + +Per-subsystem breakdown for root-cause investigation. Use this after the Overview flags an anomaly. Covers memory allocator internals, MSACC scheduler microstate accounting, Erlang distribution link health, and Mnesia. + +| Section | Panels | +|---------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Process, Port & Atom Saturation** | Process Count vs Limit, Process Utilization %, Port Count vs Limit, Port Utilization %, Atom Count vs Limit, Atom Utilization % | +| **Memory Detail** | Process Heap Memory, System Memory, Atom Memory, Binary Memory (off-heap), ETS Memory, Code Memory | +| **ETS Tables** | ETS Table Count, ETS Table Limit, ETS Utilization %, DETS (Disk ETS) Table Count | +| **MSACC Scheduler Microstate** | aux / check_io / emulator / gc, port / nif / bif / send, sleep, alloc / other / timers | +| **GC & I/O** | GC Count Rate, GC Bytes Reclaimed Rate, GC Words Reclaimed Rate, I/O Bytes In/Out Rate, Context Switches Rate, Reductions Rate | +| **Memory Allocator Fragmentation** | MBCS Pool Block Size & Count, SBC Block Size, MBC Fragmentation %, MBC Block vs Carrier Size | +| **Erlang Distribution Link** | Distribution Bytes Sent/Received Rate, Distribution Send Pending | +| **Distribution Messages** | Distribution Messages Sent/Received Rate, Distribution Packets Sent/Received Rate | +| **Mnesia** | Held Locks, Lock Queue Depth, Tx Restart/Abort/Commit/Logged/Failed Rates, Memory Usage, Unsynchronised Tables, Subscriptions, Running/Stopped DB Nodes | +| **Mnesia Extended** | Held Locks (detail), Subscriptions by Type, Transaction Health Overview, Mnesia vs VM Memory, Node Status, Unsync Tables vs Held Locks, Active Transaction Coordinators | + +### Key signals to watch + +- **MSACC `sleep` high** → schedulers are idle; check I/O wait or under-provisioned concurrency. +- **MSACC `nif` high** → long-running NIFs blocking schedulers. +- **Binary Memory monotonically growing** → binary leak (processes holding large off-heap binary references). +- **MBC Fragmentation % rising** → memory allocator fragmentation; consider allocator tuning or restart. +- **Distribution Send Pending > 0 sustained** → backpressure on an Erlang cluster link. +- **Mnesia Unsynchronised Tables > 0** → partitioned or lagging replica; check node connectivity. + +--- + +## Prerequisites + +- Metrics exported via [`prometheus_ex`](https://hex.pm/packages/prometheus_ex) (or compatible exporter). +- Metrics must be ingested into SigNoz with the resource attributes `deployment.environment`, `k8s.cluster.name`, and `k8s.pod.name` populated for the template variable filters to work. From 7b5b3d9f69d22747a097c045f4b4be77c1ba21ca Mon Sep 17 00:00:00 2001 From: Sebastian YEPES Date: Tue, 16 Jun 2026 22:46:48 +0200 Subject: [PATCH 2/2] fix the panel sections --- beam/BEAM-ErlangVM-DeepDive.json | 9292 +++++++++++++++++++++++++++++- beam/BEAM-ErlangVM-Overview.json | 2982 +++++++++- 2 files changed, 12272 insertions(+), 2 deletions(-) diff --git a/beam/BEAM-ErlangVM-DeepDive.json b/beam/BEAM-ErlangVM-DeepDive.json index 65a8fc54..ce0434be 100644 --- a/beam/BEAM-ErlangVM-DeepDive.json +++ b/beam/BEAM-ErlangVM-DeepDive.json @@ -1 +1,9291 @@ -{"description":"Per-subsystem breakdown of BEAM runtime internals: process/port/atom saturation detail, memory segments, ETS, MSACC scheduler microstate, GC, I/O, memory allocator fragmentation, Erlang distribution link health, and Mnesia. Generic for any Elixir/Erlang project using prometheus_ex.","name":"BEAM / Erlang VM - Deep Dive","tags":["beam","erlang","elixir","debug"],"image":"data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTgiIHZpZXdCb3g9IjAgMCAxOCAxOCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0Ljk0OTQgMTMuOTQyQzE2LjIzMTggMTIuNDI1OCAxNy4zMjY4IDkuNzAyMiAxNi4xOTU2IDYuNTc0ODdDMTUuNjQ0MyA1LjA1MjQ1IDE1LjAyMTkgNC4yMDI0OSAxNC4yOTY5IDMuNjYyNTJDMTMuODU1NyAzLjMzMzc5IDEyLjA5MzMgMi41MDYzMyA5Ljc1OTY1IDIuODY3NTZDOC4wNTM0OSAzLjEzMjU1IDUuNzc0ODcgNC4yMDg3NCA0LjI5MzY5IDUuOTU5OUMyLjg1NzUyIDcuNjYxMDYgMS43NDg4MyA5LjAwNDc0IDEuNjk3NTggMTAuMzA5N0MxLjYzMTMzIDExLjk4ODMgMi44OTYyNyAxMy40MzA4IDMuMDUwMDEgMTMuNjY0NUMzLjMyMzc0IDE0LjA3OTUgNS4xOTExNSAxNi40NTE4IDguNjk5NzEgMTYuNTczMUMxMS43OTcgMTYuNjc5MyAxMy44MTQ0IDE1LjI4NDQgMTQuOTQ5NCAxMy45NDJaIiBmaWxsPSIjNDAzRDNFIi8+CjxwYXRoIGQ9Ik00LjU1MzYzIDIuNzM3NDdDMi45Mzc0NiAzLjg5MTE2IDEuMTIxMzEgNi4yNTEwMyAxLjQ0NzU0IDkuNTYwODZDMS42MDYyOCAxMS4xNzIgMi4wMDI1MSAxMi4xNDk1IDIuNTcxMjMgMTIuODUwN0MyLjkxNzQ2IDEzLjI3ODIgNC40MTk4OCAxNC41NDkzIDYuNzczNTEgMTQuNzM2OEM5LjE0NTg4IDE0LjkyNTYgMTAuOTQ5NSAxNC4zOTQ0IDEyLjgzMzIgMTMuMDg0NEMxNi42NjE3IDEwLjQyMDggMTYuMDk4IDYuMzkzNTMgMTUuOTM0MyA1LjkyNDhDMTUuNzcwNSA1LjQ1NjA3IDE0LjU0NDQgMi42OTYyMiAxMS4xNzMzIDEuNzE1MDJDOC4xOTg0NCAwLjg1MDA2OCA1Ljk4MzU1IDEuNzE1MDIgNC41NTM2MyAyLjczNzQ3WiIgZmlsbD0iIzVFNjM2NyIvPgo8cGF0aCBkPSJNNy4zOTM1MyAyLjk2MTA5QzUuNjE3MzcgMi44OTczNCAzLjkxOTk2IDQuMjg4NTIgMy43NTYyMiA2LjAwNTkzQzMuNTkyNDggNy43MjIwOSA0LjY1NDkyIDkuMDI5NTIgNi4zMDk4MyA5LjI5NTc2QzcuOTY0NzUgOS41NjA3NCA5Ljg3ODM5IDguNTU1OCAxMC4yNjM0IDYuNDUwOTFDMTAuNjYwOSA0LjI4MjI3IDkuMDg5NjkgMy4wMjIzNCA3LjM5MzUzIDIuOTYxMDlaIiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBkPSJNNy45NDIxNyA1LjkwMTE1QzcuOTQyMTcgNS45MDExNSA4LjM2OTY1IDUuODEyNCA4LjQ1NDY1IDUuMTgyNDRDOC41MzgzOSA0LjU2MjQ3IDguMjMwOTEgNC4wMzM3NSA3LjUxMzQ1IDMuODQzNzZDNi43MzM0OSAzLjYzNzUyIDYuMjA0NzcgNC4wNjYyNSA2LjA2NzI3IDQuNTE3NDdDNS44NzYwMyA1LjE0NDk0IDYuMTU4NTIgNS40NDM2NyA2LjE1ODUyIDUuNDQzNjdDNi4xNTg1MiA1LjQ0MzY3IDUuMzkzNTYgNS42Mjc0MSA1LjMzMjMxIDYuNTI5ODdDNS4yNzQ4MSA3LjM4MTA3IDUuODU2MDMgNy44Mzg1NSA2LjQzOTc1IDcuOTc4NTRDNy4xNjA5NiA4LjE1MjI4IDcuOTc4NDIgNy45NTQ3OSA4LjE3ODQxIDcuMDM0ODRDOC4zNDQ2NSA2LjI3NzM4IDcuOTQyMTcgNS45MDExNSA3Ljk0MjE3IDUuOTAxMTVaIiBmaWxsPSIjMzAzMDMwIi8+CjxwYXRoIGQ9Ik02LjczOTgzIDQuNzUzNjJDNi42NzEwOSA1LjAxMjM1IDYuODA4NTggNS4yNjIzNCA3LjA3ODU3IDUuMzMxMDlDNy4zNjk4IDUuNDA0ODMgNy42MzQ3OSA1LjMwODU5IDcuNzA2MDMgNS4wMTExQzcuNzY4NTMgNC43NDczNyA3LjY0MzU0IDQuNTE0ODggNy4zMzYwNSA0LjQzOTg4QzcuMDgzNTcgNC4zNzczOSA2LjgxNDgzIDQuNDcxMTMgNi43Mzk4MyA0Ljc1MzYyWiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTYuOTU5NzggNi4wMzk3NEM2LjYzMjMgNS45Mzg0OSA2LjE5OTgyIDYuMDY0NzMgNi4xMzEwNyA2LjUwNDcxQzYuMDYyMzMgNi45NDQ2OSA2LjMyNjA2IDcuMTY5NjggNi42NzEwNCA3LjIzMjE3QzcuMDE2MDMgNy4yOTQ2NyA3LjM0MjI2IDcuMTEzNDMgNy40MDYwMSA2Ljc2MDk1QzcuNDY4NSA2LjQwOTcyIDcuMjg2MDEgNi4xMzk3MyA2Ljk1OTc4IDYuMDM5NzRaIiBmaWxsPSJ3aGl0ZSIvPgo8L3N2Zz4K","variables":{"environment":{"allSelected":true,"description":"Deployment environment filter","dynamicVariablesAttribute":"deployment.environment","dynamicVariablesSource":"Metrics","id":"var-environment","multiSelect":true,"name":"environment","order":0,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"},"k8s_cluster":{"allSelected":true,"description":"Kubernetes cluster name filter","dynamicVariablesAttribute":"k8s.cluster.name","dynamicVariablesSource":"Metrics","id":"var-k8s-cluster","multiSelect":true,"name":"k8s_cluster","order":1,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"},"k8s_pod":{"allSelected":true,"description":"Pod name filter — select specific pods to scope all panels to one application","dynamicVariablesAttribute":"k8s.pod.name","dynamicVariablesSource":"Metrics","id":"var-k8s-pod","multiSelect":true,"name":"k8s_pod","order":2,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"}},"widgets":[{"contextLinks":{"linksData":[]},"description":"","id":"r1","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Process, Port & Atom Saturation"},{"contextLinks":{"linksData":[]},"description":"Live processes (solid) vs hard limit (dashed). Gap = headroom. Shrinking gap = leak or load spike.","id":"w1","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Process Count","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_process_limit","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Process Limit","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"d745aedc-ca2a-480e-b538-2782ea4fbbb9","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Process Count vs Limit","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":">85% is critical — BEAM refuses to spawn new processes at the limit.","fillMode":"none","fillSpans":false,"id":"w2","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_process_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"d9b4c6b7-a492-42d0-a64d-55a3ec119ab2","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Process Utilization %","yAxisUnit":"percent"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"Ports represent open sockets, files, and NIF handles. Exhaustion causes connection failures.","fillMode":"none","fillSpans":false,"id":"w3","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_port_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":false,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":{"expression":""},"legend":"Port Count","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_port_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":false,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":{"expression":""},"legend":"Port Limit","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"ea4de080-95d9-46d5-bd1a-51c637c13eee","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Port Count vs Limit","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"Port exhaustion does not crash the node but causes {error, system_limit} on new connections.","fillMode":"none","fillSpans":false,"id":"w4","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_port_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_port_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"a2163961-8c39-4672-8e29-996dbb26665a","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Port Utilization %","yAxisUnit":"percent"},{"contextLinks":{"linksData":[]},"description":"Atom table is global and never shrinks. Monotonic growth without restart = atom leak (dynamic atom creation).","id":"w5","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_atom_count","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Atom Count","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_atom_limit","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Atom Limit","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"cdb7c126-a228-4297-888c-4d5dfc65a4cc","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Atom Count vs Limit","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r2","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Memory Detail"},{"contextLinks":{"linksData":[]},"description":"Memory allocated to all Erlang process stacks and heaps. Growing without a matching process count increase = per-process memory leak.","id":"w6","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_processes_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Process Heap","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"dafc56fc-e40c-4ed7-ae40-3ab34b1464ec","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Process Heap Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"BEAM VM system memory: ETS, code, stack, and VM overhead. Excludes process heap.","id":"w7","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_system_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"System","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c04b416e-405d-4773-84fd-a6cfbdd5072d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"System Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Memory consumed by the atom table. Correlated with atom count.","id":"w8","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_atom_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Atom Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"93a93556-2769-4387-8302-6119c7b5ff40","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Atom Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Off-heap reference-counted binaries. Steady monotonic growth between GCs = binary leak (processes holding large binary references).","id":"w9","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"beam_memory_binary_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Binary Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"6d8ca821-9676-4b7c-bfc8-56afaca9d9ef","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Binary Memory (off-heap)","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Total memory used by all ETS tables. Correlates with table count and row density.","id":"w10","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"beam_memory_ets_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"ETS Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"2184b14f-5e55-4a8f-af90-3451a7583b4b","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"ETS Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Memory occupied by loaded BEAM modules. Significant spikes after hot upgrades or repeated code:load calls.","id":"w11","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"beam_memory_code_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Code Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"80db8391-3a5f-47b4-9ada-912df39e90d4","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Code Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"r3","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"ETS Tables"},{"contextLinks":{"linksData":[]},"description":"Total open ETS tables. Steady growth without a corresponding application load increase = table leak (missing :ets.delete).","id":"w12","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_ets_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"ETS Tables","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"9397af8e-b8a7-4cd9-90c0-0bddb580126d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"ETS Table Count","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Hard ceiling for ETS tables (default 8192). Reference line — compare against ETS Count.","id":"w13","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_ets_limit","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"ETS Limit","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c0295751-2141-489e-8cdb-977960827e95","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"ETS Table Limit","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"ETS tables used vs limit. >80% warrants investigation into table ownership and lifecycle.","fillMode":"none","fillSpans":false,"id":"w14","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_ets_tables","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_ets_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"9b50b63a-c493-427c-bdec-ba3129d7dac1","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"ETS Utilization %","yAxisUnit":"percent"},{"contextLinks":{"linksData":[]},"description":"Disk-backed ETS tables. Normally 0 in most apps. Unexpected growth = open disk tables not being closed.","id":"w16","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_dets_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"DETS Tables","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"4d1ad25d-fe40-474f-982f-10e8f188388a","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"DETS (Disk ETS) Table Count","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r4","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"MSACC Scheduler Microstate"},{"contextLinks":{"linksData":[]},"description":"Fraction of scheduler time each MSACC state consumes. 'aux' + 'check_io' + 'emulator' should dominate; high 'sleep' = idle schedulers; high 'gc' = GC pressure.","id":"w17","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_msacc_aux_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"aux","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_check_io_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"check_io","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_emulator_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"emulator","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_gc_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"D","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"gc","orderBy":[],"queryName":"D","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"1c540b59-0c75-4c3d-ae9b-2d49a6f2ea22","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MSACC: aux / check_io / emulator / gc","yAxisUnit":"s"},{"contextLinks":{"linksData":[]},"description":"Scheduler time in port / nif / bif / send. Elevated nif = heavy NIF calls blocking schedulers; elevated send = message-passing saturation.","id":"w18","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_msacc_port_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"port","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_nif_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"nif","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_bif_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"bif","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_send_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"D","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"send","orderBy":[],"queryName":"D","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"5dad1b84-abbb-4efb-98d3-3c23e3f97fb6","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MSACC: port / nif / bif / send","yAxisUnit":"s"},{"contextLinks":{"linksData":[]},"description":"Scheduler time asleep. High sleep = under-utilised schedulers (normal during low load) or blocked waiting on I/O.","id":"w19","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_msacc_sleep_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"sleep","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c77bc2c8-af64-47ec-bc27-0f12bdb7b00c","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MSACC: sleep","yAxisUnit":"s"},{"contextLinks":{"linksData":[]},"description":"Scheduler time in alloc / other / timers. Elevated alloc = heavy allocation pressure; elevated timers = many active timer references.","id":"w20","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_msacc_alloc_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"alloc","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_other_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"other","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_msacc_timers_seconds_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"timers","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"e25c0a42-7f8d-4048-b5f1-e78c17b61c3c","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MSACC: alloc / other / timers","yAxisUnit":"s"},{"contextLinks":{"linksData":[]},"description":"","id":"r5","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"GC & I/O"},{"contextLinks":{"linksData":[]},"description":"GC runs per second. Short spikes are normal; sustained high rates with rising heap = object churn.","id":"w21","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_number_of_gcs","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"GC Count Rate","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"0a8cc42d-7082-4ee3-b35e-7a62f9a0e32b","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Count Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Bytes reclaimed per second by GC. High reclaim with high GC count = frequent short-lived allocation churn.","id":"w22","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_bytes_reclaimed","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Bytes Reclaimed/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"5e52a97e-1c5c-4b7b-a1c5-17d26640dbd5","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Bytes Reclaimed Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Words moved across generations per GC cycle. Large word counts = major GC activity; correlate with pauses.","id":"w23","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_words_reclaimed","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Words Reclaimed/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"4db37a6d-e10e-40ba-b9a8-6ea32c9c78e5","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Words Reclaimed Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Raw bytes received by all ports (sockets, pipes). Sustained high read throughput is normal; sudden drops = upstream pressure.","id":"w24","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_bytes_received_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Bytes In/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"3b4de0fd-f6d1-42be-9f36-2e91b7bde0d0","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"I/O Bytes In Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Raw bytes sent by all ports. Sustained high write throughput is normal; compare with in-rate for asymmetry.","id":"w25","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_bytes_output_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Bytes Out/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"5de6b8d3-a5bd-4b1f-8ee5-fd08b64aa9a7","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"I/O Bytes Out Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Total context switches across all schedulers per second. Sudden step-changes correlate with process spawning bursts or supervisor restarts.","id":"w26","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_context_switches","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Context Switches/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"b2da73e4-0a0e-4ead-8a03-e1c10b4fa218","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Context Switches Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Total reductions (function calls + tail calls) per second. Steady baseline; spikes indicate CPU-intensive work or hot loops.","id":"w27","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_reductions_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Reductions/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"aa98a5ca-e975-48e4-9ce8-c6ea2f524e79","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Reductions Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r6","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Memory Allocator Fragmentation"},{"contextLinks":{"linksData":[]},"description":"Bytes allocated by the BEAM allocator vs bytes actually used by live data. Gap = fragmentation overhead.","id":"w28","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_pool_blocks_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} pool_blocks_size","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"6bd1e624-1f66-4dc5-88c9-b0893b6c0bde","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Allocator MBCS Pool Block Size","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Number of multi-block carriers in the pool. Growing without load growth = fragmented carrier accumulation.","id":"w29","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_pool_blocks","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} pool_blocks","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"a9a19bb3-5069-441b-aece-5a3199b48720","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Allocator MBCS Pool Block Count","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Single-block carriers created by the allocator. SBC growth = large allocations exceeding MBC threshold; consider +MBcst tuning.","id":"w30","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_sbcs_blocks_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} sbc_blocks_size","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c44f2174-b40b-4b00-ba97-b3fe55f9a1a1","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Allocator SBC Block Size","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Fragmentation ratio = (allocated − used) / allocated. Values >40% sustained = fragmented memory; consider allocator flags or node restart.","id":"w31","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_block_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_carriers_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","expression":"(B - A) / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"id":"4e48e5b5-6e2f-489e-a2e7-72c76b90ae74","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":100,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MBC Fragmentation %","yAxisUnit":"percent"},{"contextLinks":{"linksData":[]},"description":"Allocated carrier size (total reserved) vs block size (live data). Wide gap = wasted carrier memory.","id":"w32","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_block_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} block","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_allocator_mbcs_carriers_size","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"allocator--string--tag","isColumn":false,"key":"allocator","type":"tag"}],"legend":"{{allocator}} carrier","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"b39cc3d5-6acb-4a8e-883d-84be6c7f6462","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"MBC Block vs Carrier Size","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"r7","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Erlang Distribution Link"},{"contextLinks":{"linksData":[]},"description":"Distribution bytes sent per second to remote nodes. Sudden drop = node disconnect or network partition.","id":"w33","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_send_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Bytes Sent/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"d96a4d31-80fe-415c-b2c7-a4fc1c8e86e6","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Bytes Sent Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Distribution bytes received per second from remote nodes. Spikes = large remote message deliveries or data transfers.","id":"w34","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_recv_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Bytes Recv/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"4e22b2db-83b8-4ad2-8a38-ef81f89c4eb0","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Bytes Received Rate","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"r8","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Distribution Messages"},{"contextLinks":{"linksData":[]},"description":"Messages sent across distribution links per second. High rate + growing queues = dist buffer saturation risk.","id":"w35","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_send_messages_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Messages Sent/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"f8f4c226-0c52-4d8f-8eae-f5c41a8e5e3d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Messages Sent Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Messages received over distribution links per second. Asymmetry with sent rate = one-way traffic imbalance.","id":"w36","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_recv_messages_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Messages Recv/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c17ca374-f79d-4eac-9770-4dbf19da4c37","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Messages Received Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Pending bytes in the distribution send buffer. Sustained non-zero values = congested distribution link.","id":"w37","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_send_pending_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Pending Send Bytes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"8e73f8ea-c6a9-4bef-bd86-adde23c8e4ca","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Send Pending","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Packets sent across distribution links. Low-frequency, high-byte packets = large message transfers.","id":"w38","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_send_packets_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Packets Sent/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"1e4d4d9f-b8a0-4b12-af6f-13c4cd9e0f4d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Packets Sent Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Packets received over distribution links. Compare with sent rate and bytes for flow analysis.","id":"w39","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_dist_recv_packets_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dist Packets Recv/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"c01c7098-49c3-4dd4-834e-e7ed9f97e8a0","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Distribution Packets Received Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r9","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Mnesia"},{"contextLinks":{"linksData":[]},"description":"Active Mnesia transactions. Sustained high counts = long-running transactions blocking table locks.","id":"w40","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_held_locks","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Held Locks","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"d36ff9fa-7e51-492e-aa81-61f5a7c5f7e3","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Held Locks","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Processes waiting to acquire Mnesia locks. Non-zero = lock contention; high values = potential deadlock or slow transactions.","id":"w41","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_lock_queue","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Lock Queue","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"2ec3b61e-2065-4eef-a6d3-dc9219ef6a05","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Lock Queue Depth","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia transaction restarts due to lock conflicts. Elevated restart rate = high write contention; consider table sharding or optimistic locking.","id":"w42","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_restarts","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Transaction Restarts/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"e7f3c17a-59e4-4e54-9e22-71b6d4c4d25a","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Transaction Restart Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia transactions aborted per second. Aborts can be caused by conflicts, node failures, or application errors.","id":"w43","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_aborts","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Transaction Aborts/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"bab96b77-1e6e-4fbb-a7e7-61f88a5cd07f","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Transaction Abort Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia transactions committed per second. Baseline for throughput; compare with abort and restart rates for health ratio.","id":"w44","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_commits","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Transaction Commits/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"addd1a97-d7dc-42e8-9424-2cc26789d04d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Transaction Commit Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia log entries written per second. Disk-based tables only. High log rate + low commit rate = replay overhead.","id":"w45","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_logged_transactions","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Logged Transactions/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"deea3f37-9c5b-4d99-82b4-95e285c1d47f","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Logged Transactions Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Failed Mnesia writes. Any non-zero value = data integrity issue or schema mismatch.","id":"w46","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_failed_transactions","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Failed Transactions/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"66cc4fb5-cd2a-4f42-aba6-a6c0e6c15e95","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Failed Transaction Rate","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables currently open. Steady count = stable schema; sudden drop = node partition or table deletion.","id":"w47","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_memory_usage_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Memory Usage","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"0e88bf72-e297-424b-b4a4-3aef8bd5e5e5","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Memory Usage","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables with unsynchronised data across nodes. Non-zero = split-brain risk. Should return to zero after healing.","id":"w48","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_unsynchronised_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Unsynchronised Tables","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"43c0c24d-1ecf-4af7-83e4-b8c4d7f3a3e3","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Unsynchronised Tables","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Subscribed Mnesia event listeners. Unexpected drop = subscriber process crashed.","id":"w49","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_subscriptions","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Subscriptions","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"0d9dc6bf-9b77-4b5a-a5bf-8975c38fe9d1","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Subscriptions","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Number of connected Mnesia cluster nodes. Drop below expected count = node departure or partition.","id":"w50","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_db_nodes_running","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Running DB Nodes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"bb0d8b63-fa29-4f70-a6a6-7f64fc46dca9","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Running DB Nodes","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"r10","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Mnesia Extended"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables registered in the schema. Sudden change = schema migration or table deletion.","id":"w51","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_db_nodes_stopped","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Stopped DB Nodes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"8b57d1ef-ef46-4c5f-b8ef-73f52f3499e4","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Stopped DB Nodes","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Table copies held by this node. Reduction = replica departure or table deletion.","id":"w52","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_held_locks","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Held Locks (detail)","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"b4fad1ac-d9ba-4668-a81e-26a36c2bbf74","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Held Locks (detail)","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia event subscribers. Each type of activity subscription appears as a separate line.","id":"w53","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_subscriptions","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[{"dataType":"string","id":"type--string--tag","isColumn":false,"key":"type","type":"tag"}],"legend":"{{type}} subscriptions","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"28e2e9c4-c5aa-472e-850e-ee85e8e22b49","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Subscriptions by Type","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia commit rate vs abort rate vs restart rate. High abort-to-commit ratio = significant write contention.","id":"w54","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_commits","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Commits/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_mnesia_transaction_aborts","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Aborts/s","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_mnesia_transaction_restarts","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"rate"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Restarts/s","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"1f5e5c8e-fbd2-4b9c-802e-b73b8ba56f19","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Transaction Health Overview","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Memory consumed by Mnesia data vs process heap vs system. Helps identify if Mnesia dominates memory.","id":"w55","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_memory_usage_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Mnesia Memory","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_memory_processes_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Process Heap","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_memory_system_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"System","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"7f15e2c5-07ab-49b8-a1ee-0d8edfd06fc5","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia vs VM Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables as currently reported. Any drops = partition or node failure.","id":"w56","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_db_nodes_running","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Running Nodes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_mnesia_db_nodes_stopped","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Stopped Nodes","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"e1a7b8c9-d2e3-4f56-a789-b0c1d2e3f456","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Node Status","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Mnesia tables with unsynchronised copies (split-brain indicator) vs held locks (contention indicator) over time.","id":"w57","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_unsynchronised_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Unsync Tables","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_mnesia_held_locks","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Held Locks","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"a2b3c4d5-e6f7-4890-a1b2-c3d4e5f60123","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Unsync Tables vs Held Locks","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Active Mnesia transaction coordinators (dirty vs transactional). Dominance of dirty writes = performance but no isolation.","id":"w58","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_mnesia_transaction_coordinators","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Transaction Coordinators","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"f1e2d3c4-b5a6-4789-9012-3456789abcde","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Mnesia Active Transaction Coordinators","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"% of atom table used. Atom table never shrinks — runaway growth = leak.","fillMode":"none","fillSpans":false,"id":"39eb4c4f-5129-4a3c-92fd-8786274b1159","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_atom_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":{"expression":""},"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_atom_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":{"expression":""},"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"d9b4c6b7-a492-42d0-a64d-55a3ec119ab2","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Atom Utilization %","yAxisUnit":"percent"}],"layout":[{"h":1,"i":"r1","moved":false,"static":false,"w":12,"x":0,"y":0},{"h":7,"i":"39eb4c4f-5129-4a3c-92fd-8786274b1159","moved":false,"static":false,"w":4,"x":0,"y":1},{"h":7,"i":"w2","moved":false,"static":false,"w":4,"x":4,"y":1},{"h":7,"i":"w4","moved":false,"static":false,"w":4,"x":8,"y":1},{"h":7,"i":"w5","moved":false,"static":false,"w":4,"x":0,"y":8},{"h":7,"i":"w1","moved":false,"static":false,"w":4,"x":4,"y":8},{"h":7,"i":"w3","moved":false,"static":false,"w":4,"x":8,"y":8},{"h":1,"i":"r2","moved":false,"static":false,"w":12,"x":0,"y":15},{"h":7,"i":"w6","moved":false,"static":false,"w":4,"x":0,"y":16},{"h":7,"i":"w7","moved":false,"static":false,"w":4,"x":4,"y":16},{"h":7,"i":"w8","moved":false,"static":false,"w":4,"x":8,"y":16},{"h":7,"i":"w9","moved":false,"static":false,"w":4,"x":0,"y":23},{"h":7,"i":"w10","moved":false,"static":false,"w":4,"x":4,"y":23},{"h":7,"i":"w11","moved":false,"static":false,"w":4,"x":8,"y":23},{"h":1,"i":"r3","moved":false,"static":false,"w":12,"x":0,"y":30},{"h":7,"i":"w14","moved":false,"static":false,"w":4,"x":0,"y":31},{"h":7,"i":"w12","moved":false,"static":false,"w":4,"x":4,"y":31},{"h":7,"i":"w13","moved":false,"static":false,"w":4,"x":8,"y":31},{"h":7,"i":"w16","moved":false,"static":false,"w":12,"x":0,"y":38},{"h":1,"i":"r4","moved":false,"static":false,"w":12,"x":0,"y":45},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":0,"y":46},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":3,"y":46},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":6,"y":46},{"h":7,"i":"w20","moved":false,"static":false,"w":3,"x":9,"y":46},{"h":1,"i":"r5","moved":false,"static":false,"w":12,"x":0,"y":53},{"h":7,"i":"w21","moved":false,"static":false,"w":4,"x":0,"y":54},{"h":7,"i":"w22","moved":false,"static":false,"w":4,"x":4,"y":54},{"h":7,"i":"w23","moved":false,"static":false,"w":4,"x":8,"y":54},{"h":7,"i":"w24","moved":false,"static":false,"w":4,"x":0,"y":61},{"h":7,"i":"w25","moved":false,"static":false,"w":4,"x":4,"y":61},{"h":7,"i":"w26","moved":false,"static":false,"w":4,"x":8,"y":61},{"h":7,"i":"w27","moved":false,"static":false,"w":12,"x":0,"y":68},{"h":1,"i":"r6","moved":false,"static":false,"w":12,"x":0,"y":75},{"h":7,"i":"w28","moved":false,"static":false,"w":4,"x":0,"y":76},{"h":7,"i":"w29","moved":false,"static":false,"w":4,"x":4,"y":76},{"h":7,"i":"w30","moved":false,"static":false,"w":4,"x":8,"y":76},{"h":7,"i":"w31","moved":false,"static":false,"w":6,"x":0,"y":83},{"h":7,"i":"w32","moved":false,"static":false,"w":6,"x":6,"y":83},{"h":1,"i":"r7","moved":false,"static":false,"w":12,"x":0,"y":90},{"h":7,"i":"w33","moved":false,"static":false,"w":6,"x":0,"y":91},{"h":7,"i":"w34","moved":false,"static":false,"w":6,"x":6,"y":91},{"h":1,"i":"r8","moved":false,"static":false,"w":12,"x":0,"y":98},{"h":7,"i":"w35","moved":false,"static":false,"w":6,"x":0,"y":99},{"h":7,"i":"w36","moved":false,"static":false,"w":6,"x":6,"y":99},{"h":7,"i":"w37","moved":false,"static":false,"w":4,"x":0,"y":106},{"h":7,"i":"w38","moved":false,"static":false,"w":4,"x":4,"y":106},{"h":7,"i":"w39","moved":false,"static":false,"w":4,"x":8,"y":106},{"h":1,"i":"r9","moved":false,"static":false,"w":12,"x":0,"y":113},{"h":7,"i":"w40","moved":false,"static":false,"w":6,"x":0,"y":114},{"h":7,"i":"w41","moved":false,"static":false,"w":6,"x":6,"y":114},{"h":7,"i":"w42","moved":false,"static":false,"w":6,"x":0,"y":121},{"h":7,"i":"w43","moved":false,"static":false,"w":6,"x":6,"y":121},{"h":7,"i":"w44","moved":false,"static":false,"w":6,"x":0,"y":128},{"h":7,"i":"w45","moved":false,"static":false,"w":6,"x":6,"y":128},{"h":7,"i":"w46","moved":false,"static":false,"w":6,"x":0,"y":135},{"h":7,"i":"w47","moved":false,"static":false,"w":6,"x":6,"y":135},{"h":7,"i":"w48","moved":false,"static":false,"w":4,"x":0,"y":142},{"h":7,"i":"w49","moved":false,"static":false,"w":4,"x":4,"y":142},{"h":7,"i":"w50","moved":false,"static":false,"w":4,"x":8,"y":142},{"h":1,"i":"r10","moved":false,"static":false,"w":12,"x":0,"y":149},{"h":7,"i":"w51","moved":false,"static":false,"w":3,"x":0,"y":150},{"h":7,"i":"w52","moved":false,"static":false,"w":3,"x":3,"y":150},{"h":7,"i":"w53","moved":false,"static":false,"w":6,"x":6,"y":150},{"h":7,"i":"w54","moved":false,"static":false,"w":6,"x":0,"y":157},{"h":7,"i":"w55","moved":false,"static":false,"w":6,"x":6,"y":157},{"h":7,"i":"w56","moved":false,"static":false,"w":4,"x":0,"y":164},{"h":7,"i":"w57","moved":false,"static":false,"w":4,"x":4,"y":164},{"h":7,"i":"w58","moved":false,"static":false,"w":4,"x":8,"y":164}],"panelMap":{"r1":{"collapsed":false,"widgets":[{"h":7,"i":"w1","moved":false,"static":false,"w":4,"x":0,"y":1},{"h":7,"i":"w2","moved":false,"static":false,"w":4,"x":4,"y":1},{"h":7,"i":"w3","moved":false,"static":false,"w":4,"x":8,"y":1},{"h":7,"i":"w4","moved":false,"static":false,"w":6,"x":0,"y":8},{"h":7,"i":"w5","moved":false,"static":false,"w":6,"x":6,"y":8},{"h":1,"i":"r2","moved":false,"static":false,"w":12,"x":0,"y":15},{"h":7,"i":"w6","moved":false,"static":false,"w":4,"x":0,"y":16},{"h":7,"i":"w7","moved":false,"static":false,"w":4,"x":4,"y":16},{"h":7,"i":"w8","moved":false,"static":false,"w":4,"x":8,"y":16},{"h":7,"i":"w9","moved":false,"static":false,"w":4,"x":0,"y":23},{"h":7,"i":"w10","moved":false,"static":false,"w":4,"x":4,"y":23},{"h":7,"i":"w11","moved":false,"static":false,"w":4,"x":8,"y":23},{"h":1,"i":"r3","moved":false,"static":false,"w":12,"x":0,"y":30},{"h":7,"i":"w12","moved":false,"static":false,"w":3,"x":0,"y":31},{"h":7,"i":"w13","moved":false,"static":false,"w":3,"x":3,"y":31},{"h":7,"i":"w14","moved":false,"static":false,"w":3,"x":6,"y":31},{"h":7,"i":"w15","moved":false,"static":false,"w":3,"x":9,"y":31},{"h":7,"i":"w16","moved":false,"static":false,"w":12,"x":0,"y":38},{"h":1,"i":"r4","moved":false,"static":false,"w":12,"x":0,"y":45},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":0,"y":46},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":3,"y":46},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":6,"y":46},{"h":7,"i":"w20","moved":false,"static":false,"w":3,"x":9,"y":46},{"h":1,"i":"r5","moved":false,"static":false,"w":12,"x":0,"y":53},{"h":7,"i":"w21","moved":false,"static":false,"w":4,"x":0,"y":54},{"h":7,"i":"w22","moved":false,"static":false,"w":4,"x":4,"y":54},{"h":7,"i":"w23","moved":false,"static":false,"w":4,"x":8,"y":54},{"h":7,"i":"w24","moved":false,"static":false,"w":4,"x":0,"y":61},{"h":7,"i":"w25","moved":false,"static":false,"w":4,"x":4,"y":61},{"h":7,"i":"w26","moved":false,"static":false,"w":4,"x":8,"y":61},{"h":7,"i":"w27","moved":false,"static":false,"w":12,"x":0,"y":68},{"h":1,"i":"r6","moved":false,"static":false,"w":12,"x":0,"y":75},{"h":7,"i":"w28","moved":false,"static":false,"w":4,"x":0,"y":76},{"h":7,"i":"w29","moved":false,"static":false,"w":4,"x":4,"y":76},{"h":7,"i":"w30","moved":false,"static":false,"w":4,"x":8,"y":76},{"h":7,"i":"w31","moved":false,"static":false,"w":6,"x":0,"y":83},{"h":7,"i":"w32","moved":false,"static":false,"w":6,"x":6,"y":83},{"h":1,"i":"r7","moved":false,"static":false,"w":12,"x":0,"y":90},{"h":7,"i":"w33","moved":false,"static":false,"w":6,"x":0,"y":91},{"h":7,"i":"w34","moved":false,"static":false,"w":6,"x":6,"y":91},{"h":1,"i":"r8","moved":false,"static":false,"w":12,"x":0,"y":98},{"h":7,"i":"w35","moved":false,"static":false,"w":6,"x":0,"y":99},{"h":7,"i":"w36","moved":false,"static":false,"w":6,"x":6,"y":99},{"h":7,"i":"w37","moved":false,"static":false,"w":4,"x":0,"y":106},{"h":7,"i":"w38","moved":false,"static":false,"w":4,"x":4,"y":106},{"h":7,"i":"w39","moved":false,"static":false,"w":4,"x":8,"y":106},{"h":1,"i":"r9","moved":false,"static":false,"w":12,"x":0,"y":113},{"h":7,"i":"w40","moved":false,"static":false,"w":6,"x":0,"y":114},{"h":7,"i":"w41","moved":false,"static":false,"w":6,"x":6,"y":114},{"h":7,"i":"w42","moved":false,"static":false,"w":6,"x":0,"y":121},{"h":7,"i":"w43","moved":false,"static":false,"w":6,"x":6,"y":121},{"h":7,"i":"w44","moved":false,"static":false,"w":6,"x":0,"y":128},{"h":7,"i":"w45","moved":false,"static":false,"w":6,"x":6,"y":128},{"h":7,"i":"w46","moved":false,"static":false,"w":6,"x":0,"y":135},{"h":7,"i":"w47","moved":false,"static":false,"w":6,"x":6,"y":135},{"h":7,"i":"w48","moved":false,"static":false,"w":4,"x":0,"y":142},{"h":7,"i":"w49","moved":false,"static":false,"w":4,"x":4,"y":142},{"h":7,"i":"w50","moved":false,"static":false,"w":4,"x":8,"y":142},{"h":1,"i":"r10","moved":false,"static":false,"w":12,"x":0,"y":149},{"h":7,"i":"w51","moved":false,"static":false,"w":3,"x":0,"y":150},{"h":7,"i":"w52","moved":false,"static":false,"w":3,"x":3,"y":150},{"h":7,"i":"w53","moved":false,"static":false,"w":6,"x":6,"y":150},{"h":7,"i":"w54","moved":false,"static":false,"w":6,"x":0,"y":157},{"h":7,"i":"w55","moved":false,"static":false,"w":6,"x":6,"y":157},{"h":7,"i":"w56","moved":false,"static":false,"w":4,"x":0,"y":164},{"h":7,"i":"w57","moved":false,"static":false,"w":4,"x":4,"y":164},{"h":7,"i":"w58","moved":false,"static":false,"w":4,"x":8,"y":164},{"h":7,"i":"3776cf0d-5c63-468a-9a99-d2dbc81431df","moved":false,"static":false,"w":4,"x":0,"y":171}]}},"version":"v5","title":"BEAM / Erlang VM - Deep Dive"} +{ + "description": "Per-subsystem breakdown of BEAM runtime internals: process/port/atom saturation detail, memory segments, ETS, MSACC scheduler microstate, GC, I/O, memory allocator fragmentation, Erlang distribution link health, and Mnesia. Generic for any Elixir/Erlang project using prometheus_ex.", + "image": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTgiIHZpZXdCb3g9IjAgMCAxOCAxOCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0Ljk0OTQgMTMuOTQyQzE2LjIzMTggMTIuNDI1OCAxNy4zMjY4IDkuNzAyMiAxNi4xOTU2IDYuNTc0ODdDMTUuNjQ0MyA1LjA1MjQ1IDE1LjAyMTkgNC4yMDI0OSAxNC4yOTY5IDMuNjYyNTJDMTMuODU1NyAzLjMzMzc5IDEyLjA5MzMgMi41MDYzMyA5Ljc1OTY1IDIuODY3NTZDOC4wNTM0OSAzLjEzMjU1IDUuNzc0ODcgNC4yMDg3NCA0LjI5MzY5IDUuOTU5OUMyLjg1NzUyIDcuNjYxMDYgMS43NDg4MyA5LjAwNDc0IDEuNjk3NTggMTAuMzA5N0MxLjYzMTMzIDExLjk4ODMgMi44OTYyNyAxMy40MzA4IDMuMDUwMDEgMTMuNjY0NUMzLjMyMzc0IDE0LjA3OTUgNS4xOTExNSAxNi40NTE4IDguNjk5NzEgMTYuNTczMUMxMS43OTcgMTYuNjc5MyAxMy44MTQ0IDE1LjI4NDQgMTQuOTQ5NCAxMy45NDJaIiBmaWxsPSIjNDAzRDNFIi8+CjxwYXRoIGQ9Ik00LjU1MzYzIDIuNzM3NDdDMi45Mzc0NiAzLjg5MTE2IDEuMTIxMzEgNi4yNTEwMyAxLjQ0NzU0IDkuNTYwODZDMS42MDYyOCAxMS4xNzIgMi4wMDI1MSAxMi4xNDk1IDIuNTcxMjMgMTIuODUwN0MyLjkxNzQ2IDEzLjI3ODIgNC40MTk4OCAxNC41NDkzIDYuNzczNTEgMTQuNzM2OEM5LjE0NTg4IDE0LjkyNTYgMTAuOTQ5NSAxNC4zOTQ0IDEyLjgzMzIgMTMuMDg0NEMxNi42NjE3IDEwLjQyMDggMTYuMDk4IDYuMzkzNTMgMTUuOTM0MyA1LjkyNDhDMTUuNzcwNSA1LjQ1NjA3IDE0LjU0NDQgMi42OTYyMiAxMS4xNzMzIDEuNzE1MDJDOC4xOTg0NCAwLjg1MDA2OCA1Ljk4MzU1IDEuNzE1MDIgNC41NTM2MyAyLjczNzQ3WiIgZmlsbD0iIzVFNjM2NyIvPgo8cGF0aCBkPSJNNy4zOTM1MyAyLjk2MTA5QzUuNjE3MzcgMi44OTczNCAzLjkxOTk2IDQuMjg4NTIgMy43NTYyMiA2LjAwNTkzQzMuNTkyNDggNy43MjIwOSA0LjY1NDkyIDkuMDI5NTIgNi4zMDk4MyA5LjI5NTc2QzcuOTY0NzUgOS41NjA3NCA5Ljg3ODM5IDguNTU1OCAxMC4yNjM0IDYuNDUwOTFDMTAuNjYwOSA0LjI4MjI3IDkuMDg5NjkgMy4wMjIzNCA3LjM5MzUzIDIuOTYxMDlaIiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBkPSJNNy45NDIxNyA1LjkwMTE1QzcuOTQyMTcgNS45MDExNSA4LjM2OTY1IDUuODEyNCA4LjQ1NDY1IDUuMTgyNDRDOC41MzgzOSA0LjU2MjQ3IDguMjMwOTEgNC4wMzM3NSA3LjUxMzQ1IDMuODQzNzZDNi43MzM0OSAzLjYzNzUyIDYuMjA0NzcgNC4wNjYyNSA2LjA2NzI3IDQuNTE3NDdDNS44NzYwMyA1LjE0NDk0IDYuMTU4NTIgNS40NDM2NyA2LjE1ODUyIDUuNDQzNjdDNi4xNTg1MiA1LjQ0MzY3IDUuMzkzNTYgNS42Mjc0MSA1LjMzMjMxIDYuNTI5ODdDNS4yNzQ4MSA3LjM4MTA3IDUuODU2MDMgNy44Mzg1NSA2LjQzOTc1IDcuOTc4NTRDNy4xNjA5NiA4LjE1MjI4IDcuOTc4NDIgNy45NTQ3OSA4LjE3ODQxIDcuMDM0ODRDOC4zNDQ2NSA2LjI3NzM4IDcuOTQyMTcgNS45MDExNSA3Ljk0MjE3IDUuOTAxMTVaIiBmaWxsPSIjMzAzMDMwIi8+CjxwYXRoIGQ9Ik02LjczOTgzIDQuNzUzNjJDNi42NzEwOSA1LjAxMjM1IDYuODA4NTggNS4yNjIzNCA3LjA3ODU3IDUuMzMxMDlDNy4zNjk4IDUuNDA0ODMgNy42MzQ3OSA1LjMwODU5IDcuNzA2MDMgNS4wMTExQzcuNzY4NTMgNC43NDczNyA3LjY0MzU0IDQuNTE0ODggNy4zMzYwNSA0LjQzOTg4QzcuMDgzNTcgNC4zNzczOSA2LjgxNDgzIDQuNDcxMTMgNi43Mzk4MyA0Ljc1MzYyWiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTYuOTU5NzggNi4wMzk3NEM2LjYzMjMgNS45Mzg0OSA2LjE5OTgyIDYuMDY0NzMgNi4xMzEwNyA2LjUwNDcxQzYuMDYyMzMgNi45NDQ2OSA2LjMyNjA2IDcuMTY5NjggNi42NzEwNCA3LjIzMjE3QzcuMDE2MDMgNy4yOTQ2NyA3LjM0MjI2IDcuMTEzNDMgNy40MDYwMSA2Ljc2MDk1QzcuNDY4NSA2LjQwOTcyIDcuMjg2MDEgNi4xMzk3MyA2Ljk1OTc4IDYuMDM5NzRaIiBmaWxsPSJ3aGl0ZSIvPgo8L3N2Zz4K", + "layout": [ + { + "h": 1, + "i": "r1", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 0 + }, + { + "h": 7, + "i": "39eb4c4f-5129-4a3c-92fd-8786274b1159", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 1 + }, + { + "h": 7, + "i": "w2", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 1 + }, + { + "h": 7, + "i": "w4", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 1 + }, + { + "h": 7, + "i": "w5", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 8 + }, + { + "h": 7, + "i": "w1", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 8 + }, + { + "h": 7, + "i": "w3", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 8 + }, + { + "h": 1, + "i": "r2", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 29 + }, + { + "h": 7, + "i": "w6", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 72 + }, + { + "h": 7, + "i": "w7", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 72 + }, + { + "h": 7, + "i": "w8", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 72 + }, + { + "h": 7, + "i": "w9", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 79 + }, + { + "h": 7, + "i": "w10", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 79 + }, + { + "h": 7, + "i": "w11", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 79 + }, + { + "h": 1, + "i": "r3", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 86 + }, + { + "h": 7, + "i": "w14", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 143 + }, + { + "h": 7, + "i": "w12", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 143 + }, + { + "h": 7, + "i": "w13", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 143 + }, + { + "h": 7, + "i": "w16", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 150 + }, + { + "h": 1, + "i": "r4", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 227 + }, + { + "h": 1, + "i": "r5", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 228 + }, + { + "h": 1, + "i": "r6", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 250 + }, + { + "h": 1, + "i": "r7", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 265 + }, + { + "h": 1, + "i": "r8", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 273 + }, + { + "h": 1, + "i": "r9", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 316 + }, + { + "h": 1, + "i": "r10", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 352 + } + ], + "name": "BEAM / Erlang VM - Deep Dive", + "panelMap": { + "r1": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "39eb4c4f-5129-4a3c-92fd-8786274b1159", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 1 + }, + { + "h": 7, + "i": "w2", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 1 + }, + { + "h": 7, + "i": "w4", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 1 + }, + { + "h": 7, + "i": "w5", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 8 + }, + { + "h": 7, + "i": "w1", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 8 + }, + { + "h": 7, + "i": "w3", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 8 + } + ] + }, + "r10": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w51", + "moved": false, + "static": false, + "w": 3, + "x": 0, + "y": 353 + }, + { + "h": 7, + "i": "w52", + "moved": false, + "static": false, + "w": 3, + "x": 3, + "y": 353 + }, + { + "h": 7, + "i": "w53", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 353 + }, + { + "h": 7, + "i": "w54", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 360 + }, + { + "h": 7, + "i": "w55", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 360 + }, + { + "h": 7, + "i": "w56", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 367 + }, + { + "h": 7, + "i": "w57", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 367 + }, + { + "h": 7, + "i": "w58", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 367 + } + ] + }, + "r2": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "w6", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 72 + }, + { + "h": 7, + "i": "w7", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 72 + }, + { + "h": 7, + "i": "w8", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 72 + }, + { + "h": 7, + "i": "w9", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 79 + }, + { + "h": 7, + "i": "w10", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 79 + }, + { + "h": 7, + "i": "w11", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 79 + } + ] + }, + "r3": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "w14", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 143 + }, + { + "h": 7, + "i": "w12", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 143 + }, + { + "h": 7, + "i": "w13", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 143 + }, + { + "h": 7, + "i": "w16", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 150 + } + ] + }, + "r4": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w17", + "moved": false, + "static": false, + "w": 3, + "x": 0, + "y": 298 + }, + { + "h": 7, + "i": "w18", + "moved": false, + "static": false, + "w": 3, + "x": 3, + "y": 298 + }, + { + "h": 7, + "i": "w19", + "moved": false, + "static": false, + "w": 3, + "x": 6, + "y": 298 + }, + { + "h": 7, + "i": "w20", + "moved": false, + "static": false, + "w": 3, + "x": 9, + "y": 298 + } + ] + }, + "r5": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w21", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 229 + }, + { + "h": 7, + "i": "w22", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 229 + }, + { + "h": 7, + "i": "w23", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 229 + }, + { + "h": 7, + "i": "w24", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 236 + }, + { + "h": 7, + "i": "w25", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 236 + }, + { + "h": 7, + "i": "w26", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 236 + }, + { + "h": 7, + "i": "w27", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 243 + } + ] + }, + "r6": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w28", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 251 + }, + { + "h": 7, + "i": "w29", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 251 + }, + { + "h": 7, + "i": "w30", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 251 + }, + { + "h": 7, + "i": "w31", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 258 + }, + { + "h": 7, + "i": "w32", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 258 + } + ] + }, + "r7": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w33", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 266 + }, + { + "h": 7, + "i": "w34", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 266 + } + ] + }, + "r8": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w35", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 274 + }, + { + "h": 7, + "i": "w36", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 274 + }, + { + "h": 7, + "i": "w37", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 281 + }, + { + "h": 7, + "i": "w38", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 281 + }, + { + "h": 7, + "i": "w39", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 281 + } + ] + }, + "r9": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w40", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 317 + }, + { + "h": 7, + "i": "w41", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 317 + }, + { + "h": 7, + "i": "w42", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 324 + }, + { + "h": 7, + "i": "w43", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 324 + }, + { + "h": 7, + "i": "w44", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 331 + }, + { + "h": 7, + "i": "w45", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 331 + }, + { + "h": 7, + "i": "w46", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 338 + }, + { + "h": 7, + "i": "w47", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 338 + }, + { + "h": 7, + "i": "w48", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 345 + }, + { + "h": 7, + "i": "w49", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 345 + }, + { + "h": 7, + "i": "w50", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 345 + } + ] + } + }, + "tags": [ + "beam", + "erlang", + "elixir", + "debug" + ], + "title": "BEAM / Erlang VM - Deep Dive", + "uploadedGrafana": false, + "variables": { + "environment": { + "allSelected": true, + "description": "Deployment environment filter", + "dynamicVariablesAttribute": "deployment.environment", + "dynamicVariablesSource": "Metrics", + "id": "var-environment", + "multiSelect": true, + "name": "environment", + "order": 0, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + }, + "k8s_cluster": { + "allSelected": true, + "description": "Kubernetes cluster name filter", + "dynamicVariablesAttribute": "k8s.cluster.name", + "dynamicVariablesSource": "Metrics", + "id": "var-k8s-cluster", + "multiSelect": true, + "name": "k8s_cluster", + "order": 1, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + }, + "k8s_pod": { + "allSelected": true, + "description": "Pod name filter — select specific pods to scope all panels to one application", + "dynamicVariablesAttribute": "k8s.pod.name", + "dynamicVariablesSource": "Metrics", + "id": "var-k8s-pod", + "multiSelect": true, + "name": "k8s_pod", + "order": 2, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + } + }, + "version": "v5", + "widgets": [ + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r1", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Process, Port & Atom Saturation" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Live processes (solid) vs hard limit (dashed). Gap = headroom. Shrinking gap = leak or load spike.", + "id": "w1", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Process Count", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_process_limit", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Process Limit", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "d745aedc-ca2a-480e-b538-2782ea4fbbb9", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Process Count vs Limit", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": ">85% is critical — BEAM refuses to spawn new processes at the limit.", + "fillMode": "none", + "fillSpans": false, + "id": "w2", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_process_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "d9b4c6b7-a492-42d0-a64d-55a3ec119ab2", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Process Utilization %", + "yAxisUnit": "percent" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "Ports represent open sockets, files, and NIF handles. Exhaustion causes connection failures.", + "fillMode": "none", + "fillSpans": false, + "id": "w3", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_port_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": { + "expression": "" + }, + "legend": "Port Count", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_port_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": false, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": { + "expression": "" + }, + "legend": "Port Limit", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "ea4de080-95d9-46d5-bd1a-51c637c13eee", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Port Count vs Limit", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "Port exhaustion does not crash the node but causes {error, system_limit} on new connections.", + "fillMode": "none", + "fillSpans": false, + "id": "w4", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_port_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_port_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "a2163961-8c39-4672-8e29-996dbb26665a", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Port Utilization %", + "yAxisUnit": "percent" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Atom table is global and never shrinks. Monotonic growth without restart = atom leak (dynamic atom creation).", + "id": "w5", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_count", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Atom Count", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_limit", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Atom Limit", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "cdb7c126-a228-4297-888c-4d5dfc65a4cc", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Atom Count vs Limit", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r2", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Memory Detail" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Memory allocated to all Erlang process stacks and heaps. Growing without a matching process count increase = per-process memory leak.", + "id": "w6", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_processes_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Process Heap", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "dafc56fc-e40c-4ed7-ae40-3ab34b1464ec", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Process Heap Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "BEAM VM system memory: ETS, code, stack, and VM overhead. Excludes process heap.", + "id": "w7", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_system_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "System", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c04b416e-405d-4773-84fd-a6cfbdd5072d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "System Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Memory consumed by the atom table. Correlated with atom count.", + "id": "w8", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_atom_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Atom Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "93a93556-2769-4387-8302-6119c7b5ff40", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Atom Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Off-heap reference-counted binaries. Steady monotonic growth between GCs = binary leak (processes holding large binary references).", + "id": "w9", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "beam_memory_binary_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Binary Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "6d8ca821-9676-4b7c-bfc8-56afaca9d9ef", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Binary Memory (off-heap)", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total memory used by all ETS tables. Correlates with table count and row density.", + "id": "w10", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "beam_memory_ets_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "ETS Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "2184b14f-5e55-4a8f-af90-3451a7583b4b", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "ETS Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Memory occupied by loaded BEAM modules. Significant spikes after hot upgrades or repeated code:load calls.", + "id": "w11", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "beam_memory_code_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Code Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "80db8391-3a5f-47b4-9ada-912df39e90d4", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Code Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r3", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "ETS Tables" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total open ETS tables. Steady growth without a corresponding application load increase = table leak (missing :ets.delete).", + "id": "w12", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_ets_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "ETS Tables", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "9397af8e-b8a7-4cd9-90c0-0bddb580126d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "ETS Table Count", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Hard ceiling for ETS tables (default 8192). Reference line — compare against ETS Count.", + "id": "w13", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_ets_limit", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "ETS Limit", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c0295751-2141-489e-8cdb-977960827e95", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "ETS Table Limit", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "ETS tables used vs limit. >80% warrants investigation into table ownership and lifecycle.", + "fillMode": "none", + "fillSpans": false, + "id": "w14", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_ets_tables", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_ets_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "9b50b63a-c493-427c-bdec-ba3129d7dac1", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "ETS Utilization %", + "yAxisUnit": "percent" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Disk-backed ETS tables. Normally 0 in most apps. Unexpected growth = open disk tables not being closed.", + "id": "w16", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_dets_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "DETS Tables", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "4d1ad25d-fe40-474f-982f-10e8f188388a", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "DETS (Disk ETS) Table Count", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r4", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "MSACC Scheduler Microstate" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Fraction of scheduler time each MSACC state consumes. 'aux' + 'check_io' + 'emulator' should dominate; high 'sleep' = idle schedulers; high 'gc' = GC pressure.", + "id": "w17", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_aux_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "aux", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_check_io_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "check_io", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_emulator_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "emulator", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_gc_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "D", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "gc", + "orderBy": [], + "queryName": "D", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "1c540b59-0c75-4c3d-ae9b-2d49a6f2ea22", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MSACC: aux / check_io / emulator / gc", + "yAxisUnit": "s" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Scheduler time in port / nif / bif / send. Elevated nif = heavy NIF calls blocking schedulers; elevated send = message-passing saturation.", + "id": "w18", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_port_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "port", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_nif_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "nif", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_bif_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "bif", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_send_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "D", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "send", + "orderBy": [], + "queryName": "D", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "5dad1b84-abbb-4efb-98d3-3c23e3f97fb6", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MSACC: port / nif / bif / send", + "yAxisUnit": "s" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Scheduler time asleep. High sleep = under-utilised schedulers (normal during low load) or blocked waiting on I/O.", + "id": "w19", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_sleep_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "sleep", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c77bc2c8-af64-47ec-bc27-0f12bdb7b00c", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MSACC: sleep", + "yAxisUnit": "s" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Scheduler time in alloc / other / timers. Elevated alloc = heavy allocation pressure; elevated timers = many active timer references.", + "id": "w20", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_alloc_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "alloc", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_other_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "other", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_msacc_timers_seconds_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "timers", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "e25c0a42-7f8d-4048-b5f1-e78c17b61c3c", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MSACC: alloc / other / timers", + "yAxisUnit": "s" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r5", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "GC & I/O" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "GC runs per second. Short spikes are normal; sustained high rates with rising heap = object churn.", + "id": "w21", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_number_of_gcs", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "GC Count Rate", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "0a8cc42d-7082-4ee3-b35e-7a62f9a0e32b", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Count Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Bytes reclaimed per second by GC. High reclaim with high GC count = frequent short-lived allocation churn.", + "id": "w22", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_bytes_reclaimed", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Bytes Reclaimed/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "5e52a97e-1c5c-4b7b-a1c5-17d26640dbd5", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Bytes Reclaimed Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Words moved across generations per GC cycle. Large word counts = major GC activity; correlate with pauses.", + "id": "w23", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_words_reclaimed", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Words Reclaimed/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "4db37a6d-e10e-40ba-b9a8-6ea32c9c78e5", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Words Reclaimed Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Raw bytes received by all ports (sockets, pipes). Sustained high read throughput is normal; sudden drops = upstream pressure.", + "id": "w24", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_bytes_received_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Bytes In/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "3b4de0fd-f6d1-42be-9f36-2e91b7bde0d0", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "I/O Bytes In Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Raw bytes sent by all ports. Sustained high write throughput is normal; compare with in-rate for asymmetry.", + "id": "w25", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_bytes_output_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Bytes Out/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "5de6b8d3-a5bd-4b1f-8ee5-fd08b64aa9a7", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "I/O Bytes Out Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total context switches across all schedulers per second. Sudden step-changes correlate with process spawning bursts or supervisor restarts.", + "id": "w26", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_context_switches", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Context Switches/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "b2da73e4-0a0e-4ead-8a03-e1c10b4fa218", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Context Switches Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total reductions (function calls + tail calls) per second. Steady baseline; spikes indicate CPU-intensive work or hot loops.", + "id": "w27", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_reductions_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Reductions/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "aa98a5ca-e975-48e4-9ce8-c6ea2f524e79", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Reductions Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r6", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Memory Allocator Fragmentation" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Bytes allocated by the BEAM allocator vs bytes actually used by live data. Gap = fragmentation overhead.", + "id": "w28", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_pool_blocks_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} pool_blocks_size", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "6bd1e624-1f66-4dc5-88c9-b0893b6c0bde", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Allocator MBCS Pool Block Size", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Number of multi-block carriers in the pool. Growing without load growth = fragmented carrier accumulation.", + "id": "w29", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_pool_blocks", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} pool_blocks", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "a9a19bb3-5069-441b-aece-5a3199b48720", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Allocator MBCS Pool Block Count", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Single-block carriers created by the allocator. SBC growth = large allocations exceeding MBC threshold; consider +MBcst tuning.", + "id": "w30", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_sbcs_blocks_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} sbc_blocks_size", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c44f2174-b40b-4b00-ba97-b3fe55f9a1a1", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Allocator SBC Block Size", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Fragmentation ratio = (allocated − used) / allocated. Values >40% sustained = fragmented memory; consider allocator flags or node restart.", + "id": "w31", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_block_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_carriers_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "expression": "(B - A) / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "id": "4e48e5b5-6e2f-489e-a2e7-72c76b90ae74", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 100, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MBC Fragmentation %", + "yAxisUnit": "percent" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Allocated carrier size (total reserved) vs block size (live data). Wide gap = wasted carrier memory.", + "id": "w32", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_block_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} block", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_allocator_mbcs_carriers_size", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "allocator--string--tag", + "isColumn": false, + "key": "allocator", + "type": "tag" + } + ], + "legend": "{{allocator}} carrier", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "b39cc3d5-6acb-4a8e-883d-84be6c7f6462", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "MBC Block vs Carrier Size", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r7", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Erlang Distribution Link" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Distribution bytes sent per second to remote nodes. Sudden drop = node disconnect or network partition.", + "id": "w33", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_send_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Bytes Sent/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "d96a4d31-80fe-415c-b2c7-a4fc1c8e86e6", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Bytes Sent Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Distribution bytes received per second from remote nodes. Spikes = large remote message deliveries or data transfers.", + "id": "w34", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_recv_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Bytes Recv/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "4e22b2db-83b8-4ad2-8a38-ef81f89c4eb0", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Bytes Received Rate", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r8", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Distribution Messages" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Messages sent across distribution links per second. High rate + growing queues = dist buffer saturation risk.", + "id": "w35", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_send_messages_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Messages Sent/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "f8f4c226-0c52-4d8f-8eae-f5c41a8e5e3d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Messages Sent Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Messages received over distribution links per second. Asymmetry with sent rate = one-way traffic imbalance.", + "id": "w36", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_recv_messages_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Messages Recv/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c17ca374-f79d-4eac-9770-4dbf19da4c37", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Messages Received Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Pending bytes in the distribution send buffer. Sustained non-zero values = congested distribution link.", + "id": "w37", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_send_pending_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Pending Send Bytes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "8e73f8ea-c6a9-4bef-bd86-adde23c8e4ca", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Send Pending", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Packets sent across distribution links. Low-frequency, high-byte packets = large message transfers.", + "id": "w38", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_send_packets_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Packets Sent/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "1e4d4d9f-b8a0-4b12-af6f-13c4cd9e0f4d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Packets Sent Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Packets received over distribution links. Compare with sent rate and bytes for flow analysis.", + "id": "w39", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_dist_recv_packets_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dist Packets Recv/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "c01c7098-49c3-4dd4-834e-e7ed9f97e8a0", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Distribution Packets Received Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r9", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Mnesia" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Active Mnesia transactions. Sustained high counts = long-running transactions blocking table locks.", + "id": "w40", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_held_locks", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Held Locks", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "d36ff9fa-7e51-492e-aa81-61f5a7c5f7e3", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Held Locks", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Processes waiting to acquire Mnesia locks. Non-zero = lock contention; high values = potential deadlock or slow transactions.", + "id": "w41", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_lock_queue", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Lock Queue", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "2ec3b61e-2065-4eef-a6d3-dc9219ef6a05", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Lock Queue Depth", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia transaction restarts due to lock conflicts. Elevated restart rate = high write contention; consider table sharding or optimistic locking.", + "id": "w42", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_restarts", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Transaction Restarts/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "e7f3c17a-59e4-4e54-9e22-71b6d4c4d25a", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Transaction Restart Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia transactions aborted per second. Aborts can be caused by conflicts, node failures, or application errors.", + "id": "w43", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_aborts", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Transaction Aborts/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "bab96b77-1e6e-4fbb-a7e7-61f88a5cd07f", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Transaction Abort Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia transactions committed per second. Baseline for throughput; compare with abort and restart rates for health ratio.", + "id": "w44", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_commits", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Transaction Commits/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "addd1a97-d7dc-42e8-9424-2cc26789d04d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Transaction Commit Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia log entries written per second. Disk-based tables only. High log rate + low commit rate = replay overhead.", + "id": "w45", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_logged_transactions", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Logged Transactions/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "deea3f37-9c5b-4d99-82b4-95e285c1d47f", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Logged Transactions Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Failed Mnesia writes. Any non-zero value = data integrity issue or schema mismatch.", + "id": "w46", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_failed_transactions", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Failed Transactions/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "66cc4fb5-cd2a-4f42-aba6-a6c0e6c15e95", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Failed Transaction Rate", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables currently open. Steady count = stable schema; sudden drop = node partition or table deletion.", + "id": "w47", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_memory_usage_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Memory Usage", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "0e88bf72-e297-424b-b4a4-3aef8bd5e5e5", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Memory Usage", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables with unsynchronised data across nodes. Non-zero = split-brain risk. Should return to zero after healing.", + "id": "w48", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_unsynchronised_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Unsynchronised Tables", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "43c0c24d-1ecf-4af7-83e4-b8c4d7f3a3e3", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Unsynchronised Tables", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Subscribed Mnesia event listeners. Unexpected drop = subscriber process crashed.", + "id": "w49", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_subscriptions", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Subscriptions", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "0d9dc6bf-9b77-4b5a-a5bf-8975c38fe9d1", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Subscriptions", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Number of connected Mnesia cluster nodes. Drop below expected count = node departure or partition.", + "id": "w50", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_db_nodes_running", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Running DB Nodes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "bb0d8b63-fa29-4f70-a6a6-7f64fc46dca9", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Running DB Nodes", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "r10", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Mnesia Extended" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables registered in the schema. Sudden change = schema migration or table deletion.", + "id": "w51", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_db_nodes_stopped", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Stopped DB Nodes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "8b57d1ef-ef46-4c5f-b8ef-73f52f3499e4", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Stopped DB Nodes", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Table copies held by this node. Reduction = replica departure or table deletion.", + "id": "w52", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_held_locks", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Held Locks (detail)", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "b4fad1ac-d9ba-4668-a81e-26a36c2bbf74", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Held Locks (detail)", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia event subscribers. Each type of activity subscription appears as a separate line.", + "id": "w53", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_subscriptions", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "type--string--tag", + "isColumn": false, + "key": "type", + "type": "tag" + } + ], + "legend": "{{type}} subscriptions", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "28e2e9c4-c5aa-472e-850e-ee85e8e22b49", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Subscriptions by Type", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia commit rate vs abort rate vs restart rate. High abort-to-commit ratio = significant write contention.", + "id": "w54", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_commits", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Commits/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_aborts", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Aborts/s", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_restarts", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Restarts/s", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "1f5e5c8e-fbd2-4b9c-802e-b73b8ba56f19", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Transaction Health Overview", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Memory consumed by Mnesia data vs process heap vs system. Helps identify if Mnesia dominates memory.", + "id": "w55", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_memory_usage_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Mnesia Memory", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_processes_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Process Heap", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_system_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "System", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "7f15e2c5-07ab-49b8-a1ee-0d8edfd06fc5", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia vs VM Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables as currently reported. Any drops = partition or node failure.", + "id": "w56", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_db_nodes_running", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Running Nodes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_mnesia_db_nodes_stopped", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Stopped Nodes", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "e1a7b8c9-d2e3-4f56-a789-b0c1d2e3f456", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Node Status", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Mnesia tables with unsynchronised copies (split-brain indicator) vs held locks (contention indicator) over time.", + "id": "w57", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_unsynchronised_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Unsync Tables", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_mnesia_held_locks", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Held Locks", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "a2b3c4d5-e6f7-4890-a1b2-c3d4e5f60123", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Unsync Tables vs Held Locks", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Active Mnesia transaction coordinators (dirty vs transactional). Dominance of dirty writes = performance but no isolation.", + "id": "w58", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_mnesia_transaction_coordinators", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Transaction Coordinators", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "f1e2d3c4-b5a6-4789-9012-3456789abcde", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Mnesia Active Transaction Coordinators", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "% of atom table used. Atom table never shrinks — runaway growth = leak.", + "fillMode": "none", + "fillSpans": false, + "id": "39eb4c4f-5129-4a3c-92fd-8786274b1159", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": { + "expression": "" + }, + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": { + "expression": "" + }, + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "d9b4c6b7-a492-42d0-a64d-55a3ec119ab2", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Atom Utilization %", + "yAxisUnit": "percent" + } + ] +} diff --git a/beam/BEAM-ErlangVM-Overview.json b/beam/BEAM-ErlangVM-Overview.json index 0561f951..9f6b0522 100644 --- a/beam/BEAM-ErlangVM-Overview.json +++ b/beam/BEAM-ErlangVM-Overview.json @@ -1 +1,2981 @@ -{"description":"Key BEAM runtime health indicators. First stop for on-call triage — process/port saturation, memory pressure, scheduler backlog, GC activity. Generic for any Elixir/Erlang project using prometheus_ex.","name":"BEAM / Erlang VM - Overview","tags":["beam","erlang","elixir","overview"],"image":"data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTgiIHZpZXdCb3g9IjAgMCAxOCAxOCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0Ljk0OTQgMTMuOTQyQzE2LjIzMTggMTIuNDI1OCAxNy4zMjY4IDkuNzAyMiAxNi4xOTU2IDYuNTc0ODdDMTUuNjQ0MyA1LjA1MjQ1IDE1LjAyMTkgNC4yMDI0OSAxNC4yOTY5IDMuNjYyNTJDMTMuODU1NyAzLjMzMzc5IDEyLjA5MzMgMi41MDYzMyA5Ljc1OTY1IDIuODY3NTZDOC4wNTM0OSAzLjEzMjU1IDUuNzc0ODcgNC4yMDg3NCA0LjI5MzY5IDUuOTU5OUMyLjg1NzUyIDcuNjYxMDYgMS43NDg4MyA5LjAwNDc0IDEuNjk3NTggMTAuMzA5N0MxLjYzMTMzIDExLjk4ODMgMi44OTYyNyAxMy40MzA4IDMuMDUwMDEgMTMuNjY0NUMzLjMyMzc0IDE0LjA3OTUgNS4xOTExNSAxNi40NTE4IDguNjk5NzEgMTYuNTczMUMxMS43OTcgMTYuNjc5MyAxMy44MTQ0IDE1LjI4NDQgMTQuOTQ5NCAxMy45NDJaIiBmaWxsPSIjNDAzRDNFIi8+CjxwYXRoIGQ9Ik00LjU1MzYzIDIuNzM3NDdDMi45Mzc0NiAzLjg5MTE2IDEuMTIxMzEgNi4yNTEwMyAxLjQ0NzU0IDkuNTYwODZDMS42MDYyOCAxMS4xNzIgMi4wMDI1MSAxMi4xNDk1IDIuNTcxMjMgMTIuODUwN0MyLjkxNzQ2IDEzLjI3ODIgNC40MTk4OCAxNC41NDkzIDYuNzczNTEgMTQuNzM2OEM5LjE0NTg4IDE0LjkyNTYgMTAuOTQ5NSAxNC4zOTQ0IDEyLjgzMzIgMTMuMDg0NEMxNi42NjE3IDEwLjQyMDggMTYuMDk4IDYuMzkzNTMgMTUuOTM0MyA1LjkyNDhDMTUuNzcwNSA1LjQ1NjA3IDE0LjU0NDQgMi42OTYyMiAxMS4xNzMzIDEuNzE1MDJDOC4xOTg0NCAwLjg1MDA2OCA1Ljk4MzU1IDEuNzE1MDIgNC41NTM2MyAyLjczNzQ3WiIgZmlsbD0iIzVFNjM2NyIvPgo8cGF0aCBkPSJNNy4zOTM1MyAyLjk2MTA5QzUuNjE3MzcgMi44OTczNCAzLjkxOTk2IDQuMjg4NTIgMy43NTYyMiA2LjAwNTkzQzMuNTkyNDggNy43MjIwOSA0LjY1NDkyIDkuMDI5NTIgNi4zMDk4MyA5LjI5NTc2QzcuOTY0NzUgOS41NjA3NCA5Ljg3ODM5IDguNTU1OCAxMC4yNjM0IDYuNDUwOTFDMTAuNjYwOSA0LjI4MjI3IDkuMDg5NjkgMy4wMjIzNCA3LjM5MzUzIDIuOTYxMDlaIiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBkPSJNNy45NDIxNyA1LjkwMTE1QzcuOTQyMTcgNS45MDExNSA4LjM2OTY1IDUuODEyNCA4LjQ1NDY1IDUuMTgyNDRDOC41MzgzOSA0LjU2MjQ3IDguMjMwOTEgNC4wMzM3NSA3LjUxMzQ1IDMuODQzNzZDNi43MzM0OSAzLjYzNzUyIDYuMjA0NzcgNC4wNjYyNSA2LjA2NzI3IDQuNTE3NDdDNS44NzYwMyA1LjE0NDk0IDYuMTU4NTIgNS40NDM2NyA2LjE1ODUyIDUuNDQzNjdDNi4xNTg1MiA1LjQ0MzY3IDUuMzkzNTYgNS42Mjc0MSA1LjMzMjMxIDYuNTI5ODdDNS4yNzQ4MSA3LjM4MTA3IDUuODU2MDMgNy44Mzg1NSA2LjQzOTc1IDcuOTc4NTRDNy4xNjA5NiA4LjE1MjI4IDcuOTc4NDIgNy45NTQ3OSA4LjE3ODQxIDcuMDM0ODRDOC4zNDQ2NSA2LjI3NzM4IDcuOTQyMTcgNS45MDExNSA3Ljk0MjE3IDUuOTAxMTVaIiBmaWxsPSIjMzAzMDMwIi8+CjxwYXRoIGQ9Ik02LjczOTgzIDQuNzUzNjJDNi42NzEwOSA1LjAxMjM1IDYuODA4NTggNS4yNjIzNCA3LjA3ODU3IDUuMzMxMDlDNy4zNjk4IDUuNDA0ODMgNy42MzQ3OSA1LjMwODU5IDcuNzA2MDMgNS4wMTExQzcuNzY4NTMgNC43NDczNyA3LjY0MzU0IDQuNTE0ODggNy4zMzYwNSA0LjQzOTg4QzcuMDgzNTcgNC4zNzczOSA2LjgxNDgzIDQuNDcxMTMgNi43Mzk4MyA0Ljc1MzYyWiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTYuOTU5NzggNi4wMzk3NEM2LjYzMjMgNS45Mzg0OSA2LjE5OTgyIDYuMDY0NzMgNi4xMzEwNyA2LjUwNDcxQzYuMDYyMzMgNi45NDQ2OSA2LjMyNjA2IDcuMTY5NjggNi42NzEwNCA3LjIzMjE3QzcuMDE2MDMgNy4yOTQ2NyA3LjM0MjI2IDcuMTEzNDMgNy40MDYwMSA2Ljc2MDk1QzcuNDY4NSA2LjQwOTcyIDcuMjg2MDEgNi4xMzk3MyA2Ljk1OTc4IDYuMDM5NzRaIiBmaWxsPSJ3aGl0ZSIvPgo8L3N2Zz4K","variables":{"environment":{"allSelected":true,"description":"Deployment environment filter","dynamicVariablesAttribute":"deployment.environment","dynamicVariablesSource":"Metrics","id":"var-environment","multiSelect":true,"name":"environment","order":0,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"},"k8s_cluster":{"allSelected":true,"description":"Kubernetes cluster name filter","dynamicVariablesAttribute":"k8s.cluster.name","dynamicVariablesSource":"Metrics","id":"var-k8s-cluster","multiSelect":true,"name":"k8s_cluster","order":1,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"},"k8s_pod":{"allSelected":true,"description":"Pod name filter — select specific pods to scope all panels to one application","dynamicVariablesAttribute":"k8s.pod.name","dynamicVariablesSource":"Metrics","haveCustomValuesSelected":false,"id":"var-k8s-pod","multiSelect":true,"name":"k8s_pod","order":2,"showALLOption":true,"sort":"ASC","type":"DYNAMIC"}},"widgets":[{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"% of process limit in use. >85% is critical — node will crash at 100%.","fillMode":"none","fillSpans":false,"id":"w1","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_process_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"c9912465-de59-4645-b89d-8c6625222c29","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":100,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[{"index":"0","thresholdColor":"#FFA500","thresholdFormat":"Background","thresholdLabel":"Warning","thresholdOperator":">","thresholdValue":70},{"index":"1","keyIndex":1,"thresholdColor":"#FF0000","thresholdFormat":"Background","thresholdLabel":"Critical","thresholdOperator":">","thresholdValue":85}],"timePreferance":"GLOBAL_TIME","title":"Process Utilization","yAxisUnit":"percent"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"% of port limit in use. Ports = sockets, files, NIFs.","fillMode":"none","fillSpans":false,"id":"w2","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_port_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_port_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"4e912e40-34ac-43e1-abec-0059a1d54bd2","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":100,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[{"index":"0","thresholdColor":"#FFA500","thresholdFormat":"Background","thresholdLabel":"Warning","thresholdOperator":">","thresholdValue":70},{"index":"1","keyIndex":1,"thresholdColor":"#FF0000","thresholdFormat":"Background","thresholdLabel":"Critical","thresholdOperator":">","thresholdValue":85}],"timePreferance":"GLOBAL_TIME","title":"Port Utilization","yAxisUnit":"percent"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":"% of atom table used. Atom table never shrinks — runaway growth = leak.","fillMode":"none","fillSpans":false,"id":"w3","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_atom_count","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_atom_limit","reduceTo":"avg","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[],"having":[],"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"aggregations":[],"dataSource":"metrics","disabled":false,"expression":"A / B * 100","filters":{"items":[],"op":"AND"},"functions":[],"groupBy":[],"legend":"","orderBy":[],"queryName":"F1","selectColumns":[],"stepInterval":null}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"42aacb5f-0752-4d92-a370-412be4a007ec","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":100,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[{"index":"0","thresholdColor":"#FFA500","thresholdFormat":"Background","thresholdLabel":"Warning","thresholdOperator":">","thresholdValue":70},{"index":"1","keyIndex":1,"thresholdColor":"#FF0000","thresholdFormat":"Background","thresholdLabel":"Critical","thresholdOperator":">","thresholdValue":85}],"timePreferance":"GLOBAL_TIME","title":"Atom Utilization","yAxisUnit":"percent"},{"contextLinks":{"linksData":[]},"description":"Number of ETS tables currently open.","id":"w4","nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_ets_tables","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"31a03bbe-951d-4804-b963-6467fbf4cfa3","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"ETS Tables","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"CPU scheduler threads available to the BEAM.","id":"w5","nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_schedulers_online","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"1818e786-b85d-486e-a178-2e74f3787812","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Schedulers Online","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Total BEAM memory allocation (processes + system).","id":"w6","nullZeroValues":"zero","opacity":"1","panelTypes":"value","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"034d759b-9272-4b5d-acbc-a8ecbcca65da","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Total Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"w7","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Memory"},{"contextLinks":{"linksData":[]},"description":"","id":"w8","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Total","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"a9d086d8-f2fe-471e-acf9-4e98882a5639","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Total Memory","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Stacked composition of BEAM memory — watch for any segment growing unbounded.","id":"w9","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_memory_processes_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Processes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_memory_system_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"System","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_memory_atom_bytes_total","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"C","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Atom","orderBy":[],"queryName":"C","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"beam_memory_binary_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"D","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Binary","orderBy":[],"queryName":"D","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"beam_memory_ets_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"E","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"ETS","orderBy":[],"queryName":"E","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"beam_memory_code_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"F","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Code","orderBy":[],"queryName":"F","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"deeddc90-b07a-46cd-839d-f21ac488d28d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Memory Breakdown","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"Steady monotonic growth indicates a binary leak (unreferenced binaries not GC'd).","id":"w10","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"beam_memory_binary_bytes","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Binary","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"718c7582-29a1-4453-b16e-8949bb23f0ff","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Binary Memory (off-heap)","yAxisUnit":"bytes"},{"contextLinks":{"linksData":[]},"description":"","id":"w11","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"Schedulers & Run Queues"},{"contextLinks":{"linksData":[]},"description":"Normal scheduler run queue. Sustained > 0 = CPU saturation.","id":"w12","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_run_queues_length","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Run Queue","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"51a166ed-a243-48ac-8470-bda62852a44d","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Run Queue Length","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Dirty CPU = long-running NIF pressure. Dirty IO = file/socket I/O pressure.","id":"w13","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_dirty_cpu_run_queue_length","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dirty CPU","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_statistics_dirty_io_run_queue_length","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"B","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Dirty IO","orderBy":[],"queryName":"B","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"9b84336d-4a49-4741-8415-f0234a989666","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Dirty CPU / IO Run Queues","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"Total Erlang processes. Runaway growth = process leak (missing termination or gen_server loop).","id":"w14","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","spaceAggregation":"avg","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Processes","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"da910f90-465d-4865-b6cb-02a7e33de9d9","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Live Processes","yAxisUnit":"short"},{"contextLinks":{"linksData":[]},"description":"","id":"w15","panelTypes":"row","query":{"builder":{"queryData":[],"queryFormulas":[]},"id":"","queryType":"builder"},"selectedLogFields":[],"selectedTracesFields":[],"softMax":null,"softMin":null,"title":"GC & Reductions"},{"contextLinks":{"linksData":[]},"description":"Garbage collections per second. Spikes indicate sudden allocation pressure.","id":"w16","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_number_of_gcs","spaceAggregation":"sum","temporality":"cumulative","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"GCs/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"42d2abba-afb3-437b-bcd4-2ca0584f1126","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Rate","yAxisUnit":"reqps"},{"contextLinks":{"linksData":[]},"description":"Bytes reclaimed per second by the garbage collector.","id":"w17","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_garbage_collection_bytes_reclaimed","spaceAggregation":"sum","temporality":"cumulative","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Bytes/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"f3225155-7370-4922-b824-8c259bdcf563","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"GC Bytes Reclaimed","yAxisUnit":"binBps"},{"contextLinks":{"linksData":[]},"description":"Scheduler context switches per second. High rate = many short-lived processes.","id":"w18","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_context_switches","spaceAggregation":"sum","temporality":"cumulative","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Switches/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"7c57ba19-7d01-4843-b418-72abab7d6201","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Context Switch Rate","yAxisUnit":"reqps"},{"contextLinks":{"linksData":[]},"description":"BEAM reductions per second — proxy for total CPU work across all processes.","id":"w19","legendPosition":"bottom","nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_statistics_reductions_total","spaceAggregation":"sum","temporality":"cumulative","timeAggregation":"rate"}],"dataSource":"metrics","expression":"A","filters":{"items":[{"id":"fv1","key":{"dataType":"string","id":"deployment.environment--string--resource","isColumn":false,"key":"deployment.environment","type":"resource"},"op":"IN","value":"$environment"},{"id":"fv2","key":{"dataType":"string","id":"k8s.cluster.name--string--resource","isColumn":false,"key":"k8s.cluster.name","type":"resource"},"op":"IN","value":"$k8s_cluster"},{"id":"fv3","key":{"dataType":"string","id":"k8s.pod.name--string--resource","isColumn":false,"key":"k8s.pod.name","type":"resource"},"op":"IN","value":"$k8s_pod"}],"op":"AND"},"functions":[],"groupBy":[],"legend":"Reductions/s","orderBy":[],"queryName":"A","selectColumns":[],"stepInterval":60}],"queryFormulas":[]},"id":"6a213747-fccf-44b6-b393-abae8e3a856e","queryType":"builder"},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"softMax":0,"softMin":0,"timePreferance":"GLOBAL_TIME","title":"Reductions Rate","yAxisUnit":"short"},{"bucketCount":30,"bucketWidth":0,"columnUnits":{},"contextLinks":{"linksData":[]},"customLegendColors":{},"decimalPrecision":2,"description":">85% is critical — BEAM refuses to spawn new processes at the limit.","fillMode":"none","fillSpans":false,"id":"d0d18ace-cca9-4d5a-8440-86be0469022d","isLogScale":false,"legendPosition":"bottom","lineInterpolation":"spline","lineStyle":"solid","mergeAllActiveQueries":false,"nullZeroValues":"zero","opacity":"1","panelTypes":"graph","query":{"builder":{"queryData":[{"aggregations":[{"metricName":"erlang_vm_process_count","reduceTo":"avg","spaceAggregation":"max","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"A","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[{"dataType":"string","id":"k8s.pod.name--string--tag","isColumn":false,"isJSON":false,"key":"k8s.pod.name","type":"tag"}],"having":{"expression":""},"legend":"","limit":null,"orderBy":[],"queryName":"A","selectColumns":[],"source":"","stepInterval":60},{"aggregations":[{"metricName":"erlang_vm_process_limit","reduceTo":"avg","spaceAggregation":"max","temporality":"unspecified","timeAggregation":"latest"}],"dataSource":"metrics","disabled":true,"expression":"B","filter":{"expression":"deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod"},"functions":[],"groupBy":[{"dataType":"string","id":"k8s.pod.name--string--tag","isColumn":false,"isJSON":false,"key":"k8s.pod.name","type":"tag"}],"having":{"expression":""},"legend":"","limit":null,"orderBy":[],"queryName":"B","selectColumns":[],"source":"","stepInterval":60}],"queryFormulas":[{"disabled":false,"expression":"A / B * 100","legend":"{{k8s.pod.name}}","queryName":"F1"}]},"clickhouse_sql":[{"disabled":false,"legend":"","name":"A","query":""}],"id":"6fb58024-22cd-42d0-9865-f89351f3fbbe","promql":[{"disabled":false,"legend":"","name":"A","query":""}],"queryType":"builder","unit":""},"selectedLogFields":[{"fieldContext":"log","name":"timestamp","signal":"logs","type":"log"},{"fieldContext":"log","name":"body","signal":"logs","type":"log"}],"selectedTracesFields":[{"fieldContext":"resource","fieldDataType":"string","name":"service.name","signal":"traces"},{"fieldContext":"span","fieldDataType":"string","name":"name","signal":"traces"},{"fieldContext":"span","name":"duration_nano","signal":"traces"}],"showPoints":false,"softMax":0,"softMin":0,"spanGaps":true,"stackedBarChart":false,"thresholds":[],"timePreferance":"GLOBAL_TIME","title":"Process Utilization %","yAxisUnit":"%"}],"layout":[{"h":3,"i":"w1","moved":false,"static":false,"w":2,"x":0,"y":0},{"h":3,"i":"w2","moved":false,"static":false,"w":2,"x":2,"y":0},{"h":3,"i":"w3","moved":false,"static":false,"w":2,"x":4,"y":0},{"h":3,"i":"w4","moved":false,"static":false,"w":2,"x":6,"y":0},{"h":3,"i":"w5","moved":false,"static":false,"w":2,"x":8,"y":0},{"h":3,"i":"w6","moved":false,"static":false,"w":2,"x":10,"y":0},{"h":1,"i":"w7","moved":false,"static":false,"w":12,"x":0,"y":3},{"h":7,"i":"w8","moved":false,"static":false,"w":4,"x":0,"y":4},{"h":7,"i":"w9","moved":false,"static":false,"w":4,"x":4,"y":4},{"h":7,"i":"w10","moved":false,"static":false,"w":4,"x":8,"y":4},{"h":1,"i":"w11","moved":false,"static":false,"w":12,"x":0,"y":11},{"h":7,"i":"w12","moved":false,"static":false,"w":6,"x":0,"y":12},{"h":7,"i":"w13","moved":false,"static":false,"w":6,"x":6,"y":12},{"h":7,"i":"d0d18ace-cca9-4d5a-8440-86be0469022d","moved":false,"static":false,"w":6,"x":0,"y":19},{"h":7,"i":"w14","moved":false,"static":false,"w":6,"x":6,"y":19},{"h":1,"i":"w15","moved":false,"static":false,"w":12,"x":0,"y":26},{"h":7,"i":"w16","moved":false,"static":false,"w":3,"x":0,"y":27},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":3,"y":27},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":6,"y":27},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":9,"y":27}],"panelMap":{"w11":{"collapsed":false,"widgets":[{"h":7,"i":"w12","moved":false,"static":false,"w":6,"x":0,"y":12},{"h":7,"i":"w13","moved":false,"static":false,"w":6,"x":6,"y":12},{"h":7,"i":"w14","moved":false,"static":false,"w":12,"x":0,"y":19},{"h":1,"i":"w15","moved":false,"static":false,"w":12,"x":0,"y":26},{"h":7,"i":"w16","moved":false,"static":false,"w":3,"x":0,"y":27},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":3,"y":27},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":6,"y":27},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":9,"y":27}]},"w7":{"collapsed":false,"widgets":[{"h":7,"i":"w8","moved":false,"static":false,"w":4,"x":0,"y":4},{"h":7,"i":"w9","moved":false,"static":false,"w":4,"x":4,"y":4},{"h":7,"i":"w10","moved":false,"static":false,"w":4,"x":8,"y":4},{"h":1,"i":"w11","moved":false,"static":false,"w":12,"x":0,"y":11},{"h":7,"i":"w12","moved":false,"static":false,"w":6,"x":0,"y":12},{"h":7,"i":"w13","moved":false,"static":false,"w":6,"x":6,"y":12},{"h":7,"i":"w14","moved":false,"static":false,"w":12,"x":0,"y":19},{"h":1,"i":"w15","moved":false,"static":false,"w":12,"x":0,"y":26},{"h":7,"i":"w16","moved":false,"static":false,"w":3,"x":0,"y":27},{"h":7,"i":"w17","moved":false,"static":false,"w":3,"x":3,"y":27},{"h":7,"i":"w18","moved":false,"static":false,"w":3,"x":6,"y":27},{"h":7,"i":"w19","moved":false,"static":false,"w":3,"x":9,"y":27}]}},"version":"v5","title":"BEAM / Erlang VM - Overview"} +{ + "description": "Key BEAM runtime health indicators. First stop for on-call triage — process/port saturation, memory pressure, scheduler backlog, GC activity. Generic for any Elixir/Erlang project using prometheus_ex.", + "layout": [ + { + "h": 3, + "i": "w1", + "moved": false, + "static": false, + "w": 2, + "x": 0, + "y": 0 + }, + { + "h": 3, + "i": "w2", + "moved": false, + "static": false, + "w": 2, + "x": 2, + "y": 0 + }, + { + "h": 3, + "i": "w3", + "moved": false, + "static": false, + "w": 2, + "x": 4, + "y": 0 + }, + { + "h": 3, + "i": "w4", + "moved": false, + "static": false, + "w": 2, + "x": 6, + "y": 0 + }, + { + "h": 3, + "i": "w5", + "moved": false, + "static": false, + "w": 2, + "x": 8, + "y": 0 + }, + { + "h": 3, + "i": "w6", + "moved": false, + "static": false, + "w": 2, + "x": 10, + "y": 0 + }, + { + "h": 1, + "i": "w7", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 3 + }, + { + "h": 7, + "i": "w8", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 4 + }, + { + "h": 7, + "i": "w9", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 4 + }, + { + "h": 7, + "i": "w10", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 4 + }, + { + "h": 1, + "i": "w11", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 11 + }, + { + "h": 7, + "i": "w12", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 47 + }, + { + "h": 7, + "i": "w13", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 47 + }, + { + "h": 7, + "i": "d0d18ace-cca9-4d5a-8440-86be0469022d", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 54 + }, + { + "h": 7, + "i": "w14", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 54 + }, + { + "h": 1, + "i": "w15", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 61 + } + ], + "panelMap": { + "w11": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "w12", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 47 + }, + { + "h": 7, + "i": "w13", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 47 + }, + { + "h": 7, + "i": "d0d18ace-cca9-4d5a-8440-86be0469022d", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 54 + }, + { + "h": 7, + "i": "w14", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 54 + } + ] + }, + "w15": { + "collapsed": true, + "widgets": [ + { + "h": 7, + "i": "w16", + "moved": false, + "static": false, + "w": 3, + "x": 0, + "y": 209 + }, + { + "h": 7, + "i": "w17", + "moved": false, + "static": false, + "w": 3, + "x": 3, + "y": 209 + }, + { + "h": 7, + "i": "w18", + "moved": false, + "static": false, + "w": 3, + "x": 6, + "y": 209 + }, + { + "h": 7, + "i": "w19", + "moved": false, + "static": false, + "w": 3, + "x": 9, + "y": 209 + } + ] + }, + "w7": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "w8", + "moved": false, + "static": false, + "w": 4, + "x": 0, + "y": 4 + }, + { + "h": 7, + "i": "w9", + "moved": false, + "static": false, + "w": 4, + "x": 4, + "y": 4 + }, + { + "h": 7, + "i": "w10", + "moved": false, + "static": false, + "w": 4, + "x": 8, + "y": 4 + } + ] + } + }, + "tags": [ + "beam", + "erlang", + "elixir", + "overview" + ], + "title": "BEAM / Erlang VM - Overview", + "variables": { + "environment": { + "allSelected": true, + "description": "Deployment environment filter", + "dynamicVariablesAttribute": "deployment.environment", + "dynamicVariablesSource": "Metrics", + "id": "var-environment", + "multiSelect": true, + "name": "environment", + "order": 0, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + }, + "k8s_cluster": { + "allSelected": true, + "description": "Kubernetes cluster name filter", + "dynamicVariablesAttribute": "k8s.cluster.name", + "dynamicVariablesSource": "Metrics", + "id": "var-k8s-cluster", + "multiSelect": true, + "name": "k8s_cluster", + "order": 1, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + }, + "k8s_pod": { + "allSelected": true, + "description": "Pod name filter — select specific pods to scope all panels to one application", + "dynamicVariablesAttribute": "k8s.pod.name", + "dynamicVariablesSource": "Metrics", + "haveCustomValuesSelected": false, + "id": "var-k8s-pod", + "multiSelect": true, + "name": "k8s_pod", + "order": 2, + "showALLOption": true, + "sort": "ASC", + "type": "DYNAMIC" + } + }, + "version": "v5", + "widgets": [ + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "% of process limit in use. >85% is critical — node will crash at 100%.", + "fillMode": "none", + "fillSpans": false, + "id": "w1", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_process_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "c9912465-de59-4645-b89d-8c6625222c29", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 100, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [ + { + "index": "0", + "thresholdColor": "#FFA500", + "thresholdFormat": "Background", + "thresholdLabel": "Warning", + "thresholdOperator": ">", + "thresholdValue": 70 + }, + { + "index": "1", + "keyIndex": 1, + "thresholdColor": "#FF0000", + "thresholdFormat": "Background", + "thresholdLabel": "Critical", + "thresholdOperator": ">", + "thresholdValue": 85 + } + ], + "timePreferance": "GLOBAL_TIME", + "title": "Process Utilization", + "yAxisUnit": "percent" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "% of port limit in use. Ports = sockets, files, NIFs.", + "fillMode": "none", + "fillSpans": false, + "id": "w2", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_port_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_port_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "4e912e40-34ac-43e1-abec-0059a1d54bd2", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 100, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [ + { + "index": "0", + "thresholdColor": "#FFA500", + "thresholdFormat": "Background", + "thresholdLabel": "Warning", + "thresholdOperator": ">", + "thresholdValue": 70 + }, + { + "index": "1", + "keyIndex": 1, + "thresholdColor": "#FF0000", + "thresholdFormat": "Background", + "thresholdLabel": "Critical", + "thresholdOperator": ">", + "thresholdValue": 85 + } + ], + "timePreferance": "GLOBAL_TIME", + "title": "Port Utilization", + "yAxisUnit": "percent" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": "% of atom table used. Atom table never shrinks — runaway growth = leak.", + "fillMode": "none", + "fillSpans": false, + "id": "w3", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_count", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_atom_limit", + "reduceTo": "avg", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "aggregations": [], + "dataSource": "metrics", + "disabled": false, + "expression": "A / B * 100", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "", + "orderBy": [], + "queryName": "F1", + "selectColumns": [], + "stepInterval": null + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "42aacb5f-0752-4d92-a370-412be4a007ec", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 100, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [ + { + "index": "0", + "thresholdColor": "#FFA500", + "thresholdFormat": "Background", + "thresholdLabel": "Warning", + "thresholdOperator": ">", + "thresholdValue": 70 + }, + { + "index": "1", + "keyIndex": 1, + "thresholdColor": "#FF0000", + "thresholdFormat": "Background", + "thresholdLabel": "Critical", + "thresholdOperator": ">", + "thresholdValue": 85 + } + ], + "timePreferance": "GLOBAL_TIME", + "title": "Atom Utilization", + "yAxisUnit": "percent" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Number of ETS tables currently open.", + "id": "w4", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_ets_tables", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "31a03bbe-951d-4804-b963-6467fbf4cfa3", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "ETS Tables", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "CPU scheduler threads available to the BEAM.", + "id": "w5", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_schedulers_online", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "1818e786-b85d-486e-a178-2e74f3787812", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Schedulers Online", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total BEAM memory allocation (processes + system).", + "id": "w6", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "034d759b-9272-4b5d-acbc-a8ecbcca65da", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Total Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "w7", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Memory" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "w8", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Total", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "a9d086d8-f2fe-471e-acf9-4e98882a5639", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Total Memory", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Stacked composition of BEAM memory — watch for any segment growing unbounded.", + "id": "w9", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_processes_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Processes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_system_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "System", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_memory_atom_bytes_total", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "C", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Atom", + "orderBy": [], + "queryName": "C", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "beam_memory_binary_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "D", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Binary", + "orderBy": [], + "queryName": "D", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "beam_memory_ets_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "E", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "ETS", + "orderBy": [], + "queryName": "E", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "beam_memory_code_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "F", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Code", + "orderBy": [], + "queryName": "F", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "deeddc90-b07a-46cd-839d-f21ac488d28d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Memory Breakdown", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Steady monotonic growth indicates a binary leak (unreferenced binaries not GC'd).", + "id": "w10", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "beam_memory_binary_bytes", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Binary", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "718c7582-29a1-4453-b16e-8949bb23f0ff", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Binary Memory (off-heap)", + "yAxisUnit": "bytes" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "w11", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "Schedulers & Run Queues" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Normal scheduler run queue. Sustained > 0 = CPU saturation.", + "id": "w12", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_run_queues_length", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Run Queue", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "51a166ed-a243-48ac-8470-bda62852a44d", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Run Queue Length", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Dirty CPU = long-running NIF pressure. Dirty IO = file/socket I/O pressure.", + "id": "w13", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_dirty_cpu_run_queue_length", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dirty CPU", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_dirty_io_run_queue_length", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "B", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Dirty IO", + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "9b84336d-4a49-4741-8415-f0234a989666", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Dirty CPU / IO Run Queues", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Total Erlang processes. Runaway growth = process leak (missing termination or gen_server loop).", + "id": "w14", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "spaceAggregation": "avg", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Processes", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "da910f90-465d-4865-b6cb-02a7e33de9d9", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Live Processes", + "yAxisUnit": "short" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "", + "id": "w15", + "panelTypes": "row", + "query": { + "builder": { + "queryData": [], + "queryFormulas": [] + }, + "id": "", + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": null, + "title": "GC & Reductions" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Garbage collections per second. Spikes indicate sudden allocation pressure.", + "id": "w16", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_number_of_gcs", + "spaceAggregation": "sum", + "temporality": "cumulative", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "GCs/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "42d2abba-afb3-437b-bcd4-2ca0584f1126", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Rate", + "yAxisUnit": "reqps" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Bytes reclaimed per second by the garbage collector.", + "id": "w17", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_garbage_collection_bytes_reclaimed", + "spaceAggregation": "sum", + "temporality": "cumulative", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Bytes/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "f3225155-7370-4922-b824-8c259bdcf563", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "GC Bytes Reclaimed", + "yAxisUnit": "binBps" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "Scheduler context switches per second. High rate = many short-lived processes.", + "id": "w18", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_context_switches", + "spaceAggregation": "sum", + "temporality": "cumulative", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Switches/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "7c57ba19-7d01-4843-b418-72abab7d6201", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Context Switch Rate", + "yAxisUnit": "reqps" + }, + { + "contextLinks": { + "linksData": [] + }, + "description": "BEAM reductions per second — proxy for total CPU work across all processes.", + "id": "w19", + "legendPosition": "bottom", + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_statistics_reductions_total", + "spaceAggregation": "sum", + "temporality": "cumulative", + "timeAggregation": "rate" + } + ], + "dataSource": "metrics", + "expression": "A", + "filters": { + "items": [ + { + "id": "fv1", + "key": { + "dataType": "string", + "id": "deployment.environment--string--resource", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + "op": "IN", + "value": "$environment" + }, + { + "id": "fv2", + "key": { + "dataType": "string", + "id": "k8s.cluster.name--string--resource", + "isColumn": false, + "key": "k8s.cluster.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_cluster" + }, + { + "id": "fv3", + "key": { + "dataType": "string", + "id": "k8s.pod.name--string--resource", + "isColumn": false, + "key": "k8s.pod.name", + "type": "resource" + }, + "op": "IN", + "value": "$k8s_pod" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "legend": "Reductions/s", + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "stepInterval": 60 + } + ], + "queryFormulas": [] + }, + "id": "6a213747-fccf-44b6-b393-abae8e3a856e", + "queryType": "builder" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "softMax": 0, + "softMin": 0, + "timePreferance": "GLOBAL_TIME", + "title": "Reductions Rate", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "contextLinks": { + "linksData": [] + }, + "customLegendColors": {}, + "decimalPrecision": 2, + "description": ">85% is critical — BEAM refuses to spawn new processes at the limit.", + "fillMode": "none", + "fillSpans": false, + "id": "d0d18ace-cca9-4d5a-8440-86be0469022d", + "isLogScale": false, + "legendPosition": "bottom", + "lineInterpolation": "spline", + "lineStyle": "solid", + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregations": [ + { + "metricName": "erlang_vm_process_count", + "reduceTo": "avg", + "spaceAggregation": "max", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "A", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "k8s.pod.name--string--tag", + "isColumn": false, + "isJSON": false, + "key": "k8s.pod.name", + "type": "tag" + } + ], + "having": { + "expression": "" + }, + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "selectColumns": [], + "source": "", + "stepInterval": 60 + }, + { + "aggregations": [ + { + "metricName": "erlang_vm_process_limit", + "reduceTo": "avg", + "spaceAggregation": "max", + "temporality": "unspecified", + "timeAggregation": "latest" + } + ], + "dataSource": "metrics", + "disabled": true, + "expression": "B", + "filter": { + "expression": "deployment.environment in $environment AND k8s.cluster.name in $k8s_cluster AND k8s.pod.name in $k8s_pod" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "k8s.pod.name--string--tag", + "isColumn": false, + "isJSON": false, + "key": "k8s.pod.name", + "type": "tag" + } + ], + "having": { + "expression": "" + }, + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "B", + "selectColumns": [], + "source": "", + "stepInterval": 60 + } + ], + "queryFormulas": [ + { + "disabled": false, + "expression": "A / B * 100", + "legend": "{{k8s.pod.name}}", + "queryName": "F1" + } + ] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "6fb58024-22cd-42d0-9865-f89351f3fbbe", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder", + "unit": "" + }, + "selectedLogFields": [ + { + "fieldContext": "log", + "name": "timestamp", + "signal": "logs", + "type": "log" + }, + { + "fieldContext": "log", + "name": "body", + "signal": "logs", + "type": "log" + } + ], + "selectedTracesFields": [ + { + "fieldContext": "resource", + "fieldDataType": "string", + "name": "service.name", + "signal": "traces" + }, + { + "fieldContext": "span", + "fieldDataType": "string", + "name": "name", + "signal": "traces" + }, + { + "fieldContext": "span", + "name": "duration_nano", + "signal": "traces" + } + ], + "showPoints": false, + "softMax": 0, + "softMin": 0, + "spanGaps": true, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Process Utilization %", + "yAxisUnit": "%" + } + ] +}