diff --git a/installer/install.py b/installer/install.py index 5145f28..25a0995 100644 --- a/installer/install.py +++ b/installer/install.py @@ -12,7 +12,6 @@ import json import requests - BASE_DIR = '/tmp' DKUBE_PATH = BASE_DIR + '/dkube' KUBEFLOW_APPNAME = 'my-kubeflow' @@ -46,7 +45,7 @@ def cmd_help(cmd): elif (cmd == "delete"): pretty_red("SYNTAX: %s %s --pkg "% (sys.argv[0], cmd)) elif (cmd == "operator"): - pretty_red("SYNTAX: %s %s --add --org --token "% (sys.argv[0], cmd)) + pretty_red("SYNTAX: %s %s --add --org "% (sys.argv[0], cmd)) sys.exit(1) def find_master_ip(): @@ -58,17 +57,17 @@ def find_master_ip(): return ip_addr -def operator_add(user, org, token): - master_ip = find_master_ip() - url = "http://%s:32222/GPUaaS/operator/super"%master_ip - job = {'username': user, 'organization': org, 'token':token} - data = json.dumps(job) - headers = {'Content-Type': 'application/json'} - result = requests.post(url, data=data, headers=headers) - print("status:",result.status_code) - if (result.status_code != 200): - pretty_red("Operator add Failed") - sys.exit(1) +def operator_add(user, org): + master_ip = find_master_ip() + url = "http://%s:32222/GPUaaS/operator/super"%master_ip + job = {'username': user, 'organization': org} + data = json.dumps(job) + headers = {'Content-Type': 'application/json'} + result = requests.post(url, data=data, headers=headers) + print("status:",result.status_code) + if (result.status_code != 200): + pretty_red("Operator add Failed") + sys.exit(1) def operator_delete(user, org): pretty_red("This Feature is not available for now !!! please check the installation document") @@ -468,6 +467,9 @@ def dkube_delete(): sys.exit(1) time.sleep(2) + if not sp.call("df -h | grep /var/dkube/spinner > /dev/null",shell=True, executable='/bin/bash'): + sp.call("umount /var/dkube/spinner",shell=True, executable='/bin/bash') + time.sleep(1) if os.path.isdir('/var/dkube/'): shutil.rmtree('/var/dkube/') @@ -511,12 +513,12 @@ def delete_kubeflow(): pretty_green("Kubeflow deletion is Done") def handle_operator(args): - if( (not args.add) or (not args.org) or (not args.token)): + if( (not args.add) or (not args.org)): cmd_help("operator") elif(args.add): username = args.add pretty_green("Adding operator ...") - operator_add(username, args.org, args.token) + operator_add(username, args.org) pretty_green("Operator addition is Done ...!!!") def handle_delete(args): @@ -795,7 +797,6 @@ def run(): parser.add_argument("--add", help="Username of operator to be added") #parser.add_argument("--delete", help="Username of operator to be deleted") parser.add_argument("--org", help="name of the organization for operator") - parser.add_argument("--token", help="personal token for operator") parser.add_argument("--client_id", help="Client ID for git OAuth app") parser.add_argument("--client_secret", help="Client Secret for git OAuth app") parser.add_argument("--docker_username", help="Username for docker hub") diff --git a/ksonnet/monitoring/all.libsonnet b/ksonnet/monitoring/all.libsonnet index cb82380..f6645ea 100644 --- a/ksonnet/monitoring/all.libsonnet +++ b/ksonnet/monitoring/all.libsonnet @@ -1,7 +1,8 @@ { parts(params):: { - local monitoring = import "dkube/monitoring/monitoring.libsonnet", + local grafana_config = import "dkube/monitoring/grafana_config.libsonnet", + local gpu_monitoring = import "dkube/monitoring/gpu_monitoring.libsonnet", - all:: monitoring.all(params) + all:: grafana_config.all(params) + gpu_monitoring.all(params) }, } diff --git a/ksonnet/monitoring/gpu_monitoring.libsonnet b/ksonnet/monitoring/gpu_monitoring.libsonnet new file mode 100644 index 0000000..009b125 --- /dev/null +++ b/ksonnet/monitoring/gpu_monitoring.libsonnet @@ -0,0 +1,376 @@ +{ + all(params):: [ + + { + "apiVersion": "extensions/v1beta1", + "kind": "Deployment", + "metadata": { + "annotations": null, + "labels": { + "app": "dkube-prometheus-grafana", + "chart": "grafana-0.0.37", + "heritage": "Tiller", + "release": "kube-prometheus" + }, + "name": "dkube-grafana", + "namespace": "monitoring" + }, + "spec": { + "progressDeadlineSeconds": 600, + "replicas": 1, + "revisionHistoryLimit": 10, + "selector": { + "matchLabels": { + "app": "dkube-prometheus-grafana" + } + }, + "strategy": { + "rollingUpdate": { + "maxSurge": 1, + "maxUnavailable": 1 + }, + "type": "RollingUpdate" + }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "app": "dkube-prometheus-grafana", + "release": "kube-prometheus" + } + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "GF_AUTH_BASIC_ENABLED", + "value": "true" + }, + { + "name": "GF_AUTH_ANONYMOUS_ENABLED", + "value": "true" + }, + { + "name": "GF_SECURITY_ADMIN_USER", + "valueFrom": { + "secretKeyRef": { + "key": "user", + "name": "kube-prometheus-grafana" + } + } + }, + { + "name": "GF_SECURITY_ADMIN_PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "kube-prometheus-grafana" + } + } + } + ], + "image": "grafana/grafana:5.0.0", + "imagePullPolicy": "IfNotPresent", + "name": "grafana", + "ports": [ + { + "containerPort": 3000, + "name": "web", + "protocol": "TCP" + } + ], + "readinessProbe": { + "failureThreshold": 10, + "httpGet": { + "path": "/api/health", + "port": 3000, + "scheme": "HTTP" + }, + "periodSeconds": 1, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/lib/grafana", + "name": "grafana-storage" + }, + { + "mountPath": "/etc/grafana/grafana.ini", + "name": "grafana-config", + "subPath": "grafana.ini" + } + ] + }, + { + "args": [ + "--watch-dir=/var/grafana-dashboards", + "--grafana-url=http://127.0.0.1:3000" + ], + "env": [ + { + "name": "GRAFANA_USER", + "valueFrom": { + "secretKeyRef": { + "key": "user", + "name": "kube-prometheus-grafana" + } + } + }, + { + "name": "GRAFANA_PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "kube-prometheus-grafana" + } + } + } + ], + "image": "quay.io/coreos/grafana-watcher:v0.0.8", + "imagePullPolicy": "IfNotPresent", + "name": "grafana-watcher", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/grafana-dashboards", + "name": "grafana-dashboards" + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "kube-prometheus-grafana", + "serviceAccountName": "kube-prometheus-grafana", + "terminationGracePeriodSeconds": 30, + "volumes": [ + { + "emptyDir": {}, + "name": "grafana-storage" + }, + { + "configMap": { + "defaultMode": 420, + "name": "dkube-grafana-dashboard" + }, + "name": "grafana-dashboards" + }, + { + "configMap": { + "defaultMode": 420, + "name": "dkube-grafana-init" + }, + "name": "grafana-config" + } + ] + } + } + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "annotations": { + "getambassador.io/config": "---\napiVersion: ambassador/v0\nkind: Mapping\nname: dkube_monitoring\nprefix: /dkube/grafana/\nrewrite: /\nservice: dkube-grafana.monitoring:80\ntimeout_ms: 60000\nuse_websocket: true" + }, + "labels": { + "app": "dkube-prometheus-grafana" + }, + "name": "dkube-prometheus-grafana-proxy", + "namespace": "dkube" + }, + "spec": { + "externalTrafficPolicy": "Cluster", + "ports": [ + { + "name": "http", + "nodePort": 31227, + "port": 80, + "protocol": "TCP", + "targetPort": 3000 + } + ], + "selector": { + "app": "dkube-prometheus-grafana" + }, + "sessionAffinity": "None", + "type": "NodePort" + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "annotations": null, + "labels": { + "app": "dkube-prometheus-grafana" + }, + "name": "dkube-grafana", + "namespace": "monitoring" + }, + "spec": { + "externalTrafficPolicy": "Cluster", + "ports": [ + { + "name": "http", + "nodePort": 30990, + "port": 80, + "protocol": "TCP", + "targetPort": 3000 + } + ], + "selector": { + "app": "dkube-prometheus-grafana" + }, + "sessionAffinity": "None", + "type": "NodePort" + } + }, + { + "apiVersion": "monitoring.coreos.com/v1", + "kind": "ServiceMonitor", + "metadata": { + "labels": { + "app": "prometheus", + "prometheus": "kube-prometheus" + }, + "name": "dkube-prometheus-exporter-gpu", + "namespace": "monitoring" + }, + "spec": { + "endpoints": [ + { + "interval": "15s", + "port": "http-metrics" + } + ], + "jobLabel": "k8s-app", + "namespaceSelector": { + "matchNames": [ + "monitoring" + ] + }, + "selector": { + "matchLabels": { + "app": "dkube-gpu-exporter" + } + } + } + }, + { + "apiVersion": "extensions/v1beta1", + "kind": "DaemonSet", + "metadata": { + "name": "nvidia-exporter", + "namespace": "monitoring" + }, + "spec": { + "template": { + "metadata": { + "labels": { + "app": "nvidia-exporter" + } + }, + "spec": { + "containers": [ + { + "image": "ocdr/dkube-gpu-exporter:v3", + "name": "nvidia-exporter", + "ports": [ + { + "containerPort": 9401, + "name": "http-metrics" + } + ], + "securityContext": { + "privileged": true + }, + "volumeMounts": [ + { + "mountPath": "/usr/local/nvidia", + "name": "nvidia" + }, + { + "mountPath": "/usr/lib", + "name": "nvml" + }, + { + "mountPath": "/usr/lib64", + "name": "lib64" + } + ] + } + ], + "hostNetwork": true, + "imagePullSecrets": [ + { + "name": "dkube-dockerhub-secret" + } + ], + "volumes": [ + { + "hostPath": { + "path": "/opt/nvidia/current" + }, + "name": "nvidia" + }, + { + "hostPath": { + "path": "/usr/lib" + }, + "name": "nvml" + }, + { + "hostPath": { + "path": "/usr/lib64" + }, + "name": "lib64" + } + ] + } + } + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "annotations": { + "prometheus.io/port": "9401", + "prometheus.io/scrape": "true" + }, + "labels": { + "app": "dkube-gpu-exporter" + }, + "name": "nvidia-exporter", + "namespace": "monitoring" + }, + "spec": { + "ports": [ + { + "name": "http-metrics", + "nodePort": 30001, + "port": 9401, + "protocol": "TCP", + "targetPort": 9401 + } + ], + "selector": { + "app": "nvidia-exporter" + }, + "type": "NodePort" + } + } + ] +} + diff --git a/ksonnet/monitoring/monitoring.libsonnet b/ksonnet/monitoring/grafana_config.libsonnet similarity index 79% rename from ksonnet/monitoring/monitoring.libsonnet rename to ksonnet/monitoring/grafana_config.libsonnet index 06be267..92d426e 100644 --- a/ksonnet/monitoring/monitoring.libsonnet +++ b/ksonnet/monitoring/grafana_config.libsonnet @@ -1,6 +1,5 @@ { all(params):: [ - { "apiVersion": "v1", "data": { @@ -21,8 +20,8 @@ "kubernetes-cluster-status-dashboard.json": "{\n \"dashboard\": {\n \"__inputs\": [\n {\n \"description\": \"\",\n \"label\": \"prometheus\",\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"pluginName\": \"Prometheus\",\n \"type\": \"datasource\"\n }\n ],\n \"annotations\": {\n \"list\": []\n },\n \"editable\": true,\n \"graphTooltip\": 0,\n \"hideControls\": false,\n \"links\": [],\n \"rows\": [\n {\n \"collapse\": false,\n \"height\": \"129px\",\n \"panels\": [\n {\n \"colorBackground\": false,\n \"colorValue\": true,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 5,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 6,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"sum(up{job=~\\\"apiserver|kube-scheduler|kube-controller-manager\\\"} == 0)\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"1, 3\",\n \"title\": \"Control Plane UP\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"UP\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"total\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": true,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 6,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 6,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"sum(ALERTS{alertstate=\\\"firing\\\",alertname!=\\\"DeadMansSwitch\\\"})\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"3, 5\",\n \"title\": \"Alerts Firing\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"0\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n }\n ],\n \"showTitle\": true,\n \"title\": \"Cluster Health\",\n \"titleSize\": \"h6\"\n },\n {\n \"collapse\": false,\n \"height\": \"168px\",\n \"panels\": [\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 1,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"(sum(up{job=\\\"apiserver\\\"} == 1) / count(up{job=\\\"apiserver\\\"})) * 100\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"50, 80\",\n \"title\": \"API Servers UP\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 2,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"(sum(up{job=\\\"kube-controller-manager\\\"} == 1) / count(up{job=\\\"kube-controller-manager\\\"})) * 100\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"50, 80\",\n \"title\": \"Controller Managers UP\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 3,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"(sum(up{job=\\\"kube-scheduler\\\"} == 1) / count(up{job=\\\"kube-scheduler\\\"})) * 100\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"50, 80\",\n \"title\": \"Schedulers UP\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": true,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 4,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"count(increase(kube_pod_container_status_restarts_total{namespace=~\\\"kube-system|tectonic-system\\\"}[1h]) > 5)\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"1, 3\",\n \"title\": \"Crashlooping Control Plane Pods\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"0\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n }\n ],\n \"showTitle\": true,\n \"title\": \"Control Plane Status\",\n \"titleSize\": \"h6\"\n },\n {\n \"collapse\": false,\n \"height\": \"158px\",\n \"panels\": [\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 8,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"sum(100 - (avg by (instance) (rate(node_cpu{job=\\\"node-exporter\\\",mode=\\\"idle\\\"}[5m])) * 100)) / count(node_cpu{job=\\\"node-exporter\\\",mode=\\\"idle\\\"})\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"80, 90\",\n \"title\": \"CPU Utilization\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 7,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"((sum(node_memory_MemTotal) - sum(node_memory_MemFree) - sum(node_memory_Buffers) - sum(node_memory_Cached)) / sum(node_memory_MemTotal)) * 100\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"80, 90\",\n \"title\": \"Memory Utilization\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 9,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"(sum(node_filesystem_size{device!=\\\"rootfs\\\"}) - sum(node_filesystem_free{device!=\\\"rootfs\\\"})) / sum(node_filesystem_size{device!=\\\"rootfs\\\"})\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"80, 90\",\n \"title\": \"Filesystem Utilization\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 10,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"100 - (sum(kube_node_status_capacity_pods) - sum(kube_pod_info)) / sum(kube_node_status_capacity_pods) * 100\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"80, 90\",\n \"title\": \"Pod Utilization\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n }\n ],\n \"showTitle\": true,\n \"title\": \"Capacity Planning\",\n \"titleSize\": \"h6\"\n }\n ],\n \"schemaVersion\": 14,\n \"sharedCrosshair\": false,\n \"style\": \"dark\",\n \"tags\": [],\n \"templating\": {\n \"list\": []\n },\n \"time\": {\n \"from\": \"now-6h\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"Kubernetes Cluster Status\",\n \"version\": 3\n },\n \"inputs\": [\n {\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"prometheus\"\n }\n ],\n \"overwrite\": true\n}\n", "kubernetes-control-plane-status-dashboard.json": "{\n \"dashboard\": {\n \"__inputs\": [\n {\n \"description\": \"\",\n \"label\": \"prometheus\",\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"pluginName\": \"Prometheus\",\n \"type\": \"datasource\"\n }\n ],\n \"annotations\": {\n \"list\": []\n },\n \"editable\": true,\n \"graphTooltip\": 0,\n \"hideControls\": false,\n \"links\": [],\n \"rows\": [\n {\n \"collapse\": false,\n \"editable\": true,\n \"height\": \"250px\",\n \"panels\": [\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"hideTimeOverride\": false,\n \"id\": 1,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"(sum(up{job=\\\"apiserver\\\"} == 1) / sum(up{job=\\\"apiserver\\\"})) * 100\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"50, 80\",\n \"title\": \"API Servers UP\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"hideTimeOverride\": false,\n \"id\": 2,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"(sum(up{job=\\\"kube-controller-manager\\\"} == 1) / sum(up{job=\\\"kube-controller-manager\\\"})) * 100\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"50, 80\",\n \"title\": \"Controller Managers UP\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"hideTimeOverride\": false,\n \"id\": 3,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"(sum(up{job=\\\"kube-scheduler\\\"} == 1) / sum(up{job=\\\"kube-scheduler\\\"})) * 100\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"50, 80\",\n \"title\": \"Schedulers UP\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"hideTimeOverride\": false,\n \"id\": 4,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"max(sum by(instance) (rate(apiserver_request_count{code=~\\\"5..\\\"}[5m])) / sum by(instance) (rate(apiserver_request_count[5m]))) * 100\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"\",\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"thresholds\": \"5, 10\",\n \"title\": \"API Server Request Error Rate\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"0\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n }\n ],\n \"showTitle\": false,\n \"title\": \"Dashboard Row\",\n \"titleSize\": \"h6\"\n },\n {\n \"collapse\": false,\n \"editable\": true,\n \"height\": \"250px\",\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"id\": 7,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"span\": 12,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"sum by(verb) (rate(apiserver_latency_seconds:quantile[5m]) >= 0)\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"\",\n \"refId\": \"A\",\n \"step\": 30\n }\n ],\n \"title\": \"API Server Request Latency\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"mode\": \"time\",\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n }\n ],\n \"showTitle\": false,\n \"title\": \"Dashboard Row\",\n \"titleSize\": \"h6\"\n },\n {\n \"collapse\": false,\n \"editable\": true,\n \"height\": \"250px\",\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"id\": 5,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"span\": 6,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"cluster:scheduler_e2e_scheduling_latency_seconds:quantile\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 60\n }\n ],\n \"title\": \"End to End Scheduling Latency\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"mode\": \"time\",\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"dtdurations\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"id\": 6,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"span\": 6,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"sum by(instance) (rate(apiserver_request_count{code!~\\\"2..\\\"}[5m]))\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"Error Rate\",\n \"refId\": \"A\",\n \"step\": 60\n },\n {\n \"expr\": \"sum by(instance) (rate(apiserver_request_count[5m]))\",\n \"format\": \"time_series\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"Request Rate\",\n \"refId\": \"B\",\n \"step\": 60\n }\n ],\n \"title\": \"API Server Request Rates\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"mode\": \"time\",\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n }\n ],\n \"showTitle\": false,\n \"title\": \"Dashboard Row\",\n \"titleSize\": \"h6\"\n }\n ],\n \"schemaVersion\": 14,\n \"sharedCrosshair\": false,\n \"style\": \"dark\",\n \"tags\": [],\n \"templating\": {\n \"list\": []\n },\n \"time\": {\n \"from\": \"now-6h\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"Kubernetes Control Plane Status\",\n \"version\": 3\n },\n \"inputs\": [\n {\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"prometheus\"\n }\n ],\n \"overwrite\": true\n}", "kubernetes-resource-requests-dashboard.json": "{\n \"dashboard\": {\n \"__inputs\": [\n {\n \"description\": \"\",\n \"label\": \"prometheus\",\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"pluginName\": \"Prometheus\",\n \"type\": \"datasource\"\n }\n ],\n \"annotations\": {\n \"list\": []\n },\n \"editable\": true,\n \"graphTooltip\": 0,\n \"hideControls\": false,\n \"links\": [],\n \"refresh\": false,\n \"rows\": [\n {\n \"collapse\": false,\n \"editable\": true,\n \"height\": \"300px\",\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"description\": \"This represents the total [CPU resource requests](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu) in the cluster.\\nFor comparison the total [allocatable CPU cores](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node-allocatable.md) is also shown.\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"id\": 1,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"span\": 9,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"min(sum(kube_node_status_allocatable_cpu_cores) by (instance))\",\n \"hide\": false,\n \"intervalFactor\": 2,\n \"legendFormat\": \"Allocatable CPU Cores\",\n \"refId\": \"A\",\n \"step\": 20\n },\n {\n \"expr\": \"max(sum(kube_pod_container_resource_requests_cpu_cores) by (instance))\",\n \"hide\": false,\n \"intervalFactor\": 2,\n \"legendFormat\": \"Requested CPU Cores\",\n \"refId\": \"B\",\n \"step\": 20\n }\n ],\n \"title\": \"CPU Cores\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"mode\": \"time\",\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"label\": \"CPU Cores\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"hideTimeOverride\": false,\n \"id\": 2,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"targets\": [\n {\n \"expr\": \"max(sum(kube_pod_container_resource_requests_cpu_cores) by (instance)) / min(sum(kube_node_status_allocatable_cpu_cores) by (instance)) * 100\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"\",\n \"refId\": \"A\",\n \"step\": 240\n }\n ],\n \"thresholds\": \"80, 90\",\n \"title\": \"CPU Cores\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"110%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n }\n ],\n \"showTitle\": false,\n \"title\": \"CPU Cores\",\n \"titleSize\": \"h6\"\n },\n {\n \"collapse\": false,\n \"editable\": true,\n \"height\": \"300px\",\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"description\": \"This represents the total [memory resource requests](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory) in the cluster.\\nFor comparison the total [allocatable memory](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node-allocatable.md) is also shown.\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"id\": 3,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"span\": 9,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"min(sum(kube_node_status_allocatable_memory_bytes) by (instance))\",\n \"hide\": false,\n \"intervalFactor\": 2,\n \"legendFormat\": \"Allocatable Memory\",\n \"refId\": \"A\",\n \"step\": 20\n },\n {\n \"expr\": \"max(sum(kube_pod_container_resource_requests_memory_bytes) by (instance))\",\n \"hide\": false,\n \"intervalFactor\": 2,\n \"legendFormat\": \"Requested Memory\",\n \"refId\": \"B\",\n \"step\": 20\n }\n ],\n \"title\": \"Memory\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"mode\": \"time\",\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"label\": \"Memory\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"hideTimeOverride\": false,\n \"id\": 4,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"targets\": [\n {\n \"expr\": \"max(sum(kube_pod_container_resource_requests_memory_bytes) by (instance)) / min(sum(kube_node_status_allocatable_memory_bytes) by (instance)) * 100\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"\",\n \"refId\": \"A\",\n \"step\": 240\n }\n ],\n \"thresholds\": \"80, 90\",\n \"title\": \"Memory\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"110%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n }\n ],\n \"showTitle\": false,\n \"title\": \"Memory\",\n \"titleSize\": \"h6\"\n }\n ],\n \"schemaVersion\": 14,\n \"sharedCrosshair\": false,\n \"style\": \"dark\",\n \"tags\": [],\n \"templating\": {\n \"list\": []\n },\n \"time\": {\n \"from\": \"now-3h\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"Kubernetes Resource Requests\",\n \"version\": 2\n },\n \"inputs\": [\n {\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"prometheus\"\n }\n ],\n \"overwrite\": true\n}", - "nodes-dashboard.json": "{\n \"dashboard\": {\n \"__inputs\": [\n {\n \"description\": \"\",\n \"label\": \"prometheus\",\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"pluginName\": \"Prometheus\",\n \"type\": \"datasource\"\n }\n ],\n \"annotations\": {\n \"list\": [\n {\n \"$$hashKey\": \"object:84\",\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"description\": \"Dashboard to get an overview of one server\",\n \"editable\": true,\n \"gnetId\": 22,\n \"graphTooltip\": 0,\n \"iteration\": 1534235842839,\n \"links\": [],\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 3,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"100 - (avg by (cpu) (irate(node_cpu{mode=\\\"idle\\\", instance=\\\"$server\\\"}[5m])) * 100)\",\n \"hide\": false,\n \"intervalFactor\": 10,\n \"legendFormat\": \"{{cpu}}\",\n \"refId\": \"A\",\n \"step\": 50\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Idle CPU\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"percent\",\n \"label\": \"cpu usage\",\n \"logBase\": 1,\n \"max\": 100,\n \"min\": 0,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\": 12,\n \"y\": 0\n },\n \"id\": 9,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"node_load1{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"load 1m\",\n \"refId\": \"A\",\n \"step\": 20,\n \"target\": \"\"\n },\n {\n \"expr\": \"node_load5{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"load 5m\",\n \"refId\": \"B\",\n \"step\": 20,\n \"target\": \"\"\n },\n {\n \"expr\": \"node_load15{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"load 15m\",\n \"refId\": \"C\",\n \"step\": 20,\n \"target\": \"\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"System Load\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"percentunit\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 18,\n \"x\": 0,\n \"y\": 7\n },\n \"id\": 4,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {\n \"alias\": \"node_memory_SwapFree{instance=\\\"172.17.0.1:9100\\\",job=\\\"prometheus\\\"}\",\n \"yaxis\": 2\n }\n ],\n \"spaceLength\": 10,\n \"stack\": true,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"node_memory_MemTotal{instance=\\\"$server\\\"} - node_memory_MemFree{instance=\\\"$server\\\"} - node_memory_Buffers{instance=\\\"$server\\\"} - node_memory_Cached{instance=\\\"$server\\\"}\",\n \"hide\": false,\n \"interval\": \"\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"memory used\",\n \"metric\": \"\",\n \"refId\": \"C\",\n \"step\": 10\n },\n {\n \"expr\": \"node_memory_Buffers{instance=\\\"$server\\\"}\",\n \"interval\": \"\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"memory buffers\",\n \"metric\": \"\",\n \"refId\": \"E\",\n \"step\": 10\n },\n {\n \"expr\": \"node_memory_Cached{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"memory cached\",\n \"metric\": \"\",\n \"refId\": \"F\",\n \"step\": 10\n },\n {\n \"expr\": \"node_memory_MemFree{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"memory free\",\n \"metric\": \"\",\n \"refId\": \"D\",\n \"step\": 10\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Memory Usage\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"min\": \"0\",\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"cacheTimeout\": null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 6,\n \"x\": 18,\n \"y\": 7\n },\n \"hideTimeOverride\": false,\n \"id\": 5,\n \"interval\": null,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"tableColumn\": \"\",\n \"targets\": [\n {\n \"expr\": \"((node_memory_MemTotal{instance=\\\"$server\\\"} - node_memory_MemFree{instance=\\\"$server\\\"} - node_memory_Buffers{instance=\\\"$server\\\"} - node_memory_Cached{instance=\\\"$server\\\"}) / node_memory_MemTotal{instance=\\\"$server\\\"}) * 100\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 60,\n \"target\": \"\"\n }\n ],\n \"thresholds\": \"80, 90\",\n \"title\": \"Memory Usage\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 18,\n \"x\": 0,\n \"y\": 14\n },\n \"id\": 6,\n \"isNew\": true,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {\n \"alias\": \"read\",\n \"yaxis\": 1\n },\n {\n \"alias\": \"{instance=\\\"172.17.0.1:9100\\\"}\",\n \"yaxis\": 2\n },\n {\n \"alias\": \"io time\",\n \"yaxis\": 2\n }\n ],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"sum by (instance) (rate(node_disk_bytes_read{instance=\\\"$server\\\"}[2m]))\",\n \"hide\": false,\n \"intervalFactor\": 4,\n \"legendFormat\": \"read\",\n \"refId\": \"A\",\n \"step\": 20,\n \"target\": \"\"\n },\n {\n \"expr\": \"sum by (instance) (rate(node_disk_bytes_written{instance=\\\"$server\\\"}[2m]))\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"written\",\n \"refId\": \"B\",\n \"step\": 20\n },\n {\n \"expr\": \"sum by (instance) (rate(node_disk_io_time_ms{instance=\\\"$server\\\"}[2m]))\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"io time\",\n \"refId\": \"C\",\n \"step\": 20\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Disk I/O\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"ms\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"cacheTimeout\": null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"format\": \"percentunit\",\n \"gauge\": {\n \"maxValue\": 1,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 6,\n \"x\": 18,\n \"y\": 14\n },\n \"hideTimeOverride\": false,\n \"id\": 7,\n \"interval\": null,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"tableColumn\": \"\",\n \"targets\": [\n {\n \"expr\": \"(sum(node_filesystem_size{device!=\\\"rootfs\\\",instance=\\\"$server\\\"}) - sum(node_filesystem_free{device!=\\\"rootfs\\\",instance=\\\"$server\\\"})) / sum(node_filesystem_size{device!=\\\"rootfs\\\",instance=\\\"$server\\\"})\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 60,\n \"target\": \"\"\n }\n ],\n \"thresholds\": \"0.75, 0.9\",\n \"title\": \"Disk Space Usage\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\": 0,\n \"y\": 21\n },\n \"id\": 8,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {\n \"alias\": \"transmitted\",\n \"yaxis\": 2\n }\n ],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"rate(node_network_receive_bytes{instance=\\\"$server\\\",device!~\\\"lo\\\"}[5m])\",\n \"hide\": false,\n \"intervalFactor\": 2,\n \"legendFormat\": \"{{device}}\",\n \"refId\": \"A\",\n \"step\": 10,\n \"target\": \"\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Network Received\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\": 12,\n \"y\": 21\n },\n \"id\": 10,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {\n \"alias\": \"transmitted\",\n \"yaxis\": 2\n }\n ],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"rate(node_network_transmit_bytes{instance=\\\"$server\\\",device!~\\\"lo\\\"}[5m])\",\n \"hide\": false,\n \"intervalFactor\": 2,\n \"legendFormat\": \"{{device}}\",\n \"refId\": \"B\",\n \"step\": 10,\n \"target\": \"\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Network Transmitted\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n }\n ],\n \"refresh\": false,\n \"schemaVersion\": 16,\n \"style\": \"dark\",\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"allValue\": null,\n \"current\": {\n \"text\": \"192.168.6.6:9100\",\n \"value\": \"192.168.6.6:9100\"\n },\n \"datasource\": \"prometheus\",\n \"hide\": 2,\n \"includeAll\": false,\n \"label\": null,\n \"multi\": false,\n \"name\": \"server\",\n \"options\": [],\n \"query\": \"label_values(node_boot_time, instance)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n }\n ]\n },\n \"time\": {\n \"from\": \"now-1h\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"Nodes\",\n \"uid\": \"0GobAq5mz\",\n \"version\": 2\n },\n \"inputs\": [\n {\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"prometheus\"\n }\n ],\n \"overwrite\": true\n}", - "nvidia-dashboard.json": "{\n \"dashboard\": {\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"description\": \"GPU dashboard for nvidia metrics \",\n \"editable\": true,\n \"gnetId\": 6387,\n \"graphTooltip\": 1,\n \"iteration\": 1534237424254,\n \"links\": [],\n \"panels\": [\n {\n \"gridPos\": {\n \"h\": 4,\n \"w\": 4,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 27,\n \"limit\": 10,\n \"links\": [],\n \"onlyAlertsOnDashboard\": true,\n \"show\": \"current\",\n \"sortOrder\": 1,\n \"stateFilter\": [],\n \"title\": \"Alerts\",\n \"transparent\": true,\n \"type\": \"alertlist\"\n },\n {\n \"cacheTimeout\": null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"#d44a3a\",\n \"rgba(237, 129, 40, 0.89)\",\n \"#299c46\"\n ],\n \"datasource\": \"prometheus\",\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 3,\n \"x\": 4,\n \"y\": 0\n },\n \"id\": 22,\n \"interval\": null,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n \"postfix\": \"%\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"tableColumn\": \"\",\n \"targets\": [\n {\n \"expr\": \"sum(nvidia_utilization_gpu{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}) / $count\",\n \"format\": \"time_series\",\n \"hide\": true,\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": \"\",\n \"title\": \"GPU Utilization\",\n \"transparent\": true,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n },\n {\n \"cacheTimeout\": null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"#d44a3a\",\n \"rgba(237, 129, 40, 0.89)\",\n \"#299c46\"\n ],\n \"datasource\": \"prometheus\",\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 3,\n \"x\": 7,\n \"y\": 0\n },\n \"id\": 23,\n \"interval\": null,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n \"postfix\": \"%\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"tableColumn\": \"\",\n \"targets\": [\n {\n \"expr\": \"sum(nvidia_utilization_memory{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}) / $count\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": \"\",\n \"title\": \"Memory Utilization\",\n \"transparent\": true,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 0,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 7,\n \"x\": 10,\n \"y\": 0\n },\n \"id\": 10,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": false,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {}\n ],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_utilization_gpu{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}} - gpu{{minor}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"GPU Utilization / Card\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"transparent\": true,\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"percent\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 0,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 7,\n \"x\": 17,\n \"y\": 0\n },\n \"id\": 11,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": false,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_utilization_memory{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}} - gpu{{minor}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Memory Utilization / Card\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"transparent\": true,\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"percent\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"cacheTimeout\": null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"#299c46\",\n \"rgba(237, 129, 40, 0.89)\",\n \"#d44a3a\"\n ],\n \"datasource\": \"prometheus\",\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 2,\n \"w\": 2,\n \"x\": 2,\n \"y\": 4\n },\n \"id\": 25,\n \"interval\": null,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"tableColumn\": \"\",\n \"targets\": [\n {\n \"expr\": \"sum(nvidia_device_count{instance=~\\\"$instance\\\"})\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"instant\": true,\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": \"\",\n \"title\": \"# GPUs\",\n \"transparent\": true,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"120%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 0,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 24,\n \"x\": 0,\n \"y\": 6\n },\n \"id\": 2,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": true,\n \"max\": true,\n \"min\": false,\n \"rightSide\": true,\n \"show\": true,\n \"sort\": \"current\",\n \"sortDesc\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_temperatures{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"instant\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}} - gpu{{minor}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [\n {\n \"colorMode\": \"critical\",\n \"fill\": true,\n \"line\": true,\n \"op\": \"gt\",\n \"value\": 90\n }\n ],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"GPU Temperature\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"decimals\": null,\n \"format\": \"celsius\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 0,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 24,\n \"x\": 0,\n \"y\": 12\n },\n \"id\": 12,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": true,\n \"max\": true,\n \"min\": false,\n \"rightSide\": true,\n \"show\": false,\n \"sort\": \"avg\",\n \"sortDesc\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_fanspeed{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"instant\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Fan Speed\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"decimals\": null,\n \"format\": \"percent\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 4,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 24,\n \"x\": 0,\n \"y\": 18\n },\n \"id\": 20,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": true,\n \"max\": false,\n \"min\": false,\n \"rightSide\": true,\n \"show\": false,\n \"sort\": \"current\",\n \"sortDesc\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": true,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_power_usage{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}} - gpu{{minor}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Power Usage\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"mwatt\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n }\n ],\n \"refresh\": \"30s\",\n \"schemaVersion\": 16,\n \"style\": \"dark\",\n \"tags\": [\n \"gpu\",\n \"cluster\",\n \"hardware\",\n \"infrastructure\"\n ],\n \"templating\": {\n \"list\": [\n {\n \"allValue\": null,\n \"current\": {\n \"selected\": true,\n \"text\": \"192.168.6.6:9401\",\n \"value\": \"192.168.6.6:9401\"\n },\n \"datasource\": \"prometheus\",\n \"hide\": 2,\n \"includeAll\": true,\n \"label\": null,\n \"multi\": false,\n \"name\": \"instance\",\n \"options\": [],\n \"query\": \"label_values(instance)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"allValue\": null,\n \"current\": {\n \"selected\": true,\n \"text\": \"0\",\n \"value\": \"0\"\n },\n \"datasource\": \"prometheus\",\n \"hide\": 2,\n \"includeAll\": true,\n \"label\": null,\n \"multi\": false,\n \"name\": \"minor\",\n \"options\": [],\n \"query\": \"label_values(minor)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"current\": {\n \"text\": \"1\",\n \"value\": \"1\"\n },\n \"hide\": 2,\n \"label\": null,\n \"name\": \"count\",\n \"options\": [\n {\n \"selected\": false,\n \"text\": \"\",\n \"value\": \"\"\n }\n ],\n \"query\": \"\",\n \"type\": \"constant\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now/d\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"\",\n \"title\": \"nvidia-gpu\",\n \"uid\": \"fnvhnzGmk\",\n \"version\": 9\n },\n\n \"inputs\": [{\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"prometheus\"\n }],\n \"overwrite\": true\n} ", + "nodes-dashboard.json": "{\n \"dashboard\": {\n \"__inputs\": [\n {\n \"description\": \"\",\n \"label\": \"prometheus\",\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"pluginName\": \"Prometheus\",\n \"type\": \"datasource\"\n }\n ],\n \"annotations\": {\n \"list\": [\n {\n \"$$hashKey\": \"object:84\",\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"description\": \"Dashboard to get an overview of one server\",\n \"editable\": true,\n \"gnetId\": 22,\n \"graphTooltip\": 0,\n \"iteration\": 1534765404973,\n \"links\": [],\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 3,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"100 - (avg by (cpu) (irate(node_cpu{mode=\\\"idle\\\", instance=\\\"$server\\\"}[5m])) * 100)\",\n \"hide\": false,\n \"intervalFactor\": 10,\n \"legendFormat\": \"{{cpu}}\",\n \"refId\": \"A\",\n \"step\": 50\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"CPU Usage\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"percent\",\n \"label\": \"cpu usage\",\n \"logBase\": 1,\n \"max\": 100,\n \"min\": 0,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\": 12,\n \"y\": 0\n },\n \"id\": 9,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"node_load1{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"load 1m\",\n \"refId\": \"A\",\n \"step\": 20,\n \"target\": \"\"\n },\n {\n \"expr\": \"node_load5{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"load 5m\",\n \"refId\": \"B\",\n \"step\": 20,\n \"target\": \"\"\n },\n {\n \"expr\": \"node_load15{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"load 15m\",\n \"refId\": \"C\",\n \"step\": 20,\n \"target\": \"\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"System Load\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"percentunit\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 18,\n \"x\": 0,\n \"y\": 7\n },\n \"id\": 4,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {\n \"alias\": \"node_memory_SwapFree{instance=\\\"172.17.0.1:9100\\\",job=\\\"prometheus\\\"}\",\n \"yaxis\": 2\n }\n ],\n \"spaceLength\": 10,\n \"stack\": true,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"node_memory_MemTotal{instance=\\\"$server\\\"} - node_memory_MemFree{instance=\\\"$server\\\"} - node_memory_Buffers{instance=\\\"$server\\\"} - node_memory_Cached{instance=\\\"$server\\\"}\",\n \"hide\": false,\n \"interval\": \"\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"memory used\",\n \"metric\": \"\",\n \"refId\": \"C\",\n \"step\": 10\n },\n {\n \"expr\": \"node_memory_Buffers{instance=\\\"$server\\\"}\",\n \"interval\": \"\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"memory buffers\",\n \"metric\": \"\",\n \"refId\": \"E\",\n \"step\": 10\n },\n {\n \"expr\": \"node_memory_Cached{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"memory cached\",\n \"metric\": \"\",\n \"refId\": \"F\",\n \"step\": 10\n },\n {\n \"expr\": \"node_memory_MemFree{instance=\\\"$server\\\"}\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"memory free\",\n \"metric\": \"\",\n \"refId\": \"D\",\n \"step\": 10\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Memory Usage\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"min\": \"0\",\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"cacheTimeout\": null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"format\": \"percent\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 6,\n \"x\": 18,\n \"y\": 7\n },\n \"hideTimeOverride\": false,\n \"id\": 5,\n \"interval\": null,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"tableColumn\": \"\",\n \"targets\": [\n {\n \"expr\": \"((node_memory_MemTotal{instance=\\\"$server\\\"} - node_memory_MemFree{instance=\\\"$server\\\"} - node_memory_Buffers{instance=\\\"$server\\\"} - node_memory_Cached{instance=\\\"$server\\\"}) / node_memory_MemTotal{instance=\\\"$server\\\"}) * 100\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 60,\n \"target\": \"\"\n }\n ],\n \"thresholds\": \"80, 90\",\n \"title\": \"Memory Usage\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 18,\n \"x\": 0,\n \"y\": 14\n },\n \"id\": 6,\n \"isNew\": true,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {\n \"alias\": \"read\",\n \"yaxis\": 1\n },\n {\n \"alias\": \"{instance=\\\"172.17.0.1:9100\\\"}\",\n \"yaxis\": 2\n },\n {\n \"alias\": \"io time\",\n \"yaxis\": 2\n }\n ],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"sum by (instance) (rate(node_disk_bytes_read{instance=\\\"$server\\\"}[2m]))\",\n \"hide\": false,\n \"intervalFactor\": 4,\n \"legendFormat\": \"read\",\n \"refId\": \"A\",\n \"step\": 20,\n \"target\": \"\"\n },\n {\n \"expr\": \"sum by (instance) (rate(node_disk_bytes_written{instance=\\\"$server\\\"}[2m]))\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"written\",\n \"refId\": \"B\",\n \"step\": 20\n },\n {\n \"expr\": \"sum by (instance) (rate(node_disk_io_time_ms{instance=\\\"$server\\\"}[2m]))\",\n \"intervalFactor\": 4,\n \"legendFormat\": \"io time\",\n \"refId\": \"C\",\n \"step\": 20\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Disk I/O\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"ms\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"cacheTimeout\": null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(50, 172, 45, 0.97)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(245, 54, 54, 0.9)\"\n ],\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"format\": \"percentunit\",\n \"gauge\": {\n \"maxValue\": 1,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 6,\n \"x\": 18,\n \"y\": 14\n },\n \"hideTimeOverride\": false,\n \"id\": 7,\n \"interval\": null,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"tableColumn\": \"\",\n \"targets\": [\n {\n \"expr\": \"(sum(node_filesystem_size{device!=\\\"rootfs\\\",instance=\\\"$server\\\"}) - sum(node_filesystem_free{device!=\\\"rootfs\\\",instance=\\\"$server\\\"})) / sum(node_filesystem_size{device!=\\\"rootfs\\\",instance=\\\"$server\\\"})\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 60,\n \"target\": \"\"\n }\n ],\n \"thresholds\": \"0.75, 0.9\",\n \"title\": \"Disk Space Usage\",\n \"transparent\": false,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\": 0,\n \"y\": 21\n },\n \"id\": 8,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {\n \"alias\": \"transmitted\",\n \"yaxis\": 2\n }\n ],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"rate(node_network_receive_bytes{instance=\\\"$server\\\",device!~\\\"lo\\\"}[5m])\",\n \"hide\": false,\n \"intervalFactor\": 2,\n \"legendFormat\": \"{{device}}\",\n \"refId\": \"A\",\n \"step\": 10,\n \"target\": \"\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Network Received\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \"x\": 12,\n \"y\": 21\n },\n \"id\": 10,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {\n \"alias\": \"transmitted\",\n \"yaxis\": 2\n }\n ],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"rate(node_network_transmit_bytes{instance=\\\"$server\\\",device!~\\\"lo\\\"}[5m])\",\n \"hide\": false,\n \"intervalFactor\": 2,\n \"legendFormat\": \"{{device}}\",\n \"refId\": \"B\",\n \"step\": 10,\n \"target\": \"\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Network Transmitted\",\n \"tooltip\": {\n \"msResolution\": false,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n }\n ],\n \"refresh\": false,\n \"schemaVersion\": 16,\n \"style\": \"dark\",\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"allValue\": null,\n \"current\": {\n \"text\": \"172.16.0.101:9100\",\n \"value\": \"172.16.0.101:9100\"\n },\n \"datasource\": \"prometheus\",\n \"hide\": 2,\n \"includeAll\": false,\n \"label\": null,\n \"multi\": false,\n \"name\": \"server\",\n \"options\": [],\n \"query\": \"label_values(node_boot_time, instance)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n }\n ]\n },\n \"time\": {\n \"from\": \"now-1h\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"Nodes\",\n \"uid\": \"0GobAq5mz\",\n \"version\": 1\n },\n \"inputs\": [\n {\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"prometheus\"\n }\n ],\n \"overwrite\": true\n}", + "nvidia-dashboard.json": "{\n \"dashboard\": {\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": \"-- Grafana --\",\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"description\": \"GPU dashboard for nvidia metrics \",\n \"editable\": true,\n \"gnetId\": 6387,\n \"graphTooltip\": 1,\n \"iteration\": 1534763580304,\n \"links\": [],\n \"panels\": [\n {\n \"gridPos\": {\n \"h\": 6,\n \"w\": 4,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 27,\n \"limit\": 10,\n \"links\": [],\n \"onlyAlertsOnDashboard\": true,\n \"show\": \"current\",\n \"sortOrder\": 1,\n \"stateFilter\": [],\n \"title\": \"Alerts\",\n \"transparent\": true,\n \"type\": \"alertlist\"\n },\n {\n \"cacheTimeout\": null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"#d44a3a\",\n \"rgba(237, 129, 40, 0.89)\",\n \"#299c46\"\n ],\n \"datasource\": \"prometheus\",\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 3,\n \"x\": 4,\n \"y\": 0\n },\n \"id\": 22,\n \"interval\": null,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n \"postfix\": \"%\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"tableColumn\": \"\",\n \"targets\": [\n {\n \"$$hashKey\": \"object:183\",\n \"expr\": \"sum(nvidia_utilization_gpu{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}) / $count\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": \"\",\n \"title\": \"GPU Utilization\",\n \"transparent\": true,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n },\n {\n \"cacheTimeout\": null,\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"#d44a3a\",\n \"rgba(237, 129, 40, 0.89)\",\n \"#299c46\"\n ],\n \"datasource\": \"prometheus\",\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": true,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"gridPos\": {\n \"h\": 6,\n \"w\": 3,\n \"x\": 7,\n \"y\": 0\n },\n \"id\": 23,\n \"interval\": null,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"nullText\": null,\n \"postfix\": \"%\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"tableColumn\": \"\",\n \"targets\": [\n {\n \"expr\": \"sum(nvidia_utilization_memory{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}) / $count\",\n \"format\": \"time_series\",\n \"intervalFactor\": 1,\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": \"\",\n \"title\": \"Memory Utilization\",\n \"transparent\": true,\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"current\"\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 0,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 7,\n \"x\": 10,\n \"y\": 0\n },\n \"id\": 10,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": false,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [\n {}\n ],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_utilization_gpu{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}} - gpu{{minor}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"GPU Utilization / Card\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"transparent\": true,\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"percent\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 0,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 7,\n \"x\": 17,\n \"y\": 0\n },\n \"id\": 11,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": false,\n \"total\": false,\n \"values\": false\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_utilization_memory{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}} - gpu{{minor}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Memory Utilization / Card\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"transparent\": true,\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"percent\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 0,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 24,\n \"x\": 0,\n \"y\": 6\n },\n \"id\": 2,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": true,\n \"max\": true,\n \"min\": false,\n \"rightSide\": true,\n \"show\": true,\n \"sort\": \"current\",\n \"sortDesc\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_temperatures{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"instant\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}} - gpu{{minor}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [\n {\n \"colorMode\": \"critical\",\n \"fill\": true,\n \"line\": true,\n \"op\": \"gt\",\n \"value\": 90\n }\n ],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"GPU Temperature\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"decimals\": null,\n \"format\": \"celsius\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 0,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 24,\n \"x\": 0,\n \"y\": 12\n },\n \"id\": 12,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": true,\n \"max\": true,\n \"min\": false,\n \"rightSide\": true,\n \"show\": false,\n \"sort\": \"avg\",\n \"sortDesc\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_fanspeed{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"instant\": false,\n \"interval\": \"\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Fan Speed\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"decimals\": null,\n \"format\": \"percent\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n },\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"prometheus\",\n \"fill\": 4,\n \"gridPos\": {\n \"h\": 6,\n \"w\": 24,\n \"x\": 0,\n \"y\": 18\n },\n \"id\": 20,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": true,\n \"max\": false,\n \"min\": false,\n \"rightSide\": true,\n \"show\": false,\n \"sort\": \"current\",\n \"sortDesc\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n \"linewidth\": 1,\n \"links\": [],\n \"nullPointMode\": \"null\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"stack\": true,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"nvidia_power_usage{instance=~\\\"$instance\\\", minor=~\\\"$minor\\\"}\",\n \"format\": \"time_series\",\n \"hide\": false,\n \"intervalFactor\": 1,\n \"legendFormat\": \"{{kubernetes_pod_node_name}} - gpu{{minor}}\",\n \"refId\": \"A\"\n }\n ],\n \"thresholds\": [],\n \"timeFrom\": null,\n \"timeShift\": null,\n \"title\": \"Power Usage\",\n \"tooltip\": {\n \"shared\": true,\n \"sort\": 2,\n \"value_type\": \"individual\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"buckets\": null,\n \"mode\": \"time\",\n \"name\": null,\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"mwatt\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": null,\n \"logBase\": 1,\n \"max\": null,\n \"min\": null,\n \"show\": true\n }\n ],\n \"yaxis\": {\n \"align\": false,\n \"alignLevel\": null\n }\n }\n ],\n \"refresh\": \"30s\",\n \"schemaVersion\": 16,\n \"style\": \"dark\",\n \"tags\": [\n \"gpu\",\n \"cluster\",\n \"hardware\",\n \"infrastructure\"\n ],\n \"templating\": {\n \"list\": [\n {\n \"allValue\": null,\n \"current\": {\n \"text\": \"All\",\n \"value\": \"$__all\"\n },\n \"datasource\": \"prometheus\",\n \"hide\": 2,\n \"includeAll\": true,\n \"label\": null,\n \"multi\": false,\n \"name\": \"instance\",\n \"options\": [],\n \"query\": \"label_values(instance)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"allValue\": null,\n \"current\": {\n \"selected\": true,\n \"text\": \"0\",\n \"value\": \"0\"\n },\n \"datasource\": \"prometheus\",\n \"hide\": 2,\n \"includeAll\": true,\n \"label\": null,\n \"multi\": false,\n \"name\": \"minor\",\n \"options\": [],\n \"query\": \"label_values(minor)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"current\": {\n \"text\": \"1\",\n \"value\": \"1\"\n },\n \"hide\": 2,\n \"label\": null,\n \"name\": \"count\",\n \"options\": [\n {\n \"selected\": false,\n \"text\": \"\",\n \"value\": \"\"\n }\n ],\n \"query\": \"\",\n \"type\": \"constant\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now/d\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"\",\n \"title\": \"nvidia-gpu\",\n \"uid\": \"fnvhnzGmk\",\n \"version\": 2\n\n },\n\n \"inputs\": [{\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"prometheus\"\n }],\n \"overwrite\": true\n} ", "pods-dashboard.json": "{\n \"dashboard\": {\n \"__inputs\": [\n {\n \"description\": \"\",\n \"label\": \"prometheus\",\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"pluginName\": \"Prometheus\",\n \"type\": \"datasource\"\n }\n ],\n \"annotations\": {\n \"list\": []\n },\n \"editable\": true,\n \"graphTooltip\": 1,\n \"hideControls\": false,\n \"links\": [],\n \"refresh\": false,\n \"rows\": [\n {\n \"collapse\": false,\n \"editable\": true,\n \"height\": \"250px\",\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"id\": 1,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": true,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": true,\n \"show\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"span\": 12,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"sum by(container_name) (container_memory_usage_bytes{pod_name=\\\"$pod\\\", container_name=~\\\"$container\\\", container_name!=\\\"POD\\\"})\",\n \"interval\": \"10s\",\n \"intervalFactor\": 1,\n \"legendFormat\": \"Current: {{ container_name }}\",\n \"metric\": \"container_memory_usage_bytes\",\n \"refId\": \"A\",\n \"step\": 15\n },\n {\n \"expr\": \"kube_pod_container_resource_requests_memory_bytes{pod=\\\"$pod\\\", container=~\\\"$container\\\"}\",\n \"interval\": \"10s\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"Requested: {{ container }}\",\n \"metric\": \"kube_pod_container_resource_requests_memory_bytes\",\n \"refId\": \"B\",\n \"step\": 20\n },\n {\n \"expr\": \"kube_pod_container_resource_limits_memory_bytes{pod=\\\"$pod\\\", container=~\\\"$container\\\"}\",\n \"interval\": \"10s\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"Limit: {{ container }}\",\n \"metric\": \"kube_pod_container_resource_limits_memory_bytes\",\n \"refId\": \"C\",\n \"step\": 20\n }\n ],\n \"title\": \"Memory Usage\",\n \"tooltip\": {\n \"msResolution\": true,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"mode\": \"time\",\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n }\n ],\n \"showTitle\": false,\n \"title\": \"Row\",\n \"titleSize\": \"h6\"\n },\n {\n \"collapse\": false,\n \"editable\": true,\n \"height\": \"250px\",\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"id\": 2,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": true,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": true,\n \"show\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"span\": 12,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"sum by (container_name)(rate(container_cpu_usage_seconds_total{image!=\\\"\\\",container_name!=\\\"POD\\\",pod_name=\\\"$pod\\\"}[1m]))\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"{{ container_name }}\",\n \"refId\": \"A\",\n \"step\": 30\n },\n {\n \"expr\": \"kube_pod_container_resource_requests_cpu_cores{pod=\\\"$pod\\\", container=~\\\"$container\\\"}\",\n \"interval\": \"10s\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"Requested: {{ container }}\",\n \"metric\": \"kube_pod_container_resource_requests_cpu_cores\",\n \"refId\": \"B\",\n \"step\": 20\n },\n {\n \"expr\": \"kube_pod_container_resource_limits_cpu_cores{pod=\\\"$pod\\\", container=~\\\"$container\\\"}\",\n \"interval\": \"10s\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"Limit: {{ container }}\",\n \"metric\": \"kube_pod_container_resource_limits_memory_bytes\",\n \"refId\": \"C\",\n \"step\": 20\n }\n ],\n \"title\": \"CPU Usage\",\n \"tooltip\": {\n \"msResolution\": true,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"mode\": \"time\",\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n }\n ],\n \"showTitle\": false,\n \"title\": \"Row\",\n \"titleSize\": \"h6\"\n },\n {\n \"collapse\": false,\n \"editable\": true,\n \"height\": \"250px\",\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"id\": 3,\n \"isNew\": false,\n \"legend\": {\n \"alignAsTable\": true,\n \"avg\": true,\n \"current\": true,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": true,\n \"show\": true,\n \"total\": false,\n \"values\": true\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"span\": 12,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"sort_desc(sum by (pod_name) (rate(container_network_receive_bytes_total{pod_name=\\\"$pod\\\"}[1m])))\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"{{ pod_name }}\",\n \"refId\": \"A\",\n \"step\": 30\n }\n ],\n \"title\": \"Network I/O\",\n \"tooltip\": {\n \"msResolution\": true,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"mode\": \"time\",\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"bytes\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"logBase\": 1,\n \"show\": true\n }\n ]\n }\n ],\n \"showTitle\": false,\n \"title\": \"New Row\",\n \"titleSize\": \"h6\"\n }\n ],\n \"schemaVersion\": 14,\n \"sharedCrosshair\": false,\n \"style\": \"dark\",\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"allValue\": \".*\",\n \"current\": {},\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"hide\": 0,\n \"includeAll\": true,\n \"label\": \"Namespace\",\n \"multi\": false,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": \"label_values(kube_pod_info, namespace)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"allValue\": null,\n \"current\": {},\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"hide\": 0,\n \"includeAll\": false,\n \"label\": \"Pod\",\n \"multi\": false,\n \"name\": \"pod\",\n \"options\": [],\n \"query\": \"label_values(kube_pod_info{namespace=~\\\"$namespace\\\"}, pod)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"allValue\": \".*\",\n \"current\": {},\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"hide\": 0,\n \"includeAll\": true,\n \"label\": \"Container\",\n \"multi\": false,\n \"name\": \"container\",\n \"options\": [],\n \"query\": \"label_values(kube_pod_container_info{namespace=\\\"$namespace\\\", pod=\\\"$pod\\\"}, container)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n }\n ]\n },\n \"time\": {\n \"from\": \"now-6h\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"Pods\",\n \"version\": 1\n },\n \"inputs\": [\n {\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"prometheus\"\n }\n ],\n \"overwrite\": true\n}", "prometheus-datasource.json": "{\n \"access\": \"proxy\",\n \"basicAuth\": false,\n \"name\": \"prometheus\",\n \"type\": \"prometheus\",\n \"url\": \"http://kube-prometheus:9090\"\n}\n", "statefulset-dashboard.json": "{\n \"dashboard\": {\n \"__inputs\": [\n {\n \"description\": \"\",\n \"label\": \"prometheus\",\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"pluginName\": \"Prometheus\",\n \"type\": \"datasource\"\n }\n ],\n \"annotations\": {\n \"list\": []\n },\n \"editable\": true,\n \"graphTooltip\": 1,\n \"hideControls\": false,\n \"links\": [],\n \"rows\": [\n {\n \"collapse\": false,\n \"height\": \"200px\",\n \"panels\": [\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 8,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfix\": \"cores\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 4,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"targets\": [\n {\n \"expr\": \"sum(rate(container_cpu_usage_seconds_total{namespace=\\\"$statefulset_namespace\\\",pod_name=~\\\"$statefulset_name.*\\\"}[3m]))\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"title\": \"CPU\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"110%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 9,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfix\": \"GB\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"80%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 4,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"targets\": [\n {\n \"expr\": \"sum(container_memory_usage_bytes{namespace=\\\"$statefulset_namespace\\\",pod_name=~\\\"$statefulset_name.*\\\"}) / 1024^3\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"title\": \"Memory\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"110%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"Bps\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": false\n },\n \"id\": 7,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfix\": \"\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 4,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": true\n },\n \"targets\": [\n {\n \"expr\": \"sum(rate(container_network_transmit_bytes_total{namespace=\\\"$statefulset_namespace\\\",pod_name=~\\\"$statefulset_name.*\\\"}[3m])) + sum(rate(container_network_receive_bytes_total{namespace=\\\"$statefulset_namespace\\\",pod_name=~\\\"$statefulset_name.*\\\"}[3m]))\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"title\": \"Network\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n }\n ],\n \"showTitle\": false,\n \"title\": \"Dashboard Row\",\n \"titleSize\": \"h6\"\n },\n {\n \"collapse\": false,\n \"height\": \"100px\",\n \"panels\": [\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": false\n },\n \"id\": 5,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"max(kube_statefulset_replicas{statefulset=\\\"$statefulset_name\\\",namespace=\\\"$statefulset_namespace\\\"}) without (instance, pod)\",\n \"intervalFactor\": 2,\n \"metric\": \"kube_statefulset_replicas\",\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"title\": \"Desired Replicas\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 6,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"min(kube_statefulset_status_replicas{statefulset=\\\"$statefulset_name\\\",namespace=\\\"$statefulset_namespace\\\"}) without (instance, pod)\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"title\": \"Available Replicas\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 3,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"max(kube_statefulset_status_observed_generation{statefulset=\\\"$statefulset_name\\\",namespace=\\\"$statefulset_namespace\\\"}) without (instance, pod)\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"title\": \"Observed Generation\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n },\n {\n \"colorBackground\": false,\n \"colorValue\": false,\n \"colors\": [\n \"rgba(245, 54, 54, 0.9)\",\n \"rgba(237, 129, 40, 0.89)\",\n \"rgba(50, 172, 45, 0.97)\"\n ],\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"format\": \"none\",\n \"gauge\": {\n \"maxValue\": 100,\n \"minValue\": 0,\n \"show\": false,\n \"thresholdLabels\": false,\n \"thresholdMarkers\": true\n },\n \"id\": 2,\n \"links\": [],\n \"mappingType\": 1,\n \"mappingTypes\": [\n {\n \"name\": \"value to text\",\n \"value\": 1\n },\n {\n \"name\": \"range to text\",\n \"value\": 2\n }\n ],\n \"maxDataPoints\": 100,\n \"nullPointMode\": \"connected\",\n \"postfixFontSize\": \"50%\",\n \"prefix\": \"\",\n \"prefixFontSize\": \"50%\",\n \"rangeMaps\": [\n {\n \"from\": \"null\",\n \"text\": \"N/A\",\n \"to\": \"null\"\n }\n ],\n \"span\": 3,\n \"sparkline\": {\n \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n \"full\": false,\n \"lineColor\": \"rgb(31, 120, 193)\",\n \"show\": false\n },\n \"targets\": [\n {\n \"expr\": \"max(kube_statefulset_metadata_generation{statefulset=\\\"$statefulset_name\\\",namespace=\\\"$statefulset_namespace\\\"}) without (instance, pod)\",\n \"intervalFactor\": 2,\n \"refId\": \"A\",\n \"step\": 600\n }\n ],\n \"title\": \"Metadata Generation\",\n \"type\": \"singlestat\",\n \"valueFontSize\": \"80%\",\n \"valueMaps\": [\n {\n \"op\": \"=\",\n \"text\": \"N/A\",\n \"value\": \"null\"\n }\n ],\n \"valueName\": \"avg\"\n }\n ],\n \"showTitle\": false,\n \"title\": \"Dashboard Row\",\n \"titleSize\": \"h6\"\n },\n {\n \"collapse\": false,\n \"height\": \"350px\",\n \"panels\": [\n {\n \"aliasColors\": {},\n \"bars\": false,\n \"dashLength\": 10,\n \"dashes\": false,\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"editable\": true,\n \"error\": false,\n \"fill\": 1,\n \"grid\": {\n \"threshold1Color\": \"rgba(216, 200, 27, 0.27)\",\n \"threshold2Color\": \"rgba(234, 112, 112, 0.22)\"\n },\n \"id\": 1,\n \"isNew\": true,\n \"legend\": {\n \"alignAsTable\": false,\n \"avg\": false,\n \"current\": false,\n \"hideEmpty\": false,\n \"hideZero\": false,\n \"max\": false,\n \"min\": false,\n \"rightSide\": false,\n \"show\": true,\n \"total\": false\n },\n \"lines\": true,\n \"linewidth\": 2,\n \"links\": [],\n \"nullPointMode\": \"connected\",\n \"percentage\": false,\n \"pointradius\": 5,\n \"points\": false,\n \"renderer\": \"flot\",\n \"seriesOverrides\": [],\n \"spaceLength\": 10,\n \"span\": 12,\n \"stack\": false,\n \"steppedLine\": false,\n \"targets\": [\n {\n \"expr\": \"min(kube_statefulset_status_replicas{statefulset=\\\"$statefulset_name\\\",namespace=\\\"$statefulset_namespace\\\"}) without (instance, pod)\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"available\",\n \"refId\": \"B\",\n \"step\": 30\n },\n {\n \"expr\": \"max(kube_statefulset_replicas{statefulset=\\\"$statefulset_name\\\",namespace=\\\"$statefulset_namespace\\\"}) without (instance, pod)\",\n \"intervalFactor\": 2,\n \"legendFormat\": \"desired\",\n \"refId\": \"E\",\n \"step\": 30\n }\n ],\n \"title\": \"Replicas\",\n \"tooltip\": {\n \"msResolution\": true,\n \"shared\": true,\n \"sort\": 0,\n \"value_type\": \"cumulative\"\n },\n \"type\": \"graph\",\n \"xaxis\": {\n \"mode\": \"time\",\n \"show\": true,\n \"values\": []\n },\n \"yaxes\": [\n {\n \"format\": \"none\",\n \"label\": \"\",\n \"logBase\": 1,\n \"show\": true\n },\n {\n \"format\": \"short\",\n \"label\": \"\",\n \"logBase\": 1,\n \"show\": false\n }\n ]\n }\n ],\n \"showTitle\": false,\n \"title\": \"Dashboard Row\",\n \"titleSize\": \"h6\"\n }\n ],\n \"schemaVersion\": 14,\n \"sharedCrosshair\": false,\n \"style\": \"dark\",\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"allValue\": \".*\",\n \"current\": {},\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"hide\": 0,\n \"includeAll\": false,\n \"label\": \"Namespace\",\n \"multi\": false,\n \"name\": \"statefulset_namespace\",\n \"options\": [],\n \"query\": \"label_values(kube_statefulset_metadata_generation, namespace)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": null,\n \"tags\": [],\n \"tagsQuery\": \"\",\n \"type\": \"query\",\n \"useTags\": false\n },\n {\n \"allValue\": null,\n \"current\": {},\n \"datasource\": \"${DS_PROMETHEUS}\",\n \"hide\": 0,\n \"includeAll\": false,\n \"label\": \"StatefulSet\",\n \"multi\": false,\n \"name\": \"statefulset_name\",\n \"options\": [],\n \"query\": \"label_values(kube_statefulset_metadata_generation{namespace=\\\"$statefulset_namespace\\\"}, statefulset)\",\n \"refresh\": 1,\n \"regex\": \"\",\n \"sort\": 0,\n \"tagValuesQuery\": \"\",\n \"tags\": [],\n \"tagsQuery\": \"statefulset\",\n \"type\": \"query\",\n \"useTags\": false\n }\n ]\n },\n \"time\": {\n \"from\": \"now-6h\",\n \"to\": \"now\"\n },\n \"timepicker\": {\n \"refresh_intervals\": [\n \"5s\",\n \"10s\",\n \"30s\",\n \"1m\",\n \"5m\",\n \"15m\",\n \"30m\",\n \"1h\",\n \"2h\",\n \"1d\"\n ],\n \"time_options\": [\n \"5m\",\n \"15m\",\n \"1h\",\n \"6h\",\n \"12h\",\n \"24h\",\n \"2d\",\n \"7d\",\n \"30d\"\n ]\n },\n \"timezone\": \"browser\",\n \"title\": \"StatefulSet\",\n \"version\": 1\n },\n \"inputs\": [\n {\n \"name\": \"DS_PROMETHEUS\",\n \"pluginId\": \"prometheus\",\n \"type\": \"datasource\",\n \"value\": \"prometheus\"\n }\n ],\n \"overwrite\": true\n}" @@ -35,377 +34,7 @@ "name": "dkube-grafana-dashboard", "namespace": "monitoring" } - }, - { - "apiVersion": "extensions/v1beta1", - "kind": "Deployment", - "metadata": { - "annotations": null, - "labels": { - "app": "dkube-prometheus-grafana", - "chart": "grafana-0.0.37", - "heritage": "Tiller", - "release": "kube-prometheus" - }, - "name": "dkube-grafana", - "namespace": "monitoring" - }, - "spec": { - "progressDeadlineSeconds": 600, - "replicas": 1, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "app": "dkube-prometheus-grafana" - } - }, - "strategy": { - "rollingUpdate": { - "maxSurge": 1, - "maxUnavailable": 1 - }, - "type": "RollingUpdate" - }, - "template": { - "metadata": { - "creationTimestamp": null, - "labels": { - "app": "dkube-prometheus-grafana", - "release": "kube-prometheus" - } - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "GF_AUTH_BASIC_ENABLED", - "value": "true" - }, - { - "name": "GF_AUTH_ANONYMOUS_ENABLED", - "value": "true" - }, - { - "name": "GF_SECURITY_ADMIN_USER", - "valueFrom": { - "secretKeyRef": { - "key": "user", - "name": "kube-prometheus-grafana" - } - } - }, - { - "name": "GF_SECURITY_ADMIN_PASSWORD", - "valueFrom": { - "secretKeyRef": { - "key": "password", - "name": "kube-prometheus-grafana" - } - } - } - ], - "image": "grafana/grafana:5.0.0", - "imagePullPolicy": "IfNotPresent", - "name": "grafana", - "ports": [ - { - "containerPort": 3000, - "name": "web", - "protocol": "TCP" - } - ], - "readinessProbe": { - "failureThreshold": 10, - "httpGet": { - "path": "/api/health", - "port": 3000, - "scheme": "HTTP" - }, - "periodSeconds": 1, - "successThreshold": 1, - "timeoutSeconds": 1 - }, - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/lib/grafana", - "name": "grafana-storage" - }, - { - "mountPath": "/etc/grafana/grafana.ini", - "name": "grafana-config", - "subPath": "grafana.ini" - } - ] - }, - { - "args": [ - "--watch-dir=/var/grafana-dashboards", - "--grafana-url=http://127.0.0.1:3000" - ], - "env": [ - { - "name": "GRAFANA_USER", - "valueFrom": { - "secretKeyRef": { - "key": "user", - "name": "kube-prometheus-grafana" - } - } - }, - { - "name": "GRAFANA_PASSWORD", - "valueFrom": { - "secretKeyRef": { - "key": "password", - "name": "kube-prometheus-grafana" - } - } - } - ], - "image": "quay.io/coreos/grafana-watcher:v0.0.8", - "imagePullPolicy": "IfNotPresent", - "name": "grafana-watcher", - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/grafana-dashboards", - "name": "grafana-dashboards" - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "kube-prometheus-grafana", - "serviceAccountName": "kube-prometheus-grafana", - "terminationGracePeriodSeconds": 30, - "volumes": [ - { - "emptyDir": {}, - "name": "grafana-storage" - }, - { - "configMap": { - "defaultMode": 420, - "name": "dkube-grafana-dashboard" - }, - "name": "grafana-dashboards" - }, - { - "configMap": { - "defaultMode": 420, - "name": "dkube-grafana-init" - }, - "name": "grafana-config" - } - ] - } - } - } - }, - { - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "annotations": null, - "labels": { - "app": "dkube-prometheus-grafana" - }, - "name": "dkube-grafana", - "namespace": "monitoring" - }, - "spec": { - "externalTrafficPolicy": "Cluster", - "ports": [ - { - "name": "http", - "nodePort": 30990, - "port": 80, - "protocol": "TCP", - "targetPort": 3000 - } - ], - "selector": { - "app": "dkube-prometheus-grafana" - }, - "sessionAffinity": "None", - "type": "NodePort" - } - }, - { - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "annotations": { - "getambassador.io/config": "---\napiVersion: ambassador/v0\nkind: Mapping\nname: dkube_monitoring\nprefix: /dkube/grafana/\nrewrite: /\nservice: dkube-grafana.monitoring:80\ntimeout_ms: 60000\nuse_websocket: true" - }, - "labels": { - "app": "dkube-prometheus-grafana" - }, - "name": "dkube-prometheus-grafana-proxy", - "namespace": "dkube" - }, - "spec": { - "externalTrafficPolicy": "Cluster", - "ports": [ - { - "name": "http", - "nodePort": 31227, - "port": 80, - "protocol": "TCP", - "targetPort": 3000 - } - ], - "selector": { - "app": "dkube-prometheus-grafana" - }, - "sessionAffinity": "None", - "type": "NodePort" - } - }, - { - "apiVersion": "monitoring.coreos.com/v1", - "kind": "ServiceMonitor", - "metadata": { - "labels": { - "app": "prometheus", - "prometheus": "kube-prometheus" - }, - "name": "dkube-prometheus-exporter-gpu", - "namespace": "monitoring" - }, - "spec": { - "endpoints": [ - { - "interval": "15s", - "port": "http-metrics" - } - ], - "jobLabel": "k8s-app", - "namespaceSelector": { - "matchNames": [ - "monitoring" - ] - }, - "selector": { - "matchLabels": { - "app": "dkube-gpu-exporter" - } - } - } - }, - { - "apiVersion": "extensions/v1beta1", - "kind": "DaemonSet", - "metadata": { - "name": "nvidia-exporter", - "namespace": "monitoring" - }, - "spec": { - "template": { - "metadata": { - "labels": { - "app": "nvidia-exporter" - } - }, - "spec": { - "containers": [ - { - "image": "ocdr/dkube-gpu-exporter:v3", - "name": "nvidia-exporter", - "ports": [ - { - "containerPort": 9401, - "name": "http-metrics" - } - ], - "securityContext": { - "privileged": true - }, - "volumeMounts": [ - { - "mountPath": "/usr/local/nvidia", - "name": "nvidia" - }, - { - "mountPath": "/usr/lib", - "name": "nvml" - }, - { - "mountPath": "/usr/lib64", - "name": "lib64" - } - ] - } - ], - "hostNetwork": true, - "imagePullSecrets": [ - { - "name": "dkube-dockerhub-secret" - } - ], - "volumes": [ - { - "hostPath": { - "path": "/opt/nvidia/current" - }, - "name": "nvidia" - }, - { - "hostPath": { - "path": "/usr/lib" - }, - "name": "nvml" - }, - { - "hostPath": { - "path": "/usr/lib64" - }, - "name": "lib64" - } - ] - } - } - } - }, - { - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "annotations": { - "prometheus.io/port": "9401", - "prometheus.io/scrape": "true" - }, - "labels": { - "app": "dkube-gpu-exporter" - }, - "name": "nvidia-exporter", - "namespace": "monitoring" - }, - "spec": { - "ports": [ - { - "name": "http-metrics", - "nodePort": 30001, - "port": 9401, - "protocol": "TCP", - "targetPort": 9401 - } - ], - "selector": { - "app": "nvidia-exporter" - }, - "type": "NodePort" - } - }, + } ] }