From 2a690b0e718225a7a5e899ced7fc48382901f7f2 Mon Sep 17 00:00:00 2001 From: harshalkwagh Date: Fri, 13 Mar 2020 16:11:41 +0530 Subject: [PATCH 1/8] adding node selector for dkube-exporter --- kustomize-nfs/dkube/base/dkube.yaml | 2 ++ kustomize/dkube/base/dkube.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index 5bec87e..0cdbec6 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -113,6 +113,8 @@ spec: labels: app: dkube-ext spec: + nodeSelector: + dkube.ai/gpu-accelerator: true containers: - env: - name: MYNODENAME diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 605e42c..4834b72 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -119,6 +119,8 @@ spec: labels: app: dkube-ext spec: + nodeSelector: + dkube.ai/gpu-accelerator: true containers: - env: - name: MYNODENAME From 22f37bb1934327cd4e5790198e3f1d25b6a9cc1f Mon Sep 17 00:00:00 2001 From: harshalkwagh Date: Mon, 16 Mar 2020 11:31:56 +0530 Subject: [PATCH 2/8] label fix --- kustomize-nfs/dkube/base/dkube.yaml | 2 +- kustomize/dkube/base/dkube.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index 0cdbec6..57d1126 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -114,7 +114,7 @@ spec: app: dkube-ext spec: nodeSelector: - dkube.ai/gpu-accelerator: true + dkube.ai/gpu-accelerator: "true" containers: - env: - name: MYNODENAME diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 4834b72..0f6d236 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -120,7 +120,7 @@ spec: app: dkube-ext spec: nodeSelector: - dkube.ai/gpu-accelerator: true + dkube.ai/gpu-accelerator: "true" containers: - env: - name: MYNODENAME From eef7a42bba72aeb7cf3beeb0b301a98ae7d025a6 Mon Sep 17 00:00:00 2001 From: harshalkwagh Date: Mon, 16 Mar 2020 18:45:39 +0530 Subject: [PATCH 3/8] changed node selector --- kustomize-nfs/dkube/base/dkube.yaml | 9 +++++++-- kustomize/dkube/base/dkube.yaml | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index 57d1126..c3089d1 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -113,8 +113,13 @@ spec: labels: app: dkube-ext spec: - nodeSelector: - dkube.ai/gpu-accelerator: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: Accelerator + operator: Exists containers: - env: - name: MYNODENAME diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 0f6d236..106dd2c 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -119,8 +119,13 @@ spec: labels: app: dkube-ext spec: - nodeSelector: - dkube.ai/gpu-accelerator: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: Accelerator + operator: Exists containers: - env: - name: MYNODENAME From 42eca5f81c8d364439420838cc7513ff6cb9534c Mon Sep 17 00:00:00 2001 From: jafar390 Date: Mon, 16 Mar 2020 21:22:47 +0530 Subject: [PATCH 4/8] pushgateway related chnages --- kustomize/dkube/base/dkube.yaml | 34 ++++++++++++++++++++++++++++- kustomize/service/base/service.yaml | 34 ++++++++++++++--------------- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 605e42c..96d9d12 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -156,7 +156,39 @@ spec: - hostPath: path: /var/run/docker.sock name: docker - +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: dkube-pushgateway + name: dkube-metric-pushgateway-verTag + namespace: dkube +spec: + replicas: 1 + selector: + matchLabels: + app: dkube-pushgateway + template: + metadata: + creationTimestamp: null + labels: + app: dkube-pushgateway + spec: + containers: + - image: pushgatewayImage + imagePullPolicy: IfNotPresent + name: main + ports: + - containerPort: 9401 + name: http-metrics + protocol: TCP + resources: {} + securityContext: + privileged: true + runAsUser: 0 + imagePullSecrets: + - name: dkubeDockerSecret --- apiVersion: extensions/v1beta1 kind: DaemonSet diff --git a/kustomize/service/base/service.yaml b/kustomize/service/base/service.yaml index 1e23995..ba3a373 100644 --- a/kustomize/service/base/service.yaml +++ b/kustomize/service/base/service.yaml @@ -698,39 +698,37 @@ spec: apiVersion: v1 kind: Service metadata: - annotations: - prometheus.io/port: "24231" - prometheus.io/dkube_scrape: "true" - labels: - app: dkube-log-processor - ksonnet.io/component: service - name: dkube-log-processor + name: dkube-operator-api-proxy namespace: dkube spec: ports: - - name: dkube-log-metrics - port: 24231 + - name: dfabproxy + port: 8000 protocol: TCP - targetPort: 24231 + targetPort: 8000 selector: - k8s-app: dkube-metric-collector + app: dkube-operator-proxy + sessionAffinity: None type: ClusterIP --- apiVersion: v1 kind: Service metadata: - name: dkube-operator-api-proxy + annotations: + prometheus.io/dkube_scrape: "true" + prometheus.io/port: "9401" + labels: + app: dkube-pushgateway + name: dkube-metrics-pushgateway namespace: dkube spec: ports: - - name: dfabproxy - port: 8000 + - name: http-metrics + port: 9401 protocol: TCP - targetPort: 8000 + targetPort: 9401 selector: - app: dkube-operator-proxy - sessionAffinity: None - type: ClusterIP + app: dkube-pushgateway --- apiVersion: v1 kind: Service From d54a66cdbfb2ab281aea2d48b351cdb0e5129539 Mon Sep 17 00:00:00 2001 From: jafar390 Date: Mon, 16 Mar 2020 21:40:27 +0530 Subject: [PATCH 5/8] metric collector related changes --- kustomize/service/base/service.yaml | 285 +++++++--------------------- 1 file changed, 67 insertions(+), 218 deletions(-) diff --git a/kustomize/service/base/service.yaml b/kustomize/service/base/service.yaml index ba3a373..c09dc63 100644 --- a/kustomize/service/base/service.yaml +++ b/kustomize/service/base/service.yaml @@ -201,181 +201,35 @@ metadata: namespace: dkube --- apiVersion: v1 +kind: ConfigMap +metadata: + name: dkube-metric-collector + namespace: dkube data: - accuracy.conf: "\n @type kubernetes_metadata\n\n\n\n @type grep\n \n key $.kubernetes.labels.logger\n - \ pattern /^dkube$/\n \n\n\n\n - \ @type grep\n \n key log\n pattern /accuracy/\n \n\n\n\n @type parser\n key_name $.log\n reserve_data true\n - \ remove_key_name_field true\n suppress_parse_error_log true\n \n @type - regexp\n expression /^(.*):(.*):((.*)])?((.*):)?(?(.*))$/\n \n\n\n - \n @type record_modifier\n enable_ruby\n \n - \ escaped_tag ${record[\"message\"].gsub(' ', '')}\n \n \n\n\n @type grep\n \n key escaped_tag\n - \ pattern /accuracy=/\n \n\n\n\n - \ @type parser\n key_name $.escaped_tag\n reserve_data true\n remove_key_name_field - true\n suppress_parse_error_log true\n \n @type ltsv\n delimiter_pattern - /,/\n label_delimiter =\n \n\n\n\n - \ @type record_modifier\n enable_ruby\n \n jobname ${record.dig(\"kubernetes\", - \"labels\", \"jobname\")}\n username ${record.dig(\"kubernetes\", \"labels\", - \"username\")}\n jobid ${record.dig(\"kubernetes\", \"labels\", \"jobid\")}\n - \ mode ${record.dig(\"mode\").to_s}\n step ${record.dig(\"step\").to_i}\n - \ epoch ${record.dig(\"epoch\").to_i}\n accuracy ${record.dig(\"accuracy\").to_f} - \n \n\n\n\n @type prometheus\n - \ \n name accuracy\n type gauge\n desc accuracy metric\n key - $.accuracy\n \n jobname ${jobname}\n username ${username}\n - \ jobid ${jobid}\n step ${step}\n mode ${mode}\n epoch ${epoch}\n - \ \n \n\n\n\n @type - relabel\n @label @PROMETHEUS\n\n" fluent.conf: | - - @type prometheus - - - - @type monitor_agent - - - - @type forward - - - # input plugin that collects metrics from MonitorAgent - - @type prometheus_monitor - - host ${hostname} - - - - # input plugin that collects metrics for output plugin - - @type prometheus_output_monitor - - host ${hostname} - - - - # input plugin that collects metrics for in_tail plugin - - @type prometheus_tail_monitor - - host ${hostname} - - - - - @type tail - path /var/log/containers/*_tensorflow-*.log - pos_file /var/log/fluentd-containers-accuracy.log.pos - time_format %Y-%m-%dT%H:%M:%S - tag kubernetes_accuracy.* - @label @ACCURACY - format json - read_from_head true - - @type tail - path /var/log/containers/*_tensorflow-*.log - pos_file /var/log/fluentd-containers-loss.log.pos + path /var/log/containers/*.log + pos_file /var/log/fluentd-containers-metric.log.pos time_format %Y-%m-%dT%H:%M:%S - tag kubernetes_loss.* - @label @LOSS + tag kubernetes_metric.* + refresh_interval 1s + open_on_every_update true + enable_stat_watcher false + rotate_wait 2 format json read_from_head true - - @type tail - path /var/log/containers/*_tensorflow-*.log - pos_file /var/log/fluentd-containers-step.log.pos - time_format %Y-%m-%dT%H:%M:%S - tag kubernetes_step.* - @label @STEP - format json - read_from_head true - - - - - - - - - - loss.conf: "\n @type kubernetes_metadata\n \n\n - \n @type grep\n \n key $.kubernetes.labels.logger\n - \ pattern /^dkube$/\n \n \n\n \n - \ @type grep\n \n key log\n pattern /loss/\n \n - \n\n \n @type parser\n key_name $.log\n - \ reserve_data true\n remove_key_name_field true\n suppress_parse_error_log - true\n \n @type regexp\n expression /^(.*):(.*):((.*)])?((.*):)?(?(.*))$/\n - \ \n \n\n \n @type record_modifier\n - \ enable_ruby\n \n escaped_tag ${record[\"message\"].gsub(' - ', '')}\n \n \n\n \n @type - grep\n \n key escaped_tag\n pattern /loss=/\n \n - \n\n \n @type parser\n key_name $.escaped_tag\n - \ reserve_data true\n remove_key_name_field true\n suppress_parse_error_log - true\n \n @type ltsv\n delimiter_pattern /,/\n label_delimiter - \ =\n \n \n\n \n @type record_modifier\n - \ enable_ruby\n \n jobname ${record.dig(\"kubernetes\", \"labels\", - \"jobname\")}\n username ${record.dig(\"kubernetes\", \"labels\", \"username\")}\n - \ jobuuid ${record.dig(\"kubernetes\", \"labels\", \"jobuuid\")}\n jobid - ${record.dig(\"kubernetes\", \"labels\", \"jobid\")}\n step ${record.dig(\"step\").to_i}\n - \ loss ${record.dig(\"loss\").to_f}\n mode ${record.dig(\"mode\").to_s}\n - \ epoch ${record.dig(\"epoch\").to_i} \n \n \n\n \n @type prometheus\n \n name loss\n type - gauge\n desc loss metric\n key $.loss\n \n jobuuid ${jobuuid}\n - \ jobname ${jobname}\n username ${username}\n jobid ${jobid}\n - \ step ${step}\n mode ${mode}\n epoch ${epoch}\n \n - \ \n \n\n \n @type relabel\n - \ @label @PROMETHEUS\n \n" - prometheus.conf: | - - @type copy - # for MonitorAgent sample - - @id test_forward - @type forward - buffer_type memory - flush_interval 1s - max_retry_wait 2s - send_timeout 60s - recover_wait 60s - hard_timeout 60s - - # max_retry_wait 10s - flush_interval 1s - # retry_type periodic - disable_retry_limit - - # retry_limit 3 - disable_retry_limit - - host 0.0.0.0 - port 24224 - - - - step.conf: | - - @type kubernetes_metadata + + @type kubernetes_metadata - + @type grep - key $.kubernetes.labels.logger - pattern /^dkube$/ + key $.kubernetes.labels.logger + pattern /^dkube$/ @@ -383,11 +237,19 @@ data: @type grep key log - pattern /step/ + pattern /\[dkube\]\[metrics\]/ - + + @type grep + + key $.log + pattern /(.*) = (.*),/ + + + + @type parser key_name $.log reserve_data true @@ -399,23 +261,15 @@ data: - - @type record_modifier - enable_ruby - - escaped_tag ${record["message"].gsub(' ', '')} - - - - - @type grep - - key escaped_tag - pattern /,step=/ - + + @type record_modifier + enable_ruby + + escaped_tag ${record["message"].gsub(' ', '')} + - + @type parser key_name $.escaped_tag reserve_data true @@ -428,48 +282,43 @@ data: - - @type record_modifier - enable_ruby - - jobname ${record.dig("kubernetes", "labels", "jobname")} - username ${record.dig("kubernetes", "labels", "username")} - jobid ${record.dig("kubernetes", "labels", "jobid")} - step ${record.dig("step").to_i} - accuracy ${record.dig("accuracy").to_f} - loss ${record.dig("loss").to_f} - mode ${record.dig("mode").to_s} - + + @type record_modifier + enable_ruby + + jobname ${record.dig("kubernetes", "labels", "jobname")} + username ${record.dig("kubernetes", "labels", "username")} + jobid ${record.dig("kubernetes", "labels", "jobid")} + + remove_keys log, stream, docker, kubernetes, escaped_tag, message - - @type prometheus - - name step - type gauge - desc step metric - key $.step - - jobname ${jobname} - username ${username} - jobid ${jobid} - accuracy ${accuracy} - loss ${loss} - mode ${mode} - - - - - - @type relabel - @label @PROMETHEUS + + @type copy + + @type http + endpoint http://dkube-metrics-pushgateway.dkube:9401/export-training-info + open_timeout 10 + + @type json + + + @type file + path /var/log/td-agent/jobs/metrics + flush_mode interval + flush_interval 15s + flush_thread_count 8 + retry_forever true + overflow_action throw_exception + retry_type exponential_backoff + flush_at_shutdown true + queue_limit_length 10000 + + + + @type stdout + -kind: ConfigMap -metadata: - labels: - ksonnet.io/component: service - name: dkube-metric-collector - namespace: dkube --- apiVersion: v1 kind: PersistentVolumeClaim From e0a1583f252343da1990b468b9d92650a0edc49d Mon Sep 17 00:00:00 2001 From: jafar390 Date: Tue, 17 Mar 2020 12:43:36 +0530 Subject: [PATCH 6/8] adding inside kust-nfs --- kustomize-nfs/dkube/base/dkube.yaml | 32 +++++++++++++++++++++++++++++ kustomize/dkube/base/dkube.yaml | 1 - 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index c3089d1..fd01992 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -368,6 +368,38 @@ spec: - name: dkubeDockerSecret nodeSelector: NODE_SELECTOR --- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: dkube-pushgateway + name: dkube-metric-pushgateway-verTag + namespace: dkube +spec: + replicas: 1 + selector: + matchLabels: + app: dkube-pushgateway + template: + metadata: + labels: + app: dkube-pushgateway + spec: + containers: + - image: pushgatewayImage + imagePullPolicy: IfNotPresent + name: main + ports: + - containerPort: 9401 + name: http-metrics + protocol: TCP + resources: {} + securityContext: + privileged: true + runAsUser: 0 + imagePullSecrets: + - name: dkubeDockerSecret +--- apiVersion: extensions/v1beta1 kind: Deployment metadata: diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 79e3788..670484b 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -178,7 +178,6 @@ spec: app: dkube-pushgateway template: metadata: - creationTimestamp: null labels: app: dkube-pushgateway spec: From f1c7c424b090f2af25e4f54525ae376a45a458fa Mon Sep 17 00:00:00 2001 From: jafar390 Date: Tue, 17 Mar 2020 13:00:18 +0530 Subject: [PATCH 7/8] test fix --- kustomize/dkube/base/dkube.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 670484b..ecdf875 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -169,7 +169,7 @@ kind: Deployment metadata: labels: app: dkube-pushgateway - name: dkube-metric-pushgateway-verTag + name: dkube-metrics-pushgateway-verTag namespace: dkube spec: replicas: 1 From ceb4cc032f2aa60cc67308e915fff3ca994a5932 Mon Sep 17 00:00:00 2001 From: jafar390 Date: Tue, 17 Mar 2020 13:17:43 +0530 Subject: [PATCH 8/8] nfs test fic --- kustomize-nfs/dkube/base/dkube.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index fd01992..ecdf49c 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -373,7 +373,7 @@ kind: Deployment metadata: labels: app: dkube-pushgateway - name: dkube-metric-pushgateway-verTag + name: dkube-metrics-pushgateway-verTag namespace: dkube spec: replicas: 1