diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml
index 5bec87e..ecdf49c 100644
--- a/kustomize-nfs/dkube/base/dkube.yaml
+++ b/kustomize-nfs/dkube/base/dkube.yaml
@@ -113,6 +113,13 @@ spec:
labels:
app: dkube-ext
spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: Accelerator
+ operator: Exists
containers:
- env:
- name: MYNODENAME
@@ -361,6 +368,38 @@ spec:
- name: dkubeDockerSecret
nodeSelector: NODE_SELECTOR
---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app: dkube-pushgateway
+ name: dkube-metrics-pushgateway-verTag
+ namespace: dkube
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: dkube-pushgateway
+ template:
+ metadata:
+ labels:
+ app: dkube-pushgateway
+ spec:
+ containers:
+ - image: pushgatewayImage
+ imagePullPolicy: IfNotPresent
+ name: main
+ ports:
+ - containerPort: 9401
+ name: http-metrics
+ protocol: TCP
+ resources: {}
+ securityContext:
+ privileged: true
+ runAsUser: 0
+ imagePullSecrets:
+ - name: dkubeDockerSecret
+---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml
index 605e42c..ecdf875 100644
--- a/kustomize/dkube/base/dkube.yaml
+++ b/kustomize/dkube/base/dkube.yaml
@@ -119,6 +119,13 @@ spec:
labels:
app: dkube-ext
spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: Accelerator
+ operator: Exists
containers:
- env:
- name: MYNODENAME
@@ -156,7 +163,38 @@ spec:
- hostPath:
path: /var/run/docker.sock
name: docker
-
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app: dkube-pushgateway
+ name: dkube-metrics-pushgateway-verTag
+ namespace: dkube
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: dkube-pushgateway
+ template:
+ metadata:
+ labels:
+ app: dkube-pushgateway
+ spec:
+ containers:
+ - image: pushgatewayImage
+ imagePullPolicy: IfNotPresent
+ name: main
+ ports:
+ - containerPort: 9401
+ name: http-metrics
+ protocol: TCP
+ resources: {}
+ securityContext:
+ privileged: true
+ runAsUser: 0
+ imagePullSecrets:
+ - name: dkubeDockerSecret
---
apiVersion: extensions/v1beta1
kind: DaemonSet
diff --git a/kustomize/service/base/service.yaml b/kustomize/service/base/service.yaml
index 1e23995..c09dc63 100644
--- a/kustomize/service/base/service.yaml
+++ b/kustomize/service/base/service.yaml
@@ -201,181 +201,35 @@ metadata:
namespace: dkube
---
apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: dkube-metric-collector
+ namespace: dkube
data:
- accuracy.conf: "\n @type kubernetes_metadata\n\n\n\n @type grep\n \n key $.kubernetes.labels.logger\n
- \ pattern /^dkube$/\n \n\n\n\n
- \ @type grep\n \n key log\n pattern /accuracy/\n \n\n\n\n @type parser\n key_name $.log\n reserve_data true\n
- \ remove_key_name_field true\n suppress_parse_error_log true\n \n @type
- regexp\n expression /^(.*):(.*):((.*)])?((.*):)?(?(.*))$/\n \n\n\n
- \n @type record_modifier\n enable_ruby\n \n
- \ escaped_tag ${record[\"message\"].gsub(' ', '')}\n \n \n\n\n @type grep\n \n key escaped_tag\n
- \ pattern /accuracy=/\n \n\n\n\n
- \ @type parser\n key_name $.escaped_tag\n reserve_data true\n remove_key_name_field
- true\n suppress_parse_error_log true\n \n @type ltsv\n delimiter_pattern
- /,/\n label_delimiter =\n \n\n\n\n
- \ @type record_modifier\n enable_ruby\n \n jobname ${record.dig(\"kubernetes\",
- \"labels\", \"jobname\")}\n username ${record.dig(\"kubernetes\", \"labels\",
- \"username\")}\n jobid ${record.dig(\"kubernetes\", \"labels\", \"jobid\")}\n
- \ mode ${record.dig(\"mode\").to_s}\n step ${record.dig(\"step\").to_i}\n
- \ epoch ${record.dig(\"epoch\").to_i}\n accuracy ${record.dig(\"accuracy\").to_f}
- \n \n\n\n\n @type prometheus\n
- \ \n name accuracy\n type gauge\n desc accuracy metric\n key
- $.accuracy\n \n jobname ${jobname}\n username ${username}\n
- \ jobid ${jobid}\n step ${step}\n mode ${mode}\n epoch ${epoch}\n
- \ \n \n\n\n\n @type
- relabel\n @label @PROMETHEUS\n\n"
fluent.conf: |
-
- @type prometheus
-
-
-
- @type monitor_agent
-
-
-
- @type forward
-
-
- # input plugin that collects metrics from MonitorAgent
-
- @type prometheus_monitor
-
- host ${hostname}
-
-
-
- # input plugin that collects metrics for output plugin
-
- @type prometheus_output_monitor
-
- host ${hostname}
-
-
-
- # input plugin that collects metrics for in_tail plugin
-
- @type prometheus_tail_monitor
-
- host ${hostname}
-
-
-
@type tail
- path /var/log/containers/*_tensorflow-*.log
- pos_file /var/log/fluentd-containers-accuracy.log.pos
+ path /var/log/containers/*.log
+ pos_file /var/log/fluentd-containers-metric.log.pos
time_format %Y-%m-%dT%H:%M:%S
- tag kubernetes_accuracy.*
- @label @ACCURACY
+ tag kubernetes_metric.*
+ refresh_interval 1s
+ open_on_every_update true
+ enable_stat_watcher false
+ rotate_wait 2
format json
read_from_head true
-
- @type tail
- path /var/log/containers/*_tensorflow-*.log
- pos_file /var/log/fluentd-containers-loss.log.pos
- time_format %Y-%m-%dT%H:%M:%S
- tag kubernetes_loss.*
- @label @LOSS
- format json
- read_from_head true
-
-
-
- @type tail
- path /var/log/containers/*_tensorflow-*.log
- pos_file /var/log/fluentd-containers-step.log.pos
- time_format %Y-%m-%dT%H:%M:%S
- tag kubernetes_step.*
- @label @STEP
- format json
- read_from_head true
-
-
-
-
-
-
-
-
-
- loss.conf: "\n @type kubernetes_metadata\n \n\n
- \n @type grep\n \n key $.kubernetes.labels.logger\n
- \ pattern /^dkube$/\n \n \n\n \n
- \ @type grep\n \n key log\n pattern /loss/\n \n
- \n\n \n @type parser\n key_name $.log\n
- \ reserve_data true\n remove_key_name_field true\n suppress_parse_error_log
- true\n \n @type regexp\n expression /^(.*):(.*):((.*)])?((.*):)?(?(.*))$/\n
- \ \n \n\n \n @type record_modifier\n
- \ enable_ruby\n \n escaped_tag ${record[\"message\"].gsub('
- ', '')}\n \n \n\n \n @type
- grep\n \n key escaped_tag\n pattern /loss=/\n \n
- \n\n \n @type parser\n key_name $.escaped_tag\n
- \ reserve_data true\n remove_key_name_field true\n suppress_parse_error_log
- true\n \n @type ltsv\n delimiter_pattern /,/\n label_delimiter
- \ =\n \n \n\n \n @type record_modifier\n
- \ enable_ruby\n \n jobname ${record.dig(\"kubernetes\", \"labels\",
- \"jobname\")}\n username ${record.dig(\"kubernetes\", \"labels\", \"username\")}\n
- \ jobuuid ${record.dig(\"kubernetes\", \"labels\", \"jobuuid\")}\n jobid
- ${record.dig(\"kubernetes\", \"labels\", \"jobid\")}\n step ${record.dig(\"step\").to_i}\n
- \ loss ${record.dig(\"loss\").to_f}\n mode ${record.dig(\"mode\").to_s}\n
- \ epoch ${record.dig(\"epoch\").to_i} \n \n \n\n \n @type prometheus\n \n name loss\n type
- gauge\n desc loss metric\n key $.loss\n \n jobuuid ${jobuuid}\n
- \ jobname ${jobname}\n username ${username}\n jobid ${jobid}\n
- \ step ${step}\n mode ${mode}\n epoch ${epoch}\n \n
- \ \n \n\n \n @type relabel\n
- \ @label @PROMETHEUS\n \n"
- prometheus.conf: |
-
- @type copy
- # for MonitorAgent sample
-
- @id test_forward
- @type forward
- buffer_type memory
- flush_interval 1s
- max_retry_wait 2s
- send_timeout 60s
- recover_wait 60s
- hard_timeout 60s
-
- # max_retry_wait 10s
- flush_interval 1s
- # retry_type periodic
- disable_retry_limit
-
- # retry_limit 3
- disable_retry_limit
-
- host 0.0.0.0
- port 24224
-
-
-
- step.conf: |
-
- @type kubernetes_metadata
+
+ @type kubernetes_metadata
-
+
@type grep
- key $.kubernetes.labels.logger
- pattern /^dkube$/
+ key $.kubernetes.labels.logger
+ pattern /^dkube$/
@@ -383,11 +237,19 @@ data:
@type grep
key log
- pattern /step/
+ pattern /\[dkube\]\[metrics\]/
-
+
+ @type grep
+
+ key $.log
+ pattern /(.*) = (.*),/
+
+
+
+
@type parser
key_name $.log
reserve_data true
@@ -399,23 +261,15 @@ data:
-
- @type record_modifier
- enable_ruby
-
- escaped_tag ${record["message"].gsub(' ', '')}
-
-
-
-
- @type grep
-
- key escaped_tag
- pattern /,step=/
-
+
+ @type record_modifier
+ enable_ruby
+
+ escaped_tag ${record["message"].gsub(' ', '')}
+
-
+
@type parser
key_name $.escaped_tag
reserve_data true
@@ -428,48 +282,43 @@ data:
-
- @type record_modifier
- enable_ruby
-
- jobname ${record.dig("kubernetes", "labels", "jobname")}
- username ${record.dig("kubernetes", "labels", "username")}
- jobid ${record.dig("kubernetes", "labels", "jobid")}
- step ${record.dig("step").to_i}
- accuracy ${record.dig("accuracy").to_f}
- loss ${record.dig("loss").to_f}
- mode ${record.dig("mode").to_s}
-
-
-
-
- @type prometheus
-
- name step
- type gauge
- desc step metric
- key $.step
-
- jobname ${jobname}
- username ${username}
- jobid ${jobid}
- accuracy ${accuracy}
- loss ${loss}
- mode ${mode}
-
-
+
+ @type record_modifier
+ enable_ruby
+
+ jobname ${record.dig("kubernetes", "labels", "jobname")}
+ username ${record.dig("kubernetes", "labels", "username")}
+ jobid ${record.dig("kubernetes", "labels", "jobid")}
+
+ remove_keys log, stream, docker, kubernetes, escaped_tag, message
-
- @type relabel
- @label @PROMETHEUS
+
+ @type copy
+
+ @type http
+ endpoint http://dkube-metrics-pushgateway.dkube:9401/export-training-info
+ open_timeout 10
+
+ @type json
+
+
+ @type file
+ path /var/log/td-agent/jobs/metrics
+ flush_mode interval
+ flush_interval 15s
+ flush_thread_count 8
+ retry_forever true
+ overflow_action throw_exception
+ retry_type exponential_backoff
+ flush_at_shutdown true
+ queue_limit_length 10000
+
+
+
+ @type stdout
+
-kind: ConfigMap
-metadata:
- labels:
- ksonnet.io/component: service
- name: dkube-metric-collector
- namespace: dkube
---
apiVersion: v1
kind: PersistentVolumeClaim
@@ -698,39 +547,37 @@ spec:
apiVersion: v1
kind: Service
metadata:
- annotations:
- prometheus.io/port: "24231"
- prometheus.io/dkube_scrape: "true"
- labels:
- app: dkube-log-processor
- ksonnet.io/component: service
- name: dkube-log-processor
+ name: dkube-operator-api-proxy
namespace: dkube
spec:
ports:
- - name: dkube-log-metrics
- port: 24231
+ - name: dfabproxy
+ port: 8000
protocol: TCP
- targetPort: 24231
+ targetPort: 8000
selector:
- k8s-app: dkube-metric-collector
+ app: dkube-operator-proxy
+ sessionAffinity: None
type: ClusterIP
---
apiVersion: v1
kind: Service
metadata:
- name: dkube-operator-api-proxy
+ annotations:
+ prometheus.io/dkube_scrape: "true"
+ prometheus.io/port: "9401"
+ labels:
+ app: dkube-pushgateway
+ name: dkube-metrics-pushgateway
namespace: dkube
spec:
ports:
- - name: dfabproxy
- port: 8000
+ - name: http-metrics
+ port: 9401
protocol: TCP
- targetPort: 8000
+ targetPort: 9401
selector:
- app: dkube-operator-proxy
- sessionAffinity: None
- type: ClusterIP
+ app: dkube-pushgateway
---
apiVersion: v1
kind: Service