From dcb86fbff33da35e93cbda7ab0df69f9b244843e Mon Sep 17 00:00:00 2001 From: Deepak S Date: Thu, 10 Sep 2020 08:19:50 +0000 Subject: [PATCH 01/11] csi --- kustomize-nfs/dkube/base/dkube.yaml | 4 -- kustomize/dkube/base/dkube.yaml | 60 ++++++++--------------------- 2 files changed, 16 insertions(+), 48 deletions(-) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index 2a210a9..d6297c0 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -36,8 +36,6 @@ spec: value: nfsServer - name: NFS_BASE_PATH value: nfsBasePath - - name: CEPH_MONITORS - value: 'cephMonitors' - name: DKUBE_STORAGE_PROVIDER valueFrom: configMapKeyRef: @@ -589,8 +587,6 @@ spec: value: nfsBasePath - name: DKUBE_APISERVER_ROLE value: master - - name: CEPH_MONITORS - value: 'cephMonitors' - name: DKUBE_STORAGE_PROVIDER valueFrom: configMapKeyRef: diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 86776a5..c343849 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -36,8 +36,6 @@ spec: value: nfsServer - name: NFS_BASE_PATH value: nfsBasePath - - name: CEPH_MONITORS - value: 'cephMonitors' - name: DKUBE_STORAGE_PROVIDER valueFrom: configMapKeyRef: @@ -64,10 +62,12 @@ spec: volumeMounts: - mountPath: /home/dkube/d3s name: store + subPath: dkube - mountPath: /var/run/docker.sock name: docker - mountPath: /var/log/dkube - name: dkube-logs + subPath: dkube/system/logs/dkube + name: store - mountPath: /etc/dkube/dkube_platform_cfg name: pc - image: dkubeDownloaderImage @@ -89,19 +89,8 @@ spec: serviceAccountName: dkube volumes: - name: store - cephfs: - monitors: cephMonitors - path: /dkube - user: admin - secretRef: - name: ceph-secret - - name: dkube-logs - cephfs: - monitors: cephMonitors - path: /dkube/system/logs/dkube - user: admin - secretRef: - name: ceph-secret + persistentVolumeClaim: + claimName: dkube-pvc - hostPath: path: /var/run/docker.sock name: docker @@ -289,6 +278,7 @@ spec: volumeMounts: - mountPath: /var/log/dkube name: dkube-logs + subPath: dkube/system/logs/dkube dnsPolicy: ClusterFirst imagePullSecrets: - name: dkubeDockerSecret @@ -298,12 +288,8 @@ spec: serviceAccountName: dkube volumes: - name: dkube-logs - cephfs: - monitors: cephMonitors - path: /dkube/system/logs/dkube - user: admin - secretRef: - name: ceph-secret + persistentVolumeClaim: + claimName: dkube-pvc --- apiVersion: apps/v1 kind: Deployment @@ -481,12 +467,8 @@ spec: terminationGracePeriodSeconds: 30 volumes: - name: storage - cephfs: - monitors: cephMonitors - path: / - user: admin - secretRef: - name: ceph-secret + persistentVolumeClaim: + claimName: dkube-pvc --- apiVersion: apps/v1 kind: Deployment @@ -583,6 +565,7 @@ spec: volumeMounts: - mountPath: /dkube name: store + subPath: dkube - env: - name: DKUBE_MOUNT_PATH value: /home/dkube/d3s @@ -602,8 +585,6 @@ spec: value: nfsBasePath - name: DKUBE_APISERVER_ROLE value: master - - name: CEPH_MONITORS - value: 'cephMonitors' - name: DKUBE_STORAGE_PROVIDER valueFrom: configMapKeyRef: @@ -628,10 +609,12 @@ spec: volumeMounts: - mountPath: /home/dkube/d3s name: store + subPath: dkube - mountPath: /var/run/docker.sock name: docker - mountPath: /var/log/dkube - name: dkube-logs + name: store + subPath: dkube/system/logs/dkube - mountPath: /etc/dkube/dkube_platform_cfg name: pc dnsConfig: @@ -648,19 +631,8 @@ spec: serviceAccountName: dkube volumes: - name: store - cephfs: - monitors: cephMonitors - path: /dkube - user: admin - secretRef: - name: ceph-secret - - name: dkube-logs - cephfs: - monitors: cephMonitors - path: /dkube/system/logs/dkube - user: admin - secretRef: - name: ceph-secret + persistentVolumeClaim: + claimName: dkube-pvc - hostPath: path: /var/run/docker.sock name: docker From 52db82101bbf554b041961ce75fabcc69a2cf4e2 Mon Sep 17 00:00:00 2001 From: Deepak S Date: Thu, 10 Sep 2020 18:31:36 +0000 Subject: [PATCH 02/11] env --- kustomize-nfs/dkube/base/dkube.yaml | 6 ++++++ kustomize/dkube/base/dkube.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index d6297c0..c07ee66 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -47,6 +47,9 @@ spec: value: dkubeClusterType - name: MODEL_REGISTRY value: modelRegistry + envFrom: + - configMapRef: + name: ceph-config image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main @@ -596,6 +599,9 @@ spec: value: dkubeClusterType - name: MODEL_REGISTRY value: modelRegistry + envFrom: + - configMapRef: + name: ceph-config image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index c343849..a257336 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -47,6 +47,9 @@ spec: value: dkubeClusterType - name: MODEL_REGISTRY value: modelRegistry + envFrom: + - configMapRef: + name: ceph-config image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main @@ -594,6 +597,9 @@ spec: value: dkubeClusterType - name: MODEL_REGISTRY value: modelRegistry + envFrom: + - configMapRef: + name: ceph-config image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main From f1022c4de3a99dbac53cbce984abce1b39d7f1c5 Mon Sep 17 00:00:00 2001 From: Deepak S Date: Thu, 10 Sep 2020 18:44:25 +0000 Subject: [PATCH 03/11] optional env --- kustomize-nfs/dkube/base/dkube.yaml | 2 ++ kustomize/dkube/base/dkube.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index c07ee66..6c7abe1 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -50,6 +50,7 @@ spec: envFrom: - configMapRef: name: ceph-config + optional: true image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main @@ -602,6 +603,7 @@ spec: envFrom: - configMapRef: name: ceph-config + optional: true image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index a257336..63f07a7 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -50,6 +50,7 @@ spec: envFrom: - configMapRef: name: ceph-config + optional: true image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main @@ -600,6 +601,7 @@ spec: envFrom: - configMapRef: name: ceph-config + optional: true image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main From 2c465659e81de9643804653db14982d0aa7f8131 Mon Sep 17 00:00:00 2001 From: AnsuFrancis Date: Wed, 20 Jan 2021 13:32:48 +0000 Subject: [PATCH 04/11] allow referer auth header in envoy filter --- kustomize/service/base/service.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kustomize/service/base/service.yaml b/kustomize/service/base/service.yaml index 8a265ec..f32dec1 100644 --- a/kustomize/service/base/service.yaml +++ b/kustomize/service/base/service.yaml @@ -609,6 +609,7 @@ spec: - exact: d3-license - exact: d3-uname - exact: d3-role + - exact: referer authorizationResponse: allowedUpstreamHeaders: patterns: From fcd280941cf80398bad69c56e851d919534e36c3 Mon Sep 17 00:00:00 2001 From: Deepak S Date: Fri, 5 Feb 2021 11:24:39 +0530 Subject: [PATCH 05/11] no csi --- kustomize-nfs/dkube/base/dkube.yaml | 27 +- kustomize/dkube/base/dkube.yaml | 28 +- kustomize/minio/base/minio.yaml | 4 + kustomize/monitoring/base/monitoring.yaml | 5 +- kustomize/service/base/service.yaml | 454 +++++++--------------- 5 files changed, 167 insertions(+), 351 deletions(-) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index 0f83f5f..508f2b2 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -165,18 +165,18 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: - k8s-app: dkube-metric-collector + k8s-app: dkube-log-collector ksonnet.io/component: dkube name: dkube-log-processor-verTag namespace: dkube spec: selector: matchLabels: - k8s-app: dkube-metric-collector + k8s-app: dkube-log-collector template: metadata: labels: - k8s-app: dkube-metric-collector + k8s-app: dkube-log-collector spec: tolerations: - operator: "Exists" @@ -188,21 +188,6 @@ spec: procMount: Default runAsUser: 0 volumeMounts: - - mountPath: LOG_PATH - name: varlibdockercontainers - readOnly: true - - mountPath: /fluentd/etc/ - name: dkube-metric-collector - readOnly: true - - mountPath: /var/log - name: varlog - - image: fluentdImage - imagePullPolicy: IfNotPresent - name: sidecar - securityContext: - procMount: Default - runAsUser: 0 - volumeMounts: - mountPath: LOG_PATH name: varlibdockercontainers readOnly: true @@ -231,10 +216,6 @@ spec: - hostPath: path: LOG_PATH name: varlibdockercontainers - - configMap: - defaultMode: 420 - name: dkube-metric-collector - name: dkube-metric-collector --- apiVersion: apps/v1 kind: Deployment @@ -319,6 +300,8 @@ spec: - --advertise-client-urls=http://0.0.0.0:2379 - --data-dir=/var/lib/etcd - --force-new-cluster + - --heartbeat-interval=1000 + - --election-timeout=10000 image: k8s.gcr.io/etcd-amd64:3.1.12 imagePullPolicy: IfNotPresent name: main diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index de93bbd..7271b43 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -171,18 +171,18 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: - k8s-app: dkube-metric-collector + k8s-app: dkube-log-collector ksonnet.io/component: dkube name: dkube-log-processor-verTag namespace: dkube spec: selector: matchLabels: - k8s-app: dkube-metric-collector + k8s-app: dkube-log-collector template: metadata: labels: - k8s-app: dkube-metric-collector + k8s-app: dkube-log-collector spec: tolerations: - operator: "Exists" @@ -194,21 +194,6 @@ spec: procMount: Default runAsUser: 0 volumeMounts: - - mountPath: LOG_PATH - name: varlibdockercontainers - readOnly: true - - mountPath: /fluentd/etc/ - name: dkube-metric-collector - readOnly: true - - mountPath: /var/log - name: varlog - - image: fluentdImage - imagePullPolicy: IfNotPresent - name: sidecar - securityContext: - procMount: Default - runAsUser: 0 - volumeMounts: - mountPath: LOG_PATH name: varlibdockercontainers readOnly: true @@ -237,10 +222,6 @@ spec: - hostPath: path: LOG_PATH name: varlibdockercontainers - - configMap: - defaultMode: 420 - name: dkube-metric-collector - name: dkube-metric-collector --- apiVersion: apps/v1 kind: Deployment @@ -328,7 +309,8 @@ spec: - --advertise-client-urls=http://0.0.0.0:2379 - --data-dir=/var/lib/etcd - --force-new-cluster - - --election-timeout=30000 + - --heartbeat-interval=1000 + - --election-timeout=10000 image: k8s.gcr.io/etcd-amd64:3.1.12 imagePullPolicy: IfNotPresent name: main diff --git a/kustomize/minio/base/minio.yaml b/kustomize/minio/base/minio.yaml index c7176ca..bc31cf8 100644 --- a/kustomize/minio/base/minio.yaml +++ b/kustomize/minio/base/minio.yaml @@ -51,6 +51,10 @@ spec: port: number: 9000 host: dkube-minio-server + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: apps/v1 kind: Deployment diff --git a/kustomize/monitoring/base/monitoring.yaml b/kustomize/monitoring/base/monitoring.yaml index bf3ff5d..65abe1d 100644 --- a/kustomize/monitoring/base/monitoring.yaml +++ b/kustomize/monitoring/base/monitoring.yaml @@ -429,7 +429,7 @@ spec: app: dkube-state-exporter spec: containers: - - image: gcr.io/google_containers/kube-state-metrics:v1.2.0 + - image: quay.io/coreos/kube-state-metrics:v1.9.7 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 @@ -553,10 +553,11 @@ spec: spec: containers: - name: prometheus - image: prom/prometheus:v2.12.0 + image: prom/prometheus:v2.13.1 args: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus/" + - "--storage.tsdb.retention.time=3650d" ports: - containerPort: 9090 volumeMounts: diff --git a/kustomize/service/base/service.yaml b/kustomize/service/base/service.yaml index 75eea4e..f32dec1 100644 --- a/kustomize/service/base/service.yaml +++ b/kustomize/service/base/service.yaml @@ -158,10 +158,12 @@ data: final_log ${record.dig("kubernetes", "labels", "studyuuid") ? record.dig("trial_log") : record.dig("training_log")} #for pipeline launcher logs we are not adding extra fields - message ${record.dig("kubernetes", "labels", "runid") ? record.dig("log") : record.dig("final_log") } + message ${record.dig("kubernetes", "labels", "runid") ? record.dig("log") : record.dig("final_log") } + #s3 storage path + s3path ${record.dig("fpath")}/${record.dig("username")}/${record.dig("finalid")}/${record.dig("container")} #removing unwanted and temp fields - remove_keys log, stream, docker, kubernetes, final_log, trial_log, training_log, jobuuid, studyuuid, plpath, training_path, temp_log + remove_keys log, stream, docker, kubernetes, final_log, trial_log, training_log, jobuuid, studyuuid, plpath, training_path, temp_log, , username, finalid, container @@ -170,18 +172,22 @@ data: aws_sec_key l06dands19s s3_endpoint http://dkube-minio-server.dkube:9000/ s3_bucket dkube - path system/logs/${fpath}/${username}/${finalid}/${container} - s3_object_key_format %{path}/job-log-%{index}.%{file_extension} + path system/logs/${s3path} + s3_object_key_format %{path}/job-log-%{uuid_flush}.%{file_extension} store_as text force_path_style true + ssl_verify_peer false + check_apikey_on_start false + check_bucket false + auto_create_bucket false @type single_value message_key message add_newline false - + @type file - path /var/log/td-agent/jobs/${fpath}/${username}/${finalid}/${container} + path /var/log/td-agent/jobs/${s3path} timekey 10s timekey_wait 10s timekey_use_utc true @@ -189,11 +195,10 @@ data: flush_thread_count 8 retry_forever true overflow_action throw_exception - retry_type exponential_backoff - retry_exponential_backoff_base 2 + retry_type periodic + retry_wait 10s flush_mode immediate flush_at_shutdown true - queue_limit_length 10000 kind: ConfigMap @@ -202,277 +207,6 @@ metadata: namespace: dkube --- apiVersion: v1 -data: - accuracy.conf: "\n @type kubernetes_metadata\n\n\n\n @type grep\n \n key $.kubernetes.labels.logger\n - \ pattern /^dkube$/\n \n\n\n\n - \ @type grep\n \n key log\n pattern /accuracy/\n \n\n\n\n @type parser\n key_name $.log\n reserve_data true\n - \ remove_key_name_field true\n suppress_parse_error_log true\n \n @type - regexp\n expression /^(.*):(.*):((.*)])?((.*):)?(?(.*))$/\n \n\n\n - \n @type record_modifier\n enable_ruby\n \n - \ escaped_tag ${record[\"message\"].gsub(' ', '')}\n \n \n\n\n @type grep\n \n key escaped_tag\n - \ pattern /accuracy=/\n \n\n\n\n - \ @type parser\n key_name $.escaped_tag\n reserve_data true\n remove_key_name_field - true\n suppress_parse_error_log true\n \n @type ltsv\n delimiter_pattern - /,/\n label_delimiter =\n \n\n\n\n - \ @type record_modifier\n enable_ruby\n \n jobname ${record.dig(\"kubernetes\", - \"labels\", \"jobname\")}\n username ${record.dig(\"kubernetes\", \"labels\", - \"username\")}\n jobid ${record.dig(\"kubernetes\", \"labels\", \"jobid\")}\n - \ mode ${record.dig(\"mode\").to_s}\n step ${record.dig(\"step\").to_i}\n - \ epoch ${record.dig(\"epoch\").to_i}\n accuracy ${record.dig(\"accuracy\").to_f} - \n \n\n\n\n @type prometheus\n - \ \n name accuracy\n type gauge\n desc accuracy metric\n key - $.accuracy\n \n jobname ${jobname}\n username ${username}\n - \ jobid ${jobid}\n step ${step}\n mode ${mode}\n epoch ${epoch}\n - \ \n \n\n\n\n @type - relabel\n @label @PROMETHEUS\n\n" - fluent.conf: | - - @type prometheus - - - - @type monitor_agent - - - - @type forward - - - # input plugin that collects metrics from MonitorAgent - - @type prometheus_monitor - - host ${hostname} - - - - # input plugin that collects metrics for output plugin - - @type prometheus_output_monitor - - host ${hostname} - - - - # input plugin that collects metrics for in_tail plugin - - @type prometheus_tail_monitor - - host ${hostname} - - - - - @type tail - path /var/log/containers/*_tensorflow-*.log - pos_file /var/log/fluentd-containers-accuracy.log.pos - time_format %Y-%m-%dT%H:%M:%S - tag kubernetes_accuracy.* - @label @ACCURACY - format json - read_from_head true - - - - @type tail - path /var/log/containers/*_tensorflow-*.log - pos_file /var/log/fluentd-containers-loss.log.pos - time_format %Y-%m-%dT%H:%M:%S - tag kubernetes_loss.* - @label @LOSS - format json - read_from_head true - - - - @type tail - path /var/log/containers/*_tensorflow-*.log - pos_file /var/log/fluentd-containers-step.log.pos - time_format %Y-%m-%dT%H:%M:%S - tag kubernetes_step.* - @label @STEP - format json - read_from_head true - - - - - - - - - - loss.conf: "\n @type kubernetes_metadata\n \n\n - \n @type grep\n \n key $.kubernetes.labels.logger\n - \ pattern /^dkube$/\n \n \n\n \n - \ @type grep\n \n key log\n pattern /loss/\n \n - \n\n \n @type parser\n key_name $.log\n - \ reserve_data true\n remove_key_name_field true\n suppress_parse_error_log - true\n \n @type regexp\n expression /^(.*):(.*):((.*)])?((.*):)?(?(.*))$/\n - \ \n \n\n \n @type record_modifier\n - \ enable_ruby\n \n escaped_tag ${record[\"message\"].gsub(' - ', '')}\n \n \n\n \n @type - grep\n \n key escaped_tag\n pattern /loss=/\n \n - \n\n \n @type parser\n key_name $.escaped_tag\n - \ reserve_data true\n remove_key_name_field true\n suppress_parse_error_log - true\n \n @type ltsv\n delimiter_pattern /,/\n label_delimiter - \ =\n \n \n\n \n @type record_modifier\n - \ enable_ruby\n \n jobname ${record.dig(\"kubernetes\", \"labels\", - \"jobname\")}\n username ${record.dig(\"kubernetes\", \"labels\", \"username\")}\n - \ jobuuid ${record.dig(\"kubernetes\", \"labels\", \"jobuuid\")}\n jobid - ${record.dig(\"kubernetes\", \"labels\", \"jobid\")}\n step ${record.dig(\"step\").to_i}\n - \ loss ${record.dig(\"loss\").to_f}\n mode ${record.dig(\"mode\").to_s}\n - \ epoch ${record.dig(\"epoch\").to_i} \n \n \n\n \n @type prometheus\n \n name loss\n type - gauge\n desc loss metric\n key $.loss\n \n jobuuid ${jobuuid}\n - \ jobname ${jobname}\n username ${username}\n jobid ${jobid}\n - \ step ${step}\n mode ${mode}\n epoch ${epoch}\n \n - \ \n \n\n \n @type relabel\n - \ @label @PROMETHEUS\n \n" - prometheus.conf: | - - @type copy - # for MonitorAgent sample - - @id test_forward - @type forward - buffer_type memory - flush_interval 1s - max_retry_wait 2s - send_timeout 60s - recover_wait 60s - hard_timeout 60s - - # max_retry_wait 10s - flush_interval 1s - # retry_type periodic - disable_retry_limit - - # retry_limit 3 - disable_retry_limit - - host 0.0.0.0 - port 24224 - - - - step.conf: | - - @type kubernetes_metadata - - - - @type grep - - key $.kubernetes.labels.logger - pattern /^dkube$/ - - - - - @type grep - - key log - pattern /step/ - - - - - @type parser - key_name $.log - reserve_data true - remove_key_name_field true - suppress_parse_error_log true - - @type regexp - expression /^(.*):(.*):((.*)])?((.*):)?(?(.*))$/ - - - - - @type record_modifier - enable_ruby - - escaped_tag ${record["message"].gsub(' ', '')} - - - - - @type grep - - key escaped_tag - pattern /,step=/ - - - - - @type parser - key_name $.escaped_tag - reserve_data true - remove_key_name_field true - suppress_parse_error_log true - - @type ltsv - delimiter_pattern /,/ - label_delimiter = - - - - - @type record_modifier - enable_ruby - - jobname ${record.dig("kubernetes", "labels", "jobname")} - username ${record.dig("kubernetes", "labels", "username")} - jobid ${record.dig("kubernetes", "labels", "jobid")} - step ${record.dig("step").to_i} - accuracy ${record.dig("accuracy").to_f} - loss ${record.dig("loss").to_f} - mode ${record.dig("mode").to_s} - - - - - @type prometheus - - name step - type gauge - desc step metric - key $.step - - jobname ${jobname} - username ${username} - jobid ${jobid} - accuracy ${accuracy} - loss ${loss} - mode ${mode} - - - - - - @type relabel - @label @PROMETHEUS - -kind: ConfigMap -metadata: - labels: - ksonnet.io/component: service - name: dkube-metric-collector - namespace: dkube ---- -apiVersion: v1 kind: PersistentVolumeClaim metadata: labels: @@ -706,27 +440,6 @@ spec: --- apiVersion: v1 kind: Service -metadata: - annotations: - prometheus.io/port: "24231" - prometheus.io/dkube_scrape: "true" - labels: - app: dkube-log-processor - ksonnet.io/component: service - name: dkube-log-processor - namespace: dkube -spec: - ports: - - name: dkube-log-metrics - port: 24231 - protocol: TCP - targetPort: 24231 - selector: - k8s-app: dkube-metric-collector - type: ClusterIP ---- -apiVersion: v1 -kind: Service metadata: name: dkube-operator-api-proxy namespace: dkube @@ -832,7 +545,9 @@ spec: - excludedPaths: - exact: / - prefix: /dex + - prefix: /cicd - prefix: /pipeline + - prefix: /ml_metadata - prefix: /dkube/pipeline - prefix: /static - prefix: /callback @@ -846,7 +561,6 @@ spec: - prefix: /dkube/v2/controller/notify - prefix: /dkube/v2/controller/init - prefix: /dkube/v2/controller/dkubeinfo - - prefix: /cicd - suffix: /custom - prefix: /v1/models/ - regex: .*/nb/.*|.*/nb/lab.*|.*/tb/.*|.*/inference|.*/font-roboto/.*|.*/installer|.*/report|.*/katib.*|.*/rs.*|.*/docs/.*|.*/healthz @@ -857,7 +571,9 @@ spec: - excludedPaths: - exact: / - prefix: /dex + - prefix: /cicd - prefix: /pipeline + - prefix: /ml_metadata - prefix: /dkube/pipeline - prefix: /static - prefix: /callback @@ -871,7 +587,6 @@ spec: - prefix: /dkube/v2/controller/notify - prefix: /dkube/v2/controller/init - prefix: /dkube/v2/controller/dkubeinfo - - prefix: /cicd - suffix: /custom - prefix: /v1/models/ - regex: .*/nb/.*|.*/nb/lab.*|.*/tb/.*|.*/inference|.*/font-roboto/.*|.*/installer|.*/report|.*/katib.*|.*/rs.*|.*/docs/.*|.*/healthz @@ -894,6 +609,7 @@ spec: - exact: d3-license - exact: d3-uname - exact: d3-role + - exact: referer authorizationResponse: allowedUpstreamHeaders: patterns: @@ -974,6 +690,10 @@ spec: port: number: 5556 host: dkube-auth-server + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream - match: - uri: prefix: /dkube/v2/login @@ -984,6 +704,10 @@ spec: port: number: 3001 host: dkube-auth-server + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream - match: - uri: prefix: /dkube/v2/logout @@ -994,6 +718,10 @@ spec: port: number: 3001 host: dkube-auth-server + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1016,6 +744,10 @@ spec: port: number: 2379 host: dkube-db-server + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1038,6 +770,10 @@ spec: port: number: 8888 host: dkube-docs + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1060,6 +796,10 @@ spec: port: number: 9401 host: dkube-downloader + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream - match: - uri: prefix: /dkube/pipeline/logs/ @@ -1070,6 +810,10 @@ spec: port: number: 9401 host: dkube-downloader + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1090,6 +834,10 @@ spec: port: number: 8000 host: dkube-operator-api-proxy + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1112,6 +860,10 @@ spec: port: number: 80 host: dkube-grafana + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1134,6 +886,10 @@ spec: port: number: 9090 host: dkube-prometheus.dkube.svc.cluster.local + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1154,6 +910,10 @@ spec: port: number: 8000 host: dkube-serving + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream - match: - uri: prefix: /predict @@ -1162,6 +922,10 @@ spec: port: number: 8000 host: dkube-serving + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1182,6 +946,10 @@ spec: port: number: 3000 host: dkube-ui + retries: + attempts: 10 + perTryTimeout: 1s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1204,6 +972,10 @@ spec: port: number: 80 host: argo-ui.kubeflow.svc.cluster.local + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1224,6 +996,10 @@ spec: port: number: 80 host: katib-ui.kubeflow.svc.cluster.local + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1246,6 +1022,10 @@ spec: host: dkube-controller-master port: number: 5001 + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream - match: - method: regex: POST|DELETE @@ -1256,11 +1036,33 @@ spec: host: dkube-controller-master port: number: 5001 + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream - match: - method: - regex: POST|DELETE + exact: POST + uri: + exact: /pipeline/apis/v1beta1/runs + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream + route: + - destination: + host: dkube-controller-master + port: + number: 5001 + - match: + - method: + exact: DELETE uri: prefix: /pipeline/apis/v1beta1/runs + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream route: - destination: host: dkube-controller-master @@ -1274,6 +1076,22 @@ spec: host: ml-pipeline-ui.kubeflow.svc.cluster.local port: number: 80 + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream + - match: + - uri: + prefix: /ml_metadata + route: + - destination: + host: metadata-envoy-service.kubeflow.svc.cluster.local + port: + number: 9090 + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1296,6 +1114,10 @@ spec: port: number: 8888 host: dkube-installer-service + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream - match: - uri: prefix: /report @@ -1304,6 +1126,10 @@ spec: port: number: 8888 host: dkube-installer-service + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService @@ -1326,6 +1152,10 @@ spec: port: number: 5000 host: dkube-controller-worker + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream - match: - uri: prefix: /dkube/v2/controller @@ -1334,6 +1164,22 @@ spec: port: number: 5000 host: dkube-controller-master + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream + - match: + - uri: + prefix: /api/2.0/mlflow + route: + - destination: + port: + number: 5002 + host: dkube-controller-master + retries: + attempts: 3 + perTryTimeout: 5s + retryOn: gateway-error,connect-failure,refused-stream --- apiVersion: v1 kind: Service From 62e2539d9c10ad1d68f1707d47308a9d6969c518 Mon Sep 17 00:00:00 2001 From: Deepak S Date: Fri, 5 Feb 2021 13:38:20 +0530 Subject: [PATCH 06/11] removed mesh --- kustomize/service/base/service.yaml | 66 ----------------------------- 1 file changed, 66 deletions(-) diff --git a/kustomize/service/base/service.yaml b/kustomize/service/base/service.yaml index f32dec1..0c8aeb7 100644 --- a/kustomize/service/base/service.yaml +++ b/kustomize/service/base/service.yaml @@ -526,72 +526,6 @@ spec: app: dkube-ui type: ClusterIP --- -apiVersion: authentication.istio.io/v1alpha1 -kind: MeshPolicy -metadata: - labels: - app: security - chart: security - heritage: Tiller - release: istio - name: default -spec: - originIsOptional: false - origins: - - jwt: - issuer: http://localhost:5556/dex - jwksUri: http://dkube-auth-server.dkube.svc.cluster.local:5556/dex/keys - triggerRules: - - excludedPaths: - - exact: / - - prefix: /dex - - prefix: /cicd - - prefix: /pipeline - - prefix: /ml_metadata - - prefix: /dkube/pipeline - - prefix: /static - - prefix: /callback - - prefix: /favicon - - prefix: /dkube/v2/login - - prefix: /dkube/v2/logout - - prefix: /dkube/v2/ext - - prefix: /dkube/v2/etcd - - prefix: /dkube/v2/prometheus - - prefix: /dkube/v2/controller/dlsupport - - prefix: /dkube/v2/controller/notify - - prefix: /dkube/v2/controller/init - - prefix: /dkube/v2/controller/dkubeinfo - - suffix: /custom - - prefix: /v1/models/ - - regex: .*/nb/.*|.*/nb/lab.*|.*/tb/.*|.*/inference|.*/font-roboto/.*|.*/installer|.*/report|.*/katib.*|.*/rs.*|.*/docs/.*|.*/healthz - - jwt: - issuer: DKube - jwksUri: http://dkube-auth-server.dkube.svc.cluster.local:3001/keys - triggerRules: - - excludedPaths: - - exact: / - - prefix: /dex - - prefix: /cicd - - prefix: /pipeline - - prefix: /ml_metadata - - prefix: /dkube/pipeline - - prefix: /static - - prefix: /callback - - prefix: /favicon - - prefix: /dkube/v2/login - - prefix: /dkube/v2/logout - - prefix: /dkube/v2/ext - - prefix: /dkube/v2/etcd - - prefix: /dkube/v2/prometheus - - prefix: /dkube/v2/controller/dlsupport - - prefix: /dkube/v2/controller/notify - - prefix: /dkube/v2/controller/init - - prefix: /dkube/v2/controller/dkubeinfo - - suffix: /custom - - prefix: /v1/models/ - - regex: .*/nb/.*|.*/nb/lab.*|.*/tb/.*|.*/inference|.*/font-roboto/.*|.*/installer|.*/report|.*/katib.*|.*/rs.*|.*/docs/.*|.*/healthz - principalBinding: USE_ORIGIN ---- apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: From f6ca03def823eab9f9250ea03b08b5896030a1d5 Mon Sep 17 00:00:00 2001 From: Deepak S Date: Wed, 10 Feb 2021 13:38:55 +0530 Subject: [PATCH 07/11] no csi --- kustomize-nfs/dkube/base/dkube.yaml | 4 +++ kustomize/dkube/base/dkube.yaml | 56 ++++++++++++++++++++++------- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index 5ee1513..5212967 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -36,6 +36,8 @@ spec: value: nfsServer - name: NFS_BASE_PATH value: nfsBasePath + - name: CEPH_MONITORS + value: 'cephMonitors' - name: DKUBE_STORAGE_PROVIDER valueFrom: configMapKeyRef: @@ -578,6 +580,8 @@ spec: value: nfsBasePath - name: DKUBE_APISERVER_ROLE value: master + - name: CEPH_MONITORS + value: 'cephMonitors' - name: DKUBE_STORAGE_PROVIDER valueFrom: configMapKeyRef: diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 7e3cdd4..f16590b 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -36,6 +36,8 @@ spec: value: nfsServer - name: NFS_BASE_PATH value: nfsBasePath + - name: CEPH_MONITORS + value: 'cephMonitors' - name: DKUBE_STORAGE_PROVIDER valueFrom: configMapKeyRef: @@ -70,8 +72,7 @@ spec: - mountPath: /var/run/docker.sock name: docker - mountPath: /var/log/dkube - subPath: dkube/system/logs/dkube - name: store + name: dkube-logs - mountPath: /etc/dkube/dkube_platform_cfg name: pc - image: dkubeDownloaderImage @@ -93,8 +94,19 @@ spec: serviceAccountName: dkube volumes: - name: store - persistentVolumeClaim: - claimName: dkube-pvc + cephfs: + monitors: cephMonitors + path: /dkube + user: admin + secretRef: + name: ceph-secret + - name: dkube-logs + cephfs: + monitors: cephMonitors + path: /dkube/system/logs/dkube + user: admin + secretRef: + name: ceph-secret - hostPath: path: /var/run/docker.sock name: docker @@ -273,8 +285,12 @@ spec: serviceAccountName: dkube volumes: - name: dkube-logs - persistentVolumeClaim: - claimName: dkube-pvc + cephfs: + monitors: cephMonitors + path: /dkube/system/logs/dkube + user: admin + secretRef: + name: ceph-secret --- apiVersion: apps/v1 kind: Deployment @@ -453,8 +469,12 @@ spec: terminationGracePeriodSeconds: 30 volumes: - name: storage - persistentVolumeClaim: - claimName: dkube-pvc + cephfs: + monitors: cephMonitors + path: / + user: admin + secretRef: + name: ceph-secret --- apiVersion: apps/v1 kind: Deployment @@ -575,6 +595,8 @@ spec: value: nfsBasePath - name: DKUBE_APISERVER_ROLE value: master + - name: CEPH_MONITORS + value: 'cephMonitors' - name: DKUBE_STORAGE_PROVIDER valueFrom: configMapKeyRef: @@ -607,8 +629,7 @@ spec: - mountPath: /var/run/docker.sock name: docker - mountPath: /var/log/dkube - name: store - subPath: dkube/system/logs/dkube + name: dkube-logs - mountPath: /etc/dkube/dkube_platform_cfg name: pc dnsConfig: @@ -625,8 +646,19 @@ spec: serviceAccountName: dkube volumes: - name: store - persistentVolumeClaim: - claimName: dkube-pvc + cephfs: + monitors: cephMonitors + path: /dkube + user: admin + secretRef: + name: ceph-secret + - name: dkube-logs + cephfs: + monitors: cephMonitors + path: /dkube/system/logs/dkube + user: admin + secretRef: + name: ceph-secret - hostPath: path: /var/run/docker.sock name: docker From 8ab4bf536c2ba7fb1f48fb2da3040f9011490060 Mon Sep 17 00:00:00 2001 From: Deepak S Date: Wed, 10 Feb 2021 13:46:46 +0530 Subject: [PATCH 08/11] remove env --- kustomize-nfs/dkube/base/dkube.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index 5212967..508f2b2 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -49,10 +49,6 @@ spec: value: dkubeClusterType - name: MODEL_REGISTRY value: modelRegistry - envFrom: - - configMapRef: - name: ceph-config - optional: true image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main @@ -591,10 +587,6 @@ spec: value: dkubeClusterType - name: MODEL_REGISTRY value: modelRegistry - envFrom: - - configMapRef: - name: ceph-config - optional: true image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main From 00226834066380109b79b51c77050b58002ed644 Mon Sep 17 00:00:00 2001 From: Deepak S Date: Wed, 10 Feb 2021 18:23:30 +0530 Subject: [PATCH 09/11] rm --- kustomize/dkube/base/dkube.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index f16590b..7271b43 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -49,10 +49,6 @@ spec: value: dkubeClusterType - name: MODEL_REGISTRY value: modelRegistry - envFrom: - - configMapRef: - name: ceph-config - optional: true image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main @@ -68,7 +64,6 @@ spec: volumeMounts: - mountPath: /home/dkube/d3s name: store - subPath: dkube - mountPath: /var/run/docker.sock name: docker - mountPath: /var/log/dkube @@ -275,7 +270,6 @@ spec: volumeMounts: - mountPath: /var/log/dkube name: dkube-logs - subPath: dkube/system/logs/dkube dnsPolicy: ClusterFirst imagePullSecrets: - name: dkubeDockerSecret @@ -575,7 +569,6 @@ spec: volumeMounts: - mountPath: /dkube name: store - subPath: dkube - env: - name: DKUBE_MOUNT_PATH value: /home/dkube/d3s @@ -606,10 +599,6 @@ spec: value: dkubeClusterType - name: MODEL_REGISTRY value: modelRegistry - envFrom: - - configMapRef: - name: ceph-config - optional: true image: dkubeApiServerImage imagePullPolicy: IfNotPresent name: main @@ -625,7 +614,6 @@ spec: volumeMounts: - mountPath: /home/dkube/d3s name: store - subPath: dkube - mountPath: /var/run/docker.sock name: docker - mountPath: /var/log/dkube From 4d5d2e0272b6e16ddec7b8b2d4d6d906252e0d38 Mon Sep 17 00:00:00 2001 From: tambem Date: Fri, 19 Feb 2021 14:24:55 +0530 Subject: [PATCH 10/11] running exporter with cap_sys_admin to read mig device information --- kustomize-nfs/dkube/base/dkube.yaml | 5 +++-- kustomize/dkube/base/dkube.yaml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index 508f2b2..9767672 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -137,8 +137,9 @@ spec: name: http-metrics protocol: TCP securityContext: - privileged: true - runAsUser: 0 + capabilities: + add: + - SYS_ADMIN volumeMounts: - mountPath: /usr/local/nvidia/lib64 name: nvidia-lib diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 7271b43..0e64470 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -143,8 +143,9 @@ spec: name: http-metrics protocol: TCP securityContext: - privileged: true - runAsUser: 0 + capabilities: + add: + - SYS_ADMIN volumeMounts: - mountPath: /usr/local/nvidia/lib64 name: nvidia-lib From 6751356abdd68e27663810aa20bd2e5e0dc318e6 Mon Sep 17 00:00:00 2001 From: tambem Date: Fri, 19 Feb 2021 18:33:27 +0530 Subject: [PATCH 11/11] enaling mig monitor to be able to fetch mig device data in container --- kustomize-nfs/dkube/base/dkube.yaml | 2 ++ kustomize/dkube/base/dkube.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/kustomize-nfs/dkube/base/dkube.yaml b/kustomize-nfs/dkube/base/dkube.yaml index 9767672..e879676 100644 --- a/kustomize-nfs/dkube/base/dkube.yaml +++ b/kustomize-nfs/dkube/base/dkube.yaml @@ -129,6 +129,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName + - name: NVIDIA_MIG_MONITOR_DEVICES + value: all image: dkubeExtImage imagePullPolicy: IfNotPresent name: main diff --git a/kustomize/dkube/base/dkube.yaml b/kustomize/dkube/base/dkube.yaml index 0e64470..1bc2b70 100644 --- a/kustomize/dkube/base/dkube.yaml +++ b/kustomize/dkube/base/dkube.yaml @@ -135,6 +135,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName + - name: NVIDIA_MIG_MONITOR_DEVICES + value: all image: dkubeExtImage imagePullPolicy: IfNotPresent name: main