From 871acc0b95eedc3f1b0b7d2882e4c07d5bcb9fa7 Mon Sep 17 00:00:00 2001 From: Ferran Date: Tue, 6 Oct 2020 16:19:41 +0100 Subject: [PATCH 01/10] Collect all StorageOS logs --- examples/bundle.yaml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/examples/bundle.yaml b/examples/bundle.yaml index cd126fd..a3d30a4 100644 --- a/examples/bundle.yaml +++ b/examples/bundle.yaml @@ -9,35 +9,13 @@ spec: - name=storageos-cluster-operator namespace: storageos-operator limits: - maxLines: 10000 + maxLines: 100000 - logs: selector: - app=storageos - - app.kubernetes.io/component=storageos-csi-helper namespace: kube-system limits: - maxLines: 10000 - - logs: - selector: - - app=storageos - - app.kubernetes.io/component=storageos-daemonset - namespace: kube-system - limits: - maxLines: 10000 - - logs: - selector: - - app=storageos - - app.kubernetes.io/component=storageos-api-manager - namespace: kube-system - limits: - maxLines: 10000 - - logs: - selector: - - app=storageos - - app.kubernetes.io/component=storageos-scheduler - namespace: kube-system - limits: - maxLines: 10000 + maxLines: 1000000 - clusterResources: {} analyzers: - clusterVersion: @@ -108,4 +86,4 @@ spec: when: "< 1" message: The scheduler deployment does not have any ready replicas. - pass: - message: The scheduler deployment is ready. \ No newline at end of file + message: The scheduler deployment is ready. From dd0dd15c0e2c1afd6c3c62700c00752318c831a8 Mon Sep 17 00:00:00 2001 From: Ferran Date: Tue, 6 Oct 2020 17:56:31 +0100 Subject: [PATCH 02/10] StoargeOS CLI commands - Run the cli with a command - Run the cli with multple commands - Exec into an existing cli --- examples/cli.yaml | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 examples/cli.yaml diff --git a/examples/cli.yaml b/examples/cli.yaml new file mode 100644 index 0000000..108a2f9 --- /dev/null +++ b/examples/cli.yaml @@ -0,0 +1,44 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: StorageOS +spec: + collectors: + - exec: + collectorName: storageos-nodes + name: storageos-nodes + selector: + - run=cli + namespace: kube-system + command: ["storageos"] + args: ["get", "nodes"] + timeout: 20s + - run: + collectorName: "run-storageos-cli" + image: storageos/cli:v2.2.0 + namespace: kube-system + command: ["storageos"] + args: + - --endpoints + - "http://storageos.kube-system.svc:5705" + - get + - nodes + - -o + - yaml + imagePullPolicy: IfNotPresent + - run: + collectorName: "run-storageos-cli2" + image: storageos/cli:v2.2.0 + namespace: kube-system + command: ["/bin/sh"] + args: + - -c + - " + echo STORAGEOS NODES; + storageos --endpoints http://storageos.kube-system.svc:5705 get nodes -o yaml; + echo '-----------------------------------------'; + echo STORAGEOS VOLUMES; + storageos --endpoints http://storageos.kube-system.svc:5705 get volumes --all-namespaces -oyaml; + echo '-----------------------------------------'; + " + imagePullPolicy: IfNotPresent From aa8ed951fbc4e86622a286d41fe1121c443fe624 Mon Sep 17 00:00:00 2001 From: Ferran Date: Thu, 8 Oct 2020 16:37:01 +0100 Subject: [PATCH 03/10] Get a full ps from the cluster --- examples/ps.yaml | 118 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 examples/ps.yaml diff --git a/examples/ps.yaml b/examples/ps.yaml new file mode 100644 index 0000000..39ec535 --- /dev/null +++ b/examples/ps.yaml @@ -0,0 +1,118 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: StorageOS +spec: + collectors: + - exec: + collectorName: ps + name: ps + selector: + - app=tools + namespace: kube-system + command: ["ps"] + args: ["auxwwwf"] + timeout: 20s + +# We created the following Daemonset Pod to execute on it +# kubectl create -f ./daemonset-tools.yaml +# +# cat ./daemonset-tools.yaml +#apiVersion: apps/v1 +#kind: DaemonSet +#metadata: +# name: tools +# namespace: kube-system +#spec: +# revisionHistoryLimit: 10 +# selector: +# matchLabels: +# app: tools +# template: +# metadata: +# labels: +# app: tools +# spec: +# containers: +# - command: +# - /bin/sh +# - -c +# - "while true; do sleep 3600; done" +# image: arau/tools:0.7 +# name: tools +# securityContext: +# allowPrivilegeEscalation: true +# capabilities: +# add: +# - SYS_ADMIN +# privileged: true +# volumeMounts: +# - mountPath: /sys +# name: sys +# - mountPath: /var/lib/storageos +# mountPropagation: Bidirectional +# name: state +# - mountPath: /lib/modules +# name: kernel-modules +# readOnly: true +# - mountPath: /etc/storageos +# name: config +# - mountPath: /csi +# name: plugin-dir +# - mountPath: /var/lib/csi/sockets/ +# name: registrar-socket-dir +# dnsPolicy: ClusterFirstWithHostNet +# hostNetwork: true +# hostPID: true +# priorityClassName: system-node-critical +# tolerations: +# - key: node.kubernetes.io/disk-pressure +# operator: Exists +# - key: node.kubernetes.io/memory-pressure +# operator: Exists +# - key: node.kubernetes.io/network-unavailable +# operator: Exists +# - key: node.kubernetes.io/not-ready +# operator: Exists +# - key: node.kubernetes.io/out-of-disk +# operator: Exists +# - key: node.kubernetes.io/pid-pressure +# operator: Exists +# - key: node.kubernetes.io/unreachable +# operator: Exists +# - key: node.kubernetes.io/unschedulable +# operator: Exists +# volumes: +# - hostPath: +# path: /lib/modules +# type: "" +# name: kernel-modules +# - hostPath: +# path: /sys +# type: "" +# name: sys +# - hostPath: +# path: /var/lib/storageos +# type: "" +# name: state +# - configMap: +# defaultMode: 384 +# name: storageos-node-config +# optional: false +# name: config +# - hostPath: +# path: /var/lib/kubelet +# type: Directory +# name: kubelet-dir +# - hostPath: +# path: /var/lib/kubelet/plugins_registry +# type: Directory +# name: registration-dir +# - hostPath: +# path: /var/lib/kubelet/device-plugins/ +# type: DirectoryOrCreate +# name: registrar-socket-dir +# - hostPath: +# path: /var/lib/kubelet/plugins_registry/storageos +# type: DirectoryOrCreate +# name: plugin-dir From 6918e71956a0e1bef391463f5649953ea7ebda1e Mon Sep 17 00:00:00 2001 From: Ferran Date: Fri, 9 Oct 2020 17:13:49 +0100 Subject: [PATCH 04/10] Network check for StorageOS Pods Targetting all nodes from the cluster with or without StorageOS and running a nc for all StorageOS Ports. --- examples/netcat.yaml | 29 +++++++++++++++++++++++++++++ examples/ps.yaml | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 examples/netcat.yaml diff --git a/examples/netcat.yaml b/examples/netcat.yaml new file mode 100644 index 0000000..ca4286c --- /dev/null +++ b/examples/netcat.yaml @@ -0,0 +1,29 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: StorageOS +spec: + collectors: + - run: + collectorName: "network-check" + image: arau/tools:0.9 + namespace: kube-system + command: + - "/bin/sh" + - "-c" + - " + #!/bin/bash + # + # IOPort = 5703 # DataPlane + # SupervisorPort = 5704 # For sync + # ExternalAPIPort = 5705 # REST API + # InternalAPIPort = 5710 # Grpc API + # GossipPort = 5711 # Gossip+Healthcheck + + echo \"Source node for the test:\"; + hostname -f -I; echo; + + parallel -j2 nc -vz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \ + ::: 5703 5704 5705 5710 5711 + " + timeout: 300s diff --git a/examples/ps.yaml b/examples/ps.yaml index 39ec535..0a771db 100644 --- a/examples/ps.yaml +++ b/examples/ps.yaml @@ -38,7 +38,7 @@ spec: # - /bin/sh # - -c # - "while true; do sleep 3600; done" -# image: arau/tools:0.7 +# image: arau/tools:0.8 # name: tools # securityContext: # allowPrivilegeEscalation: true From 0db7914ba4f9d2f2b58e2745fa0fd25480ed1754 Mon Sep 17 00:00:00 2001 From: Ferran Date: Fri, 9 Oct 2020 18:02:39 +0100 Subject: [PATCH 05/10] Add HostNetwork to the PodSpec --- examples/netcat.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/netcat.yaml b/examples/netcat.yaml index ca4286c..66bccde 100644 --- a/examples/netcat.yaml +++ b/examples/netcat.yaml @@ -8,6 +8,7 @@ spec: collectorName: "network-check" image: arau/tools:0.9 namespace: kube-system + hostNetwork: true command: - "/bin/sh" - "-c" @@ -23,7 +24,7 @@ spec: echo \"Source node for the test:\"; hostname -f -I; echo; - parallel -j2 nc -vz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \ + parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \ ::: 5703 5704 5705 5710 5711 " timeout: 300s From bc7ec166dc7bded11d82eaa93dce7b368ecd0a6e Mon Sep 17 00:00:00 2001 From: Ferran Date: Fri, 9 Oct 2020 18:08:14 +0100 Subject: [PATCH 06/10] Get process list 'run' with HostPID --- examples/ps.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/ps.yaml b/examples/ps.yaml index 0a771db..f1f90d7 100644 --- a/examples/ps.yaml +++ b/examples/ps.yaml @@ -4,7 +4,17 @@ metadata: name: StorageOS spec: collectors: - - exec: + - run: + collectorName: "ps-run" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + command: ["ps"] + args: ["auxwwwf"] + timeout: 20s + - exec: + # This collector would need the DaemonSet definition below to be created + # before hand collectorName: ps name: ps selector: From 0db79f200d37a378a2a64bb14f1cd9407b4a7504 Mon Sep 17 00:00:00 2001 From: Ferran Date: Sun, 11 Oct 2020 20:08:08 +0100 Subject: [PATCH 07/10] Examples to gather information --- examples/bundle-info.yaml | 103 ++++++++++++++++++ examples/bundle.yaml | 95 +++++++++++++++- examples/netcat.yaml | 4 +- ...s.yaml => ps-with-external-daemonset.yaml} | 12 +- examples/ps-workers.yaml | 18 +++ examples/runmanynodes.yaml | 38 +++++++ 6 files changed, 255 insertions(+), 15 deletions(-) create mode 100644 examples/bundle-info.yaml rename examples/{ps.yaml => ps-with-external-daemonset.yaml} (93%) create mode 100644 examples/ps-workers.yaml create mode 100644 examples/runmanynodes.yaml diff --git a/examples/bundle-info.yaml b/examples/bundle-info.yaml new file mode 100644 index 0000000..81ab406 --- /dev/null +++ b/examples/bundle-info.yaml @@ -0,0 +1,103 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: StorageOS +spec: + collectors: + - clusterResources: {} + - logs: + selector: + - name=storageos-cluster-operator + namespace: storageos-operator + limits: + maxLines: 10000 + - logs: + selector: + - app=storageos + namespace: kube-system + limits: + maxLines: 1000000 + - run: + name: "network-checks" + collectorName: "netcat" + image: arau/tools:0.9 + namespace: kube-system + hostNetwork: true + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: + - "/bin/sh" + - "-c" + - " + #!/bin/bash + # + # IOPort = 5703 # DataPlane + # SupervisorPort = 5704 # For sync + # ExternalAPIPort = 5705 # REST API + # InternalAPIPort = 5710 # Grpc API + # GossipPort = 5711 # Gossip+Healthcheck + + echo \"Source node for the test:\"; + hostname -f -I; echo; + + parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \ + ::: 5703 5704 5705 5710 5711 + " + timeout: 90s + - run: + name: "backend-disks" + collectorName: "lsblk" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: ["lsblk"] + timeout: 90s + - run: + name: "free-disk-space" + collectorName: "df" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: ["df -h"] + timeout: 90s + - run: + name: "ps-all-nodes" + collectorName: "processlist" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: ["ps"] + args: ["auxwwwf"] + timeout: 90s + - exec: + name: storageos-cli-info + collectorName: storageos-cli + selector: + - run=cli + namespace: kube-system + timeout: 90s + command: ["/bin/sh"] + args: + - -c + - " + export STORAGEOS_ENDPOINTS='http://storageos.kube-system.svc:5705'; + echo STORAGEOS CLUSTER; + storageos get cluster -ojson; + echo '-----------------------------------------'; + echo STORAGEOS LICENCE; + storageos get licence -ojson; + echo '-----------------------------------------'; + echo STORAGEOS NAMESPACE; + storageos get namespace -ojson; + echo '-----------------------------------------'; + echo STORAGEOS VOLUMES; + storageos get volumes --all-namespaces -ojson --timeout 30s; + echo '-----------------------------------------'; + " diff --git a/examples/bundle.yaml b/examples/bundle.yaml index a3d30a4..2035ae2 100644 --- a/examples/bundle.yaml +++ b/examples/bundle.yaml @@ -4,19 +4,106 @@ metadata: name: StorageOS spec: collectors: + - clusterResources: {} - logs: selector: - name=storageos-cluster-operator namespace: storageos-operator limits: - maxLines: 100000 + maxLines: 10000 - logs: selector: - app=storageos namespace: kube-system limits: maxLines: 1000000 - - clusterResources: {} + - run: + name: "network-checks" + collectorName: "netcat" + image: arau/tools:0.9 + namespace: kube-system + hostNetwork: true + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: + - "/bin/sh" + - "-c" + - " + #!/bin/bash + # + # IOPort = 5703 # DataPlane + # SupervisorPort = 5704 # For sync + # ExternalAPIPort = 5705 # REST API + # InternalAPIPort = 5710 # Grpc API + # GossipPort = 5711 # Gossip+Healthcheck + + echo \"Source node for the test:\"; + hostname -f -I; echo; + + parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \ + ::: 5703 5704 5705 5710 5711 + " + timeout: 90s + - run: + name: "backend-disks" + collectorName: "lsblk" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: ["lsblk"] + timeout: 90s + - run: + name: "free-disk-space" + collectorName: "df" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: ["df -h"] + timeout: 90s + - run: + name: "ps-all-nodes" + collectorName: "processlist" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: ["ps"] + args: ["auxwwwf"] + timeout: 90s + - exec: + name: storageos-cli-info + collectorName: storageos-cli + selector: + - run=cli + namespace: kube-system + timeout: 90s + command: ["/bin/sh"] + args: + - -c + - " + export STORAGEOS_ENDPOINTS='http://storageos.kube-system.svc:5705'; + echo STORAGEOS CLUSTER; + storageos get cluster -ojson; + echo '-----------------------------------------'; + echo STORAGEOS LICENCE; + storageos get licence -ojson; + echo '-----------------------------------------'; + echo STORAGEOS NAMESPACE; + storageos get namespace -ojson; + echo '-----------------------------------------'; + echo STORAGEOS NODES; + storageos get nodes -o json; + echo '-----------------------------------------'; + echo STORAGEOS VOLUMES; + storageos get volumes --all-namespaces -ojson; + echo '-----------------------------------------'; + " analyzers: - clusterVersion: outcomes: @@ -77,7 +164,7 @@ spec: when: "< 1" message: The CSI helper deployment does not have any ready replicas. - pass: - message: The CSI helper deployment is ready. + message: The CSI helper deployment is ready. - deploymentStatus: name: storageos-scheduler namespace: kube-system @@ -86,4 +173,4 @@ spec: when: "< 1" message: The scheduler deployment does not have any ready replicas. - pass: - message: The scheduler deployment is ready. + message: The scheduler deployment is ready. diff --git a/examples/netcat.yaml b/examples/netcat.yaml index 66bccde..a2ec089 100644 --- a/examples/netcat.yaml +++ b/examples/netcat.yaml @@ -9,6 +9,8 @@ spec: image: arau/tools:0.9 namespace: kube-system hostNetwork: true + nodeSelector: + node-role.kubernetes.io/worker: "true" command: - "/bin/sh" - "-c" @@ -27,4 +29,4 @@ spec: parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \ ::: 5703 5704 5705 5710 5711 " - timeout: 300s + timeout: 90s diff --git a/examples/ps.yaml b/examples/ps-with-external-daemonset.yaml similarity index 93% rename from examples/ps.yaml rename to examples/ps-with-external-daemonset.yaml index f1f90d7..e9895d5 100644 --- a/examples/ps.yaml +++ b/examples/ps-with-external-daemonset.yaml @@ -4,15 +4,7 @@ metadata: name: StorageOS spec: collectors: - - run: - collectorName: "ps-run" - image: arau/tools:0.9 - namespace: kube-system - hostPID: true - command: ["ps"] - args: ["auxwwwf"] - timeout: 20s - - exec: + - exec: # This collector would need the DaemonSet definition below to be created # before hand collectorName: ps @@ -22,7 +14,7 @@ spec: namespace: kube-system command: ["ps"] args: ["auxwwwf"] - timeout: 20s + timeout: 60s # We created the following Daemonset Pod to execute on it # kubectl create -f ./daemonset-tools.yaml diff --git a/examples/ps-workers.yaml b/examples/ps-workers.yaml new file mode 100644 index 0000000..f3f5314 --- /dev/null +++ b/examples/ps-workers.yaml @@ -0,0 +1,18 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: StorageOS +spec: + collectors: + - run: + # Because of the nodeSelector, this runner will create + # one Pod for each node that matches the nodeSelector Term + collectorName: "ps-all-nodes" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: ["ps"] + args: ["auxwwwf"] + timeout: 20s diff --git a/examples/runmanynodes.yaml b/examples/runmanynodes.yaml new file mode 100644 index 0000000..ea244ec --- /dev/null +++ b/examples/runmanynodes.yaml @@ -0,0 +1,38 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: StorageOS +spec: + collectors: + - run: + collectorName: "hostname-one" + name: "runonenode" + image: arau/tools:0.9 + namespace: kube-system + hostNetwork: true + command: + - "/bin/sh" + - "-c" + - " + #!/bin/bash + + hostname + " + timeout: 20s + - run: + collectorName: "hostname-worker" + name: "runmanynodes" + image: arau/tools:0.9 + namespace: kube-system + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: + - "/bin/sh" + - "-c" + - " + #!/bin/bash + + hostname + " + timeout: 30s From aef63698aec10713e16ecd7ac385d139431c6abf Mon Sep 17 00:00:00 2001 From: Ferran Date: Mon, 12 Oct 2020 13:59:20 +0100 Subject: [PATCH 08/10] Example to gather data (no analyzers) - ps - netcat - top - storageos cli get nodes, volumes, cluster, etc - lsblk - df - logs (operator + DS + scheduler + CSI) --- examples/bundle-info-alpha.yaml | 119 ++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 examples/bundle-info-alpha.yaml diff --git a/examples/bundle-info-alpha.yaml b/examples/bundle-info-alpha.yaml new file mode 100644 index 0000000..ee19e5c --- /dev/null +++ b/examples/bundle-info-alpha.yaml @@ -0,0 +1,119 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: StorageOS +spec: + collectors: + - clusterResources: {} + - logs: + name: storageos-cluster-operator-logs + selector: + - name=storageos-cluster-operator + namespace: storageos-operator + limits: + maxLines: 10000 + - logs: + name: storageos-logs + selector: + - app=storageos + namespace: kube-system + limits: + maxLines: 1000000 + - run: + name: "network-checks" + collectorName: "netcat" + image: arau/tools:0.9 + namespace: kube-system + hostNetwork: true + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: + - "/bin/sh" + - "-c" + - " + #!/bin/bash + # + # IOPort = 5703 # DataPlane + # SupervisorPort = 5704 # For sync + # ExternalAPIPort = 5705 # REST API + # InternalAPIPort = 5710 # Grpc API + # GossipPort = 5711 # Gossip+Healthcheck + + echo \"Source node for the test:\"; + hostname -f -I; echo; + + parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \ + ::: 5703 5704 5705 5710 5711 + " + timeout: 90s + - exec: + name: "backend-disks" + collectorName: "lsblk" + selector: + - kind=daemonset + - app=storageos + namespace: kube-system + command: ["lsblk"] + timeout: 90s + - exec: + name: "free-disk-space" + collectorName: "df" + namespace: kube-system + selector: + - kind=daemonset + - app=storageos + command: ["df", "-h"] + timeout: 90s + - run: + name: "ps-all-nodes" + collectorName: "processlist" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: ["ps"] + args: ["auxwwwf"] + timeout: 90s + - run: + name: "loadAvg-all-nodes" + collectorName: "top" + image: arau/tools:0.9 + namespace: kube-system + hostPID: true + nodeSelector: + node-role.kubernetes.io/worker: "true" + command: ["top"] + args: + - "-b" + - "-c" + - "-n4" + - "-d2" + - "-w500" + timeout: 90s + - exec: + name: storageos-cli-info + collectorName: storageos-cli + selector: + - run=cli + namespace: kube-system + timeout: 90s + command: ["/bin/sh"] + args: + - -c + - " + export STORAGEOS_ENDPOINTS='http://storageos.kube-system.svc:5705'; + echo STORAGEOS CLUSTER; + storageos get cluster -ojson; + echo '-----------------------------------------'; + echo STORAGEOS LICENCE; + storageos get licence -ojson; + echo '-----------------------------------------'; + echo STORAGEOS NAMESPACE; + storageos get namespace -ojson; + echo '-----------------------------------------'; + echo STORAGEOS VOLUMES; + storageos get volumes --all-namespaces -ojson --timeout 30s; + echo '-----------------------------------------'; + " From 6c5599b9ed6f0af1a5e5217e2edc813dd28ff5d3 Mon Sep 17 00:00:00 2001 From: Ferran Date: Mon, 12 Oct 2020 14:02:39 +0100 Subject: [PATCH 09/10] Replace troubleshoot library to point to a self build --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index ec52300..fb96cbb 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/croomes/kubectl-plugin go 1.12 +replace github.com/replicatedhq/troubleshoot => /Users/ferran/repos/troubleshoot + require ( github.com/ahmetalpbalkan/go-cursor v0.0.0-20131010032410-8136607ea412 github.com/fatih/color v1.7.0 From f4cb7bfe51c9a5f2f1d3e9b0384c9723dcae151c Mon Sep 17 00:00:00 2001 From: Ferran Date: Tue, 13 Oct 2020 10:54:11 +0100 Subject: [PATCH 10/10] bundle with execs and DS created --- examples/bundle-with-execs.yaml | 113 ++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 examples/bundle-with-execs.yaml diff --git a/examples/bundle-with-execs.yaml b/examples/bundle-with-execs.yaml new file mode 100644 index 0000000..8e234a3 --- /dev/null +++ b/examples/bundle-with-execs.yaml @@ -0,0 +1,113 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: StorageOS +spec: + collectors: + - clusterResources: {} + - logs: + name: storageos-cluster-operator-logs + selector: + - name=storageos-cluster-operator + namespace: storageos-operator + limits: + maxLines: 10000 + - logs: + name: storageos-logs + selector: + - app=storageos + namespace: kube-system + limits: + maxLines: 1000000 + - exec: + name: "network-checks" + collectorName: "netcat" + image: arau/tools:0.9 + namespace: kube-system + selector: + - app=tools + command: + - "/bin/sh" + - "-c" + - " + #!/bin/bash + # + # IOPort = 5703 # DataPlane + # SupervisorPort = 5704 # For sync + # ExternalAPIPort = 5705 # REST API + # InternalAPIPort = 5710 # Grpc API + # GossipPort = 5711 # Gossip+Healthcheck + + echo \"Source node for the test:\"; + hostname -f -I; echo; + + parallel -j2 nc -vnz ::: $(echo $NODES_PRIVATE_IPS| sed \"s/,/ /g\" ) \ + ::: 5703 5704 5705 5710 5711 + " + timeout: 90s + - exec: + name: "backend-disks" + collectorName: "lsblk" + selector: + - kind=daemonset + - app=storageos + namespace: kube-system + command: ["lsblk"] + timeout: 90s + - exec: + name: "free-disk-space" + collectorName: "df" + namespace: kube-system + selector: + - kind=daemonset + - app=storageos + command: ["df", "-h"] + timeout: 90s + - exec: + name: "ps-all-nodes" + collectorName: "processlist" + namespace: kube-system + selector: + - app=tools + command: ["ps"] + args: ["auxwwwf"] + timeout: 90s + - exec: + name: "loadAvg-all-nodes" + collectorName: "top" + namespace: kube-system + selector: + - app=tools + command: ["top"] + args: + - "-b" + - "-c" + - "-n4" + - "-d2" + - "-w500" + timeout: 90s + - exec: + name: storageos-cli-info + collectorName: storageos-cli + selector: + - run=cli + namespace: kube-system + timeout: 90s + command: ["/bin/sh"] + args: + - -c + - " + export STORAGEOS_ENDPOINTS='http://storageos.kube-system.svc:5705'; + echo STORAGEOS CLUSTER; + storageos get cluster -ojson; + echo '-----------------------------------------'; + echo STORAGEOS LICENCE; + storageos get licence -ojson; + echo '-----------------------------------------'; + echo STORAGEOS NAMESPACE; + storageos get namespace -ojson; + echo '-----------------------------------------'; + echo STORAGEOS VOLUMES; + storageos get volumes --all-namespaces -ojson --timeout 30s; + echo '-----------------------------------------'; + "