From f1b0ad6625d331a9583d3c461d32c85e1609f335 Mon Sep 17 00:00:00 2001 From: Rui Gao Date: Mon, 22 Jun 2026 02:01:44 +0000 Subject: [PATCH 1/3] remove non existing nodes when deploying cluster configuration --- .../deploy/start.sh.template | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/src/cluster-configuration/deploy/start.sh.template b/src/cluster-configuration/deploy/start.sh.template index 5d9e7c0f..e97fcfa9 100644 --- a/src/cluster-configuration/deploy/start.sh.template +++ b/src/cluster-configuration/deploy/start.sh.template @@ -42,36 +42,54 @@ kubectl apply --overwrite=true -f priority-class.yaml || exit $? kubectl apply -f write-etc-hosts.yaml || exit $? # Add `pai-master`, `pai-worker`, `pai-storage` label to corresponding nodes and remove irrelant labels +# Skip nodes that no longer exist in the cluster to avoid deploy failure +all_nodes=$(kubectl get nodes -o custom-columns=NAME:.metadata.name --no-headers) || exit $? +existing_nodes="" + +{%- for host in cluster_cfg['layout']['machine-list'] %} + {%- set hostname = cluster_cfg['layout']['machine-list'][host]['hostname'] %} +if echo "$all_nodes" | grep -Fxq "{{ hostname }}"; then + existing_nodes="$existing_nodes {{ hostname }}" +else + echo "WARNING: node {{ hostname }} not found, skipping" >&2 +fi +{%- endfor %} + ( {%- for host in cluster_cfg['layout']['machine-list'] %} + {%- set hostname = cluster_cfg['layout']['machine-list'][host]['hostname'] %} +if echo "$existing_nodes" | grep -Fwxq "{{ hostname }}"; then {%- if 'pai-master' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-master'] == 'true' %} -echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-master=true pai-worker=false || exit $? +echo "kubectl label --overwrite=true nodes {{ hostname }} pai-master=true pai-worker=false || exit $?" {%- else %} -echo kubectl label nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-master- || exit $? +echo "kubectl label nodes {{ hostname }} pai-master- || exit $?" {%- endif %} + {%- if 'pai-worker' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-worker'] == 'true' %} -echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-worker=true || exit $? +echo "kubectl label --overwrite=true nodes {{ hostname }} pai-worker=true || exit $?" {%- set machine_type = cluster_cfg['layout']['machine-list'][host]['machine-type'] %} {%- if machine_type in cluster_cfg['layout']['machine-sku'] and 'computing-device' in cluster_cfg['layout']['machine-sku'][machine_type] %} {%- set device_type = cluster_cfg['layout']['machine-sku'][machine_type]['computing-device']['type'] %} {%- if device_type == 'nvidia.com/gpu' %} -echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} vendor=nvidia || exit $? +echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=nvidia || exit $?" {%- elif device_type == 'amd.com/gpu' %} -echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} vendor=amd || exit $? +echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=amd || exit $?" {%- else %} -echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} vendor=unknown || exit $? +echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=unknown || exit $?" {%- endif %} {%- else %} -echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} vendor=cpu || exit $? +echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=cpu || exit $?" {%- endif %} {%- else %} -echo kubectl label nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-worker- || exit $? +echo "kubectl label nodes {{ hostname }} pai-worker- || exit $?" {%- endif %} + {%- if 'pai-storage' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-storage'] == 'true' %} -echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-storage=true || exit $? +echo "kubectl label --overwrite=true nodes {{ hostname }} pai-storage=true || exit $?" {%- else %} -echo kubectl label nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-storage- || exit $? +echo "kubectl label nodes {{ hostname }} pai-storage- || exit $?" {%- endif %} +fi {%- endfor %} ) | parallel || exit $? From fbf82d6a22022382b4bd903bba979cc432a460c3 Mon Sep 17 00:00:00 2001 From: Rui Gao Date: Wed, 24 Jun 2026 07:00:38 +0000 Subject: [PATCH 2/3] fix comments of copilot --- .../deploy/start.sh.template | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/cluster-configuration/deploy/start.sh.template b/src/cluster-configuration/deploy/start.sh.template index e97fcfa9..93e06b4d 100644 --- a/src/cluster-configuration/deploy/start.sh.template +++ b/src/cluster-configuration/deploy/start.sh.template @@ -58,36 +58,36 @@ fi ( {%- for host in cluster_cfg['layout']['machine-list'] %} {%- set hostname = cluster_cfg['layout']['machine-list'][host]['hostname'] %} -if echo "$existing_nodes" | grep -Fwxq "{{ hostname }}"; then +if echo "$existing_nodes" | grep -Fwq "{{ hostname }}"; then {%- if 'pai-master' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-master'] == 'true' %} -echo "kubectl label --overwrite=true nodes {{ hostname }} pai-master=true pai-worker=false || exit $?" +echo "kubectl label --overwrite=true nodes {{ hostname }} pai-master=true pai-worker=false || exit \$?" {%- else %} -echo "kubectl label nodes {{ hostname }} pai-master- || exit $?" +echo "kubectl label nodes {{ hostname }} pai-master- || exit \$?" {%- endif %} {%- if 'pai-worker' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-worker'] == 'true' %} -echo "kubectl label --overwrite=true nodes {{ hostname }} pai-worker=true || exit $?" +echo "kubectl label --overwrite=true nodes {{ hostname }} pai-worker=true || exit \$?" {%- set machine_type = cluster_cfg['layout']['machine-list'][host]['machine-type'] %} {%- if machine_type in cluster_cfg['layout']['machine-sku'] and 'computing-device' in cluster_cfg['layout']['machine-sku'][machine_type] %} {%- set device_type = cluster_cfg['layout']['machine-sku'][machine_type]['computing-device']['type'] %} {%- if device_type == 'nvidia.com/gpu' %} -echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=nvidia || exit $?" +echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=nvidia || exit \$?" {%- elif device_type == 'amd.com/gpu' %} -echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=amd || exit $?" +echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=amd || exit \$?" {%- else %} -echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=unknown || exit $?" +echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=unknown || exit \$?" {%- endif %} {%- else %} -echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=cpu || exit $?" +echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=cpu || exit \$?" {%- endif %} {%- else %} -echo "kubectl label nodes {{ hostname }} pai-worker- || exit $?" +echo "kubectl label nodes {{ hostname }} pai-worker- || exit \$?" {%- endif %} {%- if 'pai-storage' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-storage'] == 'true' %} -echo "kubectl label --overwrite=true nodes {{ hostname }} pai-storage=true || exit $?" +echo "kubectl label --overwrite=true nodes {{ hostname }} pai-storage=true || exit \$?" {%- else %} -echo "kubectl label nodes {{ hostname }} pai-storage- || exit $?" +echo "kubectl label nodes {{ hostname }} pai-storage- || exit \$?" {%- endif %} fi {%- endfor %} From 98c8b977b84023ebe4a6915ba20bf454b6abb94c Mon Sep 17 00:00:00 2001 From: Rui Gao Date: Wed, 24 Jun 2026 07:38:54 +0000 Subject: [PATCH 3/3] fix copilot comments --- src/cluster-configuration/deploy/start.sh.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cluster-configuration/deploy/start.sh.template b/src/cluster-configuration/deploy/start.sh.template index 93e06b4d..f15891af 100644 --- a/src/cluster-configuration/deploy/start.sh.template +++ b/src/cluster-configuration/deploy/start.sh.template @@ -49,7 +49,7 @@ existing_nodes="" {%- for host in cluster_cfg['layout']['machine-list'] %} {%- set hostname = cluster_cfg['layout']['machine-list'][host]['hostname'] %} if echo "$all_nodes" | grep -Fxq "{{ hostname }}"; then - existing_nodes="$existing_nodes {{ hostname }}" + existing_nodes="${existing_nodes}{{ hostname }}"$'\n' else echo "WARNING: node {{ hostname }} not found, skipping" >&2 fi @@ -58,7 +58,7 @@ fi ( {%- for host in cluster_cfg['layout']['machine-list'] %} {%- set hostname = cluster_cfg['layout']['machine-list'][host]['hostname'] %} -if echo "$existing_nodes" | grep -Fwq "{{ hostname }}"; then +if printf '%s' "$existing_nodes" | grep -Fxq "{{ hostname }}"; then {%- if 'pai-master' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-master'] == 'true' %} echo "kubectl label --overwrite=true nodes {{ hostname }} pai-master=true pai-worker=false || exit \$?" {%- else %}