Skip to content

remove non existing nodes when deploying cluster configuration #185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hippogr merged 3 commits into from
Jun 24, 2026
Merged

remove non existing nodes when deploying cluster configuration #185

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f1b0ad6
remove non existing nodes when deploying cluster configuration
Jun 22, 2026
fbf82d6
fix comments of copilot
Jun 24, 2026
98c8b97
fix copilot comments
Jun 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 28 additions & 10 deletions src/cluster-configuration/deploy/start.sh.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,36 +42,54 @@ kubectl apply --overwrite=true -f priority-class.yaml || exit $?
kubectl apply -f write-etc-hosts.yaml || exit $?

# Add `pai-master`, `pai-worker`, `pai-storage` label to corresponding nodes and remove irrelant labels
# Skip nodes that no longer exist in the cluster to avoid deploy failure
Comment thread
hippogr marked this conversation as resolved.
all_nodes=$(kubectl get nodes -o custom-columns=NAME:.metadata.name --no-headers) || exit $?
existing_nodes=""

{%- for host in cluster_cfg['layout']['machine-list'] %}
{%- set hostname = cluster_cfg['layout']['machine-list'][host]['hostname'] %}
if echo "$all_nodes" | grep -Fxq "{{ hostname }}"; then
existing_nodes="${existing_nodes}{{ hostname }}"$'\n'
else
echo "WARNING: node {{ hostname }} not found, skipping" >&2
fi
{%- endfor %}

(
{%- for host in cluster_cfg['layout']['machine-list'] %}
{%- set hostname = cluster_cfg['layout']['machine-list'][host]['hostname'] %}
if printf '%s' "$existing_nodes" | grep -Fxq "{{ hostname }}"; then
{%- if 'pai-master' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-master'] == 'true' %}
Comment thread
Copilot marked this conversation as resolved.
echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-master=true pai-worker=false || exit $?
echo "kubectl label --overwrite=true nodes {{ hostname }} pai-master=true pai-worker=false || exit \$?"
{%- else %}
echo kubectl label nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-master- || exit $?
echo "kubectl label nodes {{ hostname }} pai-master- || exit \$?"
{%- endif %}

{%- if 'pai-worker' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-worker'] == 'true' %}
echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-worker=true || exit $?
echo "kubectl label --overwrite=true nodes {{ hostname }} pai-worker=true || exit \$?"
{%- set machine_type = cluster_cfg['layout']['machine-list'][host]['machine-type'] %}
{%- if machine_type in cluster_cfg['layout']['machine-sku'] and 'computing-device' in cluster_cfg['layout']['machine-sku'][machine_type] %}
{%- set device_type = cluster_cfg['layout']['machine-sku'][machine_type]['computing-device']['type'] %}
{%- if device_type == 'nvidia.com/gpu' %}
echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} vendor=nvidia || exit $?
echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=nvidia || exit \$?"
{%- elif device_type == 'amd.com/gpu' %}
echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} vendor=amd || exit $?
echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=amd || exit \$?"
{%- else %}
echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} vendor=unknown || exit $?
echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=unknown || exit \$?"
{%- endif %}
{%- else %}
echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} vendor=cpu || exit $?
echo "kubectl label --overwrite=true nodes {{ hostname }} vendor=cpu || exit \$?"
{%- endif %}
{%- else %}
echo kubectl label nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-worker- || exit $?
echo "kubectl label nodes {{ hostname }} pai-worker- || exit \$?"
{%- endif %}

{%- if 'pai-storage' in cluster_cfg['layout']['machine-list'][host] and cluster_cfg['layout']['machine-list'][host]['pai-storage'] == 'true' %}
echo kubectl label --overwrite=true nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-storage=true || exit $?
echo "kubectl label --overwrite=true nodes {{ hostname }} pai-storage=true || exit \$?"
{%- else %}
echo kubectl label nodes {{ cluster_cfg['layout']['machine-list'][host]['hostname'] }} pai-storage- || exit $?
echo "kubectl label nodes {{ hostname }} pai-storage- || exit \$?"
{%- endif %}
fi
{%- endfor %}
) | parallel || exit $?

Expand Down