From e992eaab4acf49bea4ab5f545c060349b61f65e4 Mon Sep 17 00:00:00 2001 From: "Han, Chao1" Date: Mon, 21 Jul 2025 14:52:46 +0800 Subject: [PATCH 1/2] support optim cases --- .../algorithms/ddp_comm_hooks/ddp_zero_hook.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py b/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py index 2a08212dfa9cc..a8ec8a3120d41 100644 --- a/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py +++ b/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py @@ -233,12 +233,12 @@ def hook_with_zero_step( ddp_ref = weakref.ref(ddp) # NOTE: Gloo may hang with this overlapping approach, so we require - # NCCL/HCCL backend for now; see https://github.com/pytorch/pytorch/issues/62300 + # NCCL/HCCL/XCCL backend for now; see https://github.com/pytorch/pytorch/issues/62300 pg = dist.get_backend(ddp_ref().process_group) # type: ignore[union-attr] - if (pg != dist.Backend.NCCL) and (pg != "hccl"): + if (pg != dist.Backend.NCCL) and (pg != "hccl") and (pg != "xccl"): raise RuntimeError( "Overlapping DDP with ZeRO using this approach currently requires " - "NCCL/HCCL backend to avoid hangs" + "NCCL/HCCL/XCCL backend to avoid hangs" ) if shard_buckets: @@ -395,12 +395,12 @@ def hook_with_zero_step_interleaved( ddp_ref = weakref.ref(ddp) # NOTE: Gloo may hang with this overlapping approach, so we require - # NCCL/HCCL backend for now; see https://github.com/pytorch/pytorch/issues/62300 + # NCCL/HCCL/XCCL backend for now; see https://github.com/pytorch/pytorch/issues/62300 pg = dist.get_backend(ddp_ref().process_group) # type: ignore[union-attr] - if (pg != dist.Backend.NCCL) and (pg != "hccl"): + if (pg != dist.Backend.NCCL) and (pg != "hccl") and (pg != "xccl"): raise RuntimeError( "Overlapping DDP with ZeRO using this approach currently requires " - "NCCL/HCCL backend to avoid hangs" + "NCCL/HCCL/XCCL backend to avoid hangs" ) if shard_buckets: From 62eb2cacfc6fb17c4a402d46961c5877f3e74405 Mon Sep 17 00:00:00 2001 From: "Han, Chao1" Date: Tue, 29 Jul 2025 09:53:39 +0800 Subject: [PATCH 2/2] blacklist gloo --- .../algorithms/ddp_comm_hooks/ddp_zero_hook.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py b/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py index a8ec8a3120d41..6153d8e03fdff 100644 --- a/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py +++ b/torch/distributed/algorithms/ddp_comm_hooks/ddp_zero_hook.py @@ -232,13 +232,11 @@ def hook_with_zero_step( ) ddp_ref = weakref.ref(ddp) - # NOTE: Gloo may hang with this overlapping approach, so we require - # NCCL/HCCL/XCCL backend for now; see https://github.com/pytorch/pytorch/issues/62300 + # NOTE: Gloo may hang with this overlapping approach; see https://github.com/pytorch/pytorch/issues/62300 pg = dist.get_backend(ddp_ref().process_group) # type: ignore[union-attr] - if (pg != dist.Backend.NCCL) and (pg != "hccl") and (pg != "xccl"): + if pg == dist.Backend.GLOO: raise RuntimeError( - "Overlapping DDP with ZeRO using this approach currently requires " - "NCCL/HCCL/XCCL backend to avoid hangs" + "Gloo backend using Overlapping DDP with ZeRO may meet hangs" ) if shard_buckets: @@ -394,13 +392,11 @@ def hook_with_zero_step_interleaved( ) ddp_ref = weakref.ref(ddp) - # NOTE: Gloo may hang with this overlapping approach, so we require - # NCCL/HCCL/XCCL backend for now; see https://github.com/pytorch/pytorch/issues/62300 + # NOTE: Gloo may hang with this overlapping approach; see https://github.com/pytorch/pytorch/issues/62300 pg = dist.get_backend(ddp_ref().process_group) # type: ignore[union-attr] - if (pg != dist.Backend.NCCL) and (pg != "hccl") and (pg != "xccl"): + if pg == dist.Backend.GLOO: raise RuntimeError( - "Overlapping DDP with ZeRO using this approach currently requires " - "NCCL/HCCL/XCCL backend to avoid hangs" + "Gloo backend using Overlapping DDP with ZeRO may meet hangs" ) if shard_buckets: