From e957f4bb88b28b3c9df1c60b18b6dd114dc94f08 Mon Sep 17 00:00:00 2001 From: paklui Date: Wed, 27 May 2026 19:19:51 -0700 Subject: [PATCH 01/12] add NIC_TRAFFIC_CLASS_FIFO env var for control-path DSCP marking Mirrors NCCL_IB_FIFO_TC: stamps control/signaling QP traffic with a separate DSCP value from bulk data traffic so network switches can steer ctrl and data packets into different priority queues. NIC_TRAFFIC_CLASS_FIFO=0 (default) is a complete no-op: no ctrl QPs are created and no extra work is done per iteration. Co-Authored-By: Claude Sonnet 4.6 --- src/client/EnvVars.hpp | 16 +++- src/header/TransferBench.hpp | 163 ++++++++++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 7 deletions(-) diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index 34940b69..42eb1e6d 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -120,6 +120,7 @@ class EnvVars int nicChunkBytes; // Number of bytes to send per chunk for RDMA operations int nicCqPollBatch; // Number of CQ entries to poll per ibv_poll_cq call int nicRelaxedOrder; // Use relaxed ordering for RDMA + int nicFifoTrafficClass; // DSCP/traffic class byte for control (FIFO) QPs int nicServiceLevel; // IB service level (sl) for InfiniBand QPs int nicTrafficClass; // DSCP/traffic class byte for RoCE GRH int roceVersion; // RoCE version number @@ -186,11 +187,16 @@ class EnvVars ipAddressFamily = GetEnvVar("IP_ADDRESS_FAMILY" , 4); nicChunkBytes = GetEnvVar("NIC_CHUNK_BYTES" , 1073741824); nicCqPollBatch = GetEnvVar("NIC_CQ_POLL_BATCH" , 4); - nicRelaxedOrder = GetEnvVar("NIC_RELAX_ORDER" , 1); - nicServiceLevel = GetEnvVar("NIC_SERVICE_LEVEL" , 0); - nicTrafficClass = GetEnvVar("NIC_TRAFFIC_CLASS" , 0); + nicFifoTrafficClass = GetEnvVar("NIC_TRAFFIC_CLASS_FIFO", 0); + nicRelaxedOrder = GetEnvVar("NIC_RELAX_ORDER" , 1); + nicServiceLevel = GetEnvVar("NIC_SERVICE_LEVEL" , 0); + nicTrafficClass = GetEnvVar("NIC_TRAFFIC_CLASS" , 0); // Check that NIC service level and traffic class are in valid ranges + if (nicFifoTrafficClass < 0 || nicFifoTrafficClass > 255) { + printf("[ERROR] NIC_TRAFFIC_CLASS_FIFO must be in range 0..255 (got %d)\n", nicFifoTrafficClass); + exit(1); + } if (nicServiceLevel < 0 || nicServiceLevel > 15) { printf("[ERROR] NIC_SERVICE_LEVEL must be in range 0..15 (got %d)\n", nicServiceLevel); exit(1); @@ -379,6 +385,7 @@ class EnvVars #if NIC_EXEC_ENABLED printf(" NIC_CHUNK_BYTES - Number of bytes to send at a time using NIC (default = 1GB)\n"); printf(" NIC_CQ_POLL_BATCH - Number of CQ entries to poll per ibv_poll_cq call (default = 4)\n"); + printf(" NIC_TRAFFIC_CLASS_FIFO - DSCP/traffic class byte for control (FIFO) QP GRH (default=0)\n"); printf(" NIC_RELAX_ORDER - Set to non-zero to use relaxed ordering\n"); printf(" NIC_SERVICE_LEVEL - IB service level (sl) for InfiniBand QPs (default=0)\n"); printf(" NIC_TRAFFIC_CLASS - DSCP/traffic class byte for RoCE GRH (default=0)\n"); @@ -520,6 +527,8 @@ class EnvVars "Polling %d CQ entries per ibv_poll_cq call", nicCqPollBatch); Print("NIC_RELAX_ORDER", nicRelaxedOrder, "Using %s ordering for NIC RDMA", nicRelaxedOrder ? "relaxed" : "strict"); + Print("NIC_TRAFFIC_CLASS_FIFO", nicFifoTrafficClass, + "RoCE FIFO/ctrl traffic class (DSCP) set to %d", nicFifoTrafficClass); Print("NIC_SERVICE_LEVEL", nicServiceLevel, "IB service level (sl) set to %d", nicServiceLevel); Print("NIC_TRAFFIC_CLASS", nicTrafficClass, @@ -744,6 +753,7 @@ class EnvVars cfg.nic.ibGidIndex = ibGidIndex; cfg.nic.ibPort = ibPort; cfg.nic.ipAddressFamily = ipAddressFamily; + cfg.nic.fifoTrafficClass = nicFifoTrafficClass; cfg.nic.useRelaxedOrder = nicRelaxedOrder; cfg.nic.serviceLevel = nicServiceLevel; cfg.nic.trafficClass = nicTrafficClass; diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index d94eb725..e7850110 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -270,6 +270,7 @@ namespace TransferBench int queueSize = 100; ///< Completion queue size int roceVersion = 2; ///< RoCE version (used for auto GID detection) int useRelaxedOrder = 1; ///< Use relaxed ordering + uint8_t fifoTrafficClass = 0; ///< DSCP/traffic class byte for control (FIFO) QPs uint8_t serviceLevel = 0; ///< IB service level (sl) for InfiniBand QPs uint8_t trafficClass = 0; ///< DSCP/traffic class byte for RoCE GRH int useNuma = 0; ///< Switch to closest numa thread for execution @@ -2001,9 +2002,10 @@ namespace { if (nic.maxRecvWorkReq != cfg.nic.maxRecvWorkReq) ADD_ERROR("cfg.nic.maxRecvWorkReq"); if (nic.maxSendWorkReq != cfg.nic.maxSendWorkReq) ADD_ERROR("cfg.nic.maxSendWorkReq"); // nic.queueSize is permitted to be different across ranks - if (nic.roceVersion != cfg.nic.roceVersion) ADD_ERROR("cfg.nic.roceVersion"); - if (nic.serviceLevel != cfg.nic.serviceLevel) ADD_ERROR("cfg.nic.serviceLevel"); - if (nic.trafficClass != cfg.nic.trafficClass) ADD_ERROR("cfg.nic.trafficClass"); + if (nic.roceVersion != cfg.nic.roceVersion) ADD_ERROR("cfg.nic.roceVersion"); + if (nic.fifoTrafficClass != cfg.nic.fifoTrafficClass) ADD_ERROR("cfg.nic.fifoTrafficClass"); + if (nic.serviceLevel != cfg.nic.serviceLevel) ADD_ERROR("cfg.nic.serviceLevel"); + if (nic.trafficClass != cfg.nic.trafficClass) ADD_ERROR("cfg.nic.trafficClass"); if (nic.useRelaxedOrder != cfg.nic.useRelaxedOrder) ADD_ERROR("cfg.nic.useRelaxedOrder"); if (nic.useNuma != cfg.nic.useNuma) ADD_ERROR("cfg.nic.useNuma"); } @@ -2749,6 +2751,12 @@ namespace { ibv_gid dstGid; ///< GID handle for DST NIC vector srcQueuePairs; ///< Queue pairs for SRC NIC vector dstQueuePairs; ///< Queue pairs for DST NIC + ibv_cq* srcCtrlCompQueue; ///< Completion queue for SRC ctrl QPs (FIFO TC) + ibv_cq* dstCtrlCompQueue; ///< Completion queue for DST ctrl QPs (FIFO TC) + vector srcCtrlQueuePairs; ///< Control QPs on SRC NIC (FIFO TC) + vector dstCtrlQueuePairs; ///< Control QPs on DST NIC (FIFO TC) + ibv_send_wr ctrlSendWr; ///< Send WR for ctrl signal (zero-byte inline, reused per iteration) + ibv_recv_wr ctrlRecvWr; ///< Recv WR for ctrl signal (reused per iteration) ibv_mr* srcMemRegion; ///< Memory region for SRC ibv_mr* dstMemRegion; ///< Memory region for DST int srcDmabufFd; ///< DMA-BUF file descriptor for SRC (if using dmabuf) @@ -3554,6 +3562,28 @@ static bool IsConfiguredGid(union ibv_gid const& gid) rss.sendWorkRequests.resize(rss.qpCount); } + // Create control (FIFO) QPs when fifoTrafficClass differs from trafficClass + if (cfg.nic.fifoTrafficClass != cfg.nic.trafficClass) { + if (GetRank() == srcMemRank) { + IBV_PTR_CALL(rss.srcCtrlCompQueue, ibv_create_cq, + rss.srcContext, cfg.nic.queueSize, NULL, NULL, 0); + rss.srcCtrlQueuePairs.resize(rss.qpCount); + for (int i = 0; i < rss.qpCount; i++) { + ERR_CHECK(CreateQueuePair(cfg, rss.srcProtect, rss.srcCtrlCompQueue, rss.srcCtrlQueuePairs[i])); + ERR_CHECK(InitQueuePair(rss.srcCtrlQueuePairs[i], port, rdmaAccessFlags)); + } + } + if (GetRank() == dstMemRank) { + IBV_PTR_CALL(rss.dstCtrlCompQueue, ibv_create_cq, + rss.dstContext, cfg.nic.queueSize, NULL, NULL, 0); + rss.dstCtrlQueuePairs.resize(rss.qpCount); + for (int i = 0; i < rss.qpCount; i++) { + ERR_CHECK(CreateQueuePair(cfg, rss.dstProtect, rss.dstCtrlCompQueue, rss.dstCtrlQueuePairs[i])); + ERR_CHECK(InitQueuePair(rss.dstCtrlQueuePairs[i], port, rdmaAccessFlags)); + } + } + } + // Broadcast SRC/DST port link_layer so that all ranks know it so that they can be compared System::Get().Broadcast(srcMemRank, sizeof(rss.srcPortAttr.link_layer), &rss.srcPortAttr.link_layer); System::Get().Broadcast(dstMemRank, sizeof(rss.dstPortAttr.link_layer), &rss.dstPortAttr.link_layer); @@ -3675,6 +3705,70 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } } } + + // Exchange ctrl QP numbers and connect them with fifoTrafficClass + if (cfg.nic.fifoTrafficClass != cfg.nic.trafficClass) { + ConnInfo srcCtrlInfo = {}, dstCtrlInfo = {}; + for (int i = 0; i < rss.qpCount; i++) { + if (GetRank() == srcMemRank) { + srcCtrlInfo.lid = rss.srcPortAttr.lid; + srcCtrlInfo.gid = rss.srcGid; + srcCtrlInfo.gidIdx = srcGidIndex; + srcCtrlInfo.qpn = rss.srcCtrlQueuePairs[i]->qp_num; + srcCtrlInfo.rkey = 0; + srcCtrlInfo.vaddr = 0; + } + System::Get().Broadcast(srcMemRank, sizeof(srcCtrlInfo), &srcCtrlInfo); + + if (GetRank() == dstMemRank) { + dstCtrlInfo.lid = rss.dstPortAttr.lid; + dstCtrlInfo.gid = rss.dstGid; + dstCtrlInfo.gidIdx = dstGidIndex; + dstCtrlInfo.qpn = rss.dstCtrlQueuePairs[i]->qp_num; + dstCtrlInfo.rkey = 0; + dstCtrlInfo.vaddr = 0; + } + System::Get().Broadcast(dstMemRank, sizeof(dstCtrlInfo), &dstCtrlInfo); + + QpTransitionResult srcCtrlResult = {ERR_NONE, false}; + if (GetRank() == srcMemRank) { + ErrResult err = TransitionQpToRtr(rss.srcCtrlQueuePairs[i], dstCtrlInfo, port, srcIsRoCE, + rss.srcPortAttr.active_mtu, cfg.nic.fifoTrafficClass, cfg.nic.serviceLevel); + srcCtrlResult.rtrFailed = (err.errType != ERR_NONE); + if (err.errType == ERR_NONE) err = TransitionQpToRts(rss.srcCtrlQueuePairs[i]); + srcCtrlResult.errType = err.errType; + } + System::Get().Broadcast(srcMemRank, sizeof(srcCtrlResult), &srcCtrlResult); + if (srcCtrlResult.errType != ERR_NONE) + return {ERR_FATAL, "SRC rank %d failed to transition ctrl QP %d to %s", + srcMemRank, i, srcCtrlResult.rtrFailed ? "RTR" : "RTS"}; + + QpTransitionResult dstCtrlResult = {ERR_NONE, false}; + if (GetRank() == dstMemRank) { + ErrResult err = TransitionQpToRtr(rss.dstCtrlQueuePairs[i], srcCtrlInfo, port, dstIsRoCE, + rss.dstPortAttr.active_mtu, cfg.nic.fifoTrafficClass, cfg.nic.serviceLevel); + dstCtrlResult.rtrFailed = (err.errType != ERR_NONE); + if (err.errType == ERR_NONE) err = TransitionQpToRts(rss.dstCtrlQueuePairs[i]); + dstCtrlResult.errType = err.errType; + } + System::Get().Broadcast(dstMemRank, sizeof(dstCtrlResult), &dstCtrlResult); + if (dstCtrlResult.errType != ERR_NONE) + return {ERR_FATAL, "DST rank %d failed to transition ctrl QP %d to %s", + dstMemRank, i, dstCtrlResult.rtrFailed ? "RTR" : "RTS"}; + } + + // Pre-build reusable ctrl send/recv WRs (zero-byte inline, posted once per iteration) + rss.ctrlSendWr = {}; + rss.ctrlSendWr.sg_list = nullptr; + rss.ctrlSendWr.num_sge = 0; + rss.ctrlSendWr.opcode = IBV_WR_SEND; + rss.ctrlSendWr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_INLINE; + + rss.ctrlRecvWr = {}; + rss.ctrlRecvWr.sg_list = nullptr; + rss.ctrlRecvWr.num_sge = 0; + } + return ERR_NONE; } @@ -3711,6 +3805,18 @@ static bool IsConfiguredGid(union ibv_gid const& gid) rss.dstQueuePairs.clear(); } + // Destroy ctrl queue pairs and completion queues (only exist when fifoTrafficClass != trafficClass) + if (isSrcRank && !rss.srcCtrlQueuePairs.empty()) { + for (auto qp : rss.srcCtrlQueuePairs) IBV_CALL(ibv_destroy_qp, qp); + rss.srcCtrlQueuePairs.clear(); + IBV_CALL(ibv_destroy_cq, rss.srcCtrlCompQueue); + } + if (isDstRank && !rss.dstCtrlQueuePairs.empty()) { + for (auto qp : rss.dstCtrlQueuePairs) IBV_CALL(ibv_destroy_qp, qp); + rss.dstCtrlQueuePairs.clear(); + IBV_CALL(ibv_destroy_cq, rss.dstCtrlCompQueue); + } + // Destroy completion queues if (isSrcRank) IBV_CALL(ibv_destroy_cq, rss.srcCompQueue); if (isDstRank) IBV_CALL(ibv_destroy_cq, rss.dstCompQueue); @@ -4653,7 +4759,56 @@ static bool IsConfiguredGid(union ibv_gid const& gid) int const exeIndex, TransferResources& rss) { - // Loop over each of the queue pairs and post work request + // Ping-pong control path: receiver arms recv, both sync, sender fires zero-byte signal + // on ctrl QPs (stamped with fifoTrafficClass), then data transfer proceeds as normal. + if (cfg.nic.fifoTrafficClass != cfg.nic.trafficClass) { + // [1] Receiver arms one recv WR per ctrl QP before the barrier + if (!rss.srcIsExeNic) { + ibv_recv_wr* badRecvWr; + for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { + ibv_recv_wr wr = rss.ctrlRecvWr; // copy: wr.next must be null + int err = ibv_post_recv(rss.dstCtrlQueuePairs[qpIdx], &wr, &badRecvWr); + if (err) + return {ERR_FATAL, "Transfer %d: ibv_post_recv on ctrl QP %d failed (%s)", + rss.transferIdx, qpIdx, strerror(err)}; + } + } + + // [2] Barrier — guarantees receiver has armed recv before sender fires + System::Get().Barrier(); + + // [3] Sender fires one zero-byte inline IBV_WR_SEND per ctrl QP + if (rss.srcIsExeNic) { + ibv_send_wr* badSendWr; + for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { + int err = ibv_post_send(rss.srcCtrlQueuePairs[qpIdx], &rss.ctrlSendWr, &badSendWr); + if (err) + return {ERR_FATAL, "Transfer %d: ibv_post_send on ctrl QP %d failed (%s)", + rss.transferIdx, qpIdx, strerror(err)}; + } + // [4] Sender polls ctrl CQ for send completions + for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { + ibv_wc wc; + while (ibv_poll_cq(rss.srcCtrlCompQueue, 1, &wc) == 0) {} + if (wc.status != IBV_WC_SUCCESS) + return {ERR_FATAL, "Transfer %d: ctrl send CQ error on QP %d [status %d]", + rss.transferIdx, qpIdx, wc.status}; + } + } + + // [5] Receiver polls ctrl CQ — confirms signal arrived + if (!rss.srcIsExeNic) { + for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { + ibv_wc wc; + while (ibv_poll_cq(rss.dstCtrlCompQueue, 1, &wc) == 0) {} + if (wc.status != IBV_WC_SUCCESS) + return {ERR_FATAL, "Transfer %d: ctrl recv CQ error on QP %d [status %d]", + rss.transferIdx, qpIdx, wc.status}; + } + } + } + + // [6] Data path — unchanged: post all RDMA send WRs (stamped with trafficClass) ibv_send_wr* badWorkReq; for (int qpIndex = 0; qpIndex < rss.qpCount; qpIndex++) { size_t numChunks = rss.sendWorkRequests[qpIndex].size(); From 797a28dc0628f2410f02bfc6cb173a51663aa0c3 Mon Sep 17 00:00:00 2001 From: paklui Date: Wed, 27 May 2026 20:03:48 -0700 Subject: [PATCH 02/12] fix QpTransitionResult scope for ctrl QP setup Hoisted QpTransitionResult struct definition above the data QP exchange loop so it is also visible to the ctrl QP exchange loop that follows it. Co-Authored-By: Claude Sonnet 4.6 --- src/header/TransferBench.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index e7850110..f5d408e0 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -3592,6 +3592,11 @@ static bool IsConfiguredGid(union ibv_gid const& gid) rss.srcNicIndex, srcMemRank, rss.dstNicIndex, dstMemRank, rss.srcPortAttr.link_layer, rss.dstPortAttr.link_layer}; } + // Shared result type for broadcasting QP transition success/failure across MPI ranks + struct QpTransitionResult { ErrType errType; bool rtrFailed; }; + static_assert(std::is_trivially_copyable::value, + "QpTransitionResult must be trivially copyable for MPI broadcast"); + ConnInfo dstConnInfo = {}; ConnInfo srcConnInfo = {}; for (int i = 0; i < rss.qpCount; i++) { @@ -3621,8 +3626,6 @@ static bool IsConfiguredGid(union ibv_gid const& gid) // Then move them to read-to-send (RTS) // Broadcast each rank's result so all ranks fail together rather than // hanging on the next iteration's Broadcast when qpCount > 1. - struct QpTransitionResult { ErrType errType; bool rtrFailed; }; - static_assert(std::is_trivially_copyable::value, "QpTransitionResult must be trivially copyable for MPI broadcast"); QpTransitionResult srcQpResult = {ERR_NONE, false}; if (GetRank() == srcMemRank) { ErrResult err = TransitionQpToRtr(rss.srcQueuePairs[i], dstConnInfo, port, srcIsRoCE, rss.srcPortAttr.active_mtu, cfg.nic.trafficClass, cfg.nic.serviceLevel); From 038e6f01d90a665de9fec5ca90acaf3a0c73adaa Mon Sep 17 00:00:00 2001 From: paklui Date: Thu, 28 May 2026 19:06:15 -0700 Subject: [PATCH 03/12] fix NIC_TRAFFIC_CLASS_FIFO gate condition and remove barrier deadlock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate was using fifoTrafficClass != trafficClass which activated the ctrl path unintentionally when only NIC_TRAFFIC_CLASS was set. Changed all three gate points to fifoTrafficClass != 0 so default NIC_TRAFFIC_CLASS_FIFO=0 is a true no-op. Removed MPI barrier inside ExecuteNicTransfer() — dst rank never calls that function so the barrier deadlocked. Pre-posting recv WRs during setup (where both ranks participate) avoids the need for any per-iteration synchronisation. Co-Authored-By: Claude Sonnet 4.6 --- src/header/TransferBench.hpp | 92 +++++++++++++++--------------------- 1 file changed, 39 insertions(+), 53 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index f5d408e0..d4bf4d67 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -2756,7 +2756,6 @@ namespace { vector srcCtrlQueuePairs; ///< Control QPs on SRC NIC (FIFO TC) vector dstCtrlQueuePairs; ///< Control QPs on DST NIC (FIFO TC) ibv_send_wr ctrlSendWr; ///< Send WR for ctrl signal (zero-byte inline, reused per iteration) - ibv_recv_wr ctrlRecvWr; ///< Recv WR for ctrl signal (reused per iteration) ibv_mr* srcMemRegion; ///< Memory region for SRC ibv_mr* dstMemRegion; ///< Memory region for DST int srcDmabufFd; ///< DMA-BUF file descriptor for SRC (if using dmabuf) @@ -3563,7 +3562,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } // Create control (FIFO) QPs when fifoTrafficClass differs from trafficClass - if (cfg.nic.fifoTrafficClass != cfg.nic.trafficClass) { + if (cfg.nic.fifoTrafficClass != 0) { if (GetRank() == srcMemRank) { IBV_PTR_CALL(rss.srcCtrlCompQueue, ibv_create_cq, rss.srcContext, cfg.nic.queueSize, NULL, NULL, 0); @@ -3710,7 +3709,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } // Exchange ctrl QP numbers and connect them with fifoTrafficClass - if (cfg.nic.fifoTrafficClass != cfg.nic.trafficClass) { + if (cfg.nic.fifoTrafficClass != 0) { ConnInfo srcCtrlInfo = {}, dstCtrlInfo = {}; for (int i = 0; i < rss.qpCount; i++) { if (GetRank() == srcMemRank) { @@ -3760,16 +3759,33 @@ static bool IsConfiguredGid(union ibv_gid const& gid) dstMemRank, i, dstCtrlResult.rtrFailed ? "RTR" : "RTS"}; } - // Pre-build reusable ctrl send/recv WRs (zero-byte inline, posted once per iteration) + // Pre-build reusable ctrl send WR (zero-byte inline IBV_WR_SEND, posted once per iteration) rss.ctrlSendWr = {}; rss.ctrlSendWr.sg_list = nullptr; rss.ctrlSendWr.num_sge = 0; rss.ctrlSendWr.opcode = IBV_WR_SEND; rss.ctrlSendWr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_INLINE; - rss.ctrlRecvWr = {}; - rss.ctrlRecvWr.sg_list = nullptr; - rss.ctrlRecvWr.num_sge = 0; + // DST rank pre-posts recv WRs for all expected iterations so no per-iteration + // coordination (barrier) is needed between executor and non-executor ranks. + // numWarmups uses negative iteration indices; total sends = numWarmups + numIterations. + int numPrepost = cfg.general.numWarmups + + std::max(1, std::abs(cfg.general.numIterations)) + 5; + if (GetRank() == dstMemRank) { + ibv_recv_wr ctrlRecvWr = {}; + ctrlRecvWr.sg_list = nullptr; + ctrlRecvWr.num_sge = 0; + ibv_recv_wr* badRecvWr; + for (int i = 0; i < rss.qpCount; i++) { + for (int n = 0; n < numPrepost; n++) { + ibv_recv_wr wr = ctrlRecvWr; + int err = ibv_post_recv(rss.dstCtrlQueuePairs[i], &wr, &badRecvWr); + if (err) + return {ERR_FATAL, "Transfer %d: ibv_post_recv pre-post on ctrl QP %d failed (%s)", + rss.transferIdx, i, strerror(err)}; + } + } + } } return ERR_NONE; @@ -4762,56 +4778,26 @@ static bool IsConfiguredGid(union ibv_gid const& gid) int const exeIndex, TransferResources& rss) { - // Ping-pong control path: receiver arms recv, both sync, sender fires zero-byte signal - // on ctrl QPs (stamped with fifoTrafficClass), then data transfer proceeds as normal. - if (cfg.nic.fifoTrafficClass != cfg.nic.trafficClass) { - // [1] Receiver arms one recv WR per ctrl QP before the barrier - if (!rss.srcIsExeNic) { - ibv_recv_wr* badRecvWr; - for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { - ibv_recv_wr wr = rss.ctrlRecvWr; // copy: wr.next must be null - int err = ibv_post_recv(rss.dstCtrlQueuePairs[qpIdx], &wr, &badRecvWr); - if (err) - return {ERR_FATAL, "Transfer %d: ibv_post_recv on ctrl QP %d failed (%s)", - rss.transferIdx, qpIdx, strerror(err)}; - } + // Ctrl path: executor (src) fires one zero-byte inline IBV_WR_SEND per ctrl QP. + // Recv WRs are pre-posted on the dst side during setup, so no barrier is needed. + if (cfg.nic.fifoTrafficClass != 0 && rss.srcIsExeNic) { + ibv_send_wr* badSendWr; + for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { + int err = ibv_post_send(rss.srcCtrlQueuePairs[qpIdx], &rss.ctrlSendWr, &badSendWr); + if (err) + return {ERR_FATAL, "Transfer %d: ibv_post_send on ctrl QP %d failed (%s)", + rss.transferIdx, qpIdx, strerror(err)}; } - - // [2] Barrier — guarantees receiver has armed recv before sender fires - System::Get().Barrier(); - - // [3] Sender fires one zero-byte inline IBV_WR_SEND per ctrl QP - if (rss.srcIsExeNic) { - ibv_send_wr* badSendWr; - for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { - int err = ibv_post_send(rss.srcCtrlQueuePairs[qpIdx], &rss.ctrlSendWr, &badSendWr); - if (err) - return {ERR_FATAL, "Transfer %d: ibv_post_send on ctrl QP %d failed (%s)", - rss.transferIdx, qpIdx, strerror(err)}; - } - // [4] Sender polls ctrl CQ for send completions - for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { - ibv_wc wc; - while (ibv_poll_cq(rss.srcCtrlCompQueue, 1, &wc) == 0) {} - if (wc.status != IBV_WC_SUCCESS) - return {ERR_FATAL, "Transfer %d: ctrl send CQ error on QP %d [status %d]", - rss.transferIdx, qpIdx, wc.status}; - } - } - - // [5] Receiver polls ctrl CQ — confirms signal arrived - if (!rss.srcIsExeNic) { - for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { - ibv_wc wc; - while (ibv_poll_cq(rss.dstCtrlCompQueue, 1, &wc) == 0) {} - if (wc.status != IBV_WC_SUCCESS) - return {ERR_FATAL, "Transfer %d: ctrl recv CQ error on QP %d [status %d]", - rss.transferIdx, qpIdx, wc.status}; - } + for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { + ibv_wc wc; + while (ibv_poll_cq(rss.srcCtrlCompQueue, 1, &wc) == 0) {} + if (wc.status != IBV_WC_SUCCESS) + return {ERR_FATAL, "Transfer %d: ctrl send CQ error on QP %d [status %d]", + rss.transferIdx, qpIdx, wc.status}; } } - // [6] Data path — unchanged: post all RDMA send WRs (stamped with trafficClass) + // Data path — unchanged: post all RDMA send WRs (stamped with trafficClass) ibv_send_wr* badWorkReq; for (int qpIndex = 0; qpIndex < rss.qpCount; qpIndex++) { size_t numChunks = rss.sendWorkRequests[qpIndex].size(); From 578b1cc22c07dfc031da9f840f23e70960c53f9b Mon Sep 17 00:00:00 2001 From: paklui Date: Thu, 28 May 2026 19:06:25 -0700 Subject: [PATCH 04/12] fix ctrl QP max_recv_wr capacity for NIC_TRAFFIC_CLASS_FIFO pre-posted recvs The ctrl QP on the DST rank pre-posts all recv WRs during setup to avoid per-iteration barriers. However the QP was created with max_recv_wr = cfg.nic.maxRecvWorkReq (default 16), which is smaller than ctrlNumPrepost (numWarmups + numIterations + 5 = 18 with defaults), causing ibv_post_recv to fail. Fix by computing ctrlNumPrepost before ctrl QP creation and passing it as max_recv_wr for the dst ctrl QPs. The optional maxRecvWr parameter is added to CreateQueuePair (-1 means use cfg value, preserving behaviour for all existing data QP callers). Co-Authored-By: Claude Sonnet 4.6 --- src/header/TransferBench.hpp | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index d4bf4d67..360650ae 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -3237,17 +3237,18 @@ static bool IsConfiguredGid(union ibv_gid const& gid) static ErrResult CreateQueuePair(ConfigOptions const& cfg, struct ibv_pd* pd, struct ibv_cq* cq, - struct ibv_qp*& qp) + struct ibv_qp*& qp, + int maxRecvWr = -1) { // Set queue pair attributes struct ibv_qp_init_attr attr = {}; - attr.qp_type = IBV_QPT_RC; // Set type to reliable connection - attr.send_cq = cq; // Send completion queue - attr.recv_cq = cq; // Recv completion queue - attr.cap.max_send_wr = cfg.nic.maxSendWorkReq; // Max send work requests - attr.cap.max_recv_wr = cfg.nic.maxRecvWorkReq; // Max recv work requests - attr.cap.max_send_sge = 1; // Max send scatter-gather entries - attr.cap.max_recv_sge = 1; // Max recv scatter-gather entries + attr.qp_type = IBV_QPT_RC; // Set type to reliable connection + attr.send_cq = cq; // Send completion queue + attr.recv_cq = cq; // Recv completion queue + attr.cap.max_send_wr = cfg.nic.maxSendWorkReq; // Max send work requests + attr.cap.max_recv_wr = (maxRecvWr >= 0) ? maxRecvWr : cfg.nic.maxRecvWorkReq; // Max recv work requests + attr.cap.max_send_sge = 1; // Max send scatter-gather entries + attr.cap.max_recv_sge = 1; // Max recv scatter-gather entries qp = ibv_create_qp(pd, &attr); if (qp == NULL) @@ -3561,13 +3562,17 @@ static bool IsConfiguredGid(union ibv_gid const& gid) rss.sendWorkRequests.resize(rss.qpCount); } - // Create control (FIFO) QPs when fifoTrafficClass differs from trafficClass + // Create control (FIFO) QPs when fifoTrafficClass is non-zero. + // Compute numPrepost here so dst ctrl QPs are created with enough max_recv_wr capacity. + int ctrlNumPrepost = cfg.general.numWarmups + + std::max(1, std::abs(cfg.general.numIterations)) + 5; if (cfg.nic.fifoTrafficClass != 0) { if (GetRank() == srcMemRank) { IBV_PTR_CALL(rss.srcCtrlCompQueue, ibv_create_cq, rss.srcContext, cfg.nic.queueSize, NULL, NULL, 0); rss.srcCtrlQueuePairs.resize(rss.qpCount); for (int i = 0; i < rss.qpCount; i++) { + // SRC ctrl QP only posts sends, so use default max_recv_wr ERR_CHECK(CreateQueuePair(cfg, rss.srcProtect, rss.srcCtrlCompQueue, rss.srcCtrlQueuePairs[i])); ERR_CHECK(InitQueuePair(rss.srcCtrlQueuePairs[i], port, rdmaAccessFlags)); } @@ -3577,7 +3582,9 @@ static bool IsConfiguredGid(union ibv_gid const& gid) rss.dstContext, cfg.nic.queueSize, NULL, NULL, 0); rss.dstCtrlQueuePairs.resize(rss.qpCount); for (int i = 0; i < rss.qpCount; i++) { - ERR_CHECK(CreateQueuePair(cfg, rss.dstProtect, rss.dstCtrlCompQueue, rss.dstCtrlQueuePairs[i])); + // DST ctrl QP pre-posts ctrlNumPrepost recv WRs; ensure max_recv_wr is large enough + ERR_CHECK(CreateQueuePair(cfg, rss.dstProtect, rss.dstCtrlCompQueue, rss.dstCtrlQueuePairs[i], + ctrlNumPrepost)); ERR_CHECK(InitQueuePair(rss.dstCtrlQueuePairs[i], port, rdmaAccessFlags)); } } @@ -3768,16 +3775,14 @@ static bool IsConfiguredGid(union ibv_gid const& gid) // DST rank pre-posts recv WRs for all expected iterations so no per-iteration // coordination (barrier) is needed between executor and non-executor ranks. - // numWarmups uses negative iteration indices; total sends = numWarmups + numIterations. - int numPrepost = cfg.general.numWarmups + - std::max(1, std::abs(cfg.general.numIterations)) + 5; + // ctrlNumPrepost was computed above and matches the max_recv_wr used when creating ctrl QPs. if (GetRank() == dstMemRank) { ibv_recv_wr ctrlRecvWr = {}; ctrlRecvWr.sg_list = nullptr; ctrlRecvWr.num_sge = 0; ibv_recv_wr* badRecvWr; for (int i = 0; i < rss.qpCount; i++) { - for (int n = 0; n < numPrepost; n++) { + for (int n = 0; n < ctrlNumPrepost; n++) { ibv_recv_wr wr = ctrlRecvWr; int err = ibv_post_recv(rss.dstCtrlQueuePairs[i], &wr, &badRecvWr); if (err) From e4600ec5e01f383a7b5501571e974195de3e6707 Mon Sep 17 00:00:00 2001 From: Pak Nin Lui Date: Fri, 29 May 2026 09:51:50 -0700 Subject: [PATCH 05/12] fix ctrlNumPrepost to account for numSubIterations ExecuteNicTransfer() is called once per sub-iteration inside the loop in RunNicExecutor() The pre-posted recv WR count should be: (numWarmups + numIterations) * numSubIterations Previously the formula omitted the numSubIterations multiplier, which would cause ibv_post_recv failures when NUM_SUBITERS > 1. --- src/header/TransferBench.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 360650ae..c7f99c82 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -3564,8 +3564,10 @@ static bool IsConfiguredGid(union ibv_gid const& gid) // Create control (FIFO) QPs when fifoTrafficClass is non-zero. // Compute numPrepost here so dst ctrl QPs are created with enough max_recv_wr capacity. - int ctrlNumPrepost = cfg.general.numWarmups + - std::max(1, std::abs(cfg.general.numIterations)) + 5; + int ctrlNumPrepost = (cfg.general.numWarmups + + std::max(1, std::abs(cfg.general.numIterations))) * + std::max(1, cfg.general.numSubIterations); + if (cfg.nic.fifoTrafficClass != 0) { if (GetRank() == srcMemRank) { IBV_PTR_CALL(rss.srcCtrlCompQueue, ibv_create_cq, From c449d7072fcf7852d000834218b4a66bc5b5efd9 Mon Sep 17 00:00:00 2001 From: Pak Nin Lui Date: Fri, 29 May 2026 11:30:30 -0700 Subject: [PATCH 06/12] modify CHANGELOG.md for NIC_TRAFFIC_CLASS_FIFO --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84847a37..a0eab25d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,9 @@ Documentation for TransferBench is available at ## v1.67.00 ### Added -- Added NIC_TRAFFIC_CLASS to set the DSCP/traffic class byte in the RoCE GRH for QPs (RoCE only) -- Added NIC_SERVICE_LEVEL to set the IB service level (sl) for QPs (IB and RoCE) +- Added NIC_TRAFFIC_CLASS to set the DSCP/traffic class byte in the RoCE GRH for QPs (equivalent to NCCL_IB_TC) +- Added NIC_TRAFFIC_CLASS_FIFO to set a DSCP/traffic class to steer the control traffic into another priority queue (equivalent NCCL_IB_FIFO_TC) +- Added NIC_SERVICE_LEVEL to set the IB service level (sl) for QPs (equivalent to NCCL_IB_SL) - Initial support for pod communication. Requires compatible hardware / ROCm version and subject to further testing - This potentially enables GFX/DMA executors to access SRC/DST memory locations on GPUs within the same pod - Pod membership requires amd-smi however can be skipped by setting TB_FORCE_SINGLE_POD=1 From 4f7dc790cab0cbbda2d6c8a3c2a964802271634b Mon Sep 17 00:00:00 2001 From: Pak Nin Lui Date: Fri, 29 May 2026 16:44:53 -0700 Subject: [PATCH 07/12] add a missing word in CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0eab25d..3d86af2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Documentation for TransferBench is available at ## v1.67.00 ### Added - Added NIC_TRAFFIC_CLASS to set the DSCP/traffic class byte in the RoCE GRH for QPs (equivalent to NCCL_IB_TC) -- Added NIC_TRAFFIC_CLASS_FIFO to set a DSCP/traffic class to steer the control traffic into another priority queue (equivalent NCCL_IB_FIFO_TC) +- Added NIC_TRAFFIC_CLASS_FIFO to set a DSCP/traffic class to steer the control traffic into another priority queue (equivalent to NCCL_IB_FIFO_TC) - Added NIC_SERVICE_LEVEL to set the IB service level (sl) for QPs (equivalent to NCCL_IB_SL) - Initial support for pod communication. Requires compatible hardware / ROCm version and subject to further testing - This potentially enables GFX/DMA executors to access SRC/DST memory locations on GPUs within the same pod From 28b2754f51f3a7011c52864865b0d8351dfb8603 Mon Sep 17 00:00:00 2001 From: paklui Date: Fri, 29 May 2026 16:53:20 -0700 Subject: [PATCH 08/12] minor fix for a teardown comment --- src/header/TransferBench.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index c7f99c82..6c49366a 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -3831,7 +3831,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) rss.dstQueuePairs.clear(); } - // Destroy ctrl queue pairs and completion queues (only exist when fifoTrafficClass != trafficClass) + // Destroy ctrl queue pairs and completion queues (only exist when fifoTrafficClass != 0) if (isSrcRank && !rss.srcCtrlQueuePairs.empty()) { for (auto qp : rss.srcCtrlQueuePairs) IBV_CALL(ibv_destroy_qp, qp); rss.srcCtrlQueuePairs.clear(); From 470d880944d585cbc1aeff66de9a5ce05f2cda79 Mon Sep 17 00:00:00 2001 From: paklui Date: Fri, 29 May 2026 17:59:07 -0700 Subject: [PATCH 09/12] fix ctrl CQ poll error handling and out-of-order completion reporting Set wr_id = qpIdx before each ibv_post_send so completions can be identified regardless of arrival order Handle ibv_poll_cq returning < 0 (CQ error) to avoid an infinite spin Use wc.wr_id instead of qpIdx in the error message for identifying QP --- src/header/TransferBench.hpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 6c49366a..c20fd927 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -4790,6 +4790,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) if (cfg.nic.fifoTrafficClass != 0 && rss.srcIsExeNic) { ibv_send_wr* badSendWr; for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { + rss.ctrlSendWr.wr_id = qpIdx; int err = ibv_post_send(rss.srcCtrlQueuePairs[qpIdx], &rss.ctrlSendWr, &badSendWr); if (err) return {ERR_FATAL, "Transfer %d: ibv_post_send on ctrl QP %d failed (%s)", @@ -4797,10 +4798,13 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } for (int qpIdx = 0; qpIdx < rss.qpCount; qpIdx++) { ibv_wc wc; - while (ibv_poll_cq(rss.srcCtrlCompQueue, 1, &wc) == 0) {} + int nc; + while ((nc = ibv_poll_cq(rss.srcCtrlCompQueue, 1, &wc)) == 0) {} + if (nc < 0) + return {ERR_FATAL, "Transfer %d: ctrl CQ poll error", rss.transferIdx}; if (wc.status != IBV_WC_SUCCESS) - return {ERR_FATAL, "Transfer %d: ctrl send CQ error on QP %d [status %d]", - rss.transferIdx, qpIdx, wc.status}; + return {ERR_FATAL, "Transfer %d: ctrl send CQ error on QP %llu [status %d]", + rss.transferIdx, wc.wr_id, wc.status}; } } From 17ecb8acb94333da296d69fbf96edea00caecaf0 Mon Sep 17 00:00:00 2001 From: paklui Date: Fri, 29 May 2026 18:08:45 -0700 Subject: [PATCH 10/12] fix ctrl CQ sizes for src and dst ranks src ctrl CQ needs qpCount slots (drained each ExecuteNicTransfer call) dst ctrl CQ needs ctrlNumPrepost * qpCount slots since it is never polled and completions accumulate over the full run --- src/header/TransferBench.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index c20fd927..93f29fd9 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -3571,7 +3571,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) if (cfg.nic.fifoTrafficClass != 0) { if (GetRank() == srcMemRank) { IBV_PTR_CALL(rss.srcCtrlCompQueue, ibv_create_cq, - rss.srcContext, cfg.nic.queueSize, NULL, NULL, 0); + rss.srcContext, rss.qpCount, NULL, NULL, 0); rss.srcCtrlQueuePairs.resize(rss.qpCount); for (int i = 0; i < rss.qpCount; i++) { // SRC ctrl QP only posts sends, so use default max_recv_wr @@ -3581,7 +3581,7 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } if (GetRank() == dstMemRank) { IBV_PTR_CALL(rss.dstCtrlCompQueue, ibv_create_cq, - rss.dstContext, cfg.nic.queueSize, NULL, NULL, 0); + rss.dstContext, ctrlNumPrepost * rss.qpCount, NULL, NULL, 0); rss.dstCtrlQueuePairs.resize(rss.qpCount); for (int i = 0; i < rss.qpCount; i++) { // DST ctrl QP pre-posts ctrlNumPrepost recv WRs; ensure max_recv_wr is large enough From 82648c12cd1d7c94d33b28b1bbff32de9e908051 Mon Sep 17 00:00:00 2001 From: paklui Date: Sat, 30 May 2026 16:48:34 -0700 Subject: [PATCH 11/12] prevent NIC_TRAFFIC_CLASS_FIFO to run in timed and infinite iteration modes ctrlNumPrepost assumes a fixed iteration count. When NUM_ITERATIONS <= 0, such as in timed or infinite mode, when the actual send count is unbounded, it would exhaust the pre-posted recv WRs, which can cause RNR retries or hangs. To workaroundm, exit with an error instead if NIC_TRAFFIC_CLASS_FIFO is set in these modes. --- src/client/EnvVars.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/EnvVars.hpp b/src/client/EnvVars.hpp index 42eb1e6d..4319e3f1 100644 --- a/src/client/EnvVars.hpp +++ b/src/client/EnvVars.hpp @@ -197,6 +197,10 @@ class EnvVars printf("[ERROR] NIC_TRAFFIC_CLASS_FIFO must be in range 0..255 (got %d)\n", nicFifoTrafficClass); exit(1); } + if (nicFifoTrafficClass != 0 && numIterations <= 0) { + printf("[ERROR] NIC_TRAFFIC_CLASS_FIFO requires NUM_ITERATIONS > 0 (timed/infinite mode is not supported)\n"); + exit(1); + } if (nicServiceLevel < 0 || nicServiceLevel > 15) { printf("[ERROR] NIC_SERVICE_LEVEL must be in range 0..15 (got %d)\n", nicServiceLevel); exit(1); From 4d3406821ebf55ed0f9723fc1679827d0406b3a7 Mon Sep 17 00:00:00 2001 From: paklui Date: Mon, 1 Jun 2026 19:47:23 -0700 Subject: [PATCH 12/12] fix ctrl QP setup to skip RDMA_READ transfers and guard against overflow - Gate ctrl QP creation and exchange/connect blocks on rss.srcIsExeNic; RDMA_READ transfers never use ctrl QPs so creating them wastes resources and can fail near device QP/CQ limits - Compute ctrlNumPrepost in int64_t to avoid signed overflow for large iteration counts; drop std::abs (NIC_TRAFFIC_CLASS_FIFO already rejects numIterations<=0); clamp to INT_MAX before casting back to int Co-Authored-By: Claude Sonnet 4.6 --- src/header/TransferBench.hpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/header/TransferBench.hpp b/src/header/TransferBench.hpp index 93f29fd9..3cb5e92c 100644 --- a/src/header/TransferBench.hpp +++ b/src/header/TransferBench.hpp @@ -3562,13 +3562,14 @@ static bool IsConfiguredGid(union ibv_gid const& gid) rss.sendWorkRequests.resize(rss.qpCount); } - // Create control (FIFO) QPs when fifoTrafficClass is non-zero. - // Compute numPrepost here so dst ctrl QPs are created with enough max_recv_wr capacity. - int ctrlNumPrepost = (cfg.general.numWarmups + - std::max(1, std::abs(cfg.general.numIterations))) * - std::max(1, cfg.general.numSubIterations); - - if (cfg.nic.fifoTrafficClass != 0) { + // Create control (FIFO) QPs when fifoTrafficClass is non-zero and this is an RDMA_WRITE + // transfer (srcIsExeNic == true). RDMA_READ transfers never use ctrl QPs so skip them. + // Compute ctrlNumPrepost in 64-bit to avoid signed overflow for large iteration counts, + // then clamp to INT_MAX before casting (NIC_TRAFFIC_CLASS_FIFO already rejects numIterations<=0). + int64_t ctrlNumPrepost64 = (int64_t)(cfg.general.numWarmups + cfg.general.numIterations) * + std::max(1, cfg.general.numSubIterations); + int ctrlNumPrepost = (int)std::min(ctrlNumPrepost64, (int64_t)INT_MAX); + if (cfg.nic.fifoTrafficClass != 0 && rss.srcIsExeNic) { if (GetRank() == srcMemRank) { IBV_PTR_CALL(rss.srcCtrlCompQueue, ibv_create_cq, rss.srcContext, rss.qpCount, NULL, NULL, 0); @@ -3717,8 +3718,9 @@ static bool IsConfiguredGid(union ibv_gid const& gid) } } - // Exchange ctrl QP numbers and connect them with fifoTrafficClass - if (cfg.nic.fifoTrafficClass != 0) { + // Exchange ctrl QP numbers and connect them with fifoTrafficClass. + // Guard on srcIsExeNic to match the ctrl QP creation block above. + if (cfg.nic.fifoTrafficClass != 0 && rss.srcIsExeNic) { ConnInfo srcCtrlInfo = {}, dstCtrlInfo = {}; for (int i = 0; i < rss.qpCount; i++) { if (GetRank() == srcMemRank) {