From df5ae611f76d3c66a87c126ed3c25427c3aff3de Mon Sep 17 00:00:00 2001 From: Fanglai Peng Date: Tue, 13 May 2025 17:41:39 +0800 Subject: [PATCH 1/7] [iluvatar] base/vendor: update iluvatar BI150 pytorch dockerfile for base test --- base/vendors/iluvatar/pytorch_2.1/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/vendors/iluvatar/pytorch_2.1/Dockerfile b/base/vendors/iluvatar/pytorch_2.1/Dockerfile index f166ee8f8..46e3479e6 100644 --- a/base/vendors/iluvatar/pytorch_2.1/Dockerfile +++ b/base/vendors/iluvatar/pytorch_2.1/Dockerfile @@ -1,4 +1,4 @@ -FROM zibo.harbor.iluvatar.com.cn:30000/saas/bi150-4.1.0-x86-ubuntu20.04-py3.10-customer-flagperf-triton:v1.0 +FROM zibo.harbor.iluvatar.com.cn:30000/saas/mr-bi150-4.2.0-x86-ubuntu20.04-py3.10-customer-ds-671b:v1.3 #RUN /bin/bash -c "pip config set global.index-url https://mirror.baidu.com/pypi/simple" #RUN /bin/bash -c "uname -a" #RUN /bin/bash -c alias python3=python From 8b3372d16c33939b2ae4bdfc13af2288c1cacba5 Mon Sep 17 00:00:00 2001 From: Fanglai Peng Date: Wed, 4 Jun 2025 12:14:53 +0800 Subject: [PATCH 2/7] [iluvatar] update base/interconnect-MPI_interserver env.sh --- .../interconnect-MPI_interserver/iluvatar/BI150/env.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh index df2fb083d..6d17376c5 100644 --- a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh @@ -1,3 +1,4 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export NCCL_IB_HCA=mlx5_2 From cc7f93d67b9275737e9148dc4464c79deff1cfce Mon Sep 17 00:00:00 2001 From: Fanglai Peng Date: Thu, 5 Jun 2025 09:31:23 +0800 Subject: [PATCH 3/7] [iluvatar] update base envs.sh, reduce number of test loops --- .../iluvatar/BI150/case_config.yaml | 2 +- .../iluvatar/BI150/case_config.yaml | 4 ++-- .../iluvatar/BI150/case_config.yaml | 2 +- .../iluvatar/BI150/case_config.yaml | 4 ++-- .../interconnect-h2d/iluvatar/BI150/case_config.yaml | 4 ++-- .../main_memory-bandwidth/iluvatar/BI150/case_config.yaml | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml index 85c16433c..c860d7bc3 100644 --- a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 DIST_BACKEND: "nccl" -WARMUP: 100 +WARMUP: 10 ITERS: 1000 diff --git a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml index 7f3b3a76a..c860d7bc3 100644 --- a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 DIST_BACKEND: "nccl" -WARMUP: 100 -ITERS: 10000 +WARMUP: 10 +ITERS: 1000 diff --git a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml index 51f29d4a0..79a51f63e 100644 --- a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 100 +WARMUP: 10 ITERS: 1000 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml index 2a0e5e2d8..79a51f63e 100644 --- a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 100 -ITERS: 100000 +WARMUP: 10 +ITERS: 1000 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml index e29ba62a6..79a51f63e 100644 --- a/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 100 -ITERS: 10000 +WARMUP: 10 +ITERS: 1000 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml b/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml index e29ba62a6..79a51f63e 100644 --- a/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 100 -ITERS: 10000 +WARMUP: 10 +ITERS: 1000 DIST_BACKEND: "nccl" From 2582308e6c1673c1f8ede05acaf5d0e1f34875c2 Mon Sep 17 00:00:00 2001 From: Fanglai Peng Date: Wed, 11 Jun 2025 17:59:04 +0800 Subject: [PATCH 4/7] [iluvatar] update base test env.sh --- .../iluvatar/BI150/case_config.yaml | 2 +- .../interconnect-MPI_interserver/iluvatar/BI150/env.sh | 2 +- .../iluvatar/BI150/case_config.yaml | 2 +- .../interconnect-MPI_intraserver/iluvatar/BI150/env.sh | 4 +++- .../iluvatar/BI150/case_config.yaml | 2 +- .../interconnect-P2P_interserver/iluvatar/BI150/env.sh | 3 ++- .../iluvatar/BI150/case_config.yaml | 2 +- .../interconnect-P2P_intraserver/iluvatar/BI150/env.sh | 3 ++- base/benchmarks/interconnect-h2d/main.py | 2 ++ 9 files changed, 14 insertions(+), 8 deletions(-) diff --git a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml index c860d7bc3..4a3459bcb 100644 --- a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 DIST_BACKEND: "nccl" WARMUP: 10 -ITERS: 1000 +ITERS: 100 diff --git a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh index 6d17376c5..423b2f69b 100644 --- a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh @@ -1,4 +1,4 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH -export NCCL_IB_HCA=mlx5_2 +export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml index c860d7bc3..4a3459bcb 100644 --- a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 DIST_BACKEND: "nccl" WARMUP: 10 -ITERS: 1000 +ITERS: 100 diff --git a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh index df2fb083d..34c4a9e71 100644 --- a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh @@ -1,3 +1,5 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH + +export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml index 79a51f63e..87a1d1435 100644 --- a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 WARMUP: 10 -ITERS: 1000 +ITERS: 100 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh index df2fb083d..423b2f69b 100644 --- a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh @@ -1,3 +1,4 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml index 79a51f63e..87a1d1435 100644 --- a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 WARMUP: 10 -ITERS: 1000 +ITERS: 100 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh index df2fb083d..423b2f69b 100644 --- a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh @@ -1,3 +1,4 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-h2d/main.py b/base/benchmarks/interconnect-h2d/main.py index 2095db8a2..1c84805f4 100644 --- a/base/benchmarks/interconnect-h2d/main.py +++ b/base/benchmarks/interconnect-h2d/main.py @@ -52,6 +52,8 @@ def main(config, case_config, rank, world_size, local_rank): if "mthreads" in config.vendor: tensor = torch.rand(torchsize, dtype=torch.float32).pin_memory() + elif "iluvatar" in config.vendor: + tensor = torch.rand(torchsize, dtype=torch.float32, pin_memory=True) else: tensor = torch.rand(torchsize, dtype=torch.float32) #print(f"Memory address of tensor in rank {rank} and local rank {local_rank}: {tensor.data_ptr()}") From e35431b3d970ef84e39a9f2c86bf1616d1ba313b Mon Sep 17 00:00:00 2001 From: Fanglai Peng Date: Fri, 13 Jun 2025 12:29:26 +0800 Subject: [PATCH 5/7] [iluvatar] update computation-FP32/FP16/BF16 code for iluvatar BI150 [iluvatar] update main_memory-capacity/main_memory-bandwidth code for iluvatar BI150 modify unidirectional bandwidth to bidirectional bandwidth in interconnect-P2P_interserver and interconnect-MPI_interserver [iluvatar] update iluvatar base test envs.sh [iluvatar] reduce number of loops for iluvatar base test --- .../computation-BF16/iluvatar/BI150/case_config.yaml | 4 ++-- base/benchmarks/computation-BF16/main.py | 7 +++---- .../computation-FP16/iluvatar/BI150/case_config.yaml | 4 ++-- base/benchmarks/computation-FP16/main.py | 7 +++---- .../computation-FP32/iluvatar/BI150/case_config.yaml | 4 ++-- base/benchmarks/computation-FP32/main.py | 8 ++++---- .../iluvatar/BI150/case_config.yaml | 4 ++-- .../interconnect-MPI_interserver/iluvatar/BI150/env.sh | 1 + base/benchmarks/interconnect-MPI_interserver/main.py | 1 + .../interconnect-MPI_intraserver/iluvatar/BI150/env.sh | 2 +- .../iluvatar/BI150/case_config.yaml | 4 ++-- .../interconnect-P2P_interserver/iluvatar/BI150/env.sh | 1 + base/benchmarks/interconnect-P2P_interserver/main.py | 6 +++--- .../iluvatar/BI150/case_config.yaml | 4 ++-- .../interconnect-P2P_intraserver/iluvatar/BI150/env.sh | 1 + .../interconnect-h2d/iluvatar/BI150/case_config.yaml | 4 ++-- base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh | 3 ++- .../main_memory-bandwidth/iluvatar/BI150/case_config.yaml | 4 ++-- base/benchmarks/main_memory-bandwidth/main.py | 2 +- .../benchmarks/main_memory-capacity/iluvatar/BI150/env.sh | 2 +- base/benchmarks/main_memory-capacity/main.py | 5 ++++- 21 files changed, 42 insertions(+), 36 deletions(-) diff --git a/base/benchmarks/computation-BF16/iluvatar/BI150/case_config.yaml b/base/benchmarks/computation-BF16/iluvatar/BI150/case_config.yaml index b4d2b52cd..e4cf917ca 100644 --- a/base/benchmarks/computation-BF16/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/computation-BF16/iluvatar/BI150/case_config.yaml @@ -2,5 +2,5 @@ M: 6144 N: 6144 K: 6144 DIST_BACKEND: "nccl" -WARMUP: 100 -ITERS: 10000 +WARMUP: 1 +ITERS: 5 diff --git a/base/benchmarks/computation-BF16/main.py b/base/benchmarks/computation-BF16/main.py index 69810fdd4..0ab7dfc50 100644 --- a/base/benchmarks/computation-BF16/main.py +++ b/base/benchmarks/computation-BF16/main.py @@ -74,14 +74,13 @@ def main(config, case_config, rank, world_size, local_rank): for _ in range(case_config.ITERS): _result = torch.mm(matrixA, matrixB) - if "iluvatar" not in config.vendor: - host_device_sync(config.vendor) - multi_device_sync(config.vendor) + host_device_sync(config.vendor) + multi_device_sync(config.vendor) end_time = time.perf_counter() exec_time = end_time - start_time - operations = case_config.ITERS * 2 * m * n * k + operations = case_config.ITERS * 2 * m * n * k if "iluvatar" not in config.vendor else case_config.ITERS * 4 * m * n * k tflops = operations / exec_time / 1e12 return round(tflops, 2) diff --git a/base/benchmarks/computation-FP16/iluvatar/BI150/case_config.yaml b/base/benchmarks/computation-FP16/iluvatar/BI150/case_config.yaml index b4d2b52cd..e4cf917ca 100644 --- a/base/benchmarks/computation-FP16/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/computation-FP16/iluvatar/BI150/case_config.yaml @@ -2,5 +2,5 @@ M: 6144 N: 6144 K: 6144 DIST_BACKEND: "nccl" -WARMUP: 100 -ITERS: 10000 +WARMUP: 1 +ITERS: 5 diff --git a/base/benchmarks/computation-FP16/main.py b/base/benchmarks/computation-FP16/main.py index 25926e431..c3c57b9f7 100644 --- a/base/benchmarks/computation-FP16/main.py +++ b/base/benchmarks/computation-FP16/main.py @@ -74,14 +74,13 @@ def main(config, case_config, rank, world_size, local_rank): for _ in range(case_config.ITERS): _result = torch.mm(matrixA, matrixB) - if "iluvatar" not in config.vendor: - host_device_sync(config.vendor) - multi_device_sync(config.vendor) + host_device_sync(config.vendor) + multi_device_sync(config.vendor) end_time = time.perf_counter() exec_time = end_time - start_time - operations = case_config.ITERS * 2 * m * n * k + operations = case_config.ITERS * 2 * m * n * k if "iluvatar" not in config.vendor else case_config.ITERS * 4 * m * n * k tflops = operations / exec_time / 1e12 return round(tflops, 2) diff --git a/base/benchmarks/computation-FP32/iluvatar/BI150/case_config.yaml b/base/benchmarks/computation-FP32/iluvatar/BI150/case_config.yaml index b4d2b52cd..e4cf917ca 100644 --- a/base/benchmarks/computation-FP32/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/computation-FP32/iluvatar/BI150/case_config.yaml @@ -2,5 +2,5 @@ M: 6144 N: 6144 K: 6144 DIST_BACKEND: "nccl" -WARMUP: 100 -ITERS: 10000 +WARMUP: 1 +ITERS: 5 diff --git a/base/benchmarks/computation-FP32/main.py b/base/benchmarks/computation-FP32/main.py index 8cff03c32..877fcac46 100644 --- a/base/benchmarks/computation-FP32/main.py +++ b/base/benchmarks/computation-FP32/main.py @@ -74,14 +74,14 @@ def main(config, case_config, rank, world_size, local_rank): for _ in range(case_config.ITERS): _result = torch.mm(matrixA, matrixB) - if "iluvatar" not in config.vendor: - host_device_sync(config.vendor) - multi_device_sync(config.vendor) + + host_device_sync(config.vendor) + multi_device_sync(config.vendor) end_time = time.perf_counter() exec_time = end_time - start_time - operations = case_config.ITERS * 2 * m * n * k + operations = case_config.ITERS * 2 * m * n * k if "iluvatar" not in config.vendor else case_config.ITERS * 4 * m * n * k tflops = operations / exec_time / 1e12 return round(tflops, 2) diff --git a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml index 4a3459bcb..af24519bf 100644 --- a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 DIST_BACKEND: "nccl" -WARMUP: 10 -ITERS: 100 +WARMUP: 1 +ITERS: 5 diff --git a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh index 423b2f69b..eb1278178 100644 --- a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh @@ -1,4 +1,5 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-MPI_interserver/main.py b/base/benchmarks/interconnect-MPI_interserver/main.py index 64c0c9115..66a29923c 100644 --- a/base/benchmarks/interconnect-MPI_interserver/main.py +++ b/base/benchmarks/interconnect-MPI_interserver/main.py @@ -95,6 +95,7 @@ def main(config, case_config, rank, world_size, local_rank): datasize = case_config.ITERS * (Melements * 1024 * 1024 * 4 / 1E9) algbw = datasize / elapsed_time bandwidth = algbw * (2 * (config.node_size - 1) / config.node_size) + bandwidth *= 2 bandwidth_gib = bandwidth * 1E9 / (1024**3) return round(bandwidth, 2), round(bandwidth_gib, 2) diff --git a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh index 34c4a9e71..eb1278178 100644 --- a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh @@ -1,5 +1,5 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH - +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml index 87a1d1435..7cc5f78eb 100644 --- a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 10 -ITERS: 100 +WARMUP: 1 +ITERS: 5 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh index 423b2f69b..eb1278178 100644 --- a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh @@ -1,4 +1,5 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-P2P_interserver/main.py b/base/benchmarks/interconnect-P2P_interserver/main.py index b1ff6b4ab..b70dd6e28 100644 --- a/base/benchmarks/interconnect-P2P_interserver/main.py +++ b/base/benchmarks/interconnect-P2P_interserver/main.py @@ -45,9 +45,8 @@ def main(config, case_config, rank, world_size, local_rank): if "iluvatar" in config.vendor: torch.cuda.set_device(local_rank) - rank_dst = 16 - else: - rank_dst = 8 + + rank_dst = 8 Melements = case_config.Melements torchsize = (Melements, 1024, 1024) @@ -85,6 +84,7 @@ def main(config, case_config, rank, world_size, local_rank): datasize = case_config.ITERS * (Melements * 1024 * 1024 * 4 / 1E9) bandwidth = datasize / elapsed_time + bandwidth *= 2 bandwidth_gib = bandwidth * 1E9 / (1024**3) return round(bandwidth, 2), round(bandwidth_gib, 2) diff --git a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml index 87a1d1435..7cc5f78eb 100644 --- a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 10 -ITERS: 100 +WARMUP: 1 +ITERS: 5 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh index 423b2f69b..eb1278178 100644 --- a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh @@ -1,4 +1,5 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml index 79a51f63e..7cc5f78eb 100644 --- a/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 10 -ITERS: 1000 +WARMUP: 1 +ITERS: 5 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh index df2fb083d..9eba4b60d 100644 --- a/base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh @@ -1,3 +1,4 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export CUDA_VISIBLE_DEVICES=0,2,4,6,8,10,12,14 diff --git a/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml b/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml index 79a51f63e..f5a3fd58a 100644 --- a/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 10 -ITERS: 1000 +WARMUP: 1 +ITERS: 1 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/main_memory-bandwidth/main.py b/base/benchmarks/main_memory-bandwidth/main.py index bccaa4b24..04e5897ba 100644 --- a/base/benchmarks/main_memory-bandwidth/main.py +++ b/base/benchmarks/main_memory-bandwidth/main.py @@ -80,7 +80,7 @@ def main(config, case_config, rank, world_size, local_rank): elapsed_time = end_time - start_time - datasize = case_config.ITERS * 2 * (Melements * 1024 * 1024 * 4 / 1E9) + datasize = case_config.ITERS * 2 * (Melements * 1024 * 1024 * 4 / 1E9) if "iluvatar" not in config.vendor else case_config.ITERS * 2 * (Melements * 1024 * 1024 * 8 / 1E9) bandwidth = datasize / elapsed_time bandwidth_gib = bandwidth * 1E9 / (1024**3) diff --git a/base/benchmarks/main_memory-capacity/iluvatar/BI150/env.sh b/base/benchmarks/main_memory-capacity/iluvatar/BI150/env.sh index df2fb083d..906a77a4e 100644 --- a/base/benchmarks/main_memory-capacity/iluvatar/BI150/env.sh +++ b/base/benchmarks/main_memory-capacity/iluvatar/BI150/env.sh @@ -1,3 +1,3 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH diff --git a/base/benchmarks/main_memory-capacity/main.py b/base/benchmarks/main_memory-capacity/main.py index cfa012452..bd2bc0220 100644 --- a/base/benchmarks/main_memory-capacity/main.py +++ b/base/benchmarks/main_memory-capacity/main.py @@ -76,7 +76,10 @@ def main(config, case_config, rank, world_size, local_rank): if local_rank == 0: print("Test Finished") - + + if "iluvatar" in config.vendor: + total_allocated *= 2 + return total_allocated From 1d6cdc2f4deafd072b736ad0da41d41db7b118ff Mon Sep 17 00:00:00 2001 From: Fanglai Peng Date: Fri, 13 Jun 2025 16:15:04 +0800 Subject: [PATCH 6/7] [iluvatar] fix interconnect-MPI_interserver/interconnect-P2P_interserver bandwidth calculation for iluvatar --- base/benchmarks/interconnect-MPI_interserver/main.py | 3 ++- base/benchmarks/interconnect-P2P_interserver/main.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/base/benchmarks/interconnect-MPI_interserver/main.py b/base/benchmarks/interconnect-MPI_interserver/main.py index 66a29923c..f4ba3e368 100644 --- a/base/benchmarks/interconnect-MPI_interserver/main.py +++ b/base/benchmarks/interconnect-MPI_interserver/main.py @@ -95,7 +95,8 @@ def main(config, case_config, rank, world_size, local_rank): datasize = case_config.ITERS * (Melements * 1024 * 1024 * 4 / 1E9) algbw = datasize / elapsed_time bandwidth = algbw * (2 * (config.node_size - 1) / config.node_size) - bandwidth *= 2 + if "iluvatar" in config.vendor: + bandwidth *= 2 # output bidirectional bandwidth for iluvatar test bandwidth_gib = bandwidth * 1E9 / (1024**3) return round(bandwidth, 2), round(bandwidth_gib, 2) diff --git a/base/benchmarks/interconnect-P2P_interserver/main.py b/base/benchmarks/interconnect-P2P_interserver/main.py index b70dd6e28..66c506665 100644 --- a/base/benchmarks/interconnect-P2P_interserver/main.py +++ b/base/benchmarks/interconnect-P2P_interserver/main.py @@ -84,7 +84,8 @@ def main(config, case_config, rank, world_size, local_rank): datasize = case_config.ITERS * (Melements * 1024 * 1024 * 4 / 1E9) bandwidth = datasize / elapsed_time - bandwidth *= 2 + if "iluvatar" in config.vendor: + bandwidth *= 2 # output bidirectional bandwidth for iluvatar test bandwidth_gib = bandwidth * 1E9 / (1024**3) return round(bandwidth, 2), round(bandwidth_gib, 2) From 2b92c8441acfd3e69a482845ddcc826c20858c7d Mon Sep 17 00:00:00 2001 From: fanglaipeng Date: Mon, 16 Jun 2025 22:07:19 +0800 Subject: [PATCH 7/7] [iluvatar] add iluvatar BI150 output info in base test --- base/benchmarks/computation-BF16/main.py | 4 +++- base/benchmarks/computation-FP16/main.py | 4 +++- base/benchmarks/computation-FP32/main.py | 4 +++- base/benchmarks/interconnect-MPI_interserver/main.py | 5 +++-- base/benchmarks/interconnect-P2P_interserver/main.py | 5 +++-- base/benchmarks/main_memory-bandwidth/main.py | 5 ++++- base/benchmarks/main_memory-capacity/main.py | 6 +++--- 7 files changed, 22 insertions(+), 11 deletions(-) diff --git a/base/benchmarks/computation-BF16/main.py b/base/benchmarks/computation-BF16/main.py index 0ab7dfc50..a0cb2e6f0 100644 --- a/base/benchmarks/computation-BF16/main.py +++ b/base/benchmarks/computation-BF16/main.py @@ -80,7 +80,7 @@ def main(config, case_config, rank, world_size, local_rank): exec_time = end_time - start_time - operations = case_config.ITERS * 2 * m * n * k if "iluvatar" not in config.vendor else case_config.ITERS * 4 * m * n * k + operations = case_config.ITERS * 2 * m * n * k tflops = operations / exec_time / 1e12 return round(tflops, 2) @@ -106,6 +106,8 @@ def main(config, case_config, rank, world_size, local_rank): for output_rank in range(config.node_size): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s computation-BF16=".format(dist.get_rank()) + str(result) + "TFLOPS") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU computation-BF16=".format(dist.get_rank()) + str(result*2) + "TFLOPS") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/computation-FP16/main.py b/base/benchmarks/computation-FP16/main.py index c3c57b9f7..2fd922ee7 100644 --- a/base/benchmarks/computation-FP16/main.py +++ b/base/benchmarks/computation-FP16/main.py @@ -80,7 +80,7 @@ def main(config, case_config, rank, world_size, local_rank): exec_time = end_time - start_time - operations = case_config.ITERS * 2 * m * n * k if "iluvatar" not in config.vendor else case_config.ITERS * 4 * m * n * k + operations = case_config.ITERS * 2 * m * n * k tflops = operations / exec_time / 1e12 return round(tflops, 2) @@ -106,6 +106,8 @@ def main(config, case_config, rank, world_size, local_rank): for output_rank in range(config.node_size): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s computation-FP16=".format(dist.get_rank()) + str(result) + "TFLOPS") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU computation-FP16=".format(dist.get_rank()) + str(result*2) + "TFLOPS") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/computation-FP32/main.py b/base/benchmarks/computation-FP32/main.py index 877fcac46..24b9d2e52 100644 --- a/base/benchmarks/computation-FP32/main.py +++ b/base/benchmarks/computation-FP32/main.py @@ -81,7 +81,7 @@ def main(config, case_config, rank, world_size, local_rank): exec_time = end_time - start_time - operations = case_config.ITERS * 2 * m * n * k if "iluvatar" not in config.vendor else case_config.ITERS * 4 * m * n * k + operations = case_config.ITERS * 2 * m * n * k tflops = operations / exec_time / 1e12 return round(tflops, 2) @@ -107,6 +107,8 @@ def main(config, case_config, rank, world_size, local_rank): for output_rank in range(config.node_size): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s computation-FP32=".format(dist.get_rank()) + str(result) + "TFLOPS") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU computation-FP32=".format(dist.get_rank()) + str(result*2) + "TFLOPS") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/interconnect-MPI_interserver/main.py b/base/benchmarks/interconnect-MPI_interserver/main.py index f4ba3e368..d3debe671 100644 --- a/base/benchmarks/interconnect-MPI_interserver/main.py +++ b/base/benchmarks/interconnect-MPI_interserver/main.py @@ -95,8 +95,6 @@ def main(config, case_config, rank, world_size, local_rank): datasize = case_config.ITERS * (Melements * 1024 * 1024 * 4 / 1E9) algbw = datasize / elapsed_time bandwidth = algbw * (2 * (config.node_size - 1) / config.node_size) - if "iluvatar" in config.vendor: - bandwidth *= 2 # output bidirectional bandwidth for iluvatar test bandwidth_gib = bandwidth * 1E9 / (1024**3) return round(bandwidth, 2), round(bandwidth_gib, 2) @@ -123,6 +121,9 @@ def main(config, case_config, rank, world_size, local_rank): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gb) + "GB/s") print(r"[FlagPerf Result]Rank {}'s interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gib) + "GiB/s") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gb*2) + "GB/s") + print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gib*2) + "GiB/s") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/interconnect-P2P_interserver/main.py b/base/benchmarks/interconnect-P2P_interserver/main.py index 66c506665..74bd3d351 100644 --- a/base/benchmarks/interconnect-P2P_interserver/main.py +++ b/base/benchmarks/interconnect-P2P_interserver/main.py @@ -84,8 +84,6 @@ def main(config, case_config, rank, world_size, local_rank): datasize = case_config.ITERS * (Melements * 1024 * 1024 * 4 / 1E9) bandwidth = datasize / elapsed_time - if "iluvatar" in config.vendor: - bandwidth *= 2 # output bidirectional bandwidth for iluvatar test bandwidth_gib = bandwidth * 1E9 / (1024**3) return round(bandwidth, 2), round(bandwidth_gib, 2) @@ -115,6 +113,9 @@ def main(config, case_config, rank, world_size, local_rank): if dist.get_rank() % config.node_size == 0: print(r"[FlagPerf Result]Rank {}'s inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gb) + "GB/s") print(r"[FlagPerf Result]Rank {}'s inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gib) + "GiB/s") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gb*2) + "GB/s") + print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gib*2) + "GiB/s") dist.destroy_process_group() diff --git a/base/benchmarks/main_memory-bandwidth/main.py b/base/benchmarks/main_memory-bandwidth/main.py index 04e5897ba..f9388ea3b 100644 --- a/base/benchmarks/main_memory-bandwidth/main.py +++ b/base/benchmarks/main_memory-bandwidth/main.py @@ -80,7 +80,7 @@ def main(config, case_config, rank, world_size, local_rank): elapsed_time = end_time - start_time - datasize = case_config.ITERS * 2 * (Melements * 1024 * 1024 * 4 / 1E9) if "iluvatar" not in config.vendor else case_config.ITERS * 2 * (Melements * 1024 * 1024 * 8 / 1E9) + datasize = case_config.ITERS * 2 * (Melements * 1024 * 1024 * 4 / 1E9) bandwidth = datasize / elapsed_time bandwidth_gib = bandwidth * 1E9 / (1024**3) @@ -108,6 +108,9 @@ def main(config, case_config, rank, world_size, local_rank): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s main_memory-bindwidth=".format(dist.get_rank()) + str(gb) + "GB/s") print(r"[FlagPerf Result]Rank {}'s main_memory-bindwidth=".format(dist.get_rank()) + str(gib) + "GiB/s") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-bindwidth=".format(dist.get_rank()) + str(gb*2) + "GB/s") + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-bindwidth=".format(dist.get_rank()) + str(gib*2) + "GiB/s") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/main_memory-capacity/main.py b/base/benchmarks/main_memory-capacity/main.py index bd2bc0220..75076fd08 100644 --- a/base/benchmarks/main_memory-capacity/main.py +++ b/base/benchmarks/main_memory-capacity/main.py @@ -77,9 +77,6 @@ def main(config, case_config, rank, world_size, local_rank): if local_rank == 0: print("Test Finished") - if "iluvatar" in config.vendor: - total_allocated *= 2 - return total_allocated @@ -107,6 +104,9 @@ def main(config, case_config, rank, world_size, local_rank): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s main_memory-capacity=".format(dist.get_rank()) + str(gb) + "GB") print(r"[FlagPerf Result]Rank {}'s main_memory-capacity=".format(dist.get_rank()) + str(gib) + "GiB") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-capacity=".format(dist.get_rank()) + str(gb*2) + "GB") + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-capacity=".format(dist.get_rank()) + str(gib*2) + "GiB") if "iluvatar" not in config.vendor: multi_device_sync(config.vendor)