diff --git a/base/benchmarks/computation-BF16/iluvatar/BI150/case_config.yaml b/base/benchmarks/computation-BF16/iluvatar/BI150/case_config.yaml index b4d2b52cd..e4cf917ca 100644 --- a/base/benchmarks/computation-BF16/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/computation-BF16/iluvatar/BI150/case_config.yaml @@ -2,5 +2,5 @@ M: 6144 N: 6144 K: 6144 DIST_BACKEND: "nccl" -WARMUP: 100 -ITERS: 10000 +WARMUP: 1 +ITERS: 5 diff --git a/base/benchmarks/computation-BF16/main.py b/base/benchmarks/computation-BF16/main.py index 69810fdd4..a0cb2e6f0 100644 --- a/base/benchmarks/computation-BF16/main.py +++ b/base/benchmarks/computation-BF16/main.py @@ -74,9 +74,8 @@ def main(config, case_config, rank, world_size, local_rank): for _ in range(case_config.ITERS): _result = torch.mm(matrixA, matrixB) - if "iluvatar" not in config.vendor: - host_device_sync(config.vendor) - multi_device_sync(config.vendor) + host_device_sync(config.vendor) + multi_device_sync(config.vendor) end_time = time.perf_counter() exec_time = end_time - start_time @@ -107,6 +106,8 @@ def main(config, case_config, rank, world_size, local_rank): for output_rank in range(config.node_size): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s computation-BF16=".format(dist.get_rank()) + str(result) + "TFLOPS") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU computation-BF16=".format(dist.get_rank()) + str(result*2) + "TFLOPS") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/computation-FP16/iluvatar/BI150/case_config.yaml b/base/benchmarks/computation-FP16/iluvatar/BI150/case_config.yaml index b4d2b52cd..e4cf917ca 100644 --- a/base/benchmarks/computation-FP16/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/computation-FP16/iluvatar/BI150/case_config.yaml @@ -2,5 +2,5 @@ M: 6144 N: 6144 K: 6144 DIST_BACKEND: "nccl" -WARMUP: 100 -ITERS: 10000 +WARMUP: 1 +ITERS: 5 diff --git a/base/benchmarks/computation-FP16/main.py b/base/benchmarks/computation-FP16/main.py index 25926e431..2fd922ee7 100644 --- a/base/benchmarks/computation-FP16/main.py +++ b/base/benchmarks/computation-FP16/main.py @@ -74,9 +74,8 @@ def main(config, case_config, rank, world_size, local_rank): for _ in range(case_config.ITERS): _result = torch.mm(matrixA, matrixB) - if "iluvatar" not in config.vendor: - host_device_sync(config.vendor) - multi_device_sync(config.vendor) + host_device_sync(config.vendor) + multi_device_sync(config.vendor) end_time = time.perf_counter() exec_time = end_time - start_time @@ -107,6 +106,8 @@ def main(config, case_config, rank, world_size, local_rank): for output_rank in range(config.node_size): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s computation-FP16=".format(dist.get_rank()) + str(result) + "TFLOPS") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU computation-FP16=".format(dist.get_rank()) + str(result*2) + "TFLOPS") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/computation-FP32/iluvatar/BI150/case_config.yaml b/base/benchmarks/computation-FP32/iluvatar/BI150/case_config.yaml index b4d2b52cd..e4cf917ca 100644 --- a/base/benchmarks/computation-FP32/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/computation-FP32/iluvatar/BI150/case_config.yaml @@ -2,5 +2,5 @@ M: 6144 N: 6144 K: 6144 DIST_BACKEND: "nccl" -WARMUP: 100 -ITERS: 10000 +WARMUP: 1 +ITERS: 5 diff --git a/base/benchmarks/computation-FP32/main.py b/base/benchmarks/computation-FP32/main.py index 8cff03c32..24b9d2e52 100644 --- a/base/benchmarks/computation-FP32/main.py +++ b/base/benchmarks/computation-FP32/main.py @@ -74,9 +74,9 @@ def main(config, case_config, rank, world_size, local_rank): for _ in range(case_config.ITERS): _result = torch.mm(matrixA, matrixB) - if "iluvatar" not in config.vendor: - host_device_sync(config.vendor) - multi_device_sync(config.vendor) + + host_device_sync(config.vendor) + multi_device_sync(config.vendor) end_time = time.perf_counter() exec_time = end_time - start_time @@ -107,6 +107,8 @@ def main(config, case_config, rank, world_size, local_rank): for output_rank in range(config.node_size): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s computation-FP32=".format(dist.get_rank()) + str(result) + "TFLOPS") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU computation-FP32=".format(dist.get_rank()) + str(result*2) + "TFLOPS") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml index 85c16433c..af24519bf 100644 --- a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 DIST_BACKEND: "nccl" -WARMUP: 100 -ITERS: 1000 +WARMUP: 1 +ITERS: 5 diff --git a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh index 6d17376c5..eb1278178 100644 --- a/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh @@ -1,4 +1,5 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH -export NCCL_IB_HCA=mlx5_2 +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-MPI_interserver/main.py b/base/benchmarks/interconnect-MPI_interserver/main.py index 64c0c9115..d3debe671 100644 --- a/base/benchmarks/interconnect-MPI_interserver/main.py +++ b/base/benchmarks/interconnect-MPI_interserver/main.py @@ -121,6 +121,9 @@ def main(config, case_config, rank, world_size, local_rank): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gb) + "GB/s") print(r"[FlagPerf Result]Rank {}'s interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gib) + "GiB/s") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gb*2) + "GB/s") + print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gib*2) + "GiB/s") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml index 7f3b3a76a..4a3459bcb 100644 --- a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 DIST_BACKEND: "nccl" -WARMUP: 100 -ITERS: 10000 +WARMUP: 10 +ITERS: 100 diff --git a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh index df2fb083d..eb1278178 100644 --- a/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh @@ -1,3 +1,5 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml index 51f29d4a0..7cc5f78eb 100644 --- a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 100 -ITERS: 1000 +WARMUP: 1 +ITERS: 5 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh index df2fb083d..eb1278178 100644 --- a/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh @@ -1,3 +1,5 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-P2P_interserver/main.py b/base/benchmarks/interconnect-P2P_interserver/main.py index b1ff6b4ab..74bd3d351 100644 --- a/base/benchmarks/interconnect-P2P_interserver/main.py +++ b/base/benchmarks/interconnect-P2P_interserver/main.py @@ -45,9 +45,8 @@ def main(config, case_config, rank, world_size, local_rank): if "iluvatar" in config.vendor: torch.cuda.set_device(local_rank) - rank_dst = 16 - else: - rank_dst = 8 + + rank_dst = 8 Melements = case_config.Melements torchsize = (Melements, 1024, 1024) @@ -114,6 +113,9 @@ def main(config, case_config, rank, world_size, local_rank): if dist.get_rank() % config.node_size == 0: print(r"[FlagPerf Result]Rank {}'s inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gb) + "GB/s") print(r"[FlagPerf Result]Rank {}'s inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gib) + "GiB/s") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gb*2) + "GB/s") + print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gib*2) + "GiB/s") dist.destroy_process_group() diff --git a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml index 2a0e5e2d8..7cc5f78eb 100644 --- a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 100 -ITERS: 100000 +WARMUP: 1 +ITERS: 5 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh index df2fb083d..eb1278178 100644 --- a/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh @@ -1,3 +1,5 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export NCCL_SHM_DISABLE=1 diff --git a/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml b/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml index e29ba62a6..7cc5f78eb 100644 --- a/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 100 -ITERS: 10000 +WARMUP: 1 +ITERS: 5 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh b/base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh index df2fb083d..9eba4b60d 100644 --- a/base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh +++ b/base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh @@ -1,3 +1,4 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH +export CUDA_VISIBLE_DEVICES=0,2,4,6,8,10,12,14 diff --git a/base/benchmarks/interconnect-h2d/main.py b/base/benchmarks/interconnect-h2d/main.py index 2095db8a2..1c84805f4 100644 --- a/base/benchmarks/interconnect-h2d/main.py +++ b/base/benchmarks/interconnect-h2d/main.py @@ -52,6 +52,8 @@ def main(config, case_config, rank, world_size, local_rank): if "mthreads" in config.vendor: tensor = torch.rand(torchsize, dtype=torch.float32).pin_memory() + elif "iluvatar" in config.vendor: + tensor = torch.rand(torchsize, dtype=torch.float32, pin_memory=True) else: tensor = torch.rand(torchsize, dtype=torch.float32) #print(f"Memory address of tensor in rank {rank} and local rank {local_rank}: {tensor.data_ptr()}") diff --git a/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml b/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml index e29ba62a6..f5a3fd58a 100644 --- a/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml +++ b/base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml @@ -1,4 +1,4 @@ Melements: 1024 -WARMUP: 100 -ITERS: 10000 +WARMUP: 1 +ITERS: 1 DIST_BACKEND: "nccl" diff --git a/base/benchmarks/main_memory-bandwidth/main.py b/base/benchmarks/main_memory-bandwidth/main.py index bccaa4b24..f9388ea3b 100644 --- a/base/benchmarks/main_memory-bandwidth/main.py +++ b/base/benchmarks/main_memory-bandwidth/main.py @@ -108,6 +108,9 @@ def main(config, case_config, rank, world_size, local_rank): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s main_memory-bindwidth=".format(dist.get_rank()) + str(gb) + "GB/s") print(r"[FlagPerf Result]Rank {}'s main_memory-bindwidth=".format(dist.get_rank()) + str(gib) + "GiB/s") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-bindwidth=".format(dist.get_rank()) + str(gb*2) + "GB/s") + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-bindwidth=".format(dist.get_rank()) + str(gib*2) + "GiB/s") multi_device_sync(config.vendor) dist.destroy_process_group() diff --git a/base/benchmarks/main_memory-capacity/iluvatar/BI150/env.sh b/base/benchmarks/main_memory-capacity/iluvatar/BI150/env.sh index df2fb083d..906a77a4e 100644 --- a/base/benchmarks/main_memory-capacity/iluvatar/BI150/env.sh +++ b/base/benchmarks/main_memory-capacity/iluvatar/BI150/env.sh @@ -1,3 +1,3 @@ export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH -export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH \ No newline at end of file +export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH diff --git a/base/benchmarks/main_memory-capacity/main.py b/base/benchmarks/main_memory-capacity/main.py index cfa012452..75076fd08 100644 --- a/base/benchmarks/main_memory-capacity/main.py +++ b/base/benchmarks/main_memory-capacity/main.py @@ -76,7 +76,7 @@ def main(config, case_config, rank, world_size, local_rank): if local_rank == 0: print("Test Finished") - + return total_allocated @@ -104,6 +104,9 @@ def main(config, case_config, rank, world_size, local_rank): if local_rank == output_rank: print(r"[FlagPerf Result]Rank {}'s main_memory-capacity=".format(dist.get_rank()) + str(gb) + "GB") print(r"[FlagPerf Result]Rank {}'s main_memory-capacity=".format(dist.get_rank()) + str(gib) + "GiB") + if "iluvatar" in config.vendor: + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-capacity=".format(dist.get_rank()) + str(gb*2) + "GB") + print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-capacity=".format(dist.get_rank()) + str(gib*2) + "GiB") if "iluvatar" not in config.vendor: multi_device_sync(config.vendor)