Skip to content

[iluvatar] add iluvatar BI150 output info in base test #813

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tianxiao-baai merged 11 commits into from
Jun 17, 2025
Merged

[iluvatar] add iluvatar BI150 output info in base test #813

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
df5ae61
[iluvatar] base/vendor: update iluvatar BI150 pytorch dockerfile for …
May 13, 2025
d1e885b
Merge remote-tracking branch 'upstream/main' into main
Jun 4, 2025
8b3372d
[iluvatar] update base/interconnect-MPI_interserver env.sh
Jun 4, 2025
20dc699
Merge remote-tracking branch 'upstream/main' into main
Jun 5, 2025
cc7f93d
[iluvatar] update base envs.sh, reduce number of test loops
Jun 5, 2025
ce916e1
Merge remote-tracking branch 'upstream/main' into main
Jun 11, 2025
2582308
[iluvatar] update base test env.sh
Jun 11, 2025
466c554
Merge remote-tracking branch 'upstream/main' into main
Jun 13, 2025
e35431b
[iluvatar] update computation-FP32/FP16/BF16 code for iluvatar BI150
Jun 13, 2025
1d6cdc2
[iluvatar] fix interconnect-MPI_interserver/interconnect-P2P_interser…
Jun 13, 2025
2b92c84
[iluvatar] add iluvatar BI150 output info in base test
DannyP0 Jun 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions base/benchmarks/computation-BF16/iluvatar/BI150/case_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ M: 6144
N: 6144
K: 6144
DIST_BACKEND: "nccl"
WARMUP: 100
ITERS: 10000
WARMUP: 1
ITERS: 5
7 changes: 4 additions & 3 deletions base/benchmarks/computation-BF16/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,8 @@ def main(config, case_config, rank, world_size, local_rank):
for _ in range(case_config.ITERS):
_result = torch.mm(matrixA, matrixB)

if "iluvatar" not in config.vendor:
host_device_sync(config.vendor)
multi_device_sync(config.vendor)
host_device_sync(config.vendor)
multi_device_sync(config.vendor)
end_time = time.perf_counter()

exec_time = end_time - start_time
Expand Down Expand Up @@ -107,6 +106,8 @@ def main(config, case_config, rank, world_size, local_rank):
for output_rank in range(config.node_size):
if local_rank == output_rank:
print(r"[FlagPerf Result]Rank {}'s computation-BF16=".format(dist.get_rank()) + str(result) + "TFLOPS")
if "iluvatar" in config.vendor:
print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU computation-BF16=".format(dist.get_rank()) + str(result*2) + "TFLOPS")
multi_device_sync(config.vendor)

dist.destroy_process_group()
Expand Down
4 changes: 2 additions & 2 deletions base/benchmarks/computation-FP16/iluvatar/BI150/case_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ M: 6144
N: 6144
K: 6144
DIST_BACKEND: "nccl"
WARMUP: 100
ITERS: 10000
WARMUP: 1
ITERS: 5
7 changes: 4 additions & 3 deletions base/benchmarks/computation-FP16/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,8 @@ def main(config, case_config, rank, world_size, local_rank):
for _ in range(case_config.ITERS):
_result = torch.mm(matrixA, matrixB)

if "iluvatar" not in config.vendor:
host_device_sync(config.vendor)
multi_device_sync(config.vendor)
host_device_sync(config.vendor)
multi_device_sync(config.vendor)
end_time = time.perf_counter()

exec_time = end_time - start_time
Expand Down Expand Up @@ -107,6 +106,8 @@ def main(config, case_config, rank, world_size, local_rank):
for output_rank in range(config.node_size):
if local_rank == output_rank:
print(r"[FlagPerf Result]Rank {}'s computation-FP16=".format(dist.get_rank()) + str(result) + "TFLOPS")
if "iluvatar" in config.vendor:
print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU computation-FP16=".format(dist.get_rank()) + str(result*2) + "TFLOPS")
multi_device_sync(config.vendor)

dist.destroy_process_group()
Expand Down
4 changes: 2 additions & 2 deletions base/benchmarks/computation-FP32/iluvatar/BI150/case_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ M: 6144
N: 6144
K: 6144
DIST_BACKEND: "nccl"
WARMUP: 100
ITERS: 10000
WARMUP: 1
ITERS: 5
8 changes: 5 additions & 3 deletions base/benchmarks/computation-FP32/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ def main(config, case_config, rank, world_size, local_rank):

for _ in range(case_config.ITERS):
_result = torch.mm(matrixA, matrixB)
if "iluvatar" not in config.vendor:
host_device_sync(config.vendor)
multi_device_sync(config.vendor)

host_device_sync(config.vendor)
multi_device_sync(config.vendor)
end_time = time.perf_counter()

exec_time = end_time - start_time
Expand Down Expand Up @@ -107,6 +107,8 @@ def main(config, case_config, rank, world_size, local_rank):
for output_rank in range(config.node_size):
if local_rank == output_rank:
print(r"[FlagPerf Result]Rank {}'s computation-FP32=".format(dist.get_rank()) + str(result) + "TFLOPS")
if "iluvatar" in config.vendor:
print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU computation-FP32=".format(dist.get_rank()) + str(result*2) + "TFLOPS")
multi_device_sync(config.vendor)

dist.destroy_process_group()
Expand Down
4 changes: 2 additions & 2 deletions base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/case_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Melements: 1024
DIST_BACKEND: "nccl"
WARMUP: 100
ITERS: 1000
WARMUP: 1
ITERS: 5
3 changes: 2 additions & 1 deletion base/benchmarks/interconnect-MPI_interserver/iluvatar/BI150/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH
export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export NCCL_IB_HCA=mlx5_2
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export NCCL_SHM_DISABLE=1
3 changes: 3 additions & 0 deletions base/benchmarks/interconnect-MPI_interserver/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ def main(config, case_config, rank, world_size, local_rank):
if local_rank == output_rank:
print(r"[FlagPerf Result]Rank {}'s interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gb) + "GB/s")
print(r"[FlagPerf Result]Rank {}'s interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gib) + "GiB/s")
if "iluvatar" in config.vendor:
print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gb*2) + "GB/s")
print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and interconnect-MPI_interserver-bandwidth=".format(dist.get_rank()) + str(gib*2) + "GiB/s")
multi_device_sync(config.vendor)

dist.destroy_process_group()
4 changes: 2 additions & 2 deletions base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/case_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Melements: 1024
DIST_BACKEND: "nccl"
WARMUP: 100
ITERS: 10000
WARMUP: 10
ITERS: 100
4 changes: 3 additions & 1 deletion base/benchmarks/interconnect-MPI_intraserver/iluvatar/BI150/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH
export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export NCCL_SHM_DISABLE=1
4 changes: 2 additions & 2 deletions base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/case_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Melements: 1024
WARMUP: 100
ITERS: 1000
WARMUP: 1
ITERS: 5
DIST_BACKEND: "nccl"
4 changes: 3 additions & 1 deletion base/benchmarks/interconnect-P2P_interserver/iluvatar/BI150/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH
export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export NCCL_SHM_DISABLE=1
8 changes: 5 additions & 3 deletions base/benchmarks/interconnect-P2P_interserver/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,8 @@ def main(config, case_config, rank, world_size, local_rank):

if "iluvatar" in config.vendor:
torch.cuda.set_device(local_rank)
rank_dst = 16
else:
rank_dst = 8

rank_dst = 8

Melements = case_config.Melements
torchsize = (Melements, 1024, 1024)
Expand Down Expand Up @@ -114,6 +113,9 @@ def main(config, case_config, rank, world_size, local_rank):
if dist.get_rank() % config.node_size == 0:
print(r"[FlagPerf Result]Rank {}'s inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gb) + "GB/s")
print(r"[FlagPerf Result]Rank {}'s inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gib) + "GiB/s")
if "iluvatar" in config.vendor:
print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gb*2) + "GB/s")
print(r"[FlagPerf Result]Rank {} BI-V150 output bidirectional bandwidth and inferconnect-P2P_intraserver-bandwidth=".format(dist.get_rank()) + str(gib*2) + "GiB/s")

dist.destroy_process_group()

Expand Down
4 changes: 2 additions & 2 deletions base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/case_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Melements: 1024
WARMUP: 100
ITERS: 100000
WARMUP: 1
ITERS: 5
DIST_BACKEND: "nccl"
4 changes: 3 additions & 1 deletion base/benchmarks/interconnect-P2P_intraserver/iluvatar/BI150/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH
export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
export NCCL_SHM_DISABLE=1
4 changes: 2 additions & 2 deletions base/benchmarks/interconnect-h2d/iluvatar/BI150/case_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Melements: 1024
WARMUP: 100
ITERS: 10000
WARMUP: 1
ITERS: 5
DIST_BACKEND: "nccl"
3 changes: 2 additions & 1 deletion base/benchmarks/interconnect-h2d/iluvatar/BI150/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH
export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export CUDA_VISIBLE_DEVICES=0,2,4,6,8,10,12,14
2 changes: 2 additions & 0 deletions base/benchmarks/interconnect-h2d/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ def main(config, case_config, rank, world_size, local_rank):

if "mthreads" in config.vendor:
tensor = torch.rand(torchsize, dtype=torch.float32).pin_memory()
elif "iluvatar" in config.vendor:
tensor = torch.rand(torchsize, dtype=torch.float32, pin_memory=True)
else:
tensor = torch.rand(torchsize, dtype=torch.float32)
#print(f"Memory address of tensor in rank {rank} and local rank {local_rank}: {tensor.data_ptr()}")
Expand Down
4 changes: 2 additions & 2 deletions base/benchmarks/main_memory-bandwidth/iluvatar/BI150/case_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Melements: 1024
WARMUP: 100
ITERS: 10000
WARMUP: 1
ITERS: 1
DIST_BACKEND: "nccl"
3 changes: 3 additions & 0 deletions base/benchmarks/main_memory-bandwidth/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,9 @@ def main(config, case_config, rank, world_size, local_rank):
if local_rank == output_rank:
print(r"[FlagPerf Result]Rank {}'s main_memory-bindwidth=".format(dist.get_rank()) + str(gb) + "GB/s")
print(r"[FlagPerf Result]Rank {}'s main_memory-bindwidth=".format(dist.get_rank()) + str(gib) + "GiB/s")
if "iluvatar" in config.vendor:
print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-bindwidth=".format(dist.get_rank()) + str(gb*2) + "GB/s")
print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-bindwidth=".format(dist.get_rank()) + str(gib*2) + "GiB/s")
multi_device_sync(config.vendor)

dist.destroy_process_group()
Expand Down
2 changes: 1 addition & 1 deletion base/benchmarks/main_memory-capacity/iluvatar/BI150/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages:$PYTHONPATH
export LD_LIBRARY_PATH=/usr/local/corex/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/corex/lib64/python3/dist-packages/bin:$PATH
5 changes: 4 additions & 1 deletion base/benchmarks/main_memory-capacity/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def main(config, case_config, rank, world_size, local_rank):

if local_rank == 0:
print("Test Finished")

return total_allocated


Expand Down Expand Up @@ -104,6 +104,9 @@ def main(config, case_config, rank, world_size, local_rank):
if local_rank == output_rank:
print(r"[FlagPerf Result]Rank {}'s main_memory-capacity=".format(dist.get_rank()) + str(gb) + "GB")
print(r"[FlagPerf Result]Rank {}'s main_memory-capacity=".format(dist.get_rank()) + str(gib) + "GiB")
if "iluvatar" in config.vendor:
print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-capacity=".format(dist.get_rank()) + str(gb*2) + "GB")
print(r"[FlagPerf Result]Rank {} BI-V150 has 2 chips and overall GPU main_memory-capacity=".format(dist.get_rank()) + str(gib*2) + "GiB")
if "iluvatar" not in config.vendor:
multi_device_sync(config.vendor)

Expand Down