Skip to content

feat: Add AMD accelerator detection and tracking in libfabric topology #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mpashkovskii wants to merge 2 commits into from
Closed

feat: Add AMD accelerator detection and tracking in libfabric topology #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
334207c
feat: Add AMD accelerator detection and tracking in libfabric topology
mpashkovskii Feb 26, 2026
92de8c9
fix: Update AMD accelerator device ID list for MI300 and MI355 series
mpashkovskii Feb 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 79 additions & 14 deletions src/utils/libfabric/libfabric_topology.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
nixlLibfabricTopology::nixlLibfabricTopology()
: num_aws_accel(0),
num_nvidia_accel(0),
num_amd_accel(0),
num_numa_nodes(0),
num_devices(0),
topology_discovered(false),
Expand Down Expand Up @@ -86,8 +87,8 @@ nixlLibfabricTopology::discoverTopology() {
NIXL_ERROR << "Failed to discover hwloc topology";
return status;
}
// Build nVidia accelerator to EFA mapping based on PCIe topology
if (num_nvidia_accel > 0) {
// Build accelerator to EFA mapping based on PCIe topology
if (num_nvidia_accel > 0 || num_amd_accel > 0) {
status = buildAccelToEfaMapping();
if (status != NIXL_SUCCESS) {
NIXL_ERROR << "Failed to build accelerator to EFA mapping";
Expand Down Expand Up @@ -189,7 +190,9 @@ void
nixlLibfabricTopology::printTopologyInfo() const {
NIXL_TRACE << "=== Libfabric Topology Information ===";
NIXL_TRACE << "Topology discovered: " << (topology_discovered ? "Yes" : "No");
NIXL_TRACE << "Number of AWS accelerators: " << num_aws_accel;
NIXL_TRACE << "Number of NVIDIA accelerators: " << num_nvidia_accel;
NIXL_TRACE << "Number of AMD accelerators: " << num_amd_accel;
NIXL_TRACE << "Number of AWS Neuron accelerators: " << num_aws_accel;
NIXL_TRACE << "Number of NUMA nodes: " << num_numa_nodes;
NIXL_TRACE << "Number of EFA devices: " << num_devices;
NIXL_TRACE << "EFA devices: ";
Expand All @@ -215,7 +218,9 @@ std::string
nixlLibfabricTopology::getTopologyString() const {
std::stringstream ss;
ss << "Libfabric Topology: ";
ss << "AWS_Accelerators=" << num_aws_accel << ", ";
ss << "NVIDIA_GPUs=" << num_nvidia_accel << ", ";
ss << "AMD_GPUs=" << num_amd_accel << ", ";
ss << "Neuron_Accelerators=" << num_aws_accel << ", ";
ss << "NUMA=" << num_numa_nodes << ", ";
ss << "EFA=" << num_devices << ", ";
ss << "Discovered=" << (topology_discovered ? "Yes" : "No");
Expand Down Expand Up @@ -328,29 +333,41 @@ nixl_status_t
nixlLibfabricTopology::discoverAccelWithHwloc() {
num_aws_accel = 0;
num_nvidia_accel = 0;
num_amd_accel = 0;
// Find all PCI devices and log detailed information
static const char *vendor_names[2] = {"NEURON", "NVIDIA"};
hwloc_obj_t pci_obj = nullptr;
while ((pci_obj = hwloc_get_next_pcidev(hwloc_topology, pci_obj)) != nullptr) {
const bool is_nvidia_accel = isNvidiaAccel(pci_obj);
if (is_nvidia_accel || isNeuronAccel(pci_obj)) {
const bool is_neuron_accel = isNeuronAccel(pci_obj);
const bool is_amd_accel = isAmdAccel(pci_obj);

if (is_nvidia_accel || is_neuron_accel || is_amd_accel) {
std::string pcie_addr = getPcieAddressFromHwlocObj(pci_obj);
// Get device and vendor info
uint16_t vendor_id = pci_obj->attr->pcidev.vendor_id;
uint16_t device_id = pci_obj->attr->pcidev.device_id;
uint16_t class_id = pci_obj->attr->pcidev.class_id;

NIXL_TRACE << "Found " << vendor_names[is_nvidia_accel] << " accelerator "
<< num_aws_accel << ": " << pcie_addr << " (vendor=" << std::hex << vendor_id
const char *vendor_name = is_nvidia_accel ? "NVIDIA" :
is_amd_accel ? "AMD" : "NEURON";

NIXL_TRACE << "Found " << vendor_name << " accelerator: " << pcie_addr
<< " (vendor=" << std::hex << vendor_id
<< ", device=" << device_id << ", class=" << class_id << std::dec << ")";

num_aws_accel++;
num_nvidia_accel += is_nvidia_accel;
if (is_nvidia_accel) {
num_nvidia_accel++;
} else if (is_amd_accel) {
num_amd_accel++;
} else {
num_aws_accel++;
}
}
}

NIXL_TRACE << "Discovered " << num_aws_accel << " "
<< vendor_names[num_aws_accel == num_nvidia_accel] << " devices via hwloc";
NIXL_TRACE << "Discovered " << num_nvidia_accel << " NVIDIA, "
<< num_amd_accel << " AMD, and "
<< num_aws_accel << " Neuron accelerators via hwloc";

// If we found more than 8 NVIDIA accelerators on P5en, investigate further
if (num_nvidia_accel > 8) {
Expand Down Expand Up @@ -514,10 +531,10 @@ nixlLibfabricTopology::buildTopologyAwareGrouping() {
NIXL_WARN << "Could not find hwloc object for PCIe address: " << pcie_addr;
}
}
// Step 2: Discover accelerators
// Step 2: Discover accelerators (NVIDIA and AMD)
hwloc_obj_t pci_obj = nullptr;
while ((pci_obj = hwloc_get_next_pcidev(hwloc_topology, pci_obj)) != nullptr) {
if (isNvidiaAccel(pci_obj)) {
if (isNvidiaAccel(pci_obj) || isAmdAccel(pci_obj)) {
AccelInfo accel;
accel.hwloc_node = pci_obj;
accel.domain_id = pci_obj->attr->pcidev.domain;
Expand Down Expand Up @@ -643,6 +660,54 @@ nixlLibfabricTopology::isNeuronAccel(hwloc_obj_t obj) const {
obj->attr->pcidev.device_id) != std::end(NEURON_DEVICE_IDS);
}

bool
nixlLibfabricTopology::isAmdAccel(hwloc_obj_t obj) const {
if (!obj || obj->type != HWLOC_OBJ_PCI_DEVICE) {
return false;
}
// AMD vendor ID is 0x1002
if (obj->attr->pcidev.vendor_id != 0x1002) {
return false;
}
// AMD Instinct MI300/MI355 series device IDs
// Sources:
// - https://github.com/GPUOpen-Tools/device_info/blob/master/DeviceInfo.cpp
// - https://github.com/openbsd/src/blob/master/sys/dev/pci/drm/amd/amdgpu/amdgpu_devlist.h
// - https://github.com/ROCm/k8s-device-plugin/issues/112
// - https://github.com/ROCm/ROCm/issues/5891
// Architecture: CDNA3 (gfx942) and CDNA4 (gfx950)
static const uint16_t AMD_GPU_DEVICE_IDS[] = {
// MI300 Series (CDNA3 - gfx942)
0x74a0, // MI300A APU
0x74a1, // MI300X dGPU (most common)
0x74a2, // MI308X
0x74a5, // MI325X
0x74a9, // MI300XHF
0x74b5, // MI300X VF (Virtual Function)

// MI355 Series (CDNA4 - gfx950)
0x75a0, // MI355X
0x75a1, // MI355X variant
0x75a3, // MI355X variant (Chip ID 30115)

// Note: MI455X (CDNA5) device IDs not yet available (H2 2026 release)
};

uint16_t device_id = obj->attr->pcidev.device_id;

// Check if it's in the known device ID list
if (std::find(std::begin(AMD_GPU_DEVICE_IDS),
std::end(AMD_GPU_DEVICE_IDS),
device_id) != std::end(AMD_GPU_DEVICE_IDS)) {
return true;
}

// Fallback: Check PCI class ID for display controller (GPU)
// Class 0x300-0x3ff for display controllers (similar to NVIDIA check)
uint16_t class_id = obj->attr->pcidev.class_id;
return (class_id >= 0x300 && class_id < 0x400);
}

bool
nixlLibfabricTopology::isEfaDevice(hwloc_obj_t obj) const {
if (!obj || obj->type != HWLOC_OBJ_PCI_DEVICE) {
Expand Down
16 changes: 15 additions & 1 deletion src/utils/libfabric/libfabric_topology.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class nixlLibfabricTopology {
// System information
int num_aws_accel; // AWS Trainium accelerators
int num_nvidia_accel; // NVIDIA GPU accelerators
int num_amd_accel; // AMD GPU accelerators (MI300x, MI355, MI455)
int num_numa_nodes;
int num_devices;

Expand Down Expand Up @@ -122,6 +123,8 @@ class nixlLibfabricTopology {
bool
isNeuronAccel(hwloc_obj_t obj) const;
bool
isAmdAccel(hwloc_obj_t obj) const;
bool
isEfaDevice(hwloc_obj_t obj) const;

public:
Expand All @@ -143,6 +146,11 @@ class nixlLibfabricTopology {
return num_nvidia_accel;
}

int
getNumAmdAccel() const {
return num_amd_accel;
}

const std::vector<std::string> &
getAllDevices() const {
return all_devices;
Expand All @@ -164,7 +172,13 @@ class nixlLibfabricTopology {

enum fi_hmem_iface
getMrAttrIface(int device_id) const {
return (device_id < num_nvidia_accel) ? FI_HMEM_CUDA : FI_HMEM_NEURON;
if (device_id < num_nvidia_accel) {
return FI_HMEM_CUDA;
} else if (device_id < num_nvidia_accel + num_amd_accel) {
return FI_HMEM_ROCR;
} else {
return FI_HMEM_NEURON;
}
}

// Debug/info
Expand Down