diff --git a/CMakeLists.txt b/CMakeLists.txt index d4c18fe..8de9b5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ project (konro) list (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/") option(TIMING "Enable low level timing" OFF) +option(NVIDIA "Enable NVIDIA GPU management" OFF) if (NOT DEFINED CGROUPBASEDIR) set(CGROUPBASEDIR "/sys/fs/cgroup" CACHE STRING "The cgroup base directory") @@ -22,6 +23,10 @@ if (TIMING) add_definitions(-DTIMING) endif() +if (NVIDIA) + add_definitions(-DNVIDIA) +endif() + add_subdirectory (common) add_subdirectory (lib) add_subdirectory (rm) @@ -30,6 +35,7 @@ add_subdirectory (konrolib) add_subdirectory (testplatformcontrol) add_subdirectory (testthreadsafequeue) add_subdirectory (testsensors) +add_subdirectory (testnvml) add_subdirectory (testhwloc) add_subdirectory (testkonrolib) add_subdirectory (testtimer) diff --git a/README.md b/README.md index 9e289a5..8c0a31f 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,19 @@ A modern resource manager - libhwloc-dev - libsensors-dev - libopencv-dev +- Cuda Toolkit OpenCV is required only for the peopledetect demo. If it is not installed, peopledetect is not compiled. +# Optional: Nvidia GPU support +nvidia-cuda-toolkit is required in order to compile with NVIDIA GPU monitoring. It can be installed from your package manager or by manually by following this link https://developer.nvidia.com/cuda-downloads. +In order to compile, you need to add the NVIDIA flag to cmake: +* `mkdir build && cd build` +* `cmake -DNVIDIA=ON ..` +* `make` +If no NVIDIA GPUs are present, the manager will still work, but the monitor will just skip them. +If nothing is specified, NVIDIA GPU support will be disabled. # Development QtCreator has been used as IDE for development on Ubuntu 22.04.2 LTS. diff --git a/rm/CMakeLists.txt b/rm/CMakeLists.txt index 21aaab3..7a7dcd6 100644 --- a/rm/CMakeLists.txt +++ b/rm/CMakeLists.txt @@ -9,6 +9,10 @@ add_subdirectory (platformcontrol) add_subdirectory (workloadmanager) add_subdirectory (policymanager) add_subdirectory (platformmonitor) +if(NVIDIA) + # Enable nvml code + add_subdirectory (nvidiamonitor) +endif(NVIDIA) file(GLOB konro_SOURCES "src/*.cpp") file(GLOB konro_HEADERS "src/*.h") @@ -17,6 +21,18 @@ include_directories(${PROJECT_SOURCE_DIR}/rm/include) add_executable (konro ${konro_SOURCES} ${konro_HEADERS}) target_compile_definitions(konro PUBLIC KONRO_RM) + +if(NVIDIA) +target_link_libraries(konro PUBLIC + konro-common + rmcommon + platformcontrol + workloadmanager + policymanager + platformmonitor + nvidiamonitor + ${LOG4CPP_LIBRARIES}) +else(NVIDIA) target_link_libraries(konro PUBLIC konro-common rmcommon @@ -25,5 +41,6 @@ target_link_libraries(konro PUBLIC policymanager platformmonitor ${LOG4CPP_LIBRARIES}) +endif(NVIDIA) configure_file(konro.ini.in konro.ini) diff --git a/rm/librmcommon/events/monitorgpuevent.h b/rm/librmcommon/events/monitorgpuevent.h new file mode 100644 index 0000000..c768f97 --- /dev/null +++ b/rm/librmcommon/events/monitorgpuevent.h @@ -0,0 +1,52 @@ +#ifndef MONITORGPUEVENT_H +#define MONITORGPUEVENT_H + +#include "baseevent.h" +#include "../gpuload.h" +#include "../gpupower.h" +#include "../gputemperature.h" +#include +#include + +namespace rmcommon { + +/*! + * \class event generated by the periodical sampling of machine status information. + */ +class MonitorGpuEvent : public BaseEvent { + GpuTemperature gpuTemp_; + GpuPower gpuActualPower_; + GpuLoad gpuLoad_; +public: + MonitorGpuEvent(GpuTemperature temp, GpuPower power, GpuLoad load) : + BaseEvent("MonitorEvent"), + gpuTemp_(temp), + gpuActualPower_(power), + gpuLoad_(load) + {} + virtual ~MonitorGpuEvent() = default; + + GpuTemperature getGpuTemperature() const { + return gpuTemp_; + } + + GpuPower getGpuPower() const { + return gpuActualPower_; + } + + GpuLoad getGpuLoad() const { + return gpuLoad_; + } + + virtual void printOnOstream(std::ostream &os) const { + os << "{"; + os << "\"gpuTemp\":" << gpuTemp_; + os << ",\"gpuPower\":" << gpuActualPower_; + os << ",\"gpuLoad\":" << gpuLoad_; + os << "}"; + } +}; + +} // namespace rmcommon + +#endif // MONITORGPUEVENT_H diff --git a/rm/librmcommon/gpuload.h b/rm/librmcommon/gpuload.h new file mode 100644 index 0000000..17f0025 --- /dev/null +++ b/rm/librmcommon/gpuload.h @@ -0,0 +1,46 @@ +#ifndef GPULOAD_H +#define GPULOAD_H + +#include +#include + +namespace rmcommon { +/*! + * \class encapsulates GPU utilization information + */ +class GpuLoad { + /*! % Usage of each GPU */ + std::vector gpu_; + +public: + void addGpuLoad(int load) { + gpu_.push_back(load); + } + + int getGpuLoad(int idx) { + return gpu_[idx]; + } + + + friend std::ostream &operator << (std::ostream &os, const GpuLoad &pt) { + bool first; + os << '{' + << "\"gpus\":"; + os << '['; + first = true; + for (const auto &gpu: pt.gpu_) { + if (first) + first = false; + else + os << ','; + os << gpu; + } + os << ']'; + os << '}'; + return os; + } +}; + +} // namespace rmcommon + +#endif // GPULOAD_H diff --git a/rm/librmcommon/gpupower.h b/rm/librmcommon/gpupower.h new file mode 100644 index 0000000..61eeb94 --- /dev/null +++ b/rm/librmcommon/gpupower.h @@ -0,0 +1,44 @@ +#ifndef GPUPOWER_H +#define GPUPOWER_H + +#include + +namespace rmcommon { +/*! + * \class encapsulates power information about the GPU + */ +class GpuPower { + /* Current power in mW */ + std::vector gpu_; +public: + + void addGpuPower(int val) { + gpu_.push_back(val); + } + + int getGpuPower(int index) { + return gpu_[index]; + } + + friend std::ostream &operator << (std::ostream &os, const GpuPower &pp) { + bool first; + os << '{' + << "\"gpus\":"; + os << '['; + first = true; + for (const auto &gpu: pp.gpu_) { + if (first) + first = false; + else + os << ','; + os << gpu; + } + os << ']'; + os << '}'; + return os; + } +}; + +} // namespace rmcommon + +#endif // GPUPOWER_H diff --git a/rm/librmcommon/gputemperature.h b/rm/librmcommon/gputemperature.h new file mode 100644 index 0000000..2e8df81 --- /dev/null +++ b/rm/librmcommon/gputemperature.h @@ -0,0 +1,44 @@ +#ifndef GPUTEMPERATURE_H +#define GPUTEMPERATURE_H + +#include + +namespace rmcommon { +/*! + * \class encapsulates temperature information about the GPU + */ + class GpuTemperature { + /* Current temperature in CELSIUS */ + std::vector gpu_; + public: + + void addGpuTemperature(int val) { + gpu_.push_back(val); + } + + int getGpuTemperature(int index) { + return gpu_[index]; + } + + friend std::ostream &operator << (std::ostream &os, const GpuTemperature &pp) { + bool first; + os << '{' + << "\"gpus\":"; + os << '['; + first = true; + for (const auto &gpu: pp.gpu_) { + if (first) + first = false; + else + os << ','; + os << gpu; + } + os << ']'; + os << '}'; + return os; + } + }; + +} // namespace rmcommon + +#endif // GPUTEMPERATURE_H diff --git a/rm/nvidiamonitor/CMakeLists.txt b/rm/nvidiamonitor/CMakeLists.txt new file mode 100644 index 0000000..8370273 --- /dev/null +++ b/rm/nvidiamonitor/CMakeLists.txt @@ -0,0 +1,13 @@ +set(CMAKE_CXX_STANDARD 14) + +find_package(CUDAToolkit REQUIRED) + +file(GLOB nvidiamonitor_SOURCES "*.cpp") +file(GLOB nvidiamonitor_HEADERS "*.h") +add_library(nvidiamonitor + ${nvidiamonitor_HEADERS} + ${nvidiamonitor_SOURCES} +) + +target_include_directories(nvidiamonitor INTERFACE .) +target_link_libraries(nvidiamonitor PUBLIC policymanager CUDA::nvml) diff --git a/rm/nvidiamonitor/nvidiamonitor.cpp b/rm/nvidiamonitor/nvidiamonitor.cpp new file mode 100644 index 0000000..f0b38d9 --- /dev/null +++ b/rm/nvidiamonitor/nvidiamonitor.cpp @@ -0,0 +1,143 @@ +#include "nvidiamonitor.h" +#include +#include +#include +#include +#include +#include +#include +#include "monitorevent.h" +#include "platformtemperature.h" +#include "platformpower.h" +#include "platformload.h" +#include "threadname.h" +#include "tsplit.h" +#include "monitorgpuevent.h" + +using namespace std; + +struct NvidiaMonitor::NvidiaMonitorImpl { + unsigned int nvml_devices_; + bool initialized = false; + vector nvidiaChips; + nvmlReturn_t result; + + NvidiaMonitorImpl() { + init(); + } + + ~NvidiaMonitorImpl() { + fini(); + } + + void init() { + result = nvmlInit_v2(); + if (result != NVML_SUCCESS) { + log4cpp::Category::getRoot().info("Failed to initialize NVML: %s", nvmlErrorString(result)); + this->initialized = false; + } else { + this->initialized = true; + result = nvmlDeviceGetCount(&nvml_devices_); + log4cpp::Category::getRoot().info("Found %d NVML devices", nvml_devices_); + for (unsigned int i = 0; i < nvml_devices_; i++) { + nvmlDevice_t device; + result = nvmlDeviceGetHandleByIndex(i, &device); + if (result != NVML_SUCCESS) { + log4cpp::Category::getRoot().info("Failed to get device handle: %s", nvmlErrorString(result)); + continue; // Continue with the next device on error + } + char name[NVML_DEVICE_NAME_BUFFER_SIZE]; + result = nvmlDeviceGetName(device, name, NVML_DEVICE_NAME_BUFFER_SIZE); + + nvidiaChips.emplace_back(name); + } + } + } + + void fini() { + if (this->initialized) { + result = nvmlShutdown(); + if (result != NVML_SUCCESS) { + log4cpp::Category::getRoot().info("Failed to initialize NVML: %s", nvmlErrorString(result)); + } + } + } + + bool isNvidiaChip(string chip) { + return find(begin(nvidiaChips), end(nvidiaChips), chip) != end(nvidiaChips); + } + + void handleNvidiaTemp(nvmlDevice_t device, rmcommon::GpuTemperature &gpuTemp) { + unsigned int temp; + result = nvmlDeviceGetTemperature(device, NVML_TEMPERATURE_GPU, &temp); + if (result == NVML_SUCCESS) { + gpuTemp.addGpuTemperature(temp); + } + } + + void handleNvidiaPower(nvmlDevice_t device, rmcommon::GpuPower &gpuPower) { + unsigned int power; + result = nvmlDeviceGetPowerUsage(device, &power); + if (result == NVML_SUCCESS) { + gpuPower.addGpuPower(power); + } + } + + void handleNvidiaLoad(nvmlDevice_t device, rmcommon::GpuLoad &gpuLoad) { + nvmlUtilization_t utilization; + result = nvmlDeviceGetUtilizationRates(device, &utilization); + if (result == NVML_SUCCESS) { + gpuLoad.addGpuLoad(utilization.gpu); + } + } + + /*! + * Detect each nvidia gpu available on the machine and calls the + * appropriate handler function to store information about its + * current status. + */ + void + handleNVGPUs(rmcommon::GpuLoad &gpuLoad, rmcommon::GpuPower &gpuPower, rmcommon::GpuTemperature &gpuTemperature) { + if (this->initialized) { + for (unsigned int i = 0; i < nvml_devices_; i++) { + nvmlDevice_t device; + result = nvmlDeviceGetHandleByIndex(i, &device); + if (result != NVML_SUCCESS) { + log4cpp::Category::getRoot().info("Failed to get device handle: %s", nvmlErrorString(result)); + continue; // Continue with the next device on error + } + handleNvidiaTemp(device, gpuTemperature); + handleNvidiaPower(device, gpuPower); + handleNvidiaLoad(device, gpuLoad); + } + } + } +}; + +NvidiaMonitor::NvidiaMonitor(rmcommon::EventBus &eventBus, int monitorPeriod) : + pimpl_(new NvidiaMonitorImpl()), + cat_(log4cpp::Category::getRoot()), + monitorPeriod_(monitorPeriod), + bus_(eventBus) { +} + +NvidiaMonitor::~NvidiaMonitor() { +} + +void NvidiaMonitor::run() { + setThreadName("NVIDIAMONITOR"); + cat_.info("NVIDIAMONITOR running"); + while (!stopped()) { + for (int i = 0; i < monitorPeriod_ && !stopped(); ++i) { + this_thread::sleep_for(chrono::seconds(1)); + } + if (!stopped()) { + rmcommon::GpuTemperature gpuTemp; + rmcommon::GpuPower gpuPower; + rmcommon::GpuLoad gpuLoad; + pimpl_->handleNVGPUs(gpuLoad, gpuPower, gpuTemp); + bus_.publish(new rmcommon::MonitorGpuEvent(gpuTemp, gpuPower, gpuLoad)); + } + } + cat_.info("NVIDIAMONITOR exiting"); +} diff --git a/rm/nvidiamonitor/nvidiamonitor.h b/rm/nvidiamonitor/nvidiamonitor.h new file mode 100644 index 0000000..6588f34 --- /dev/null +++ b/rm/nvidiamonitor/nvidiamonitor.h @@ -0,0 +1,38 @@ +#ifndef NVIDIAMONITOR_H +#define NVIDIAMONITOR_H + +#include "eventbus.h" +#include "basethread.h" +#include "gpuload.h" +#include "gpupower.h" +#include "gputemperature.h" +#include +#include +#include +#include +#include +#include + +/*! + * \class periodically samples information about the platform status. + * This information is then encapsulated in a MonitorEvent and published + * to the EventBus. + * NvidiaMonitor runs in a dedicated thread. + */ +class NvidiaMonitor : public rmcommon::BaseThread { + struct NvidiaMonitorImpl; + std::unique_ptr pimpl_; + log4cpp::Category &cat_; + int monitorPeriod_; + rmcommon::EventBus &bus_; + + virtual void run() override; + +public: + NvidiaMonitor(rmcommon::EventBus &eventBus, int monitorPeriod); + + ~NvidiaMonitor(); + +}; + +#endif // NVIDIAMONITOR_H \ No newline at end of file diff --git a/rm/policymanager/policymanager.cpp b/rm/policymanager/policymanager.cpp index 40c56e6..595c613 100644 --- a/rm/policymanager/policymanager.cpp +++ b/rm/policymanager/policymanager.cpp @@ -74,6 +74,7 @@ void PolicyManager::subscribeToEvents() bus_.subscribe(this, &PolicyManager::addEvent); bus_.subscribe(this, &PolicyManager::addEvent); bus_.subscribe(this, &PolicyManager::addEvent); + bus_.subscribe(this, &PolicyManager::addEvent); } bool PolicyManager::processEvent(std::shared_ptr event) @@ -93,6 +94,8 @@ bool PolicyManager::processEvent(std::shared_ptr even processTimerEvent(static_pointer_cast(event)); } else if (const MonitorEvent *e = dynamic_cast(event.get())) { processMonitorEvent(static_pointer_cast(event)); + } else if (const MonitorGpuEvent *e = dynamic_cast(event.get())) { + processMonitorGpuEvent(static_pointer_cast(event)); } else if (const FeedbackEvent *e = dynamic_cast(event.get())) { processFeedbackEvent(static_pointer_cast(event)); } @@ -135,6 +138,14 @@ void PolicyManager::processMonitorEvent(std::shared_ptrmonitor(event); } +void PolicyManager::processMonitorGpuEvent(std::shared_ptr event) +{ + ostringstream os; + os << "POLICYMANAGER GPU monitor event received => " << *event; + cat_.info(os.str()); +// policy_->monitor(event); +} + void PolicyManager::processFeedbackEvent(std::shared_ptr event) { using namespace rmcommon; diff --git a/rm/policymanager/policymanager.h b/rm/policymanager/policymanager.h index e337873..ca283ad 100644 --- a/rm/policymanager/policymanager.h +++ b/rm/policymanager/policymanager.h @@ -8,6 +8,7 @@ #include "removeevent.h" #include "timerevent.h" #include "monitorevent.h" +#include "monitorgpuevent.h" #include "feedbackevent.h" #include "appmapping.h" #include "policies/ibasepolicy.h" @@ -78,6 +79,12 @@ class PolicyManager : public rmcommon::BaseEventReceiver { */ void processMonitorEvent(std::shared_ptr event); + /*! + * Processes a MonitorGpuEvent + * \param ev the event to process + */ + void processMonitorGpuEvent(std::shared_ptr event); + /*! * Processes a FeedbackEvent * \param ev the event to process diff --git a/rm/src/konromanager.cpp b/rm/src/konromanager.cpp index e9f4c21..267ae2d 100644 --- a/rm/src/konromanager.cpp +++ b/rm/src/konromanager.cpp @@ -7,6 +7,9 @@ #include "policymanager.h" #include "workloadmanager.h" #include "platformmonitor.h" +#ifdef NVIDIA +#include "nvidiamonitor.h" +#endif #include "proclistener.h" #include "konrohttp.h" #include "policytimer.h" @@ -49,6 +52,9 @@ struct KonroManager::KonroManagerImpl { rp::PolicyManager *policyManager; rp::PolicyTimer *policyTimer; PlatformMonitor *platformMonitor; +#ifdef NVIDIA + NvidiaMonitor *nvidiaMonitor; +#endif KonroManagerImpl() { procListener = nullptr; @@ -57,6 +63,9 @@ struct KonroManager::KonroManagerImpl { policyManager = nullptr; policyTimer = nullptr; platformMonitor = nullptr; +#ifdef NVIDIA + nvidiaMonitor = nullptr; +#endif } ~KonroManagerImpl() { @@ -66,6 +75,9 @@ struct KonroManager::KonroManagerImpl { delete policyManager; delete policyTimer; delete platformMonitor; +#ifdef NVIDIA + delete nvidiaMonitor; +#endif } }; @@ -108,6 +120,9 @@ void KonroManager::loadConfiguration(std::string configFile) cfgMonitorPeriod_ = configRead(config, "platformmonitor", "monitorperiod", 20); cfgCpuModuleNames_ = configRead(config, "platformmonitor", "kernelcpumodulenames", std::string("coretemp,k10temp,k8temp,cputemp")); cfgBatteryModuleNames_ = configRead(config, "platformmonitor", "kernelbatterymodulenames", std::string("BAT")); +#ifdef NVIDIA + cfgNvidiaMonitorPeriod_ = configRead(config, "nvidiamonitor", "monitorperiod", 20); +#endif httpListenHost_ = configRead(config, "http", "listenhost", std::string("localhost")); httpListenPort_ = configRead(config, "http", "listenport", 8080); changeContainerCgroup_ = configRead(config, "container", "changecontainercgroup", 1); @@ -115,8 +130,11 @@ void KonroManager::loadConfiguration(std::string configFile) cat_.info("MAIN configuration: policy = %s", cfgPolicyName_.c_str()); cat_.info("MAIN configuration: policy timer seconds = %d", cfgTimerSeconds_); - cat_.info("MAIN configuration: monitor period seconds = %d", cfgMonitorPeriod_); + cat_.info("MAIN configuration: CPU monitor period seconds = %d", cfgMonitorPeriod_); cat_.info("MAIN configuration: CPU module names = %s", cfgCpuModuleNames_.c_str()); +#ifdef NVIDIA + cat_.info("MAIN configuration: NVIDIA GPU monitor period seconds = %d", cfgNvidiaMonitorPeriod_); +#endif cat_.info("MAIN configuration: battery module names = %s", cfgBatteryModuleNames_.c_str()); cat_.info("MAIN configuration: HTTP listen on %s:%d", httpListenHost_.c_str(), httpListenPort_); cat_.info("MAIN configuration: change container cgroup = %s", @@ -137,6 +155,9 @@ void KonroManager::run() pimpl_->workloadManager = new wm::WorkloadManager(pimpl_->eventBus, pimpl_->cgc); pimpl_->procListener = new wm::ProcListener(pimpl_->eventBus); pimpl_->platformMonitor = new PlatformMonitor(pimpl_->eventBus, pimpl_->platformDescription, cfgMonitorPeriod_); +#ifdef NVIDIA + pimpl_->nvidiaMonitor = new NvidiaMonitor(pimpl_->eventBus, cfgNvidiaMonitorPeriod_); +#endif pimpl_->policyTimer = new rp::PolicyTimer(pimpl_->eventBus, cfgTimerSeconds_); pimpl_->platformDescription.logTopology(); @@ -149,8 +170,9 @@ void KonroManager::run() // 2. WorkloadManager runs in a separate thread // 3. PolicyManager runs in a separate thread // 4. PlatformMonitor runs in a separate thread - // 5. KonroHttp runs in a separate thread - // 6. PolicyTimer runs in a separate thread + // 5. NvidiaMonitor runs in a separate thread, if present + // 6. KonroHttp runs in a separate thread + // 7. PolicyTimer runs in a separate thread cat_.info("MAIN starting WorkloadManager thread"); pimpl_->workloadManager->start(); @@ -169,6 +191,11 @@ void KonroManager::run() cat_.info("MAIN starting PlatformMonitor thread"); pimpl_->platformMonitor->start(); +#ifdef NVIDIA + cat_.info("MAIN starting NvidiaMonitor thread"); + pimpl_->nvidiaMonitor->start(); +#endif + cat_.info("MAIN starting HTTP thread"); pimpl_->http->start(); @@ -184,6 +211,9 @@ void KonroManager::run() pimpl_->policyTimer->stop(); } pimpl_->platformMonitor->stop(); +#ifdef NVIDIA + pimpl_->nvidiaMonitor->stop(); +#endif pimpl_->workloadManager->stop(); pimpl_->policyManager->stop(); @@ -196,6 +226,9 @@ void KonroManager::run() pimpl_->policyTimer->join(); } pimpl_->platformMonitor->join(); +#ifdef NVIDIA + pimpl_->nvidiaMonitor->join(); +#endif pimpl_->workloadManager->join(); pimpl_->policyManager->join(); diff --git a/rm/src/konromanager.h b/rm/src/konromanager.h index ead3be4..bc0466a 100644 --- a/rm/src/konromanager.h +++ b/rm/src/konromanager.h @@ -14,6 +14,7 @@ class KonroManager { std::string cfgPolicyName_; int cfgTimerSeconds_; // 0 means "no timer" int cfgMonitorPeriod_; + int cfgNvidiaMonitorPeriod_; std::string cfgCpuModuleNames_; std::string cfgBatteryModuleNames_; std::string httpListenHost_; diff --git a/testnvml/CMakeLists.txt b/testnvml/CMakeLists.txt new file mode 100644 index 0000000..f1a3770 --- /dev/null +++ b/testnvml/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.21) +set(CMAKE_CXX_STANDARD 14) +file(GLOB testnvml_SOURCES "*.cpp") +file(GLOB testnvml_HEADERS "*.h") + +# Find NVML library without specifying the name +#find_library(NVML_LIBRARY REQUIRED PATHS "/usr/lib/nvidia") +find_package(CUDAToolkit REQUIRED) + +add_executable(testnvml ${testnvml_SOURCES} ${testnvml_HEADERS}) +#target_include_directories(testnvml PRIVATE ${CUDA_INCLUDE_DIRS}) +target_link_libraries(testnvml PRIVATE CUDA::nvml) diff --git a/testnvml/testnvml.cpp b/testnvml/testnvml.cpp new file mode 100644 index 0000000..12092b6 --- /dev/null +++ b/testnvml/testnvml.cpp @@ -0,0 +1,195 @@ +#include +#include + +// Function to handle device-specific operations +void performDeviceOperations(nvmlDevice_t device, unsigned int index) +{ + nvmlReturn_t result; + + char name[NVML_DEVICE_NAME_BUFFER_SIZE]; + result = nvmlDeviceGetName(device, name, NVML_DEVICE_NAME_BUFFER_SIZE); + if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device name: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " name: " << name << std::endl; + +// // turn on the GPU +// result = nvmlDeviceSetPersistenceMode(device, nvmlEnableState_enum::NVML_FEATURE_DISABLED); + + + unsigned int temperature; + result = nvmlDeviceGetTemperature(device, NVML_TEMPERATURE_GPU, &temperature); + if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device temperature: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " temperature: " << temperature << " [C]" << std::endl; + } + + // Fan speed + unsigned int speed; + result = nvmlDeviceGetFanSpeed(device, &speed); + if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device fan speed: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " fan speed: " << speed << std::endl; + } + // Utilization rate + nvmlUtilization_t utilization; + result = nvmlDeviceGetUtilizationRates(device, &utilization); + if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device utilization: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " utilization: " << utilization.gpu << " [%GPU] " << utilization.memory << " [%memory]" << std::endl; + } + + // Battery info + nvmlPstates_t powerState; + nvmlDeviceGetPowerState(device, &powerState); + if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device power state: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " power state: " << powerState << " [0-15(minimum performance]" << std::endl; + } + + + unsigned int power; + result = nvmlDeviceGetPowerUsage(device, &power); + if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device power usage: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " power usage: " << power << "[mW]" << std::endl; + } + + unsigned int power_limit; + result = nvmlDeviceGetPowerManagementLimit(device, &power_limit); + if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device power limit: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " power limit: " << power_limit << "[mW]" << std::endl; + } + + unsigned int clock; + result = nvmlDeviceGetClockInfo(device, NVML_CLOCK_GRAPHICS, &clock); + if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device clock: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " clock: " << clock << "[MHz]" << std::endl; + } + + // PCI info + nvmlPciInfo_t pci; + result = nvmlDeviceGetPciInfo(device, &pci); + if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device PCI info: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " PCI BUS id: " << pci.busId << std::endl; + } + + // Compute mode + nvmlComputeMode_t compute_mode; + result = nvmlDeviceGetComputeMode(device, &compute_mode); + if (result == NVML_ERROR_NOT_SUPPORTED) + { + std::cout << "This is not a CUDA-capable device" << std::endl; + } + else if (result != NVML_SUCCESS) + { + std::cout << "Device " << index << " - Failed to get device compute mode: " << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "Device " << index << " compute mode: " << compute_mode << std::endl; + + // Try to change compute mode + std::cout<< "Changing device's compute mode from '" << compute_mode << "' to '" << NVML_COMPUTEMODE_PROHIBITED << "'" << std::endl; + + result = nvmlDeviceSetComputeMode(device, NVML_COMPUTEMODE_PROHIBITED); + if (NVML_ERROR_NO_PERMISSION == result) + std::cout<< "\t\t Need root privileges to do that: " << nvmlErrorString(result) << std::endl; + else if (NVML_ERROR_NOT_SUPPORTED == result) + std::cout<< " Compute mode prohibited not supported. You might be running on\n" + "windows in WDDM driver model or on non-CUDA capable GPU.\n"; + else if (NVML_SUCCESS != result) + { + std::cout << "\t\t Failed to set compute mode for device %i: %s\n" << index << nvmlErrorString(result) << std::endl; + } + else + { + std::cout << "\t\t restored compute mode to '" << compute_mode << "'" << std::endl; + result = nvmlDeviceSetComputeMode(device, compute_mode); + if (NVML_SUCCESS != result) + { + std::cout << "\t\t Failed to restore compute mode for device %i: %s\n" << index << nvmlErrorString(result) << std::endl; + } + } + } + } +} + +int main() +{ + nvmlReturn_t result; + + result = nvmlInit_v2(); + if (result != NVML_SUCCESS) + { + std::cout << "Failed to initialize NVML: " << nvmlErrorString(result) << std::endl; + return 1; + } + + // Get the number of devices + unsigned int nvml_devices; + result = nvmlDeviceGetCount(&nvml_devices); + std::cout << "Found " << nvml_devices << " devices" << std::endl; + + for (unsigned int i = 0; i < nvml_devices; i++) + { + nvmlDevice_t device; + result = nvmlDeviceGetHandleByIndex(i, &device); + if (result != NVML_SUCCESS) + { + std::cout << "Failed to get device handle: " << nvmlErrorString(result) << std::endl; + continue; // Continue with the next device on error + } + + // Perform device-specific operations + performDeviceOperations(device, i); + } + + result = nvmlShutdown(); + + if (result != NVML_SUCCESS) + { + std::cout << "Failed to initialize NVML: " << nvmlErrorString(result) << std::endl; + return 1; + } + + return 0; +} \ No newline at end of file