diff --git a/pyproject.toml b/pyproject.toml index a750462b..744d7149 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ ] [project.scripts] -keep-gpu = "keep_gpu.cli:main" +keep-gpu = "keep_gpu.cli:app" [project.optional-dependencies] dev = [ diff --git a/src/keep_gpu/cli.py b/src/keep_gpu/cli.py index 18e1ee54..d64e1abf 100644 --- a/src/keep_gpu/cli.py +++ b/src/keep_gpu/cli.py @@ -1,20 +1,78 @@ """Console script for keep_gpu.""" -from .keep_gpu import run +import os +import time +from typing import Optional +import torch import typer from rich.console import Console +from keep_gpu.global_gpu_controller.global_gpu_controller import GlobalGPUController +from keep_gpu.utilities.logger import setup_logger + app = typer.Typer() console = Console() +logger = setup_logger(__name__) @app.command() -def main(): - """Console script for keep_gpu.""" - console.print("Replace this message by putting your code into " "keep_gpu.cli.main") - console.print("See Typer documentation at https://typer.tiangolo.com/") - run() +def main( + interval: int = typer.Option( + 300, help="Interval in seconds between GPU usage checks" + ), + gpu_ids: Optional[str] = typer.Option( + None, + help="Comma-separated list of GPU IDs to monitor and benchmark on (default: all)", + ), + vram: str = typer.Option( + "1GiB", + help="Amount of VRAM to keep occupied (e.g., '500MB', '1GiB', or integer in bytes)", + ), + threshold: int = typer.Option( + -1, + help="Max GPU utilization threshold to trigger keeping GPU awake", + ), +): + """ + Keep specified GPUs awake by allocating VRAM and monitoring usage. + """ + # Process GPU IDs + if gpu_ids: + try: + gpu_id_list = [int(i.strip()) for i in gpu_ids.split(",")] + except ValueError: + console.print(f"[bold red]Error: Invalid characters in --gpu-ids '{gpu_ids}'. Please use comma-separated integers.[/bold red]") + raise typer.Exit(code=1) + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpu_id_list)) + logger.info(f"Using specified GPUs: {gpu_id_list}") + gpu_count = len(gpu_id_list) + else: + gpu_id_list = None + gpu_count = torch.cuda.device_count() + logger.info("Using all available GPUs") + + # Log settings + logger.info(f"GPU count: {gpu_count}") + logger.info(f"VRAM to keep occupied: {vram}") + logger.info(f"Check interval: {interval} seconds") + logger.info(f"Busy threshold: {threshold}%") + + # Create and start Global GPU Controller + global_controller = GlobalGPUController( + gpu_ids=gpu_id_list, + interval=interval, + vram_to_keep=vram, + busy_threshold=threshold, + ) + + with global_controller: + logger.info("Keeping GPUs awake. Press Ctrl+C to exit.") + try: + while True: + time.sleep(3600) + except KeyboardInterrupt: + logger.info("Interruption received. Releasing GPUs...") if __name__ == "__main__": diff --git a/src/keep_gpu/keep_gpu.py b/src/keep_gpu/keep_gpu.py deleted file mode 100644 index 8d9a0443..00000000 --- a/src/keep_gpu/keep_gpu.py +++ /dev/null @@ -1,72 +0,0 @@ -import argparse -import os -import subprocess -import time -import torch - -from .benchmark import run_benchmark -from keep_gpu.utilities.logger import setup_logger - -logger = setup_logger(__name__) - - -def parse_args(): - parser = argparse.ArgumentParser( - description="GPU Idle Monitor and Benchmark Trigger" - ) - parser.add_argument( - "--interval", - type=int, - default=300, - help="Interval in seconds between GPU usage checks", - ) - parser.add_argument( - "--gpu-ids", - type=str, - default=None, - help="Comma-separated list of GPU IDs to monitor and benchmark on (default: all)", - ) - return parser.parse_args() - - -def check_gpu_usage(gpu_ids=None): - result = subprocess.run( - ["nvidia-smi", "--query-gpu=utilization.gpu", "--format=csv,noheader,nounits"], - capture_output=True, - text=True, - ) - usage_lines = result.stdout.strip().split("\n") - usages = [int(line.strip()) for line in usage_lines] - - if gpu_ids is not None: - usages = [usages[i] for i in gpu_ids if i < len(usages)] - - return any(usage > 0 for usage in usages) - - -def run(): - args = parse_args() - - if args.gpu_ids: - gpu_ids = [int(i.strip()) for i in args.gpu_ids.split(",")] - os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpu_ids)) - logger.info(f"Using specified GPUs: {gpu_ids}") - gpu_count = len(gpu_ids) - else: - gpu_ids = None - gpu_count = torch.cuda.device_count() - logger.info("Using all available GPUs") - - idle_count = 0 - logger.info(f"GPU count: {gpu_count}") - while True: - if not check_gpu_usage(gpu_ids): - idle_count += 1 - else: - idle_count = 0 - - if idle_count >= 1: - run_benchmark(gpu_count) - idle_count = 0 - - time.sleep(args.interval)