Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ dependencies = [
]

[project.scripts]
keep-gpu = "keep_gpu.cli:main"
keep-gpu = "keep_gpu.cli:app"

[project.optional-dependencies]
dev = [
Expand Down
70 changes: 64 additions & 6 deletions src/keep_gpu/cli.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,78 @@
"""Console script for keep_gpu."""

from .keep_gpu import run
import os
import time
from typing import Optional

import torch
import typer
from rich.console import Console

from keep_gpu.global_gpu_controller.global_gpu_controller import GlobalGPUController
from keep_gpu.utilities.logger import setup_logger

app = typer.Typer()
console = Console()
logger = setup_logger(__name__)


@app.command()
def main():
"""Console script for keep_gpu."""
console.print("Replace this message by putting your code into " "keep_gpu.cli.main")
console.print("See Typer documentation at https://typer.tiangolo.com/")
run()
def main(
interval: int = typer.Option(
300, help="Interval in seconds between GPU usage checks"
),
gpu_ids: Optional[str] = typer.Option(
None,
help="Comma-separated list of GPU IDs to monitor and benchmark on (default: all)",
),
vram: str = typer.Option(
"1GiB",
help="Amount of VRAM to keep occupied (e.g., '500MB', '1GiB', or integer in bytes)",
),
threshold: int = typer.Option(
-1,
help="Max GPU utilization threshold to trigger keeping GPU awake",
),
):
"""
Keep specified GPUs awake by allocating VRAM and monitoring usage.
"""
# Process GPU IDs
if gpu_ids:
try:
gpu_id_list = [int(i.strip()) for i in gpu_ids.split(",")]
except ValueError:
console.print(f"[bold red]Error: Invalid characters in --gpu-ids '{gpu_ids}'. Please use comma-separated integers.[/bold red]")
raise typer.Exit(code=1)
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpu_id_list))
logger.info(f"Using specified GPUs: {gpu_id_list}")
gpu_count = len(gpu_id_list)
else:
gpu_id_list = None
gpu_count = torch.cuda.device_count()
logger.info("Using all available GPUs")

# Log settings
logger.info(f"GPU count: {gpu_count}")
logger.info(f"VRAM to keep occupied: {vram}")
logger.info(f"Check interval: {interval} seconds")
logger.info(f"Busy threshold: {threshold}%")

# Create and start Global GPU Controller
global_controller = GlobalGPUController(
gpu_ids=gpu_id_list,
interval=interval,
vram_to_keep=vram,
busy_threshold=threshold,
)
Comment on lines +62 to +67

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The GlobalGPUController may raise a ValueError if the vram string is in an invalid format. This exception is currently unhandled in the main function, which will cause the application to crash with a stack trace. It's better to catch this exception and display a user-friendly error message.

    try:
        global_controller = GlobalGPUController(
            gpu_ids=gpu_id_list,
            interval=interval,
            vram_to_keep=vram,
            busy_threshold=threshold,
        )
    except ValueError as e:
        console.print(f"[bold red]Error: {e}[/bold red]")
        raise typer.Exit(code=1)


with global_controller:
logger.info("Keeping GPUs awake. Press Ctrl+C to exit.")
try:
while True:
time.sleep(3600)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The main thread sleeps for 3600 seconds (1 hour). This makes the application unresponsive to termination signals like Ctrl+C. Since the actual work is done in background threads managed by the GlobalGPUController, the main thread only needs to wait. Using a much shorter sleep interval in the loop will allow for a prompt shutdown when requested.

                time.sleep(1)

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mark this for later.

except KeyboardInterrupt:
logger.info("Interruption received. Releasing GPUs...")


if __name__ == "__main__":
Expand Down
72 changes: 0 additions & 72 deletions src/keep_gpu/keep_gpu.py

This file was deleted.

Loading