[SD-151] Run the experiment for measuring impact of convolutional parameters on power

jetson/power_logging/Dockerfile.jetson

@@ -6,13 +6,13 @@ RUN apt update && apt install --no-install-recommends -y libgl1-mesa-glx
 RUN groupadd --gid 1000 appuser \
     && useradd --uid 1000 --gid 1000 -ms /bin/bash appuser
 
+# modelopt dependency comes pre-packaged as part of the image
+RUN pip install ultralytics
+
 USER appuser
 
 WORKDIR /app
 
 COPY --chown=appuser:appuser . ./
 
-# modelopt dependency comes pre-packaged as part of the image
-RUN pip install ultralytics
-
 CMD ["/bin/bash", "./run_experiment.sh"]

jetson/power_logging/measure_inference_power.py

@@ -7,7 +7,7 @@
 from multiprocessing.synchronize import Event as EventClass
 from pathlib import Path
 
-from model.benchmark import benchmark
+from model.benchmark import benchmark_classify, benchmark_detection, benchmark_trt
 
 multiprocessing.set_start_method("spawn", force=True)
 
@@ -54,7 +54,11 @@ def inference(event: EventClass, args: argparse.Namespace) -> None:
         event: An object that manages a flag for communication among processes.
         args: Arguments from CLI.
     """
-    benchmark(args)
+    if args.command == "detect":
+        benchmark_detection(args)
+    elif args.command == "classify":
+        # benchmark_classify(args)  # For Pytorch
+        benchmark_trt(args)  # For TensorRT
     event.set()
 
 
@@ -64,28 +68,89 @@ def inference(event: EventClass, args: argparse.Namespace) -> None:
         description="Collect power usage data during inference cycles for ImageNet pretrained CNN models.",
     )
     parser.add_argument(
+        "--result-dir",
+        type=str,
+        default="results",
+        help="The directory to save the log result.",
+    )
+    parser.add_argument(
+        "--disable-power-measurement",
+        action="store_true",
+        help="Disable power measurement during benchmark execution.",
+    )
+    subparsers = parser.add_subparsers(help="Types of arguments", dest="command")
+    detection_parser = subparsers.add_parser(
+        "detect", help="Run benchmarking for object detection models"
+    )
+    detection_parser.add_argument(
         "--model",
         type=str,
         default="yolov5n",
         help="Specify name of pretrained CNN model from ultralytics.",
     )
-    parser.add_argument(
+    detection_parser.add_argument(
         "--dataset-name",
         type=str,
         default="coco.yaml",
         help="Specify name of dataset from ultralytics.",
     )
-    parser.add_argument(
-        "--result-dir",
+    classify_parser = subparsers.add_parser(
+        "classify", help="Run benchmarking for image classification models"
+    )
+    classify_parser.add_argument(
+        "--model",
         type=str,
-        default="results",
-        help="The directory to save the log result.",
+        default="mobilenet_v2",
+        help="Specify name of pretrained CNN model from PyTorch Hub."
+        "For more information on PyTorch Hub visit: "
+        "https://pytorch.org/hub/research-models",
     )
-    parser.add_argument(
-        "--disable-power-measurement",
-        action="store_true",
-        help="Disable power measurement during benchmark execution.",
+    classify_parser.add_argument(
+        "--model-repo",
+        type=str,
+        default="pytorch/vision:v0.16.0",  # This version should have all the models we want
+        help="Specify path and version to model repository from PyTorch Hub.",
     )
+    classify_parser.add_argument(
+        "--dtype",
+        type=str,
+        default="float16",
+        choices=["float16", "bfloat16", "float32"],
+        help="Data type for model weights and activations.\n\n"
+        '* "float16" is the same as "half".\n'
+        '* "bfloat16" for a balance between precision and range.\n'
+        '* "float32" for FP32 precision.',
+    )
+    classify_parser.add_argument(
+        "--input-shape",
+        type=int,
+        nargs="+",
+        default=[1, 3, 224, 224],
+        help="Input shape BCHW",
+    )
+    classify_parser.add_argument(
+        "--warmup",
+        type=int,
+        default=50,
+        help="Number of iterations to perform warmup before benchmarking",
+    )
+    classify_parser.add_argument(
+        "--runs", type=int, default=30000, help="Number of inference cycle to run"
+    )
+    classify_parser.add_argument(
+        "--optimization-level",
+        type=int,
+        default=5,
+        help="Builder optimization 0-5, higher levels imply longer build time, "
+        "searching for more optimization options.",
+    )
+    classify_parser.add_argument(
+        "--min-block-size",
+        type=int,
+        default=5,
+        help="Minimum number of operators per TRT-Engine Block",
+    )
+
     args = parser.parse_args()
 
     event = Event()

jetson/power_logging/model/benchmark.py

@@ -13,10 +13,14 @@
 from pathlib import Path
 from typing import Any
 
+import numpy as np
 import torch
+import torch_tensorrt
 from pydantic import BaseModel
+from tqdm import tqdm
 
 from model.model_utils import get_layers, load_model
+from model.trt_utils import CustomProfiler, save_engine_info, save_layer_wise_profiling
 
 """
 Wrapper class for Torch.cuda.event for non-CUDA supported devices
@@ -60,14 +64,23 @@ def get_time_stamp(self):
 DEVICE = "cuda" if IS_GPU else "cpu"
 
 
-class BenchmarkMetrics(BaseModel):
+class DetectionBenchmarkMetrics(BaseModel):
     config: dict[str, Any]
     total_time: float  # in seconds
     timestamp: str
     start_time: float
     end_time: float
 
 
+class ClassifyBenchmarkMetrics(BaseModel):
+    config: dict[str, Any]
+    total_time: float  # in seconds
+    timestamp: str
+    latencies: list[float]  # in seconds
+    avg_latency: float  # in seconds
+    avg_throughput: float
+
+
 def define_and_register_hooks(model, device) -> dict:
     """
         Define and register hooks with CUDA or CPU timing.
@@ -136,8 +149,8 @@ def layer_time_hook(
     layer_time_dict[layer_name]["start_time"] = start_event.get_time_stamp()
 
 
-def benchmark(args: argparse.Namespace) -> None:
-    """Benchmark latency and throughput across all backends.
+def benchmark_detection(args: argparse.Namespace) -> None:
+    """Benchmark latency and throughput for object detection models.
 
     Args:
         args: Arguments from CLI.
@@ -180,7 +193,7 @@ def benchmark(args: argparse.Namespace) -> None:
         print("Benchmarking complete ...")
         total_time = start_event.elapsed_time(end_event)
 
-        results = BenchmarkMetrics(
+        results = DetectionBenchmarkMetrics(
             config=vars(args),
             total_time=total_time,  # in seconds
             timestamp=timestamp,
@@ -208,3 +221,196 @@ def benchmark(args: argparse.Namespace) -> None:
     except Exception as e:
         print(f"An error has occurred during benchmarking: {e}")
         raise e
+
+
+def benchmark_classify(args: argparse.Namespace) -> None:
+    """Benchmark latency and throughput across all backends.
+
+    Args:
+        args: Arguments from CLI.
+    """
+    print("Starting benchmark...")
+
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+
+    try:
+        input_data = torch.randn(args.input_shape, device=DEVICE)
+        model = load_model(args.model)
+        model.eval().to(DEVICE)
+
+        if args.dtype == "float16":
+            dtype = torch.float16
+        if args.dtype == "bfloat16":
+            dtype = torch.bfloat16
+        if args.dtype == "float32":
+            dtype = torch.float32
+
+        input_data = input_data.to(dtype)
+        model = model.to(dtype)
+        print(f"Using {DEVICE=} for benchmarking")
+        if DEVICE == "cpu":
+            print("Warning: Running on CPU.")
+
+        st = time.perf_counter()
+        print("Warm up ...")
+        with torch.no_grad():
+            for _ in range(args.warmup):
+                _ = model(input_data)
+        print(f"Warm complete in {time.perf_counter() - st:.2f} sec ...")
+
+        layer_profiles = []
+        layer_profile = define_and_register_hooks(model, DEVICE)
+
+        print("Starting timing inference ...")
+        latencies = []
+        start_events = [CudaEvent(enable_timing=True) for _ in range(args.runs)]
+        end_events = [CudaEvent(enable_timing=True) for _ in range(args.runs)]
+
+        with torch.no_grad():
+            for i in tqdm(range(args.runs)):
+                start_events[i].record()
+                _ = model(input_data)
+                end_events[i].record()
+
+                if IS_GPU:
+                    torch.cuda.synchronize()
+
+                latency = start_events[i].elapsed_time(end_events[i])
+                latencies.append(latency * 1.0e-3)
+                layer_profiles.append(layer_profile.copy())
+
+        print("Benchmarking complete ...")
+
+        total_time = sum(latencies)
+        avg_latency = total_time / len(latencies)
+        avg_throughput = args.input_shape[0] / avg_latency
+
+        results = ClassifyBenchmarkMetrics(
+            config=vars(args),
+            total_time=total_time,  # in seconds
+            timestamp=timestamp,
+            latencies=latencies,  # in seconds
+            avg_throughput=avg_throughput,
+            avg_latency=avg_latency,  # in seconds
+        )
+
+        model_dir = f"{args.result_dir}/{args.model}"
+        Path(model_dir).mkdir(exist_ok=True, parents=True)
+        file_name = f"{args.model}_pytorch.json"
+        file_path = f"{model_dir}/{file_name}"
+        with open(file_path, "w", encoding="utf-8") as outfile:
+            json.dump(results.model_dump(), outfile, indent=4)
+        with open(
+            f"{model_dir}/{args.model}_layerwise_latency.json", "w"
+        ) as layer_profiles_file:
+            json.dump(layer_profiles, layer_profiles_file)
+    except Exception as e:
+        print(f"An error has occurred during benchmarking: {e}")
+        return
+
+
+def benchmark_trt(args: argparse.Namespace) -> None:
+    """Benchmark latency and throughput across all backends.
+
+    Additionally for tensorrt backend, we calculate layer-wise
+    latency.
+
+    Args:
+        args: Arguments from CLI.
+    """
+    start = torch.cuda.Event(enable_timing=True)
+    end = torch.cuda.Event(enable_timing=True)
+    start.record()
+
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    input_data = torch.randn(args.input_shape, device=DEVICE)
+    model = load_model(args.model)
+    model.eval().to(DEVICE)
+
+    if args.dtype == "float16":
+        dtype = torch.float16
+    if args.dtype == "bfloat16":
+        dtype = torch.bfloat16
+    if args.dtype == "float32":
+        dtype = torch.float32
+
+    input_data = input_data.to(dtype)
+    model = model.to(dtype)
+    print(f"Using {DEVICE=} for benchmarking")
+
+    exp_program = torch.export.export(model, tuple([input_data]))
+    model = torch_tensorrt.dynamo.compile(
+        exported_program=exp_program,
+        inputs=[input_data],
+        min_block_size=args.min_block_size,
+        optimization_level=args.optimization_level,
+        enabled_precisions={dtype},
+        # Set to True for verbose output
+        # NOTE: Performance Regression when rich library is available
+        # https://github.com/pytorch/TensorRT/issues/3215
+        debug=True,
+        # Setting it to True returns PythonTorchTensorRTModule which has different profiling approach
+        use_python_runtime=True,
+    )
+
+    print("Sleeping for 5 seconds to cool down...")
+    time.sleep(5)
+
+    st = time.perf_counter()
+    print("Warm up ...")
+    with torch.no_grad():
+        for _ in range(args.warmup):
+            _ = model(input_data)
+    print(f"Warm complete in {time.perf_counter() - st:.2f} sec ...")
+
+    print("Start timing using tensorrt backend ...")
+    torch.cuda.synchronize()
+    # Recorded in milliseconds
+    start_events = [torch.cuda.Event(enable_timing=True) for _ in range(args.runs)]
+    end_events = [torch.cuda.Event(enable_timing=True) for _ in range(args.runs)]
+
+    with torch.no_grad():
+        for i in tqdm(range(args.runs)):
+            # Hack for enabling profiling
+            # https://github.com/pytorch/TensorRT/issues/1467
+            profiling_dir = f"{args.result_dir}/{args.model}/trt_profiling"
+            Path(profiling_dir).mkdir(exist_ok=True, parents=True)
+
+            # Records traces in milliseconds
+            # https://docs.nvidia.com/deeplearning/tensorrt/api/python_api/infer/Core/Profiler.html#tensorrt.Profiler
+            mod = list(model.named_children())[0][1]
+            mod.enable_profiling(profiler=CustomProfiler())
+
+            start_events[i].record()
+            _ = model(input_data)
+            end_events[i].record()
+
+        end.record()
+        torch.cuda.synchronize()
+
+    save_layer_wise_profiling(mod, profiling_dir)
+    save_engine_info(mod, profiling_dir)
+
+    # Convert milliseconds to seconds
+    timings = [s.elapsed_time(e) * 1.0e-3 for s, e in zip(start_events, end_events)]
+    avg_throughput = args.input_shape[0] / np.mean(timings)
+    print("Benchmarking complete ...")
+    # Convert milliseconds to seconds
+    total_exp_time = start.elapsed_time(end) * 1.0e-3
+    print(f"Total time for experiment: {total_exp_time} sec")
+
+    results = ClassifyBenchmarkMetrics(
+        config=vars(args),
+        total_time=total_exp_time,  # in seconds
+        timestamp=timestamp,
+        latencies=timings,  # in seconds
+        avg_throughput=avg_throughput,
+        avg_latency=np.mean(timings),  # in seconds
+    )
+
+    model_dir = f"{args.result_dir}/{args.model}"
+    Path(model_dir).mkdir(exist_ok=True, parents=True)
+    file_name = f"{args.model}_tensorrt.json"
+    file_path = f"{model_dir}/{file_name}"
+    with open(file_path, "w", encoding="utf-8") as outfile:
+        json.dump(results.model_dump(), outfile, indent=4)