under test, not sure no errors

ixformer_sdk/utils/benchmark/cuda_benchmark.py

@@ -0,0 +1,69 @@
+import time
+from collections import namedtuple, OrderedDict
+from typing import List, Dict, Any
+
+import tabulate
+import torch
+import torch.distributed as dist
+
+DeviceTime = namedtuple("DeviceTime", ["cpu", "gpu"])
+
+
+class Functor:
+
+    def __init__(self, fn, *args, **kwargs):
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+
+    def __call__(self):
+        return self.fn(*self.args, **self.kwargs)
+
+
+def cuda_timeit(fn: Functor, dist_barrier=False) -> DeviceTime:
+    torch.cuda.synchronize()
+
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record(torch.cuda.current_stream())
+
+    t0 = time.time()
+    fn()
+    t1 = time.time()
+
+    stop.record(torch.cuda.current_stream())
+
+    torch.cuda.synchronize()
+    if dist_barrier:
+        dist.barrier()
+
+    gpu_time = start.elapsed_time(stop)
+    cpu_time = t1 - t0
+    return DeviceTime(cpu_time, gpu_time)
+
+
+def cuda_benchmark(fn: Functor, num_repeated=10, num_warmup=1, dist_barrier=False) -> DeviceTime:
+    [fn() for _ in range(num_warmup)]
+    times = [cuda_timeit(fn, dist_barrier=dist_barrier) for _ in range(num_repeated)]
+    times.sort(key=lambda t: t.gpu)
+
+    if num_repeated >= 10:
+        times = times[3:-3]
+
+    avg_gpu_time = sum([t.gpu for t in times]) / len(times)
+    avg_cpu_time = sum([t.cpu for t in times]) / len(times)
+
+    return DeviceTime(avg_cpu_time * 1000, avg_gpu_time)
+
+
+def show_benchmark_results(times: List[DeviceTime], extra_info: Dict[Any, List]=None):
+    data = extra_info or OrderedDict()
+
+    if len(times) != 0:
+        cpu_times = [round(t.cpu, 6) for t in times]
+        gpu_times = [round(t.gpu, 6) for t in times]
+
+        data["CPU Time(ms)"] = cpu_times
+        data["GPU Time(ms)"] = gpu_times
+
+    print(tabulate.tabulate(extra_info, headers=data.keys()))