From 682da0658216a9e8aef1e26cd3c85874b35181b2 Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Tue, 4 Mar 2025 03:01:32 +0000 Subject: [PATCH 1/2] fix: remove double instance of cudaProfilerStart() Signed-off-by: Brayden Zhong --- benchmarks/kernels/benchmark_layernorm.py | 2 +- benchmarks/kernels/benchmark_paged_attention.py | 2 +- benchmarks/kernels/benchmark_quant.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/kernels/benchmark_layernorm.py b/benchmarks/kernels/benchmark_layernorm.py index d265c91bfeff..e12d74c01e43 100644 --- a/benchmarks/kernels/benchmark_layernorm.py +++ b/benchmarks/kernels/benchmark_layernorm.py @@ -40,7 +40,7 @@ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float: end_time = time.perf_counter() if profile: - torch.cuda.cudart().cudaProfilerStart() + torch.cuda.cudart().cudaProfilerStop() return (end_time - start_time) / num_iters # Warmup. diff --git a/benchmarks/kernels/benchmark_paged_attention.py b/benchmarks/kernels/benchmark_paged_attention.py index 221d7b7d5d91..48b351bc4814 100644 --- a/benchmarks/kernels/benchmark_paged_attention.py +++ b/benchmarks/kernels/benchmark_paged_attention.py @@ -176,7 +176,7 @@ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float: end_time = time.perf_counter() if profile: - torch.cuda.cudart().cudaProfilerStart() + torch.cuda.cudart().cudaProfilerStop() return (end_time - start_time) / num_iters # Warmup. diff --git a/benchmarks/kernels/benchmark_quant.py b/benchmarks/kernels/benchmark_quant.py index 0ddea9390d77..b643897a60ee 100644 --- a/benchmarks/kernels/benchmark_quant.py +++ b/benchmarks/kernels/benchmark_quant.py @@ -40,7 +40,7 @@ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float: end_time = time.perf_counter() if profile: - torch.cuda.cudart().cudaProfilerStart() + torch.cuda.cudart().cudaProfilerStop() return (end_time - start_time) / num_iters # Warmup. From 211cb70742390f8e3ea921c4f2e9814b1aff33b1 Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Tue, 4 Mar 2025 04:48:47 +0000 Subject: [PATCH 2/2] feat: various fixes Signed-off-by: Brayden Zhong --- benchmarks/kernels/benchmark_lora.py | 1 - benchmarks/kernels/benchmark_machete.py | 3 +-- benchmarks/kernels/benchmark_moe.py | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/benchmarks/kernels/benchmark_lora.py b/benchmarks/kernels/benchmark_lora.py index 5eaeec017053..3c4d6a6aa464 100644 --- a/benchmarks/kernels/benchmark_lora.py +++ b/benchmarks/kernels/benchmark_lora.py @@ -153,7 +153,6 @@ def ref_group_gemm(ref_out: torch.Tensor, input: torch.Tensor, result = torch.nn.functional.linear(x, w) result *= scaling out_list.append(result) - torch.cat(out_list, dim=0) cat_result = torch.cat(out_list, dim=0) diff --git a/benchmarks/kernels/benchmark_machete.py b/benchmarks/kernels/benchmark_machete.py index 3fa57bd7b233..a661ea9d7e60 100644 --- a/benchmarks/kernels/benchmark_machete.py +++ b/benchmarks/kernels/benchmark_machete.py @@ -45,7 +45,6 @@ def terse_type_name(dt): torch.float16: "fp16", torch.int8: "int8", torch.float8_e4m3fn: "fp8", - torch.bfloat16: "bf16", torch.float: "float", torch.int: "int", }[dt] @@ -259,7 +258,7 @@ def machete_create_bench_fn(bt: BenchmarkTensors, return lambda: ops.machete_mm( a=bt.a, - b_q=bt.w_q, + b_q=w_q, b_type=bt.wtype, b_group_scales=bt.w_g_s, b_group_zeros=w_g_zp, diff --git a/benchmarks/kernels/benchmark_moe.py b/benchmarks/kernels/benchmark_moe.py index c862dec81fcc..b488c3b44507 100644 --- a/benchmarks/kernels/benchmark_moe.py +++ b/benchmarks/kernels/benchmark_moe.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 import argparse +import json import time from datetime import datetime from itertools import product