From e75c02904d9c36ecdaf89cfa97bfd9a0f72b3978 Mon Sep 17 00:00:00 2001 From: "zhou.weiguo" Date: Mon, 15 Apr 2024 10:26:54 +0800 Subject: [PATCH 1/2] ggml: add hw_accel in data structure --- ggml.c | 2 ++ ggml.h | 1 + whisper.cpp | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index 793b67f4c70..843896dc0b9 100644 --- a/ggml.c +++ b/ggml.c @@ -2195,6 +2195,7 @@ struct ggml_context { bool mem_buffer_owned; bool no_alloc; bool no_alloc_save; // this is used to save the no_alloc state when using scratch buffers + bool use_hwaccel; int n_objects; @@ -2754,6 +2755,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) { /*.mem_buffer_owned =*/ params.mem_buffer ? false : true, /*.no_alloc =*/ params.no_alloc, /*.no_alloc_save =*/ params.no_alloc, + /*.use_hwaccel =*/ params.use_hwaccel, /*.n_objects =*/ 0, /*.objects_begin =*/ NULL, /*.objects_end =*/ NULL, diff --git a/ggml.h b/ggml.h index abe3767f224..71fcdb741be 100644 --- a/ggml.h +++ b/ggml.h @@ -657,6 +657,7 @@ extern "C" { size_t mem_size; // bytes void * mem_buffer; // if NULL, memory will be allocated internally bool no_alloc; // don't allocate memory for the tensor data + bool use_hwaccel; }; diff --git a/whisper.cpp b/whisper.cpp index b9e1ef2ced1..0a8524a7a5f 100644 --- a/whisper.cpp +++ b/whisper.cpp @@ -4066,6 +4066,14 @@ static int whisper_has_openvino(void) { #endif } +static int whisper_has_qnn(void) { +#ifdef GGML_USE_QNN + return 1; +#else + return 0; +#endif +} + const char * whisper_print_system_info(void) { static std::string s; @@ -4086,7 +4094,9 @@ const char * whisper_print_system_info(void) { s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | "; s += "CUDA = " + std::to_string(ggml_cpu_has_cuda()) + " | "; s += "COREML = " + std::to_string(whisper_has_coreml()) + " | "; - s += "OPENVINO = " + std::to_string(whisper_has_openvino()) ; + s += "OPENVINO = " + std::to_string(whisper_has_openvino()) + " | "; + s += "QNN = " + std::to_string(whisper_has_qnn()) ; + return s.c_str(); } @@ -6518,6 +6528,9 @@ WHISPER_API const char * whisper_bench_ggml_mul_mat_str(int n_threads) { /*.no_alloc =*/ false, }; +#ifdef GGML_USE_QNN + gparams.use_hwaccel = true; +#endif struct ggml_context * ctx0 = ggml_init(gparams); struct ggml_tensor * a = ggml_new_tensor_2d(ctx0, wtype, N, N); From d648e0f4365968b98c0e1b85500ba750f34d2916 Mon Sep 17 00:00:00 2001 From: "zhou.weiguo" Date: Mon, 22 Apr 2024 08:01:55 +0800 Subject: [PATCH 2/2] make github CI happy --- ggml.c | 4 ++++ ggml.h | 4 +++- whisper.cpp | 12 +----------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/ggml.c b/ggml.c index 843896dc0b9..8f672871695 100644 --- a/ggml.c +++ b/ggml.c @@ -2987,9 +2987,13 @@ static struct ggml_tensor * ggml_new_tensor_impl( /*.data =*/ obj_alloc_size > 0 ? (void *)(result + 1) : data, /*.name =*/ { 0 }, /*.extra =*/ NULL, + /*.rank =*/ n_dims, /*.padding =*/ { 0 }, }; + if (ctx->use_hwaccel) + result->backend = GGML_BACKEND_TYPE_GPU; + // TODO: this should not be needed as long as we don't rely on aligned SIMD loads //ggml_assert_aligned(result->data); diff --git a/ggml.h b/ggml.h index 71fcdb741be..76bc63dea71 100644 --- a/ggml.h +++ b/ggml.h @@ -591,7 +591,9 @@ extern "C" { void * extra; // extra things e.g. for ggml-cuda.cu - char padding[8]; + int32_t rank; + + char padding[20]; }; static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); diff --git a/whisper.cpp b/whisper.cpp index 0a8524a7a5f..0cabf3273c0 100644 --- a/whisper.cpp +++ b/whisper.cpp @@ -4066,14 +4066,6 @@ static int whisper_has_openvino(void) { #endif } -static int whisper_has_qnn(void) { -#ifdef GGML_USE_QNN - return 1; -#else - return 0; -#endif -} - const char * whisper_print_system_info(void) { static std::string s; @@ -4094,9 +4086,7 @@ const char * whisper_print_system_info(void) { s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | "; s += "CUDA = " + std::to_string(ggml_cpu_has_cuda()) + " | "; s += "COREML = " + std::to_string(whisper_has_coreml()) + " | "; - s += "OPENVINO = " + std::to_string(whisper_has_openvino()) + " | "; - s += "QNN = " + std::to_string(whisper_has_qnn()) ; - + s += "OPENVINO = " + std::to_string(whisper_has_openvino()) ; return s.c_str(); }