From de8035af2ca3dba2954ddffd419c1a2f00beb06a Mon Sep 17 00:00:00 2001 From: JohannesGaessler Date: Mon, 18 Sep 2023 18:14:53 +0200 Subject: [PATCH] CUDA: use only 1 thread if fully offloaded --- llama.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/llama.cpp b/llama.cpp index 79b48897d8bbe..10cff5ff7124c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -3777,6 +3777,15 @@ static bool llama_eval_internal( n_threads = std::min(4, n_threads); } + // If all tensors can be run on the GPU then using more than 1 thread is detrimental. + const bool full_offload_supported = model.arch == LLM_ARCH_LLAMA || + model.arch == LLM_ARCH_BAICHUAN || + model.arch == LLM_ARCH_FALCON; + const bool fully_offloaded = model.n_gpu_layers >= (int) hparams.n_layer + 3; + if (ggml_cpu_has_cublas() && full_offload_supported && fully_offloaded) { + n_threads = 1; + } + struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 2];