Fixed OpenLLaMA 3b CUDA mul_mat_vec_q

Author: JohannesGaessler

ggml-cuda.cu

@@ -216,7 +216,7 @@ static_assert(sizeof(block_q6_K) == sizeof(ggml_fp16_t) + 13*QK_K/16, "wrong q6_
 #define CUDA_SCALE_BLOCK_SIZE 256
 #define CUDA_ROPE_BLOCK_SIZE 256
 #define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32
-#define CUDA_QUANTIZE_BLOCK_SIZE 256
+#define CUDA_QUANTIZE_BLOCK_SIZE 128
 #define CUDA_DEQUANTIZE_BLOCK_SIZE 256
 
 // dmmv = dequantize_mul_mat_vec
@@ -1174,16 +1174,12 @@ static __device__ void convert_f16(const void * vx, const int ib, const int iqs,
 static __global__ void quantize_q8_1(const float * __restrict__ x, void * __restrict__ vy, const int k) {
     const int i = blockDim.x*blockIdx.x + threadIdx.x;
 
-    if (i >= k) {
-        return;
-    }
-
     block_q8_1 * y = (block_q8_1 *) vy;
 
-    const int ib = i / QK8_0; // block index
-    const int iqs = i % QK8_0; // quant index
+    const int ib = i / QK8_1; // block index
+    const int iqs = i % QK8_1; // quant index
 
-    const float xi = x[i];
+    const float xi = i < k ? x[i] : 0.0f;
     float amax = fabsf(xi);
     float sum = xi;
 
@@ -2359,8 +2355,10 @@ inline void ggml_cuda_op_mul_mat_vec(
 #endif
 
     if (use_mul_mat_vec_q) {
+        int64_t padded_row_size = ne00 + CUDA_QUANTIZE_BLOCK_SIZE - 1;
+        padded_row_size -= padded_row_size % CUDA_QUANTIZE_BLOCK_SIZE;
         size_t as;
-        void * src1_q8_1 = ggml_cuda_pool_malloc(ne00*sizeof(block_q8_1)/QK8_1, &as);
+        void * src1_q8_1 = ggml_cuda_pool_malloc(padded_row_size*sizeof(block_q8_1)/QK8_1, &as);
         quantize_row_q8_1_cuda(src1_ddf_i, src1_q8_1, ne00, cudaStream_main);
 
         switch (src0->type) {
@@ -3105,7 +3103,11 @@ void ggml_cuda_nop(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tens
 
 void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor) {
     int nrows = ggml_nrows(tensor);
+
+    const int64_t ne0 = tensor->ne[0];
+
     const size_t nb1 = tensor->nb[1];
+
     ggml_backend backend = tensor->backend;
     struct ggml_tensor_extra_gpu * extra = new struct ggml_tensor_extra_gpu;
     memset(extra, 0, sizeof(*extra));
@@ -3134,7 +3136,11 @@ void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor) {
         int64_t nrows_split = row_high - row_low;
 
         const size_t offset_split = row_low*nb1;
-        const size_t size = ggml_nbytes_split(tensor, nrows_split);
+        size_t size = ggml_nbytes_split(tensor, nrows_split);
+        if (ne0 % CUDA_QUANTIZE_BLOCK_SIZE != 0) {
+            size += (CUDA_QUANTIZE_BLOCK_SIZE - ne0 % CUDA_QUANTIZE_BLOCK_SIZE)
+                * ggml_type_size(tensor->type)/ggml_blck_size(tensor->type);
+        }
 
         void * buf;
         CUDA_CHECK(cudaMalloc(&buf, size));