From 735ffe3d2f8f57fd0b1066e48ff690dd31b67854 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Tue, 14 Nov 2023 16:48:33 -0500 Subject: [PATCH 1/2] Revert "dont add space when using special tokens" This reverts commit 1c28116de410ad689fdf26d637be6f0530107cb0. --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 01522fdb4e74f..fe4ff736970fd 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6283,7 +6283,7 @@ static std::vector llama_tokenize_internal(const llama_vocab & // by modifying llm_tokenizer_x to operate with string offsets like pre-tokenizer // and passing 'add space prefix' as bool argument // - auto raw_text = (special ? "" : " ") + fragment.raw_text.substr(fragment.offset, fragment.length); + auto raw_text = " " + fragment.raw_text.substr(fragment.offset, fragment.length); #ifdef PRETOKENIZERDEBUG fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); From 5e899428cb180841e8e14599b64fbb4830a2ca15 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Tue, 14 Nov 2023 19:21:14 -0500 Subject: [PATCH 2/2] do not add space prefix if the first token is special --- llama.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index fe4ff736970fd..92c4536cb948e 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6283,7 +6283,10 @@ static std::vector llama_tokenize_internal(const llama_vocab & // by modifying llm_tokenizer_x to operate with string offsets like pre-tokenizer // and passing 'add space prefix' as bool argument // - auto raw_text = " " + fragment.raw_text.substr(fragment.offset, fragment.length); + auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); + if (&fragment == &fragment_buffer.front()) { + raw_text = " " + raw_text; // prefix with space if the first token is not special + } #ifdef PRETOKENIZERDEBUG fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());