diff --git a/common/common.cpp b/common/common.cpp
index afc9b8a55bfae..fd47dbc2db683 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -804,10 +804,10 @@ std::vector<llama_token> llama_tokenize(
// upper limit for the number of tokens
int n_tokens = text.length() + add_bos;
std::vector<llama_token> result(n_tokens);
- n_tokens = llama_tokenize(ctx, text.c_str(), result.data(), result.size(), add_bos);
+ n_tokens = llama_tokenize(ctx, text.data(), text.length(), result.data(), result.size(), add_bos);
if (n_tokens < 0) {
result.resize(-n_tokens);
- int check = llama_tokenize(ctx, text.c_str(), result.data(), result.size(), add_bos);
+ int check = llama_tokenize(ctx, text.data(), text.length(), result.data(), result.size(), add_bos);
GGML_ASSERT(check == -n_tokens);
} else {
result.resize(n_tokens);
diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp
index 947aa7ed3bd3e..59c90c7ba654d 100644
--- a/examples/train-text-from-scratch/train-text-from-scratch.cpp
+++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp
@@ -965,10 +965,10 @@ int tokenize_file(struct llama_context * lctx, const char * filename, std::vecto
buf[size] = '\0';
- int n_tokens = llama_tokenize(lctx, buf.data(), out.data(), out.size(), false);
+ int n_tokens = llama_tokenize(lctx, buf.data(), buf.size(), out.data(), out.size(), false);
if (n_tokens < 0) {
out.resize(-n_tokens);
- n_tokens = llama_tokenize(lctx, buf.data(), out.data(), out.size(), false);
+ n_tokens = llama_tokenize(lctx, buf.data(), buf.size(), out.data(), out.size(), false);
}
GGML_ASSERT(n_tokens >= 0);
out.resize(n_tokens);
diff --git a/llama.cpp b/llama.cpp
index cbaf8edac0ba8..2fea1cac10c88 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -6202,19 +6202,21 @@ llama_token llama_token_nl(const struct llama_context * ctx) {
int llama_tokenize(
struct llama_context * ctx,
const char * text,
+ int text_len,
llama_token * tokens,
int n_max_tokens,
bool add_bos) {
- return llama_tokenize_with_model(&ctx->model, text, tokens, n_max_tokens, add_bos);
+ return llama_tokenize_with_model(&ctx->model, text, text_len, tokens, n_max_tokens, add_bos);
}
int llama_tokenize_with_model(
const struct llama_model * model,
const char * text,
+ int text_len,
llama_token * tokens,
int n_max_tokens,
bool add_bos) {
- auto res = llama_tokenize_internal(model->vocab, text, add_bos);
+ auto res = llama_tokenize_internal(model->vocab, std::string(text, text_len), add_bos);
if (n_max_tokens < (int) res.size()) {
// LLAMA_LOG_ERROR("%s: too many tokens\n", __func__);
diff --git a/llama.h b/llama.h
index 37975bebed22e..795389d8d7d8f 100644
--- a/llama.h
+++ b/llama.h
@@ -374,6 +374,7 @@ extern "C" {
LLAMA_API int llama_tokenize(
struct llama_context * ctx,
const char * text,
+ int text_len,
llama_token * tokens,
int n_max_tokens,
bool add_bos);
@@ -381,6 +382,7 @@ extern "C" {
LLAMA_API int llama_tokenize_with_model(
const struct llama_model * model,
const char * text,
+ int text_len,
llama_token * tokens,
int n_max_tokens,
bool add_bos);
diff --git a/tests/test-tokenizer-0-llama.cpp b/tests/test-tokenizer-0-llama.cpp
index edbd86f8514d6..dfb2e81a9bc6f 100644
--- a/tests/test-tokenizer-0-llama.cpp
+++ b/tests/test-tokenizer-0-llama.cpp
@@ -36,6 +36,7 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests() {
{ " Hello" , { 1678, 15043, }, },
{ " Hello" , { 268, 15043, }, },
{ " Hello\n Hello" , { 268, 15043, 13, 1678, 15043, }, },
+ { " (" , { 29871, 313, }, },
};
return _k_tests;
diff --git a/tests/test-tokenizer-1-llama.cpp b/tests/test-tokenizer-1-llama.cpp
index ab3d822f2b1cd..817acacea4228 100644
--- a/tests/test-tokenizer-1-llama.cpp
+++ b/tests/test-tokenizer-1-llama.cpp
@@ -87,10 +87,9 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens);
if (check != str) {
- fprintf(stderr, "%s : error: token %d detokenizes to >%s<(%llu) but tokenization of this detokenizes to >%s<(%llu)\n",
+ fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
__func__, i, str.c_str(), str.length(), check.c_str(), check.length());
- if(i != 3)
- return 2;
+ return 2;
}
}
@@ -99,11 +98,10 @@ int main(int argc, char **argv) {
std::string str = codepoint_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens);
- if (str != check) {
- fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%llu) instead of >%s<(%llu)\n",
+ if (cp != 9601 && str != check) {
+ fprintf(stderr, "%s : error: codepoint %d detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
__func__, cp, check.c_str(), check.length(), str.c_str(), str.length());
- if(cp != 0 && cp != 9601)
- return 3;
+ return 3;
}
}
}
@@ -112,7 +110,7 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens);
if (str != check) {
- fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%llu) instead of >%s<(%llu)\n",
+ fprintf(stderr, "%s : error: codepoint %d detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
__func__, cp, check.c_str(), check.length(), str.c_str(), str.length());
return 4;
}