tests : fix build + warnings (test-tokenizer-1 still fails)

Author: ggerganov

tests/test-tokenizer-0.cpp

@@ -8,14 +8,13 @@
 
 static std::string unescape_whitespace(llama_context* ctx, const std::vector<llama_token>& tokens) {
     std::string result;
-    for (int i = 0; i < tokens.size(); ++i) {
+    for (size_t i = 0; i < tokens.size(); ++i) {
         result += llama_token_to_str(ctx, tokens[i]);
     }
     return result;
 }
 
-static const std::map<std::string, std::vector<llama_token>> & k_tests()
-{
+static const std::map<std::string, std::vector<llama_token>> & k_tests() {
     static std::map<std::string, std::vector<llama_token>> _k_tests = {
         { " ",                      {1,    259, }, },
         { "\t",                     { 1,    29871,   12, }, },
@@ -29,17 +28,18 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests()
         { " this is 🦙.cpp",        { 1,  29871,    445,    338,  29871,    243,    162,    169,    156,  29889,   8223, }, },
         { "w048 7tuijk dsdfhu",     { 1,    281,  29900,  29946,  29947,  29871,  29955,   9161,  13535,  18031,   2176,   6905, }, },
         { "нещо на Български",      { 1,   1538,   4851,    665,   1386,  29713,   1305, }, },
-        { "កាន់តែពិសេសអាចខលចេញ",   { 1,  29871,  31849,  31324,  31934,    228,    162,    142,    228,    161,    
-                                        146,    228,    162,    133,    228,    161,    153,    228,    161,    186,  
-                                        31708,    228,    162,    132,  31708,    228,    161,    165,  31324,    228,    
-                                        161,    136,    228,    161,    132,    228,    161,    158,    228,    161,    
-                                        136,    228,    162,    132,    228,    161,    140, }, },
+        { "កាន់តែពិសេសអាចខលចេញ",   { 1,  29871,  31849,  31324,  31934,    228,    162,    142,    228,    161,
+                                     146,    228,    162,    133,    228,    161,    153,    228,    161,    186,
+                                     31708,    228,    162,    132,  31708,    228,    161,    165,  31324,    228,
+                                     161,    136,    228,    161,    132,    228,    161,    158,    228,    161,
+                                     136,    228,    162,    132,    228,    161,    140, }, },
         { "🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)",
-            { 1,  29871,    243,    162,    157,    131,    313,   8945,  29897,  29871,    
-              243,    162,    155,    185,  30722,    243,    162,    143,    174,  30598,    
-              313,  20787,    953,   3848,    275,  16125,    630,  29897,  29871,  31681,    
-              313,   6194,    953,  29877,   2397,    393,    756,    967,   1914,   5993,  29897, }, },
-     };
+            { 1,  29871,    243,    162,    157,    131,    313,   8945,  29897,  29871,
+                243,    162,    155,    185,  30722,    243,    162,    143,    174,  30598,
+                313,  20787,    953,   3848,    275,  16125,    630,  29897,  29871,  31681,
+                313,   6194,    953,  29877,   2397,    393,    756,    967,   1914,   5993,  29897, }, },
+    };
+
     return _k_tests;
 };
 
@@ -90,8 +90,8 @@ int main(int argc, char **argv) {
     }
 
     for (const auto & test_kv : k_tests()) {
-        std::vector<llama_token> res = llama_tokenize(ctx, test_kv.first.c_str(), true);
-        fprintf(stderr, "%s : '%s' tokenized to '%s'\n", 
+        std::vector<llama_token> res = llama_tokenize(ctx, test_kv.first, true);
+        fprintf(stderr, "%s : '%s' tokenized to '%s'\n",
             __func__, test_kv.first.c_str(), unescape_whitespace(ctx, res).c_str());
 
         bool correct = res.size() == test_kv.second.size();

tests/test-tokenizer-1.cpp

@@ -8,6 +8,7 @@
 #include <codecvt>
 #include <map>
 #include <vector>
+#include <locale>
 
 static std::string vocab_type(llama_context* ctx) {
     return llama_n_vocab(ctx) == 32000 ? "spm": "bpe";
@@ -85,17 +86,17 @@ int main(int argc, char **argv) {
         if (tokens.size() == 1) {
             if (i != tokens[0]) {
                 std::string backward = llama_token_to_str(ctx, tokens[0]);
-                fprintf(stderr, "%s : error: token %d is string %s but bpe returns token %d %s\n", 
+                fprintf(stderr, "%s : error: token %d is string %s but bpe returns token %d %s\n",
                     __func__, i, llama_token_to_str(ctx, i).c_str(), tokens[0], backward.c_str());
                 return 2;
             }
         } else {
-            if ((vocab_type(ctx) == "spm" && i <= 258) || 
+            if ((vocab_type(ctx) == "spm" && i <= 258) ||
                 (vocab_type(ctx) == "bpe" && (i == 0 || i >= 100000))) {
-                fprintf(stderr, "%s : info: token %d is string %s and bpe returns tokens %s\n", 
+                fprintf(stderr, "%s : info: token %d is string %s and bpe returns tokens %s\n",
                     __func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens).c_str());
             } else {
-                fprintf(stderr, "%s : error: token %d is string %s but bpe returns tokens %s\n", 
+                fprintf(stderr, "%s : error: token %d is string %s but bpe returns tokens %s\n",
                     __func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens).c_str());
                 return 2;
             }
@@ -108,7 +109,7 @@ int main(int argc, char **argv) {
         std::string str = converter.to_bytes(wstr);
         std::vector<llama_token> tokens = llama_tokenize(ctx, escape_whitespace(str).c_str(), false);
         if (tokens.size() == 1) {
-            fprintf(stderr, "%s : info: %s tokenized to %d \n", 
+            fprintf(stderr, "%s : info: %s tokenized to %d \n",
                 __func__, str.c_str(), tokens[0]);
         }
     }