Skip to content

Commit 0acb5f5

Browse files
committed
mild cleanup
don't need the space-prefixing here rn since main.cpp already does it
1 parent 8d32441 commit 0acb5f5

File tree

1 file changed

+3
-12
lines changed

1 file changed

+3
-12
lines changed

utils.cpp

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ struct llama_sp_symbol {
303303

304304
struct llama_sp_bigram {
305305
struct comparator {
306-
const bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) {
306+
bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) {
307307
return (l.score < r.score) || (l.score == r.score && l.left > r.left);
308308
}
309309
};
@@ -409,7 +409,7 @@ struct llama_tokenizer {
409409
work_queue_.push(bigram);
410410
}
411411

412-
gpt_vocab const& vocab_;
412+
const gpt_vocab & vocab_;
413413
std::vector<llama_sp_symbol> symbols_;
414414
llama_sp_bigram::queue work_queue_;
415415
};
@@ -426,16 +426,7 @@ std::vector<gpt_vocab::id> llama_tokenize(const gpt_vocab & vocab, std::string_v
426426
output.push_back(1);
427427
}
428428

429-
if (text[0] != ' ') {
430-
std::string copy;
431-
432-
copy += ' ';
433-
copy += text;
434-
tokenizer.tokenize(copy, output);
435-
} else {
436-
tokenizer.tokenize(text, output);
437-
}
438-
429+
tokenizer.tokenize(text, output);
439430
return output;
440431
}
441432

0 commit comments

Comments
 (0)