Skip to content

Commit 012cf34

Browse files
authored
server : send token probs for "stream == false" (#4714)
1 parent a919280 commit 012cf34

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

examples/server/server.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1265,7 +1265,7 @@ struct llama_server_context
12651265
{
12661266
std::vector<completion_token_output> probs_output = {};
12671267
const std::vector<llama_token> to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false);
1268-
size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size());
1268+
size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size());
12691269
size_t probs_stop_pos = std::min(slot.sent_token_probs_index + to_send_toks.size(), slot.generated_token_probs.size());
12701270
if (probs_pos < probs_stop_pos)
12711271
{
@@ -1325,7 +1325,7 @@ struct llama_server_context
13251325
{
13261326
probs = std::vector<completion_token_output>(
13271327
slot.generated_token_probs.begin(),
1328-
slot.generated_token_probs.begin() + slot.sent_token_probs_index);
1328+
slot.generated_token_probs.end());
13291329
}
13301330
res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs);
13311331
}

0 commit comments

Comments
 (0)