From ba210e4bc781d37d437feb51c063c315e9a5720d Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Wed, 21 Jun 2023 14:21:35 -0700
Subject: [PATCH 01/10] server: add option to output probabilities for
completion
---
examples/common.h | 1 +
examples/server/server.cpp | 126 +++++++++++++++++++++++++++++--------
2 files changed, 100 insertions(+), 27 deletions(-)
diff --git a/examples/common.h b/examples/common.h
index 6c2953cb2a7c6..6393c85636f2a 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -31,6 +31,7 @@ struct gpt_params {
int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors
float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs
bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance
+ int32_t n_probs = 0; // if greater than 1, output the probabilities of top n_probs tokens. Max 5
// sampling parameters
std::unordered_map<llama_token, float> logit_bias; // logit bias for specific tokens
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index c0984aadb92ba..16f5bac0abf35 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -26,6 +26,28 @@ struct server_params {
int32_t write_timeout = 600;
};
+// completion string output with probabilities
+struct completion_string_output {
+ struct token_prob {
+ std::string tok_str;
+ float prob;
+ };
+
+ std::vector<token_prob> probs;
+ std::string tok_str;
+};
+
+// completion token output with probabilities
+struct completion_token_output {
+ struct token_prob {
+ llama_token tok;
+ float prob;
+ };
+
+ std::vector<token_prob> probs;
+ llama_token tok;
+};
+
static size_t common_part(const std::vector<llama_token> & a, const std::vector<llama_token> & b) {
size_t i;
for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) {}
@@ -107,6 +129,7 @@ struct llama_server_context {
bool stream = false;
bool has_next_token = false;
std::string generated_text;
+ std::vector<completion_string_output> generated_text_probs;
size_t num_tokens_predicted = 0;
size_t n_past = 0;
@@ -137,6 +160,7 @@ struct llama_server_context {
num_tokens_predicted = 0;
generated_text = "";
generated_text.reserve(params.n_ctx);
+ generated_text_probs.clear();
truncated = false;
stopped_eos = false;
stopped_word = false;
@@ -216,8 +240,9 @@ struct llama_server_context {
llama_set_rng_seed(ctx, params.seed);
}
- llama_token nextToken() {
- llama_token result = -1;
+ completion_token_output nextToken() {
+ completion_token_output result;
+ result.tok = -1;
if (embd.size() >= (size_t)params.n_ctx) {
// Reset context
@@ -256,7 +281,8 @@ struct llama_server_context {
if (params.n_predict == 0) {
has_next_token = false;
- return llama_token_eos();
+ result.tok = llama_token_eos();
+ return result;
}
// out of user input, sample next token
@@ -273,7 +299,7 @@ struct llama_server_context {
const float mirostat_tau = params.mirostat_tau;
const float mirostat_eta = params.mirostat_eta;
const bool penalize_nl = params.penalize_nl;
- llama_token id = 0;
+ const int32_t n_probs = params.n_probs;
{
auto * logits = llama_get_logits(ctx);
@@ -307,17 +333,17 @@ struct llama_server_context {
if (temp <= 0) {
// Greedy sampling
- id = llama_sample_token_greedy(ctx, &candidates_p);
+ result.tok = llama_sample_token_greedy(ctx, &candidates_p);
} else {
if (mirostat == 1) {
static float mirostat_mu = 2.0f * mirostat_tau;
const int mirostat_m = 100;
llama_sample_temperature(ctx, &candidates_p, temp);
- id = llama_sample_token_mirostat(ctx, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);
+ result.tok = llama_sample_token_mirostat(ctx, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);
} else if (mirostat == 2) {
static float mirostat_mu = 2.0f * mirostat_tau;
llama_sample_temperature(ctx, &candidates_p, temp);
- id = llama_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
+ result.tok = llama_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
} else {
// Temperature sampling
llama_sample_tail_free(ctx, &candidates_p, tfs_z, 1);
@@ -325,17 +351,19 @@ struct llama_server_context {
llama_sample_top_p(ctx, &candidates_p, top_p, 1);
llama_sample_top_k(ctx, &candidates_p, top_k, 1);
llama_sample_temperature(ctx, &candidates_p, temp);
- id = llama_sample_token(ctx, &candidates_p);
+ result.tok = llama_sample_token(ctx, &candidates_p);
}
}
+ for (size_t i = 0; i < std::min(candidates_p.size, std::min((size_t) n_probs, size_t(5))); ++i) {
+ result.probs.push_back({candidates_p.data[i].id, candidates_p.data[i].p});
+ }
last_n_tokens.erase(last_n_tokens.begin());
- last_n_tokens.push_back(id);
+ last_n_tokens.push_back(result.tok);
num_tokens_predicted++;
}
// add it to the context
- embd.push_back(id);
- result = id;
+ embd.push_back(result.tok);
// decrement remaining sampling budget
--n_remain;
@@ -377,12 +405,22 @@ struct llama_server_context {
return stop_pos;
}
- std::string doCompletion() {
- const llama_token token = nextToken();
+ completion_string_output doCompletion() {
+ const completion_token_output token_with_probs = nextToken();
+ completion_string_output result;
- const std::string token_text = token == -1 ? "" : llama_token_to_str(ctx, token);
+ const std::string token_text = token_with_probs.tok == -1 ? "" : llama_token_to_str(ctx, token_with_probs.tok);
+ result.tok_str = token_text;
generated_text += token_text;
+ // iterate through token_with_probs.probs, if tok is valid, convert it to string and add to result.prob
+ for (const auto & prob : token_with_probs.probs) {
+ const std::string prob_text = prob.tok == -1 ? "" : llama_token_to_str(ctx, prob.tok);
+ result.probs.push_back({prob_text, prob.prob});
+ }
+
+ generated_text_probs.push_back(result);
+
if (multibyte_pending > 0) {
multibyte_pending -= token_text.size();
} else if (token_text.size() == 1) {
@@ -411,8 +449,8 @@ struct llama_server_context {
}
LOG_VERBOSE("next token", {
- { "token", token },
- { "token_text", llama_token_to_str(ctx, token) },
+ { "token", token_with_probs.tok },
+ { "token_text", llama_token_to_str(ctx, token_with_probs.tok) },
{ "has_next_token", has_next_token },
{ "n_remain", n_remain },
{ "num_tokens_predicted", num_tokens_predicted },
@@ -422,7 +460,7 @@ struct llama_server_context {
{ "stopping_word", stopping_word },
});
- return token_text;
+ return result;
}
std::vector<float> getEmbedding() {
@@ -664,6 +702,7 @@ static json format_generation_settings(llama_server_context & llama) {
{ "ignore_eos", ignore_eos },
{ "stream", llama.stream },
{ "logit_bias", llama.params.logit_bias },
+ { "n_probs", llama.params.n_probs },
};
}
@@ -673,9 +712,26 @@ static json format_embedding_response(llama_server_context & llama) {
};
}
-static json format_final_response(llama_server_context & llama, const std::string & content) {
+static json format_final_response(llama_server_context & llama, const std::string & content, const std::vector<completion_string_output> & probs) {
+
+ json completion_probabilities_json = json::array();
+ for (const auto & prob : probs) {
+ json probs_for_token = json::array();
+ for (const auto & p : prob.probs) {
+ probs_for_token.push_back(json {
+ { "tok_str", p.tok_str },
+ { "prob", p.prob },
+ });
+ }
+ completion_probabilities_json.push_back(json {
+ {"content", prob.tok_str},
+ {"probs", probs_for_token},
+ });
+ }
+
return json {
{ "content", content },
+ { "completion_probabilities", completion_probabilities_json},
{ "stop", true },
{ "model", llama.params.model_alias },
{ "tokens_predicted", llama.num_tokens_predicted },
@@ -689,11 +745,25 @@ static json format_final_response(llama_server_context & llama, const std::strin
};
}
-static json format_partial_response(const std::string & content) {
- return json {
+static json format_partial_response(const std::string & content, const completion_string_output & probs) {
+ json res = json {
{ "content", content },
{ "stop", false },
};
+
+ // iterate through probs.probs, and add to res
+ json probs_json = json::array();
+ for (const auto & prob : probs.probs) {
+ probs_json.push_back(json {
+ { "tok_str", prob.tok_str },
+ { "prob", prob.prob },
+ });
+ }
+ if (probs.probs.size() > 0) {
+ res["probs"] = probs_json;
+ }
+
+ return res;
}
static json format_tokenizer_response(const std::vector<llama_token> & tokens) {
@@ -723,6 +793,7 @@ static void parse_options_completion(const json & body, llama_server_context & l
llama.params.n_keep = body.value("n_keep", default_params.n_keep);
llama.params.seed = body.value("seed", default_params.seed);
llama.params.prompt = body.value("prompt", default_params.prompt);
+ llama.params.n_probs = body.value("n_probs", default_params.n_probs);
llama.params.logit_bias.clear();
if (body.value("ignore_eos", false)) {
@@ -825,7 +896,8 @@ int main(int argc, char ** argv) {
size_t stop_pos = std::string::npos;
while (llama.has_next_token) {
- const std::string token_text = llama.doCompletion();
+ const completion_string_output token_text_with_probs = llama.doCompletion();
+ const std::string token_text = token_text_with_probs.tok_str;
stop_pos = llama.findStoppingStrings(llama.generated_text,
token_text.size(), STOP_FULL);
@@ -839,7 +911,7 @@ int main(int argc, char ** argv) {
llama.generated_text.end());
}
- const json data = format_final_response(llama, llama.generated_text);
+ const json data = format_final_response(llama, llama.generated_text, llama.generated_text_probs);
llama_print_timings(llama.ctx);
@@ -850,7 +922,7 @@ int main(int argc, char ** argv) {
size_t sent_count = 0;
while (llama.has_next_token) {
- const std::string token_text = llama.doCompletion();
+ const completion_string_output token_text_with_probs = llama.doCompletion();
if (llama.multibyte_pending > 0) {
continue;
}
@@ -859,14 +931,14 @@ int main(int argc, char ** argv) {
const std::string str_test = llama.generated_text.substr(pos);
size_t stop_pos =
- llama.findStoppingStrings(str_test, token_text.size(), STOP_FULL);
+ llama.findStoppingStrings(str_test, token_text_with_probs.tok_str.size(), STOP_FULL);
if (stop_pos != std::string::npos) {
llama.generated_text.erase(
llama.generated_text.begin() + pos + stop_pos,
llama.generated_text.end());
pos = std::min(sent_count, llama.generated_text.size());
} else {
- stop_pos = llama.findStoppingStrings(str_test, token_text.size(),
+ stop_pos = llama.findStoppingStrings(str_test, token_text_with_probs.tok_str.size(),
STOP_PARTIAL);
}
@@ -874,9 +946,9 @@ int main(int argc, char ** argv) {
sent_count += to_send.size();
const json data = llama.has_next_token
- ? format_partial_response(to_send)
+ ? format_partial_response(to_send, token_text_with_probs)
// Generation is done, send extra information.
- : format_final_response(llama, to_send);
+ : format_final_response(llama, to_send, {token_text_with_probs});
const std::string str =
"data: " +
From ccf254bd4450049dc623dd3a8cabfda70d452483 Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Thu, 22 Jun 2023 08:57:35 -0700
Subject: [PATCH 02/10] server: fix comment about max n_probs
---
examples/common.h | 2 +-
examples/server/server.cpp | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/examples/common.h b/examples/common.h
index 6393c85636f2a..5ceac53c5233d 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -31,7 +31,7 @@ struct gpt_params {
int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors
float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs
bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance
- int32_t n_probs = 0; // if greater than 1, output the probabilities of top n_probs tokens. Max 5
+ int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens.
// sampling parameters
std::unordered_map<llama_token, float> logit_bias; // logit bias for specific tokens
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 16f5bac0abf35..4cad658d3eb3a 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -354,6 +354,7 @@ struct llama_server_context {
result.tok = llama_sample_token(ctx, &candidates_p);
}
}
+ // Add maximum of 5 most probable tokens to the result
for (size_t i = 0; i < std::min(candidates_p.size, std::min((size_t) n_probs, size_t(5))); ++i) {
result.probs.push_back({candidates_p.data[i].id, candidates_p.data[i].p});
}
From cf7619522335100284f35b94be5104dded20b407 Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Thu, 22 Jun 2023 21:35:37 -0700
Subject: [PATCH 03/10] server: fix issue when handling probability output for
incomplete tokens for multibyte character generation
---
examples/server/server.cpp | 135 ++++++++++++++++++++-----------------
1 file changed, 73 insertions(+), 62 deletions(-)
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 4cad658d3eb3a..d9af1308e3d8a 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -26,17 +26,6 @@ struct server_params {
int32_t write_timeout = 600;
};
-// completion string output with probabilities
-struct completion_string_output {
- struct token_prob {
- std::string tok_str;
- float prob;
- };
-
- std::vector<token_prob> probs;
- std::string tok_str;
-};
-
// completion token output with probabilities
struct completion_token_output {
struct token_prob {
@@ -108,6 +97,36 @@ static void server_log(const char * level, const char * function, int line,
fflush(stdout);
}
+// format incomplete utf-8 multibyte character for output
+static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) {
+ const std::string out = token == -1 ? "" : llama_token_to_str(ctx, token);
+ if (out[0] > 127) {
+ out = "byte: \\x" + std::format("{:x}", out[0]);
+ }
+ return out;
+}
+
+// convert a vector of completion_token_output to json
+static json probs_vector_to_json(const llama_context * ctx, const vector<completion_token_output> probs) {
+ json out = json::array();
+ for (const auto & prob : probs) {
+ json probs_for_token = json::array();
+ for (const auto & p : prob.probs) {
+ std::string tok_str = tokens_to_output_formatted_string(ctx, p.tok);
+ probs_for_token.push_back(json {
+ { "tok_str", tok_str },
+ { "prob", p.prob },
+ });
+ }
+ std::string tok_str = tokens_to_output_formatted_string(ctx, prob.tok);
+ out.push_back(json {
+ {"content", tok_str},
+ {"probs", probs_for_token},
+ });
+ }
+ return out;
+}
+
static bool server_verbose = false;
#if SERVER_VERBOSE != 1
@@ -129,7 +148,7 @@ struct llama_server_context {
bool stream = false;
bool has_next_token = false;
std::string generated_text;
- std::vector<completion_string_output> generated_text_probs;
+ std::vector<completion_token_output> generated_token_probs;
size_t num_tokens_predicted = 0;
size_t n_past = 0;
@@ -160,7 +179,7 @@ struct llama_server_context {
num_tokens_predicted = 0;
generated_text = "";
generated_text.reserve(params.n_ctx);
- generated_text_probs.clear();
+ generated_token_probs.clear();
truncated = false;
stopped_eos = false;
stopped_word = false;
@@ -406,22 +425,16 @@ struct llama_server_context {
return stop_pos;
}
- completion_string_output doCompletion() {
+ completion_token_output doCompletion() {
const completion_token_output token_with_probs = nextToken();
- completion_string_output result;
const std::string token_text = token_with_probs.tok == -1 ? "" : llama_token_to_str(ctx, token_with_probs.tok);
- result.tok_str = token_text;
generated_text += token_text;
- // iterate through token_with_probs.probs, if tok is valid, convert it to string and add to result.prob
- for (const auto & prob : token_with_probs.probs) {
- const std::string prob_text = prob.tok == -1 ? "" : llama_token_to_str(ctx, prob.tok);
- result.probs.push_back({prob_text, prob.prob});
+ if (params.n_probs > 0) {
+ generated_token_probs.push_back(token_with_probs);
}
- generated_text_probs.push_back(result);
-
if (multibyte_pending > 0) {
multibyte_pending -= token_text.size();
} else if (token_text.size() == 1) {
@@ -451,7 +464,7 @@ struct llama_server_context {
LOG_VERBOSE("next token", {
{ "token", token_with_probs.tok },
- { "token_text", llama_token_to_str(ctx, token_with_probs.tok) },
+ { "token_text", tokens_to_output_formatted_string(ctx, token_with_probs.tok) },
{ "has_next_token", has_next_token },
{ "n_remain", n_remain },
{ "num_tokens_predicted", num_tokens_predicted },
@@ -461,7 +474,7 @@ struct llama_server_context {
{ "stopping_word", stopping_word },
});
- return result;
+ return token_with_probs;
}
std::vector<float> getEmbedding() {
@@ -713,26 +726,10 @@ static json format_embedding_response(llama_server_context & llama) {
};
}
-static json format_final_response(llama_server_context & llama, const std::string & content, const std::vector<completion_string_output> & probs) {
+static json format_final_response(llama_server_context & llama, const std::string & content, const std::vector<completion_token_output> & probs) {
- json completion_probabilities_json = json::array();
- for (const auto & prob : probs) {
- json probs_for_token = json::array();
- for (const auto & p : prob.probs) {
- probs_for_token.push_back(json {
- { "tok_str", p.tok_str },
- { "prob", p.prob },
- });
- }
- completion_probabilities_json.push_back(json {
- {"content", prob.tok_str},
- {"probs", probs_for_token},
- });
- }
-
- return json {
+ json res = json {
{ "content", content },
- { "completion_probabilities", completion_probabilities_json},
{ "stop", true },
{ "model", llama.params.model_alias },
{ "tokens_predicted", llama.num_tokens_predicted },
@@ -743,25 +740,25 @@ static json format_final_response(llama_server_context & llama, const std::strin
{ "stopped_word", llama.stopped_word },
{ "stopped_limit", llama.stopped_limit },
{ "stopping_word", llama.stopping_word },
- };
+ }
+
+ if (llama.params.n_probs > 0) {
+ json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs);
+ res["completion_probabilities"] = completion_probabilities_json;
+ }
+
+ return res;
}
-static json format_partial_response(const std::string & content, const completion_string_output & probs) {
+static json format_partial_response(llama_server_context & llama, const std::string & content, const std::vector<completion_token_output> & probs) {
json res = json {
{ "content", content },
{ "stop", false },
};
- // iterate through probs.probs, and add to res
- json probs_json = json::array();
- for (const auto & prob : probs.probs) {
- probs_json.push_back(json {
- { "tok_str", prob.tok_str },
- { "prob", prob.prob },
- });
- }
- if (probs.probs.size() > 0) {
- res["probs"] = probs_json;
+ if (llama.params.n_probs > 0) {
+ json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs);
+ res["completion_probabilities"] = completion_probabilities_json;
}
return res;
@@ -897,8 +894,8 @@ int main(int argc, char ** argv) {
size_t stop_pos = std::string::npos;
while (llama.has_next_token) {
- const completion_string_output token_text_with_probs = llama.doCompletion();
- const std::string token_text = token_text_with_probs.tok_str;
+ const completion_token_output token_with_probs = llama.doCompletion();
+ const std::string token_text = llama_token_to_str(llama.ctx, token_with_probs.tok);
stop_pos = llama.findStoppingStrings(llama.generated_text,
token_text.size(), STOP_FULL);
@@ -912,7 +909,7 @@ int main(int argc, char ** argv) {
llama.generated_text.end());
}
- const json data = format_final_response(llama, llama.generated_text, llama.generated_text_probs);
+ const json data = format_final_response(llama, llama.generated_text, llama.generated_token_probs);
llama_print_timings(llama.ctx);
@@ -921,9 +918,11 @@ int main(int argc, char ** argv) {
} else {
const auto chunked_content_provider = [&](size_t, DataSink & sink) {
size_t sent_count = 0;
+ size_t sent_token_probs_index = 0;
while (llama.has_next_token) {
- const completion_string_output token_text_with_probs = llama.doCompletion();
+ const completion_token_output token_with_probs = llama.doCompletion();
+ const std::string token_text = llama_token_to_str(llama.ctx, token_with_probs.tok);
if (llama.multibyte_pending > 0) {
continue;
}
@@ -932,24 +931,36 @@ int main(int argc, char ** argv) {
const std::string str_test = llama.generated_text.substr(pos);
size_t stop_pos =
- llama.findStoppingStrings(str_test, token_text_with_probs.tok_str.size(), STOP_FULL);
+ llama.findStoppingStrings(str_test, token_text.size(), STOP_FULL);
if (stop_pos != std::string::npos) {
llama.generated_text.erase(
llama.generated_text.begin() + pos + stop_pos,
llama.generated_text.end());
pos = std::min(sent_count, llama.generated_text.size());
} else {
- stop_pos = llama.findStoppingStrings(str_test, token_text_with_probs.tok_str.size(),
+ stop_pos = llama.findStoppingStrings(str_test, token_text.size(),
STOP_PARTIAL);
}
const std::string to_send = llama.generated_text.substr(pos, stop_pos);
sent_count += to_send.size();
+ std::vector<completion_token_output> probs_output = {};
+
+ if (llama.params.n_probs > 0) {
+ const std::vector<llama_token> to_send_toks = llama_tokenize(llama.ctx, to_send, false);
+ size_t probs_pos = std::min(sent_token_probs_index, llama.generated_token_probs.size());
+ size_t probs_stop_pos = std::min(sent_token_probs_index + to_send_toks.size(), llama.generated_token_probs.size());
+ if (probs_pos < probs_stop_pos) {
+ probs_output = std::vector<completion_token_output>(llama.generated_token_probs.begin() + probs_pos, llama.generated_token_probs.begin() + probs_stop_pos);
+ }
+ sent_token_probs_index = probs_stop_pos;
+ }
+
const json data = llama.has_next_token
- ? format_partial_response(to_send, token_text_with_probs)
+ ? format_partial_response(llama, to_send, probs_output)
// Generation is done, send extra information.
- : format_final_response(llama, to_send, {token_text_with_probs});
+ : format_final_response(llama, to_send, probs_output);
const std::string str =
"data: " +
From 7b93b248efa828129b14353a2f3a9f0a790b6014 Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Thu, 22 Jun 2023 21:59:12 -0700
Subject: [PATCH 04/10] server: fix some beginner mistakes
---
examples/server/server.cpp | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index d9af1308e3d8a..ea484560eb996 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -99,15 +99,19 @@ static void server_log(const char * level, const char * function, int line,
// format incomplete utf-8 multibyte character for output
static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) {
- const std::string out = token == -1 ? "" : llama_token_to_str(ctx, token);
- if (out[0] > 127) {
- out = "byte: \\x" + std::format("{:x}", out[0]);
+ std::string out = token == -1 ? "" : llama_token_to_str(ctx, token);
+ // if first bit is 1, meaning it's a partial character
+ if ((out[0] & 0x80) == 0x80) {
+ std::stringstream ss;
+ ss<< std::hex << (out[0] & 0xff);
+ std::string res ( ss.str() );
+ out = "byte: \\x" + res;
}
return out;
}
// convert a vector of completion_token_output to json
-static json probs_vector_to_json(const llama_context * ctx, const vector<completion_token_output> probs) {
+static json probs_vector_to_json(const llama_context * ctx, const std::vector<completion_token_output> probs) {
json out = json::array();
for (const auto & prob : probs) {
json probs_for_token = json::array();
@@ -740,7 +744,7 @@ static json format_final_response(llama_server_context & llama, const std::strin
{ "stopped_word", llama.stopped_word },
{ "stopped_limit", llama.stopped_limit },
{ "stopping_word", llama.stopping_word },
- }
+ };
if (llama.params.n_probs > 0) {
json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs);
From 02c96a4cbbfde81f8f0ce67ec8275f1a7990b573 Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Sat, 24 Jun 2023 07:54:26 -0700
Subject: [PATCH 05/10] server: remove trailling white space
---
examples/server/server.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index ea484560eb996..4ac8d6fbdadb5 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -103,7 +103,7 @@ static std::string tokens_to_output_formatted_string(const llama_context * ctx,
// if first bit is 1, meaning it's a partial character
if ((out[0] & 0x80) == 0x80) {
std::stringstream ss;
- ss<< std::hex << (out[0] & 0xff);
+ ss<< std::hex << (out[0] & 0xff);
std::string res ( ss.str() );
out = "byte: \\x" + res;
}
From e815b695792aa8b3e744a8eb825549590aa3e7f3 Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Sun, 25 Jun 2023 14:15:14 -0700
Subject: [PATCH 06/10] server: remove n_probs upper limit of 5
---
examples/server/server.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 4ac8d6fbdadb5..3d076060ca747 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -101,7 +101,7 @@ static void server_log(const char * level, const char * function, int line,
static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) {
std::string out = token == -1 ? "" : llama_token_to_str(ctx, token);
// if first bit is 1, meaning it's a partial character
- if ((out[0] & 0x80) == 0x80) {
+ if (out.size() > 0 && (out[0] & 0x80) == 0x80) {
std::stringstream ss;
ss<< std::hex << (out[0] & 0xff);
std::string res ( ss.str() );
@@ -377,8 +377,8 @@ struct llama_server_context {
result.tok = llama_sample_token(ctx, &candidates_p);
}
}
- // Add maximum of 5 most probable tokens to the result
- for (size_t i = 0; i < std::min(candidates_p.size, std::min((size_t) n_probs, size_t(5))); ++i) {
+
+ for (size_t i = 0; i < std::min(candidates_p.size, (size_t) n_probs); ++i) {
result.probs.push_back({candidates_p.data[i].id, candidates_p.data[i].p});
}
last_n_tokens.erase(last_n_tokens.begin());
From c9e6642cf7eae4223e96f224a6c50d0712f6a2c2 Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Sun, 25 Jun 2023 16:29:34 -0700
Subject: [PATCH 07/10] server: handle probs output when temp=0; handle final
response probs output
---
examples/server/server.cpp | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 3d076060ca747..e349489f80f67 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -357,6 +357,9 @@ struct llama_server_context {
if (temp <= 0) {
// Greedy sampling
result.tok = llama_sample_token_greedy(ctx, &candidates_p);
+ if (n_probs > 0) {
+ llama_sample_softmax(ctx, &candidates_p);
+ }
} else {
if (mirostat == 1) {
static float mirostat_mu = 2.0f * mirostat_tau;
@@ -369,10 +372,11 @@ struct llama_server_context {
result.tok = llama_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
} else {
// Temperature sampling
- llama_sample_tail_free(ctx, &candidates_p, tfs_z, 1);
- llama_sample_typical(ctx, &candidates_p, typical_p, 1);
- llama_sample_top_p(ctx, &candidates_p, top_p, 1);
- llama_sample_top_k(ctx, &candidates_p, top_k, 1);
+ size_t min_keep = std::max(1, n_probs);
+ llama_sample_tail_free(ctx, &candidates_p, tfs_z, min_keep);
+ llama_sample_typical(ctx, &candidates_p, typical_p, min_keep);
+ llama_sample_top_p(ctx, &candidates_p, top_p, min_keep);
+ llama_sample_top_k(ctx, &candidates_p, top_k, min_keep);
llama_sample_temperature(ctx, &candidates_p, temp);
result.tok = llama_sample_token(ctx, &candidates_p);
}
@@ -747,8 +751,7 @@ static json format_final_response(llama_server_context & llama, const std::strin
};
if (llama.params.n_probs > 0) {
- json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs);
- res["completion_probabilities"] = completion_probabilities_json;
+ res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs);
}
return res;
@@ -761,8 +764,7 @@ static json format_partial_response(llama_server_context & llama, const std::str
};
if (llama.params.n_probs > 0) {
- json completion_probabilities_json = probs_vector_to_json(llama.ctx, probs);
- res["completion_probabilities"] = completion_probabilities_json;
+ res["completion_probabilities"] = probs_vector_to_json(llama.ctx, probs);
}
return res;
@@ -964,7 +966,7 @@ int main(int argc, char ** argv) {
const json data = llama.has_next_token
? format_partial_response(llama, to_send, probs_output)
// Generation is done, send extra information.
- : format_final_response(llama, to_send, probs_output);
+ : format_final_response(llama, to_send, llama.generated_token_probs);
const std::string str =
"data: " +
From bc88fece87460e0121942a314aa4de71b97bad32 Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Mon, 26 Jun 2023 18:11:27 -0700
Subject: [PATCH 08/10] server: fix llama_sample_top_k order
---
examples/server/server.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index c0ed4b016c64b..e4ddbe9865506 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -378,10 +378,10 @@ struct llama_server_context {
} else {
// Temperature sampling
size_t min_keep = std::max(1, n_probs);
+ llama_sample_top_k(ctx, &candidates_p, top_k, min_keep);
llama_sample_tail_free(ctx, &candidates_p, tfs_z, min_keep);
llama_sample_typical(ctx, &candidates_p, typical_p, min_keep);
llama_sample_top_p(ctx, &candidates_p, top_p, min_keep);
- llama_sample_top_k(ctx, &candidates_p, top_k, min_keep);
llama_sample_temperature(ctx, &candidates_p, temp);
result.tok = llama_sample_token(ctx, &candidates_p);
}
From 1a70a80369d64e61d658e82451634e932caddc29 Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Sun, 2 Jul 2023 08:00:13 +0800
Subject: [PATCH 09/10] examples/common.h: put all bool variables in gpt_params
together
---
examples/common.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/common.h b/examples/common.h
index 951511f8a5746..03312365eb27c 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -31,7 +31,6 @@ struct gpt_params {
int32_t n_gpu_layers = 0; // number of layers to store in VRAM
int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors
float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs
- bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance
int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens.
// sampling parameters
@@ -60,6 +59,7 @@ struct gpt_params {
std::string lora_adapter = ""; // lora adapter path
std::string lora_base = ""; // base model path for the lora adapter
+ bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance
bool memory_f16 = true; // use f16 instead of f32 for memory kv
bool random_prompt = false; // do not randomize prompt if none provided
bool use_color = false; // use color to distinguish generations and inputs
From 71f829678aec335dac97cb44f255e5eb26544cf8 Mon Sep 17 00:00:00 2001
From: "Wang Haoran(Robin)" <notrealrobin@gmail.com>
Date: Sun, 2 Jul 2023 08:01:19 +0800
Subject: [PATCH 10/10] examples/common.h: put all bool variables in gpt_params
together
---
examples/common.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/common.h b/examples/common.h
index 03312365eb27c..96f2228f8677b 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -59,7 +59,7 @@ struct gpt_params {
std::string lora_adapter = ""; // lora adapter path
std::string lora_base = ""; // base model path for the lora adapter
- bool low_vram = 0; // if true, reduce VRAM usage at the cost of performance
+ bool low_vram = false; // if true, reduce VRAM usage at the cost of performance
bool memory_f16 = true; // use f16 instead of f32 for memory kv
bool random_prompt = false; // do not randomize prompt if none provided
bool use_color = false; // use color to distinguish generations and inputs