tokenize example: Respect normal add BOS token behavior (#4126)
Allow building with Makefile
This commit is contained in:
1 parent
0b5c3b0457
commit
28a2e6e7d4
2 files changed
+5
-2
No files matched your search
@@ -26,7 +26,7 @@ int main(int argc, char ** argv) {
|
||||
llama_context_params ctx_params = llama_context_default_params();
|
||||
llama_context * ctx = llama_new_context_with_model(model, ctx_params);
|
||||
|
||||
const bool add_bos = true;
|
||||
const bool add_bos = llama_should_add_bos_token(model);
|
||||
|
||||
std::vector<llama_token> tokens;
|
||||
|
||||
|
||||
Reference in new issue
Block a user