millaguie/llama.cpp
1
0 Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

llama : rename jina tokenizers to v2 (#7249)

Browse Source 
* refactor: rename jina tokenizers to v2

* refactor: keep refactoring non-breaking
This commit is contained in:
Joan Fontanals authored and GitHub committed 2024-05-13 11:35:14 +03:00
1 parent b1f8af1886
commit 9aa672490c
3 files changed
+9 -7
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

llama.cpp
+3 -1
View File
@@ -4424,7 +4424,9 @@ static void llm_load_vocab(
} else if (
tokenizer_pre == "gpt-2" ||
tokenizer_pre == "jina-es" ||
tokenizer_pre == "jina-de") {
tokenizer_pre == "jina-de" ||
tokenizer_pre == "jina-v2-es" ||
tokenizer_pre == "jina-v2-de") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;
} else if (
tokenizer_pre == "refact") {