Added support for Viking pre-tokenizer (#8135)

Co-authored-by: kustaaya <kustaaya@protonmail.com>
This commit is contained in:
kustaayaandkustaaya authored and GitHub committed 2024-06-27 10:58:54 +02:00
1 parent 911e35bb8b
commit f675b20a3b
4 files changed
+14

No files matched your search

+1
View File
@@ -88,6 +88,7 @@ extern "C" {
LLAMA_VOCAB_PRE_TYPE_DBRX = 13,
LLAMA_VOCAB_PRE_TYPE_SMAUG = 14,
LLAMA_VOCAB_PRE_TYPE_PORO = 15,
LLAMA_VOCAB_PRE_TYPE_VIKING = 16,
};
// note: these values should be synchronized with ggml_rope