llama : add pre-tokenizer regexes for BLOOM and gpt3-finnish (#8850)

This commit is contained in:
Esko Toivonen authored and GitHub committed 2024-08-15 10:17:12 +03:00
1 parent d5492f0525
commit 6bda7ce6c3
5 files changed
+19 -1

No files matched your search

+2
View File
@@ -93,6 +93,8 @@ extern "C" {
LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20,
LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,
LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,
LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,
LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,
};
enum llama_rope_type {