llama : add pre-tokenizer regexes for BLOOM and gpt3-finnish (#8850)
This commit is contained in:
1 parent
d5492f0525
commit
6bda7ce6c3
5 files changed
+19
-1
No files matched your search
@@ -93,6 +93,8 @@ extern "C" {
|
||||
LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20,
|
||||
LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,
|
||||
LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,
|
||||
LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,
|
||||
LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,
|
||||
};
|
||||
|
||||
enum llama_rope_type {
|
||||
|
||||
Reference in new issue
Block a user