imatrix : Add --parse-special for enabling parsing of special tokens in imatrix calculation (#13389)

* Add --parse-special for enabling parsing of special tokens in imatrix calculation

* whitespace
This commit is contained in:
Bartowski authored and GitHub committed 2025-05-09 11:53:58 +02:00
1 parent 0527771dd8
commit efb8b47eda
3 files changed
+11 -2

No files matched your search

+1
View File
@@ -409,6 +409,7 @@ struct common_params {
bool process_output = false; // collect data for the output tensor
bool compute_ppl = true; // whether to compute perplexity
bool parse_special = false; // whether to parse special tokens during imatrix tokenization
// cvector-generator params
int n_pca_batch = 100;