Fix unicode in grammars (fixes #2501) (#2553)

* Fix unicode in grammars (fixes #2501)

* add more comments

* fix test-llama-grammar

tests/test-llama-grammar.cpp

@@ -199,7 +199,7 @@ int main()
         uint32_t *cp = new uint32_t[2]; // dynamically allocate memory for code_point
         cp[0] = 37 + i;
         cp[1] = 0;
-        next_candidates[i] = {i, cp};
+        next_candidates[i] = {i, cp, {}};
     }
 
     std::vector<std::vector<std::pair<uint32_t, uint16_t>>> expected_reject = {