Skip to content

Commit 971f245

Browse files
authored
llama : recognize IBM Granite 3.3 FIM tokens (#12988)
The Granite's FIM tokens are very similar to Qwen's; it's just that they use underscore instead of a dash. So <fim_middle> for example instead of <fim-middle>. Opening up tokenizer_config.json in ibm-granite/granite-3.3-8b-base shows: ``` "<fim_prefix>", "<fim_middle>", "<fim_suffix>", "<fim_pad>", ... "<reponame>", ```
1 parent 12b1750 commit 971f245

File tree

1 file changed

+5
-0
lines changed

1 file changed

+5
-0
lines changed

src/llama-vocab.cpp

+5
Original file line numberDiff line numberDiff line change
@@ -1841,6 +1841,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
18411841
if (false
18421842
|| t.first == "<|fim_prefix|>" // Qwen
18431843
|| t.first == "<fim-prefix>"
1844+
|| t.first == "<fim_prefix>" // Granite
18441845
|| t.first == "<|fim▁begin|>" // DeepSeek
18451846
|| t.first == "<PRE>"
18461847
|| t.first == "▁<PRE>" // CodeLlama
@@ -1859,6 +1860,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
18591860
if (false
18601861
|| t.first == "<|fim_suffix|>" // Qwen
18611862
|| t.first == "<fim-suffix>"
1863+
|| t.first == "<fim_suffix>" // Granite
18621864
|| t.first == "<|fim▁hole|>" // DeepSeek
18631865
|| t.first == "<SUF>"
18641866
|| t.first == "▁<SUF>" // CodeLlama
@@ -1877,6 +1879,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
18771879
if (false
18781880
|| t.first == "<|fim_middle|>" // Qwen
18791881
|| t.first == "<fim-middle>"
1882+
|| t.first == "<fim_middle>" // Granite
18801883
|| t.first == "<|fim▁end|>" // DeepSeek
18811884
|| t.first == "<MID>"
18821885
|| t.first == "▁<MID>" // CodeLlama
@@ -1895,6 +1898,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
18951898
if (false
18961899
|| t.first == "<|fim_pad|>" // Qwen
18971900
|| t.first == "<fim-pad>"
1901+
|| t.first == "<fim_pad>" // Granite
18981902
|| t.first == "<PAD>"
18991903
) {
19001904
special_fim_pad_id = t.second;
@@ -1913,6 +1917,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
19131917
|| t.first == "<|repo_name|>"
19141918
|| t.first == "<fim-repo>"
19151919
|| t.first == "<REPO>"
1920+
|| t.first == "<reponame>" // Granite
19161921
) {
19171922
special_fim_rep_id = t.second;
19181923
if ((id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {

0 commit comments

Comments
 (0)