remove vocab hack

ngxson · ngxson · commit a543ddfd02e2 · 2025-08-01T23:56:32.000+02:00
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
@@ -2330,13 +2330,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             }
         }
 
-        // @ngxson : quick hack for gpt-oss
-        for (const auto & t : token_to_id) {
-            if (t.first == "<|channel|>" || t.first == "<|message|>") {
-                id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_NORMAL;
-            }
-        }
-
         // sanity checks
         if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {
             special_eog_ids.insert(special_eos_id);

Original file line number	Diff line number	Diff line change
`@@ -2330,13 +2330,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {`
`2330`	`2330`	`}`
`2331`	`2331`	`}`
`2332`	`2332`
`2333`		`- // @ngxson : quick hack for gpt-oss`
`2334`		`- for (const auto & t : token_to_id) {`
`2335`		`- if (t.first == "<\|channel\|>" \|\| t.first == "<\|message\|>") {`
`2336`		`- id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_NORMAL;`
`2337`		`- }`
`2338`		`- }`
`2339`		`-`
`2340`	`2333`	`// sanity checks`
`2341`	`2334`	`if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {`
`2342`	`2335`	`special_eog_ids.insert(special_eos_id);`