vendor/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llama: handle Gemma 4 and LFM2 BOS override in llama server (#16367)

Browse Source
This commit is contained in:
Jeffrey Morgan
2026-05-31 14:05:39 -07:00
committed by GitHub
parent 11be8f6ac8
commit be7de10c41
4 changed files with 74 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
llm/llama_server.go
View File

@@ -241,7 +241,21 @@ func (s *llamaServerRunner) tokenizerAddsBOS() bool {
return false
}
return s.ggml.KV().Bool("tokenizer.ggml.add_bos_token")
kv := s.ggml.KV()
if kv.String("tokenizer.ggml.pre") == "lfm2" {
return true
}
// llama.cpp forces add_bos on for Gemma4 at load time, even for GGUFs
// whose tokenizer.ggml.add_bos_token metadata is explicitly false. Some
// GGUFs omit tokenizer.ggml.pre and are still treated as Gemma4 from
// tokenizer.ggml.model.
if kv.String("tokenizer.ggml.pre") == "gemma4" || kv.String("tokenizer.ggml.model") == "gemma4" {
return true
}
return kv.Bool("tokenizer.ggml.add_bos_token")
}
func (s *llamaServerRunner) completionPromptForRequest(ctx context.Context, req CompletionRequest) (any, error) {

58
llm/llama_server_test.go
View File

@@ -981,6 +981,7 @@ func TestLlamaServerCompletionBOSOwnership(t *testing.T) {
name string
leadingBOS string
tokenizerAddsBOS bool
ggmlKV ggml.KV
prompt string
wantPrompt string
}{
@@ -1020,6 +1021,59 @@ func TestLlamaServerCompletionBOSOwnership(t *testing.T) {
prompt: "<bos>hello<bos>",
wantPrompt: "hello<bos>",
},
{
name: "gemma4 llama.cpp runtime bos override",
leadingBOS: "<bos>",
ggmlKV: ggml.KV{
"general.architecture": "gemma4",
"tokenizer.ggml.pre": "gemma4",
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.bos_token_id": uint32(2),
"tokenizer.ggml.eos_token_id": uint32(1),
"tokenizer.ggml.unknown_token_id": uint32(0),
},
prompt: "<bos><|turn>user\nhello<turn|>\n<|turn>model\n",
wantPrompt: "<|turn>user\nhello<turn|>\n<|turn>model\n",
},
{
name: "gemma4 model runtime bos override",
leadingBOS: "<bos>",
ggmlKV: ggml.KV{
"general.architecture": "gemma4",
"tokenizer.ggml.model": "gemma4",
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.bos_token_id": uint32(2),
"tokenizer.ggml.eos_token_id": uint32(1),
"tokenizer.ggml.unknown_token_id": uint32(0),
},
prompt: "<bos><|turn>user\nhello<turn|>\n<|turn>model\n",
wantPrompt: "<|turn>user\nhello<turn|>\n<|turn>model\n",
},
{
name: "lfm2 strips renderer bos",
leadingBOS: "<|startoftext|>",
ggmlKV: ggml.KV{
"general.architecture": "lfm2",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "lfm2",
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.bos_token_id": uint32(0),
},
prompt: "<|startoftext|><|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",
wantPrompt: "<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",
},
{
name: "lfm2 missing bos metadata uses llama.cpp default",
leadingBOS: "<|startoftext|>",
ggmlKV: ggml.KV{
"general.architecture": "lfm2",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "lfm2",
"tokenizer.ggml.bos_token_id": uint32(0),
},
prompt: "<|startoftext|><|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",
wantPrompt: "<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n",
},
}
for _, tt := range tests {
@@ -1053,7 +1107,9 @@ func TestLlamaServerCompletionBOSOwnership(t *testing.T) {
sem: semaphore.NewWeighted(1),
options: api.Options{Runner: api.Runner{NumCtx: 2048}},
}
if tt.tokenizerAddsBOS {
if tt.ggmlKV != nil {
runner.ggml = loadTestGGML(t, tt.ggmlKV)
} else if tt.tokenizerAddsBOS {
runner.ggml = loadTestGGML(t, ggml.KV{
"general.architecture": "gemma3",
"tokenizer.ggml.add_bos_token": true,