vendor/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

server: classify mmproj GGUFs as projector layers (#16472)

Browse Source
This commit is contained in:
Jeffrey Morgan
2026-06-03 12:59:34 -07:00
committed by GitHub
parent ad8cda255d
commit 5f56a289b3
2 changed files with 53 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
server/create.go
View File

@@ -1232,8 +1232,7 @@ func ggufLayersWithMediaType(digest, sourceName, mediaType string, fn func(resp
mediaType = "application/vnd.ollama.image.model"
if f.KV().Kind() == "adapter" {
mediaType = "application/vnd.ollama.image.adapter"
} else if (f.KV().Uint("block_count") == 0 && f.KV().Uint("vision.block_count") > 0) || f.KV().Kind() == "projector" {
// if a model has vision.block_count but not block_count, it is a standalone vision model
} else if isProjectorGGUF(f.KV()) {
mediaType = "application/vnd.ollama.image.projector"
}
}
@@ -1249,6 +1248,20 @@ func ggufLayersWithMediaType(digest, sourceName, mediaType string, fn func(resp
return layers, nil
}
func isProjectorGGUF(kv ggml.KV) bool {
switch kv.Kind() {
case "projector", "mmproj":
return true
}
// If a model has vision.block_count but not block_count, it is a standalone vision model.
if kv.Uint("block_count") == 0 && kv.Uint("vision.block_count") > 0 {
return true
}
return kv.Architecture() == "clip" && kv.Uint("block_count") == 0 && (kv.Bool("has_vision_encoder") || kv.Bool("has_audio_encoder"))
}
func removeLayer(layers []manifest.Layer, mediatype string) []manifest.Layer {
return slices.DeleteFunc(layers, func(layer manifest.Layer) bool {
if layer.MediaType != mediatype {

38
server/routes_create_test.go
View File

@@ -465,6 +465,44 @@ func TestCreateModelAddsDefaultLlavaProjectorType(t *testing.T) {
}
}
func TestGGUFLayersClassifiesMMProjAsProjector(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
_, digest := createBinFile(t, map[string]any{
"general.architecture": "clip",
"general.type": "mmproj",
"clip.has_vision_encoder": true,
"clip.vision.block_count": uint32(0),
"clip.vision.projector_type": "gemma4uv",
"clip.has_audio_encoder": true,
"clip.audio.block_count": uint32(0),
"clip.audio.projector_type": "gemma4ua",
"clip.vision.projection_dim": uint32(3840),
"clip.vision.embedding_length": uint32(3840),
"clip.audio.projection_dim": uint32(3840),
"clip.audio.embedding_length": uint32(3840),
"clip.vision.feed_forward_length": uint32(0),
"clip.audio.feed_forward_length": uint32(0),
}, []*ggml.Tensor{
{
Name: "mm.input_projection.weight",
Kind: uint32(ggml.TensorTypeF32),
Shape: []uint64{1, 1},
WriterTo: bytes.NewReader(make([]byte, 4)),
},
})
layers, err := ggufLayers(digest, "mmproj-gemma-4-12B-it-bf16.gguf", func(api.ProgressResponse) {})
if err != nil {
t.Fatal(err)
}
if len(layers) != 1 {
t.Fatalf("layers = %d, want 1", len(layers))
}
if got := layers[0].MediaType; got != "application/vnd.ollama.image.projector" {
t.Fatalf("media type = %q, want projector", got)
}
}
func TestCreateModelQuantizeRestoresEmbeddedCompatibilityTensors(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
oldRun := runLlamaQuantize