From 5f56a289b3e5284fba1270c0f654397cc5721c99 Mon Sep 17 00:00:00 2001
From: Jeffrey Morgan <jmorganca@gmail.com>
Date: Wed, 3 Jun 2026 12:59:34 -0700
Subject: [PATCH] server: classify mmproj GGUFs as projector layers (#16472)

---
 server/create.go             | 17 ++++++++++++++--
 server/routes_create_test.go | 38 ++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/server/create.go b/server/create.go
index 122a5805..fb07e725 100644
--- a/server/create.go
+++ b/server/create.go
@@ -1232,8 +1232,7 @@ func ggufLayersWithMediaType(digest, sourceName, mediaType string, fn func(resp
 		mediaType = "application/vnd.ollama.image.model"
 		if f.KV().Kind() == "adapter" {
 			mediaType = "application/vnd.ollama.image.adapter"
-		} else if (f.KV().Uint("block_count") == 0 && f.KV().Uint("vision.block_count") > 0) || f.KV().Kind() == "projector" {
-			// if a model has vision.block_count but not block_count, it is a standalone vision model
+		} else if isProjectorGGUF(f.KV()) {
 			mediaType = "application/vnd.ollama.image.projector"
 		}
 	}
@@ -1249,6 +1248,20 @@ func ggufLayersWithMediaType(digest, sourceName, mediaType string, fn func(resp
 	return layers, nil
 }
 
+func isProjectorGGUF(kv ggml.KV) bool {
+	switch kv.Kind() {
+	case "projector", "mmproj":
+		return true
+	}
+
+	// If a model has vision.block_count but not block_count, it is a standalone vision model.
+	if kv.Uint("block_count") == 0 && kv.Uint("vision.block_count") > 0 {
+		return true
+	}
+
+	return kv.Architecture() == "clip" && kv.Uint("block_count") == 0 && (kv.Bool("has_vision_encoder") || kv.Bool("has_audio_encoder"))
+}
+
 func removeLayer(layers []manifest.Layer, mediatype string) []manifest.Layer {
 	return slices.DeleteFunc(layers, func(layer manifest.Layer) bool {
 		if layer.MediaType != mediatype {
diff --git a/server/routes_create_test.go b/server/routes_create_test.go
index 524da0e5..1a6205b7 100644
--- a/server/routes_create_test.go
+++ b/server/routes_create_test.go
@@ -465,6 +465,44 @@ func TestCreateModelAddsDefaultLlavaProjectorType(t *testing.T) {
 	}
 }
 
+func TestGGUFLayersClassifiesMMProjAsProjector(t *testing.T) {
+	t.Setenv("OLLAMA_MODELS", t.TempDir())
+
+	_, digest := createBinFile(t, map[string]any{
+		"general.architecture":            "clip",
+		"general.type":                    "mmproj",
+		"clip.has_vision_encoder":         true,
+		"clip.vision.block_count":         uint32(0),
+		"clip.vision.projector_type":      "gemma4uv",
+		"clip.has_audio_encoder":          true,
+		"clip.audio.block_count":          uint32(0),
+		"clip.audio.projector_type":       "gemma4ua",
+		"clip.vision.projection_dim":      uint32(3840),
+		"clip.vision.embedding_length":    uint32(3840),
+		"clip.audio.projection_dim":       uint32(3840),
+		"clip.audio.embedding_length":     uint32(3840),
+		"clip.vision.feed_forward_length": uint32(0),
+		"clip.audio.feed_forward_length":  uint32(0),
+	}, []*ggml.Tensor{
+		{
+			Name:     "mm.input_projection.weight",
+			Kind:     uint32(ggml.TensorTypeF32),
+			Shape:    []uint64{1, 1},
+			WriterTo: bytes.NewReader(make([]byte, 4)),
+		},
+	})
+	layers, err := ggufLayers(digest, "mmproj-gemma-4-12B-it-bf16.gguf", func(api.ProgressResponse) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(layers) != 1 {
+		t.Fatalf("layers = %d, want 1", len(layers))
+	}
+	if got := layers[0].MediaType; got != "application/vnd.ollama.image.projector" {
+		t.Fatalf("media type = %q, want projector", got)
+	}
+}
+
 func TestCreateModelQuantizeRestoresEmbeddedCompatibilityTensors(t *testing.T) {
 	t.Setenv("OLLAMA_MODELS", t.TempDir())
 	oldRun := runLlamaQuantize