vendor/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm: size mmproj offload by projector memory (#16866)

Browse Source 
* llm: size mmproj offload by projector memory

Replace the blanket 10 GiB VRAM cutoff with a projector tensor-size estimate plus backend headroom, while preserving the existing CPU-only, partial text offload, shared-memory GPU, and startup OOM retry gates.

This is a stopgap until fit accounts for mmproj memory directly.

The same limited-vram path appears in the qwen3.5 vision hang report: the logs show --no-mmproj-offload on a 7.5 GiB RTX 5050 with about 6.4 GiB free while llama-server estimates the inline mmproj at about 962 MiB.

Fixes #16496

Fixes #16570

* review comments
This commit is contained in:
Daniel Hiltgen
2026-06-23 13:04:02 -07:00
committed by GitHub
parent 46bc1bcb4c
commit 836507378b
2 changed files with 204 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
llm/llama_server.go
View File

@@ -160,6 +160,7 @@ type llamaServerLaunchConfig struct {
modelPath string
modelArch string
projectors []string
mmprojMemory uint64
modelLayers uint64
adapters []string
opts api.Options
@@ -602,7 +603,10 @@ func appendMainGPUArgs(params []string, opts api.Options) []string {
return append(params, "--split-mode", "none", "--main-gpu", strconv.Itoa(*opts.MainGPU))
}
const limitedMMProjOffloadMemory = 10 << 30
const (
// mmprojOffloadHeadroom leaves 1 GiB for backend buffers beyond projector weights.
mmprojOffloadHeadroom = 1 << 30
)
func appendMMProjArgs(params []string, launch llamaServerLaunchConfig) []string {
if len(launch.projectors) == 0 {
@@ -622,10 +626,10 @@ func (launch llamaServerLaunchConfig) mmprojOffloadDisabled() (bool, string) {
if launch.forceNoMMProjOffload {
return true, "startup-oom-retry"
}
return shouldDisableMMProjOffload(launch.opts, launch.gpus, launch.modelLayers)
return shouldDisableMMProjOffload(launch.opts, launch.gpus, launch.modelLayers, launch.mmprojMemory)
}
func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLayers uint64) (bool, string) {
func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLayers, mmprojMemory uint64) (bool, string) {
if opts.NumGPU == 0 {
return true, "cpu-only"
}
@@ -633,6 +637,8 @@ func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLay
return true, "partial-text-offload"
}
requiredMemory := mmprojMemory + mmprojOffloadHeadroom
for _, gpu := range gpus {
if gpu.Integrated && gpu.Library != "Metal" {
return true, "shared-memory-gpu"
@@ -641,7 +647,7 @@ func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLay
if memory == 0 || (gpu.TotalMemory > 0 && gpu.TotalMemory < memory) {
memory = gpu.TotalMemory
}
if memory > 0 && memory <= limitedMMProjOffloadMemory {
if memory > 0 && memory < requiredMemory {
return true, "limited-vram"
}
}
@@ -649,6 +655,48 @@ func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLay
return false, ""
}
// mmprojMemoryRequirement is a stopgap until fit accounts for mmproj memory directly.
func mmprojMemoryRequirement(modelPath string, f *ggml.GGML, projectors []string) (uint64, error) {
if len(projectors) == 0 {
return 0, nil
}
if projectors[0] == modelPath {
if f == nil {
return 0, errors.New("read inline mmproj metadata: missing model metadata")
}
var size uint64
for _, prefix := range []string{"v.", "mm.", "a."} {
for _, tensor := range f.Tensors().Items(prefix) {
size += tensor.Size()
}
}
if size == 0 {
return 0, errors.New("read inline mmproj metadata: no projector tensors found")
}
return size, nil
}
file, err := os.Open(projectors[0])
if err != nil {
return 0, fmt.Errorf("read mmproj metadata %q: %w", projectors[0], err)
}
defer file.Close()
projector, err := ggml.Decode(file, 1024)
if err != nil {
return 0, fmt.Errorf("read mmproj metadata %q: %w", projectors[0], err)
}
var size uint64
for _, tensor := range projector.Tensors().Items() {
size += tensor.Size()
}
if size == 0 {
return 0, fmt.Errorf("read mmproj metadata %q: no projector tensors found", projectors[0])
}
return size, nil
}
func appendJinjaArgs(params []string, config LlamaServerConfig) []string {
if config.DisableJinja {
// Go-rendered chat paths send already-rendered prompts through completion
@@ -752,6 +800,10 @@ func NewLlamaServerRunner(
compatClipArches[arch] {
projectors = []string{modelPath}
}
mmprojMemory, err := mmprojMemoryRequirement(modelPath, f, projectors)
if err != nil {
return nil, err
}
if config.DraftModelPath == "" && hasMTPDraft(f) {
config.EnableMTP = true
}
@@ -771,19 +823,20 @@ func NewLlamaServerRunner(
serverEnvs["LLAMA_MEDIA_MARKER"] = mediaMarker
launch := llamaServerLaunchConfig{
modelPath: modelPath,
modelArch: arch,
projectors: slices.Clone(projectors),
modelLayers: f.KV().BlockCount() + 1,
adapters: slices.Clone(adapters),
opts: opts,
numParallel: numParallel,
kvCacheType: kvCacheType,
embedding: isEmbedding,
config: config,
gpus: slices.Clone(gpus),
gpuLibs: slices.Clone(gpuLibs),
extraEnvs: cloneStringMap(serverEnvs),
modelPath: modelPath,
modelArch: arch,
projectors: slices.Clone(projectors),
mmprojMemory: mmprojMemory,
modelLayers: f.KV().BlockCount() + 1,
adapters: slices.Clone(adapters),
opts: opts,
numParallel: numParallel,
kvCacheType: kvCacheType,
embedding: isEmbedding,
config: config,
gpus: slices.Clone(gpus),
gpuLibs: slices.Clone(gpuLibs),
extraEnvs: cloneStringMap(serverEnvs),
}
s := &llamaServerRunner{

192
llm/llama_server_test.go
View File

@@ -1,6 +1,7 @@
package llm
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
@@ -1961,13 +1962,14 @@ func TestAppendMMProjArgs(t *testing.T) {
cpuOpts.NumGPU = 0
tests := []struct {
name string
projectors []string
opts api.Options
gpus []ml.DeviceInfo
modelLayers uint64
retry bool
want []string
name string
projectors []string
opts api.Options
gpus []ml.DeviceInfo
mmprojMemory uint64
modelLayers uint64
retry bool
want []string
}{
{
name: "no projector leaves args unchanged",
@@ -1975,69 +1977,86 @@ func TestAppendMMProjArgs(t *testing.T) {
want: []string{"base"},
},
{
name: "large discrete gpu keeps projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf"},
name: "large discrete gpu keeps projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
mmprojMemory: 933 << 20,
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf"},
},
{
name: "small discrete gpu disables projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, TotalMemory: 8 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
name: "small discrete gpu keeps projector offload when projector fits",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "ROCm"}, FreeMemory: 7900 << 20, TotalMemory: 8 << 30}},
mmprojMemory: 933 << 20,
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf"},
},
{
name: "integrated rocm gpu disables projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "ROCm"}, Integrated: true, FreeMemory: 32 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
name: "tight discrete gpu disables projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 1500 << 20, TotalMemory: 8 << 30}},
mmprojMemory: 933 << 20,
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
{
name: "integrated metal gpu keeps projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "Metal"}, Integrated: true, FreeMemory: 32 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf"},
name: "integrated rocm gpu disables projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "ROCm"}, Integrated: true, FreeMemory: 32 << 30}},
mmprojMemory: 933 << 20,
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
{
name: "cpu only request disables projector offload",
projectors: []string{"model.gguf"},
opts: cpuOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
name: "integrated metal gpu keeps projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "Metal"}, Integrated: true, FreeMemory: 32 << 30}},
mmprojMemory: 933 << 20,
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf"},
},
{
name: "partial text offload disables projector offload",
projectors: []string{"model.gguf"},
opts: partialOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
name: "cpu only request disables projector offload",
projectors: []string{"model.gguf"},
opts: cpuOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
mmprojMemory: 933 << 20,
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
{
name: "explicit full text offload keeps projector offload",
projectors: []string{"model.gguf"},
opts: fullOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf"},
name: "partial text offload disables projector offload",
projectors: []string{"model.gguf"},
opts: partialOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
mmprojMemory: 933 << 20,
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
{
name: "startup oom retry disables projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
modelLayers: 81,
retry: true,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
name: "explicit full text offload keeps projector offload",
projectors: []string{"model.gguf"},
opts: fullOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
mmprojMemory: 933 << 20,
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf"},
},
{
name: "startup oom retry disables projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 24 << 30}},
mmprojMemory: 933 << 20,
modelLayers: 81,
retry: true,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
}
@@ -2046,6 +2065,7 @@ func TestAppendMMProjArgs(t *testing.T) {
got := appendMMProjArgs([]string{"base"}, llamaServerLaunchConfig{
modelPath: "model.gguf",
projectors: tt.projectors,
mmprojMemory: tt.mmprojMemory,
opts: tt.opts,
gpus: tt.gpus,
modelLayers: tt.modelLayers,
@@ -2058,6 +2078,45 @@ func TestAppendMMProjArgs(t *testing.T) {
}
}
func TestMMProjMemoryRequirement(t *testing.T) {
if got, err := mmprojMemoryRequirement("model.gguf", nil, nil); err != nil || got != 0 {
t.Fatalf("no projector memory = %d, %v; want 0, nil", got, err)
}
modelPath, model := writeTestGGML(t, ggml.KV{"general.architecture": "gemma3"}, []*ggml.Tensor{
testGGMLTensor("blk.0.attn_q.weight", ggml.TensorTypeF32, []uint64{4}),
testGGMLTensor("v.patch_embd.weight", ggml.TensorTypeF16, []uint64{16}),
testGGMLTensor("mm.0.weight", ggml.TensorTypeF32, []uint64{8}),
testGGMLTensor("a.encoder.weight", ggml.TensorTypeF32, []uint64{2}),
})
wantInline := uint64(16*2 + 8*4 + 2*4)
if got, err := mmprojMemoryRequirement(modelPath, model, []string{modelPath}); err != nil || got != wantInline {
t.Fatalf("inline mmproj memory = %d, %v; want %d, nil", got, err, wantInline)
}
projectorPath, _ := writeTestGGML(t, ggml.KV{"general.architecture": "clip"}, []*ggml.Tensor{
testGGMLTensor("vision.weight", ggml.TensorTypeF16, []uint64{32}),
testGGMLTensor("audio.weight", ggml.TensorTypeF32, []uint64{4}),
})
wantProjector := uint64(32*2 + 4*4)
if got, err := mmprojMemoryRequirement(modelPath, model, []string{projectorPath}); err != nil || got != wantProjector {
t.Fatalf("projector file memory = %d, %v; want %d, nil", got, err, wantProjector)
}
if _, err := mmprojMemoryRequirement(modelPath, nil, []string{modelPath}); err == nil {
t.Fatal("inline mmproj with nil model error = nil, want error")
}
if _, err := mmprojMemoryRequirement(modelPath, model, []string{filepath.Join(t.TempDir(), "missing.gguf")}); err == nil {
t.Fatal("missing projector error = nil, want error")
}
emptyProjectorPath, _ := writeTestGGML(t, ggml.KV{"general.architecture": "clip"}, nil)
if _, err := mmprojMemoryRequirement(modelPath, model, []string{emptyProjectorPath}); err == nil {
t.Fatal("empty projector error = nil, want error")
}
}
func TestAppendJinjaArgs(t *testing.T) {
tests := []struct {
name string
@@ -3145,11 +3204,18 @@ func TestFindLlamaServer(t *testing.T) {
func loadTestGGML(t *testing.T, kv ggml.KV) *ggml.GGML {
t.Helper()
_, model := writeTestGGML(t, kv, nil)
return model
}
func writeTestGGML(t *testing.T, kv ggml.KV, tensors []*ggml.Tensor) (string, *ggml.GGML) {
t.Helper()
f, err := os.CreateTemp(t.TempDir(), "*.gguf")
if err != nil {
t.Fatal(err)
}
if err := ggml.WriteGGUF(f, kv, nil); err != nil {
if err := ggml.WriteGGUF(f, kv, tensors); err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
@@ -3160,7 +3226,17 @@ func loadTestGGML(t *testing.T, kv ggml.KV) *ggml.GGML {
if err != nil {
t.Fatal(err)
}
return model
return f.Name(), model
}
func testGGMLTensor(name string, kind ggml.TensorType, shape []uint64) *ggml.Tensor {
tensor := &ggml.Tensor{
Name: name,
Kind: uint32(kind),
Shape: shape,
}
tensor.WriterTo = bytes.NewReader(make([]byte, tensor.Size()))
return tensor
}
// fakeRunningCmd returns an exec.Cmd that looks like it's still running