vendor/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New models (#15861)

Browse Source 
* mlx: add laguna model support

* convert: support fp8 safetensors import

Decode HF F8_E4M3 safetensors with block scale companions into GGUF-supported tensor types, and record which output tensors came from FP8 source weights.

Use that source-precision metadata during create quantization: default FP8-sourced GGUFs to Q8_0, keep non-FP8 tensors at their original precision for Q8_0, and promote non-FP8 quantizable tensors to Q8_0 for Q4_K requests.

* ggml: add laguna model support

* server: preserve generate logprobs with builtin parsers

Generate requests were dropping logprob-only chunks whenever a builtin parser buffered visible content. Chat already handled this case, but generate only forwarded chunks with visible response, thinking, or tool-call output.

Keep generate chunks that carry logprobs even when the builtin parser has not flushed visible content yet, and add a regression test that exercises the behavior with a generic thinking parser.

* review comments - perf improvements

* ggml: implement nemotron 3 nano omni

* add poolside integration

* update poolside doc

* adapt to new cache setup

* fix test

* fix test

---------

Co-authored-by: Eva Ho <hoyyeva@gmail.com>
This commit is contained in:
Daniel Hiltgen
2026-04-28 11:50:12 -07:00
committed by GitHub
parent 2bbe2405fe
commit 87288ced4f
62 changed files with 11284 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
server/quantization.go
View File

@@ -7,6 +7,7 @@ import (
"maps"
"os"
"slices"
"strconv"
"strings"
"unsafe"
@@ -51,11 +52,14 @@ func (q quantizer) WriteTo(w io.Writer) (int64, error) {
}
type quantizeState struct {
nAttnV int // Number of attn_*v* weight tensors
nFfnDown int // Number of ffn_down tensors
iAttnV int // Running counter of number of attn_v tensors that have been processed
iFfnDown int // Running counter of number of ffn_down tensors that have been processed
hasOutput bool // used to figure out if a model shares tok_embd with the output weight
nAttnV int // Number of attn_*v* weight tensors
nFfnDown int // Number of ffn_down tensors
iAttnV int // Running counter of number of attn_v tensors that have been processed
iFfnDown int // Running counter of number of ffn_down tensors that have been processed
hasOutput bool // used to figure out if a model shares tok_embd with the output weight
preserveSourceFP8ToQ8 bool
preserveSourceQ4 bool
sourceFP8Tensors map[string]struct{}
}
func useMoreBits(iLayer, nLayers int) bool {
@@ -108,6 +112,53 @@ func qwen3LinearAttnQuantType(name string) (fsggml.TensorType, bool) {
return 0, false
}
func isLagunaGGUFRoutedExpertWeight(name string) bool {
return strings.HasSuffix(name, ".weight") && (strings.Contains(name, "ffn_gate_exps") ||
strings.Contains(name, "ffn_up_exps") ||
strings.Contains(name, "ffn_down_exps"))
}
func lagunaGGUFBlockIndex(name string) (int, bool) {
if !strings.HasPrefix(name, "blk.") {
return 0, false
}
parts := strings.SplitN(strings.TrimPrefix(name, "blk."), ".", 2)
if len(parts) != 2 {
return 0, false
}
i, err := strconv.Atoi(parts[0])
if err != nil {
return 0, false
}
return i, true
}
func lagunaGGUFQuantization(name string, originalType, requestedType fsggml.TensorType, ftype fsggml.FileType, blockCount int) (fsggml.TensorType, bool) {
if !isLagunaGGUFRoutedExpertWeight(name) {
return originalType, false
}
if strings.HasSuffix(name, ".ffn_down_exps.weight") {
if i, ok := lagunaGGUFBlockIndex(name); ok && blockCount > 0 {
switch ftype {
case fsggml.FileTypeQ4_K_M:
if requestedType != fsggml.TensorTypeQ8_0 && useMoreBits(i, blockCount) {
return fsggml.TensorTypeQ6_K, true
}
case fsggml.FileTypeQ4_K_S:
if requestedType != fsggml.TensorTypeQ8_0 && i < blockCount/8 {
return fsggml.TensorTypeQ5_K, true
}
}
}
}
return requestedType, true
}
func getTensorNewType(kv fsggml.KV, qs *quantizeState, newType fsggml.TensorType, name string, shape []uint64, ftype fsggml.FileType) fsggml.TensorType {
// Ported from llama_tensor_get_type, removed unsupported quantization types
nExperts := max(1, kv.Uint("expert_count", 0))
@@ -120,10 +171,10 @@ func getTensorNewType(kv fsggml.KV, qs *quantizeState, newType fsggml.TensorType
newType = fsggml.TensorTypeQ6_K
}
} else if strings.Contains(name, "attn_v.weight") {
if (ftype == fsggml.FileTypeQ4_K_M) &&
if newType != fsggml.TensorTypeQ8_0 && (ftype == fsggml.FileTypeQ4_K_M) &&
useMoreBits(qs.iAttnV, qs.nAttnV) {
newType = fsggml.TensorTypeQ6_K
} else if ftype == fsggml.FileTypeQ4_K_S && qs.iAttnV < 4 {
} else if newType != fsggml.TensorTypeQ8_0 && ftype == fsggml.FileTypeQ4_K_S && qs.iAttnV < 4 {
newType = fsggml.TensorTypeQ5_K
}
@@ -158,31 +209,35 @@ func getTensorNewType(kv fsggml.KV, qs *quantizeState, newType fsggml.TensorType
// expert alphabetically, so dense increments the counter and expert uses counter-1.
var iLayer int
if strings.Contains(name, "_exps") {
if kv.Architecture() == "laguna" {
goto finalize
}
iLayer = max(0, qs.iFfnDown-1)
} else {
iLayer = qs.iFfnDown
qs.iFfnDown++
}
n_layer := qs.nFfnDown
if ftype == fsggml.FileTypeQ4_K_M {
if newType != fsggml.TensorTypeQ8_0 && ftype == fsggml.FileTypeQ4_K_M {
if useMoreBits(iLayer, n_layer) {
newType = fsggml.TensorTypeQ6_K
}
} else if ftype == fsggml.FileTypeQ4_K_S && iLayer < n_layer/8 {
} else if newType != fsggml.TensorTypeQ8_0 && ftype == fsggml.FileTypeQ4_K_S && iLayer < n_layer/8 {
newType = fsggml.TensorTypeQ5_K
}
} else if strings.Contains(name, "attn_output.weight") {
if nExperts == 8 {
if newType != fsggml.TensorTypeQ8_0 && nExperts == 8 {
if ftype == fsggml.FileTypeQ4_K_S || ftype == fsggml.FileTypeQ4_K_M {
newType = fsggml.TensorTypeQ5_K
}
}
} else if strings.Contains(name, "attn_qkv.weight") {
if ftype == fsggml.FileTypeQ4_K_M {
if newType != fsggml.TensorTypeQ8_0 && ftype == fsggml.FileTypeQ4_K_M {
newType = fsggml.TensorTypeQ5_K
}
}
finalize:
if newType.IsQuantized() {
nx := shape[0]
qk_k := newType.BlockSize()
@@ -218,7 +273,12 @@ func quantize(in, out *os.File, orig *fsggml.GGML, newFileType fsggml.FileType,
kv := maps.Clone(orig.KV())
kv["general.file_type"] = newFileType
// kv["general.quantization_version"] = ggml.QuantizationVersion()
qs := &quantizeState{}
qs := &quantizeState{
sourceFP8Tensors: sourceFP8TensorSet(kv),
}
hasSourceFP8 := hasSourceFP8Tensors(kv)
qs.preserveSourceFP8ToQ8 = hasSourceFP8 && newFileType == fsggml.FileTypeQ8_0
qs.preserveSourceQ4 = hasSourceFP8 && slices.Contains([]fsggml.FileType{fsggml.FileTypeQ4_K_M, fsggml.FileTypeQ4_K_S}, newFileType)
// Build up the quantize state so newType can adjust types
layerCount := 0
for k, l := range orig.Tensors().GroupLayers() {
@@ -304,13 +364,34 @@ func newType(t *fsggml.Tensor, kv fsggml.KV, qs *quantizeState, ftype fsggml.Fil
newType := fsggml.TensorType(t.Kind)
if quantize {
if qs.preserveSourceFP8ToQ8 {
if _, ok := qs.sourceFP8Tensors[name]; !ok {
return newType
}
}
if slices.Contains([]string{"qwen3next", "qwen35", "qwen35moe"}, kv.Architecture()) && (ftype == fsggml.FileTypeQ4_K_M || ftype == fsggml.FileTypeQ4_K_S) {
if qt, ok := qwen3LinearAttnQuantType(name); ok {
return qt
}
}
// TODO: Consider extracting architecture-specific GGUF quantization policy
// from server so different quantization backends can share one source of
// truth for model-family specializations.
// get more optimal quantization type based on the tensor shape, layer, etc.
if qs.preserveSourceQ4 {
if _, ok := qs.sourceFP8Tensors[name]; !ok {
defaultType = fsggml.TensorTypeQ8_0
}
}
if kv.Architecture() == "laguna" {
var ok bool
defaultType, ok = lagunaGGUFQuantization(name, newType, defaultType, ftype, int(kv.Uint("block_count", 0)))
if !ok {
return newType
}
}
newType = getTensorNewType(kv, qs, defaultType, t.Name, t.Shape, ftype)
if newType != defaultType {
slog.Debug("tensor quantization adjusted for better quality", "name", t.Name, "requested", defaultType, "quantization", newType)
@@ -318,3 +399,16 @@ func newType(t *fsggml.Tensor, kv fsggml.KV, qs *quantizeState, ftype fsggml.Fil
}
return newType
}
func sourceFP8TensorSet(kv fsggml.KV) map[string]struct{} {
names := kv.Strings("source_fp8_tensors")
if len(names) == 0 {
return nil
}
out := make(map[string]struct{}, len(names))
for _, name := range names {
out[name] = struct{}{}
}
return out
}