vendor/ollama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm: decouple prompt caching from context shift (#16639)

Browse Source 
This PR separates prompt caching from the public shift request option for native llama-server requests.

Previously, shift controlled two different mechanisms:

context shifting / overflow behavior
per-request llama-server cache_prompt
That meant callers could not request shift: false without also disabling prompt caching.

Fixes #16635
This commit is contained in:
Parafee41
2026-06-12 07:05:24 +08:00
committed by GitHub
parent 82e0ddb6fe
commit f8a48df24d
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
llm/llama_server.go
View File

@@ -1395,7 +1395,7 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques
lsReq := llamaServerCompletionRequest{ lsReq := llamaServerCompletionRequest{
Prompt: prompt, Prompt: prompt,
Stream: true, Stream: true,
CachePrompt: req.Shift, CachePrompt: true,
NPredict: req.Options.NumPredict, NPredict: req.Options.NumPredict,
NKeep: req.Options.NumKeep, NKeep: req.Options.NumKeep,
Temperature: req.Options.Temperature, Temperature: req.Options.Temperature,
@@ -1984,7 +1984,7 @@ func (s *llamaServerRunner) llamaServerChatRequest(req ChatRequest, stream bool)
body := map[string]any{ body := map[string]any{
"messages": messages, "messages": messages,
"stream": stream, "stream": stream,
"cache_prompt": req.Shift, "cache_prompt": true,
"n_predict": req.Options.NumPredict, "n_predict": req.Options.NumPredict,
"n_keep": req.Options.NumKeep, "n_keep": req.Options.NumKeep,
"temperature": req.Options.Temperature, "temperature": req.Options.Temperature,