llm: decouple prompt caching from context shift (#16639)
This PR separates prompt caching from the public shift request option for native llama-server requests. Previously, shift controlled two different mechanisms: context shifting / overflow behavior per-request llama-server cache_prompt That meant callers could not request shift: false without also disabling prompt caching. Fixes #16635
This commit is contained in:
@@ -1395,7 +1395,7 @@ func (s *llamaServerRunner) Completion(ctx context.Context, req CompletionReques
|
||||
lsReq := llamaServerCompletionRequest{
|
||||
Prompt: prompt,
|
||||
Stream: true,
|
||||
CachePrompt: req.Shift,
|
||||
CachePrompt: true,
|
||||
NPredict: req.Options.NumPredict,
|
||||
NKeep: req.Options.NumKeep,
|
||||
Temperature: req.Options.Temperature,
|
||||
@@ -1984,7 +1984,7 @@ func (s *llamaServerRunner) llamaServerChatRequest(req ChatRequest, stream bool)
|
||||
body := map[string]any{
|
||||
"messages": messages,
|
||||
"stream": stream,
|
||||
"cache_prompt": req.Shift,
|
||||
"cache_prompt": true,
|
||||
"n_predict": req.Options.NumPredict,
|
||||
"n_keep": req.Options.NumKeep,
|
||||
"temperature": req.Options.Temperature,
|
||||
|
||||
Reference in New Issue
Block a user