From 3af1a008e2dd4740874f07968b373db3bc723344 Mon Sep 17 00:00:00 2001 From: Eva H <63033505+hoyyeva@users.noreply.github.com> Date: Tue, 12 May 2026 12:51:46 -0700 Subject: [PATCH] launch/opencode: add image modalities for vision models (#15922) --- cmd/launch/opencode.go | 48 +++++++++++++--- cmd/launch/opencode_test.go | 108 ++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 9 deletions(-) diff --git a/cmd/launch/opencode.go b/cmd/launch/opencode.go index 18b04eb9..3fcaec90 100644 --- a/cmd/launch/opencode.go +++ b/cmd/launch/opencode.go @@ -1,6 +1,7 @@ package launch import ( + "context" "encoding/json" "fmt" "os" @@ -8,9 +9,12 @@ import ( "path/filepath" "runtime" "slices" + "time" + "github.com/ollama/ollama/api" "github.com/ollama/ollama/cmd/internal/fileutil" "github.com/ollama/ollama/envconfig" + "github.com/ollama/ollama/types/model" ) // OpenCode implements Runner and Editor for OpenCode integration. @@ -20,6 +24,8 @@ type OpenCode struct { configContent string // JSON config built by Edit, passed to Run via env var } +const openCodeModelShowTimeout = 2 * time.Second + func (o *OpenCode) String() string { return "OpenCode" } // findOpenCode returns the opencode binary path, checking PATH first then the @@ -176,6 +182,12 @@ func buildInlineConfig(primary string, models []string) (string, error) { if primary == "" || len(models) == 0 { return "", fmt.Errorf("buildInlineConfig: primary and models are required") } + + client, err := api.ClientFromEnvironment() + if err != nil { + client = nil + } + config := map[string]any{ "$schema": "https://opencode.ai/config.json", "provider": map[string]any{ @@ -185,7 +197,7 @@ func buildInlineConfig(primary string, models []string) (string, error) { "options": map[string]any{ "baseURL": envconfig.Host().String() + "/v1", }, - "models": buildModelEntries(models), + "models": buildModelEntries(context.Background(), client, models), }, }, "model": "ollama/" + primary, @@ -228,21 +240,39 @@ func readModelJSONModels() []string { return models } -func buildModelEntries(modelList []string) map[string]any { - models := make(map[string]any) - for _, model := range modelList { - entry := map[string]any{ - "name": model, +func buildModelEntries(ctx context.Context, client *api.Client, modelList []string) map[string]any { + if client != nil { + var cancel context.CancelFunc + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + ctx, cancel = context.WithTimeout(ctx, openCodeModelShowTimeout) + defer cancel() } - if isCloudModelName(model) { - if l, ok := lookupCloudModelLimit(model); ok { + } + + models := make(map[string]any) + for _, modelID := range modelList { + entry := map[string]any{ + "name": modelID, + } + if client != nil { + if resp, err := client.Show(ctx, &api.ShowRequest{Model: modelID}); err == nil { + if slices.Contains(resp.Capabilities, model.CapabilityVision) { + entry["modalities"] = map[string]any{ + "input": []string{"text", "image"}, + "output": []string{"text"}, + } + } + } + } + if isCloudModelName(modelID) { + if l, ok := lookupCloudModelLimit(modelID); ok { entry["limit"] = map[string]any{ "context": l.Context, "output": l.Output, } } } - models[model] = entry + models[modelID] = entry } return models } diff --git a/cmd/launch/opencode_test.go b/cmd/launch/opencode_test.go index 9d3b378a..54d721e8 100644 --- a/cmd/launch/opencode_test.go +++ b/cmd/launch/opencode_test.go @@ -1,12 +1,21 @@ package launch import ( + "context" "encoding/json" "fmt" + "io" + "net/http" + "net/url" "os" "path/filepath" "runtime" + "strings" + "sync" "testing" + "time" + + "github.com/ollama/ollama/api" ) func TestOpenCodeIntegration(t *testing.T) { @@ -157,6 +166,105 @@ func TestOpenCodeEdit(t *testing.T) { t.Errorf("local model should not have limit, got %v", entry["limit"]) } }) + + t.Run("vision model gets image input modalities", func(t *testing.T) { + u, err := url.Parse("http://ollama.example") + if err != nil { + t.Fatalf("parse test URL: %v", err) + } + client := api.NewClient(u, &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "/api/show" { + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(strings.NewReader("not found")), + Header: make(http.Header), + }, nil + } + + var body struct { + Model string `json:"model"` + } + if err := json.NewDecoder(req.Body).Decode(&body); err != nil { + t.Fatalf("decode show request: %v", err) + } + if body.Model != "gemma4:26b" { + t.Fatalf("show request model = %q, want gemma4:26b", body.Model) + } + + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"capabilities":["vision"],"model_info":{}}`)), + Header: make(http.Header), + }, nil + })}) + + models := buildModelEntries(context.Background(), client, []string{"gemma4:26b"}) + entry, _ := models["gemma4:26b"].(map[string]any) + modalities, _ := entry["modalities"].(map[string]any) + input, _ := modalities["input"].([]string) + output, _ := modalities["output"].([]string) + + if len(input) != 2 || input[0] != "text" || input[1] != "image" { + t.Fatalf("modalities.input = %v, want [text image]", input) + } + if len(output) != 1 || output[0] != "text" { + t.Fatalf("modalities.output = %v, want [text]", output) + } + }) +} + +func TestBuildModelEntries(t *testing.T) { + t.Run("defaults to model name when capabilities cannot be probed", func(t *testing.T) { + models := buildModelEntries(context.Background(), nil, []string{"llama3.2"}) + entry, _ := models["llama3.2"].(map[string]any) + if entry["name"] != "llama3.2" { + t.Fatalf("name = %v, want llama3.2", entry["name"]) + } + if _, ok := entry["modalities"]; ok { + t.Fatalf("modalities should not be set without an API client, got %v", entry["modalities"]) + } + }) + + t.Run("uses one timeout budget across capability probes", func(t *testing.T) { + u, err := url.Parse("http://ollama.example") + if err != nil { + t.Fatalf("parse test URL: %v", err) + } + + var mu sync.Mutex + waited := 0 + + client := api.NewClient(u, &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + mu.Lock() + if req.Context().Err() == nil { + waited++ + } + mu.Unlock() + + <-req.Context().Done() + return nil, req.Context().Err() + })}) + + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + + models := buildModelEntries(ctx, client, []string{"slow-1", "slow-2"}) + for _, modelID := range []string{"slow-1", "slow-2"} { + entry, _ := models[modelID].(map[string]any) + if entry["name"] != modelID { + t.Fatalf("name for %q = %v, want %q", modelID, entry["name"], modelID) + } + if _, ok := entry["modalities"]; ok { + t.Fatalf("modalities for %q should not be set after probe timeout, got %v", modelID, entry["modalities"]) + } + } + + mu.Lock() + defer mu.Unlock() + if waited != 1 { + t.Fatalf("expected shared timeout to block one probe, waited on %d probes", waited) + } + }) } func TestOpenCodeModels_ReturnsNil(t *testing.T) {