vendor/opencode
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

fix(core): make V2 reads media-aware and binary-safe (#31038)

Browse Source
This commit is contained in:
Kit Langton
2026-06-05 19:48:34 -04:00
committed by GitHub
parent f750deaa3e
commit 83dca45dd5
26 changed files with 1709 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
packages/llm/src/protocols/anthropic-messages.ts
View File

@@ -303,14 +303,17 @@ const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult
})
const lowerImage = Effect.fn("AnthropicMessages.lowerImage")(function* (part: MediaPart) {
if (!part.mediaType.startsWith("image/"))
return yield* invalid(`Anthropic Messages user media content only supports images`)
const media = yield* ProviderShared.validateMedia(
"Anthropic Messages",
part,
new Set<string>(ProviderShared.IMAGE_MIMES),
)
return {
type: "image" as const,
source: {
type: "base64" as const,
media_type: part.mediaType,
data: ProviderShared.mediaBase64(part),
media_type: media.mime,
data: media.base64,
},
} satisfies AnthropicImageBlock
})
@@ -321,16 +324,19 @@ const lowerToolResultContentItem = Effect.fn("AnthropicMessages.lowerToolResultC
item: ToolResultContentPart,
) {
if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock
if (item.mediaType.startsWith("image/"))
return {
type: "image" as const,
source: {
type: "base64" as const,
media_type: item.mediaType,
data: ProviderShared.mediaBase64(item),
},
} satisfies AnthropicImageBlock
return yield* invalid(`Anthropic Messages tool-result media content only supports images, got ${item.mediaType}`)
const media = yield* ProviderShared.validateMedia(
"Anthropic Messages",
item,
new Set<string>(ProviderShared.IMAGE_MIMES),
)
return {
type: "image" as const,
source: {
type: "base64" as const,
media_type: media.mime,
data: media.base64,
},
} satisfies AnthropicImageBlock
})
const lowerToolResultContent = Effect.fn("AnthropicMessages.lowerToolResultContent")(function* (part: ToolResultPart) {

34
packages/llm/src/protocols/gemini.ts
View File

@@ -14,12 +14,14 @@ import {
type TextPart,
type ToolCallPart,
type ToolDefinition,
type ToolResultContentPart,
} from "../schema"
import { JsonObject, optionalArray, ProviderShared } from "./shared"
import { GeminiToolSchema } from "./utils/gemini-tool-schema"
import { Lifecycle } from "./utils/lifecycle"
const ADAPTER = "gemini"
const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
// =============================================================================
@@ -140,8 +142,6 @@ interface ParserState {
readonly reasoningSignature?: string
}
const mediaData = ProviderShared.mediaBytes
// =============================================================================
// Tool Schema Conversion
// =============================================================================
@@ -180,8 +180,11 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
})
const lowerUserPart = (part: TextPart | MediaPart) =>
part.type === "text" ? { text: part.text } : { inlineData: { mimeType: part.mediaType, data: mediaData(part) } }
const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPart | MediaPart) {
if (part.type === "text") return { text: part.text }
const media = yield* ProviderShared.validateMedia("Gemini", part, IMAGE_MIMES)
return { inlineData: { mimeType: media.mime, data: media.base64 } }
})
const googleMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ google: metadata })
@@ -215,7 +218,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["text", "media"]))
return yield* ProviderShared.unsupportedContent("Gemini", "user", ["text", "media"])
parts.push(lowerUserPart(part))
parts.push(yield* lowerUserPart(part))
}
contents.push({ role: "user", parts })
continue
@@ -247,15 +250,34 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["tool-result"]))
return yield* ProviderShared.unsupportedContent("Gemini", "tool", ["tool-result"])
if (part.result.type !== "content") {
parts.push({
functionResponse: {
name: part.name,
response: {
name: part.name,
content: ProviderShared.toolResultText(part),
},
},
})
continue
}
const content: ReadonlyArray<ToolResultContentPart> = part.result.value
const text = content.filter((item) => item.type === "text").map((item) => item.text)
parts.push({
functionResponse: {
name: part.name,
response: {
name: part.name,
content: ProviderShared.toolResultText(part),
content: text.join("\n"),
},
},
})
for (const item of content) {
if (item.type === "text") continue
const media = yield* ProviderShared.validateMedia("Gemini", item, IMAGE_MIMES)
parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } })
}
}
contents.push({ role: "user", parts })
}

82
packages/llm/src/protocols/openai-chat.ts
View File

@@ -9,10 +9,12 @@ import {
Usage,
type FinishReason,
type LLMRequest,
type MediaPart,
type ReasoningPart,
type TextPart,
type ToolCallPart,
type ToolDefinition,
type ToolResultContentPart,
} from "../schema"
import { isRecord, JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared"
import { OpenAIOptions } from "./utils/openai-options"
@@ -20,6 +22,7 @@ import { Lifecycle } from "./utils/lifecycle"
import { ToolStream } from "./utils/tool-stream"
const ADAPTER = "openai-chat"
const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/chat/completions"
@@ -51,9 +54,20 @@ const OpenAIChatAssistantToolCall = Schema.Struct({
})
type OpenAIChatAssistantToolCall = Schema.Schema.Type<typeof OpenAIChatAssistantToolCall>
const OpenAIChatUserContent = Schema.Union([
Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }),
Schema.Struct({
type: Schema.Literal("image_url"),
image_url: Schema.Struct({ url: Schema.String }),
}),
])
const OpenAIChatMessage = Schema.Union([
Schema.Struct({ role: Schema.Literal("system"), content: Schema.String }),
Schema.Struct({ role: Schema.Literal("user"), content: Schema.String }),
Schema.Struct({
role: Schema.Literal("user"),
content: Schema.Union([Schema.String, Schema.Array(OpenAIChatUserContent)]),
}),
Schema.Struct({
role: Schema.Literal("assistant"),
content: Schema.NullOr(Schema.String),
@@ -186,17 +200,32 @@ const lowerToolCall = (part: ToolCallPart): OpenAIChatAssistantToolCall => ({
},
})
const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (
part: Extract<MediaPart | ToolResultContentPart, { type: "media" }>,
) {
const media = yield* ProviderShared.validateMedia("OpenAI Chat", part, IMAGE_MIMES)
return { type: "image_url" as const, image_url: { url: media.dataUrl } }
})
const openAICompatibleReasoningContent = (native: unknown) =>
isRecord(native) && typeof native.reasoning_content === "string" ? native.reasoning_content : undefined
const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (message: OpenAIChatRequestMessage) {
const content: TextPart[] = []
const content: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["text"]))
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text"])
content.push(part)
if (part.type === "text") {
content.push({ type: "text", text: part.text })
continue
}
if (part.type === "media") {
content.push(yield* lowerMedia(part))
continue
}
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text", "media"])
}
return { role: "user" as const, content: ProviderShared.joinText(content) }
if (content.every((part) => part.type === "text"))
return { role: "user" as const, content: content.map((part) => part.text).join("") }
return { role: "user" as const, content }
})
const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(function* (
@@ -234,35 +263,68 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (message: OpenAIChatRequestMessage) {
const messages: OpenAIChatMessage[] = []
const images: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["tool-result"]))
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"])
messages.push({ role: "tool", tool_call_id: part.id, content: ProviderShared.toolResultText(part) })
if (part.result.type !== "content") {
messages.push({ role: "tool", tool_call_id: part.id, content: ProviderShared.toolResultText(part) })
continue
}
const content: ReadonlyArray<ToolResultContentPart> = part.result.value
const text = content.filter((item) => item.type === "text").map((item) => item.text)
messages.push({ role: "tool", tool_call_id: part.id, content: text.join("\n") })
const media = content.filter(
(item): item is Extract<ToolResultContentPart, { type: "media" }> => item.type === "media",
)
images.push(...(yield* Effect.forEach(media, lowerMedia)))
}
return messages
return { messages, images }
})
const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (message: OpenAIChatRequestMessage) {
if (message.role === "user") return [yield* lowerUserMessage(message)]
if (message.role === "assistant") return [yield* lowerAssistantMessage(message)]
return yield* lowerToolMessages(message)
return (yield* lowerToolMessages(message)).messages
})
const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: LLMRequest) {
const system: OpenAIChatMessage[] =
request.system.length === 0 ? [] : [{ role: "system", content: ProviderShared.joinText(request.system) }]
const messages = [...system]
const pendingImages: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const flushImages = () => {
if (pendingImages.length === 0) return
messages.push({ role: "user", content: pendingImages.splice(0) })
}
for (const message of request.messages) {
if (message.role === "system") {
const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message)
if (pendingImages.length > 0) {
messages.push({ role: "user", content: [...pendingImages.splice(0), { type: "text", text: part.text }] })
continue
}
const previous = messages.at(-1)
if (previous?.role === "user")
if (previous?.role === "user" && typeof previous.content === "string")
messages[messages.length - 1] = { role: "user", content: `${previous.content}\n${part.text}` }
else if (previous?.role === "user" && Array.isArray(previous.content))
messages[messages.length - 1] = {
role: "user",
content: [...previous.content, { type: "text", text: part.text }],
}
else messages.push({ role: "user", content: part.text })
continue
}
if (message.role === "tool") {
const lowered = yield* lowerToolMessages(message)
messages.push(...lowered.messages)
pendingImages.push(...lowered.images)
continue
}
flushImages()
messages.push(...(yield* lowerMessage(message)))
}
flushImages()
return messages
})

22
packages/llm/src/protocols/openai-responses.ts
View File

@@ -304,10 +304,14 @@ const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function*
part: LLMRequest["messages"][number]["content"][number],
) {
if (part.type === "text") return { type: "input_text" as const, text: part.text }
if (part.type === "media" && part.mediaType.startsWith("image/")) {
return { type: "input_image" as const, image_url: ProviderShared.mediaDataUrl(part) }
if (part.type === "media") {
const media = yield* ProviderShared.validateMedia(
"OpenAI Responses",
part,
new Set<string>(ProviderShared.IMAGE_MIMES),
)
return { type: "input_image" as const, image_url: media.dataUrl }
}
if (part.type === "media") return yield* invalid("OpenAI Responses user media content only supports images")
return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"])
})
@@ -317,12 +321,12 @@ const lowerToolResultContentItem = Effect.fn("OpenAIResponses.lowerToolResultCon
item: ToolResultContentPart,
) {
if (item.type === "text") return { type: "input_text" as const, text: item.text }
if (item.mediaType.startsWith("image/"))
return {
type: "input_image" as const,
image_url: ProviderShared.mediaDataUrl(item),
}
return yield* invalid(`OpenAI Responses tool-result media content only supports images, got ${item.mediaType}`)
const media = yield* ProviderShared.validateMedia(
"OpenAI Responses",
item,
new Set<string>(ProviderShared.IMAGE_MIMES),
)
return { type: "input_image" as const, image_url: media.dataUrl }
})
const lowerToolResultOutput = Effect.fn("OpenAIResponses.lowerToolResultOutput")(function* (part: ToolResultPart) {

58
packages/llm/src/protocols/shared.ts
View File

@@ -186,24 +186,52 @@ export const wrappedSystemUpdate = Effect.fn("ProviderShared.wrappedSystemUpdate
export const parseToolInput = (route: string, name: string, raw: string) =>
parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
/**
* Encode a `MediaPart`'s raw bytes for inclusion in a JSON request body.
* `data: string` is assumed to already be base64 (matches caller convention
* across Gemini / Bedrock); `data: Uint8Array` is base64-encoded here. Used
* by every route that supports image / document inputs.
*/
export const mediaBytes = (part: MediaPart) =>
typeof part.data === "string" ? part.data : Buffer.from(part.data).toString("base64")
export const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"] as const
export const MAX_MEDIA_ENCODED_BYTES = 8 * 1024 * 1024
export const MAX_MEDIA_DECODED_BYTES = 6 * 1024 * 1024
export const mediaBase64 = (part: MediaPart) => {
if (typeof part.data !== "string" || !part.data.startsWith("data:")) return mediaBytes(part)
return part.data.slice(part.data.indexOf(",") + 1)
const base64Pattern = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/
export interface ValidatedMedia {
readonly mime: string
readonly base64: string
readonly dataUrl: string
readonly bytes: Uint8Array
}
export const mediaDataUrl = (part: MediaPart) =>
typeof part.data === "string" && part.data.startsWith("data:")
? part.data
: `data:${part.mediaType};base64,${mediaBytes(part)}`
export const validateMedia = Effect.fn("ProviderShared.validateMedia")(function* (
route: string,
part: MediaPart,
supportedMimes: ReadonlySet<string>,
) {
const mime = part.mediaType.toLowerCase()
if (!supportedMimes.has(mime)) return yield* invalidRequest(`${route} does not support media type ${part.mediaType}`)
let base64: string
if (typeof part.data !== "string") {
if (part.data.byteLength > MAX_MEDIA_DECODED_BYTES)
return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
base64 = Buffer.from(part.data).toString("base64")
} else if (part.data.startsWith("data:")) {
const match = /^data:([^;,]+);base64,([A-Za-z0-9+/]*={0,2})$/s.exec(part.data)
if (!match) return yield* invalidRequest(`${route} media data URL must contain valid base64`)
if (match[1]!.toLowerCase() !== mime)
return yield* invalidRequest(`${route} media type ${part.mediaType} does not match data URL type ${match[1]}`)
base64 = match[2]!
} else {
base64 = part.data
}
if (Buffer.byteLength(base64, "utf8") > MAX_MEDIA_ENCODED_BYTES)
return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_ENCODED_BYTES} byte encoded limit`)
if (!base64 || base64.length % 4 !== 0 || !base64Pattern.test(base64))
return yield* invalidRequest(`${route} media must contain valid base64`)
const bytes = Buffer.from(base64, "base64")
if (bytes.byteLength > MAX_MEDIA_DECODED_BYTES)
return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
if (bytes.toString("base64") !== base64) return yield* invalidRequest(`${route} media must contain canonical base64`)
return { mime, base64, dataUrl: `data:${mime};base64,${base64}`, bytes } satisfies ValidatedMedia
})
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")

34
packages/llm/src/protocols/utils/bedrock-media.ts
View File

@@ -49,15 +49,11 @@ const DOCUMENT_FORMATS = {
"text/markdown": "md",
} as const satisfies Record<string, DocumentFormat>
const imageBlock = (part: MediaPart, format: ImageFormat): ImageBlock => ({
image: { format, source: { bytes: ProviderShared.mediaBytes(part) } },
})
const documentBlock = (part: MediaPart, format: DocumentFormat): DocumentBlock => ({
const documentBlock = (part: MediaPart, format: DocumentFormat, bytes: string): DocumentBlock => ({
document: {
format,
name: part.filename ?? `document.${format}`,
source: { bytes: ProviderShared.mediaBytes(part) },
source: { bytes },
},
})
@@ -66,15 +62,29 @@ const documentBlock = (part: MediaPart, format: DocumentFormat): DocumentBlock =
// document block. Image MIME types not in `IMAGE_FORMATS` (e.g. `image/svg+xml`)
// get an image-specific error so the caller knows it's a format-support issue,
// not a kind-detection issue.
export const lower = (part: MediaPart) => {
export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart) {
const mime = part.mediaType.toLowerCase()
const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
if (imageFormat) return Effect.succeed(imageBlock(part, imageFormat))
if (imageFormat) {
const media = yield* ProviderShared.validateMedia(
"Bedrock Converse",
part,
new Set<string>(Object.keys(IMAGE_FORMATS)),
)
return { image: { format: imageFormat, source: { bytes: media.base64 } } } satisfies ImageBlock
}
if (mime.startsWith("image/"))
return ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
const documentFormat = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS]
if (documentFormat) return Effect.succeed(documentBlock(part, documentFormat))
return ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
}
if (documentFormat) {
const media = yield* ProviderShared.validateMedia(
"Bedrock Converse",
part,
new Set<string>(Object.keys(DOCUMENT_FORMATS)),
)
return documentBlock(part, documentFormat, media.base64)
}
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
})
export * as BedrockMedia from "./bedrock-media"

20
packages/llm/src/tool.ts
View File

@@ -51,6 +51,7 @@ export interface Tool<Parameters extends ToolSchema<any>, Success extends ToolSc
readonly success: Success
readonly execute?: ToolExecute<Parameters, Success>
readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
/** @internal */
readonly _decode: (input: unknown) => Effect.Effect<Schema.Schema.Type<Parameters>, Schema.SchemaError>
/** @internal */
@@ -86,6 +87,7 @@ type TypedToolConfig = {
readonly success: ToolSchema<any>
readonly execute?: ToolExecute<ToolSchema<any>, ToolSchema<any>>
readonly toModelOutput?: ToolToModelOutput<ToolSchema<any>, ToolSchema<any>>
readonly toStructuredOutput?: (output: unknown) => unknown
}
type DynamicToolConfig = {
@@ -94,6 +96,7 @@ type DynamicToolConfig = {
readonly outputSchema?: JsonSchema.JsonSchema
readonly execute?: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
readonly toStructuredOutput?: (output: unknown) => unknown
}
/**
@@ -133,6 +136,7 @@ export function make<Parameters extends ToolSchema<any>, Success extends ToolSch
readonly success: Success
readonly execute: ToolExecute<Parameters, Success>
readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
}): ExecutableTool<Parameters, Success>
export function make<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>>(config: {
readonly description: string
@@ -140,6 +144,7 @@ export function make<Parameters extends ToolSchema<any>, Success extends ToolSch
readonly success: Success
readonly execute?: undefined
readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
}): Tool<Parameters, Success>
export function make(config: {
readonly description: string
@@ -147,6 +152,7 @@ export function make(config: {
readonly outputSchema?: JsonSchema.JsonSchema
readonly execute: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
readonly toStructuredOutput?: (output: unknown) => unknown
}): AnyExecutableTool
export function make(config: {
readonly description: string
@@ -154,6 +160,7 @@ export function make(config: {
readonly outputSchema?: JsonSchema.JsonSchema
readonly execute?: undefined
readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
readonly toStructuredOutput?: (output: unknown) => unknown
}): AnyTool
export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
if ("jsonSchema" in config) {
@@ -163,10 +170,12 @@ export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
success: Schema.Unknown as ToolSchema<unknown>,
execute: config.execute,
toModelOutput: config.toModelOutput,
toStructuredOutput: config.toStructuredOutput,
_decode: Effect.succeed,
_encode: Effect.succeed,
_project: (parameters, callID, output) => project(config.toModelOutput, parameters, callID, output),
_legacyResult: config.toModelOutput === undefined,
_project: (parameters, callID, output) =>
project(config.toModelOutput, config.toStructuredOutput, parameters, callID, output),
_legacyResult: config.toModelOutput === undefined && config.toStructuredOutput === undefined,
_definition: new ToolDefinition({
name: "",
description: config.description,
@@ -181,9 +190,11 @@ export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
success: config.success,
execute: config.execute,
toModelOutput: config.toModelOutput,
toStructuredOutput: config.toStructuredOutput,
_decode: Schema.decodeUnknownEffect(config.parameters),
_encode: Schema.encodeEffect(config.success),
_project: (parameters, callID, output) => project(config.toModelOutput, parameters, callID, output),
_project: (parameters, callID, output) =>
project(config.toModelOutput, config.toStructuredOutput, parameters, callID, output),
_legacyResult: false,
_definition: new ToolDefinition({
name: "",
@@ -226,12 +237,13 @@ const toJsonSchema = (schema: Schema.Top): JsonSchema.JsonSchema => {
const project = (
toModelOutput: ((input: ToolModelOutputInput<any, any>) => ReadonlyArray<ToolContent>) | undefined,
toStructuredOutput: ((output: unknown) => unknown) | undefined,
parameters: unknown,
callID: ToolCallPart["id"],
output: unknown,
): ToolOutputType =>
ToolOutput.make(
output,
toStructuredOutput?.(output) ?? output,
toModelOutput?.({ callID, parameters, output }) ??
(typeof output === "string" ? [toolText({ type: "text", text: output })] : []),
)

8
packages/llm/test/provider/bedrock-converse.test.ts
View File

@@ -509,8 +509,8 @@ describe("Bedrock Converse route", () => {
model,
messages: [
Message.user([
{ type: "media", mediaType: "application/pdf", data: "PDFDATA", filename: "report.pdf" },
{ type: "media", mediaType: "text/csv", data: "CSVDATA" },
{ type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==", filename: "report.pdf" },
{ type: "media", mediaType: "text/csv", data: "Q1NWREFUQQ==" },
]),
],
}),
@@ -522,9 +522,9 @@ describe("Bedrock Converse route", () => {
role: "user",
content: [
// Filename round-trips when supplied.
{ document: { format: "pdf", name: "report.pdf", source: { bytes: "PDFDATA" } } },
{ document: { format: "pdf", name: "report.pdf", source: { bytes: "UERGREFUQQ==" } } },
// Falls back to a stable placeholder when filename is missing.
{ document: { format: "csv", name: "document.csv", source: { bytes: "CSVDATA" } } },
{ document: { format: "csv", name: "document.csv", source: { bytes: "Q1NWREFUQQ==" } } },
],
},
],

105
packages/llm/test/provider/gemini.test.ts
View File

@@ -3,6 +3,7 @@ import { Effect } from "effect"
import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { Auth, LLMClient } from "../../src/route"
import * as Gemini from "../../src/protocols/gemini"
import { ProviderShared } from "../../src/protocols/shared"
import { it } from "../lib/effect"
import { fixedResponse } from "../lib/http"
import { sseEvents, sseRaw } from "../lib/sse"
@@ -109,6 +110,110 @@ describe("Gemini route", () => {
}),
)
it.effect("continues image tool results as inline vision input without base64 text", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<Gemini.GeminiBody>(
LLM.request({
model,
messages: [
Message.assistant([ToolCallPart.make({ id: "call_image", name: "read", input: { path: "pixel.png" } })]),
Message.tool({
id: "call_image",
name: "read",
result: {
type: "content",
value: [
{ type: "text", text: "Image read successfully" },
{ type: "media", mediaType: "image/png", data: "AAECAw==", filename: "pixel.png" },
],
},
}),
],
}),
)
expect(prepared.body.contents).toEqual([
{ role: "model", parts: [{ functionCall: { name: "read", args: { path: "pixel.png" } } }] },
{
role: "user",
parts: [
{
functionResponse: {
name: "read",
response: { name: "read", content: "Image read successfully" },
},
},
{ inlineData: { mimeType: "image/png", data: "AAECAw==" } },
],
},
])
expect(JSON.stringify(prepared.body.contents)).not.toContain('"content":"AAECAw=="')
}),
)
it.effect("strips matching data URLs to raw base64 inlineData", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<Gemini.GeminiBody>(
LLM.request({
model,
messages: [
Message.user({ type: "media", mediaType: "image/png", data: "data:image/png;base64,AAEC" }),
Message.tool({
id: "call_image",
name: "read",
result: {
type: "content",
value: [{ type: "media", mediaType: "image/jpeg", data: "data:image/jpeg;base64,/9j/" }],
},
}),
],
}),
)
expect(prepared.body.contents).toEqual([
{ role: "user", parts: [{ inlineData: { mimeType: "image/png", data: "AAEC" } }] },
{
role: "user",
parts: [
{ functionResponse: { name: "read", response: { name: "read", content: "" } } },
{ inlineData: { mimeType: "image/jpeg", data: "/9j/" } },
],
},
])
}),
)
for (const [name, media] of [
["mismatched data URL MIME", { mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" }],
["malformed base64", { mediaType: "image/png", data: "%%%=" }],
["unsupported SVG", { mediaType: "image/svg+xml", data: "PHN2Zz4=" }],
] as const)
it.effect(`rejects ${name}`, () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
LLM.request({ model, messages: [Message.user({ type: "media", ...media })] }),
).pipe(Effect.flip)
expect(error.message).toMatch(/does not support|does not match|valid base64/)
}),
)
it.effect("rejects oversized image input", () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
LLM.request({
model,
messages: [
Message.user({
type: "media",
mediaType: "image/png",
data: "A".repeat(ProviderShared.MAX_MEDIA_ENCODED_BYTES + 4),
}),
],
}),
).pipe(Effect.flip)
expect(error.message).toContain("encoded limit")
}),
)
it.effect("omits tools when tool choice is none", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(

9
packages/llm/test/provider/golden.recorded.test.ts
View File

@@ -73,7 +73,7 @@ describeRecordedGoldenScenarios([
prefix: "openai-chat",
model: openAIChat,
requires: ["OPENAI_API_KEY"],
scenarios: ["text", "tool-call", "tool-loop"],
scenarios: ["text", "tool-call", "tool-loop", { id: "image-tool-result", maxTokens: 40 }],
},
{
name: "OpenAI Responses gpt-5.5",
@@ -123,7 +123,12 @@ describeRecordedGoldenScenarios([
prefix: "gemini",
model: gemini,
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
scenarios: [{ id: "text", maxTokens: 80 }, "tool-call", { id: "image", maxTokens: 160 }],
scenarios: [
{ id: "text", maxTokens: 80 },
"tool-call",
{ id: "image", maxTokens: 160 },
{ id: "image-tool-result", maxTokens: 40 },
],
},
{
name: "xAI Grok 3 Mini",

200
packages/llm/test/provider/openai-chat.test.ts
View File

@@ -5,6 +5,7 @@ import { LLM, LLMError, Message, Model, ToolCallPart, Usage } from "../../src"
import * as Azure from "../../src/providers/azure"
import * as OpenAI from "../../src/providers/openai"
import * as OpenAIChat from "../../src/protocols/openai-chat"
import { ProviderShared } from "../../src/protocols/shared"
import { Auth, LLMClient } from "../../src/route"
import { it } from "../lib/effect"
import { dynamicResponse, fixedResponse, truncatedStream } from "../lib/http"
@@ -223,17 +224,208 @@ describe("OpenAI Chat route", () => {
}),
)
it.effect("rejects unsupported user media content", () =>
it.effect("continues image tool results as vision input without base64 text", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
model,
messages: [
Message.assistant([ToolCallPart.make({ id: "call_image", name: "read", input: { path: "pixel.png" } })]),
Message.tool({
id: "call_image",
name: "read",
result: {
type: "content",
value: [
{ type: "text", text: "Image read successfully" },
{ type: "media", mediaType: "image/png", data: "AAECAw==", filename: "pixel.png" },
],
},
}),
],
}),
)
expect(prepared.body.messages).toEqual([
{
role: "assistant",
content: null,
tool_calls: [
{
id: "call_image",
type: "function",
function: { name: "read", arguments: encodeJson({ path: "pixel.png" }) },
},
],
},
{ role: "tool", tool_call_id: "call_image", content: "Image read successfully" },
{
role: "user",
content: [{ type: "image_url", image_url: { url: "data:image/png;base64,AAECAw==" } }],
},
])
expect(JSON.stringify(prepared.body.messages)).not.toContain('"content":"AAECAw=="')
}),
)
it.effect("orders parallel tool responses before one aggregated vision message", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
model,
messages: [
Message.assistant([
ToolCallPart.make({ id: "call_1", name: "read", input: {} }),
ToolCallPart.make({ id: "call_2", name: "read", input: {} }),
]),
Message.make({
role: "tool",
content: [
{
type: "tool-result",
id: "call_1",
name: "read",
result: { type: "content", value: [{ type: "media", mediaType: "image/png", data: "AAEC" }] },
},
{
type: "tool-result",
id: "call_2",
name: "read",
result: { type: "content", value: [{ type: "media", mediaType: "image/jpeg", data: "/9j/" }] },
},
],
}),
],
}),
)
expect(prepared.body.messages.slice(1)).toEqual([
{ role: "tool", tool_call_id: "call_1", content: "" },
{ role: "tool", tool_call_id: "call_2", content: "" },
{
role: "user",
content: [
{ type: "image_url", image_url: { url: "data:image/png;base64,AAEC" } },
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,/9j/" } },
],
},
])
}),
)
it.effect("aggregates consecutive tool images with a following system update", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
model,
messages: [
Message.tool({
id: "call_1",
name: "read",
result: { type: "content", value: [{ type: "media", mediaType: "image/png", data: "AAEC" }] },
}),
Message.tool({
id: "call_2",
name: "read",
result: { type: "content", value: [{ type: "media", mediaType: "image/webp", data: "UklG" }] },
}),
Message.system("Inspect both images."),
],
}),
)
expect(prepared.body.messages).toEqual([
{ role: "tool", tool_call_id: "call_1", content: "" },
{ role: "tool", tool_call_id: "call_2", content: "" },
{
role: "user",
content: [
{ type: "image_url", image_url: { url: "data:image/png;base64,AAEC" } },
{ type: "image_url", image_url: { url: "data:image/webp;base64,UklG" } },
{ type: "text", text: "<system-update>\nInspect both images.\n</system-update>" },
],
},
])
}),
)
it.effect("appends system updates without replacing multipart user content", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
model,
messages: [
Message.user({ type: "media", mediaType: "image/png", data: "AAEC" }),
Message.system("Keep the image."),
],
}),
)
expect(prepared.body.messages).toEqual([
{
role: "user",
content: [
{ type: "image_url", image_url: { url: "data:image/png;base64,AAEC" } },
{ type: "text", text: "<system-update>\nKeep the image.\n</system-update>" },
],
},
])
}),
)
for (const [name, media] of [
["mismatched data URL MIME", { mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" }],
["malformed base64", { mediaType: "image/png", data: "not-base64" }],
["unsupported SVG", { mediaType: "image/svg+xml", data: "PHN2Zz4=" }],
] as const)
it.effect(`rejects ${name}`, () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
LLM.request({ model, messages: [Message.user({ type: "media", ...media })] }),
).pipe(Effect.flip)
expect(error.message).toMatch(/does not support|does not match|valid base64/)
}),
)
it.effect("rejects oversized image input", () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
LLM.request({
id: "req_media",
model,
messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
messages: [
Message.user({
type: "media",
mediaType: "image/png",
data: "A".repeat(ProviderShared.MAX_MEDIA_ENCODED_BYTES + 4),
}),
],
}),
).pipe(Effect.flip)
expect(error.message).toContain("encoded limit")
}),
)
expect(error.message).toContain("OpenAI Chat user messages only support text content for now")
it.effect("prepares raw and data URL image media as vision input", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
LLM.request({
id: "req_media",
model,
messages: [
Message.user([
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
{ type: "media", mediaType: "image/jpeg", data: "data:image/jpeg;base64,/9j/" },
]),
],
}),
)
expect(prepared.body.messages).toEqual([
{
role: "user",
content: [
{ type: "image_url", image_url: { url: "data:image/png;base64,AAECAw==" } },
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,/9j/" } },
],
},
])
}),
)

2
packages/llm/test/provider/openai-responses.test.ts
View File

@@ -1254,7 +1254,7 @@ describe("OpenAI Responses route", () => {
}),
).pipe(Effect.flip)
expect(error.message).toContain("OpenAI Responses user media content only supports images")
expect(error.message).toContain("OpenAI Responses does not support media type application/pdf")
}),
)

25
packages/llm/test/tool-runtime.test.ts
View File

@@ -208,6 +208,31 @@ describe("LLMClient tools", () => {
}),
)
it.effect("can retain model media while redacting duplicated structured payloads", () =>
Effect.gen(function* () {
const image = Tool.make({
description: "Return an image.",
parameters: Schema.Struct({}),
success: Schema.Struct({ mime: Schema.String, data: Schema.String }),
execute: () => Effect.succeed({ mime: "image/png", data: "AAECAw==" }),
toStructuredOutput: (output) => ({ mime: output.mime }),
toModelOutput: ({ output }) => [
{ type: "file", source: { type: "data", data: output.data }, mime: output.mime },
],
})
const dispatched = yield* ToolRuntime.dispatch(
{ image },
LLMEvent.toolCall({ id: "call_image", name: "image", input: {} }),
)
expect(dispatched.output).toEqual({
structured: { mime: "image/png" },
content: [{ type: "file", source: { type: "data", data: "AAECAw==" }, mime: "image/png" }],
})
}),
)
it.effect("models canonical tool files with explicit data, url, and file sources", () =>
Effect.sync(() => {
const decode = Schema.decodeUnknownSync(ToolContent)