diff --git a/packages/@ant/model-provider/src/shared/__tests__/openaiConvertMessages.test.ts b/packages/@ant/model-provider/src/shared/__tests__/openaiConvertMessages.test.ts index 1830c58d4..17d3ee99c 100644 --- a/packages/@ant/model-provider/src/shared/__tests__/openaiConvertMessages.test.ts +++ b/packages/@ant/model-provider/src/shared/__tests__/openaiConvertMessages.test.ts @@ -121,6 +121,69 @@ describe('anthropicMessagesToOpenAI', () => { ]) }) + test('converts image content nested in tool_result to follow-up user image', () => { + const result = anthropicMessagesToOpenAI( + [ + makeAssistantMsg([ + { + type: 'tool_use' as const, + id: 'toolu_img', + name: 'Read', + input: { file_path: '1.png' }, + }, + ]), + makeUserMsg([ + { + type: 'tool_result' as const, + tool_use_id: 'toolu_img', + content: [ + { + type: 'image' as const, + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgo=', + }, + }, + ], + }, + ]), + ], + [] as any, + ) + + expect(result).toEqual([ + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'toolu_img', + type: 'function', + function: { + name: 'Read', + arguments: '{"file_path":"1.png"}', + }, + }, + ], + }, + { + role: 'tool', + tool_call_id: 'toolu_img', + content: '[image]', + }, + { + role: 'user', + content: [ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + }, + ], + }, + ]) + }) + test('preserves thinking blocks as reasoning_content', () => { const result = anthropicMessagesToOpenAI( [ diff --git a/packages/@ant/model-provider/src/shared/openaiConvertMessages.ts b/packages/@ant/model-provider/src/shared/openaiConvertMessages.ts index f896a6743..f0773281e 100644 --- a/packages/@ant/model-provider/src/shared/openaiConvertMessages.ts +++ b/packages/@ant/model-provider/src/shared/openaiConvertMessages.ts @@ -106,7 +106,7 @@ function convertInternalUserMessage( // message with tool_calls. If we emit a user message first, the API will // reject the request with "insufficient tool messages following tool_calls". for (const tr of toolResults) { - result.push(convertToolResult(tr)) + result.push(...convertToolResult(tr)) } // 如果有图片，构建多模态 content 数组 @@ -136,8 +136,11 @@ function convertInternalUserMessage( function convertToolResult( block: BetaToolResultBlockParam, -): ChatCompletionToolMessageParam { +): ChatCompletionMessageParam[] { let content: string + const imageParts: Array<{ type: 'image_url'; image_url: { url: string } }> = + [] + if (typeof block.content === 'string') { content = block.content } else if (Array.isArray(block.content)) { @@ -145,6 +148,15 @@ function convertToolResult( .map(c => { if (typeof c === 'string') return c if ('text' in c) return c.text + if ('type' in c && c.type === 'image') { + const imagePart = convertImageBlockToOpenAI( + c as unknown as Record, + ) + if (imagePart) { + imageParts.push(imagePart) + } + return '' + } return '' }) .filter(Boolean) @@ -153,11 +165,20 @@ function convertToolResult( content = '' } - return { + const result: ChatCompletionMessageParam[] = [{ role: 'tool', tool_call_id: block.tool_use_id, - content, - } satisfies ChatCompletionToolMessageParam + content: content || (imageParts.length > 0 ? '[image]' : ''), + } satisfies ChatCompletionToolMessageParam] + + if (imageParts.length > 0) { + result.push({ + role: 'user', + content: imageParts, + } satisfies ChatCompletionUserMessageParam) + } + + return result } function convertInternalAssistantMessage( @@ -214,7 +235,7 @@ function convertInternalAssistantMessage( const thinkingText = (block as unknown as Record) .thinking if (typeof thinkingText === 'string') { - reasoningParts.push(thinkingText) + reasoningParts.push(thinkingText.length > 0 ? thinkingText : ' ') } } // Skip redacted_thinking, server_tool_use, etc. diff --git a/src/costrict/provider/index.test.ts b/src/costrict/provider/index.test.ts index eba953416..18386f643 100644 --- a/src/costrict/provider/index.test.ts +++ b/src/costrict/provider/index.test.ts @@ -1,9 +1,6 @@ import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test' import type { BetaRawMessageStreamEvent } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs' -import type { - AssistantMessage, - StreamEvent, -} from '../../types/message.js' +import type { AssistantMessage, StreamEvent } from '../../types/message.js' function makeMessageStart( overrides: Record = {}, @@ -88,6 +85,8 @@ async function* eventStream(events: BetaRawMessageStreamEvent[]) { let _nextEvents: BetaRawMessageStreamEvent[] = [] let _lastCreateArgs: Record | null = null let _mockModelMaxTokens: number | undefined +let _mockModelSupportsImages: boolean | undefined +let _mockCreateError: Error | undefined mock.module('openai', () => ({ default: class OpenAI { @@ -95,6 +94,7 @@ mock.module('openai', () => ({ completions: { create: async (args: Record) => { _lastCreateArgs = args + if (_mockCreateError) throw _mockCreateError return { [Symbol.asyncIterator]: async function* () {} } }, }, @@ -172,6 +172,7 @@ mock.module('./models.js', () => ({ id: 'test-model', maxTokens: _mockModelMaxTokens, maxTokensKey: 'max_completion_tokens', + supportsImages: _mockModelSupportsImages, }, ], })) @@ -196,6 +197,7 @@ mock.module('../../utils/context.js', () => ({ async function runQueryModel( events: BetaRawMessageStreamEvent[], optionsOverrides: Record = {}, + messages: any[] = [], ) { _nextEvents = events const { queryModelCoStrict } = await import('./index.js') @@ -211,7 +213,7 @@ async function runQueryModel( } for await (const item of queryModelCoStrict( - [], + messages, { type: 'text', text: '' } as any, [], new AbortController().signal, @@ -231,11 +233,15 @@ beforeEach(() => { _nextEvents = [] _lastCreateArgs = null _mockModelMaxTokens = undefined + _mockModelSupportsImages = undefined + _mockCreateError = undefined }) afterEach(() => { _nextEvents = [] _mockModelMaxTokens = undefined + _mockModelSupportsImages = undefined + _mockCreateError = undefined }) describe('queryModelCoStrict', () => { @@ -257,9 +263,7 @@ describe('queryModelCoStrict', () => { expect(assistantMessages).toHaveLength(1) expect(assistantMessages[0]!.message.stop_reason).toBe('end_turn') expect( - (assistantMessages[0]!.message.content as any[]).map( - block => block.type, - ), + (assistantMessages[0]!.message.content as any[]).map(block => block.type), ).toEqual(['thinking', 'text']) }) @@ -272,4 +276,82 @@ describe('queryModelCoStrict', () => { expect(_lastCreateArgs).not.toBeNull() expect(_lastCreateArgs!.max_completion_tokens).toBe(2048) }) + + test('returns a clear error before sending images to non-multimodal model', async () => { + _mockModelSupportsImages = false + const messages = [ + { + type: 'user', + uuid: 'user-img', + timestamp: new Date().toISOString(), + message: { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_img', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgo=', + }, + }, + ], + }, + ], + }, + }, + ] + + const { assistantMessages } = await runQueryModel([], {}, messages) + + expect(_lastCreateArgs).toBeNull() + expect(assistantMessages).toHaveLength(1) + expect((assistantMessages[0]!.message.content as any[])[0].text).toContain( + 'does not support image input', + ) + }) + + test('allows images when model metadata declares multimodal support', async () => { + _mockModelSupportsImages = true + const events = [makeMessageStart(), makeMessageStop()] + const messages = [ + { + type: 'user', + uuid: 'user-img', + timestamp: new Date().toISOString(), + message: { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgo=', + }, + }, + ], + }, + }, + ] + + await runQueryModel(events, {}, messages) + + expect(_lastCreateArgs).not.toBeNull() + }) + + test('normalizes backend non-multimodal model errors', async () => { + _mockCreateError = new Error('/mnt/model is not a multimodal model') + + const { assistantMessages } = await runQueryModel([], {}) + + expect(assistantMessages).toHaveLength(1) + expect((assistantMessages[0]!.message.content as any[])[0].text).toBe( + 'CoStrict API Error: The current model does not support image input. Switch to a multimodal or vision-capable model and try again.', + ) + }) }) diff --git a/src/costrict/provider/index.ts b/src/costrict/provider/index.ts index 924a387c7..f3857c0dd 100644 --- a/src/costrict/provider/index.ts +++ b/src/costrict/provider/index.ts @@ -41,13 +41,47 @@ import { isOpenAIThinkingEnabled, resolveOpenAIMaxTokens, } from '../../services/api/openai/requestBody.js' -import { fetchCoStrictModels } from './models.js' +import { fetchCoStrictModels, type CoStrictModel } from './models.js' import { getMainThreadAgentType, getActiveSkillName, } from '../../bootstrap/state.js' import { getModelMaxOutputTokens } from '../../utils/context.js' +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' +} + +function contentContainsImage(content: unknown): boolean { + if (!Array.isArray(content)) return false + + return content.some(block => { + if (!isRecord(block)) return false + if (block.type === 'image') return true + if (block.type === 'tool_result') { + return contentContainsImage(block.content) + } + return false + }) +} + +function messagesContainImages(messages: Message[]): boolean { + return messages.some(message => { + if (message.type !== 'user') return false + if (!message.message) return false + return contentContainsImage(message.message.content) + }) +} + +function isNonMultimodalModelError(message: string): boolean { + const normalized = message.toLowerCase() + return ( + normalized.includes('not a multimodal model') || + normalized.includes('does not support image') || + normalized.includes('does not support images') + ) +} + /** * CoStrict 查询路径 * 与 queryModelOpenAI 结构相同，使用 CoStrict 自定义 fetch 和 baseURL @@ -74,10 +108,11 @@ export async function* queryModelCoStrict( // 3. 从模型列表获取 maxTokens 相关参数 let defaultMaxTokens = getModelMaxOutputTokens(costrictModel).upperLimit let maxTokensParamKey: string = 'max_tokens' + let modelInfo: CoStrictModel | undefined if (creds?.access_token) { try { const modelList = await fetchCoStrictModels(baseUrl, creds.access_token) - const modelInfo = modelList.find(m => m.id === costrictModel) + modelInfo = modelList.find(m => m.id === costrictModel) if (modelInfo) { maxTokensParamKey = modelInfo.maxTokensKey || 'max_tokens' if (modelInfo.maxTokens != null) { @@ -95,6 +130,16 @@ export async function* queryModelCoStrict( // 4. 规范化消息 const messagesForAPI = normalizeMessagesForAPI(messages, tools) + if ( + modelInfo?.supportsImages === false && + messagesContainImages(messagesForAPI) + ) { + yield createAssistantAPIErrorMessage({ + content: `CoStrict API Error: Model ${costrictModel} does not support image input. Switch to a multimodal or vision-capable model and try again.`, + apiError: 'api_error', + }) + return + } // 5. 构建工具 schema const toolSchemas = await Promise.all( @@ -323,7 +368,9 @@ export async function* queryModelCoStrict( const errorMsg = error instanceof Error ? error.message : String(error) logForDebugging(`[CoStrict] Error: ${errorMsg}`, { level: 'error' }) yield createAssistantAPIErrorMessage({ - content: `CoStrict API Error: ${errorMsg}`, + content: isNonMultimodalModelError(errorMsg) + ? 'CoStrict API Error: The current model does not support image input. Switch to a multimodal or vision-capable model and try again.' + : `CoStrict API Error: ${errorMsg}`, apiError: 'api_error', error: error instanceof Error