Merge pull request #129 from IronRookieCoder/fix/image-multimodal-model-support

fix(costrict): handle image input for model compatibility

packages/@ant/model-provider/src/shared/__tests__/openaiConvertMessages.test.ts

@@ -121,6 +121,69 @@ describe('anthropicMessagesToOpenAI', () => {
     ])
   })
 
+  test('converts image content nested in tool_result to follow-up user image', () => {
+    const result = anthropicMessagesToOpenAI(
+      [
+        makeAssistantMsg([
+          {
+            type: 'tool_use' as const,
+            id: 'toolu_img',
+            name: 'Read',
+            input: { file_path: '1.png' },
+          },
+        ]),
+        makeUserMsg([
+          {
+            type: 'tool_result' as const,
+            tool_use_id: 'toolu_img',
+            content: [
+              {
+                type: 'image' as const,
+                source: {
+                  type: 'base64',
+                  media_type: 'image/png',
+                  data: 'iVBORw0KGgo=',
+                },
+              },
+            ],
+          },
+        ]),
+      ],
+      [] as any,
+    )
+
+    expect(result).toEqual([
+      {
+        role: 'assistant',
+        content: null,
+        tool_calls: [
+          {
+            id: 'toolu_img',
+            type: 'function',
+            function: {
+              name: 'Read',
+              arguments: '{"file_path":"1.png"}',
+            },
+          },
+        ],
+      },
+      {
+        role: 'tool',
+        tool_call_id: 'toolu_img',
+        content: '[image]',
+      },
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'image_url',
+            image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' },
+          },
+        ],
+      },
+    ])
+  })
+
   test('preserves thinking blocks as reasoning_content', () => {
     const result = anthropicMessagesToOpenAI(
       [

packages/@ant/model-provider/src/shared/openaiConvertMessages.ts

@@ -106,7 +106,7 @@ function convertInternalUserMessage(
     // message with tool_calls. If we emit a user message first, the API will
     // reject the request with "insufficient tool messages following tool_calls".
     for (const tr of toolResults) {
-      result.push(convertToolResult(tr))
+      result.push(...convertToolResult(tr))
     }
 
     // 如果有图片，构建多模态 content 数组
@@ -136,8 +136,11 @@ function convertInternalUserMessage(
 
 function convertToolResult(
   block: BetaToolResultBlockParam,
-): ChatCompletionToolMessageParam {
+): ChatCompletionMessageParam[] {
   let content: string
+  const imageParts: Array<{ type: 'image_url'; image_url: { url: string } }> =
+    []
+
   if (typeof block.content === 'string') {
     content = block.content
   } else if (Array.isArray(block.content)) {
@@ -145,6 +148,15 @@ function convertToolResult(
       .map(c => {
         if (typeof c === 'string') return c
         if ('text' in c) return c.text
+        if ('type' in c && c.type === 'image') {
+          const imagePart = convertImageBlockToOpenAI(
+            c as unknown as Record<string, unknown>,
+          )
+          if (imagePart) {
+            imageParts.push(imagePart)
+          }
+          return ''
+        }
         return ''
       })
       .filter(Boolean)
@@ -153,11 +165,20 @@ function convertToolResult(
     content = ''
   }
 
-  return {
+  const result: ChatCompletionMessageParam[] = [{
     role: 'tool',
     tool_call_id: block.tool_use_id,
-    content,
+    content: content || (imageParts.length > 0 ? '[image]' : ''),
-  } satisfies ChatCompletionToolMessageParam
+  } satisfies ChatCompletionToolMessageParam]
+
+  if (imageParts.length > 0) {
+    result.push({
+      role: 'user',
+      content: imageParts,
+    } satisfies ChatCompletionUserMessageParam)
+  }
+
+  return result
 }
 
 function convertInternalAssistantMessage(
@@ -214,7 +235,7 @@ function convertInternalAssistantMessage(
       const thinkingText = (block as unknown as Record<string, unknown>)
         .thinking
       if (typeof thinkingText === 'string') {
-        reasoningParts.push(thinkingText)
+        reasoningParts.push(thinkingText.length > 0 ? thinkingText : ' ')
       }
     }
     // Skip redacted_thinking, server_tool_use, etc.

src/costrict/provider/index.test.ts

@@ -1,9 +1,6 @@
 import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'
 import type { BetaRawMessageStreamEvent } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs'
-import type {
+import type { AssistantMessage, StreamEvent } from '../../types/message.js'
-  AssistantMessage,
-  StreamEvent,
-} from '../../types/message.js'
 
 function makeMessageStart(
   overrides: Record<string, any> = {},
@@ -88,6 +85,8 @@ async function* eventStream(events: BetaRawMessageStreamEvent[]) {
 let _nextEvents: BetaRawMessageStreamEvent[] = []
 let _lastCreateArgs: Record<string, any> | null = null
 let _mockModelMaxTokens: number | undefined
+let _mockModelSupportsImages: boolean | undefined
+let _mockCreateError: Error | undefined
 
 mock.module('openai', () => ({
   default: class OpenAI {
@@ -95,6 +94,7 @@ mock.module('openai', () => ({
       completions: {
         create: async (args: Record<string, any>) => {
           _lastCreateArgs = args
+          if (_mockCreateError) throw _mockCreateError
           return { [Symbol.asyncIterator]: async function* () {} }
         },
       },
@@ -172,6 +172,7 @@ mock.module('./models.js', () => ({
       id: 'test-model',
       maxTokens: _mockModelMaxTokens,
       maxTokensKey: 'max_completion_tokens',
+      supportsImages: _mockModelSupportsImages,
     },
   ],
 }))
@@ -196,6 +197,7 @@ mock.module('../../utils/context.js', () => ({
 async function runQueryModel(
   events: BetaRawMessageStreamEvent[],
   optionsOverrides: Record<string, unknown> = {},
+  messages: any[] = [],
 ) {
   _nextEvents = events
   const { queryModelCoStrict } = await import('./index.js')
@@ -211,7 +213,7 @@ async function runQueryModel(
   }
 
   for await (const item of queryModelCoStrict(
-    [],
+    messages,
     { type: 'text', text: '' } as any,
     [],
     new AbortController().signal,
@@ -231,11 +233,15 @@ beforeEach(() => {
   _nextEvents = []
   _lastCreateArgs = null
   _mockModelMaxTokens = undefined
+  _mockModelSupportsImages = undefined
+  _mockCreateError = undefined
 })
 
 afterEach(() => {
   _nextEvents = []
   _mockModelMaxTokens = undefined
+  _mockModelSupportsImages = undefined
+  _mockCreateError = undefined
 })
 
 describe('queryModelCoStrict', () => {
@@ -257,9 +263,7 @@ describe('queryModelCoStrict', () => {
     expect(assistantMessages).toHaveLength(1)
     expect(assistantMessages[0]!.message.stop_reason).toBe('end_turn')
     expect(
-      (assistantMessages[0]!.message.content as any[]).map(
+      (assistantMessages[0]!.message.content as any[]).map(block => block.type),
-        block => block.type,
-      ),
     ).toEqual(['thinking', 'text'])
   })
 
@@ -272,4 +276,82 @@ describe('queryModelCoStrict', () => {
     expect(_lastCreateArgs).not.toBeNull()
     expect(_lastCreateArgs!.max_completion_tokens).toBe(2048)
   })
+
+  test('returns a clear error before sending images to non-multimodal model', async () => {
+    _mockModelSupportsImages = false
+    const messages = [
+      {
+        type: 'user',
+        uuid: 'user-img',
+        timestamp: new Date().toISOString(),
+        message: {
+          role: 'user',
+          content: [
+            {
+              type: 'tool_result',
+              tool_use_id: 'toolu_img',
+              content: [
+                {
+                  type: 'image',
+                  source: {
+                    type: 'base64',
+                    media_type: 'image/png',
+                    data: 'iVBORw0KGgo=',
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      },
+    ]
+
+    const { assistantMessages } = await runQueryModel([], {}, messages)
+
+    expect(_lastCreateArgs).toBeNull()
+    expect(assistantMessages).toHaveLength(1)
+    expect((assistantMessages[0]!.message.content as any[])[0].text).toContain(
+      'does not support image input',
+    )
+  })
+
+  test('allows images when model metadata declares multimodal support', async () => {
+    _mockModelSupportsImages = true
+    const events = [makeMessageStart(), makeMessageStop()]
+    const messages = [
+      {
+        type: 'user',
+        uuid: 'user-img',
+        timestamp: new Date().toISOString(),
+        message: {
+          role: 'user',
+          content: [
+            {
+              type: 'image',
+              source: {
+                type: 'base64',
+                media_type: 'image/png',
+                data: 'iVBORw0KGgo=',
+              },
+            },
+          ],
+        },
+      },
+    ]
+
+    await runQueryModel(events, {}, messages)
+
+    expect(_lastCreateArgs).not.toBeNull()
+  })
+
+  test('normalizes backend non-multimodal model errors', async () => {
+    _mockCreateError = new Error('/mnt/model is not a multimodal model')
+
+    const { assistantMessages } = await runQueryModel([], {})
+
+    expect(assistantMessages).toHaveLength(1)
+    expect((assistantMessages[0]!.message.content as any[])[0].text).toBe(
+      'CoStrict API Error: The current model does not support image input. Switch to a multimodal or vision-capable model and try again.',
+    )
+  })
 })

src/costrict/provider/index.ts

@@ -41,13 +41,47 @@ import {
   isOpenAIThinkingEnabled,
   resolveOpenAIMaxTokens,
 } from '../../services/api/openai/requestBody.js'
-import { fetchCoStrictModels } from './models.js'
+import { fetchCoStrictModels, type CoStrictModel } from './models.js'
 import {
   getMainThreadAgentType,
   getActiveSkillName,
 } from '../../bootstrap/state.js'
 import { getModelMaxOutputTokens } from '../../utils/context.js'
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object'
+}
+
+function contentContainsImage(content: unknown): boolean {
+  if (!Array.isArray(content)) return false
+
+  return content.some(block => {
+    if (!isRecord(block)) return false
+    if (block.type === 'image') return true
+    if (block.type === 'tool_result') {
+      return contentContainsImage(block.content)
+    }
+    return false
+  })
+}
+
+function messagesContainImages(messages: Message[]): boolean {
+  return messages.some(message => {
+    if (message.type !== 'user') return false
+    if (!message.message) return false
+    return contentContainsImage(message.message.content)
+  })
+}
+
+function isNonMultimodalModelError(message: string): boolean {
+  const normalized = message.toLowerCase()
+  return (
+    normalized.includes('not a multimodal model') ||
+    normalized.includes('does not support image') ||
+    normalized.includes('does not support images')
+  )
+}
+
 /**
  * CoStrict 查询路径
  * 与 queryModelOpenAI 结构相同，使用 CoStrict 自定义 fetch 和 baseURL
@@ -74,10 +108,11 @@ export async function* queryModelCoStrict(
     // 3. 从模型列表获取 maxTokens 相关参数
     let defaultMaxTokens = getModelMaxOutputTokens(costrictModel).upperLimit
     let maxTokensParamKey: string = 'max_tokens'
+    let modelInfo: CoStrictModel | undefined
     if (creds?.access_token) {
       try {
         const modelList = await fetchCoStrictModels(baseUrl, creds.access_token)
-        const modelInfo = modelList.find(m => m.id === costrictModel)
+        modelInfo = modelList.find(m => m.id === costrictModel)
         if (modelInfo) {
           maxTokensParamKey = modelInfo.maxTokensKey || 'max_tokens'
           if (modelInfo.maxTokens != null) {
@@ -95,6 +130,16 @@ export async function* queryModelCoStrict(
 
     // 4. 规范化消息
     const messagesForAPI = normalizeMessagesForAPI(messages, tools)
+    if (
+      modelInfo?.supportsImages === false &&
+      messagesContainImages(messagesForAPI)
+    ) {
+      yield createAssistantAPIErrorMessage({
+        content: `CoStrict API Error: Model ${costrictModel} does not support image input. Switch to a multimodal or vision-capable model and try again.`,
+        apiError: 'api_error',
+      })
+      return
+    }
 
     // 5. 构建工具 schema
     const toolSchemas = await Promise.all(
@@ -323,7 +368,9 @@ export async function* queryModelCoStrict(
     const errorMsg = error instanceof Error ? error.message : String(error)
     logForDebugging(`[CoStrict] Error: ${errorMsg}`, { level: 'error' })
     yield createAssistantAPIErrorMessage({
-      content: `CoStrict API Error: ${errorMsg}`,
+      content: isNonMultimodalModelError(errorMsg)
+        ? 'CoStrict API Error: The current model does not support image input. Switch to a multimodal or vision-capable model and try again.'
+        : `CoStrict API Error: ${errorMsg}`,
       apiError: 'api_error',
       error:
         error instanceof Error