github/claude-code-best
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(costrict): handle image input for model compatibility

Browse Source 
## Bug 详情
CoStrict 在读取图片后会把图片放在 tool_result.content 中，OpenAI 兼容层只保留文本，导致模型没有收到真实图片内容。

非多模态模型收到图片请求时，后端会返回底层 400 错误，例如 not a multimodal model。

## 根因
OpenAI 消息转换逻辑没有处理 tool_result 中嵌套的 image block。

CoStrict 查询路径没有在发送请求前检查模型的图片输入能力，也没有归一化非多模态模型错误。

## 修复方案
将 tool_result 中的图片转换为后续 user 多模态消息，确保兼容层能传递 image_url。

在 CoStrict 模型元数据明确声明 supportsImages=false 时，提前返回英文错误提示；同时对后端非多模态错误做英文提示归一化。

## 变更要点
- Preserve image blocks nested in tool_result for OpenAI-compatible providers
- Add CoStrict image-capability gate based on supportsImages metadata
- Normalize non-multimodal backend errors to a clear English message
- Add regression tests for nested tool_result images and non-multimodal model handling

## 自测
- bun test src/costrict/provider/index.test.ts
- bun test packages/@ant/model-provider/src/shared/__tests__/openaiConvertMessages.test.ts
This commit is contained in:
IronRookieCoder 2026-05-19 11:38:16 +08:00
parent b7153a12a7
commit b36e8e4ff2
4 changed files with 230 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
packages/@ant/model-provider/src/shared/__tests__/openaiConvertMessages.test.ts
View File

@ -121,6 +121,69 @@ describe('anthropicMessagesToOpenAI', () => {
])
})
test('converts image content nested in tool_result to follow-up user image', () => {
const result = anthropicMessagesToOpenAI(
[
makeAssistantMsg([
{
type: 'tool_use' as const,
id: 'toolu_img',
name: 'Read',
input: { file_path: '1.png' },
},
]),
makeUserMsg([
{
type: 'tool_result' as const,
tool_use_id: 'toolu_img',
content: [
{
type: 'image' as const,
source: {
type: 'base64',
media_type: 'image/png',
data: 'iVBORw0KGgo=',
},
},
],
},
]),
],
[] as any,
)
expect(result).toEqual([
{
role: 'assistant',
content: null,
tool_calls: [
{
id: 'toolu_img',
type: 'function',
function: {
name: 'Read',
arguments: '{"file_path":"1.png"}',
},
},
],
},
{
role: 'tool',
tool_call_id: 'toolu_img',
content: '[image]',
},
{
role: 'user',
content: [
{
type: 'image_url',
image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' },
},
],
},
])
})
test('preserves thinking blocks as reasoning_content', () => {
const result = anthropicMessagesToOpenAI(
[

33
packages/@ant/model-provider/src/shared/openaiConvertMessages.ts
View File

@ -106,7 +106,7 @@ function convertInternalUserMessage(
// message with tool_calls. If we emit a user message first, the API will
// reject the request with "insufficient tool messages following tool_calls".
for (const tr of toolResults) {
result.push(convertToolResult(tr))
result.push(...convertToolResult(tr))
}
// 如果有图片，构建多模态 content 数组
@ -136,8 +136,11 @@ function convertInternalUserMessage(
function convertToolResult(
block: BetaToolResultBlockParam,
): ChatCompletionToolMessageParam {
): ChatCompletionMessageParam[] {
let content: string
const imageParts: Array<{ type: 'image_url'; image_url: { url: string } }> =
[]
if (typeof block.content === 'string') {
content = block.content
} else if (Array.isArray(block.content)) {
@ -145,6 +148,15 @@ function convertToolResult(
.map(c => {
if (typeof c === 'string') return c
if ('text' in c) return c.text
if ('type' in c && c.type === 'image') {
const imagePart = convertImageBlockToOpenAI(
c as unknown as Record<string, unknown>,
)
if (imagePart) {
imageParts.push(imagePart)
}
return ''
}
return ''
})
.filter(Boolean)
@ -153,11 +165,20 @@ function convertToolResult(
content = ''
}
return {
const result: ChatCompletionMessageParam[] = [{
role: 'tool',
tool_call_id: block.tool_use_id,
content,
} satisfies ChatCompletionToolMessageParam
content: content || (imageParts.length > 0 ? '[image]' : ''),
} satisfies ChatCompletionToolMessageParam]
if (imageParts.length > 0) {
result.push({
role: 'user',
content: imageParts,
} satisfies ChatCompletionUserMessageParam)
}
return result
}
function convertInternalAssistantMessage(
@ -214,7 +235,7 @@ function convertInternalAssistantMessage(
const thinkingText = (block as unknown as Record<string, unknown>)
.thinking
if (typeof thinkingText === 'string') {
reasoningParts.push(thinkingText)
reasoningParts.push(thinkingText.length > 0 ? thinkingText : ' ')
}
}
// Skip redacted_thinking, server_tool_use, etc.

98
src/costrict/provider/index.test.ts
View File

@ -1,9 +1,6 @@
import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'
import type { BetaRawMessageStreamEvent } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs'
import type {
AssistantMessage,
StreamEvent,
} from '../../types/message.js'
import type { AssistantMessage, StreamEvent } from '../../types/message.js'
function makeMessageStart(
overrides: Record<string, any> = {},
@ -88,6 +85,8 @@ async function* eventStream(events: BetaRawMessageStreamEvent[]) {
let _nextEvents: BetaRawMessageStreamEvent[] = []
let _lastCreateArgs: Record<string, any> | null = null
let _mockModelMaxTokens: number | undefined
let _mockModelSupportsImages: boolean | undefined
let _mockCreateError: Error | undefined
mock.module('openai', () => ({
default: class OpenAI {
@ -95,6 +94,7 @@ mock.module('openai', () => ({
completions: {
create: async (args: Record<string, any>) => {
_lastCreateArgs = args
if (_mockCreateError) throw _mockCreateError
return { [Symbol.asyncIterator]: async function* () {} }
},
},
@ -172,6 +172,7 @@ mock.module('./models.js', () => ({
id: 'test-model',
maxTokens: _mockModelMaxTokens,
maxTokensKey: 'max_completion_tokens',
supportsImages: _mockModelSupportsImages,
},
],
}))
@ -196,6 +197,7 @@ mock.module('../../utils/context.js', () => ({
async function runQueryModel(
events: BetaRawMessageStreamEvent[],
optionsOverrides: Record<string, unknown> = {},
messages: any[] = [],
) {
_nextEvents = events
const { queryModelCoStrict } = await import('./index.js')
@ -211,7 +213,7 @@ async function runQueryModel(
}
for await (const item of queryModelCoStrict(
[],
messages,
{ type: 'text', text: '' } as any,
[],
new AbortController().signal,
@ -231,11 +233,15 @@ beforeEach(() => {
_nextEvents = []
_lastCreateArgs = null
_mockModelMaxTokens = undefined
_mockModelSupportsImages = undefined
_mockCreateError = undefined
})
afterEach(() => {
_nextEvents = []
_mockModelMaxTokens = undefined
_mockModelSupportsImages = undefined
_mockCreateError = undefined
})
describe('queryModelCoStrict', () => {
@ -257,9 +263,7 @@ describe('queryModelCoStrict', () => {
expect(assistantMessages).toHaveLength(1)
expect(assistantMessages[0]!.message.stop_reason).toBe('end_turn')
expect(
(assistantMessages[0]!.message.content as any[]).map(
block => block.type,
),
(assistantMessages[0]!.message.content as any[]).map(block => block.type),
).toEqual(['thinking', 'text'])
})
@ -272,4 +276,82 @@ describe('queryModelCoStrict', () => {
expect(_lastCreateArgs).not.toBeNull()
expect(_lastCreateArgs!.max_completion_tokens).toBe(2048)
})
test('returns a clear error before sending images to non-multimodal model', async () => {
_mockModelSupportsImages = false
const messages = [
{
type: 'user',
uuid: 'user-img',
timestamp: new Date().toISOString(),
message: {
role: 'user',
content: [
{
type: 'tool_result',
tool_use_id: 'toolu_img',
content: [
{
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'iVBORw0KGgo=',
},
},
],
},
],
},
},
]
const { assistantMessages } = await runQueryModel([], {}, messages)
expect(_lastCreateArgs).toBeNull()
expect(assistantMessages).toHaveLength(1)
expect((assistantMessages[0]!.message.content as any[])[0].text).toContain(
'does not support image input',
)
})
test('allows images when model metadata declares multimodal support', async () => {
_mockModelSupportsImages = true
const events = [makeMessageStart(), makeMessageStop()]
const messages = [
{
type: 'user',
uuid: 'user-img',
timestamp: new Date().toISOString(),
message: {
role: 'user',
content: [
{
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'iVBORw0KGgo=',
},
},
],
},
},
]
await runQueryModel(events, {}, messages)
expect(_lastCreateArgs).not.toBeNull()
})
test('normalizes backend non-multimodal model errors', async () => {
_mockCreateError = new Error('/mnt/model is not a multimodal model')
const { assistantMessages } = await runQueryModel([], {})
expect(assistantMessages).toHaveLength(1)
expect((assistantMessages[0]!.message.content as any[])[0].text).toBe(
'CoStrict API Error: The current model does not support image input. Switch to a multimodal or vision-capable model and try again.',
)
})
})

53
src/costrict/provider/index.ts
View File

@ -41,13 +41,47 @@ import {
isOpenAIThinkingEnabled,
resolveOpenAIMaxTokens,
} from '../../services/api/openai/requestBody.js'
import { fetchCoStrictModels } from './models.js'
import { fetchCoStrictModels, type CoStrictModel } from './models.js'
import {
getMainThreadAgentType,
getActiveSkillName,
} from '../../bootstrap/state.js'
import { getModelMaxOutputTokens } from '../../utils/context.js'
function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === 'object'
}
function contentContainsImage(content: unknown): boolean {
if (!Array.isArray(content)) return false
return content.some(block => {
if (!isRecord(block)) return false
if (block.type === 'image') return true
if (block.type === 'tool_result') {
return contentContainsImage(block.content)
}
return false
})
}
function messagesContainImages(messages: Message[]): boolean {
return messages.some(message => {
if (message.type !== 'user') return false
if (!message.message) return false
return contentContainsImage(message.message.content)
})
}
function isNonMultimodalModelError(message: string): boolean {
const normalized = message.toLowerCase()
return (
normalized.includes('not a multimodal model') ||
normalized.includes('does not support image') ||
normalized.includes('does not support images')
)
}
/**
* CoStrict 查询路径
* 与 queryModelOpenAI 结构相同，使用 CoStrict 自定义 fetch 和 baseURL
@ -74,10 +108,11 @@ export async function* queryModelCoStrict(
// 3. 从模型列表获取 maxTokens 相关参数
let defaultMaxTokens = getModelMaxOutputTokens(costrictModel).upperLimit
let maxTokensParamKey: string = 'max_tokens'
let modelInfo: CoStrictModel | undefined
if (creds?.access_token) {
try {
const modelList = await fetchCoStrictModels(baseUrl, creds.access_token)
const modelInfo = modelList.find(m => m.id === costrictModel)
modelInfo = modelList.find(m => m.id === costrictModel)
if (modelInfo) {
maxTokensParamKey = modelInfo.maxTokensKey || 'max_tokens'
if (modelInfo.maxTokens != null) {
@ -95,6 +130,16 @@ export async function* queryModelCoStrict(
// 4. 规范化消息
const messagesForAPI = normalizeMessagesForAPI(messages, tools)
if (
modelInfo?.supportsImages === false &&
messagesContainImages(messagesForAPI)
) {
yield createAssistantAPIErrorMessage({
content: `CoStrict API Error: Model ${costrictModel} does not support image input. Switch to a multimodal or vision-capable model and try again.`,
apiError: 'api_error',
})
return
}
// 5. 构建工具 schema
const toolSchemas = await Promise.all(
@ -323,7 +368,9 @@ export async function* queryModelCoStrict(
const errorMsg = error instanceof Error ? error.message : String(error)
logForDebugging(`[CoStrict] Error: ${errorMsg}`, { level: 'error' })
yield createAssistantAPIErrorMessage({
content: `CoStrict API Error: ${errorMsg}`,
content: isNonMultimodalModelError(errorMsg)
? 'CoStrict API Error: The current model does not support image input. Switch to a multimodal or vision-capable model and try again.'
: `CoStrict API Error: ${errorMsg}`,
apiError: 'api_error',
error:
error instanceof Error