fix(security): extract shared stripHtmlToText() utility (fixes #18-24)

- Create src/utils/stripHtml.ts with he-based HTML-to-text conversion
- Replace bingAdapter's inline regex+decodeHtmlEntities with stripHtmlToText()
- Replace WebBrowserTool's inline regex chain with stripHtmlToText()
- Add stripHtml.test.ts with 6 test cases
- Update bingAdapter test expectation for whitespace normalization

Test: 3068 pass, 0 fail
This commit is contained in:
James Feng 2026-06-03 17:05:49 +08:00
parent 368dd99d01
commit a249bbe9a6
5 changed files with 66 additions and 12 deletions

View File

@ -2,6 +2,7 @@ import { z } from 'zod/v4'
import type { ToolResultBlockParam } from 'src/Tool.js'
import { buildTool } from 'src/Tool.js'
import { lazySchema } from 'src/utils/lazySchema.js'
import { stripHtmlToText } from 'src/utils/stripHtml.js'
const WEB_BROWSER_TOOL_NAME = 'WebBrowser'
@ -117,12 +118,7 @@ Use this for:
const title = titleMatch?.[1]?.trim() ?? ''
// Extract text content (strip HTML tags, scripts, styles)
let textContent = html
.replace(/<script[\s\S]*?<\/script>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
.replace(/<[^>]+>/g, ' ')
.replace(/\s+/g, ' ')
.trim()
let textContent = stripHtmlToText(html)
// Truncate to reasonable size
if (textContent.length > 50_000) {

View File

@ -244,7 +244,7 @@ describe('extractBingResults', () => {
`
const results = extractBingResults(html)
expect(results).toHaveLength(1)
expect(results[0].title).toBe('Whitespace Title')
expect(results[0].title).toBe('Whitespace Title')
})
test('handles snippet with HTML entities', () => {

View File

@ -6,6 +6,7 @@
import axios from 'axios'
import he from 'he'
import { AbortError } from 'src/utils/errors.js'
import { stripHtmlToText } from 'src/utils/stripHtml.js'
import type { SearchResult, SearchOptions, WebSearchAdapter } from './types.js'
const FETCH_TIMEOUT_MS = 30_000
@ -137,7 +138,7 @@ export function extractBingResults(html: string): SearchResult[] {
const url = resolveBingUrl(rawUrl)
if (!url) continue
const title = decodeHtmlEntities(titleHtml.replace(/<[^>]+>/g, '').trim())
const title = stripHtmlToText(titleHtml)
// Extract snippet: try b_lineclamp → b_caption <p> → b_caption fallback
const snippet = extractSnippet(block)
@ -153,7 +154,7 @@ function extractSnippet(block: string): string | undefined {
const lineclampRegex = /<p[^>]*class="b_lineclamp[^"]*"[^>]*>([\s\S]*?)<\/p>/i
let match = lineclampRegex.exec(block)
if (match) {
return decodeHtmlEntities(match[1].replace(/<[^>]+>/g, '').trim())
return stripHtmlToText(match[1])
}
// 2. Try <p> inside b_caption
@ -161,7 +162,7 @@ function extractSnippet(block: string): string | undefined {
/<div[^>]*class="b_caption[^"]*"[^>]*>[\s\S]*?<p[^>]*>([\s\S]*?)<\/p>/i
match = captionPRegex.exec(block)
if (match) {
return decodeHtmlEntities(match[1].replace(/<[^>]+>/g, '').trim())
return stripHtmlToText(match[1])
}
// 3. Fallback: any text inside b_caption <div>
@ -169,8 +170,8 @@ function extractSnippet(block: string): string | undefined {
/<div[^>]*class="b_caption[^"]*"[^>]*>([\s\S]*?)<\/div>/i
const fallbackMatch = fallbackRegex.exec(block)
if (fallbackMatch) {
const text = fallbackMatch[1].replace(/<[^>]+>/g, '').trim()
if (text) return decodeHtmlEntities(text)
const text = stripHtmlToText(fallbackMatch[1])
if (text) return text
}
return undefined

View File

@ -0,0 +1,30 @@
import { describe, expect, test } from 'bun:test'
import { stripHtmlToText } from '../stripHtml.js'
describe('stripHtmlToText', () => {
test('removes HTML tags', () => {
expect(stripHtmlToText('<p>Hello <b>world</b></p>')).toBe('Hello world')
})
test('decodes HTML entities', () => {
expect(stripHtmlToText('&lt;script&gt;alert(1)&lt;/script&gt;')).toBe(
'<script>alert(1)</script>',
)
})
test('removes script blocks', () => {
expect(stripHtmlToText('<script>evil()</script>safe')).toBe('safe')
})
test('removes style blocks', () => {
expect(stripHtmlToText('<style>body{}</style>text')).toBe('text')
})
test('handles empty input', () => {
expect(stripHtmlToText('')).toBe('')
})
test('collapses excess whitespace', () => {
expect(stripHtmlToText('a b\n\n\nc')).toBe('a b\n\nc')
})
})

27
src/utils/stripHtml.ts Normal file
View File

@ -0,0 +1,27 @@
/**
* Strip HTML tags and decode entities, producing plain text.
* Uses the `he` library for entity decoding (already a dependency).
*/
import he from 'he'
export function stripHtmlToText(html: string): string {
return (
he
.decode(
html
// Remove script/style blocks and their content
.replace(/<script[\s\S]*?<\/script>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
// Replace <br>, <p>, <li>, <tr> with newlines
.replace(/<(br|p|li|tr)\b[^>]*\/?>/gi, '\n')
// Replace </p>, </li>, </tr>, </div>, </h[1-6]> with newlines
.replace(/<\/(p|li|tr|div|h[1-6])>/gi, '\n')
// Remove remaining HTML tags
.replace(/<[^>]*>/g, ''),
)
// Collapse multiple whitespace/newlines
.replace(/\n{3,}/g, '\n\n')
.replace(/[ \t]{2,}/g, ' ')
.trim()
)
}