fix(security): extract shared stripHtmlToText() utility (fixes #18-24)
- Create src/utils/stripHtml.ts with he-based HTML-to-text conversion - Replace bingAdapter's inline regex+decodeHtmlEntities with stripHtmlToText() - Replace WebBrowserTool's inline regex chain with stripHtmlToText() - Add stripHtml.test.ts with 6 test cases - Update bingAdapter test expectation for whitespace normalization Test: 3068 pass, 0 fail
This commit is contained in:
parent
368dd99d01
commit
a249bbe9a6
|
|
@ -2,6 +2,7 @@ import { z } from 'zod/v4'
|
|||
import type { ToolResultBlockParam } from 'src/Tool.js'
|
||||
import { buildTool } from 'src/Tool.js'
|
||||
import { lazySchema } from 'src/utils/lazySchema.js'
|
||||
import { stripHtmlToText } from 'src/utils/stripHtml.js'
|
||||
|
||||
const WEB_BROWSER_TOOL_NAME = 'WebBrowser'
|
||||
|
||||
|
|
@ -117,12 +118,7 @@ Use this for:
|
|||
const title = titleMatch?.[1]?.trim() ?? ''
|
||||
|
||||
// Extract text content (strip HTML tags, scripts, styles)
|
||||
let textContent = html
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
let textContent = stripHtmlToText(html)
|
||||
|
||||
// Truncate to reasonable size
|
||||
if (textContent.length > 50_000) {
|
||||
|
|
|
|||
|
|
@ -244,7 +244,7 @@ describe('extractBingResults', () => {
|
|||
`
|
||||
const results = extractBingResults(html)
|
||||
expect(results).toHaveLength(1)
|
||||
expect(results[0].title).toBe('Whitespace Title')
|
||||
expect(results[0].title).toBe('Whitespace Title')
|
||||
})
|
||||
|
||||
test('handles snippet with HTML entities', () => {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
import axios from 'axios'
|
||||
import he from 'he'
|
||||
import { AbortError } from 'src/utils/errors.js'
|
||||
import { stripHtmlToText } from 'src/utils/stripHtml.js'
|
||||
import type { SearchResult, SearchOptions, WebSearchAdapter } from './types.js'
|
||||
|
||||
const FETCH_TIMEOUT_MS = 30_000
|
||||
|
|
@ -137,7 +138,7 @@ export function extractBingResults(html: string): SearchResult[] {
|
|||
const url = resolveBingUrl(rawUrl)
|
||||
if (!url) continue
|
||||
|
||||
const title = decodeHtmlEntities(titleHtml.replace(/<[^>]+>/g, '').trim())
|
||||
const title = stripHtmlToText(titleHtml)
|
||||
|
||||
// Extract snippet: try b_lineclamp → b_caption <p> → b_caption fallback
|
||||
const snippet = extractSnippet(block)
|
||||
|
|
@ -153,7 +154,7 @@ function extractSnippet(block: string): string | undefined {
|
|||
const lineclampRegex = /<p[^>]*class="b_lineclamp[^"]*"[^>]*>([\s\S]*?)<\/p>/i
|
||||
let match = lineclampRegex.exec(block)
|
||||
if (match) {
|
||||
return decodeHtmlEntities(match[1].replace(/<[^>]+>/g, '').trim())
|
||||
return stripHtmlToText(match[1])
|
||||
}
|
||||
|
||||
// 2. Try <p> inside b_caption
|
||||
|
|
@ -161,7 +162,7 @@ function extractSnippet(block: string): string | undefined {
|
|||
/<div[^>]*class="b_caption[^"]*"[^>]*>[\s\S]*?<p[^>]*>([\s\S]*?)<\/p>/i
|
||||
match = captionPRegex.exec(block)
|
||||
if (match) {
|
||||
return decodeHtmlEntities(match[1].replace(/<[^>]+>/g, '').trim())
|
||||
return stripHtmlToText(match[1])
|
||||
}
|
||||
|
||||
// 3. Fallback: any text inside b_caption <div>
|
||||
|
|
@ -169,8 +170,8 @@ function extractSnippet(block: string): string | undefined {
|
|||
/<div[^>]*class="b_caption[^"]*"[^>]*>([\s\S]*?)<\/div>/i
|
||||
const fallbackMatch = fallbackRegex.exec(block)
|
||||
if (fallbackMatch) {
|
||||
const text = fallbackMatch[1].replace(/<[^>]+>/g, '').trim()
|
||||
if (text) return decodeHtmlEntities(text)
|
||||
const text = stripHtmlToText(fallbackMatch[1])
|
||||
if (text) return text
|
||||
}
|
||||
|
||||
return undefined
|
||||
|
|
|
|||
30
src/utils/__tests__/stripHtml.test.ts
Normal file
30
src/utils/__tests__/stripHtml.test.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import { describe, expect, test } from 'bun:test'
|
||||
import { stripHtmlToText } from '../stripHtml.js'
|
||||
|
||||
describe('stripHtmlToText', () => {
|
||||
test('removes HTML tags', () => {
|
||||
expect(stripHtmlToText('<p>Hello <b>world</b></p>')).toBe('Hello world')
|
||||
})
|
||||
|
||||
test('decodes HTML entities', () => {
|
||||
expect(stripHtmlToText('<script>alert(1)</script>')).toBe(
|
||||
'<script>alert(1)</script>',
|
||||
)
|
||||
})
|
||||
|
||||
test('removes script blocks', () => {
|
||||
expect(stripHtmlToText('<script>evil()</script>safe')).toBe('safe')
|
||||
})
|
||||
|
||||
test('removes style blocks', () => {
|
||||
expect(stripHtmlToText('<style>body{}</style>text')).toBe('text')
|
||||
})
|
||||
|
||||
test('handles empty input', () => {
|
||||
expect(stripHtmlToText('')).toBe('')
|
||||
})
|
||||
|
||||
test('collapses excess whitespace', () => {
|
||||
expect(stripHtmlToText('a b\n\n\nc')).toBe('a b\n\nc')
|
||||
})
|
||||
})
|
||||
27
src/utils/stripHtml.ts
Normal file
27
src/utils/stripHtml.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
/**
|
||||
* Strip HTML tags and decode entities, producing plain text.
|
||||
* Uses the `he` library for entity decoding (already a dependency).
|
||||
*/
|
||||
import he from 'he'
|
||||
|
||||
export function stripHtmlToText(html: string): string {
|
||||
return (
|
||||
he
|
||||
.decode(
|
||||
html
|
||||
// Remove script/style blocks and their content
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, '')
|
||||
// Replace <br>, <p>, <li>, <tr> with newlines
|
||||
.replace(/<(br|p|li|tr)\b[^>]*\/?>/gi, '\n')
|
||||
// Replace </p>, </li>, </tr>, </div>, </h[1-6]> with newlines
|
||||
.replace(/<\/(p|li|tr|div|h[1-6])>/gi, '\n')
|
||||
// Remove remaining HTML tags
|
||||
.replace(/<[^>]*>/g, ''),
|
||||
)
|
||||
// Collapse multiple whitespace/newlines
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.replace(/[ \t]{2,}/g, ' ')
|
||||
.trim()
|
||||
)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user