fix(security): extract shared stripHtmlToText() utility (fixes #18-24)
- Create src/utils/stripHtml.ts with he-based HTML-to-text conversion - Replace bingAdapter's inline regex+decodeHtmlEntities with stripHtmlToText() - Replace WebBrowserTool's inline regex chain with stripHtmlToText() - Add stripHtml.test.ts with 6 test cases - Update bingAdapter test expectation for whitespace normalization Test: 3068 pass, 0 fail
This commit is contained in:
parent
368dd99d01
commit
a249bbe9a6
|
|
@ -2,6 +2,7 @@ import { z } from 'zod/v4'
|
||||||
import type { ToolResultBlockParam } from 'src/Tool.js'
|
import type { ToolResultBlockParam } from 'src/Tool.js'
|
||||||
import { buildTool } from 'src/Tool.js'
|
import { buildTool } from 'src/Tool.js'
|
||||||
import { lazySchema } from 'src/utils/lazySchema.js'
|
import { lazySchema } from 'src/utils/lazySchema.js'
|
||||||
|
import { stripHtmlToText } from 'src/utils/stripHtml.js'
|
||||||
|
|
||||||
const WEB_BROWSER_TOOL_NAME = 'WebBrowser'
|
const WEB_BROWSER_TOOL_NAME = 'WebBrowser'
|
||||||
|
|
||||||
|
|
@ -117,12 +118,7 @@ Use this for:
|
||||||
const title = titleMatch?.[1]?.trim() ?? ''
|
const title = titleMatch?.[1]?.trim() ?? ''
|
||||||
|
|
||||||
// Extract text content (strip HTML tags, scripts, styles)
|
// Extract text content (strip HTML tags, scripts, styles)
|
||||||
let textContent = html
|
let textContent = stripHtmlToText(html)
|
||||||
.replace(/<script[\s\S]*?<\/script>/gi, '')
|
|
||||||
.replace(/<style[\s\S]*?<\/style>/gi, '')
|
|
||||||
.replace(/<[^>]+>/g, ' ')
|
|
||||||
.replace(/\s+/g, ' ')
|
|
||||||
.trim()
|
|
||||||
|
|
||||||
// Truncate to reasonable size
|
// Truncate to reasonable size
|
||||||
if (textContent.length > 50_000) {
|
if (textContent.length > 50_000) {
|
||||||
|
|
|
||||||
|
|
@ -244,7 +244,7 @@ describe('extractBingResults', () => {
|
||||||
`
|
`
|
||||||
const results = extractBingResults(html)
|
const results = extractBingResults(html)
|
||||||
expect(results).toHaveLength(1)
|
expect(results).toHaveLength(1)
|
||||||
expect(results[0].title).toBe('Whitespace Title')
|
expect(results[0].title).toBe('Whitespace Title')
|
||||||
})
|
})
|
||||||
|
|
||||||
test('handles snippet with HTML entities', () => {
|
test('handles snippet with HTML entities', () => {
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
import axios from 'axios'
|
import axios from 'axios'
|
||||||
import he from 'he'
|
import he from 'he'
|
||||||
import { AbortError } from 'src/utils/errors.js'
|
import { AbortError } from 'src/utils/errors.js'
|
||||||
|
import { stripHtmlToText } from 'src/utils/stripHtml.js'
|
||||||
import type { SearchResult, SearchOptions, WebSearchAdapter } from './types.js'
|
import type { SearchResult, SearchOptions, WebSearchAdapter } from './types.js'
|
||||||
|
|
||||||
const FETCH_TIMEOUT_MS = 30_000
|
const FETCH_TIMEOUT_MS = 30_000
|
||||||
|
|
@ -137,7 +138,7 @@ export function extractBingResults(html: string): SearchResult[] {
|
||||||
const url = resolveBingUrl(rawUrl)
|
const url = resolveBingUrl(rawUrl)
|
||||||
if (!url) continue
|
if (!url) continue
|
||||||
|
|
||||||
const title = decodeHtmlEntities(titleHtml.replace(/<[^>]+>/g, '').trim())
|
const title = stripHtmlToText(titleHtml)
|
||||||
|
|
||||||
// Extract snippet: try b_lineclamp → b_caption <p> → b_caption fallback
|
// Extract snippet: try b_lineclamp → b_caption <p> → b_caption fallback
|
||||||
const snippet = extractSnippet(block)
|
const snippet = extractSnippet(block)
|
||||||
|
|
@ -153,7 +154,7 @@ function extractSnippet(block: string): string | undefined {
|
||||||
const lineclampRegex = /<p[^>]*class="b_lineclamp[^"]*"[^>]*>([\s\S]*?)<\/p>/i
|
const lineclampRegex = /<p[^>]*class="b_lineclamp[^"]*"[^>]*>([\s\S]*?)<\/p>/i
|
||||||
let match = lineclampRegex.exec(block)
|
let match = lineclampRegex.exec(block)
|
||||||
if (match) {
|
if (match) {
|
||||||
return decodeHtmlEntities(match[1].replace(/<[^>]+>/g, '').trim())
|
return stripHtmlToText(match[1])
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Try <p> inside b_caption
|
// 2. Try <p> inside b_caption
|
||||||
|
|
@ -161,7 +162,7 @@ function extractSnippet(block: string): string | undefined {
|
||||||
/<div[^>]*class="b_caption[^"]*"[^>]*>[\s\S]*?<p[^>]*>([\s\S]*?)<\/p>/i
|
/<div[^>]*class="b_caption[^"]*"[^>]*>[\s\S]*?<p[^>]*>([\s\S]*?)<\/p>/i
|
||||||
match = captionPRegex.exec(block)
|
match = captionPRegex.exec(block)
|
||||||
if (match) {
|
if (match) {
|
||||||
return decodeHtmlEntities(match[1].replace(/<[^>]+>/g, '').trim())
|
return stripHtmlToText(match[1])
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Fallback: any text inside b_caption <div>
|
// 3. Fallback: any text inside b_caption <div>
|
||||||
|
|
@ -169,8 +170,8 @@ function extractSnippet(block: string): string | undefined {
|
||||||
/<div[^>]*class="b_caption[^"]*"[^>]*>([\s\S]*?)<\/div>/i
|
/<div[^>]*class="b_caption[^"]*"[^>]*>([\s\S]*?)<\/div>/i
|
||||||
const fallbackMatch = fallbackRegex.exec(block)
|
const fallbackMatch = fallbackRegex.exec(block)
|
||||||
if (fallbackMatch) {
|
if (fallbackMatch) {
|
||||||
const text = fallbackMatch[1].replace(/<[^>]+>/g, '').trim()
|
const text = stripHtmlToText(fallbackMatch[1])
|
||||||
if (text) return decodeHtmlEntities(text)
|
if (text) return text
|
||||||
}
|
}
|
||||||
|
|
||||||
return undefined
|
return undefined
|
||||||
|
|
|
||||||
30
src/utils/__tests__/stripHtml.test.ts
Normal file
30
src/utils/__tests__/stripHtml.test.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
import { describe, expect, test } from 'bun:test'
|
||||||
|
import { stripHtmlToText } from '../stripHtml.js'
|
||||||
|
|
||||||
|
describe('stripHtmlToText', () => {
|
||||||
|
test('removes HTML tags', () => {
|
||||||
|
expect(stripHtmlToText('<p>Hello <b>world</b></p>')).toBe('Hello world')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('decodes HTML entities', () => {
|
||||||
|
expect(stripHtmlToText('<script>alert(1)</script>')).toBe(
|
||||||
|
'<script>alert(1)</script>',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
test('removes script blocks', () => {
|
||||||
|
expect(stripHtmlToText('<script>evil()</script>safe')).toBe('safe')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('removes style blocks', () => {
|
||||||
|
expect(stripHtmlToText('<style>body{}</style>text')).toBe('text')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('handles empty input', () => {
|
||||||
|
expect(stripHtmlToText('')).toBe('')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('collapses excess whitespace', () => {
|
||||||
|
expect(stripHtmlToText('a b\n\n\nc')).toBe('a b\n\nc')
|
||||||
|
})
|
||||||
|
})
|
||||||
27
src/utils/stripHtml.ts
Normal file
27
src/utils/stripHtml.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
/**
|
||||||
|
* Strip HTML tags and decode entities, producing plain text.
|
||||||
|
* Uses the `he` library for entity decoding (already a dependency).
|
||||||
|
*/
|
||||||
|
import he from 'he'
|
||||||
|
|
||||||
|
export function stripHtmlToText(html: string): string {
|
||||||
|
return (
|
||||||
|
he
|
||||||
|
.decode(
|
||||||
|
html
|
||||||
|
// Remove script/style blocks and their content
|
||||||
|
.replace(/<script[\s\S]*?<\/script>/gi, '')
|
||||||
|
.replace(/<style[\s\S]*?<\/style>/gi, '')
|
||||||
|
// Replace <br>, <p>, <li>, <tr> with newlines
|
||||||
|
.replace(/<(br|p|li|tr)\b[^>]*\/?>/gi, '\n')
|
||||||
|
// Replace </p>, </li>, </tr>, </div>, </h[1-6]> with newlines
|
||||||
|
.replace(/<\/(p|li|tr|div|h[1-6])>/gi, '\n')
|
||||||
|
// Remove remaining HTML tags
|
||||||
|
.replace(/<[^>]*>/g, ''),
|
||||||
|
)
|
||||||
|
// Collapse multiple whitespace/newlines
|
||||||
|
.replace(/\n{3,}/g, '\n\n')
|
||||||
|
.replace(/[ \t]{2,}/g, ' ')
|
||||||
|
.trim()
|
||||||
|
)
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user