const MAX_BYTES = 50_000; const TIMEOUT_MS = 15_000; type FetchResult = { content: string; strategy: string; }; function truncate(text: string): { text: string; truncated: boolean; originalBytes: number } { const encoder = new TextEncoder(); const bytes = encoder.encode(text); if (bytes.length <= MAX_BYTES) return { text, truncated: false, originalBytes: bytes.length }; const decoder = new TextDecoder(); return { text: decoder.decode(bytes.slice(0, MAX_BYTES)), truncated: true, originalBytes: bytes.length, }; } function stripHtml(html: string): string { // Remove script and style blocks entirely let text = html .replace(//gi, '') .replace(//gi, '') .replace(//gi, '') .replace(//gi, '') .replace(//gi, '') .replace(//gi, ''); // Replace block-level tags with newlines text = text .replace(//gi, '\n') .replace(/<\/p>/gi, '\n\n') .replace(/<\/div>/gi, '\n') .replace(/<\/li>/gi, '\n') .replace(/<\/h[1-6]>/gi, '\n\n') .replace(/<\/tr>/gi, '\n') .replace(/<\/td>/gi, '\t') .replace(/<\/th>/gi, '\t'); // Strip remaining tags text = text.replace(/<[^>]+>/g, ''); // Decode common HTML entities text = text .replace(/&/g, '&') .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') .replace(/'/g, "'") .replace(/ /g, ' ') .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code))); // Collapse excessive whitespace but preserve paragraph breaks text = text .replace(/[ \t]+/g, ' ') .replace(/\n{3,}/g, '\n\n') .trim(); return text; } async function tryFetch(url: string, options: RequestInit = {}): Promise { try { const signal = AbortSignal.timeout(TIMEOUT_MS); const res = await fetch(url, { ...options, signal, redirect: 'follow' }); if (res.ok) return res; return null; } catch { return null; } } async function strategyMarkdown(url: string): Promise { const mdUrl = url.endsWith('.md') ? null : `${url.replace(/\/$/, '')}.md`; if (!mdUrl) return null; const res = await tryFetch(mdUrl); if (!res) return null; const contentType = res.headers.get('content-type') ?? ''; if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null; const text = await res.text(); if (text.trim().startsWith('<')) return null; // Got HTML anyway return { content: text, strategy: `markdown (${mdUrl})` }; } async function strategyLlmsTxt(url: string): Promise { const { origin } = new URL(url); const llmsUrl = `${origin}/llms.txt`; const res = await tryFetch(llmsUrl); if (!res) return null; const text = await res.text(); if (!text.trim() || text.trim().startsWith('<')) return null; return { content: text, strategy: `llms.txt (${llmsUrl})` }; } async function strategyPlainText(url: string): Promise { const res = await tryFetch(url, { headers: { Accept: 'text/plain, text/markdown, */*;q=0.8' } }); if (!res) return null; const contentType = res.headers.get('content-type') ?? ''; if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null; const text = await res.text(); if (text.trim().startsWith('<')) return null; return { content: text, strategy: 'plain text response' }; } async function strategyHtmlStrip(url: string): Promise { const res = await tryFetch(url, { headers: { Accept: 'text/html,*/*;q=0.8' } }); if (!res) return null; const html = await res.text(); const text = stripHtml(html); if (!text.trim()) return null; return { content: text, strategy: 'HTML (stripped)' }; } export async function execute( _toolCallId: string, params: { url: string }, _signal: AbortSignal | undefined, onUpdate?: (partial: { content: Array<{ type: string; text: string }> }) => void, ) { const { url } = params; // Validate URL let parsedUrl: URL; try { parsedUrl = new URL(url); } catch { return { content: [{ type: 'text', text: `Invalid URL: ${url}` }], details: { error: 'invalid_url' }, isError: true, }; } if (!['http:', 'https:'].includes(parsedUrl.protocol)) { return { content: [{ type: 'text', text: `Unsupported protocol: ${parsedUrl.protocol}` }], details: { error: 'unsupported_protocol' }, isError: true, }; } const strategies = [ { name: 'markdown', fn: () => strategyMarkdown(url) }, { name: 'llms.txt', fn: () => strategyLlmsTxt(url) }, { name: 'plain text', fn: () => strategyPlainText(url) }, { name: 'HTML strip', fn: () => strategyHtmlStrip(url) }, ]; for (const { name, fn } of strategies) { onUpdate?.({ content: [{ type: 'text', text: `Trying ${name} strategy...` }] }); const result = await fn(); if (!result) continue; const { text, truncated, originalBytes } = truncate(result.content); let output = `[Fetched via ${result.strategy}]\n\n${text}`; if (truncated) { output += `\n\n[Content truncated: showing ${MAX_BYTES.toLocaleString()} of ${originalBytes.toLocaleString()} bytes]`; } return { content: [{ type: 'text', text: output }], details: { url, strategy: result.strategy, truncated, originalBytes }, }; } return { content: [{ type: 'text', text: `Failed to fetch content from: ${url}` }], details: { error: 'all_strategies_failed', url }, isError: true, }; }