tools system - web-fetch

2026-02-24 00:06:32 +00:00
parent 2126f3912e
commit 071e2decc3
8 changed files with 561 additions and 4 deletions
@@ -0,0 +1,25 @@
+---
+name: web_fetch
+label: Web Fetch
+description: Fetch and return the text content of a URL. Tries to get clean readable content using a cascade of strategies in order: appending .md to the URL, fetching a /llms.txt discovery file, requesting plain text via Accept header, then falling back to stripping HTML. Use when the user provides a URL and wants to read, summarize, or extract information from web content.
+language: typescript
+inputs:
+  url:
+    type: string
+    description: The URL to fetch content from
+---
+
+# Web Fetch
+
+Fetches web content with cascading fallback strategies to get the cleanest possible text.
+
+## Strategies (in order)
+
+1. **Markdown version** — Appends `.md` to the URL (works on GitHub, many docs sites)
+2. **llms.txt discovery** — Checks `/llms.txt` at the root (sites that publish LLM-friendly content)
+3. **Plain text request** — Sends `Accept: text/plain` header
+4. **HTML strip** — Fetches HTML and strips tags, scripts, and styles
+
+## Output
+
+Returns the text content with a note indicating which strategy succeeded.
@@ -0,0 +1,188 @@
+const MAX_BYTES = 50_000;
+const TIMEOUT_MS = 15_000;
+
+type FetchResult = {
+  content: string;
+  strategy: string;
+};
+
+function truncate(text: string): { text: string; truncated: boolean; originalBytes: number } {
+  const encoder = new TextEncoder();
+  const bytes = encoder.encode(text);
+  if (bytes.length <= MAX_BYTES) return { text, truncated: false, originalBytes: bytes.length };
+  const decoder = new TextDecoder();
+  return {
+    text: decoder.decode(bytes.slice(0, MAX_BYTES)),
+    truncated: true,
+    originalBytes: bytes.length,
+  };
+}
+
+function stripHtml(html: string): string {
+  // Remove script and style blocks entirely
+  let text = html
+    .replace(/<script[\s\S]*?<\/script>/gi, '')
+    .replace(/<style[\s\S]*?<\/style>/gi, '')
+    .replace(/<noscript[\s\S]*?<\/noscript>/gi, '')
+    .replace(/<nav[\s\S]*?<\/nav>/gi, '')
+    .replace(/<footer[\s\S]*?<\/footer>/gi, '')
+    .replace(/<header[\s\S]*?<\/header>/gi, '');
+
+  // Replace block-level tags with newlines
+  text = text
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/p>/gi, '\n\n')
+    .replace(/<\/div>/gi, '\n')
+    .replace(/<\/li>/gi, '\n')
+    .replace(/<\/h[1-6]>/gi, '\n\n')
+    .replace(/<\/tr>/gi, '\n')
+    .replace(/<\/td>/gi, '\t')
+    .replace(/<\/th>/gi, '\t');
+
+  // Strip remaining tags
+  text = text.replace(/<[^>]+>/g, '');
+
+  // Decode common HTML entities
+  text = text
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)));
+
+  // Collapse excessive whitespace but preserve paragraph breaks
+  text = text
+    .replace(/[ \t]+/g, ' ')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+
+  return text;
+}
+
+async function tryFetch(url: string, options: RequestInit = {}): Promise<Response | null> {
+  try {
+    const signal = AbortSignal.timeout(TIMEOUT_MS);
+    const res = await fetch(url, { ...options, signal, redirect: 'follow' });
+    if (res.ok) return res;
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+async function strategyMarkdown(url: string): Promise<FetchResult | null> {
+  const mdUrl = url.endsWith('.md') ? null : `${url.replace(/\/$/, '')}.md`;
+  if (!mdUrl) return null;
+
+  const res = await tryFetch(mdUrl);
+  if (!res) return null;
+
+  const contentType = res.headers.get('content-type') ?? '';
+  if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null;
+
+  const text = await res.text();
+  if (text.trim().startsWith('<')) return null; // Got HTML anyway
+
+  return { content: text, strategy: `markdown (${mdUrl})` };
+}
+
+async function strategyLlmsTxt(url: string): Promise<FetchResult | null> {
+  const { origin } = new URL(url);
+  const llmsUrl = `${origin}/llms.txt`;
+
+  const res = await tryFetch(llmsUrl);
+  if (!res) return null;
+
+  const text = await res.text();
+  if (!text.trim() || text.trim().startsWith('<')) return null;
+
+  return { content: text, strategy: `llms.txt (${llmsUrl})` };
+}
+
+async function strategyPlainText(url: string): Promise<FetchResult | null> {
+  const res = await tryFetch(url, { headers: { Accept: 'text/plain, text/markdown, */*;q=0.8' } });
+  if (!res) return null;
+
+  const contentType = res.headers.get('content-type') ?? '';
+  if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null;
+
+  const text = await res.text();
+  if (text.trim().startsWith('<')) return null;
+
+  return { content: text, strategy: 'plain text response' };
+}
+
+async function strategyHtmlStrip(url: string): Promise<FetchResult | null> {
+  const res = await tryFetch(url, { headers: { Accept: 'text/html,*/*;q=0.8' } });
+  if (!res) return null;
+
+  const html = await res.text();
+  const text = stripHtml(html);
+
+  if (!text.trim()) return null;
+
+  return { content: text, strategy: 'HTML (stripped)' };
+}
+
+export async function execute(
+  _toolCallId: string,
+  params: { url: string },
+  _signal: AbortSignal | undefined,
+  onUpdate?: (partial: { content: Array<{ type: string; text: string }> }) => void,
+) {
+  const { url } = params;
+
+  // Validate URL
+  let parsedUrl: URL;
+  try {
+    parsedUrl = new URL(url);
+  } catch {
+    return {
+      content: [{ type: 'text', text: `Invalid URL: ${url}` }],
+      details: { error: 'invalid_url' },
+      isError: true,
+    };
+  }
+
+  if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
+    return {
+      content: [{ type: 'text', text: `Unsupported protocol: ${parsedUrl.protocol}` }],
+      details: { error: 'unsupported_protocol' },
+      isError: true,
+    };
+  }
+
+  const strategies = [
+    { name: 'markdown', fn: () => strategyMarkdown(url) },
+    { name: 'llms.txt', fn: () => strategyLlmsTxt(url) },
+    { name: 'plain text', fn: () => strategyPlainText(url) },
+    { name: 'HTML strip', fn: () => strategyHtmlStrip(url) },
+  ];
+
+  for (const { name, fn } of strategies) {
+    onUpdate?.({ content: [{ type: 'text', text: `Trying ${name} strategy...` }] });
+
+    const result = await fn();
+    if (!result) continue;
+
+    const { text, truncated, originalBytes } = truncate(result.content);
+
+    let output = `[Fetched via ${result.strategy}]\n\n${text}`;
+    if (truncated) {
+      output += `\n\n[Content truncated: showing ${MAX_BYTES.toLocaleString()} of ${originalBytes.toLocaleString()} bytes]`;
+    }
+
+    return {
+      content: [{ type: 'text', text: output }],
+      details: { url, strategy: result.strategy, truncated, originalBytes },
+    };
+  }
+
+  return {
+    content: [{ type: 'text', text: `Failed to fetch content from: ${url}` }],
+    details: { error: 'all_strategies_failed', url },
+    isError: true,
+  };
+}