platform/seed/tools/web-fetch/index.ts

const MAX_BYTES = 50_000;
const TIMEOUT_MS = 15_000;

type FetchResult = {
  content: string;
  strategy: string;
};

function truncate(text: string): { text: string; truncated: boolean; originalBytes: number } {
  const encoder = new TextEncoder();
  const bytes = encoder.encode(text);
  if (bytes.length <= MAX_BYTES) return { text, truncated: false, originalBytes: bytes.length };
  const decoder = new TextDecoder();
  return {
    text: decoder.decode(bytes.slice(0, MAX_BYTES)),
    truncated: true,
    originalBytes: bytes.length,
  };
}

function stripHtml(html: string): string {
  // Remove script and style blocks entirely
  let text = html
    .replace(/<script[\s\S]*?<\/script>/gi, '')
    .replace(/<style[\s\S]*?<\/style>/gi, '')
    .replace(/<noscript[\s\S]*?<\/noscript>/gi, '')
    .replace(/<nav[\s\S]*?<\/nav>/gi, '')
    .replace(/<footer[\s\S]*?<\/footer>/gi, '')
    .replace(/<header[\s\S]*?<\/header>/gi, '');

  // Replace block-level tags with newlines
  text = text
    .replace(/<br\s*\/?>/gi, '\n')
    .replace(/<\/p>/gi, '\n\n')
    .replace(/<\/div>/gi, '\n')
    .replace(/<\/li>/gi, '\n')
    .replace(/<\/h[1-6]>/gi, '\n\n')
    .replace(/<\/tr>/gi, '\n')
    .replace(/<\/td>/gi, '\t')
    .replace(/<\/th>/gi, '\t');

  // Strip remaining tags
  text = text.replace(/<[^>]+>/g, '');

  // Decode common HTML entities
  text = text
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&nbsp;/g, ' ')
    .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)));

  // Collapse excessive whitespace but preserve paragraph breaks
  text = text
    .replace(/[ \t]+/g, ' ')
    .replace(/\n{3,}/g, '\n\n')
    .trim();

  return text;
}

async function tryFetch(url: string, options: RequestInit = {}): Promise<Response | null> {
  try {
    const signal = AbortSignal.timeout(TIMEOUT_MS);
    const res = await fetch(url, { ...options, signal, redirect: 'follow' });
    if (res.ok) return res;
    return null;
  } catch {
    return null;
  }
}

async function strategyMarkdown(url: string): Promise<FetchResult | null> {
  const mdUrl = url.endsWith('.md') ? null : `${url.replace(/\/$/, '')}.md`;
  if (!mdUrl) return null;

  const res = await tryFetch(mdUrl);
  if (!res) return null;

  const contentType = res.headers.get('content-type') ?? '';
  if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null;

  const text = await res.text();
  if (text.trim().startsWith('<')) return null; // Got HTML anyway

  return { content: text, strategy: `markdown (${mdUrl})` };
}

async function strategyLlmsTxt(url: string): Promise<FetchResult | null> {
  const { origin } = new URL(url);
  const llmsUrl = `${origin}/llms.txt`;

  const res = await tryFetch(llmsUrl);
  if (!res) return null;

  const text = await res.text();
  if (!text.trim() || text.trim().startsWith('<')) return null;

  return { content: text, strategy: `llms.txt (${llmsUrl})` };
}

async function strategyPlainText(url: string): Promise<FetchResult | null> {
  const res = await tryFetch(url, { headers: { Accept: 'text/plain, text/markdown, */*;q=0.8' } });
  if (!res) return null;

  const contentType = res.headers.get('content-type') ?? '';
  if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null;

  const text = await res.text();
  if (text.trim().startsWith('<')) return null;

  return { content: text, strategy: 'plain text response' };
}

async function strategyHtmlStrip(url: string): Promise<FetchResult | null> {
  const res = await tryFetch(url, { headers: { Accept: 'text/html,*/*;q=0.8' } });
  if (!res) return null;

  const html = await res.text();
  const text = stripHtml(html);

  if (!text.trim()) return null;

  return { content: text, strategy: 'HTML (stripped)' };
}

export async function execute(
  _toolCallId: string,
  params: { url: string },
  _signal: AbortSignal | undefined,
  onUpdate?: (partial: { content: Array<{ type: string; text: string }> }) => void,
) {
  const { url } = params;

  // Validate URL
  let parsedUrl: URL;
  try {
    parsedUrl = new URL(url);
  } catch {
    return {
      content: [{ type: 'text', text: `Invalid URL: ${url}` }],
      details: { error: 'invalid_url' },
      isError: true,
    };
  }

  if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
    return {
      content: [{ type: 'text', text: `Unsupported protocol: ${parsedUrl.protocol}` }],
      details: { error: 'unsupported_protocol' },
      isError: true,
    };
  }

  const strategies = [
    { name: 'markdown', fn: () => strategyMarkdown(url) },
    { name: 'llms.txt', fn: () => strategyLlmsTxt(url) },
    { name: 'plain text', fn: () => strategyPlainText(url) },
    { name: 'HTML strip', fn: () => strategyHtmlStrip(url) },
  ];

  for (const { name, fn } of strategies) {
    onUpdate?.({ content: [{ type: 'text', text: `Trying ${name} strategy...` }] });

    const result = await fn();
    if (!result) continue;

    const { text, truncated, originalBytes } = truncate(result.content);

    let output = `[Fetched via ${result.strategy}]\n\n${text}`;
    if (truncated) {
      output += `\n\n[Content truncated: showing ${MAX_BYTES.toLocaleString()} of ${originalBytes.toLocaleString()} bytes]`;
    }

    return {
      content: [{ type: 'text', text: output }],
      details: { url, strategy: result.strategy, truncated, originalBytes },
    };
  }

  return {
    content: [{ type: 'text', text: `Failed to fetch content from: ${url}` }],
    details: { error: 'all_strategies_failed', url },
    isError: true,
  };
}