tools system - web-fetch

seed/extensions/tool-loader/index.ts

@@ -0,0 +1,237 @@
+import type { ExtensionAPI } from '@mariozechner/pi-coding-agent';
+import { Type, type TSchema } from '@sinclair/typebox';
+import { readdirSync, existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+type ToolParamType = 'string' | 'number' | 'boolean' | 'enum';
+
+type ToolParam = {
+  type: ToolParamType;
+  description: string;
+  values?: string[];
+  default?: unknown;
+  optional?: boolean;
+};
+
+type ToolMeta = {
+  name: string;
+  label: string;
+  description: string;
+  language: 'typescript' | 'bash' | 'python';
+  inputs: Record<string, ToolParam>;
+};
+
+function parseFrontmatter(content: string): { meta: Partial<ToolMeta>; body: string } {
+  const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+  if (!match) return { meta: {}, body: content };
+
+  const yamlBlock = match[1]!;
+  const body = match[2]!;
+  const meta: Record<string, unknown> = {};
+
+  const lines = yamlBlock.split('\n');
+  let currentKey: string | null = null;
+  let currentObj: Record<string, unknown> | null = null;
+  let currentSubKey: string | null = null;
+  let currentSubObj: Record<string, unknown> | null = null;
+
+  for (const line of lines) {
+    const topMatch = line.match(/^(\w[\w-]*):\s*(.*)$/);
+    if (topMatch && !line.startsWith('  ')) {
+      if (currentSubObj && currentSubKey && currentObj) {
+        currentObj[currentSubKey] = currentSubObj;
+        currentSubObj = null;
+        currentSubKey = null;
+      }
+      if (currentObj && currentKey) {
+        meta[currentKey] = currentObj;
+        currentObj = null;
+        currentKey = null;
+      }
+      const [, key, value] = topMatch;
+      if (!value || value.trim() === '') {
+        currentKey = key!;
+        currentObj = {};
+      } else {
+        meta[key!] = value.trim();
+      }
+      continue;
+    }
+
+    const midMatch = line.match(/^  (\w[\w-]*):\s*(.*)$/);
+    if (midMatch && currentObj !== null) {
+      if (currentSubObj && currentSubKey) {
+        currentObj[currentSubKey] = currentSubObj;
+        currentSubObj = null;
+        currentSubKey = null;
+      }
+      const [, key, value] = midMatch;
+      if (!value || value.trim() === '') {
+        currentSubKey = key!;
+        currentSubObj = {};
+      } else {
+        currentObj[key!] = value.trim();
+      }
+      continue;
+    }
+
+    const deepMatch = line.match(/^    (\w[\w-]*):\s*(.*)$/);
+    if (deepMatch && currentSubObj !== null) {
+      const [, key, value] = deepMatch;
+      currentSubObj[key!] = value!.trim();
+      continue;
+    }
+
+    const arrayMatch = line.match(/^    - (.+)$/);
+    if (arrayMatch && currentSubObj !== null) {
+      const key = Object.keys(currentSubObj).at(-1);
+      if (key) {
+        const arr = currentSubObj[key];
+        if (Array.isArray(arr)) {
+          arr.push(arrayMatch[1]!.trim());
+        } else {
+          currentSubObj[key] = [arrayMatch[1]!.trim()];
+        }
+      }
+    }
+  }
+
+  if (currentSubObj && currentSubKey && currentObj) {
+    currentObj[currentSubKey] = currentSubObj;
+  }
+  if (currentObj && currentKey) {
+    meta[currentKey] = currentObj;
+  }
+
+  return { meta: meta as Partial<ToolMeta>, body };
+}
+
+function buildSchema(inputs: Record<string, ToolParam>): TSchema {
+  const props: Record<string, TSchema> = {};
+
+  for (const [paramName, param] of Object.entries(inputs)) {
+    let schema: TSchema;
+
+    switch (param.type) {
+      case 'enum': {
+        const values = param.values ?? [];
+        schema = Type.Union(values.map((v) => Type.Literal(v)), {
+          description: param.description,
+        });
+        break;
+      }
+      case 'number':
+        schema = Type.Number({ description: param.description });
+        break;
+      case 'boolean':
+        schema = Type.Boolean({ description: param.description });
+        break;
+      default:
+        schema = Type.String({ description: param.description });
+    }
+
+    props[paramName] = param.optional ? Type.Optional(schema) : schema;
+  }
+
+  return Type.Object(props);
+}
+
+function discoverTools(dir: string): Array<{ toolDir: string; entryFile: string; meta: ToolMeta }> {
+  if (!existsSync(dir)) return [];
+
+  const discovered: Array<{ toolDir: string; entryFile: string; meta: ToolMeta }> = [];
+  const entries = readdirSync(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+
+    const toolDir = join(dir, entry.name);
+    const toolMdPath = join(toolDir, 'TOOL.md');
+    if (!existsSync(toolMdPath)) continue;
+
+    const indexTs = join(toolDir, 'index.ts');
+    const indexJs = join(toolDir, 'index.js');
+    const entryFile = existsSync(indexTs) ? indexTs : existsSync(indexJs) ? indexJs : null;
+    if (!entryFile) {
+      console.warn(`[tool-loader] Skipping ${entry.name}: no index.ts or index.js found`);
+      continue;
+    }
+
+    const content = readFileSync(toolMdPath, 'utf-8');
+    const { meta } = parseFrontmatter(content);
+
+    if (!meta.name || !meta.description) {
+      console.warn(`[tool-loader] Skipping ${entry.name}: missing name or description in TOOL.md`);
+      continue;
+    }
+
+    discovered.push({ toolDir, entryFile, meta: meta as ToolMeta });
+  }
+
+  return discovered;
+}
+
+export default function (pi: ExtensionAPI) {
+  const rawDirs = process.env.PI_TOOLS_DIRS ?? '';
+  const toolDirs = rawDirs.split(':').filter(Boolean);
+
+  if (toolDirs.length === 0) {
+    console.warn('[tool-loader] PI_TOOLS_DIRS not set — no custom tools will be loaded');
+    return;
+  }
+
+  // Register synchronously in the factory function so tools appear in the system prompt.
+  // Implementations are lazy-loaded on first call to avoid async import issues at startup.
+  const seen = new Set<string>();
+
+  for (const dir of toolDirs) {
+    const tools = discoverTools(dir);
+
+    for (const { entryFile, meta } of tools) {
+      // User dirs come after global — last writer wins, so skip if already registered
+      if (seen.has(meta.name)) continue;
+      seen.add(meta.name);
+
+      const schema = buildSchema(meta.inputs ?? {});
+
+      // Capture entryFile in closure for lazy load
+      const capturedEntry = entryFile;
+
+      pi.registerTool({
+        name: meta.name,
+        label: meta.label ?? meta.name,
+        description: meta.description,
+        parameters: schema,
+
+        async execute(toolCallId, params, signal, onUpdate, ctx) {
+          // Lazy-load the implementation on first actual call.
+          // Dynamic import works here because we're already in an async tool execution
+          // context — jiti/Node has had time to set up its module hooks.
+          let executeFn: Function | undefined;
+          try {
+            const mod = await import(capturedEntry);
+            executeFn = mod.execute ?? mod.default?.execute;
+          } catch (err) {
+            return {
+              content: [{ type: 'text', text: `[tool-loader] Failed to load ${meta.name}: ${String(err)}` }],
+              details: { error: String(err) },
+              isError: true,
+            };
+          }
+
+          if (typeof executeFn !== 'function') {
+            return {
+              content: [{ type: 'text', text: `[tool-loader] ${meta.name}/index.ts must export an "execute" function` }],
+              details: {},
+              isError: true,
+            };
+          }
+
+          return executeFn(toolCallId, params, signal, onUpdate, ctx);
+        },
+      });
+
+      console.log(`[tool-loader] Registered tool: ${meta.name} (${capturedEntry})`);
+    }
+  }
+}

seed/tools/web-fetch/TOOL.md

@@ -0,0 +1,25 @@
+---
+name: web_fetch
+label: Web Fetch
+description: Fetch and return the text content of a URL. Tries to get clean readable content using a cascade of strategies in order: appending .md to the URL, fetching a /llms.txt discovery file, requesting plain text via Accept header, then falling back to stripping HTML. Use when the user provides a URL and wants to read, summarize, or extract information from web content.
+language: typescript
+inputs:
+  url:
+    type: string
+    description: The URL to fetch content from
+---
+
+# Web Fetch
+
+Fetches web content with cascading fallback strategies to get the cleanest possible text.
+
+## Strategies (in order)
+
+1. **Markdown version** — Appends `.md` to the URL (works on GitHub, many docs sites)
+2. **llms.txt discovery** — Checks `/llms.txt` at the root (sites that publish LLM-friendly content)
+3. **Plain text request** — Sends `Accept: text/plain` header
+4. **HTML strip** — Fetches HTML and strips tags, scripts, and styles
+
+## Output
+
+Returns the text content with a note indicating which strategy succeeded.

seed/tools/web-fetch/index.ts

@@ -0,0 +1,188 @@
+const MAX_BYTES = 50_000;
+const TIMEOUT_MS = 15_000;
+
+type FetchResult = {
+  content: string;
+  strategy: string;
+};
+
+function truncate(text: string): { text: string; truncated: boolean; originalBytes: number } {
+  const encoder = new TextEncoder();
+  const bytes = encoder.encode(text);
+  if (bytes.length <= MAX_BYTES) return { text, truncated: false, originalBytes: bytes.length };
+  const decoder = new TextDecoder();
+  return {
+    text: decoder.decode(bytes.slice(0, MAX_BYTES)),
+    truncated: true,
+    originalBytes: bytes.length,
+  };
+}
+
+function stripHtml(html: string): string {
+  // Remove script and style blocks entirely
+  let text = html
+    .replace(/<script[\s\S]*?<\/script>/gi, '')
+    .replace(/<style[\s\S]*?<\/style>/gi, '')
+    .replace(/<noscript[\s\S]*?<\/noscript>/gi, '')
+    .replace(/<nav[\s\S]*?<\/nav>/gi, '')
+    .replace(/<footer[\s\S]*?<\/footer>/gi, '')
+    .replace(/<header[\s\S]*?<\/header>/gi, '');
+
+  // Replace block-level tags with newlines
+  text = text
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/p>/gi, '\n\n')
+    .replace(/<\/div>/gi, '\n')
+    .replace(/<\/li>/gi, '\n')
+    .replace(/<\/h[1-6]>/gi, '\n\n')
+    .replace(/<\/tr>/gi, '\n')
+    .replace(/<\/td>/gi, '\t')
+    .replace(/<\/th>/gi, '\t');
+
+  // Strip remaining tags
+  text = text.replace(/<[^>]+>/g, '');
+
+  // Decode common HTML entities
+  text = text
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)));
+
+  // Collapse excessive whitespace but preserve paragraph breaks
+  text = text
+    .replace(/[ \t]+/g, ' ')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+
+  return text;
+}
+
+async function tryFetch(url: string, options: RequestInit = {}): Promise<Response | null> {
+  try {
+    const signal = AbortSignal.timeout(TIMEOUT_MS);
+    const res = await fetch(url, { ...options, signal, redirect: 'follow' });
+    if (res.ok) return res;
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+async function strategyMarkdown(url: string): Promise<FetchResult | null> {
+  const mdUrl = url.endsWith('.md') ? null : `${url.replace(/\/$/, '')}.md`;
+  if (!mdUrl) return null;
+
+  const res = await tryFetch(mdUrl);
+  if (!res) return null;
+
+  const contentType = res.headers.get('content-type') ?? '';
+  if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null;
+
+  const text = await res.text();
+  if (text.trim().startsWith('<')) return null; // Got HTML anyway
+
+  return { content: text, strategy: `markdown (${mdUrl})` };
+}
+
+async function strategyLlmsTxt(url: string): Promise<FetchResult | null> {
+  const { origin } = new URL(url);
+  const llmsUrl = `${origin}/llms.txt`;
+
+  const res = await tryFetch(llmsUrl);
+  if (!res) return null;
+
+  const text = await res.text();
+  if (!text.trim() || text.trim().startsWith('<')) return null;
+
+  return { content: text, strategy: `llms.txt (${llmsUrl})` };
+}
+
+async function strategyPlainText(url: string): Promise<FetchResult | null> {
+  const res = await tryFetch(url, { headers: { Accept: 'text/plain, text/markdown, */*;q=0.8' } });
+  if (!res) return null;
+
+  const contentType = res.headers.get('content-type') ?? '';
+  if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null;
+
+  const text = await res.text();
+  if (text.trim().startsWith('<')) return null;
+
+  return { content: text, strategy: 'plain text response' };
+}
+
+async function strategyHtmlStrip(url: string): Promise<FetchResult | null> {
+  const res = await tryFetch(url, { headers: { Accept: 'text/html,*/*;q=0.8' } });
+  if (!res) return null;
+
+  const html = await res.text();
+  const text = stripHtml(html);
+
+  if (!text.trim()) return null;
+
+  return { content: text, strategy: 'HTML (stripped)' };
+}
+
+export async function execute(
+  _toolCallId: string,
+  params: { url: string },
+  _signal: AbortSignal | undefined,
+  onUpdate?: (partial: { content: Array<{ type: string; text: string }> }) => void,
+) {
+  const { url } = params;
+
+  // Validate URL
+  let parsedUrl: URL;
+  try {
+    parsedUrl = new URL(url);
+  } catch {
+    return {
+      content: [{ type: 'text', text: `Invalid URL: ${url}` }],
+      details: { error: 'invalid_url' },
+      isError: true,
+    };
+  }
+
+  if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
+    return {
+      content: [{ type: 'text', text: `Unsupported protocol: ${parsedUrl.protocol}` }],
+      details: { error: 'unsupported_protocol' },
+      isError: true,
+    };
+  }
+
+  const strategies = [
+    { name: 'markdown', fn: () => strategyMarkdown(url) },
+    { name: 'llms.txt', fn: () => strategyLlmsTxt(url) },
+    { name: 'plain text', fn: () => strategyPlainText(url) },
+    { name: 'HTML strip', fn: () => strategyHtmlStrip(url) },
+  ];
+
+  for (const { name, fn } of strategies) {
+    onUpdate?.({ content: [{ type: 'text', text: `Trying ${name} strategy...` }] });
+
+    const result = await fn();
+    if (!result) continue;
+
+    const { text, truncated, originalBytes } = truncate(result.content);
+
+    let output = `[Fetched via ${result.strategy}]\n\n${text}`;
+    if (truncated) {
+      output += `\n\n[Content truncated: showing ${MAX_BYTES.toLocaleString()} of ${originalBytes.toLocaleString()} bytes]`;
+    }
+
+    return {
+      content: [{ type: 'text', text: output }],
+      details: { url, strategy: result.strategy, truncated, originalBytes },
+    };
+  }
+
+  return {
+    content: [{ type: 'text', text: `Failed to fetch content from: ${url}` }],
+    details: { error: 'all_strategies_failed', url },
+    isError: true,
+  };
+}

src/servers/api/pi/pi-bridge.ts

@@ -3,7 +3,7 @@ import { readdirSync, existsSync, mkdirSync } from "node:fs";
 import type { Subprocess } from "bun";
 import type { PiEvent, MessageCost } from "./types";
 import { readApiKeys } from "../server-settings/pi-mono";
-import { PI_CONFIG_DIR, getGlobalSkillsDir, getUserSkillsDir } from "../../data-path";
+import { PI_CONFIG_DIR, getGlobalSkillsDir, getUserSkillsDir, getGlobalExtensionsDir, getUserExtensionsDir, getGlobalToolsDir, getUserToolsDir } from "../../data-path";
 import { ensureDockerContainer } from "../terminal/websocket";
 import { logger } from "./logger";
 
@@ -28,6 +28,25 @@ function collectSkillFlags(email: string): string[] {
   return flags;
 }
 
+function collectExtensionFlags(email: string): string[] {
+  const flags: string[] = [];
+  const dirs = [getGlobalExtensionsDir(), getUserExtensionsDir(email)];
+
+  for (const dir of dirs) {
+    if (!existsSync(dir)) continue;
+    const entries = readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+      const entryFile = join(dir, entry.name, 'index.ts');
+      if (existsSync(entryFile)) {
+        flags.push('--extension', entryFile);
+      }
+    }
+  }
+
+  return flags;
+}
+
 type SandboxOptions = {
   userId: number;
   username: string;
@@ -81,22 +100,29 @@ export async function spawnPi(
   } else {
     const storedKeys = await readApiKeys();
     const skillFlags = collectSkillFlags(email);
-    const args = ['pi', '--mode', 'rpc', '--no-extensions', '--no-skills', '--no-prompt-templates', '--no-themes', ...skillFlags];
+    const extensionFlags = collectExtensionFlags(email);
+    const args = ['pi', '--mode', 'rpc', '--no-skills', '--no-prompt-templates', '--no-themes', ...skillFlags, ...extensionFlags];
     if (model) args.push('--model', model);
 
     if (!existsSync(cwd)) {
       mkdirSync(cwd, { recursive: true });
     }
 
+    const toolsDirs = [getGlobalToolsDir(), getUserToolsDir(email)].join(':');
+
     proc = Bun.spawn(args, {
       cwd,
       stdin: 'pipe',
       stdout: 'pipe',
       stderr: 'pipe',
-      env: { ...process.env, ...storedKeys, PI_CODING_AGENT_DIR: PI_CONFIG_DIR },
+      env: { ...process.env, ...storedKeys, PI_CODING_AGENT_DIR: PI_CONFIG_DIR, PI_TOOLS_DIRS: toolsDirs },
     });
 
-    logger.info('Spawned Pi locally', { model, skills: skillFlags.filter((f) => f !== '--skill').length });
+    logger.info('Spawned Pi locally', {
+      model,
+      skills: skillFlags.filter((f) => f !== '--skill').length,
+      extensions: extensionFlags.filter((f) => f !== '--extension').length,
+    });
   }
 
   // Read stdout JSON event stream (runs in background)

src/servers/bootstrap.ts

@@ -6,6 +6,8 @@ import { syncLocalProvidersToPiConfig } from './api/server-settings/sync-pi-conf
 import { syncAllUserPiConfigs } from './api/server-settings/sync-user-pi-config';
 import { initAuthStore } from 'officerdb';
 import { syncSeedSkills } from './sync-skills';
+import { syncSeedTools } from './sync-tools';
+import { syncSeedExtensions } from './sync-extensions';
 
 mkdirSync(DATA_PATH, { recursive: true });
 mkdirSync(PI_CONFIG_DIR, { recursive: true });
@@ -71,6 +73,8 @@ function seedPiConfig(): void {
 
   seedPiConfig();
   syncSeedSkills();
+  syncSeedTools();
+  syncSeedExtensions();
 
   await syncLocalProvidersToPiConfig().catch(err => {
     console.error('[bootstrap] Failed to sync local providers to Pi config:', err);

src/servers/data-path.ts

@@ -50,6 +50,18 @@ export const getGlobalSkillsDir = () => join(DATA_PATH, 'skills');
 
 export const getUserSkillsDir = (email: string) => join(DATA_PATH, email, 'skills');
 
+export const getNativeToolsDir = () => join(SEED_PATH, 'tools');
+
+export const getGlobalToolsDir = () => join(DATA_PATH, 'tools');
+
+export const getUserToolsDir = (email: string) => join(DATA_PATH, email, 'tools');
+
+export const getNativeExtensionsDir = () => join(SEED_PATH, 'extensions');
+
+export const getGlobalExtensionsDir = () => join(DATA_PATH, 'extensions');
+
+export const getUserExtensionsDir = (email: string) => join(DATA_PATH, email, 'extensions');
+
 export const getNativeTasksDir = () => join(SEED_PATH, 'tasks');
 
 export const getGlobalTasksDir = () => join(DATA_PATH, 'tasks');

src/servers/sync-extensions.ts

@@ -0,0 +1,33 @@
+import { readdirSync, existsSync, mkdirSync, cpSync } from 'node:fs';
+import { join } from 'node:path';
+import { SEED_PATH, DATA_PATH } from './data-path';
+
+const SEED_EXTENSIONS_DIR = join(SEED_PATH, 'extensions');
+const GLOBAL_EXTENSIONS_DIR = join(DATA_PATH, 'extensions');
+
+export function syncSeedExtensions(): void {
+  if (!existsSync(SEED_EXTENSIONS_DIR)) return;
+
+  mkdirSync(GLOBAL_EXTENSIONS_DIR, { recursive: true });
+
+  const seedEntries = readdirSync(SEED_EXTENSIONS_DIR, { withFileTypes: true });
+
+  for (const entry of seedEntries) {
+    if (!entry.isDirectory()) continue;
+
+    const seedExtensionDir = join(SEED_EXTENSIONS_DIR, entry.name);
+    const entryFile = join(seedExtensionDir, 'index.ts');
+    if (!existsSync(entryFile)) continue;
+
+    const targetDir = join(GLOBAL_EXTENSIONS_DIR, entry.name);
+
+    if (existsSync(targetDir)) {
+      // Extension already exists in DATA_PATH — always overwrite (managed by us, not the user)
+      cpSync(seedExtensionDir, targetDir, { recursive: true });
+      continue;
+    }
+
+    cpSync(seedExtensionDir, targetDir, { recursive: true });
+    console.log(`[extensions] Synced seed extension: ${entry.name}`);
+  }
+}

src/servers/sync-tools.ts

@@ -0,0 +1,32 @@
+import { readdirSync, existsSync, mkdirSync, cpSync } from 'node:fs';
+import { join } from 'node:path';
+import { SEED_PATH, DATA_PATH } from './data-path';
+
+const SEED_TOOLS_DIR = join(SEED_PATH, 'tools');
+const GLOBAL_TOOLS_DIR = join(DATA_PATH, 'tools');
+
+export function syncSeedTools(): void {
+  if (!existsSync(SEED_TOOLS_DIR)) return;
+
+  mkdirSync(GLOBAL_TOOLS_DIR, { recursive: true });
+
+  const seedEntries = readdirSync(SEED_TOOLS_DIR, { withFileTypes: true });
+
+  for (const entry of seedEntries) {
+    if (!entry.isDirectory()) continue;
+
+    const seedToolDir = join(SEED_TOOLS_DIR, entry.name);
+    const toolFile = join(seedToolDir, 'TOOL.md');
+    if (!existsSync(toolFile)) continue;
+
+    const targetDir = join(GLOBAL_TOOLS_DIR, entry.name);
+
+    if (existsSync(targetDir)) {
+      // Tool already exists in DATA_PATH — skip to preserve user edits
+      continue;
+    }
+
+    cpSync(seedToolDir, targetDir, { recursive: true });
+    console.log(`[tools] Synced seed tool: ${entry.name}`);
+  }
+}