diff --git a/seed/extensions/tool-loader/index.ts b/seed/extensions/tool-loader/index.ts new file mode 100644 index 00000000..456d7321 --- /dev/null +++ b/seed/extensions/tool-loader/index.ts @@ -0,0 +1,237 @@ +import type { ExtensionAPI } from '@mariozechner/pi-coding-agent'; +import { Type, type TSchema } from '@sinclair/typebox'; +import { readdirSync, existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +type ToolParamType = 'string' | 'number' | 'boolean' | 'enum'; + +type ToolParam = { + type: ToolParamType; + description: string; + values?: string[]; + default?: unknown; + optional?: boolean; +}; + +type ToolMeta = { + name: string; + label: string; + description: string; + language: 'typescript' | 'bash' | 'python'; + inputs: Record; +}; + +function parseFrontmatter(content: string): { meta: Partial; body: string } { + const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); + if (!match) return { meta: {}, body: content }; + + const yamlBlock = match[1]!; + const body = match[2]!; + const meta: Record = {}; + + const lines = yamlBlock.split('\n'); + let currentKey: string | null = null; + let currentObj: Record | null = null; + let currentSubKey: string | null = null; + let currentSubObj: Record | null = null; + + for (const line of lines) { + const topMatch = line.match(/^(\w[\w-]*):\s*(.*)$/); + if (topMatch && !line.startsWith(' ')) { + if (currentSubObj && currentSubKey && currentObj) { + currentObj[currentSubKey] = currentSubObj; + currentSubObj = null; + currentSubKey = null; + } + if (currentObj && currentKey) { + meta[currentKey] = currentObj; + currentObj = null; + currentKey = null; + } + const [, key, value] = topMatch; + if (!value || value.trim() === '') { + currentKey = key!; + currentObj = {}; + } else { + meta[key!] = value.trim(); + } + continue; + } + + const midMatch = line.match(/^ (\w[\w-]*):\s*(.*)$/); + if (midMatch && currentObj !== null) { + if (currentSubObj && currentSubKey) { + currentObj[currentSubKey] = currentSubObj; + currentSubObj = null; + currentSubKey = null; + } + const [, key, value] = midMatch; + if (!value || value.trim() === '') { + currentSubKey = key!; + currentSubObj = {}; + } else { + currentObj[key!] = value.trim(); + } + continue; + } + + const deepMatch = line.match(/^ (\w[\w-]*):\s*(.*)$/); + if (deepMatch && currentSubObj !== null) { + const [, key, value] = deepMatch; + currentSubObj[key!] = value!.trim(); + continue; + } + + const arrayMatch = line.match(/^ - (.+)$/); + if (arrayMatch && currentSubObj !== null) { + const key = Object.keys(currentSubObj).at(-1); + if (key) { + const arr = currentSubObj[key]; + if (Array.isArray(arr)) { + arr.push(arrayMatch[1]!.trim()); + } else { + currentSubObj[key] = [arrayMatch[1]!.trim()]; + } + } + } + } + + if (currentSubObj && currentSubKey && currentObj) { + currentObj[currentSubKey] = currentSubObj; + } + if (currentObj && currentKey) { + meta[currentKey] = currentObj; + } + + return { meta: meta as Partial, body }; +} + +function buildSchema(inputs: Record): TSchema { + const props: Record = {}; + + for (const [paramName, param] of Object.entries(inputs)) { + let schema: TSchema; + + switch (param.type) { + case 'enum': { + const values = param.values ?? []; + schema = Type.Union(values.map((v) => Type.Literal(v)), { + description: param.description, + }); + break; + } + case 'number': + schema = Type.Number({ description: param.description }); + break; + case 'boolean': + schema = Type.Boolean({ description: param.description }); + break; + default: + schema = Type.String({ description: param.description }); + } + + props[paramName] = param.optional ? Type.Optional(schema) : schema; + } + + return Type.Object(props); +} + +function discoverTools(dir: string): Array<{ toolDir: string; entryFile: string; meta: ToolMeta }> { + if (!existsSync(dir)) return []; + + const discovered: Array<{ toolDir: string; entryFile: string; meta: ToolMeta }> = []; + const entries = readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + + const toolDir = join(dir, entry.name); + const toolMdPath = join(toolDir, 'TOOL.md'); + if (!existsSync(toolMdPath)) continue; + + const indexTs = join(toolDir, 'index.ts'); + const indexJs = join(toolDir, 'index.js'); + const entryFile = existsSync(indexTs) ? indexTs : existsSync(indexJs) ? indexJs : null; + if (!entryFile) { + console.warn(`[tool-loader] Skipping ${entry.name}: no index.ts or index.js found`); + continue; + } + + const content = readFileSync(toolMdPath, 'utf-8'); + const { meta } = parseFrontmatter(content); + + if (!meta.name || !meta.description) { + console.warn(`[tool-loader] Skipping ${entry.name}: missing name or description in TOOL.md`); + continue; + } + + discovered.push({ toolDir, entryFile, meta: meta as ToolMeta }); + } + + return discovered; +} + +export default function (pi: ExtensionAPI) { + const rawDirs = process.env.PI_TOOLS_DIRS ?? ''; + const toolDirs = rawDirs.split(':').filter(Boolean); + + if (toolDirs.length === 0) { + console.warn('[tool-loader] PI_TOOLS_DIRS not set — no custom tools will be loaded'); + return; + } + + // Register synchronously in the factory function so tools appear in the system prompt. + // Implementations are lazy-loaded on first call to avoid async import issues at startup. + const seen = new Set(); + + for (const dir of toolDirs) { + const tools = discoverTools(dir); + + for (const { entryFile, meta } of tools) { + // User dirs come after global — last writer wins, so skip if already registered + if (seen.has(meta.name)) continue; + seen.add(meta.name); + + const schema = buildSchema(meta.inputs ?? {}); + + // Capture entryFile in closure for lazy load + const capturedEntry = entryFile; + + pi.registerTool({ + name: meta.name, + label: meta.label ?? meta.name, + description: meta.description, + parameters: schema, + + async execute(toolCallId, params, signal, onUpdate, ctx) { + // Lazy-load the implementation on first actual call. + // Dynamic import works here because we're already in an async tool execution + // context — jiti/Node has had time to set up its module hooks. + let executeFn: Function | undefined; + try { + const mod = await import(capturedEntry); + executeFn = mod.execute ?? mod.default?.execute; + } catch (err) { + return { + content: [{ type: 'text', text: `[tool-loader] Failed to load ${meta.name}: ${String(err)}` }], + details: { error: String(err) }, + isError: true, + }; + } + + if (typeof executeFn !== 'function') { + return { + content: [{ type: 'text', text: `[tool-loader] ${meta.name}/index.ts must export an "execute" function` }], + details: {}, + isError: true, + }; + } + + return executeFn(toolCallId, params, signal, onUpdate, ctx); + }, + }); + + console.log(`[tool-loader] Registered tool: ${meta.name} (${capturedEntry})`); + } + } +} diff --git a/seed/tools/web-fetch/TOOL.md b/seed/tools/web-fetch/TOOL.md new file mode 100644 index 00000000..8bcec9b8 --- /dev/null +++ b/seed/tools/web-fetch/TOOL.md @@ -0,0 +1,25 @@ +--- +name: web_fetch +label: Web Fetch +description: Fetch and return the text content of a URL. Tries to get clean readable content using a cascade of strategies in order: appending .md to the URL, fetching a /llms.txt discovery file, requesting plain text via Accept header, then falling back to stripping HTML. Use when the user provides a URL and wants to read, summarize, or extract information from web content. +language: typescript +inputs: + url: + type: string + description: The URL to fetch content from +--- + +# Web Fetch + +Fetches web content with cascading fallback strategies to get the cleanest possible text. + +## Strategies (in order) + +1. **Markdown version** — Appends `.md` to the URL (works on GitHub, many docs sites) +2. **llms.txt discovery** — Checks `/llms.txt` at the root (sites that publish LLM-friendly content) +3. **Plain text request** — Sends `Accept: text/plain` header +4. **HTML strip** — Fetches HTML and strips tags, scripts, and styles + +## Output + +Returns the text content with a note indicating which strategy succeeded. diff --git a/seed/tools/web-fetch/index.ts b/seed/tools/web-fetch/index.ts new file mode 100644 index 00000000..93b450c2 --- /dev/null +++ b/seed/tools/web-fetch/index.ts @@ -0,0 +1,188 @@ +const MAX_BYTES = 50_000; +const TIMEOUT_MS = 15_000; + +type FetchResult = { + content: string; + strategy: string; +}; + +function truncate(text: string): { text: string; truncated: boolean; originalBytes: number } { + const encoder = new TextEncoder(); + const bytes = encoder.encode(text); + if (bytes.length <= MAX_BYTES) return { text, truncated: false, originalBytes: bytes.length }; + const decoder = new TextDecoder(); + return { + text: decoder.decode(bytes.slice(0, MAX_BYTES)), + truncated: true, + originalBytes: bytes.length, + }; +} + +function stripHtml(html: string): string { + // Remove script and style blocks entirely + let text = html + .replace(//gi, '') + .replace(//gi, '') + .replace(//gi, '') + .replace(//gi, '') + .replace(//gi, '') + .replace(//gi, ''); + + // Replace block-level tags with newlines + text = text + .replace(//gi, '\n') + .replace(/<\/p>/gi, '\n\n') + .replace(/<\/div>/gi, '\n') + .replace(/<\/li>/gi, '\n') + .replace(/<\/h[1-6]>/gi, '\n\n') + .replace(/<\/tr>/gi, '\n') + .replace(/<\/td>/gi, '\t') + .replace(/<\/th>/gi, '\t'); + + // Strip remaining tags + text = text.replace(/<[^>]+>/g, ''); + + // Decode common HTML entities + text = text + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/ /g, ' ') + .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code))); + + // Collapse excessive whitespace but preserve paragraph breaks + text = text + .replace(/[ \t]+/g, ' ') + .replace(/\n{3,}/g, '\n\n') + .trim(); + + return text; +} + +async function tryFetch(url: string, options: RequestInit = {}): Promise { + try { + const signal = AbortSignal.timeout(TIMEOUT_MS); + const res = await fetch(url, { ...options, signal, redirect: 'follow' }); + if (res.ok) return res; + return null; + } catch { + return null; + } +} + +async function strategyMarkdown(url: string): Promise { + const mdUrl = url.endsWith('.md') ? null : `${url.replace(/\/$/, '')}.md`; + if (!mdUrl) return null; + + const res = await tryFetch(mdUrl); + if (!res) return null; + + const contentType = res.headers.get('content-type') ?? ''; + if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null; + + const text = await res.text(); + if (text.trim().startsWith('<')) return null; // Got HTML anyway + + return { content: text, strategy: `markdown (${mdUrl})` }; +} + +async function strategyLlmsTxt(url: string): Promise { + const { origin } = new URL(url); + const llmsUrl = `${origin}/llms.txt`; + + const res = await tryFetch(llmsUrl); + if (!res) return null; + + const text = await res.text(); + if (!text.trim() || text.trim().startsWith('<')) return null; + + return { content: text, strategy: `llms.txt (${llmsUrl})` }; +} + +async function strategyPlainText(url: string): Promise { + const res = await tryFetch(url, { headers: { Accept: 'text/plain, text/markdown, */*;q=0.8' } }); + if (!res) return null; + + const contentType = res.headers.get('content-type') ?? ''; + if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null; + + const text = await res.text(); + if (text.trim().startsWith('<')) return null; + + return { content: text, strategy: 'plain text response' }; +} + +async function strategyHtmlStrip(url: string): Promise { + const res = await tryFetch(url, { headers: { Accept: 'text/html,*/*;q=0.8' } }); + if (!res) return null; + + const html = await res.text(); + const text = stripHtml(html); + + if (!text.trim()) return null; + + return { content: text, strategy: 'HTML (stripped)' }; +} + +export async function execute( + _toolCallId: string, + params: { url: string }, + _signal: AbortSignal | undefined, + onUpdate?: (partial: { content: Array<{ type: string; text: string }> }) => void, +) { + const { url } = params; + + // Validate URL + let parsedUrl: URL; + try { + parsedUrl = new URL(url); + } catch { + return { + content: [{ type: 'text', text: `Invalid URL: ${url}` }], + details: { error: 'invalid_url' }, + isError: true, + }; + } + + if (!['http:', 'https:'].includes(parsedUrl.protocol)) { + return { + content: [{ type: 'text', text: `Unsupported protocol: ${parsedUrl.protocol}` }], + details: { error: 'unsupported_protocol' }, + isError: true, + }; + } + + const strategies = [ + { name: 'markdown', fn: () => strategyMarkdown(url) }, + { name: 'llms.txt', fn: () => strategyLlmsTxt(url) }, + { name: 'plain text', fn: () => strategyPlainText(url) }, + { name: 'HTML strip', fn: () => strategyHtmlStrip(url) }, + ]; + + for (const { name, fn } of strategies) { + onUpdate?.({ content: [{ type: 'text', text: `Trying ${name} strategy...` }] }); + + const result = await fn(); + if (!result) continue; + + const { text, truncated, originalBytes } = truncate(result.content); + + let output = `[Fetched via ${result.strategy}]\n\n${text}`; + if (truncated) { + output += `\n\n[Content truncated: showing ${MAX_BYTES.toLocaleString()} of ${originalBytes.toLocaleString()} bytes]`; + } + + return { + content: [{ type: 'text', text: output }], + details: { url, strategy: result.strategy, truncated, originalBytes }, + }; + } + + return { + content: [{ type: 'text', text: `Failed to fetch content from: ${url}` }], + details: { error: 'all_strategies_failed', url }, + isError: true, + }; +} diff --git a/src/servers/api/pi/pi-bridge.ts b/src/servers/api/pi/pi-bridge.ts index cbb10b17..8d06a8ee 100644 --- a/src/servers/api/pi/pi-bridge.ts +++ b/src/servers/api/pi/pi-bridge.ts @@ -3,7 +3,7 @@ import { readdirSync, existsSync, mkdirSync } from "node:fs"; import type { Subprocess } from "bun"; import type { PiEvent, MessageCost } from "./types"; import { readApiKeys } from "../server-settings/pi-mono"; -import { PI_CONFIG_DIR, getGlobalSkillsDir, getUserSkillsDir } from "../../data-path"; +import { PI_CONFIG_DIR, getGlobalSkillsDir, getUserSkillsDir, getGlobalExtensionsDir, getUserExtensionsDir, getGlobalToolsDir, getUserToolsDir } from "../../data-path"; import { ensureDockerContainer } from "../terminal/websocket"; import { logger } from "./logger"; @@ -28,6 +28,25 @@ function collectSkillFlags(email: string): string[] { return flags; } +function collectExtensionFlags(email: string): string[] { + const flags: string[] = []; + const dirs = [getGlobalExtensionsDir(), getUserExtensionsDir(email)]; + + for (const dir of dirs) { + if (!existsSync(dir)) continue; + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const entryFile = join(dir, entry.name, 'index.ts'); + if (existsSync(entryFile)) { + flags.push('--extension', entryFile); + } + } + } + + return flags; +} + type SandboxOptions = { userId: number; username: string; @@ -81,22 +100,29 @@ export async function spawnPi( } else { const storedKeys = await readApiKeys(); const skillFlags = collectSkillFlags(email); - const args = ['pi', '--mode', 'rpc', '--no-extensions', '--no-skills', '--no-prompt-templates', '--no-themes', ...skillFlags]; + const extensionFlags = collectExtensionFlags(email); + const args = ['pi', '--mode', 'rpc', '--no-skills', '--no-prompt-templates', '--no-themes', ...skillFlags, ...extensionFlags]; if (model) args.push('--model', model); if (!existsSync(cwd)) { mkdirSync(cwd, { recursive: true }); } + const toolsDirs = [getGlobalToolsDir(), getUserToolsDir(email)].join(':'); + proc = Bun.spawn(args, { cwd, stdin: 'pipe', stdout: 'pipe', stderr: 'pipe', - env: { ...process.env, ...storedKeys, PI_CODING_AGENT_DIR: PI_CONFIG_DIR }, + env: { ...process.env, ...storedKeys, PI_CODING_AGENT_DIR: PI_CONFIG_DIR, PI_TOOLS_DIRS: toolsDirs }, }); - logger.info('Spawned Pi locally', { model, skills: skillFlags.filter((f) => f !== '--skill').length }); + logger.info('Spawned Pi locally', { + model, + skills: skillFlags.filter((f) => f !== '--skill').length, + extensions: extensionFlags.filter((f) => f !== '--extension').length, + }); } // Read stdout JSON event stream (runs in background) diff --git a/src/servers/bootstrap.ts b/src/servers/bootstrap.ts index a550f540..0eec05df 100644 --- a/src/servers/bootstrap.ts +++ b/src/servers/bootstrap.ts @@ -6,6 +6,8 @@ import { syncLocalProvidersToPiConfig } from './api/server-settings/sync-pi-conf import { syncAllUserPiConfigs } from './api/server-settings/sync-user-pi-config'; import { initAuthStore } from 'officerdb'; import { syncSeedSkills } from './sync-skills'; +import { syncSeedTools } from './sync-tools'; +import { syncSeedExtensions } from './sync-extensions'; mkdirSync(DATA_PATH, { recursive: true }); mkdirSync(PI_CONFIG_DIR, { recursive: true }); @@ -71,6 +73,8 @@ function seedPiConfig(): void { seedPiConfig(); syncSeedSkills(); + syncSeedTools(); + syncSeedExtensions(); await syncLocalProvidersToPiConfig().catch(err => { console.error('[bootstrap] Failed to sync local providers to Pi config:', err); diff --git a/src/servers/data-path.ts b/src/servers/data-path.ts index 493e1688..7bb213c9 100644 --- a/src/servers/data-path.ts +++ b/src/servers/data-path.ts @@ -50,6 +50,18 @@ export const getGlobalSkillsDir = () => join(DATA_PATH, 'skills'); export const getUserSkillsDir = (email: string) => join(DATA_PATH, email, 'skills'); +export const getNativeToolsDir = () => join(SEED_PATH, 'tools'); + +export const getGlobalToolsDir = () => join(DATA_PATH, 'tools'); + +export const getUserToolsDir = (email: string) => join(DATA_PATH, email, 'tools'); + +export const getNativeExtensionsDir = () => join(SEED_PATH, 'extensions'); + +export const getGlobalExtensionsDir = () => join(DATA_PATH, 'extensions'); + +export const getUserExtensionsDir = (email: string) => join(DATA_PATH, email, 'extensions'); + export const getNativeTasksDir = () => join(SEED_PATH, 'tasks'); export const getGlobalTasksDir = () => join(DATA_PATH, 'tasks'); diff --git a/src/servers/sync-extensions.ts b/src/servers/sync-extensions.ts new file mode 100644 index 00000000..f75bc8bb --- /dev/null +++ b/src/servers/sync-extensions.ts @@ -0,0 +1,33 @@ +import { readdirSync, existsSync, mkdirSync, cpSync } from 'node:fs'; +import { join } from 'node:path'; +import { SEED_PATH, DATA_PATH } from './data-path'; + +const SEED_EXTENSIONS_DIR = join(SEED_PATH, 'extensions'); +const GLOBAL_EXTENSIONS_DIR = join(DATA_PATH, 'extensions'); + +export function syncSeedExtensions(): void { + if (!existsSync(SEED_EXTENSIONS_DIR)) return; + + mkdirSync(GLOBAL_EXTENSIONS_DIR, { recursive: true }); + + const seedEntries = readdirSync(SEED_EXTENSIONS_DIR, { withFileTypes: true }); + + for (const entry of seedEntries) { + if (!entry.isDirectory()) continue; + + const seedExtensionDir = join(SEED_EXTENSIONS_DIR, entry.name); + const entryFile = join(seedExtensionDir, 'index.ts'); + if (!existsSync(entryFile)) continue; + + const targetDir = join(GLOBAL_EXTENSIONS_DIR, entry.name); + + if (existsSync(targetDir)) { + // Extension already exists in DATA_PATH — always overwrite (managed by us, not the user) + cpSync(seedExtensionDir, targetDir, { recursive: true }); + continue; + } + + cpSync(seedExtensionDir, targetDir, { recursive: true }); + console.log(`[extensions] Synced seed extension: ${entry.name}`); + } +} diff --git a/src/servers/sync-tools.ts b/src/servers/sync-tools.ts new file mode 100644 index 00000000..e6b91ec2 --- /dev/null +++ b/src/servers/sync-tools.ts @@ -0,0 +1,32 @@ +import { readdirSync, existsSync, mkdirSync, cpSync } from 'node:fs'; +import { join } from 'node:path'; +import { SEED_PATH, DATA_PATH } from './data-path'; + +const SEED_TOOLS_DIR = join(SEED_PATH, 'tools'); +const GLOBAL_TOOLS_DIR = join(DATA_PATH, 'tools'); + +export function syncSeedTools(): void { + if (!existsSync(SEED_TOOLS_DIR)) return; + + mkdirSync(GLOBAL_TOOLS_DIR, { recursive: true }); + + const seedEntries = readdirSync(SEED_TOOLS_DIR, { withFileTypes: true }); + + for (const entry of seedEntries) { + if (!entry.isDirectory()) continue; + + const seedToolDir = join(SEED_TOOLS_DIR, entry.name); + const toolFile = join(seedToolDir, 'TOOL.md'); + if (!existsSync(toolFile)) continue; + + const targetDir = join(GLOBAL_TOOLS_DIR, entry.name); + + if (existsSync(targetDir)) { + // Tool already exists in DATA_PATH — skip to preserve user edits + continue; + } + + cpSync(seedToolDir, targetDir, { recursive: true }); + console.log(`[tools] Synced seed tool: ${entry.name}`); + } +}