tools system - web-fetch
This commit is contained in:
@@ -0,0 +1,237 @@
|
||||
import type { ExtensionAPI } from '@mariozechner/pi-coding-agent';
|
||||
import { Type, type TSchema } from '@sinclair/typebox';
|
||||
import { readdirSync, existsSync, readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
type ToolParamType = 'string' | 'number' | 'boolean' | 'enum';
|
||||
|
||||
type ToolParam = {
|
||||
type: ToolParamType;
|
||||
description: string;
|
||||
values?: string[];
|
||||
default?: unknown;
|
||||
optional?: boolean;
|
||||
};
|
||||
|
||||
type ToolMeta = {
|
||||
name: string;
|
||||
label: string;
|
||||
description: string;
|
||||
language: 'typescript' | 'bash' | 'python';
|
||||
inputs: Record<string, ToolParam>;
|
||||
};
|
||||
|
||||
function parseFrontmatter(content: string): { meta: Partial<ToolMeta>; body: string } {
|
||||
const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
||||
if (!match) return { meta: {}, body: content };
|
||||
|
||||
const yamlBlock = match[1]!;
|
||||
const body = match[2]!;
|
||||
const meta: Record<string, unknown> = {};
|
||||
|
||||
const lines = yamlBlock.split('\n');
|
||||
let currentKey: string | null = null;
|
||||
let currentObj: Record<string, unknown> | null = null;
|
||||
let currentSubKey: string | null = null;
|
||||
let currentSubObj: Record<string, unknown> | null = null;
|
||||
|
||||
for (const line of lines) {
|
||||
const topMatch = line.match(/^(\w[\w-]*):\s*(.*)$/);
|
||||
if (topMatch && !line.startsWith(' ')) {
|
||||
if (currentSubObj && currentSubKey && currentObj) {
|
||||
currentObj[currentSubKey] = currentSubObj;
|
||||
currentSubObj = null;
|
||||
currentSubKey = null;
|
||||
}
|
||||
if (currentObj && currentKey) {
|
||||
meta[currentKey] = currentObj;
|
||||
currentObj = null;
|
||||
currentKey = null;
|
||||
}
|
||||
const [, key, value] = topMatch;
|
||||
if (!value || value.trim() === '') {
|
||||
currentKey = key!;
|
||||
currentObj = {};
|
||||
} else {
|
||||
meta[key!] = value.trim();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const midMatch = line.match(/^ (\w[\w-]*):\s*(.*)$/);
|
||||
if (midMatch && currentObj !== null) {
|
||||
if (currentSubObj && currentSubKey) {
|
||||
currentObj[currentSubKey] = currentSubObj;
|
||||
currentSubObj = null;
|
||||
currentSubKey = null;
|
||||
}
|
||||
const [, key, value] = midMatch;
|
||||
if (!value || value.trim() === '') {
|
||||
currentSubKey = key!;
|
||||
currentSubObj = {};
|
||||
} else {
|
||||
currentObj[key!] = value.trim();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const deepMatch = line.match(/^ (\w[\w-]*):\s*(.*)$/);
|
||||
if (deepMatch && currentSubObj !== null) {
|
||||
const [, key, value] = deepMatch;
|
||||
currentSubObj[key!] = value!.trim();
|
||||
continue;
|
||||
}
|
||||
|
||||
const arrayMatch = line.match(/^ - (.+)$/);
|
||||
if (arrayMatch && currentSubObj !== null) {
|
||||
const key = Object.keys(currentSubObj).at(-1);
|
||||
if (key) {
|
||||
const arr = currentSubObj[key];
|
||||
if (Array.isArray(arr)) {
|
||||
arr.push(arrayMatch[1]!.trim());
|
||||
} else {
|
||||
currentSubObj[key] = [arrayMatch[1]!.trim()];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (currentSubObj && currentSubKey && currentObj) {
|
||||
currentObj[currentSubKey] = currentSubObj;
|
||||
}
|
||||
if (currentObj && currentKey) {
|
||||
meta[currentKey] = currentObj;
|
||||
}
|
||||
|
||||
return { meta: meta as Partial<ToolMeta>, body };
|
||||
}
|
||||
|
||||
function buildSchema(inputs: Record<string, ToolParam>): TSchema {
|
||||
const props: Record<string, TSchema> = {};
|
||||
|
||||
for (const [paramName, param] of Object.entries(inputs)) {
|
||||
let schema: TSchema;
|
||||
|
||||
switch (param.type) {
|
||||
case 'enum': {
|
||||
const values = param.values ?? [];
|
||||
schema = Type.Union(values.map((v) => Type.Literal(v)), {
|
||||
description: param.description,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'number':
|
||||
schema = Type.Number({ description: param.description });
|
||||
break;
|
||||
case 'boolean':
|
||||
schema = Type.Boolean({ description: param.description });
|
||||
break;
|
||||
default:
|
||||
schema = Type.String({ description: param.description });
|
||||
}
|
||||
|
||||
props[paramName] = param.optional ? Type.Optional(schema) : schema;
|
||||
}
|
||||
|
||||
return Type.Object(props);
|
||||
}
|
||||
|
||||
function discoverTools(dir: string): Array<{ toolDir: string; entryFile: string; meta: ToolMeta }> {
|
||||
if (!existsSync(dir)) return [];
|
||||
|
||||
const discovered: Array<{ toolDir: string; entryFile: string; meta: ToolMeta }> = [];
|
||||
const entries = readdirSync(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
|
||||
const toolDir = join(dir, entry.name);
|
||||
const toolMdPath = join(toolDir, 'TOOL.md');
|
||||
if (!existsSync(toolMdPath)) continue;
|
||||
|
||||
const indexTs = join(toolDir, 'index.ts');
|
||||
const indexJs = join(toolDir, 'index.js');
|
||||
const entryFile = existsSync(indexTs) ? indexTs : existsSync(indexJs) ? indexJs : null;
|
||||
if (!entryFile) {
|
||||
console.warn(`[tool-loader] Skipping ${entry.name}: no index.ts or index.js found`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = readFileSync(toolMdPath, 'utf-8');
|
||||
const { meta } = parseFrontmatter(content);
|
||||
|
||||
if (!meta.name || !meta.description) {
|
||||
console.warn(`[tool-loader] Skipping ${entry.name}: missing name or description in TOOL.md`);
|
||||
continue;
|
||||
}
|
||||
|
||||
discovered.push({ toolDir, entryFile, meta: meta as ToolMeta });
|
||||
}
|
||||
|
||||
return discovered;
|
||||
}
|
||||
|
||||
export default function (pi: ExtensionAPI) {
|
||||
const rawDirs = process.env.PI_TOOLS_DIRS ?? '';
|
||||
const toolDirs = rawDirs.split(':').filter(Boolean);
|
||||
|
||||
if (toolDirs.length === 0) {
|
||||
console.warn('[tool-loader] PI_TOOLS_DIRS not set — no custom tools will be loaded');
|
||||
return;
|
||||
}
|
||||
|
||||
// Register synchronously in the factory function so tools appear in the system prompt.
|
||||
// Implementations are lazy-loaded on first call to avoid async import issues at startup.
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const dir of toolDirs) {
|
||||
const tools = discoverTools(dir);
|
||||
|
||||
for (const { entryFile, meta } of tools) {
|
||||
// User dirs come after global — last writer wins, so skip if already registered
|
||||
if (seen.has(meta.name)) continue;
|
||||
seen.add(meta.name);
|
||||
|
||||
const schema = buildSchema(meta.inputs ?? {});
|
||||
|
||||
// Capture entryFile in closure for lazy load
|
||||
const capturedEntry = entryFile;
|
||||
|
||||
pi.registerTool({
|
||||
name: meta.name,
|
||||
label: meta.label ?? meta.name,
|
||||
description: meta.description,
|
||||
parameters: schema,
|
||||
|
||||
async execute(toolCallId, params, signal, onUpdate, ctx) {
|
||||
// Lazy-load the implementation on first actual call.
|
||||
// Dynamic import works here because we're already in an async tool execution
|
||||
// context — jiti/Node has had time to set up its module hooks.
|
||||
let executeFn: Function | undefined;
|
||||
try {
|
||||
const mod = await import(capturedEntry);
|
||||
executeFn = mod.execute ?? mod.default?.execute;
|
||||
} catch (err) {
|
||||
return {
|
||||
content: [{ type: 'text', text: `[tool-loader] Failed to load ${meta.name}: ${String(err)}` }],
|
||||
details: { error: String(err) },
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (typeof executeFn !== 'function') {
|
||||
return {
|
||||
content: [{ type: 'text', text: `[tool-loader] ${meta.name}/index.ts must export an "execute" function` }],
|
||||
details: {},
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
return executeFn(toolCallId, params, signal, onUpdate, ctx);
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`[tool-loader] Registered tool: ${meta.name} (${capturedEntry})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
---
|
||||
name: web_fetch
|
||||
label: Web Fetch
|
||||
description: Fetch and return the text content of a URL. Tries to get clean readable content using a cascade of strategies in order: appending .md to the URL, fetching a /llms.txt discovery file, requesting plain text via Accept header, then falling back to stripping HTML. Use when the user provides a URL and wants to read, summarize, or extract information from web content.
|
||||
language: typescript
|
||||
inputs:
|
||||
url:
|
||||
type: string
|
||||
description: The URL to fetch content from
|
||||
---
|
||||
|
||||
# Web Fetch
|
||||
|
||||
Fetches web content with cascading fallback strategies to get the cleanest possible text.
|
||||
|
||||
## Strategies (in order)
|
||||
|
||||
1. **Markdown version** — Appends `.md` to the URL (works on GitHub, many docs sites)
|
||||
2. **llms.txt discovery** — Checks `/llms.txt` at the root (sites that publish LLM-friendly content)
|
||||
3. **Plain text request** — Sends `Accept: text/plain` header
|
||||
4. **HTML strip** — Fetches HTML and strips tags, scripts, and styles
|
||||
|
||||
## Output
|
||||
|
||||
Returns the text content with a note indicating which strategy succeeded.
|
||||
@@ -0,0 +1,188 @@
|
||||
const MAX_BYTES = 50_000;
|
||||
const TIMEOUT_MS = 15_000;
|
||||
|
||||
type FetchResult = {
|
||||
content: string;
|
||||
strategy: string;
|
||||
};
|
||||
|
||||
function truncate(text: string): { text: string; truncated: boolean; originalBytes: number } {
|
||||
const encoder = new TextEncoder();
|
||||
const bytes = encoder.encode(text);
|
||||
if (bytes.length <= MAX_BYTES) return { text, truncated: false, originalBytes: bytes.length };
|
||||
const decoder = new TextDecoder();
|
||||
return {
|
||||
text: decoder.decode(bytes.slice(0, MAX_BYTES)),
|
||||
truncated: true,
|
||||
originalBytes: bytes.length,
|
||||
};
|
||||
}
|
||||
|
||||
function stripHtml(html: string): string {
|
||||
// Remove script and style blocks entirely
|
||||
let text = html
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<noscript[\s\S]*?<\/noscript>/gi, '')
|
||||
.replace(/<nav[\s\S]*?<\/nav>/gi, '')
|
||||
.replace(/<footer[\s\S]*?<\/footer>/gi, '')
|
||||
.replace(/<header[\s\S]*?<\/header>/gi, '');
|
||||
|
||||
// Replace block-level tags with newlines
|
||||
text = text
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<\/p>/gi, '\n\n')
|
||||
.replace(/<\/div>/gi, '\n')
|
||||
.replace(/<\/li>/gi, '\n')
|
||||
.replace(/<\/h[1-6]>/gi, '\n\n')
|
||||
.replace(/<\/tr>/gi, '\n')
|
||||
.replace(/<\/td>/gi, '\t')
|
||||
.replace(/<\/th>/gi, '\t');
|
||||
|
||||
// Strip remaining tags
|
||||
text = text.replace(/<[^>]+>/g, '');
|
||||
|
||||
// Decode common HTML entities
|
||||
text = text
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)));
|
||||
|
||||
// Collapse excessive whitespace but preserve paragraph breaks
|
||||
text = text
|
||||
.replace(/[ \t]+/g, ' ')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
async function tryFetch(url: string, options: RequestInit = {}): Promise<Response | null> {
|
||||
try {
|
||||
const signal = AbortSignal.timeout(TIMEOUT_MS);
|
||||
const res = await fetch(url, { ...options, signal, redirect: 'follow' });
|
||||
if (res.ok) return res;
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function strategyMarkdown(url: string): Promise<FetchResult | null> {
|
||||
const mdUrl = url.endsWith('.md') ? null : `${url.replace(/\/$/, '')}.md`;
|
||||
if (!mdUrl) return null;
|
||||
|
||||
const res = await tryFetch(mdUrl);
|
||||
if (!res) return null;
|
||||
|
||||
const contentType = res.headers.get('content-type') ?? '';
|
||||
if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null;
|
||||
|
||||
const text = await res.text();
|
||||
if (text.trim().startsWith('<')) return null; // Got HTML anyway
|
||||
|
||||
return { content: text, strategy: `markdown (${mdUrl})` };
|
||||
}
|
||||
|
||||
async function strategyLlmsTxt(url: string): Promise<FetchResult | null> {
|
||||
const { origin } = new URL(url);
|
||||
const llmsUrl = `${origin}/llms.txt`;
|
||||
|
||||
const res = await tryFetch(llmsUrl);
|
||||
if (!res) return null;
|
||||
|
||||
const text = await res.text();
|
||||
if (!text.trim() || text.trim().startsWith('<')) return null;
|
||||
|
||||
return { content: text, strategy: `llms.txt (${llmsUrl})` };
|
||||
}
|
||||
|
||||
async function strategyPlainText(url: string): Promise<FetchResult | null> {
|
||||
const res = await tryFetch(url, { headers: { Accept: 'text/plain, text/markdown, */*;q=0.8' } });
|
||||
if (!res) return null;
|
||||
|
||||
const contentType = res.headers.get('content-type') ?? '';
|
||||
if (!contentType.includes('text/plain') && !contentType.includes('text/markdown')) return null;
|
||||
|
||||
const text = await res.text();
|
||||
if (text.trim().startsWith('<')) return null;
|
||||
|
||||
return { content: text, strategy: 'plain text response' };
|
||||
}
|
||||
|
||||
async function strategyHtmlStrip(url: string): Promise<FetchResult | null> {
|
||||
const res = await tryFetch(url, { headers: { Accept: 'text/html,*/*;q=0.8' } });
|
||||
if (!res) return null;
|
||||
|
||||
const html = await res.text();
|
||||
const text = stripHtml(html);
|
||||
|
||||
if (!text.trim()) return null;
|
||||
|
||||
return { content: text, strategy: 'HTML (stripped)' };
|
||||
}
|
||||
|
||||
export async function execute(
|
||||
_toolCallId: string,
|
||||
params: { url: string },
|
||||
_signal: AbortSignal | undefined,
|
||||
onUpdate?: (partial: { content: Array<{ type: string; text: string }> }) => void,
|
||||
) {
|
||||
const { url } = params;
|
||||
|
||||
// Validate URL
|
||||
let parsedUrl: URL;
|
||||
try {
|
||||
parsedUrl = new URL(url);
|
||||
} catch {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Invalid URL: ${url}` }],
|
||||
details: { error: 'invalid_url' },
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Unsupported protocol: ${parsedUrl.protocol}` }],
|
||||
details: { error: 'unsupported_protocol' },
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
const strategies = [
|
||||
{ name: 'markdown', fn: () => strategyMarkdown(url) },
|
||||
{ name: 'llms.txt', fn: () => strategyLlmsTxt(url) },
|
||||
{ name: 'plain text', fn: () => strategyPlainText(url) },
|
||||
{ name: 'HTML strip', fn: () => strategyHtmlStrip(url) },
|
||||
];
|
||||
|
||||
for (const { name, fn } of strategies) {
|
||||
onUpdate?.({ content: [{ type: 'text', text: `Trying ${name} strategy...` }] });
|
||||
|
||||
const result = await fn();
|
||||
if (!result) continue;
|
||||
|
||||
const { text, truncated, originalBytes } = truncate(result.content);
|
||||
|
||||
let output = `[Fetched via ${result.strategy}]\n\n${text}`;
|
||||
if (truncated) {
|
||||
output += `\n\n[Content truncated: showing ${MAX_BYTES.toLocaleString()} of ${originalBytes.toLocaleString()} bytes]`;
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: output }],
|
||||
details: { url, strategy: result.strategy, truncated, originalBytes },
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: `Failed to fetch content from: ${url}` }],
|
||||
details: { error: 'all_strategies_failed', url },
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user