unify agent items into a flat file-based store, drop the marketplace

Replace the marketplace service dependency and the native/global/user scope tiers with a single external directory ($OFFICER_ITEMS_DIR) holding skills, tools, tasks, processes and extensions as plain files. - tasks move from Postgres to TASK.md files (new file-backed task layer); task editing now works, which the DB path never supported - skills/tools/processes collapse into one shared file router (single dir) - remove the marketplace client (sync-marketplace/sync-version) and the boot-time sync; pi-bridge/pi-manager/sandbox point at the flat store - drop the dead tasks + vestigial skills/tools/processes/extensions + item_chats tables (migration 0004) - one-time migration script exports DB tasks and consolidates disk items Migration verified: all 6 tasks round-trip through the runtime parser identically to their DB rows (pipeline steps, triggers, script impls and agentic bodies all intact). NOTE: not yet functionally tested end-to-end — every item (each task mode, tool, skill, extension) still needs to be run/exercised in the app before this is trusted. To be done manually. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-21 00:39:17 +00:00
parent 5fd3d4faac
commit f3492512ba
29 changed files with 2366 additions and 1357 deletions
@@ -0,0 +1,247 @@
+import { readdir, mkdir, rm } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { itemsDir } from '../../data-path';
+
+// File-backed task store. Every task is a directory under $OFFICER_ITEMS_DIR/tasks/<dirName>/ with a
+// TASK.md (metadata + prose body) and, for script-mode tasks, a sibling implementation file. There are
+// no scope tiers and no database — the directory name is the task's identity.
+
+const YAML = (Bun as unknown as { YAML: { parse(input: string): unknown } }).YAML;
+
+export type TaskFrontmatter = {
+  name: string;
+  description: string | null;
+  version: number;
+  mode: string;
+  language: string | null;
+  args: string[] | null;
+  tags: string[] | null;
+  tools: string[] | null;
+  skills: string[] | null;
+  inputs: unknown;
+  outputs: unknown;
+  dependencies: unknown;
+  config: unknown;
+  trigger: unknown;
+};
+
+export type TaskRecord = TaskFrontmatter & {
+  dirName: string;
+  body: string;
+  implementation: string | null;
+  filePath: string;
+};
+
+export type TaskSummary = {
+  dirName: string;
+  name: string;
+  description: string | null;
+  mode: string;
+  version: number;
+  trigger: unknown;
+};
+
+// Frontmatter keys, in write order. These mirror the old `tasks` table columns 1:1.
+const FM_KEYS = [
+  'name',
+  'description',
+  'version',
+  'mode',
+  'language',
+  'args',
+  'tags',
+  'tools',
+  'skills',
+  'inputs',
+  'outputs',
+  'dependencies',
+  'config',
+  'trigger',
+] as const;
+
+// Script implementation lives in a sibling file named by language, matching the marketplace registry.
+const IMPL_FILE: Record<string, string> = {
+  bash: 'run.sh',
+  python: 'run.py',
+  typescript: 'index.ts',
+  javascript: 'index.js',
+};
+
+const implFileName = (language: string | null) => IMPL_FILE[language ?? 'bash'] ?? 'run.sh';
+
+const tasksRoot = () => itemsDir('tasks');
+const taskDir = (dirName: string) => join(tasksRoot(), dirName);
+const taskFile = (dirName: string) => join(taskDir(dirName), 'TASK.md');
+
+const asArray = (v: unknown): string[] | null => (Array.isArray(v) ? v.map(String) : null);
+
+export const slugify = (name: string) =>
+  name
+    .trim()
+    .toLowerCase()
+    .replace(/\s+/g, '-')
+    .replace(/[^a-z0-9-]/g, '');
+
+function parseTaskMd(raw: string): { fm: TaskFrontmatter; body: string } {
+  const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+  const body = match ? match[2]! : raw;
+  const yaml = match ? match[1]! : '';
+
+  let parsed: Record<string, unknown> = {};
+  if (yaml.trim()) {
+    try {
+      parsed = (YAML.parse(yaml) as Record<string, unknown>) ?? {};
+    } catch {
+      parsed = {};
+    }
+  }
+
+  const versionRaw = parsed.version;
+  const version = typeof versionRaw === 'number' ? versionRaw : Number(versionRaw) || 1;
+
+  return {
+    fm: {
+      name: parsed.name == null ? '' : String(parsed.name),
+      description: parsed.description == null ? null : String(parsed.description),
+      version,
+      mode: parsed.mode == null ? 'agentic' : String(parsed.mode),
+      language: parsed.language == null ? null : String(parsed.language),
+      args: asArray(parsed.args),
+      tags: asArray(parsed.tags),
+      tools: asArray(parsed.tools),
+      skills: asArray(parsed.skills),
+      inputs: parsed.inputs ?? null,
+      outputs: parsed.outputs ?? null,
+      dependencies: parsed.dependencies ?? null,
+      config: parsed.config ?? null,
+      trigger: parsed.trigger ?? null,
+    },
+    body,
+  };
+}
+
+// Serialize as JSON-flow YAML: every value is JSON.stringify'd, which is valid YAML and round-trips
+// losslessly. Bun.YAML.parse also accepts hand-written multi-line YAML, so files stay editable.
+function buildTaskMd(fm: Partial<TaskFrontmatter>, body: string): string {
+  const lines: string[] = ['---'];
+  for (const key of FM_KEYS) {
+    const val = (fm as Record<string, unknown>)[key];
+    if (val === undefined || val === null) continue;
+    if (Array.isArray(val) && val.length === 0) continue;
+    lines.push(`${key}: ${JSON.stringify(val)}`);
+  }
+  lines.push('---', '');
+  lines.push(body ?? '');
+  return lines.join('\n');
+}
+
+async function readImplementation(dirName: string, language: string | null): Promise<string | null> {
+  const implPath = join(taskDir(dirName), implFileName(language));
+  const file = Bun.file(implPath);
+  return (await file.exists()) ? file.text() : null;
+}
+
+export async function listTasks(): Promise<TaskSummary[]> {
+  let entries;
+  try {
+    entries = await readdir(tasksRoot(), { withFileTypes: true });
+  } catch {
+    return [];
+  }
+
+  const summaries: TaskSummary[] = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const file = Bun.file(taskFile(entry.name));
+    if (!(await file.exists())) continue;
+    const { fm } = parseTaskMd(await file.text());
+    summaries.push({
+      dirName: entry.name,
+      name: fm.name || entry.name,
+      description: fm.description,
+      mode: fm.mode,
+      version: fm.version,
+      trigger: fm.trigger ?? [],
+    });
+  }
+
+  summaries.sort((a, b) => a.name.localeCompare(b.name));
+  return summaries;
+}
+
+export async function getTaskByDirName(dirName: string): Promise<TaskRecord | null> {
+  const file = Bun.file(taskFile(dirName));
+  if (!(await file.exists())) return null;
+  const { fm, body } = parseTaskMd(await file.text());
+  const implementation = await readImplementation(dirName, fm.language);
+  return { ...fm, dirName, body, implementation, filePath: taskFile(dirName) };
+}
+
+type CreateTaskInput = {
+  name: string;
+  description?: string | null;
+  mode?: string;
+  language?: string | null;
+};
+
+export async function createTask(input: CreateTaskInput): Promise<TaskSummary & { filePath: string }> {
+  const name = input.name.trim();
+  const dirName = slugify(name);
+  if (!dirName) throw new Error('Invalid name');
+  if (await Bun.file(taskFile(dirName)).exists()) throw new Error('Task already exists');
+
+  const mode = input.mode ?? 'agentic';
+  const language = input.language ?? null;
+
+  await mkdir(taskDir(dirName), { recursive: true });
+  await Bun.write(
+    taskFile(dirName),
+    buildTaskMd({ name, description: input.description ?? null, version: 1, mode, language }, ''),
+  );
+  if (mode === 'script') await Bun.write(join(taskDir(dirName), implFileName(language)), '');
+
+  return {
+    dirName,
+    name,
+    description: input.description ?? null,
+    mode,
+    version: 1,
+    trigger: [],
+    filePath: taskFile(dirName),
+  };
+}
+
+export async function updateTask(
+  dirName: string,
+  patch: Partial<TaskFrontmatter> & { body?: string; implementation?: string | null },
+): Promise<TaskRecord | null> {
+  const existing = await getTaskByDirName(dirName);
+  if (!existing) return null;
+
+  const merged: TaskFrontmatter = { ...existing, ...patch };
+  const body = patch.body ?? existing.body;
+  await Bun.write(taskFile(dirName), buildTaskMd(merged, body));
+
+  if (patch.implementation !== undefined && patch.implementation !== null) {
+    await Bun.write(join(taskDir(dirName), implFileName(merged.language)), patch.implementation);
+  }
+
+  return getTaskByDirName(dirName);
+}
+
+export async function deleteTask(dirName: string): Promise<void> {
+  await rm(taskDir(dirName), { recursive: true, force: true });
+}
+
+// Write a full task record to disk (used by the one-time DB→files migration). Overwrites any
+// existing task with the same dirName so callers can apply their own precedence via write order.
+export async function importTask(
+  dirName: string,
+  record: Partial<TaskFrontmatter> & { body?: string; implementation?: string | null },
+): Promise<void> {
+  await mkdir(taskDir(dirName), { recursive: true });
+  await Bun.write(taskFile(dirName), buildTaskMd(record, record.body ?? ''));
+  if (record.implementation != null && record.implementation !== '') {
+    await Bun.write(join(taskDir(dirName), implFileName(record.language ?? null)), record.implementation);
+  }
+}