unify agent items into a flat file-based store, drop the marketplace

Replace the marketplace service dependency and the native/global/user scope tiers with a single external directory ($OFFICER_ITEMS_DIR) holding skills, tools, tasks, processes and extensions as plain files. - tasks move from Postgres to TASK.md files (new file-backed task layer); task editing now works, which the DB path never supported - skills/tools/processes collapse into one shared file router (single dir) - remove the marketplace client (sync-marketplace/sync-version) and the boot-time sync; pi-bridge/pi-manager/sandbox point at the flat store - drop the dead tasks + vestigial skills/tools/processes/extensions + item_chats tables (migration 0004) - one-time migration script exports DB tasks and consolidates disk items Migration verified: all 6 tasks round-trip through the runtime parser identically to their DB rows (pipeline steps, triggers, script impls and agentic bodies all intact). NOTE: not yet functionally tested end-to-end — every item (each task mode, tool, skill, extension) still needs to be run/exercised in the app before this is trusted. To be done manually. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-21 00:39:17 +00:00
parent 5fd3d4faac
commit f3492512ba
29 changed files with 2366 additions and 1357 deletions
@@ -0,0 +1,112 @@
+/**
+ * One-time migration: consolidate every agent item into the flat, file-based store
+ * ($OFFICER_ITEMS_DIR) and export the DB-backed `tasks` table to TASK.md files.
+ *
+ * Idempotent — safe to re-run. Run this BEFORE applying the drop-tables DB migration
+ * (it reads the `tasks` table, which still exists until that migration runs).
+ *
+ * Sources, in precedence order (later overwrites earlier on a dirName collision):
+ *   - tasks:   officer_db.tasks rows (native → global → user)
+ *   - skills / tools / processes / extensions: $DATA_PATH/<type> then $DATA_PATH/<email>/<type>
+ *   - tools:   marketplace registry tools not already present (archive safety)
+ *
+ * Usage: bun run scripts/migrate-items-to-files.ts
+ */
+
+import { join, resolve } from 'node:path';
+import { readdir, cp } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { db } from 'officerdb/db';
+import { sql } from 'drizzle-orm';
+import { itemsDir, ensureItemDirs, DATA_PATH, OFFICER_ITEMS_DIR, type ItemType } from '../src/servers/data-path';
+import { importTask } from '../src/servers/api/tasks/task-files';
+
+ensureItemDirs();
+console.log(`Target store: ${OFFICER_ITEMS_DIR}`);
+
+async function listSubdirs(dir: string): Promise<string[]> {
+  try {
+    return (await readdir(dir, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => e.name);
+  } catch {
+    return [];
+  }
+}
+
+// ── 1. Tasks: Postgres → TASK.md files ──
+// Order native → global → user so user/global overwrite native on a dirName collision.
+const scopeRank = (s: string) => (s === 'user' ? 2 : s === 'global' ? 1 : 0);
+
+console.log('\n── Tasks (DB → files) ──');
+let taskRows: Record<string, unknown>[] = [];
+try {
+  taskRows = (await db.execute(sql.raw('SELECT * FROM tasks'))) as unknown as Record<string, unknown>[];
+} catch (err) {
+  console.log(`  could not read tasks table (already dropped?): ${err instanceof Error ? err.message : err}`);
+}
+taskRows.sort((a, b) => scopeRank(String(a.scope)) - scopeRank(String(b.scope)));
+
+for (const row of taskRows) {
+  const dirName = String(row.dir_name);
+  await importTask(dirName, {
+    name: String(row.name ?? dirName),
+    description: row.description == null ? null : String(row.description),
+    version: Number(row.version) || 1,
+    mode: String(row.mode ?? 'agentic'),
+    language: row.language == null ? null : String(row.language),
+    args: (row.args as string[] | null) ?? null,
+    tags: (row.tags as string[] | null) ?? null,
+    tools: (row.tools as string[] | null) ?? null,
+    skills: (row.skills as string[] | null) ?? null,
+    inputs: row.inputs ?? null,
+    outputs: row.outputs ?? null,
+    dependencies: row.dependencies ?? null,
+    config: row.config ?? null,
+    trigger: row.trigger ?? null,
+    body: row.body == null ? '' : String(row.body),
+    implementation: row.implementation == null ? null : String(row.implementation),
+  });
+  console.log(`  ${dirName} (${row.scope})`);
+}
+console.log(`  ${taskRows.length} task file(s) written`);
+
+// ── 2. On-disk items → flat store ──
+const DISK_TYPES: ItemType[] = ['skills', 'tools', 'processes', 'extensions'];
+
+async function copyItemsFrom(srcTypeDir: string, type: ItemType): Promise<number> {
+  let n = 0;
+  for (const name of await listSubdirs(srcTypeDir)) {
+    await cp(join(srcTypeDir, name), join(itemsDir(type), name), { recursive: true, force: true });
+    n++;
+  }
+  return n;
+}
+
+console.log('\n── Disk items (DATA_PATH → flat store) ──');
+const emailDirs = (await listSubdirs(DATA_PATH)).filter((n) => n.includes('@'));
+
+for (const type of DISK_TYPES) {
+  let n = await copyItemsFrom(join(DATA_PATH, type), type); // global
+  for (const email of emailDirs) n += await copyItemsFrom(join(DATA_PATH, email, type), type); // user (overwrites)
+  console.log(`  ${type}: ${n} item(s) copied`);
+}
+
+// ── 3. Marketplace registry tools not already present (archive safety) ──
+const MARKETPLACE_REGISTRY = process.env.MARKETPLACE_REGISTRY ?? resolve(import.meta.dir, '../../marketplace/registry');
+console.log(`\n── Marketplace registry (${MARKETPLACE_REGISTRY}) ──`);
+if (existsSync(MARKETPLACE_REGISTRY)) {
+  let n = 0;
+  for (const name of await listSubdirs(join(MARKETPLACE_REGISTRY, 'tools'))) {
+    const target = join(itemsDir('tools'), name);
+    if (existsSync(target)) continue; // don't clobber a synced/user version
+    await cp(join(MARKETPLACE_REGISTRY, 'tools', name), target, { recursive: true });
+    n++;
+    console.log(`  tool ${name} (from registry)`);
+  }
+  console.log(`  ${n} registry tool(s) added`);
+  console.log('  registry tasks come from the DB export above (native scope) — skipped here');
+} else {
+  console.log('  registry not found, skipping');
+}
+
+console.log('\nDone. Verify counts in the UI, then apply the drop-tables DB migration.');
+process.exit(0);
@@ -5,7 +5,6 @@
 * - DB: user_settings, user_state, user_integrations, dock_configs,
 *        chat_sessions (cascades chat_messages), chat_groups,
 *        dashboards, screens, projects,
- *        tasks, skills, processes, tools, extensions (+ item_chats),
 *        task_logs, queue_jobs, terminal_containers
 * - Filesystem: entire $DATA_PATH/<email>/ directory
 *        (home, settings, state, dashboards, chat_sessions, emails.db,
@@ -76,7 +75,7 @@ const tables = [
  'user_state',
  'user_integrations',
  'dock_configs',
-  'chat_sessions',  // cascades chat_messages
+  'chat_sessions', // cascades chat_messages
  'chat_groups',
  'dashboards',
  'screens',
@@ -92,21 +91,8 @@ for (const table of tables) {
  console.log(`  ${table}: ${count} rows deleted`);
 }

-// Agent items: tasks, skills, processes, tools, extensions
-// These have nullable user_id — delete only rows belonging to this user
-const agentTables = ['tasks', 'skills', 'processes', 'tools', 'extensions'];
-
-for (const table of agentTables) {
-  // First collect item IDs to clean up item_chats
-  const items = await db.execute(sql.raw(`SELECT id FROM ${table} WHERE user_id = ${userId}`));
-  if (items.length > 0) {
-    const ids = items.map((r: Record<string, unknown>) => r.id).join(',');
-    const chatResult = await db.execute(sql.raw(`DELETE FROM item_chats WHERE item_type = '${table}' AND item_id IN (${ids})`));
-    console.log(`  item_chats (${table}): ${chatResult.length ?? 0} rows deleted`);
-  }
-  const result = await db.execute(sql.raw(`DELETE FROM ${table} WHERE user_id = ${userId}`));
-  console.log(`  ${table}: ${result.length ?? 0} rows deleted`);
-}
+// Agent items (skills, tools, tasks, processes, extensions) are now flat files in
+// $OFFICER_ITEMS_DIR, shared and not user-owned — intentionally left untouched by a user reset.

 // ── Queue job files ──