From f3492512badf5c81c94506588b5e2de5175f40f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Tue, 21 Jul 2026 00:39:17 +0000 Subject: [PATCH] unify agent items into a flat file-based store, drop the marketplace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the marketplace service dependency and the native/global/user scope tiers with a single external directory ($OFFICER_ITEMS_DIR) holding skills, tools, tasks, processes and extensions as plain files. - tasks move from Postgres to TASK.md files (new file-backed task layer); task editing now works, which the DB path never supported - skills/tools/processes collapse into one shared file router (single dir) - remove the marketplace client (sync-marketplace/sync-version) and the boot-time sync; pi-bridge/pi-manager/sandbox point at the flat store - drop the dead tasks + vestigial skills/tools/processes/extensions + item_chats tables (migration 0004) - one-time migration script exports DB tasks and consolidates disk items Migration verified: all 6 tasks round-trip through the runtime parser identically to their DB rows (pipeline steps, triggers, script impls and agentic bodies all intact). NOTE: not yet functionally tested end-to-end — every item (each task mode, tool, skill, extension) still needs to be run/exercised in the app before this is trusted. To be done manually. Co-Authored-By: Claude Opus 4.8 --- .env.example | 1 + scripts/migrate-items-to-files.ts | 112 ++ scripts/reset-user-data.ts | 20 +- .../Dashboard/Automation/AutomationDetail.tsx | 77 +- .../Screens/Dashboard/CapabilityPage.tsx | 37 +- .../migrations/0004_true_annihilus.sql | 6 + .../migrations/meta/0004_snapshot.json | 1685 +++++++++++++++++ .../officer_db/migrations/meta/_journal.json | 7 + src/databases/officer_db/src/index.ts | 11 - src/databases/officer_db/src/queries/tasks.ts | 100 - .../officer_db/src/schema/agent-items.ts | 159 -- src/databases/officer_db/src/schema/index.ts | 1 - src/databases/officer_db/src/types.ts | 30 - src/servers/api/item-router.ts | 158 ++ src/servers/api/pi/pi-bridge.ts | 27 +- src/servers/api/processes/processes.ts | 212 +-- src/servers/api/skills/skills.ts | 215 +-- src/servers/api/tasks/pipeline-executor.ts | 9 +- src/servers/api/tasks/task-executor.ts | 8 +- src/servers/api/tasks/task-files.ts | 247 +++ src/servers/api/tasks/tasks.ts | 59 +- src/servers/api/tools/tools.ts | 218 +-- src/servers/bootstrap.ts | 6 +- src/servers/data-path.ts | 46 +- src/servers/generate-container-context.ts | 33 +- src/servers/sidecar/pi/pi-manager.ts | 40 +- src/servers/sidecar/sandbox.ts | 7 +- src/servers/sync-marketplace.ts | 182 -- src/servers/sync-version.ts | 10 - 29 files changed, 2366 insertions(+), 1357 deletions(-) create mode 100644 scripts/migrate-items-to-files.ts create mode 100644 src/databases/officer_db/migrations/0004_true_annihilus.sql create mode 100644 src/databases/officer_db/migrations/meta/0004_snapshot.json delete mode 100644 src/databases/officer_db/src/queries/tasks.ts delete mode 100644 src/databases/officer_db/src/schema/agent-items.ts create mode 100644 src/servers/api/item-router.ts create mode 100644 src/servers/api/tasks/task-files.ts delete mode 100644 src/servers/sync-marketplace.ts delete mode 100644 src/servers/sync-version.ts diff --git a/.env.example b/.env.example index 301ea540..591ecb63 100644 --- a/.env.example +++ b/.env.example @@ -4,5 +4,6 @@ POSTGRES_URL="postgres://postgres:password@localhost:5432/officer" MAIL_TRANSPORT="smtp://localhost:1025" PUBLIC_URL=http://localhost:9000 DATA_PATH=/path/to/data +OFFICER_ITEMS_DIR=/path/to/officer-items HOME_DIR=/home/user BROWSER_RELAY_PORT=18792 diff --git a/scripts/migrate-items-to-files.ts b/scripts/migrate-items-to-files.ts new file mode 100644 index 00000000..07d26dd1 --- /dev/null +++ b/scripts/migrate-items-to-files.ts @@ -0,0 +1,112 @@ +/** + * One-time migration: consolidate every agent item into the flat, file-based store + * ($OFFICER_ITEMS_DIR) and export the DB-backed `tasks` table to TASK.md files. + * + * Idempotent — safe to re-run. Run this BEFORE applying the drop-tables DB migration + * (it reads the `tasks` table, which still exists until that migration runs). + * + * Sources, in precedence order (later overwrites earlier on a dirName collision): + * - tasks: officer_db.tasks rows (native → global → user) + * - skills / tools / processes / extensions: $DATA_PATH/ then $DATA_PATH// + * - tools: marketplace registry tools not already present (archive safety) + * + * Usage: bun run scripts/migrate-items-to-files.ts + */ + +import { join, resolve } from 'node:path'; +import { readdir, cp } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { db } from 'officerdb/db'; +import { sql } from 'drizzle-orm'; +import { itemsDir, ensureItemDirs, DATA_PATH, OFFICER_ITEMS_DIR, type ItemType } from '../src/servers/data-path'; +import { importTask } from '../src/servers/api/tasks/task-files'; + +ensureItemDirs(); +console.log(`Target store: ${OFFICER_ITEMS_DIR}`); + +async function listSubdirs(dir: string): Promise { + try { + return (await readdir(dir, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => e.name); + } catch { + return []; + } +} + +// ── 1. Tasks: Postgres → TASK.md files ── +// Order native → global → user so user/global overwrite native on a dirName collision. +const scopeRank = (s: string) => (s === 'user' ? 2 : s === 'global' ? 1 : 0); + +console.log('\n── Tasks (DB → files) ──'); +let taskRows: Record[] = []; +try { + taskRows = (await db.execute(sql.raw('SELECT * FROM tasks'))) as unknown as Record[]; +} catch (err) { + console.log(` could not read tasks table (already dropped?): ${err instanceof Error ? err.message : err}`); +} +taskRows.sort((a, b) => scopeRank(String(a.scope)) - scopeRank(String(b.scope))); + +for (const row of taskRows) { + const dirName = String(row.dir_name); + await importTask(dirName, { + name: String(row.name ?? dirName), + description: row.description == null ? null : String(row.description), + version: Number(row.version) || 1, + mode: String(row.mode ?? 'agentic'), + language: row.language == null ? null : String(row.language), + args: (row.args as string[] | null) ?? null, + tags: (row.tags as string[] | null) ?? null, + tools: (row.tools as string[] | null) ?? null, + skills: (row.skills as string[] | null) ?? null, + inputs: row.inputs ?? null, + outputs: row.outputs ?? null, + dependencies: row.dependencies ?? null, + config: row.config ?? null, + trigger: row.trigger ?? null, + body: row.body == null ? '' : String(row.body), + implementation: row.implementation == null ? null : String(row.implementation), + }); + console.log(` ${dirName} (${row.scope})`); +} +console.log(` ${taskRows.length} task file(s) written`); + +// ── 2. On-disk items → flat store ── +const DISK_TYPES: ItemType[] = ['skills', 'tools', 'processes', 'extensions']; + +async function copyItemsFrom(srcTypeDir: string, type: ItemType): Promise { + let n = 0; + for (const name of await listSubdirs(srcTypeDir)) { + await cp(join(srcTypeDir, name), join(itemsDir(type), name), { recursive: true, force: true }); + n++; + } + return n; +} + +console.log('\n── Disk items (DATA_PATH → flat store) ──'); +const emailDirs = (await listSubdirs(DATA_PATH)).filter((n) => n.includes('@')); + +for (const type of DISK_TYPES) { + let n = await copyItemsFrom(join(DATA_PATH, type), type); // global + for (const email of emailDirs) n += await copyItemsFrom(join(DATA_PATH, email, type), type); // user (overwrites) + console.log(` ${type}: ${n} item(s) copied`); +} + +// ── 3. Marketplace registry tools not already present (archive safety) ── +const MARKETPLACE_REGISTRY = process.env.MARKETPLACE_REGISTRY ?? resolve(import.meta.dir, '../../marketplace/registry'); +console.log(`\n── Marketplace registry (${MARKETPLACE_REGISTRY}) ──`); +if (existsSync(MARKETPLACE_REGISTRY)) { + let n = 0; + for (const name of await listSubdirs(join(MARKETPLACE_REGISTRY, 'tools'))) { + const target = join(itemsDir('tools'), name); + if (existsSync(target)) continue; // don't clobber a synced/user version + await cp(join(MARKETPLACE_REGISTRY, 'tools', name), target, { recursive: true }); + n++; + console.log(` tool ${name} (from registry)`); + } + console.log(` ${n} registry tool(s) added`); + console.log(' registry tasks come from the DB export above (native scope) — skipped here'); +} else { + console.log(' registry not found, skipping'); +} + +console.log('\nDone. Verify counts in the UI, then apply the drop-tables DB migration.'); +process.exit(0); diff --git a/scripts/reset-user-data.ts b/scripts/reset-user-data.ts index 5d531ff3..15a90f85 100644 --- a/scripts/reset-user-data.ts +++ b/scripts/reset-user-data.ts @@ -5,7 +5,6 @@ * - DB: user_settings, user_state, user_integrations, dock_configs, * chat_sessions (cascades chat_messages), chat_groups, * dashboards, screens, projects, - * tasks, skills, processes, tools, extensions (+ item_chats), * task_logs, queue_jobs, terminal_containers * - Filesystem: entire $DATA_PATH// directory * (home, settings, state, dashboards, chat_sessions, emails.db, @@ -76,7 +75,7 @@ const tables = [ 'user_state', 'user_integrations', 'dock_configs', - 'chat_sessions', // cascades chat_messages + 'chat_sessions', // cascades chat_messages 'chat_groups', 'dashboards', 'screens', @@ -92,21 +91,8 @@ for (const table of tables) { console.log(` ${table}: ${count} rows deleted`); } -// Agent items: tasks, skills, processes, tools, extensions -// These have nullable user_id — delete only rows belonging to this user -const agentTables = ['tasks', 'skills', 'processes', 'tools', 'extensions']; - -for (const table of agentTables) { - // First collect item IDs to clean up item_chats - const items = await db.execute(sql.raw(`SELECT id FROM ${table} WHERE user_id = ${userId}`)); - if (items.length > 0) { - const ids = items.map((r: Record) => r.id).join(','); - const chatResult = await db.execute(sql.raw(`DELETE FROM item_chats WHERE item_type = '${table}' AND item_id IN (${ids})`)); - console.log(` item_chats (${table}): ${chatResult.length ?? 0} rows deleted`); - } - const result = await db.execute(sql.raw(`DELETE FROM ${table} WHERE user_id = ${userId}`)); - console.log(` ${table}: ${result.length ?? 0} rows deleted`); -} +// Agent items (skills, tools, tasks, processes, extensions) are now flat files in +// $OFFICER_ITEMS_DIR, shared and not user-owned — intentionally left untouched by a user reset. // ── Queue job files ── diff --git a/src/apps/officer-web/Screens/Dashboard/Automation/AutomationDetail.tsx b/src/apps/officer-web/Screens/Dashboard/Automation/AutomationDetail.tsx index f6f34945..e6ba8d25 100644 --- a/src/apps/officer-web/Screens/Dashboard/Automation/AutomationDetail.tsx +++ b/src/apps/officer-web/Screens/Dashboard/Automation/AutomationDetail.tsx @@ -8,25 +8,21 @@ import { Play, Trash2, Terminal, Bot, Workflow } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from '@/components/ui/dialog'; import { useClient } from 'hooks/useClient'; -import { useAuth } from 'hooks/useAuth'; import { usePanelChannel } from 'hooks/usePanelChannel'; import { Card } from '@/components/Card'; import { TaskRunnerModal } from 'officerdev'; import type { TaskSummary } from 'officerdev'; type TaskDetail = { - id: number; dirName: string; name: string; description: string; - scope: string; mode: string; language: string | null; body: string | null; inputs: Record | null; config: { steps?: Array<{ task: string; foreach?: string }> } | null; version: number | null; - userId: number | null; }; const modeLabels: Record = { @@ -38,13 +34,14 @@ const modeLabels: Record { const client = useClient(); const qc = useQueryClient(); - const { user } = useAuth(); const [selected, setSelected] = usePanelChannel('automation:selected-task', null); const [deleteConfirm, setDeleteConfirm] = useState(false); const [runTask, setRunTask] = useState(null); // Reset delete confirm when selection changes - useEffect(() => { setDeleteConfirm(false); }, [selected?.dirName]); + useEffect(() => { + setDeleteConfirm(false); + }, [selected?.dirName]); const { data: detail } = useQuery({ queryKey: ['tasks', selected?.dirName], @@ -73,7 +70,6 @@ export const AutomationDetail = () => { ); } - const canModify = selected.scope === 'user' || user?.role === 'Super Admin'; const mode = modeLabels[selected.mode] ?? modeLabels.agentic!; const ModeIcon = mode.icon; @@ -88,7 +84,9 @@ export const AutomationDetail = () => { {detail?.name ?? selected.name} - + {mode.label} @@ -99,15 +97,13 @@ export const AutomationDetail = () => { > - {canModify && ( - - )} + {/* Content */} @@ -119,11 +115,6 @@ export const AutomationDetail = () => { {/* Meta badges */}
- {detail?.scope && detail.scope !== 'user' && ( - - {detail.scope} - - )} {detail?.language && ( {detail.language} @@ -139,13 +130,24 @@ export const AutomationDetail = () => { {/* Pipeline steps */} {hasSteps && (
-

Pipeline Steps

+

+ Pipeline Steps +

{detail!.config!.steps!.map((step, i) => ( -
- {i + 1} +
+ + {i + 1} + {step.task} - {step.foreach && (foreach: {step.foreach})} + {step.foreach && ( + + (foreach: {step.foreach}) + + )}
))}
@@ -155,15 +157,26 @@ export const AutomationDetail = () => { {/* Inputs */} {hasInputs && (
-

Inputs

+

+ Inputs +

{Object.entries(detail!.inputs!).map(([key, def]) => { const d = def as { type?: string; description?: string; default?: string }; return ( -
+
{key} - {d.type && {d.type}} - {d.description && {d.description}} + {d.type && ( + {d.type} + )} + {d.description && ( + + {d.description} + + )}
); })} @@ -208,7 +221,9 @@ export const AutomationDetail = () => { {runTask && ( { if (!open) setRunTask(null); }} + onOpenChange={(open) => { + if (!open) setRunTask(null); + }} task={runTask} /> )} diff --git a/src/apps/officer-web/Screens/Dashboard/CapabilityPage.tsx b/src/apps/officer-web/Screens/Dashboard/CapabilityPage.tsx index c5059554..40b9cc72 100644 --- a/src/apps/officer-web/Screens/Dashboard/CapabilityPage.tsx +++ b/src/apps/officer-web/Screens/Dashboard/CapabilityPage.tsx @@ -8,14 +8,12 @@ import { ArrowLeft, Pencil, Plus, Check, X, Trash2, Search, ChevronRight } from import { Button } from '@/components/ui/button'; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from '@/components/ui/dialog'; import { useClient } from 'hooks/useClient'; -import { useAuth } from 'hooks/useAuth'; import { Card } from '@/components/Card'; import { usePiChat, EmbeddableChat } from 'officerdev'; type CapabilitySummary = { dirName: string; name: string; description: string; - scope: 'native' | 'global' | 'user'; }; export type CapabilityDetail = CapabilitySummary & { @@ -276,9 +274,9 @@ export const CapabilityChat = ({ }: CapabilityChatProps) => { const seedFile = `${kind.toUpperCase()}.md`; const genericPrefix = `\ninput file: ${filePath}\n${seedFile}: ${filePath}\ndir: ${resourceDir}\n\nBe aware of any extra files alongside the same dir as the ${kind} file we're handling, for possible extra context. You can also, if pertinent, create scripts or other files that will help you in the future.\n`; - const promptFrontmatter = isNew ? buildCreationPrefix(kind, filePath, resourceDir) ?? genericPrefix : genericPrefix; + const promptFrontmatter = isNew ? (buildCreationPrefix(kind, filePath, resourceDir) ?? genericPrefix) : genericPrefix; const defaultInput = isNew - ? description ?? `Help me create the content for this new ${kind} file` + ? (description ?? `Help me create the content for this new ${kind} file`) : `Help me understand and improve this ${kind} file`; const pi = usePiChat(undefined, undefined, { replaceUrl: false }); @@ -294,14 +292,7 @@ export const CapabilityChat = ({ wasGenerating.current = pi.isGenerating; }, [pi.isGenerating]); - return ( - - ); + return ; }; export const FrontmatterBlock = ({ yaml }: { yaml: string }) => { @@ -323,7 +314,17 @@ export const FrontmatterBlock = ({ yaml }: { yaml: string }) => { ); }; -export const CapabilityList = ({ kind, endpoint, queryKey, selected, onSelect, onCreate, search: externalSearch, showCreate, onShowCreateChange }: CapabilityListProps) => { +export const CapabilityList = ({ + kind, + endpoint, + queryKey, + selected, + onSelect, + onCreate, + search: externalSearch, + showCreate, + onShowCreateChange, +}: CapabilityListProps) => { const client = useClient(); const qc = useQueryClient(); const [internalCreating, setInternalCreating] = useState(false); @@ -437,13 +438,6 @@ export const CapabilityList = ({ kind, endpoint, queryKey, selected, onSelect, o >
{item.name} - - {item.scope} -
{item.description &&

{item.description}

} @@ -462,7 +456,6 @@ export const CapabilityList = ({ kind, endpoint, queryKey, selected, onSelect, o export const CapabilityPage = ({ kind, endpoint, queryKey }: CapabilityPageProps) => { const client = useClient(); const qc = useQueryClient(); - const { user } = useAuth(); const [selected, setSelected] = useState(null); const [editing, setEditing] = useState(false); const [isNew, setIsNew] = useState(false); @@ -544,7 +537,7 @@ export const CapabilityPage = ({ kind, endpoint, queryKey }: CapabilityPageProps {detail?.name ?? `Select a ${kind.toLowerCase()}`} - {detail && (detail.scope === 'user' || user?.role === 'Super Admin') && ( + {detail && ( <>