task logs: migrate from filesystem to postgresql; refactor sidecars into submodules

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-06 11:49:39 +00:00
co-authored by Claude Opus 4.6
parent 5129f7827f
commit d88fe3cac7
22 changed files with 621 additions and 545 deletions
@@ -0,0 +1,388 @@
import { join } from 'node:path';
import type { Subprocess } from 'bun';
import type { PiEvent, MessageCost } from '../../api/pi/types';
import type { ClaudeSpawnParams, ClaudeSpawnStreamingParams, ClaudeCodeResult } from '../protocol';
import { getState, setClaudeSession, clearClaudeSession, getClaudeSession } from './state';
import { getProxySecret } from './proxy';
const SEND_TIMEOUT_MS = 5 * 60 * 1000;
const PROXY_PORT = process.env.ANTHROPIC_PROXY_PORT ?? '5051';
const DATA_PATH = process.env.DATA_PATH ?? join(process.cwd(), 'data');
const getHomeDir = (email: string) => join(DATA_PATH, email, 'home');
const toShellUsername = (username: string, email: string): string => {
const raw = username || email.split('@')[0]!;
return (
raw
.replace(/@.*$/, '')
.replace(/[^a-zA-Z0-9._-]/g, '_')
.toLowerCase()
.slice(0, 32) || 'officer'
);
};
async function hasOwnCredentials(homeDir: string): Promise<boolean> {
try {
return await Bun.file(join(homeDir, '.claude', '.credentials.json')).exists();
} catch {
return false;
}
}
// Resolve absolute path to claude binary
const CLAUDE_BIN = (() => {
const result = Bun.spawnSync({ cmd: ['which', 'claude'], stdout: 'pipe', stderr: 'ignore' });
return result.stdout.toString().trim() || 'claude';
})();
// Active streaming processes
const activeProcs = new Map<string, Subprocess>();
function buildAuthEnv(
shellUsername: string,
homeDir: string,
isServiceUser: boolean,
userHasCredentials: boolean,
): Record<string, string> {
if (isServiceUser) return { HOME: process.env.HOME ?? '' };
if (userHasCredentials) return { HOME: homeDir };
return { ANTHROPIC_BASE_URL: `http://127.0.0.1:${PROXY_PORT}`, ANTHROPIC_API_KEY: getProxySecret() };
}
// ── Blocking send ──
type ClaudeCodeOutput = {
result: string;
session_id: string;
cost_usd: number;
input_tokens: number;
output_tokens: number;
is_error: boolean;
};
export async function spawnClaude(params: ClaudeSpawnParams): Promise<ClaudeCodeResult> {
const { userId, email, username, prompt, sessionKey } = params;
const homeDir = getHomeDir(email);
const shellUsername = toShellUsername(username, email);
const claudeArgs = [CLAUDE_BIN, '-p', prompt, '--dangerously-skip-permissions', '--output-format', 'json'];
const subModel = params.model?.split('/')[1];
if (subModel) claudeArgs.push('--model', subModel);
const existingSession = getClaudeSession(sessionKey);
if (existingSession) {
claudeArgs.push('--resume', existingSession);
}
const isServiceUser = shellUsername === (process.env.USER ?? '');
const userHasCredentials = !isServiceUser && (await hasOwnCredentials(homeDir));
const authEnv = buildAuthEnv(shellUsername, homeDir, isServiceUser, userHasCredentials);
const env: Record<string, string> = {
...authEnv,
PATH: process.env.PATH ?? '',
TERM: 'xterm-256color',
};
const proc = isServiceUser
? Bun.spawn(claudeArgs, { stdin: 'pipe', stdout: 'pipe', stderr: 'pipe', env: { ...process.env, ...env } })
: Bun.spawn(
['sudo', '-u', shellUsername, 'env', ...Object.entries(env).map(([k, v]) => `${k}=${v}`), ...claudeArgs],
{ stdin: 'pipe', stdout: 'pipe', stderr: 'pipe' },
);
const timeout = setTimeout(() => {
try {
proc.kill();
} catch {
/* already dead */
}
}, SEND_TIMEOUT_MS);
try {
const [stdout, stderr] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text()]);
const exitCode = await proc.exited;
clearTimeout(timeout);
if (exitCode !== 0 && !stdout.trim()) {
throw new Error(`claude exited with code ${exitCode}: ${stderr.trim().slice(0, 200)}`);
}
let output: ClaudeCodeOutput;
try {
output = JSON.parse(stdout) as ClaudeCodeOutput;
} catch {
return {
text: stdout.trim() || '(no response)',
sessionId: sessionKey,
model: 'claude-code',
cost: { inputTokens: 0, outputTokens: 0, totalUSD: 0 },
};
}
if (output.is_error) {
throw new Error(output.result || 'Claude Code returned an error');
}
if (output.session_id) {
setClaudeSession(sessionKey, output.session_id);
}
return {
text: output.result || '(no response)',
sessionId: sessionKey,
model: 'claude-code',
cost: {
inputTokens: output.input_tokens ?? 0,
outputTokens: output.output_tokens ?? 0,
totalUSD: output.cost_usd ?? 0,
},
};
} catch (err) {
clearTimeout(timeout);
throw err;
}
}
// ── Streaming send ──
export async function spawnClaudeStreaming(
params: ClaudeSpawnStreamingParams,
onEvent: (event: PiEvent) => void,
): Promise<void> {
const { userId, email, username, prompt, sessionKey, cwd } = params;
const shellUsername = toShellUsername(username, email);
const homeDir = getHomeDir(email);
const workDir = cwd ?? homeDir;
const claudeArgs = [
CLAUDE_BIN,
'-p',
prompt,
'--dangerously-skip-permissions',
'--output-format',
'stream-json',
'--verbose',
'--include-partial-messages',
];
const subModel = params.model?.split('/')[1];
if (subModel) claudeArgs.push('--model', subModel);
const existingSession = getClaudeSession(sessionKey);
if (existingSession) {
claudeArgs.push('--resume', existingSession);
}
const { CLAUDECODE: _, ...cleanEnv } = process.env;
const isServiceUser = shellUsername === (process.env.USER ?? '');
const userHasCredentials = !isServiceUser && (await hasOwnCredentials(homeDir));
const authEnv = buildAuthEnv(shellUsername, homeDir, isServiceUser, userHasCredentials);
const env: Record<string, string> = {
...authEnv,
PATH: cleanEnv.PATH ?? '',
TERM: 'xterm-256color',
};
const proc = isServiceUser
? Bun.spawn(claudeArgs, {
cwd: workDir,
stdin: 'ignore',
stdout: 'pipe',
stderr: 'pipe',
env: { ...cleanEnv, ...env },
})
: Bun.spawn(
['sudo', '-u', shellUsername, 'env', ...Object.entries(env).map(([k, v]) => `${k}=${v}`), ...claudeArgs],
{ cwd: workDir, stdin: 'ignore', stdout: 'pipe', stderr: 'pipe' },
);
activeProcs.set(sessionKey, proc);
const timeout = setTimeout(() => {
try {
proc.kill();
} catch {
/* already dead */
}
onEvent({ type: 'error', message: 'Claude Code timed out after 5 minutes' });
}, SEND_TIMEOUT_MS);
// Process NDJSON stream
try {
const stdout = proc.stdout as ReadableStream<Uint8Array>;
const reader = stdout.getReader();
const decoder = new TextDecoder();
let buffer = '';
let textBuffer = '';
let gotResult = false;
const processLine = (line: string) => {
if (!line.trim()) return;
try {
const msg = JSON.parse(line) as Record<string, unknown>;
const type = msg.type as string;
if (type === 'stream_event') {
const event = msg.event as Record<string, unknown> | undefined;
if (event?.type === 'content_block_delta') {
const delta = event.delta as Record<string, unknown> | undefined;
if (delta?.type === 'text_delta' && typeof delta.text === 'string') {
textBuffer += delta.text;
onEvent({ type: 'delta', text: delta.text });
}
}
} else if (type === 'assistant') {
const message = msg.message as Record<string, unknown> | undefined;
const content = message?.content as Array<Record<string, unknown>> | undefined;
if (Array.isArray(content)) {
for (const block of content) {
if (block.type === 'text' && typeof block.text === 'string') {
onEvent({ type: 'text', text: block.text });
textBuffer = '';
} else if (block.type === 'tool_use') {
if (textBuffer) {
onEvent({ type: 'text', text: textBuffer });
textBuffer = '';
}
onEvent({
type: 'tool:start',
toolCallId: (block.id as string) ?? '',
toolName: (block.name as string) ?? 'unknown',
toolInput: (block.input as Record<string, unknown>) ?? {},
});
}
}
}
} else if (type === 'user') {
const message = msg.message as Record<string, unknown> | undefined;
const content = message?.content as Array<Record<string, unknown>> | undefined;
if (Array.isArray(content)) {
for (const block of content) {
if (block.type === 'tool_result') {
let output = '';
if (typeof block.content === 'string') {
output = block.content;
} else if (Array.isArray(block.content)) {
output = (block.content as Array<Record<string, unknown>>)
.filter((c) => c.type === 'text')
.map((c) => c.text as string)
.join('\n');
}
onEvent({
type: 'tool:result',
toolCallId: (block.tool_use_id as string) ?? '',
output,
isError: (block.is_error as boolean) ?? false,
});
}
}
}
} else if (type === 'system' && msg.subtype === 'init') {
const sessionId = msg.session_id as string | undefined;
if (sessionId) {
setClaudeSession(sessionKey, sessionId);
}
} else if (type === 'result') {
gotResult = true;
clearTimeout(timeout);
const isError = (msg.is_error as boolean) ?? false;
const resultText = (msg.result as string) ?? '';
if (isError) {
if (textBuffer) {
onEvent({ type: 'text', text: textBuffer });
textBuffer = '';
}
onEvent({ type: 'error', message: resultText || 'Claude Code returned an error' });
return;
}
if (textBuffer) {
onEvent({ type: 'text', text: textBuffer });
textBuffer = '';
}
const usage = msg.usage as Record<string, number> | undefined;
const cost: MessageCost = {
inputTokens: usage?.input_tokens ?? 0,
outputTokens: usage?.output_tokens ?? 0,
totalUSD: (msg.total_cost_usd as number) ?? 0,
};
const sessionId = msg.session_id as string | undefined;
if (sessionId) {
setClaudeSession(sessionKey, sessionId);
}
onEvent({ type: 'result', cost });
}
} catch {
// Skip malformed JSON lines
}
};
while (true) {
const { done, value } = await reader.read();
if (done) break;
const chunk = decoder.decode(value, { stream: true });
buffer += chunk;
const lines = buffer.split('\n');
buffer = lines.pop()!;
for (const line of lines) {
processLine(line);
}
}
if (buffer.trim()) {
processLine(buffer);
}
clearTimeout(timeout);
if (!gotResult) {
const exitCode = await proc.exited;
const stderr = await new Response(proc.stderr as ReadableStream<Uint8Array>).text();
if (textBuffer) {
onEvent({ type: 'text', text: textBuffer });
}
if (exitCode !== 0) {
onEvent({ type: 'error', message: `Claude Code exited with code ${exitCode}: ${stderr.trim().slice(0, 200)}` });
} else {
onEvent({ type: 'result', cost: { inputTokens: 0, outputTokens: 0, totalUSD: 0 } });
}
}
} catch (err) {
clearTimeout(timeout);
onEvent({ type: 'error', message: String(err) });
} finally {
activeProcs.delete(sessionKey);
}
}
export function killClaudeSession(sessionKey: string): boolean {
const proc = activeProcs.get(sessionKey);
if (proc) {
try {
proc.kill();
} catch {
/* already dead */
}
activeProcs.delete(sessionKey);
return true;
}
return false;
}
export function clearSession(sessionKey: string): void {
clearClaudeSession(sessionKey);
}
export function getActiveSessionKeys(): string[] {
return Array.from(activeProcs.keys());
}
+119
View File
@@ -0,0 +1,119 @@
import type { SidecarCommand, SidecarEvent } from '../protocol';
import { loadState, flushAndSave, acquireLock, releaseLock, getState } from './state';
import { startAnthropicProxy, getProxySecret, ensureProxySecret } from './proxy';
import * as claudeManager from './claude-manager';
import { createSidecarConnector } from '../connect';
const API_URL = process.env.API_URL ?? `ws://127.0.0.1:${process.env.PORT ?? '5000'}`;
// ── Startup ──
if (!acquireLock()) {
console.error('[claude] another instance is already running (lock file exists with live PID)');
process.exit(1);
}
loadState();
ensureProxySecret();
// Start Anthropic proxy
try {
startAnthropicProxy();
} catch (err) {
console.error('[claude] failed to start Anthropic proxy:', err instanceof Error ? err.message : err);
}
// ── Command handlers ──
type ReplyFn = (msg: SidecarEvent) => void;
async function handleCommand(cmd: SidecarCommand, reply: ReplyFn) {
switch (cmd.type) {
case 'ping':
reply({ type: 'pong', id: cmd.id });
break;
case 'state:sync':
reply({
type: 'state:sync',
id: cmd.id,
state: {
proxySecret: getProxySecret(),
claudeSessions: { ...getState().claudeSessions },
},
});
break;
case 'proxy:secret':
reply({ type: 'proxy:secret', id: cmd.id, secret: getProxySecret() });
break;
case 'claude:spawn': {
try {
const result = await claudeManager.spawnClaude(cmd.params);
reply({ type: 'claude:result', id: cmd.id, result });
} catch (err) {
reply({ type: 'claude:error', id: cmd.id, error: err instanceof Error ? err.message : String(err) });
}
break;
}
case 'claude:spawn-streaming': {
reply({ type: 'claude:spawned', id: cmd.id, sessionKey: cmd.params.sessionKey });
const onEvent = (event: import('../../api/pi/types').PiEvent) => {
connection.send({ type: 'claude:event', sessionKey: cmd.params.sessionKey, event });
};
claudeManager.spawnClaudeStreaming(cmd.params, onEvent).catch((err) => {
connection.send({
type: 'claude:event',
sessionKey: cmd.params.sessionKey,
event: { type: 'error', message: err instanceof Error ? err.message : String(err) },
});
});
break;
}
case 'claude:kill':
claudeManager.killClaudeSession(cmd.sessionKey);
reply({ type: 'claude:killed', id: cmd.id });
break;
case 'claude:clear-session':
claudeManager.clearSession(cmd.sessionKey);
reply({ type: 'claude:session-cleared', id: cmd.id });
break;
default:
reply({
type: 'error',
id: (cmd as SidecarCommand).id,
error: `Unknown command type: ${(cmd as Record<string, unknown>).type}`,
});
}
}
// ── Connect to API server ──
const connection = createSidecarConnector({
apiUrl: `${API_URL}/api/sidecar/register`,
name: 'claude',
capabilities: ['claude', 'proxy'],
onCommand(cmd, reply) {
handleCommand(cmd as SidecarCommand, reply as ReplyFn);
},
});
// ── Graceful shutdown ──
async function shutdown(signal: string) {
console.log(`[claude] ${signal} received, saving state...`);
connection.destroy();
await flushAndSave();
releaseLock();
process.exit(0);
}
process.on('SIGTERM', () => shutdown('SIGTERM'));
process.on('SIGINT', () => shutdown('SIGINT'));
+104
View File
@@ -0,0 +1,104 @@
import { join } from 'node:path';
import { homedir } from 'node:os';
import { getState, updateState } from './state';
const PROXY_PORT = Number(process.env.ANTHROPIC_PROXY_PORT ?? '5051');
const ANTHROPIC_API_BASE = 'https://api.anthropic.com';
const CREDENTIALS_PATH = join(homedir(), '.claude', '.credentials.json');
type CredentialsFile = {
claudeAiOauth?: {
accessToken?: string;
};
};
async function readOAuthToken(): Promise<string | null> {
try {
const file = Bun.file(CREDENTIALS_PATH);
if (!(await file.exists())) return null;
const data = (await file.json()) as CredentialsFile;
return data.claudeAiOauth?.accessToken?.trim() || null;
} catch {
return null;
}
}
export function getProxySecret(): string {
return getState().proxySecret;
}
export function ensureProxySecret(): string {
const state = getState();
if (state.proxySecret) return state.proxySecret;
// Generate once, persist forever
const secret = `sk-ant-api03-${crypto.randomUUID()}`;
updateState({ proxySecret: secret });
return secret;
}
export function startAnthropicProxy() {
const secret = ensureProxySecret();
Bun.serve({
port: PROXY_PORT,
hostname: '127.0.0.1',
idleTimeout: 0,
async fetch(req) {
const url = new URL(req.url);
// Validate proxy secret
const incomingKey = req.headers.get('x-api-key');
if (incomingKey !== secret) {
return new Response(JSON.stringify({ error: 'Unauthorized' }), {
status: 401,
headers: { 'Content-Type': 'application/json' },
});
}
// Read fresh OAuth token
const token = await readOAuthToken();
if (!token) {
return new Response(JSON.stringify({ error: 'No OAuth token available' }), {
status: 502,
headers: { 'Content-Type': 'application/json' },
});
}
// Build upstream URL
const upstream = `${ANTHROPIC_API_BASE}${url.pathname}${url.search}`;
// Clone headers, replace proxy secret with real OAuth token
const headers = new Headers(req.headers);
headers.set('x-api-key', token);
headers.delete('host');
// Read request body fully before forwarding
const body = req.body ? await req.arrayBuffer() : null;
// Forward request
const upstreamRes = await fetch(upstream, {
method: req.method,
headers,
body,
});
// Build clean response headers
const resHeaders = new Headers();
for (const [key, value] of upstreamRes.headers) {
const lower = key.toLowerCase();
if (lower === 'content-encoding' || lower === 'content-length' || lower === 'transfer-encoding') continue;
resHeaders.set(key, value);
}
return new Response(upstreamRes.body, {
status: upstreamRes.status,
statusText: upstreamRes.statusText,
headers: resHeaders,
});
},
});
console.log(`[claude:proxy] listening on 127.0.0.1:${PROXY_PORT}`);
}
+124
View File
@@ -0,0 +1,124 @@
import { join } from 'node:path';
import { mkdirSync, existsSync } from 'node:fs';
const DATA_PATH = process.env.DATA_PATH ?? join(process.cwd(), 'data');
const STATE_DIR = join(DATA_PATH, 'sidecar');
const STATE_FILE = join(STATE_DIR, 'claude-state.json');
const LOCK_FILE = join(STATE_DIR, 'claude.lock');
export type PersistedState = {
proxySecret: string;
claudeSessions: Record<string, string>; // sessionKey → Claude Code session_id
};
const DEFAULT_STATE: PersistedState = {
proxySecret: '',
claudeSessions: {},
};
let currentState: PersistedState = { ...DEFAULT_STATE };
let saveTimer: Timer | null = null;
function ensureDir() {
if (!existsSync(STATE_DIR)) {
mkdirSync(STATE_DIR, { recursive: true });
}
}
export function loadState(): PersistedState {
ensureDir();
try {
if (!existsSync(STATE_FILE)) {
currentState = { ...DEFAULT_STATE };
return currentState;
}
const text = require('node:fs').readFileSync(STATE_FILE, 'utf-8');
currentState = { ...DEFAULT_STATE, ...JSON.parse(text) };
return currentState;
} catch {
currentState = { ...DEFAULT_STATE };
return currentState;
}
}
export async function saveState(): Promise<void> {
ensureDir();
await Bun.write(STATE_FILE, JSON.stringify(currentState, null, 2));
}
export function getState(): PersistedState {
return currentState;
}
export function updateState(patch: Partial<PersistedState>): void {
Object.assign(currentState, patch);
scheduleSave();
}
export function setClaudeSession(sessionKey: string, sessionId: string): void {
currentState.claudeSessions[sessionKey] = sessionId;
scheduleSave();
}
export function clearClaudeSession(sessionKey: string): void {
delete currentState.claudeSessions[sessionKey];
scheduleSave();
}
export function getClaudeSession(sessionKey: string): string | undefined {
return currentState.claudeSessions[sessionKey];
}
function scheduleSave() {
if (saveTimer) return;
saveTimer = setTimeout(async () => {
saveTimer = null;
await saveState();
}, 30_000);
}
export async function flushAndSave(): Promise<void> {
if (saveTimer) {
clearTimeout(saveTimer);
saveTimer = null;
}
await saveState();
}
// ── Lockfile ──
export function acquireLock(): boolean {
ensureDir();
try {
if (existsSync(LOCK_FILE)) {
const pidStr = require('node:fs').readFileSync(LOCK_FILE, 'utf-8').trim();
const pid = Number(pidStr);
if (pid && isProcessAlive(pid)) {
return false;
}
}
require('node:fs').writeFileSync(LOCK_FILE, String(process.pid));
return true;
} catch {
return false;
}
}
export function releaseLock(): void {
try {
if (existsSync(LOCK_FILE)) {
require('node:fs').unlinkSync(LOCK_FILE);
}
} catch {
// best effort
}
}
function isProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}