officerdev/platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pipeline executor improvements, proxy refresh, task runner step list

Browse Source 
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
pastilhas
2026-03-12 07:25:11 +00:00
co-authored by Claude Opus 4.6
parent 0f1e1f35f1
commit 92da03fcff
6 changed files with 280 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/servers/api/tasks/pipeline-executor.ts
+46 -69
View File
@@ -31,7 +31,7 @@ type PipelineStep = {
inputs?: Record<string, string>;
foreach?: 'subdirectory';
skip_if?: string;
concurrency?: string;
concurrency?: string | boolean;
};
type PipelineConfig = {
@@ -70,17 +70,29 @@ type RunStepParams = {
taskDirName: string;
prompt: string;
cwd: string;
model: string;
abortSignal: AbortSignal;
emit: EmitEvent;
stepIndex: number;
iterationLabel?: string;
};
async function runAgenticStep({ userId, email, username, role, taskDirName, prompt, cwd, abortSignal, emit, stepIndex, iterationLabel }: RunStepParams): Promise<MessageCost> {
async function refreshProxyToken(): Promise<void> {
const port = process.env.ANTHROPIC_PROXY_PORT ?? '5051';
try {
await fetch(`http://127.0.0.1:${port}/refresh`, { method: 'POST' });
} catch {
// Best effort — proxy may not be running (e.g. using API key directly)
}
}
async function runAgenticStep({ userId, email, username, role, taskDirName, prompt, cwd, model, abortSignal, emit, stepIndex, iterationLabel }: RunStepParams): Promise<MessageCost> {
const sessionId = randomUUID();
const model = await resolveModel(userId);
const isClaudeCode = model.startsWith('claude-code');
// Ensure fresh OAuth token before spawning Claude Code
if (isClaudeCode) await refreshProxyToken();
console.log(`[pipeline] starting step ${stepIndex} for session ${sessionId} (model=${model})${iterationLabel ? ` [${iterationLabel}]` : ''}`);
return new Promise<MessageCost>(async (resolve, reject) => {
@@ -312,7 +324,7 @@ async function runScriptStep({ email, role, task, inputs, cwd, abortSignal, emit
// ── Parallel foreach ──
type ParallelForeachParams = {
type ForeachParams = {
userId: number;
email: string;
username: string;
@@ -327,13 +339,14 @@ type ParallelForeachParams = {
abortSignal: AbortSignal;
totalCost: MessageCost;
emit: EmitEvent;
concurrency: number;
model: string;
};
async function runParallelForeach({
async function runForeach({
userId, email, username, role, stepIdx, step, stepTask, subdirs, baseCwd,
inputs, cwd, abortSignal, totalCost, emit,
}: ParallelForeachParams) {
const concurrency = Math.max(1, parseInt(step.concurrency ?? '1', 10));
inputs, cwd, abortSignal, totalCost, emit, concurrency, model,
}: ForeachParams) {
// Determine skip vs run
const toSkip: string[] = [];
@@ -389,6 +402,7 @@ async function runParallelForeach({
taskDirName: step.task,
prompt,
cwd: resolvedCwd,
model,
abortSignal,
emit,
stepIndex: stepIdx,
@@ -428,11 +442,13 @@ export type ExecutePipelineParams = {
taskDirName: string;
inputs: Record<string, string>;
cwd?: string;
model?: string;
startAt?: number;
abortSignal: AbortSignal;
emit: EmitEvent;
};
export async function executePipeline({ userId, email, username, role, taskDirName, inputs, cwd, abortSignal, emit }: ExecutePipelineParams): Promise<void> {
export async function executePipeline({ userId, email, username, role, taskDirName, inputs, cwd, model: modelOverride, startAt, abortSignal, emit }: ExecutePipelineParams): Promise<void> {
const pipelineTask = await getTaskByDirName(taskDirName, userId);
if (!pipelineTask) {
emit({ type: 'error', message: `Task not found: ${taskDirName}` });
@@ -450,20 +466,30 @@ export async function executePipeline({ userId, email, username, role, taskDirNa
}
const baseCwd = resolveBaseCwd(email, role, cwd);
const model = modelOverride || (await resolveModel(userId));
const totalCost: MessageCost = { inputTokens: 0, outputTokens: 0, totalUSD: 0 };
// Resolve concurrency from user input (default 1)
const runtimeConcurrency = Math.max(1, parseInt(inputs._concurrency ?? '1', 10));
emit({
type: 'pipeline:init',
steps: config.steps.map((s) => ({
task: s.task,
foreach: s.foreach,
concurrency: s.concurrency ? parseInt(s.concurrency, 10) : undefined,
concurrency: s.concurrency ? runtimeConcurrency : undefined,
})),
});
for (let stepIdx = 0; stepIdx < config.steps.length; stepIdx++) {
if (abortSignal.aborted) break;
// Skip steps before startAt
if (startAt && stepIdx < startAt) {
emit({ type: 'step:skip', stepIndex: stepIdx, label: config.steps[stepIdx]!.task, reason: 'skipped (start at)' });
continue;
}
const step = config.steps[stepIdx]!;
const stepTask = await getTaskByDirName(step.task, userId);
@@ -527,65 +553,13 @@ export async function executePipeline({ userId, email, username, role, taskDirNa
return;
}
const concurrency = parseInt(step.concurrency ?? '1', 10);
const concurrency = step.concurrency ? runtimeConcurrency : 1;
if (concurrency > 1) {
await runParallelForeach({
userId, email, username, role,
stepIdx, step, stepTask: { name: stepTask.name, body: stepTask.body! },
subdirs, baseCwd, inputs, cwd, abortSignal, totalCost, emit,
});
} else {
// Sequential
for (let i = 0; i < subdirs.length; i++) {
if (abortSignal.aborted) break;
const subdir = subdirs[i]!;
const subdirPath = join(baseCwd, subdir);
if (step.skip_if && existsSync(join(subdirPath, step.skip_if))) {
emit({ type: 'step:skip', stepIndex: stepIdx, label: subdir, reason: `${step.skip_if} already exists` });
continue;
}
const iterVars = { ...inputs, folder_name: subdir };
const iterInputs: Record<string, string> = {};
if (step.inputs) {
for (const [key, template] of Object.entries(step.inputs)) {
iterInputs[key] = resolveInputTemplate(template, iterVars);
}
}
emit({
type: 'step:start',
stepIndex: stepIdx,
taskName: stepTask.name,
iteration: { current: i + 1, total: subdirs.length, label: subdir },
});
const cwdRelative = cwd ? `${cwd}/${subdir}` : subdir;
const resolvedCwd = resolveBaseCwd(email, role, cwdRelative);
const targetDir = toAgentPath(resolvedCwd, email, role);
const prompt = buildStepPrompt(stepTask.body!,iterInputs, targetDir);
const cost = await runAgenticStep({
userId, email, username, role,
taskDirName: step.task,
prompt,
cwd: resolvedCwd,
abortSignal,
emit,
stepIndex: stepIdx,
iterationLabel: subdir,
});
totalCost.inputTokens += cost.inputTokens;
totalCost.outputTokens += cost.outputTokens;
totalCost.totalUSD += cost.totalUSD;
emit({ type: 'step:complete', stepIndex: stepIdx, cost });
}
}
await runForeach({
userId, email, username, role,
stepIdx, step, stepTask: { name: stepTask.name, body: stepTask.body! },
subdirs, baseCwd, inputs, cwd, abortSignal, totalCost, emit, concurrency, model,
});
} else {
// Single execution step
emit({ type: 'step:start', stepIndex: stepIdx, taskName: stepTask.name });
@@ -598,6 +572,7 @@ export async function executePipeline({ userId, email, username, role, taskDirNa
taskDirName: step.task,
prompt,
cwd: baseCwd,
model,
abortSignal,
emit,
stepIndex: stepIdx,
@@ -629,7 +604,7 @@ type WSData = {
};
type ClientMessage =
| { type: 'run'; taskDirName: string; inputs: Record<string, string>; cwd?: string }
| { type: 'run'; taskDirName: string; inputs: Record<string, string>; cwd?: string; model?: string; startAt?: number }
| { type: 'stop'; jobId: string }
| { type: 'attach'; jobId: string }
| { type: 'list' };
@@ -667,6 +642,8 @@ export async function message(ws: ServerWebSocket<WSData>, raw: string | Buffer)
inputs: msg.inputs,
cwd: msg.cwd,
config: task.config,
model: msg.model,
startAt: msg.startAt,
});
send(ws, { type: 'job:created', jobId });
src/servers/api/tasks/pipeline-job-manager.ts
+4
View File
@@ -67,6 +67,8 @@ type StartJobParams = {
inputs: Record<string, string>;
cwd?: string;
config: unknown;
model?: string;
startAt?: number;
};
export async function startJob(params: StartJobParams): Promise<string> {
@@ -146,6 +148,8 @@ export async function startJob(params: StartJobParams): Promise<string> {
taskDirName: params.taskDirName,
inputs: params.inputs,
cwd: params.cwd,
model: params.model,
startAt: params.startAt,
abortSignal: job.abortSignal,
emit,
}).then(async () => {
src/servers/sidecar/claude/claude-manager.ts
+1 -1
View File
@@ -6,7 +6,7 @@ import { buildSandboxPrefix, buildRunuserSuffix, SANDBOX_HOME } from '../sandbox
import { setClaudeSession, clearClaudeSession, getClaudeSession } from './state';
import { parseStream } from './stream-parser';
const SEND_TIMEOUT_MS = 5 * 60 * 1000;
const SEND_TIMEOUT_MS = 30 * 60 * 1000;
// Use /usr/local/bin/claude so it's visible inside bwrap sandbox (which ro-binds /usr).
// The actual binary lives at ~/.local/bin/claude, symlinked from /usr/local/bin/claude.
src/servers/sidecar/claude/proxy.ts
+152 -9
View File
@@ -5,24 +5,119 @@ import { getState, updateState } from './state';
const PROXY_PORT = Number(process.env.ANTHROPIC_PROXY_PORT ?? '5051');
const ANTHROPIC_API_BASE = 'https://api.anthropic.com';
const CREDENTIALS_PATH = join(homedir(), '.claude', '.credentials.json');
const TOKEN_URL = 'https://platform.claude.com/v1/oauth/token';
const CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
type CredentialsFile = {
claudeAiOauth?: {
accessToken?: string;
};
// Buffer: refresh 5 minutes before expiry
const EXPIRY_BUFFER_MS = 5 * 60 * 1000;
type OAuthCredentials = {
accessToken?: string;
refreshToken?: string;
expiresAt?: number;
};
async function readOAuthToken(): Promise<string | null> {
type CredentialsFile = {
claudeAiOauth?: OAuthCredentials;
};
async function readCredentials(): Promise<CredentialsFile | null> {
try {
const file = Bun.file(CREDENTIALS_PATH);
if (!(await file.exists())) return null;
const data = (await file.json()) as CredentialsFile;
return data.claudeAiOauth?.accessToken?.trim() || null;
return (await file.json()) as CredentialsFile;
} catch {
return null;
}
}
async function writeCredentials(creds: CredentialsFile): Promise<void> {
try {
await Bun.write(CREDENTIALS_PATH, JSON.stringify(creds, null, 2));
} catch (err) {
console.error('[claude:proxy] failed to write credentials:', err);
}
}
async function refreshOAuthToken(refreshToken: string): Promise<OAuthCredentials | null> {
try {
console.log('[claude:proxy] refreshing OAuth token...');
const res = await fetch(TOKEN_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({
grant_type: 'refresh_token',
refresh_token: refreshToken,
client_id: CLIENT_ID,
}),
});
if (!res.ok) {
const text = await res.text();
console.error(`[claude:proxy] token refresh failed (${res.status}): ${text}`);
return null;
}
const data = (await res.json()) as { access_token?: string; refresh_token?: string; expires_in?: number };
if (!data.access_token) {
console.error('[claude:proxy] token refresh response missing access_token');
return null;
}
const expiresAt = data.expires_in ? Date.now() + data.expires_in * 1000 : undefined;
console.log(`[claude:proxy] token refreshed, expires in ${data.expires_in ?? '?'}s`);
return {
accessToken: data.access_token,
refreshToken: data.refresh_token ?? refreshToken,
expiresAt,
};
} catch (err) {
console.error('[claude:proxy] token refresh error:', err);
return null;
}
}
async function getValidToken(): Promise<string | null> {
const creds = await readCredentials();
if (!creds?.claudeAiOauth) return null;
const oauth = creds.claudeAiOauth;
const now = Date.now();
// Check if token is still valid (with buffer)
if (oauth.accessToken && oauth.expiresAt && oauth.expiresAt - now > EXPIRY_BUFFER_MS) {
return oauth.accessToken.trim();
}
// Token expired or about to expire — try refresh
if (oauth.refreshToken) {
const refreshed = await refreshOAuthToken(oauth.refreshToken);
if (refreshed?.accessToken) {
creds.claudeAiOauth = { ...oauth, ...refreshed };
await writeCredentials(creds);
return refreshed.accessToken.trim();
}
}
// Fallback: return current token even if potentially expired (let upstream decide)
return oauth.accessToken?.trim() || null;
}
async function forceRefresh(): Promise<string | null> {
const creds = await readCredentials();
const refreshToken = creds?.claudeAiOauth?.refreshToken;
if (!refreshToken) return null;
const refreshed = await refreshOAuthToken(refreshToken);
if (refreshed?.accessToken && creds?.claudeAiOauth) {
creds.claudeAiOauth = { ...creds.claudeAiOauth, ...refreshed };
await writeCredentials(creds);
return refreshed.accessToken;
}
return null;
}
export function getProxySecret(): string {
return getState().proxySecret;
}
@@ -48,6 +143,18 @@ export function startAnthropicProxy() {
async fetch(req) {
const url = new URL(req.url);
// Internal refresh endpoint (no auth needed — localhost only)
if (url.pathname === '/refresh') {
const token = await forceRefresh();
if (token) {
return new Response(JSON.stringify({ ok: true }), { headers: { 'Content-Type': 'application/json' } });
}
return new Response(JSON.stringify({ error: 'Refresh failed' }), {
status: 502,
headers: { 'Content-Type': 'application/json' },
});
}
// Validate proxy secret
const incomingKey = req.headers.get('x-api-key');
if (incomingKey !== secret) {
@@ -57,8 +164,8 @@ export function startAnthropicProxy() {
});
}
// Read fresh OAuth token
const token = await readOAuthToken();
// Get valid token (refreshes if needed)
const token = await getValidToken();
if (!token) {
return new Response(JSON.stringify({ error: 'No OAuth token available' }), {
status: 502,
@@ -84,6 +191,42 @@ export function startAnthropicProxy() {
body,
});
// If upstream returns 401, try one refresh and retry
if (upstreamRes.status === 401) {
const creds = await readCredentials();
const refreshToken = creds?.claudeAiOauth?.refreshToken;
if (refreshToken) {
const refreshed = await refreshOAuthToken(refreshToken);
if (refreshed?.accessToken) {
if (creds?.claudeAiOauth) {
creds.claudeAiOauth = { ...creds.claudeAiOauth, ...refreshed };
await writeCredentials(creds);
}
headers.set('x-api-key', refreshed.accessToken.trim());
const retryBody = body ? new Uint8Array(body) : null;
const retryRes = await fetch(upstream, {
method: req.method,
headers,
body: retryBody,
});
const retryHeaders = new Headers();
for (const [key, value] of retryRes.headers) {
const lower = key.toLowerCase();
if (lower === 'content-encoding' || lower === 'content-length' || lower === 'transfer-encoding') continue;
retryHeaders.set(key, value);
}
return new Response(retryRes.body, {
status: retryRes.status,
statusText: retryRes.statusText,
headers: retryHeaders,
});
}
}
}
// Build clean response headers
const resHeaders = new Headers();
for (const [key, value] of upstreamRes.headers) {
src/workspaces/officerdev/src/apps/FileBrowser/FileBrowserApp/components/TaskRunnerModal.tsx
+75 -5
View File
@@ -491,11 +491,18 @@ const PipelineRunner = ({ taskDirName, context, cwd }: PipelineRunnerProps) => {
const bottomRef = useRef<HTMLDivElement | null>(null);
const [inputDefs, setInputDefs] = useState<Record<string, TaskInputDef> | null>(null);
const [formValues, setFormValues] = useState<Record<string, string>>({});
const [hasConcurrentSteps, setHasConcurrentSteps] = useState(false);
const [concurrency, setConcurrency] = useState('1');
const [pipelineSteps, setPipelineSteps] = useState<Array<{ task: string; foreach?: string }>>([]);
const [startAt, setStartAt] = useState(0);
const { settings } = useSettings();
const availableModels = useUserVisibleModels();
const [selectedModel, setSelectedModel] = useState(settings.tasks.defaultModel ?? 'claude-haiku-4-5-20251001');
// Fetch task detail for inputs
// Fetch task detail for inputs + check for concurrent steps
useEffect(() => {
client.get<{ inputs?: Record<string, TaskInputDef> }>(`/tasks/${taskDirName}`).then((task) => {
const defs = task.inputs ?? {};
client.get<{ inputs?: Record<string, TaskInputDef>; config?: { steps?: Array<{ task: string; foreach?: string; concurrency?: string | boolean }> } }>(`/tasks/${taskDirName}`).then((task) => {
const defs: Record<string, TaskInputDef> = task.inputs ?? {};
setInputDefs(defs);
const initial: Record<string, string> = {};
for (const [key, def] of Object.entries(defs)) {
@@ -503,6 +510,10 @@ const PipelineRunner = ({ taskDirName, context, cwd }: PipelineRunnerProps) => {
else if (def.default !== undefined) initial[key] = def.default;
}
setFormValues(initial);
const steps = task.config?.steps ?? [];
setPipelineSteps(steps.map((s) => ({ task: s.task, foreach: s.foreach })));
setHasConcurrentSteps(steps.some((s: { concurrency?: string | boolean }) => s.concurrency));
});
}, [taskDirName]);
@@ -525,7 +536,11 @@ const PipelineRunner = ({ taskDirName, context, cwd }: PipelineRunnerProps) => {
}, [pipeline.phase]);
const handleRun = () => {
pipeline.run(taskDirName, formValues, cwd);
const allInputs = { ...formValues };
if (hasConcurrentSteps && parseInt(concurrency, 10) > 1) {
allInputs._concurrency = concurrency;
}
pipeline.run(taskDirName, allInputs, cwd, selectedModel, startAt > 0 ? startAt : undefined);
};
const hasConfigurableInputs = inputDefs && Object.keys(inputDefs).length > 0;
@@ -541,6 +556,61 @@ const PipelineRunner = ({ taskDirName, context, cwd }: PipelineRunnerProps) => {
autoFilledKeys={new Set()}
/>
)}
{hasConcurrentSteps && (
<div className="px-5 py-3 border-b border-duck-dark/10 flex items-center justify-between gap-4">
<div className="min-w-0">
<span className="text-sm font-medium text-duck-dark dark:text-foreground">Concurrency</span>
<span className="text-xs text-duck-dark/50 dark:text-foreground/50 ml-2">parallel runners per step</span>
</div>
<input
type="number"
min={1}
max={20}
value={concurrency}
onChange={(e) => setConcurrency(e.target.value)}
className="w-16 px-2 py-1 text-sm text-center rounded-lg border border-duck-dark/15 dark:border-foreground/15 bg-background focus:outline-none focus:ring-1 focus:ring-duck-teal"
/>
</div>
)}
<div className="px-5 py-3 border-b border-duck-dark/10 flex items-center justify-between gap-4">
<span className="text-sm font-medium text-duck-dark dark:text-foreground">Model</span>
<select
value={selectedModel}
onChange={(ev) => setSelectedModel(ev.target.value)}
className="px-2 py-1 text-sm rounded-lg border border-duck-dark/15 dark:border-foreground/15 bg-background focus:outline-none focus:ring-1 focus:ring-duck-teal"
>
{availableModels.map((m) => (
<option key={`${m.provider}:${m.id}`} value={m.id}>
{m.name}
</option>
))}
</select>
</div>
{pipelineSteps.length > 0 && (
<div className="px-5 py-3 border-b border-duck-dark/10">
<span className="text-sm font-medium text-duck-dark dark:text-foreground mb-2 block">Steps</span>
<div className="flex flex-col gap-0.5">
{pipelineSteps.map((step, i) => (
<button
key={i}
onClick={() => setStartAt(i)}
className={`text-left text-xs px-2.5 py-1.5 rounded-md transition-colors cursor-pointer flex items-center gap-2 ${
i < startAt
? 'text-duck-dark/30 dark:text-foreground/30 line-through'
: i === startAt
? 'bg-duck-teal/10 text-duck-teal font-medium'
: 'text-duck-dark/70 dark:text-foreground/70 hover:bg-duck-dark/5'
}`}
>
<span className="w-4 text-center font-mono text-[10px]">{i + 1}</span>
<span>{step.task}</span>
{step.foreach && <span className="text-duck-dark/40 dark:text-foreground/40">(foreach)</span>}
{i === startAt && i > 0 && <span className="ml-auto text-[10px] text-duck-teal/70">start here</span>}
</button>
))}
</div>
</div>
)}
<div className="flex-1 flex items-center justify-center">
<button
onClick={handleRun}
@@ -548,7 +618,7 @@ const PipelineRunner = ({ taskDirName, context, cwd }: PipelineRunnerProps) => {
className="flex items-center gap-2 px-6 py-2.5 rounded-lg bg-duck-teal text-white font-medium text-sm hover:bg-duck-teal/90 transition-colors disabled:opacity-50 disabled:cursor-not-allowed cursor-pointer"
>
<Play className="h-4 w-4" />
Run Pipeline
{startAt > 0 ? `Run from step ${startAt + 1}` : 'Run Pipeline'}
</button>
</div>
</div>
src/workspaces/officerdev/src/apps/FileBrowser/FileBrowserApp/components/usePipelineRunner.ts
+2 -2
View File
@@ -290,7 +290,7 @@ export function usePipelineRunner() {
};
}, []);
const run = useCallback((taskDirName: string, inputs: Record<string, string>, cwd?: string) => {
const run = useCallback((taskDirName: string, inputs: Record<string, string>, cwd?: string, model?: string, startAt?: number) => {
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return;
setPhase('running');
@@ -313,7 +313,7 @@ export function usePipelineRunner() {
setElapsed(Math.floor((Date.now() - startTimeRef.current) / 1000));
}, 1000);
wsRef.current.send(JSON.stringify({ type: 'run', taskDirName, inputs, cwd }));
wsRef.current.send(JSON.stringify({ type: 'run', taskDirName, inputs, cwd, model, startAt }));
}, []);
const stop = useCallback(() => {