From 671dcd66b39ddfb8ba690e0d8571ca140efbcc75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20F=C3=B6rster?= <103369858+sfo2001@users.noreply.github.com> Date: Sun, 15 Feb 2026 11:29:04 +0100 Subject: [PATCH 1/4] fix(stdlib): remove unused Transport type and transport variable Dead code left over from the clawd-only refactor in 5679042. Co-Authored-By: Claude Opus 4.6 --- src/commands/stdlib/llm_task_invoke.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/commands/stdlib/llm_task_invoke.ts b/src/commands/stdlib/llm_task_invoke.ts index 86ece36..700b48a 100644 --- a/src/commands/stdlib/llm_task_invoke.ts +++ b/src/commands/stdlib/llm_task_invoke.ts @@ -146,8 +146,6 @@ type CacheEntry = { storedAt: string; }; -type Transport = 'clawd'; - export const llmTaskInvokeCommand = { name: 'llm_task.invoke', meta: { @@ -198,7 +196,6 @@ export const llmTaskInvokeCommand = { const env = ctx.env ?? process.env; const clawdUrl = String(env.CLAWD_URL ?? '').trim(); - const transport: Transport = 'clawd'; if (!clawdUrl) { throw new Error('llm_task.invoke requires CLAWD_URL (run via Clawdbot gateway)'); } From 5d78f6c3165af4a152e5f388461642fa022fd248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20F=C3=B6rster?= <103369858+sfo2001@users.noreply.github.com> Date: Sun, 15 Feb 2026 11:24:46 +0100 Subject: [PATCH 2/4] refactor(stdlib): extract shared template_utils from map and template Extract getByPath and renderTemplate into template_utils.ts. Fix ReDoS vulnerability in template regex and add Object.hasOwn guard to block prototype chain traversal in getByPath. Co-Authored-By: Claude Opus 4.6 --- src/commands/stdlib/map.ts | 21 +-------------------- src/commands/stdlib/template.ts | 22 +--------------------- src/commands/stdlib/template_utils.ts | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 41 deletions(-) create mode 100644 src/commands/stdlib/template_utils.ts diff --git a/src/commands/stdlib/map.ts b/src/commands/stdlib/map.ts index e47332c..9e23733 100644 --- a/src/commands/stdlib/map.ts +++ b/src/commands/stdlib/map.ts @@ -1,23 +1,4 @@ -function getByPath(obj: any, path: string): any { - if (path === '.' || path === 'this') return obj; - const parts = path.split('.').filter(Boolean); - let cur: any = obj; - for (const p of parts) { - if (cur == null) return undefined; - cur = cur[p]; - } - return cur; -} - -function renderTemplate(tpl: string, ctx: any): string { - return tpl.replace(/\{\{\s*([^}]+?)\s*\}\}/g, (_m, expr) => { - const key = String(expr ?? '').trim(); - const val = getByPath(ctx, key); - if (val === undefined || val === null) return ''; - if (typeof val === 'string') return val; - return JSON.stringify(val); - }); -} +import { renderTemplate } from './template_utils.js'; function parseAssignments(tokens: any[]): Array<{ key: string; value: string }> { const out: Array<{ key: string; value: string }> = []; diff --git a/src/commands/stdlib/template.ts b/src/commands/stdlib/template.ts index f857a95..ecfbc50 100644 --- a/src/commands/stdlib/template.ts +++ b/src/commands/stdlib/template.ts @@ -1,25 +1,5 @@ import fs from 'node:fs/promises'; - -function getByPath(obj: any, path: string): any { - if (path === '.' || path === 'this') return obj; - const parts = path.split('.').filter(Boolean); - let cur: any = obj; - for (const p of parts) { - if (cur == null) return undefined; - cur = cur[p]; - } - return cur; -} - -function renderTemplate(tpl: string, ctx: any): string { - return tpl.replace(/\{\{\s*([^}]+?)\s*\}\}/g, (_m, expr) => { - const key = String(expr ?? '').trim(); - const val = getByPath(ctx, key); - if (val === undefined || val === null) return ''; - if (typeof val === 'string') return val; - return JSON.stringify(val); - }); -} +import { renderTemplate } from './template_utils.js'; export const templateCommand = { name: 'template', diff --git a/src/commands/stdlib/template_utils.ts b/src/commands/stdlib/template_utils.ts new file mode 100644 index 0000000..68c61ab --- /dev/null +++ b/src/commands/stdlib/template_utils.ts @@ -0,0 +1,21 @@ +export function getByPath(obj: any, path: string): any { + if (path === '.' || path === 'this') return obj; + const parts = path.split('.').filter(Boolean); + let cur: any = obj; + for (const p of parts) { + if (cur == null || typeof cur !== 'object') return undefined; + if (!Object.hasOwn(cur, p)) return undefined; + cur = cur[p]; + } + return cur; +} + +export function renderTemplate(tpl: string, ctx: any): string { + return tpl.replace(/\{\{([^}]+)\}\}/g, (_m, expr) => { + const key = String(expr ?? '').trim(); + const val = getByPath(ctx, key); + if (val === undefined || val === null) return ''; + if (typeof val === 'string') return val; + return JSON.stringify(val); + }); +} From ee6b629861b69c60b46587a32d46ddf6b20ddf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20F=C3=B6rster?= <103369858+sfo2001@users.noreply.github.com> Date: Sun, 15 Feb 2026 11:24:59 +0100 Subject: [PATCH 3/4] feat(parser): add brace-delimited body syntax for sub-pipelines Add { } body parsing with backward-compatible bare brace handling, recursion depth limit (50), and truncated error messages. Bare } at depth 0 is treated as literal to avoid breaking existing pipelines. Co-Authored-By: Claude Opus 4.6 --- src/parser.ts | 114 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 107 insertions(+), 7 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 845adfc..88d65eb 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -6,6 +6,7 @@ function splitPipes(input) { const parts = []; let current = ''; let quote = null; + let braceDepth = 0; for (let i = 0; i < input.length; i++) { const ch = input[i]; @@ -32,7 +33,24 @@ function splitPipes(input) { continue; } - if (ch === '|') { + if (ch === '{') { + braceDepth++; + current += ch; + continue; + } + + if (ch === '}') { + if (braceDepth > 0) { + braceDepth--; + current += ch; + continue; + } + // braceDepth === 0: treat as literal character (backward compat) + current += ch; + continue; + } + + if (ch === '|' && braceDepth === 0) { parts.push(current.trim()); current = ''; continue; @@ -95,7 +113,7 @@ function tokenizeCommand(input) { } function parseArgs(tokens) { - const args = { _: [] }; + const args: Record = { _: [] }; for (let i = 0; i < tokens.length; i++) { const tok = tokens[i]; @@ -126,15 +144,97 @@ function parseArgs(tokens) { return args; } -export function parsePipeline(input) { +/** Find the first lone `{` that is not inside quotes or a `{{...}}` template. Returns index or -1. */ +function findUnquotedBrace(text) { + let quote = null; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (quote) { + if (ch === '\\' && text[i + 1]) { i++; continue; } + if (ch === quote) quote = null; + continue; + } + if (ch === '"' || ch === "'") { quote = ch; continue; } + if (ch === '{') { + if (text[i + 1] === '{') { + // Skip past the {{ ... }} template expression + const close = text.indexOf('}}', i + 2); + i = close !== -1 ? close + 1 : i + 1; + continue; + } + return i; + } + } + return -1; +} + +/** Find the `}` matching the `{` at openPos, respecting quotes, nesting, and `{{...}}` templates. */ +function findMatchingBrace(text, openPos) { + let depth = 1; + let quote = null; + for (let i = openPos + 1; i < text.length; i++) { + const ch = text[i]; + if (quote) { + if (ch === '\\' && text[i + 1]) { i++; continue; } + if (ch === quote) quote = null; + continue; + } + if (ch === '"' || ch === "'") { quote = ch; continue; } + if (ch === '{') { + if (text[i + 1] === '{') { + const close = text.indexOf('}}', i + 2); + i = close !== -1 ? close + 1 : i + 1; + continue; + } + depth++; + continue; + } + if (ch === '}') { depth--; if (depth === 0) return i; } + } + return -1; +} + +const MAX_PIPELINE_DEPTH = 50; + +export function parsePipeline(input, _depth = 0) { + if (_depth > MAX_PIPELINE_DEPTH) { + throw new Error(`Pipeline nesting exceeds maximum depth of ${MAX_PIPELINE_DEPTH}`); + } const stages = splitPipes(input); if (stages.length === 0) throw new Error('Empty pipeline'); return stages.map((stage) => { - const tokens = tokenizeCommand(stage); - if (tokens.length === 0) throw new Error('Empty command stage'); - const name = tokens[0]; - const args = parseArgs(tokens.slice(1)); + const braceStart = findUnquotedBrace(stage); + if (braceStart === -1) { + // No brace syntax -- normal parse + const tokens = tokenizeCommand(stage); + if (tokens.length === 0) throw new Error('Empty command stage'); + const name = tokens[0]; + const args = parseArgs(tokens.slice(1)); + return { name, args, raw: stage }; + } + + // Brace syntax: extract prefix, body, suffix + const prefix = stage.slice(0, braceStart); + const braceEnd = findMatchingBrace(stage, braceStart); + if (braceEnd === -1) throw new Error('Unclosed brace'); + + const bodyRaw = stage.slice(braceStart + 1, braceEnd).trim(); + if (!bodyRaw) throw new Error('Empty body in { } block'); + + const suffix = stage.slice(braceEnd + 1).trim(); + const preview = suffix.length > 50 ? suffix.slice(0, 50) + '...' : suffix; + if (suffix) throw new Error(`Unexpected content after closing brace: ${preview}`); + + const prefixTokens = tokenizeCommand(prefix); + if (prefixTokens.length === 0) throw new Error('Empty command before { }'); + const name = prefixTokens[0]; + const args = parseArgs(prefixTokens.slice(1)); + + // Recursively parse the sub-pipeline body + args._body = parsePipeline(bodyRaw, _depth + 1); + args._bodyRaw = bodyRaw; + return { name, args, raw: stage }; }); } From 542e2f694b3471212d68fe506c70a641084986e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20F=C3=B6rster?= <103369858+sfo2001@users.noreply.github.com> Date: Sun, 15 Feb 2026 11:25:16 +0100 Subject: [PATCH 4/4] feat(stdlib): add each command for per-item sub-pipeline iteration Add each { ... } pipeline operator that runs a sub-pipeline per input item with {{.field}} interpolation. Includes _bodyRaw pass-through, sideEffects metadata, and tests for runtime errors, prototype denylist, and recursion depth. Co-Authored-By: Claude Opus 4.6 --- src/cli.ts | 2 +- src/commands/registry.ts | 2 + src/commands/stdlib/each.ts | 85 +++++++++++++++++ test/each.test.ts | 181 ++++++++++++++++++++++++++++++++++++ 4 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 src/commands/stdlib/each.ts create mode 100644 test/each.test.ts diff --git a/src/cli.ts b/src/cli.ts index 3f78b7a..80ff7e5 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -466,5 +466,5 @@ function helpText() { ` lobster 'exec --json "echo [1,2,3]" | json'\n` + ` lobster run --mode tool 'exec --json "echo [1]" | approve --prompt "ok?"'\n\n` + `Commands:\n` + - ` exec, head, json, pick, table, where, approve, clawd.invoke, state.get, state.set, diff.last, commands.list, workflows.list, workflows.run\n`; + ` each, exec, head, json, map, pick, table, template, where, approve, clawd.invoke, state.get, state.set, diff.last, commands.list, workflows.list, workflows.run\n`; } diff --git a/src/commands/registry.ts b/src/commands/registry.ts index 74ef5f5..fe988da 100644 --- a/src/commands/registry.ts +++ b/src/commands/registry.ts @@ -20,6 +20,7 @@ import { commandsListCommand } from "./commands_list.js"; import { gogGmailSearchCommand } from "./stdlib/gog_gmail_search.js"; import { gogGmailSendCommand } from "./stdlib/gog_gmail_send.js"; import { emailTriageCommand } from "./stdlib/email_triage.js"; +import { eachCommand } from "./stdlib/each.js"; export function createDefaultRegistry() { const commands = new Map(); @@ -48,6 +49,7 @@ export function createDefaultRegistry() { gogGmailSearchCommand, gogGmailSendCommand, emailTriageCommand, + eachCommand, ]) { commands.set(cmd.name, cmd); } diff --git a/src/commands/stdlib/each.ts b/src/commands/stdlib/each.ts new file mode 100644 index 0000000..abc7682 --- /dev/null +++ b/src/commands/stdlib/each.ts @@ -0,0 +1,85 @@ +import { runPipeline } from '../../runtime.js'; +import { renderTemplate } from './template_utils.js'; + +function interpolateStages(stages: any[], item: any): any[] { + return stages.map((stage) => { + const args = interpolateArgs(stage.args, item); + return { ...stage, args }; + }); +} + +function interpolateArgs(args: any, item: any): any { + const out: any = {}; + for (const [key, value] of Object.entries(args)) { + if (key === '_body') { + out._body = interpolateStages(value as any[], item); + } else if (key === '_bodyRaw') { + out._bodyRaw = value; // raw text, not a template + } else if (key === '_') { + out._ = (value as any[]).map((v) => + typeof v === 'string' ? renderTemplate(v, item) : v, + ); + } else if (typeof value === 'string') { + out[key] = renderTemplate(value, item); + } else { + out[key] = value; + } + } + return out; +} + +export const eachCommand = { + name: 'each', + meta: { + description: 'Run a sub-pipeline for each input item', + argsSchema: { + type: 'object', + properties: { + _body: { description: 'Parsed sub-pipeline stages (injected by parser)' }, + }, + required: [], + }, + sideEffects: ['delegates_to_sub_pipeline'], + }, + help() { + return ( + `each — run a sub-pipeline for each input item\n\n` + + `Usage:\n` + + ` ... | each { template --text "hello {{.name}}" }\n` + + ` ... | each { map --unwrap url | exec curl "{{.}}" }\n\n` + + `Notes:\n` + + ` - Each item is fed into the sub-pipeline as a single-element stream.\n` + + ` - {{.field}} interpolation is applied to all string args per item.\n` + + ` - Template patterns ({{...}}) in item field values will be interpolated.\n` + + ` - Errors in any iteration propagate immediately (fail-fast).\n` + + ` - Items are processed sequentially.\n` + ); + }, + async run({ input, args, ctx }: any) { + const bodyStages = args._body; + if (!Array.isArray(bodyStages) || bodyStages.length === 0) { + throw new Error('each requires a { sub-pipeline } body'); + } + + return { + output: (async function* () { + for await (const item of input) { + const interpolated = interpolateStages(bodyStages, item); + const result = await runPipeline({ + pipeline: interpolated, + registry: ctx.registry, + stdin: ctx.stdin, + stdout: ctx.stdout, + stderr: ctx.stderr, + env: ctx.env, + mode: ctx.mode, + input: (async function* () { yield item; })(), + }); + for (const out of result.items) { + yield out; + } + } + })(), + }; + }, +}; diff --git a/test/each.test.ts b/test/each.test.ts new file mode 100644 index 0000000..4c794a1 --- /dev/null +++ b/test/each.test.ts @@ -0,0 +1,181 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { runPipeline } from '../src/runtime.js'; +import { createDefaultRegistry } from '../src/commands/registry.js'; +import { parsePipeline } from '../src/parser.js'; + +async function run(pipelineText: string, input: any[]) { + const pipeline = parsePipeline(pipelineText); + const registry = createDefaultRegistry(); + const res = await runPipeline({ + pipeline, + registry, + stdin: process.stdin, + stdout: process.stdout, + stderr: process.stderr, + env: process.env, + mode: 'tool', + input: (async function* () { for (const x of input) yield x; })(), + }); + return res.items; +} + +// -- Parser tests -- + +test('parsePipeline parses each { ... } as single stage with _body', () => { + const p = parsePipeline('each { head --n 1 }'); + assert.equal(p.length, 1); + assert.equal(p[0].name, 'each'); + assert.ok(Array.isArray(p[0].args._body)); + assert.equal(p[0].args._body.length, 1); + assert.equal(p[0].args._body[0].name, 'head'); + assert.equal(p[0].args._body[0].args.n, '1'); + assert.equal(p[0].args._bodyRaw, 'head --n 1'); +}); + +test('parsePipeline parses multi-stage sub-pipeline in braces', () => { + const p = parsePipeline('each { map --unwrap x | head --n 1 }'); + assert.equal(p.length, 1); + assert.equal(p[0].args._body.length, 2); + assert.equal(p[0].args._body[0].name, 'map'); + assert.equal(p[0].args._body[1].name, 'head'); +}); + +test('parsePipeline handles each in a larger pipeline', () => { + const p = parsePipeline('head --n 5 | each { template --text "hi" } | json'); + assert.equal(p.length, 3); + assert.equal(p[0].name, 'head'); + assert.equal(p[1].name, 'each'); + assert.ok(Array.isArray(p[1].args._body)); + assert.equal(p[2].name, 'json'); +}); + +test('braces inside quoted strings do not trigger body parsing', () => { + const p = parsePipeline("template --text 'hello {world}'"); + assert.equal(p.length, 1); + assert.equal(p[0].name, 'template'); + assert.equal(p[0].args.text, 'hello {world}'); + assert.equal(p[0].args._body, undefined); +}); + +test('bare closing brace without matching open is treated as literal', () => { + const p = parsePipeline('exec echo }'); + assert.equal(p.length, 1); + assert.equal(p[0].name, 'exec'); + assert.deepEqual(p[0].args._, ['echo', '}']); +}); + +test('unclosed brace throws', () => { + assert.throws(() => parsePipeline('each { foo'), /Unclosed brace/); +}); + +test('empty body throws', () => { + assert.throws(() => parsePipeline('each { }'), /Empty body in \{ \} block/); +}); + +test('nested braces parse correctly', () => { + const p = parsePipeline('each { each { head --n 1 } }'); + assert.equal(p.length, 1); + assert.equal(p[0].name, 'each'); + const inner = p[0].args._body; + assert.equal(inner.length, 1); + assert.equal(inner[0].name, 'each'); + assert.equal(inner[0].args._body.length, 1); + assert.equal(inner[0].args._body[0].name, 'head'); +}); + +// -- Functional tests -- + +test('each passes each item through a single-command sub-pipeline', async () => { + const out = await run('each { map --wrap item }', ['a', 'b', 'c']); + assert.deepEqual(out, [{ item: 'a' }, { item: 'b' }, { item: 'c' }]); +}); + +test('each runs multi-stage sub-pipeline', async () => { + const out = await run('each { map --wrap x | map --unwrap x }', [1, 2, 3]); + assert.deepEqual(out, [1, 2, 3]); +}); + +test('each interpolates {{.field}} in sub-pipeline args', async () => { + const out = await run( + 'each { template --text "hello {{.name}}" }', + [{ name: 'alice' }, { name: 'bob' }], + ); + assert.deepEqual(out, ['hello alice', 'hello bob']); +}); + +test('each interpolates {{.nested.path}}', async () => { + const out = await run( + 'each { template --text "{{.user.name}}" }', + [{ user: { name: 'deep' } }], + ); + assert.deepEqual(out, ['deep']); +}); + +test('each interpolates {{.}} for whole item', async () => { + const out = await run( + 'each { template --text "val={{.}}" }', + [42, 'hi'], + ); + assert.deepEqual(out, ['val=42', 'val=hi']); +}); + +test('missing {{.field}} renders as empty string', async () => { + const out = await run( + 'each { template --text "x={{.nope}}" }', + [{ a: 1 }], + ); + assert.deepEqual(out, ['x=']); +}); + +test('each with empty input yields nothing', async () => { + const out = await run('each { map --wrap x }', []); + assert.deepEqual(out, []); +}); + +test('error in sub-pipeline propagates (fail-fast)', async () => { + await assert.rejects( + () => run('each { nonexistent_command }', [1]), + /Unknown command: nonexistent_command/, + ); +}); + +test('nested each works', async () => { + const input = [{ name: 'alice' }, { name: 'bob' }]; + // Redundant nesting but validates parser handles nested braces + // and runtime handles nested each invocations + const out = await run( + 'each { each { template --text "hi {{.name}}" } }', + input, + ); + assert.deepEqual(out, ['hi alice', 'hi bob']); +}); + +test('each without body throws at runtime', async () => { + await assert.rejects( + () => run('each', [1, 2]), + /each requires a \{ sub-pipeline \} body/, + ); +}); + +test('each does not traverse prototype properties', async () => { + const out = await run( + 'each { template --text "x={{.constructor}}" }', + [{ name: 'test' }], + ); + assert.deepEqual(out, ['x=']); +}); + +test('each yields multiple items per input when sub-pipeline fans out', async () => { + const out = await run( + "each { exec --json --shell 'echo \"[1,2]\"' }", + ['x', 'y'], + ); + assert.deepEqual(out, [1, 2, 1, 2]); +}); + +test('deeply nested braces throw recursion depth error', () => { + const deep = 'each { '.repeat(60) + 'head --n 1' + ' }'.repeat(60); + assert.throws(() => parsePipeline(deep), /maximum depth/); +});