From 6a7d8393dcdd0b47555975723cde9b44ccf86a0e Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Sun, 18 Oct 2026 15:17:54 +0200 Subject: [PATCH] fix(story-mode): what the cold readers believed that was not so (review round 8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two agents read only the comic of the eleven recorded sessions (one who has never written code, one engineering manager skimming), wrote down what they believed, then compared with the transcripts. Both were misled in the same places, and the cause was the same: the final panel kept Claude's first sentence or two, and what a reader must not lose sat further into its last message. - Claude's last words now keep key sentences from anywhere in its final message (read in full, past the label's clip): an open problem or a quirk, something not yet done, checked or confirmed, work still out, what you asked for or expected, a correction of an earlier warning (with the sentence it refers to). A bare "I didn't" counts only when nothing stronger is there; table rows, list intros and questions never. "I built the pomodoro timer. Opening the page: it won't work if you double-click index.html…"; "…That note makes the current behaviour clear, but it doesn't fix it."; "As you expected, git describe --tags then failed because the repo has no tags." - A turn that ended with helpers or a background command still out says so: "Done for now", and "Claude's part is done in 36 seconds. Its 2 helpers finished later and reported back." (`finishedLater`). - A failure you asked for, or that Claude said it expected, reads as the plan: "Test first: the first run fails as planned, the second passes.", "The tests don't even load, as planned." A loop that failed once on a test and once on another check says both, and the commits made inside a loop are part of its story; the final panel counts the turn's commits ("5 commits, uploaded"). - One or two files read are named ("Claude checks README.md." in a turn a report started); a tool lookup reads "Claude gets the tools it needs ready." again (it was overwritten) without its query; the advisor is "a stronger AI model", and one that answered with an error "gets no answer"; a commit message loses its `feat:` prefix in the plain caption. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/story-mode.ts | 188 +++++++++++++++--- src/types/story.ts | 9 + src/web/public/story-mode.js | 17 +- .../background-helpers-reports.expected.json | 27 ++- ...ash-script-background-helper.expected.json | 15 +- .../compact-then-plan-mode.expected.json | 15 +- .../errors-push-append.expected.json | 24 ++- .../failing-test-on-purpose.expected.json | 12 +- .../long-turn-prompt-added.expected.json | 18 +- .../python-cli-fix-loop.expected.json | 11 +- .../question-tests-first-push.expected.json | 9 +- .../teammates-then-interrupt.expected.json | 33 +-- .../text-plan-dry-run-helper.expected.json | 10 +- .../web-page-node-test.expected.json | 11 +- test/story-corpus.test.ts | 38 +++- test/story-mode-ui.test.ts | 26 ++- test/story-mode.test.ts | 27 ++- 17 files changed, 372 insertions(+), 118 deletions(-) diff --git a/src/story-mode.ts b/src/story-mode.ts index bef315b2..3333e684 100644 --- a/src/story-mode.ts +++ b/src/story-mode.ts @@ -127,7 +127,7 @@ export type StoryEntry = /** Teammates (helpers started as a team) reporting back, matched to their helper by name. */ teammates?: StoryTeammateReport[]; } - | { k: 'text'; at: string; text: string } + | { k: 'text'; at: string; text: string; keyPoints?: string[] } | { k: 'tool'; at: string; id: string; name: string; input: ReducedInput } | { k: 'result'; @@ -269,11 +269,75 @@ export function balloonText(text: string): string { return clip(two && REPORTS_PROBLEM.test(one) && two.length <= 140 ? `${one} ${two}` : one, 260); } -/** Claude's last words: the first sentence, and the next when it carries a caveat. */ -export function closingText(text: string): string { +/** + * A sentence a reader of Claude's last words must not lose. STRONG: an open problem or quirk, + * something not yet done, checked or confirmed, work still out, a word on what you asked for or + * expected, a correction of an earlier warning. WEAK: a bare negation ("I didn't push"), kept only when + * nothing strong is there. + */ +const KEY_POINT_STRONG = new RegExp( + [ + String.raw`\b(problem|known (issue|quirk)s?|quirks?|caveat|limitation|heads up|warning|not yet|haven't|have not|never|unfixed|left (open|unfixed|out)|still (fails?|failing|broken|missing|open|running|working)|must be (served|run)|(won't|doesn't|does not|will not) work|can't tell|isn't (confirmed|committed|pushed|tested|done|checked)|not (checked|tested|verified|confirmed|committed|pushed|opened))\b`, + String.raw`\b(as you (asked|expected|said|wanted|suggested)|you asked|your (--?[\w-]+|request|option|flag|change))\b`, + String.raw`\b(out of date|looked too early|turned out|was wrong)\b`, + ].join('|'), + 'i' +); +const KEY_POINT_WEAK = /\b(didn't|did not|couldn't|could not|wasn't able|won't|doesn't)\b/i; + +/** + * Sentences from further into a long message that carry a key point, at most two strong ones (or one + * weak one when none is strong), in order. Read on the whole message (the event's label is clipped), + * its first 8,000 characters. Table rows, list intros ending in `:` and questions are skipped: none of + * them reads on its own in a balloon. + */ +function keyPointsOf(text: string): string[] { + if (text.length <= 160) return []; + const spans: string[] = []; + const plain = plainMarkdown(text.slice(0, 8000), spans); + const strong: string[] = []; + let weak: string | undefined; + let first = true; + let previous = ''; + for (const line of plain.split('\n')) { + if (strong.length >= 2) break; + const trimmed = line.trim(); + if (trimmed.startsWith('|') || (trimmed.match(/\|/g)?.length ?? 0) >= 2) continue; // a table row + let rest = trimmed.replace(/\s+/g, ' '); + while (rest && strong.length < 2) { + const m = SENTENCE_END.exec(rest); + const sentence = (m ? m[1] : rest).trim(); + rest = m ? rest.slice(m[1].length).trim() : ''; + // The first sentence is the balloon's anyway. + if (first) { + first = false; + continue; + } + const before = previous; + previous = sentence; + if (sentence.length < 12 || /[:?]$/.test(sentence)) continue; + // "That's out of date" needs what "that" was: the sentence before comes along. + const context = /^(that|this|it)\b/i.test(sentence) && before && !strong.includes(before) ? `${before} ` : ''; + const shown = clip(restoreSpans(context + sentence, spans), context ? 300 : 180); + if (KEY_POINT_STRONG.test(sentence)) strong.push(shown); + else if (!weak && KEY_POINT_WEAK.test(sentence)) weak = shown; + } + } + return strong.length ? strong : weak ? [weak] : []; +} + +/** + * Claude's last words: the first sentence, the next when it carries a caveat, and the key points from + * further in (an open problem, what was or was not done), so the final panel never reads as a clean + * finish when Claude said otherwise. + */ +export function closingText(text: string, keyPoints: string[] = []): string { const [one, two] = firstTwoSentences(text); // A very short first sentence ("Not done.") says too little alone. - return clip(two && (CAVEAT.test(two) || one.length < 24) && two.length <= 200 ? `${one} ${two}` : one, 300); + const parts = [one]; + if (two && (CAVEAT.test(two) || one.length < 24) && two.length <= 200) parts.push(two); + for (const point of keyPoints) if (!parts.some((p) => p.includes(point) || point.includes(p))) parts.push(point); + return clip(parts.join(' '), parts.length > 2 ? 420 : 300); } /** The first sentence of a message, without markdown, for a speech balloon. */ @@ -1283,14 +1347,17 @@ export function createStoryReducer({ sidechain = false }: { sidechain?: boolean function reduceAssistant(entry: Record, at: string): StoryEntry[] { const message = isRecord(entry.message) ? entry.message : {}; const content = message.content; - if (typeof content === 'string') - return content.trim() ? [{ k: 'text', at, text: clip(content, STORY_LIMITS.textChars) }] : []; + const textEntry = (text: string): StoryEntry => { + const keyPoints = keyPointsOf(text); + return { k: 'text', at, text: clip(text, STORY_LIMITS.textChars), ...(keyPoints.length ? { keyPoints } : {}) }; + }; + if (typeof content === 'string') return content.trim() ? [textEntry(content)] : []; if (!Array.isArray(content)) return []; const out: StoryEntry[] = []; for (const block of content) { if (!isRecord(block)) continue; if (block.type === 'text' && typeof block.text === 'string' && block.text.trim()) { - out.push({ k: 'text', at, text: clip(block.text, STORY_LIMITS.textChars) }); + out.push(textEntry(block.text)); } else if (block.type === 'tool_use' && typeof block.id === 'string' && typeof block.name === 'string') { const input = reduceInput(block.name, block.input, str(entry.cwd)); remember(block.id, block.name, input); @@ -1581,6 +1648,12 @@ export function buildStoryTurns(entries: StoryEntry[]): StoryTurn[] { /** The same helpers by name: a teammate reports back by name, not by call. */ const helpersByName = new Map(); let say = 0; + /** The turn each helper or command call belongs to: a report after that turn ended came back later. */ + const turnOfCall = new WeakMap(); + const cameBackLater = (ev: StoryEvent) => { + const own = turnOfCall.get(ev); + return !!own && (own !== cur || own.ended); + }; const teammatesBack = (reports: StoryTeammateReport[], prompt: StoryEvent) => { const told = reports.filter((r) => r.result || r.summary); if (told.length) { @@ -1593,6 +1666,7 @@ export function buildStoryTurns(entries: StoryEntry[]): StoryTurn[] { const ev = helpersByName.get(r.from); if (!ev?.helper) continue; const helper: StoryHelper = { ...ev.helper, running: undefined }; + if (helper.background && cameBackLater(ev)) ev.finishedLater = true; if (r.result) helper.result = r.result; ev.helper = helper; } @@ -1614,6 +1688,7 @@ export function buildStoryTurns(entries: StoryEntry[]): StoryTurn[] { if (ev?.running && !ev.helper) { // A background command: done now, and its summary carries the exit code. ev.running = undefined; + if (cameBackLater(ev)) ev.finishedLater = true; const code = /\(exit code (\d+)\)/.exec(prompt.label)?.[1]; ev.ok = code !== undefined ? code === '0' : !report.status || /^(completed|done|finished|success)/i.test(report.status); @@ -1622,6 +1697,7 @@ export function buildStoryTurns(entries: StoryEntry[]): StoryTurn[] { } if (!ev?.helper) return; const helper: StoryHelper = { ...ev.helper, running: undefined }; + if (helper.background && cameBackLater(ev)) ev.finishedLater = true; if (report.result) helper.result = report.result; if (report.toolUses !== undefined) helper.toolUses = report.toolUses; if (report.durationMs !== undefined) helper.durationMs = report.durationMs; @@ -1712,6 +1788,7 @@ export function buildStoryTurns(entries: StoryEntry[]): StoryTurn[] { tool: 'Text', label: e.text, intent: 'say', + ...(e.keyPoints ? { keyPoints: e.keyPoints } : {}), }); break; case 'tool': { @@ -1721,6 +1798,7 @@ export function buildStoryTurns(entries: StoryEntry[]): StoryTurn[] { if (ev.helper || ev.tool === 'Bash') { // A helper, or a command that may turn out to run in the background: both report back later. helpers.set(e.id, ev); + turnOfCall.set(ev, ensure(e.at)); if (helpers.size > 2000) helpers.delete(helpers.keys().next().value as string); if (ev.helper) { helpersByName.set(ev.helper.name, ev); @@ -1916,11 +1994,11 @@ const ANNOUNCES = /\b(I'll|I will|let me|let's|I'm going to|next,? I|now I)\b/i; function groupPanels(events: StoryEvent[]): { panels: DraftPanel[]; trailingSay?: string } { const panels: DraftPanel[] = []; let pendingSay: string | undefined; - let lastSaid: string | undefined; + let lastSaid: StoryEvent | undefined; events.forEach((ev, idx) => { if (ev.intent === 'say' && !ev.todos) { pendingSay ??= balloonText(ev.label); - lastSaid ??= ev.label; + lastSaid ??= ev; return; } lastSaid = undefined; @@ -1959,7 +2037,7 @@ function groupPanels(events: StoryEvent[]): { panels: DraftPanel[]; trailingSay? pendingSay = undefined; }); // The closing words keep a caveat that follows them ("Nothing is committed: this is no git repo."). - return { panels, trailingSay: lastSaid !== undefined ? closingText(lastSaid) : pendingSay }; + return { panels, trailingSay: lastSaid !== undefined ? closingText(lastSaid.label, lastSaid.keyPoints) : pendingSay }; } /** Fix → check → fix → check (4 panels or more) is one "loop" panel: try, test, repeat. */ @@ -2076,6 +2154,16 @@ function montageSummary(evs: StoryEvent[]): string { return parts.length ? parts.join(', ') : 'a few more steps'; } +/** You asked for a failing run first (test first), or Claude said the failure was expected. */ +const PLANNED_IN_PROMPT = + /\b(test[- ]first|tdd|fail(s|ing)? (first|on purpose)|failing tests? first|on purpose|red first|before (the )?(code|module|function|implementation) exists)\b/i; +const EXPECTED_BY_CLAUDE = + /\b(as expected|as planned|as intended|expected (failure|to fail|red)|deliberately fail\w*|on purpose|test[- ]first|the red state)\b/i; +function plannedFailure(turn: StoryTurn, events: StoryEvent[]): boolean { + const prompt = events.find((e) => e.intent === 'prompt')?.label ?? turn.prompt; + return PLANNED_IN_PROMPT.test(prompt) || events.some((e) => e.intent === 'say' && EXPECTED_BY_CLAUDE.test(e.label)); +} + function decorate(draft: DraftPanel, events: StoryEvent[], turn: StoryTurn, index: number): StoryPanel { const evs = draft.events.map((i) => events[i]); const panel: StoryPanel = { kind: draft.kind, events: draft.events, caption: { story: '', director: '' } }; @@ -2139,15 +2227,23 @@ function decorate(draft: DraftPanel, events: StoryEvent[], turn: StoryTurn, inde .filter((e) => e.intent !== 'read') .map((e) => e.description || shortCommand(e)) .slice(0, 4); - if (evs.every((e) => e.tool === 'ToolSearch')) story = 'Claude gets the tools it needs ready.'; if (evs.every((e) => e.tool === 'Bash')) director = `${tools} · ${shortCommand(evs[0])}`; if (errors) director += ` · ${plural(errors, 'error')}`; const shown = files.filter((f) => !/(^|\/)\.codeman[-.]/.test(f)); - story = shown.length - ? `Claude reads ${plural(shown.length, 'file')} to find its way around.` - : evs.every((e) => e.intent === 'search') - ? 'Claude searches the code.' - : 'Claude looks around the project.'; + // A file or two by name; in a turn a report started, reading is checking what came back. + const checking = turn.origin === 'task' || turn.origin === 'peer'; + const named = shown.map((f) => basename(f)); + story = evs.every((e) => e.tool === 'ToolSearch') + ? 'Claude gets the tools it needs ready.' + : shown.length && shown.length <= 2 + ? `Claude ${checking ? 'checks' : 'reads'} ${named.join(' and ')}.` + : shown.length + ? `Claude reads ${plural(shown.length, 'file')} to find its way around.` + : evs.every((e) => e.intent === 'search') + ? 'Claude searches the code.' + : 'Claude looks around the project.'; + // A tool lookup's query (`select:EnterPlanMode,ExitPlanMode`) means nothing to a reader. + if (evs.every((e) => e.tool === 'ToolSearch')) panel.notes = []; break; } case 'research': @@ -2188,7 +2284,9 @@ function decorate(draft: DraftPanel, events: StoryEvent[], turn: StoryTurn, inde const advisor = evs.filter((e) => e.tool === 'Advisor').length; story = helpers.length ? advisor === helpers.length - ? 'Claude asks the advisor for a second opinion.' + ? evs.every((e) => e.tool !== 'Advisor' || e.ok === false) + ? 'Claude asks a stronger AI model for a second opinion, but gets no answer.' + : 'Claude asks a stronger AI model for a second opinion.' : behind === helpers.length ? `Claude sends ${plural(helpers.length, 'helper')} off to work in the background.` : `Claude calls in ${plural(helpers.length, 'helper')}.` @@ -2246,20 +2344,32 @@ function decorate(draft: DraftPanel, events: StoryEvent[], turn: StoryTurn, inde (e) => !!e.tests && e.tests.passed > 0 && !e.tests.failed && !e.tests.errors ).length; const n = panel.rounds; + const testFails = fails - lintFails; + // A failure you asked for (test first), or that Claude said it expected, is the plan working. + const planned = plannedFailure(turn, events) && passed(checks[0]) === false; story = panel.ok === true && !fails ? `Build and test, ${plural(n, 'round')}. Every check passes.` : panel.ok === true && fails === lintFails - ? `Build and test, ${plural(n, 'round')}. A check fails ${fails === 1 ? 'once' : `${fails} times`}, then everything passes.` - : panel.ok === true && checks.length === 2 && fails === 1 - ? 'Fix and test: the first run fails, the second passes.' - : panel.ok === true - ? `Fix and test, ${plural(n, 'round')}: ${fails === 1 ? 'one run fails' : `${fails} runs fail`}, then it passes.` - : panel.ok === false && tests && tests.passed && !tests.failed && !tests.errors - ? `Fix and test, ${plural(n, 'round')}. The tests pass, but another check fails.` - : panel.ok === false - ? `Fix and test, ${plural(n, 'round')}. Still failing.` - : `Fix and test, ${plural(n, 'round')}.`; + ? `Build and test, ${plural(n, 'round')}. The tests pass every time; another check fails ${fails === 1 ? 'once' : `${fails} times`}, then passes.` + : panel.ok === true && planned && checks.length === 2 + ? 'Test first: the first run fails as planned, the second passes.' + : panel.ok === true && planned + ? `Test first, ${plural(n, 'round')}: the first run fails as planned, then it passes.` + : panel.ok === true && checks.length === 2 && fails === 1 + ? 'Fix and test: the first run fails, the second passes.' + : panel.ok === true && testFails && lintFails + ? `Fix and test, ${plural(n, 'round')}: ${testFails === 1 ? 'a test run fails' : `${testFails} test runs fail`}, then another check, then everything passes.` + : panel.ok === true + ? `Fix and test, ${plural(n, 'round')}: ${fails === 1 ? 'one run fails' : `${fails} runs fail`}, then it passes.` + : panel.ok === false && tests && tests.passed && !tests.failed && !tests.errors + ? `Fix and test, ${plural(n, 'round')}. The tests pass, but another check fails.` + : panel.ok === false + ? `Fix and test, ${plural(n, 'round')}. Still failing.` + : `Fix and test, ${plural(n, 'round')}.`; + // Commits made inside the loop (a test one-liner that ends in one) are part of its story. + const saved = evs.reduce((k, e) => k + (e.commits ?? 0), 0); + if (saved) story += ` Claude saves the work${saved > 1 ? ` in ${saved} commits` : ''} along the way.`; director = `${plural(checks.length, 'check')} · ${plural(edits.length, 'edit')} · ${fmtCounts(panel.added, panel.removed)} · last: ${ lastCheck ? outcome(lastCheck) : 'running' }`; @@ -2291,6 +2401,7 @@ function decorate(draft: DraftPanel, events: StoryEvent[], turn: StoryTurn, inde ? `${tests.failed} tests fail.` : 'A test fails.' : 'A check fails.'; + if (ok === false && plannedFailure(turn, events)) story = `${story.replace(/[.]$/, '')}, as planned.`; // The same command with the same outcome twice in a row reads once, with a count. const runs: Array<{ text: string; n: number }> = []; for (const e of checks) { @@ -2355,7 +2466,9 @@ function decorate(draft: DraftPanel, events: StoryEvent[], turn: StoryTurn, inde const uploadOnly = ships === 'push' && !commits; // One commit: its message says what was saved. const subject = commits === 1 ? evs.find((e) => e.subject)?.subject : undefined; - const said = subject ? `: "${clip(subject, 60).replace(/[.]$/, '')}"` : ''; + // Its conventional prefix (`feat:`, `chore(deps):`) is for git, not for the reader. + const plainSubject = subject?.replace(/^[a-z]+(\([^)]*\))?!?:\s*/i, ''); + const said = plainSubject ? `: "${clip(plainSubject, 60).replace(/[.]$/, '')}"` : ''; story = savedAnyway ? uploadOnly ? 'Claude uploads the work, but a step after that fails.' @@ -2420,6 +2533,9 @@ function decorate(draft: DraftPanel, events: StoryEvent[], turn: StoryTurn, inde function completePanel(turn: StoryTurn, page: Omit, trailingSay?: string): StoryPanel { const running = page.helpersRunning; const commands = page.events.filter((e) => e.running && !e.helper).length; + const later = page.events.filter((e) => e.finishedLater && e.intent !== 'prompt'); + const laterHelpers = later.filter((e) => e.helper).length; + const laterCommands = later.length - laterHelpers; const done = turn.durationMs !== undefined ? `Done in ${roughDuration(turn.durationMs)}.` : 'Done.'; const story = turn.interrupted ? 'You stop Claude here.' @@ -2436,7 +2552,18 @@ function completePanel(turn: StoryTurn, page: Omit, trailin ] .filter(Boolean) .join(' ')}` - : done; + : later.length + ? `Claude's part is done${turn.durationMs !== undefined ? ` in ${roughDuration(turn.durationMs)}` : ''}. ${[ + laterHelpers + ? `${laterHelpers === 1 ? 'Its helper' : `Its ${laterHelpers} helpers`} finished later and reported back.` + : '', + laterCommands + ? `${laterCommands === 1 ? 'Its background command' : `${laterCommands} background commands`} finished later.` + : '', + ] + .filter(Boolean) + .join(' ')}` + : done; const director = [ turn.endedBy === 'next prompt' ? 'superseded by the next prompt' : (turn.endedBy ?? 'turn_duration'), turn.durationMs !== undefined ? exactDuration(turn.durationMs) : '', @@ -2501,6 +2628,9 @@ export function buildStoryPage(turn: StoryTurn): StoryPage { // The advisor is a second opinion, not a helper that does work. helpers: events.filter((e) => e.helper && e.tool !== 'Advisor' && e.intent !== 'prompt').length, helpersRunning: events.filter((e) => e.helper?.running).length, + ...(events.some((e) => e.finishedLater && e.intent !== 'prompt') + ? { finishedLater: events.filter((e) => e.finishedLater && e.intent !== 'prompt').length } + : {}), }; const calls = events.filter((e) => !NOT_TOOL_CALLS.has(e.intent)); if (calls.length && calls.every((e) => e.ok === false)) base.noneWorked = true; diff --git a/src/types/story.ts b/src/types/story.ts index de980a61..35a84476 100644 --- a/src/types/story.ts +++ b/src/types/story.ts @@ -130,6 +130,13 @@ export interface StoryEvent { running?: boolean; /** A command that ran in the background (its result came back later, as a report of its own). */ background?: boolean; + /** Background work (this command, or this helper call's helper) that finished after its turn ended. */ + finishedLater?: boolean; + /** + * Sentences from further into a long message that a reader must not lose: an open problem, a step + * not taken, what you asked for. From the whole message, past the clip of `label`. + */ + keyPoints?: string[]; /** A save that ships the work: `gh pr create` (`pr`) or `git push` (`push`). */ ships?: 'pr' | 'push'; /** How many commits a Bash call makes (`git commit` segments). */ @@ -201,6 +208,8 @@ export interface StoryPage extends StoryTurnSummary { helpers: number; /** Background helpers that had not reported back when this page was drawn. */ helpersRunning: number; + /** Helpers and background commands of this turn that finished only after it ended. */ + finishedLater?: number; /** Every tool call of the turn failed (a blocked command, nothing that worked): no celebration. */ noneWorked?: boolean; } diff --git a/src/web/public/story-mode.js b/src/web/public/story-mode.js index 9e63807b..d94da8ff 100644 --- a/src/web/public/story-mode.js +++ b/src/web/public/story-mode.js @@ -2481,8 +2481,9 @@ Object.assign(CodemanApp.prototype, { ? 'To be continued…' : page.noneWorked ? 'Turn over' - : page.helpersRunning || page.events.some((e) => e.running && !e.helper) - ? 'To be continued…' + : page.helpersRunning || page.finishedLater || page.events.some((e) => e.running && !e.helper) + ? // Work was still out when the turn ended (and maybe still is): not a clean finish. + 'Done for now' : 'Mission complete!'; const chips = rt.splash.querySelector('.story-chips'); const items = []; @@ -2493,6 +2494,18 @@ Object.assign(CodemanApp.prototype, { open: 'diff:page', }); items.push({ text: `${page.toolCalls} tool call${page.toolCalls === 1 ? '' : 's'}`, open: 'list:page' }); + // What was saved, wherever in the turn: commits inside a loop or a test one-liner count too. + const commits = page.events.reduce((n, e) => n + (e.commits || 0), 0); + // An upload that went through, even when a step after it in the same command failed. + const shipped = page.events.some( + (e) => e.ships === 'push' && (e.ok !== false || /\[new branch\]|\S+\.\.\S+\s+\S+\s+->/.test(e.output || '')) + ); + if (commits || shipped) + items.push({ + text: [commits ? `${commits} commit${commits === 1 ? '' : 's'}` : '', shipped ? 'uploaded' : ''] + .filter(Boolean) + .join(', '), + }); if (page.helpers) items.push({ text: `${page.helpers} helper${page.helpers === 1 ? '' : 's'}${ diff --git a/test/fixtures/story-corpus/background-helpers-reports.expected.json b/test/fixtures/story-corpus/background-helpers-reports.expected.json index 4f706153..10ddad2f 100644 --- a/test/fixtures/story-corpus/background-helpers-reports.expected.json +++ b/test/fixtures/story-corpus/background-helpers-reports.expected.json @@ -11,6 +11,7 @@ "removed": 0, "helpers": 2, "helpersRunning": 0, + "finishedLater": 2, "panels": [ { "kind": "mission", @@ -58,9 +59,9 @@ { "kind": "complete", "ok": null, - "story": "Done in 36 seconds.", + "story": "Claude's part is done in 36 seconds. Its 2 helpers finished later and reported back.", "director": "turn_duration · 36 s · 5 tool calls · 2 files changed · 2 helpers", - "balloon": "I wrote wordcount.py and tests/test_basic.py, and the one python3 -m pytest -q run passed 4 of 4 tests.", + "balloon": "I wrote wordcount.py and tests/test_basic.py, and the one python3 -m pytest -q run passed 4 of 4 tests. Only 4 tests ran, so tests/test_extra.py from helper B wasn't there yet and those tests haven't been run.", "tests": null, "roundsOk": null, "events": 0 @@ -79,6 +80,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -93,7 +95,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude reads 1 file to find its way around.", + "story": "Claude checks README.md.", "director": "Bash · cat /work/cases/story-r6-b/README.md", "balloon": null, "tests": null, @@ -105,7 +107,7 @@ "ok": null, "story": "Done in 14 seconds.", "director": "turn_duration · 14 s · 1 tool call", - "balloon": "Helper A has finished writing README.md, and I checked it against the code.", + "balloon": "Helper A has finished writing README.md, and I checked it against the code. It describes the command-line usage, count(text) and how to run the tests correctly, and its example ((2, 4, 24)) is right. It doesn't mention that running the tool without exactly one file prints a usage message and exits…", "tests": null, "roundsOk": null, "events": 0 @@ -124,6 +126,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -172,6 +175,7 @@ "removed": 4, "helpers": 1, "helpersRunning": 0, + "finishedLater": 2, "panels": [ { "kind": "mission", @@ -216,9 +220,9 @@ { "kind": "complete", "ok": null, - "story": "Done in 52 seconds.", + "story": "Claude's part is done in 52 seconds. Its helper finished later and reported back. Its background command finished later.", "director": "turn_duration · 52 s · 5 tool calls · 3 files changed · 1 helper", - "balloon": "I added the --json flag and started the helper and the test run as you asked.", + "balloon": "I added the --json flag and started the helper and the test run as you asked. The helper and the tests are still running in the background; I haven't waited for either, so I don't have results yet. No git, as you asked.", "tests": null, "roundsOk": null, "events": 0 @@ -237,6 +241,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -285,6 +290,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -299,7 +305,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude reads 1 file to find its way around.", + "story": "Claude checks CHANGELOG.md.", "director": "Bash · cat /work/cases/story-r6-b/CHANGELOG.md", "balloon": null, "tests": null, @@ -311,7 +317,7 @@ "ok": null, "story": "Done in 8 seconds.", "director": "turn_duration · 8 s · 1 tool call", - "balloon": "Everything you asked for this round is done.", + "balloon": "Everything you asked for this round is done. The helper used four tools, one for each step you asked for, so it looks like it followed the sleep, write, sleep, edit sequence.", "tests": null, "roundsOk": null, "events": 0 @@ -330,6 +336,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -357,7 +364,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude reads 1 file to find its way around.", + "story": "Claude reads CHANGELOG.md.", "director": "Bash · head -5 /work/cases/story-r6-b/CHANGELOG.md", "balloon": null, "tests": null, @@ -388,6 +395,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -423,6 +431,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", diff --git a/test/fixtures/story-corpus/bash-script-background-helper.expected.json b/test/fixtures/story-corpus/bash-script-background-helper.expected.json index 897b5fbf..3e5b835f 100644 --- a/test/fixtures/story-corpus/bash-script-background-helper.expected.json +++ b/test/fixtures/story-corpus/bash-script-background-helper.expected.json @@ -11,6 +11,7 @@ "removed": 3, "helpers": 1, "helpersRunning": 0, + "finishedLater": 1, "panels": [ { "kind": "mission", @@ -45,7 +46,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion, but gets no answer.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -85,7 +86,7 @@ { "kind": "complete", "ok": null, - "story": "Done in 2 minutes.", + "story": "Claude's part is done in 2 minutes. Its helper finished later and reported back.", "director": "turn_duration · 2 min 1 s · 14 tool calls · 2 files changed · 1 helper", "balloon": "All the script tests pass.", "tests": null, @@ -106,6 +107,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -120,7 +122,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude reads 1 file to find its way around.", + "story": "Claude checks README.md.", "director": "Read", "balloon": null, "tests": null, @@ -142,7 +144,7 @@ "ok": null, "story": "Done in 21 seconds.", "director": "turn_duration · 21 s · 2 tool calls", - "balloon": "I built rename_by_date.sh, the helper agent wrote README.md, and the script passed every test I ran on sample files in samples/. Nothing is committed because this folder isn't a git repository, even though the session context said it was.", + "balloon": "I built rename_by_date.sh, the helper agent wrote README.md, and the script passed every test I ran on sample files in samples/. Nothing is committed because this folder isn't a git repository, even though the session context said it was. Only lowercase .jpg is matched, as you asked.", "tests": null, "roundsOk": null, "events": 0 @@ -161,6 +163,7 @@ "removed": 8, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -175,7 +178,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude reads 1 file to find its way around.", + "story": "Claude reads rename_by_date.sh.", "director": "Read", "balloon": null, "tests": null, @@ -217,7 +220,7 @@ "ok": null, "story": "Done in 1 minute.", "director": "turn_duration · 1 min 14 s · 9 tool calls · 2 files changed", - "balloon": "I added a --prefix option to rename_by_date.sh.", + "balloon": "I added a --prefix option to rename_by_date.sh. The samples folder didn't change.", "tests": null, "roundsOk": null, "events": 0 diff --git a/test/fixtures/story-corpus/compact-then-plan-mode.expected.json b/test/fixtures/story-corpus/compact-then-plan-mode.expected.json index c1b3d7ca..6748142e 100644 --- a/test/fixtures/story-corpus/compact-then-plan-mode.expected.json +++ b/test/fixtures/story-corpus/compact-then-plan-mode.expected.json @@ -11,6 +11,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -35,7 +36,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -68,7 +69,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"feat: add clamp() with pytest coverage\".", + "story": "Claude saves the work: \"add clamp() with pytest coverage\".", "director": "git commit -q -F -", "balloon": null, "tests": null, @@ -80,7 +81,7 @@ "ok": null, "story": "Done in 1 minute.", "director": "turn_duration · 1 min 12 s · 7 tool calls · 2 files changed", - "balloon": "I wrote clamp and its five tests, and all five pass.", + "balloon": "I wrote clamp and its five tests, and all five pass. I didn't commit CLAUDE.md, .claude/ or .codeman-agent-case.json.", "tests": null, "roundsOk": null, "events": 0 @@ -99,6 +100,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -144,6 +146,7 @@ "removed": 1, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -158,7 +161,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude looks around the project.", + "story": "Claude gets the tools it needs ready.", "director": "ToolSearch", "balloon": null, "tests": null, @@ -178,7 +181,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -231,7 +234,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"feat: add wrap() for modular wrapping into a range\".", + "story": "Claude saves the work: \"add wrap() for modular wrapping into a range\".", "director": "git commit -q -F -", "balloon": null, "tests": null, diff --git a/test/fixtures/story-corpus/errors-push-append.expected.json b/test/fixtures/story-corpus/errors-push-append.expected.json index 00824f5f..87913dac 100644 --- a/test/fixtures/story-corpus/errors-push-append.expected.json +++ b/test/fixtures/story-corpus/errors-push-append.expected.json @@ -11,6 +11,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -35,7 +36,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -65,7 +66,7 @@ { "kind": "check", "ok": false, - "story": "The tests don't even load.", + "story": "The tests don't even load, as planned.", "director": "python3 -m pytest -q → error · 0 passed, 1 error", "balloon": "Now the red run, once, with calc_stats.py confirmed absent:", "tests": { @@ -79,7 +80,7 @@ { "kind": "save", "ok": true, - "story": "Claude writes 1 file and saves the work: \"test: add failing tests for calc_stats mean/median\".", + "story": "Claude writes 1 file and saves the work: \"add failing tests for calc_stats mean/median\".", "director": "git commit -q -F - · writes 1 file", "balloon": "Red as expected: collection error, exit code 2.", "tests": null, @@ -112,7 +113,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"feat: implement calc_stats mean and median\".", + "story": "Claude saves the work: \"implement calc_stats mean and median\".", "director": "git commit -q -F -", "balloon": null, "tests": null, @@ -122,7 +123,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -134,7 +135,7 @@ "ok": null, "story": "Done in 2 minutes.", "director": "turn_duration · 2 min 10 s · 12 tool calls · 3 files changed", - "balloon": "I built it test-first: the suite failed at collection before calc_stats.py existed, then passed once the module was written.", + "balloon": "I built it test-first: the suite failed at collection before calc_stats.py existed, then passed once the module was written. I didn't push, didn't touch origin.git/ and didn't create a README, since none existed to keep in sync.", "tests": null, "roundsOk": null, "events": 0 @@ -153,6 +154,7 @@ "removed": 1, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -167,7 +169,7 @@ { "kind": "loop", "ok": true, - "story": "Fix and test: the first run fails, the second passes.", + "story": "Test first: the first run fails as planned, the second passes.", "director": "2 checks · 3 edits · +40 −1 · last: ok", "balloon": "Green: 18 passed and ruff is clean.", "tests": { @@ -183,7 +185,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work and uploads it: \"feat: add calc_stats.mode with smallest-value tie-breaking\".", + "story": "Claude saves the work and uploads it: \"add calc_stats.mode with smallest-value tie-breaking\".", "director": "git commit -q -F - · git push -u origin HEAD", "balloon": null, "tests": null, @@ -205,7 +207,7 @@ "ok": null, "story": "Done in 37 seconds.", "director": "turn_duration · 37 s · 8 tool calls · 2 files changed", - "balloon": "mode(xs) is in calc_stats.py, committed once after the tests went green, and pushed.", + "balloon": "mode(xs) is in calc_stats.py, committed once after the tests went green, and pushed. I didn't use the advisor, gh or the network.", "tests": null, "roundsOk": null, "events": 0 @@ -224,6 +226,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -283,7 +286,7 @@ "ok": null, "story": "Done in 27 seconds.", "director": "turn_duration · 27 s · 5 tool calls · 1 file changed", - "balloon": "NOTES.md has the two lines and is committed and pushed.", + "balloon": "NOTES.md has the two lines and is committed and pushed. As you expected, git describe --tags then failed because the repo has no tags.", "tests": null, "roundsOk": null, "events": 0 @@ -302,6 +305,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", diff --git a/test/fixtures/story-corpus/failing-test-on-purpose.expected.json b/test/fixtures/story-corpus/failing-test-on-purpose.expected.json index d1993218..8afe5fa9 100644 --- a/test/fixtures/story-corpus/failing-test-on-purpose.expected.json +++ b/test/fixtures/story-corpus/failing-test-on-purpose.expected.json @@ -11,6 +11,7 @@ "removed": 1, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -35,7 +36,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -45,7 +46,7 @@ { "kind": "loop", "ok": true, - "story": "Fix and test: the first run fails, the second passes.", + "story": "Test first: the first run fails as planned, the second passes.", "director": "2 checks · 3 edits · +35 −1 · last: ok", "balloon": "All 4 pass.", "tests": { @@ -71,7 +72,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"feat: add slugify module with node:test suite\".", + "story": "Claude saves the work: \"add slugify module with node:test suite\".", "director": "git commit -q -F -", "balloon": null, "tests": null, @@ -102,6 +103,7 @@ "removed": 1, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -116,7 +118,7 @@ { "kind": "loop", "ok": true, - "story": "Fix and test: the first run fails, the second passes.", + "story": "Test first: the first run fails as planned, the second passes.", "director": "2 checks · 2 edits · +5 −1 · last: ok", "balloon": "All 5 pass.", "tests": { @@ -132,7 +134,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"test: cover empty-string input for slugify\".", + "story": "Claude saves the work: \"cover empty-string input for slugify\".", "director": "git commit -q -F -", "balloon": null, "tests": null, diff --git a/test/fixtures/story-corpus/long-turn-prompt-added.expected.json b/test/fixtures/story-corpus/long-turn-prompt-added.expected.json index b2c893cd..89ef9a42 100644 --- a/test/fixtures/story-corpus/long-turn-prompt-added.expected.json +++ b/test/fixtures/story-corpus/long-turn-prompt-added.expected.json @@ -11,6 +11,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -35,7 +36,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -65,7 +66,7 @@ { "kind": "loop", "ok": true, - "story": "Build and test, 4 rounds. A check fails once, then everything passes.", + "story": "Build and test, 4 rounds. The tests pass every time; another check fails once, then passes.", "director": "5 checks · 10 edits · +694 · last: ok", "balloon": "All 4 modules are done: 63 tests pass and ruff is clean.", "tests": { @@ -127,7 +128,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -139,7 +140,7 @@ "ok": null, "story": "Done in 6 minutes.", "director": "turn_duration · 6 min 25 s · 26 tool calls · 13 files changed", - "balloon": "The inventory/ package is built and committed.", + "balloon": "The inventory/ package is built and committed. --csv is part of the CLI commit rather than its own commit, because you asked for it before that module existed.", "tests": null, "roundsOk": null, "events": 0 @@ -158,6 +159,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -202,7 +204,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"docs: changelog\".", + "story": "Claude saves the work: \"changelog\".", "director": "git commit -m \"docs: changelog\" -m \"Co-Authored-By: Claude…", "balloon": "Step 10:", "tests": null, @@ -212,7 +214,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude reads 1 file to find its way around.", + "story": "Claude reads cli.py.", "director": "Read", "balloon": "Step 11:", "tests": null, @@ -255,7 +257,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"docs: changelog entry\".", + "story": "Claude saves the work: \"changelog entry\".", "director": "git commit -am \"docs: changelog entry\" -m \"Co-Authored-By:…", "balloon": "Step 15:", "tests": null, @@ -277,7 +279,7 @@ "ok": null, "story": "Done in 1 minute.", "director": "turn_duration · 1 min 16 s · 16 tool calls · 2 files changed", - "balloon": "I ran all 16 steps in order, one tool call each.", + "balloon": "I ran all 16 steps in order, one tool call each. I didn't re-run ruff, because only README.md and CHANGELOG.md changed, and ruff doesn't check Markdown.", "tests": null, "roundsOk": null, "events": 0 diff --git a/test/fixtures/story-corpus/python-cli-fix-loop.expected.json b/test/fixtures/story-corpus/python-cli-fix-loop.expected.json index e54a8983..e79330e8 100644 --- a/test/fixtures/story-corpus/python-cli-fix-loop.expected.json +++ b/test/fixtures/story-corpus/python-cli-fix-loop.expected.json @@ -11,6 +11,7 @@ "removed": 1, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -35,7 +36,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -45,7 +46,7 @@ { "kind": "loop", "ok": true, - "story": "Fix and test, 2 rounds: 2 runs fail, then it passes.", + "story": "Fix and test, 2 rounds: a test run fails, then another check, then everything passes. Claude saves the work in 4 commits along the way.", "director": "3 checks · 5 edits · +218 −1 · last: ok", "balloon": "All 19 tests pass, ruff check is clean, and the output matches real wc column for column.", "tests": { @@ -62,7 +63,7 @@ { "kind": "save", "ok": true, - "story": "Claude writes 1 file and saves the work: \"chore: ignore Python and pytest caches\".", + "story": "Claude writes 1 file and saves the work: \"ignore Python and pytest caches\".", "director": "git commit -q -m \"chore: ignore Python and pytest caches · writes 1 file", "balloon": null, "tests": null, @@ -72,7 +73,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -94,7 +95,7 @@ "ok": null, "story": "Done in 3 minutes.", "director": "turn_duration · 3 min 13 s · 13 tool calls · 3 files changed", - "balloon": "I built wcount.py and test_wcount.py.", + "balloon": "I built wcount.py and test_wcount.py. With a missing file plus a readable one, the total row didn't appear because the code counted files that could be read rather than files that were named.", "tests": null, "roundsOk": null, "events": 0 diff --git a/test/fixtures/story-corpus/question-tests-first-push.expected.json b/test/fixtures/story-corpus/question-tests-first-push.expected.json index 0f16595f..2db19109 100644 --- a/test/fixtures/story-corpus/question-tests-first-push.expected.json +++ b/test/fixtures/story-corpus/question-tests-first-push.expected.json @@ -11,6 +11,7 @@ "removed": 2, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -45,7 +46,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -55,7 +56,7 @@ { "kind": "loop", "ok": true, - "story": "Fix and test: the first run fails, the second passes.", + "story": "Test first: the first run fails as planned, the second passes.", "director": "2 checks · 4 edits · +151 −1 · last: ok", "balloon": "All 19 test cases pass (13 test functions), and ruff check and ruff format both come back clean.", "tests": { @@ -114,7 +115,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -126,7 +127,7 @@ "ok": null, "story": "Done in 5 minutes.", "director": "turn_duration · 4 min 57 s · 20 tool calls · 6 files changed", - "balloon": "I've built the kebab slug tool in this folder, committed it, and pushed it to the new bare repo remote.git.", + "balloon": "I've built the kebab slug tool in this folder, committed it, and pushed it to the new bare repo remote.git. Not tested: the README says pip install . gives you a kebab command.", "tests": null, "roundsOk": null, "events": 0 diff --git a/test/fixtures/story-corpus/teammates-then-interrupt.expected.json b/test/fixtures/story-corpus/teammates-then-interrupt.expected.json index 6d342f89..7e674072 100644 --- a/test/fixtures/story-corpus/teammates-then-interrupt.expected.json +++ b/test/fixtures/story-corpus/teammates-then-interrupt.expected.json @@ -11,6 +11,7 @@ "removed": 0, "helpers": 2, "helpersRunning": 0, + "finishedLater": 2, "panels": [ { "kind": "mission", @@ -35,7 +36,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -45,7 +46,7 @@ { "kind": "save", "ok": true, - "story": "Claude writes 2 files and saves the work: \"feat: add package.json and index.js with add and slugify\".", + "story": "Claude writes 2 files and saves the work: \"add package.json and index.js with add and slugify\".", "director": "git commit -q -m \"feat: add package.json and index.js with… · writes 2 files", "balloon": null, "tests": null, @@ -78,7 +79,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"test: add node:test suite for add and slugify\".", + "story": "Claude saves the work: \"add node:test suite for add and slugify\".", "director": "git commit -q -m \"test: add node:test suite for add and slu…", "balloon": null, "tests": null, @@ -88,9 +89,9 @@ { "kind": "complete", "ok": null, - "story": "Done in 2 minutes.", + "story": "Claude's part is done in 2 minutes. Its 2 helpers finished later and reported back.", "director": "stop_hook_summary · 1 min 44 s · 8 tool calls · 3 files changed · 2 helpers", - "balloon": "I've set up the project, started both helpers, and the tests pass: node --test index.test.mjs ran 8 tests, 8 passed, 0 failed.", + "balloon": "I've set up the project, started both helpers, and the tests pass: node --test index.test.mjs ran 8 tests, 8 passed, 0 failed. No linter is set up in this project, so I didn't run one.", "tests": null, "roundsOk": null, "events": 0 @@ -109,6 +110,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -123,7 +125,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude reads 1 file to find its way around.", + "story": "Claude checks CONTRIBUTING.md.", "director": "Bash · ls -la && git status --short && cat CONTRIBUTING.md", "balloon": null, "tests": null, @@ -133,7 +135,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"docs: add CONTRIBUTING.md\".", + "story": "Claude saves the work: \"add CONTRIBUTING.md\".", "director": "git commit -q -m \"docs: add CONTRIBUTING.md", "balloon": null, "tests": null, @@ -145,7 +147,7 @@ "ok": null, "story": "Done in 15 seconds.", "director": "stop_hook_summary · 15 s · 2 tool calls", - "balloon": "The CONTRIBUTING.md helper has finished, and I've committed its file (docs: add CONTRIBUTING.md).", + "balloon": "The CONTRIBUTING.md helper has finished, and I've committed its file (docs: add CONTRIBUTING.md). The helper reported that index.test.mjs and README.md didn't exist yet. That's out of date: it must have looked before I wrote the tests, and both files are there now. README.md is already on disk, but its helper hasn't reported back yet, so I haven't committed it in case it's still being written.", "tests": null, "roundsOk": null, "events": 0 @@ -164,6 +166,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -178,7 +181,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude reads 1 file to find its way around.", + "story": "Claude checks README.md.", "director": "Bash · cat README.md && ls index.test.mjs", "balloon": null, "tests": null, @@ -203,7 +206,7 @@ "ok": null, "story": "Done in 16 seconds.", "director": "stop_hook_summary · 16 s · 2 tool calls", - "balloon": "The project is done: both helpers finished, and I checked their files and committed them.", + "balloon": "The project is done: both helpers finished, and I checked their files and committed them. Both helpers warned that index.test.mjs didn't exist, but they had looked before I created it, so the README's Testing section is correct.", "tests": null, "roundsOk": null, "events": 0 @@ -222,6 +225,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -236,7 +240,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -259,7 +263,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"refactor: split index.js into lib/ modules behind an index.…\".", + "story": "Claude saves the work: \"split index.js into lib/ modules behind an index.js barrel\".", "director": "git commit -q -m \"refactor: split index.js into lib/ module…", "balloon": null, "tests": null, @@ -282,7 +286,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"docs: add full JSDoc to every function and module\".", + "story": "Claude saves the work: \"add full JSDoc to every function and module\".", "director": "git commit -q -m \"docs: add full JSDoc to every function an…", "balloon": null, "tests": null, @@ -313,6 +317,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -339,7 +344,7 @@ "ok": null, "story": "Nothing Claude tried worked, so the job is not done.", "director": "stop_hook_summary · 6 s · 1 tool call", - "balloon": "Not done. The harness blocked sleep 65: a sleep on its own in the foreground isn't allowed.", + "balloon": "Not done. The harness blocked sleep 65: a sleep on its own in the foreground isn't allowed. I didn't try to get around the block, for example by chaining shorter sleeps or running it in the background.", "tests": null, "roundsOk": null, "events": 0 diff --git a/test/fixtures/story-corpus/text-plan-dry-run-helper.expected.json b/test/fixtures/story-corpus/text-plan-dry-run-helper.expected.json index 3d9c8318..ee5edd81 100644 --- a/test/fixtures/story-corpus/text-plan-dry-run-helper.expected.json +++ b/test/fixtures/story-corpus/text-plan-dry-run-helper.expected.json @@ -11,6 +11,7 @@ "removed": 0, "helpers": 1, "helpersRunning": 0, + "finishedLater": 1, "panels": [ { "kind": "mission", @@ -45,7 +46,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -98,9 +99,9 @@ { "kind": "complete", "ok": null, - "story": "Done in 3 minutes.", + "story": "Claude's part is done in 3 minutes. Its helper finished later and reported back.", "director": "turn_duration · 2 min 44 s · 11 tool calls · 5 files changed · 1 helper", - "balloon": "I've built csvstat.py, a command-line tool that prints a CSV's row count and the min, max and mean of each column.", + "balloon": "I've built csvstat.py, a command-line tool that prints a CSV's row count and the min, max and mean of each column. A background helper is writing README.md; I didn't wait for it, so it isn't confirmed yet. The population mean shows a problem, described below.", "tests": null, "roundsOk": null, "events": 0 @@ -119,6 +120,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -145,7 +147,7 @@ "ok": null, "story": "Done in 9 seconds.", "director": "turn_duration · 9 s · 1 tool call", - "balloon": "The background helper has finished README.md, and I've checked its work.", + "balloon": "The background helper has finished README.md, and I've checked its work. Rounding: the README now explains that numbers are shown to 6 significant digits, so the population mean of 214574.2 appears as 214574. That note makes the current behaviour clear, but it doesn't fix it.", "tests": null, "roundsOk": null, "events": 0 diff --git a/test/fixtures/story-corpus/web-page-node-test.expected.json b/test/fixtures/story-corpus/web-page-node-test.expected.json index e1e51e8d..27bd343f 100644 --- a/test/fixtures/story-corpus/web-page-node-test.expected.json +++ b/test/fixtures/story-corpus/web-page-node-test.expected.json @@ -11,6 +11,7 @@ "removed": 0, "helpers": 0, "helpersRunning": 0, + "finishedLater": 0, "panels": [ { "kind": "mission", @@ -25,7 +26,7 @@ { "kind": "investigate", "ok": null, - "story": "Claude reads 1 file to find its way around.", + "story": "Claude reads CLAUDE.md.", "director": "Bash ×2 · ls -la && git status 2>&1 | head; git log --oneline 2>&1 |…", "balloon": null, "tests": null, @@ -45,7 +46,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -78,7 +79,7 @@ { "kind": "save", "ok": true, - "story": "Claude saves the work: \"feat: add pomodoro timer with tests\".", + "story": "Claude saves the work: \"add pomodoro timer with tests\".", "director": "git commit -q -m \"feat: add pomodoro timer with tests\" -m \"…", "balloon": null, "tests": null, @@ -88,7 +89,7 @@ { "kind": "backup", "ok": null, - "story": "Claude asks the advisor for a second opinion.", + "story": "Claude asks a stronger AI model for a second opinion.", "director": "Advisor · a second opinion", "balloon": null, "tests": null, @@ -100,7 +101,7 @@ "ok": null, "story": "Done in 3 minutes.", "director": "turn_duration · 2 min 39 s · 10 tool calls · 3 files changed", - "balloon": "I built the pomodoro timer.", + "balloon": "I built the pomodoro timer. Opening the page: it won't work if you double-click index.html. The page loads timer.js as a module, and Chrome and Firefox block that for local files, so you'd get a timer that never moves.", "tests": null, "roundsOk": null, "events": 0 diff --git a/test/story-corpus.test.ts b/test/story-corpus.test.ts index 806f7639..f8cf4138 100644 --- a/test/story-corpus.test.ts +++ b/test/story-corpus.test.ts @@ -39,6 +39,7 @@ function pagesOf(name: string) { removed: p.removed, helpers: p.helpers, helpersRunning: p.helpersRunning, + finishedLater: p.finishedLater ?? 0, panels: p.panels.map((x) => ({ kind: x.kind, ok: x.ok ?? null, @@ -71,7 +72,9 @@ describe('Story Mode on real sessions', () => { it('a fix, a failing test (exit code hidden by a pipe), a fix and a pass fold into one loop', () => { // Three runs in two rounds: the failing test asked for, then ruff, then a pass (the marks: ✗ ✗ ✓). expect(panel('python-cli-fix-loop', 0, 'loop')).toMatchObject({ - story: 'Fix and test, 2 rounds: 2 runs fail, then it passes.', + // The failing test was a real bug, the second failure only ruff: said apart, with the 4 commits made inside. + story: + 'Fix and test, 2 rounds: a test run fails, then another check, then everything passes. Claude saves the work in 4 commits along the way.', roundsOk: [false, false, true], }); }); @@ -123,13 +126,13 @@ describe('Story Mode on real sessions', () => { it('tests that do not load, a commit then a push in its own command, a push with a failing step after it', () => { const [first, second, third] = pagesOf('errors-push-append'); - expect(first.panels.find((p) => p.kind === 'check')?.story).toBe("The tests don't even load."); + expect(first.panels.find((p) => p.kind === 'check')?.story).toBe("The tests don't even load, as planned."); expect(second.panels.find((p) => p.kind === 'loop')).toMatchObject({ - story: 'Fix and test: the first run fails, the second passes.', + story: 'Test first: the first run fails as planned, the second passes.', roundsOk: [false, true], }); expect(second.panels.find((p) => p.kind === 'save')?.story).toBe( - 'Claude saves the work and uploads it: "feat: add calc_stats.mode with smallest-value tie-breaking".' + 'Claude saves the work and uploads it: "add calc_stats.mode with smallest-value tie-breaking".' ); expect(third.panels.find((p) => p.kind === 'save')?.story).toBe( 'Claude saves the work and uploads it, but a step after that fails.' @@ -152,6 +155,33 @@ describe('Story Mode on real sessions', () => { expect(pages[3].panels.map((p) => p.story)).toContain('Claude changes 3 files.'); }); + it("Claude's last words keep what a reader must not lose (round 8, the cold readers)", () => { + const last = (name: string, turn: number) => pagesOf(name)[turn].panels.at(-1)!; + // An open problem, said further into the final message than its first sentence. + expect(last('web-page-node-test', 0).balloon).toMatch(/won't work if you double-click index\.html/); + expect(last('text-plan-dry-run-helper', 1).balloon).toMatch(/doesn't fix it/); + // What you expected, and what you asked for mid-turn. + expect(last('errors-push-append', 2).balloon).toMatch(/As you expected, git describe --tags then failed/); + expect(last('long-turn-prompt-added', 0).balloon).toMatch(/--csv is part of the CLI commit/); + // A helper's warning that turned out to be out of date, with what it was about. + expect(last('teammates-then-interrupt', 1).balloon).toMatch(/didn't exist yet\. That's out of date/); + expect(last('teammates-then-interrupt', 2).balloon).toMatch(/they had looked before I created it/); + }); + + it('helpers still out when a turn ended finished later, and the page says so', () => { + expect(pagesOf('background-helpers-reports')[0].panels.at(-1)!.story).toBe( + "Claude's part is done in 36 seconds. Its 2 helpers finished later and reported back." + ); + expect(pagesOf('background-helpers-reports')[0].finishedLater).toBe(2); + }); + + it('a lookup of tools reads as getting them ready, without their names', () => { + const panel = pagesOf('compact-then-plan-mode') + .flatMap((p) => p.panels) + .find((p) => p.story === 'Claude gets the tools it needs ready.'); + expect(panel).toBeTruthy(); + }); + it('a blocked command ends a turn where nothing worked, without a celebration', () => { const last = pagesOf('teammates-then-interrupt')[4]; expect(last.panels.map((p) => p.story)).toEqual([ diff --git a/test/story-mode-ui.test.ts b/test/story-mode-ui.test.ts index 05ce50e7..2d884cd2 100644 --- a/test/story-mode-ui.test.ts +++ b/test/story-mode-ui.test.ts @@ -159,7 +159,7 @@ describe('story-mode.js overlay', () => { expect($('.story-panel[data-kind="mission"]')!.getAttribute('aria-label')).toBe('Panel 1 of 7'); expect($('.story-panel[data-kind="mission"] .story-balloon')!.className).toContain('story-balloon--you'); expect($('.story-panel[data-kind="investigate"] .story-cap-story')!.textContent).toBe( - 'Claude reads 2 files to find its way around.' + 'Claude reads CheckoutButton.tsx and banner.css.' ); expect($('.story-splash-title')!.textContent).toBe('Mission complete!'); expect($$('.story-turns option').map((o) => o.textContent)).toEqual([ @@ -781,7 +781,7 @@ describe('story-mode.js overlay', () => { sizeAndDraw(); expect($('.story-panel[data-kind="complete"] .story-chips')!.textContent).toContain('1 helper, 1 still working'); // Work still out is not a finished mission. - expect($('.story-panel[data-kind="complete"] .story-splash-title')!.textContent).toBe('To be continued…'); + expect($('.story-panel[data-kind="complete"] .story-splash-title')!.textContent).toBe('Done for now'); $('.story-panel[data-kind="backup"] .story-hit')!.click(); await settle(); expect($('.story-drawer-sub')!.textContent).toContain('in the background'); @@ -851,6 +851,28 @@ describe('story-mode.js overlay', () => { expect($$('.story-ln--add').map((l) => l.textContent)).toEqual(['+more']); }); + it('the final panel counts the commits of the whole turn, and says when the work was uploaded', async () => { + boot( + storyOf([ + prompt('ship it'), + ...edit('e1', '/work/demo-shop/a.py', ['+a = 1']), + ...bash('b1', 'pytest -q && git commit -qam "fix: a"', 'Test and commit', '3 passed in 0.1s'), + ...bash( + 'b2', + 'git commit -qam "docs: b" && git push', + 'Commit and push', + 'To ./origin.git\n 1a2b3c4..5d6e7f8 main -> main' + ), + end(1000), + ]) + ); + app.openStoryMode(SID); + await settle(); + expect($$('.story-panel[data-kind="complete"] .story-chips li').map((li) => li.textContent)).toContain( + '2 commits, uploaded' + ); + }); + it('a chip that opened the drawer takes focus back, even when the page was redrawn meanwhile', async () => { boot(storyOf(demoTurn())); app.openStoryMode(SID); diff --git a/test/story-mode.test.ts b/test/story-mode.test.ts index b2b266cb..7d33b527 100644 --- a/test/story-mode.test.ts +++ b/test/story-mode.test.ts @@ -797,14 +797,14 @@ describe('buildStoryPage', () => { const page = pageOf(demoTurn()); const byKind = Object.fromEntries(page.panels.map((p) => [p.kind, p])); expect(byKind.mission.caption.story).toBe('You give Claude a job.'); - expect(byKind.investigate.caption.story).toBe('Claude reads 2 files to find its way around.'); + expect(byKind.investigate.caption.story).toBe('Claude reads CheckoutButton.tsx and banner.css.'); expect(byKind.investigate.caption.director).toBe('Read ×2 · Bash'); expect(byKind.plan.caption.story).toBe('Claude writes a plan: 2 steps.'); expect(byKind.backup.caption.story).toBe('Claude calls in 1 helper.'); expect(byKind.backup.caption.director).toContain('3 calls inside'); expect(byKind.loop).toMatchObject({ rounds: 2, ok: true, added: 2, removed: 1 }); expect(byKind.loop.caption.story).toBe('Fix and test: the first run fails, the second passes.'); - expect(byKind.save.caption.story).toBe('Claude saves the work: "fix: banner".'); + expect(byKind.save.caption.story).toBe('Claude saves the work: "banner".'); expect(byKind.complete.caption.story).toBe('Done in 7 minutes.'); expect(byKind.complete.caption.director).toContain('7 min 12 s'); }); @@ -969,7 +969,7 @@ describe('buildStoryPage', () => { ...bash('b2', 'git push origin main', 'Push', 'To ./origin.git\n f80bb31..3e0fb80 main -> main'), end(1000), ]) - ).toBe('Claude saves the work and uploads it: "feat: add mode()".'); + ).toBe('Claude saves the work and uploads it: "add mode()".'); expect(save([prompt('push'), ...bash('b1', 'git push', 'Push', 'Everything up-to-date'), end(1000)])).toBe( 'Claude uploads the work.' ); @@ -1019,6 +1019,21 @@ describe('buildStoryPage', () => { expect(page.panels.at(-1)!.kind).toBe('complete'); }); + it('the closing balloon keeps an open problem from far into the message, never a table row or a question', () => { + const long = [ + 'I built the csvstat tool for your CSV files.', + 'It reads the file and prints the counts for each column, one line per column. '.repeat(4), + '| column | state |', + '| amount | not yet rounded |', + 'Want me to add a --json flag?', + 'One known issue: big numbers are shown rounded.', + ].join('\n'); + const balloon = pageOf([prompt('build it'), said(long), end(1000)]).panels.at(-1)!.balloon!.text; + expect(balloon).toBe( + 'I built the csvstat tool for your CSV files. One known issue: big numbers are shown rounded.' + ); + }); + it('a helper messaged twice reads as one finished run, not as superseded', () => { const r = createStoryReducer({ sidechain: true }); const lines = [prompt('write it'), said('Done.'), prompt('also the tests'), said('Done too.')].map((l) => ({ @@ -1158,7 +1173,9 @@ describe('buildStoryPage', () => { ]); const loop = page.panels.find((p) => p.kind === 'loop')!; expect(loop.roundsOk).toEqual([true, false, true]); - expect(loop.caption.story).toBe('Build and test, 2 rounds. A check fails once, then everything passes.'); + expect(loop.caption.story).toBe( + 'Build and test, 2 rounds. The tests pass every time; another check fails once, then passes.' + ); }); it('a live loop marks the runs that finished, and the one still going as null', () => { @@ -1220,7 +1237,7 @@ describe('buildStoryPage', () => { end(1000), ]); const backup = page.panels.find((p) => p.kind === 'backup')!; - expect(backup.caption.story).toBe('Claude asks the advisor for a second opinion.'); + expect(backup.caption.story).toBe('Claude asks a stronger AI model for a second opinion.'); expect(page.events[1]).toMatchObject({ tool: 'Advisor', ok: true, helper: { name: 'advisor' } }); expect(page.events[1].helper!.result).toMatch(/encrypted/); expect(page.helpers).toBe(0);