Files
Codeman/test/story-corpus.test.ts
T
Codeman maintainer 6a7d8393dc fix(story-mode): what the cold readers believed that was not so (review round 8)
Two agents read only the comic of the eleven recorded sessions (one who
has never written code, one engineering manager skimming), wrote down
what they believed, then compared with the transcripts. Both were misled
in the same places, and the cause was the same: the final panel kept
Claude's first sentence or two, and what a reader must not lose sat
further into its last message.

- Claude's last words now keep key sentences from anywhere in its final
  message (read in full, past the label's clip): an open problem or a
  quirk, something not yet done, checked or confirmed, work still out,
  what you asked for or expected, a correction of an earlier warning
  (with the sentence it refers to). A bare "I didn't" counts only when
  nothing stronger is there; table rows, list intros and questions never.
  "I built the pomodoro timer. Opening the page: it won't work if you
  double-click index.html…"; "…That note makes the current behaviour
  clear, but it doesn't fix it."; "As you expected, git describe --tags
  then failed because the repo has no tags."
- A turn that ended with helpers or a background command still out says
  so: "Done for now", and "Claude's part is done in 36 seconds. Its 2
  helpers finished later and reported back." (`finishedLater`).
- A failure you asked for, or that Claude said it expected, reads as the
  plan: "Test first: the first run fails as planned, the second
  passes.", "The tests don't even load, as planned." A loop that failed
  once on a test and once on another check says both, and the commits
  made inside a loop are part of its story; the final panel counts the
  turn's commits ("5 commits, uploaded").
- One or two files read are named ("Claude checks README.md." in a
  turn a report started); a tool lookup reads "Claude gets the tools it
  needs ready." again (it was overwritten) without its query; the
  advisor is "a stronger AI model", and one that answered with an error
  "gets no answer"; a commit message loses its `feat:` prefix in the
  plain caption.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-18 15:17:54 +02:00

194 lines
9.2 KiB
TypeScript

/**
* @fileoverview Story Mode against real sessions. Each fixture in test/fixtures/story-corpus/ is the
* transcript of a real Claude Code worker that a review agent had build a small program while Story
* Mode drew it (2026-10-18), cut down to the lines and fields the reducer reads, with every path,
* user name and email rewritten and file contents of Reads left out. The expected file beside each
* pins every page: panel kinds, captions, balloons, counts. A change that alters one fails here, and
* the diff shows exactly which story changed. After an intended change, regenerate with
* `STORY_CORPUS_UPDATE=1 npm test -- test/story-corpus.test.ts` and read the diff before committing.
*/
import { readdirSync, readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import { buildStoryPage, buildStoryTurns, createStoryReducer } from '../src/story-mode.js';
const DIR = join(import.meta.dirname, 'fixtures', 'story-corpus');
const NAMES = readdirSync(DIR)
.filter((f) => f.endsWith('.jsonl'))
.map((f) => f.slice(0, -'.jsonl'.length))
.sort();
/** What a reader sees of each turn: the page's facts and every panel's kind, words and counts. */
function pagesOf(name: string) {
const reducer = createStoryReducer();
const entries = readFileSync(join(DIR, `${name}.jsonl`), 'utf8')
.split('\n')
.filter(Boolean)
.flatMap((line) => reducer.reduce(JSON.parse(line)));
return buildStoryTurns(entries).map((turn) => {
const p = buildStoryPage(turn);
return {
prompt: p.prompt.slice(0, 100),
origin: p.origin,
complete: p.complete,
interrupted: p.interrupted,
durationMs: p.durationMs ?? null,
toolCalls: p.toolCalls,
filesChanged: p.filesChanged,
added: p.added,
removed: p.removed,
helpers: p.helpers,
helpersRunning: p.helpersRunning,
finishedLater: p.finishedLater ?? 0,
panels: p.panels.map((x) => ({
kind: x.kind,
ok: x.ok ?? null,
story: x.caption.story,
director: x.caption.director,
balloon: x.balloon?.text ?? null,
tests: x.tests ?? null,
roundsOk: x.roundsOk ?? null,
events: x.events.length,
})),
};
});
}
describe('Story Mode on real sessions', () => {
it('has the corpus', () => {
expect(NAMES.length).toBeGreaterThanOrEqual(11);
});
it.each(NAMES)('%s draws as it did when it was checked by hand', (name) => {
const pages = pagesOf(name);
const file = join(DIR, `${name}.expected.json`);
if (process.env.STORY_CORPUS_UPDATE === '1') writeFileSync(file, `${JSON.stringify(pages, null, 2)}\n`);
expect(pages).toEqual(JSON.parse(readFileSync(file, 'utf8')));
});
// The beats the review rounds were about, stated outright (the expected files hold everything else).
const panel = (name: string, turn: number, kind: string) => pagesOf(name)[turn].panels.find((p) => p.kind === kind);
it('a fix, a failing test (exit code hidden by a pipe), a fix and a pass fold into one loop', () => {
// Three runs in two rounds: the failing test asked for, then ruff, then a pass (the marks: ✗ ✗ ✓).
expect(panel('python-cli-fix-loop', 0, 'loop')).toMatchObject({
// The failing test was a real bug, the second failure only ruff: said apart, with the 4 commits made inside.
story:
'Fix and test, 2 rounds: a test run fails, then another check, then everything passes. Claude saves the work in 4 commits along the way.',
roundsOk: [false, false, true],
});
});
it('a prompt typed mid-turn (a queued_command attachment) is drawn where Claude took it in', () => {
expect(
panel('long-turn-prompt-added', 0, 'mission') && pagesOf('long-turn-prompt-added')[0].panels[4]
).toMatchObject({
kind: 'mission',
story: 'You add something while Claude works.',
});
});
it("teammates' reports are their own turns, and every turn ends without turn_duration", () => {
const pages = pagesOf('teammates-then-interrupt');
expect(pages.map((p) => [p.origin, p.complete || p.interrupted])).toEqual([
['human', true],
['task', true],
['task', true],
['human', true],
['human', true],
]);
expect(pages[3].panels.at(-1)?.story).toBe('You stop Claude here.');
});
it('a commit at the end of a test one-liner is kept, and a background helper reports back in 9 s', () => {
expect(panel('text-plan-dry-run-helper', 0, 'check')?.story).toBe(
'Claude writes 2 files; the tests pass, and Claude saves the work.'
);
expect(panel('text-plan-dry-run-helper', 0, 'plan')?.story).toBe('Claude writes a plan: 4 steps.');
expect(pagesOf('text-plan-dry-run-helper')[1]).toMatchObject({ origin: 'task', durationMs: 9008 });
});
it('/compact is one turn of its own, and an approved plan says so', () => {
const pages = pagesOf('compact-then-plan-mode');
expect(pages[1].panels.map((p) => p.kind)).toEqual(['mission', 'compact', 'complete']);
expect(pages[1].panels[0].story).toBe('You run /compact.');
expect(panel('compact-then-plan-mode', 2, 'ask')?.story).toBe('Claude shows you its plan, and you approve it.');
expect(pages[2].filesChanged).toBe(2);
});
it('a push that went through, with a step after it that failed, says both', () => {
expect(panel('question-tests-first-push', 0, 'save')).toMatchObject({
ok: true,
// One commit, then a push in a command of its own: one commit, not two.
story: 'Claude saves the work and uploads it, but a step after that fails.',
});
});
it('tests that do not load, a commit then a push in its own command, a push with a failing step after it', () => {
const [first, second, third] = pagesOf('errors-push-append');
expect(first.panels.find((p) => p.kind === 'check')?.story).toBe("The tests don't even load, as planned.");
expect(second.panels.find((p) => p.kind === 'loop')).toMatchObject({
story: 'Test first: the first run fails as planned, the second passes.',
roundsOk: [false, true],
});
expect(second.panels.find((p) => p.kind === 'save')?.story).toBe(
'Claude saves the work and uploads it: "add calc_stats.mode with smallest-value tie-breaking".'
);
expect(third.panels.find((p) => p.kind === 'save')?.story).toBe(
'Claude saves the work and uploads it, but a step after that fails.'
);
});
it('helpers and a command in the background: their reports are turns of their own, counted where they started', () => {
const pages = pagesOf('background-helpers-reports');
expect(pages[0].panels.find((p) => p.kind === 'backup')?.story).toBe(
'Claude sends 2 helpers off to work in the background.'
);
expect(pages[0].helpers).toBe(2);
for (const report of [pages[1], pages[2], pages[4], pages[5]]) {
expect(report.origin).toBe('task');
expect(report.helpers).toBe(0);
expect(report.panels[0].story).toBe('A background job reports back.');
}
expect(pages[3].panels.map((p) => p.story)).toContain('The tests run in the background and pass.');
// A python script that rewrote README.md counts as a change, beside the two files written.
expect(pages[3].panels.map((p) => p.story)).toContain('Claude changes 3 files.');
});
it("Claude's last words keep what a reader must not lose (round 8, the cold readers)", () => {
const last = (name: string, turn: number) => pagesOf(name)[turn].panels.at(-1)!;
// An open problem, said further into the final message than its first sentence.
expect(last('web-page-node-test', 0).balloon).toMatch(/won't work if you double-click index\.html/);
expect(last('text-plan-dry-run-helper', 1).balloon).toMatch(/doesn't fix it/);
// What you expected, and what you asked for mid-turn.
expect(last('errors-push-append', 2).balloon).toMatch(/As you expected, git describe --tags then failed/);
expect(last('long-turn-prompt-added', 0).balloon).toMatch(/--csv is part of the CLI commit/);
// A helper's warning that turned out to be out of date, with what it was about.
expect(last('teammates-then-interrupt', 1).balloon).toMatch(/didn't exist yet\. That's out of date/);
expect(last('teammates-then-interrupt', 2).balloon).toMatch(/they had looked before I created it/);
});
it('helpers still out when a turn ended finished later, and the page says so', () => {
expect(pagesOf('background-helpers-reports')[0].panels.at(-1)!.story).toBe(
"Claude's part is done in 36 seconds. Its 2 helpers finished later and reported back."
);
expect(pagesOf('background-helpers-reports')[0].finishedLater).toBe(2);
});
it('a lookup of tools reads as getting them ready, without their names', () => {
const panel = pagesOf('compact-then-plan-mode')
.flatMap((p) => p.panels)
.find((p) => p.story === 'Claude gets the tools it needs ready.');
expect(panel).toBeTruthy();
});
it('a blocked command ends a turn where nothing worked, without a celebration', () => {
const last = pagesOf('teammates-then-interrupt')[4];
expect(last.panels.map((p) => p.story)).toEqual([
'You give Claude a job.',
'Claude tries a command, and it is blocked.',
'Nothing Claude tried worked, so the job is not done.',
]);
});
});