mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-16 03:09:33 +01:00
Nine transcripts from the review rounds, each a real Claude Code worker building a small program (a Python CLI with a fix loop, a web page with node:test, a bash script with a background helper, a long turn with a prompt added mid-turn, teammates then an interrupt, a question then tests first then a push, a text plan with a scratch dry run, /compact then plan mode, a test failed on purpose). Each is cut down to the lines and fields the reducer reads, with every path, user name and email rewritten and the file contents of Reads left out; replaying them before and after gives the same pages. The expected file beside each pins every page (panel kinds, captions, balloons, counts), so a change that alters a story fails with a diff of exactly what changed; STORY_CORPUS_UPDATE=1 regenerates them. The beats the reviews were about are also stated outright. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
116 lines
4.9 KiB
TypeScript
116 lines
4.9 KiB
TypeScript
/**
|
|
* @fileoverview Story Mode against real sessions. Each fixture in test/fixtures/story-corpus/ is the
|
|
* transcript of a real Claude Code worker that a review agent had build a small program while Story
|
|
* Mode drew it (2026-10-18), cut down to the lines and fields the reducer reads, with every path,
|
|
* user name and email rewritten and file contents of Reads left out. The expected file beside each
|
|
* pins every page: panel kinds, captions, balloons, counts. A change that alters one fails here, and
|
|
* the diff shows exactly which story changed. After an intended change, regenerate with
|
|
* `STORY_CORPUS_UPDATE=1 npm test -- test/story-corpus.test.ts` and read the diff before committing.
|
|
*/
|
|
import { readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { describe, expect, it } from 'vitest';
|
|
import { buildStoryPage, buildStoryTurns, createStoryReducer } from '../src/story-mode.js';
|
|
|
|
const DIR = join(import.meta.dirname, 'fixtures', 'story-corpus');
|
|
const NAMES = readdirSync(DIR)
|
|
.filter((f) => f.endsWith('.jsonl'))
|
|
.map((f) => f.slice(0, -'.jsonl'.length))
|
|
.sort();
|
|
|
|
/** What a reader sees of each turn: the page's facts and every panel's kind, words and counts. */
|
|
function pagesOf(name: string) {
|
|
const reducer = createStoryReducer();
|
|
const entries = readFileSync(join(DIR, `${name}.jsonl`), 'utf8')
|
|
.split('\n')
|
|
.filter(Boolean)
|
|
.flatMap((line) => reducer.reduce(JSON.parse(line)));
|
|
return buildStoryTurns(entries).map((turn) => {
|
|
const p = buildStoryPage(turn);
|
|
return {
|
|
prompt: p.prompt.slice(0, 100),
|
|
origin: p.origin,
|
|
complete: p.complete,
|
|
interrupted: p.interrupted,
|
|
durationMs: p.durationMs ?? null,
|
|
toolCalls: p.toolCalls,
|
|
filesChanged: p.filesChanged,
|
|
added: p.added,
|
|
removed: p.removed,
|
|
helpers: p.helpers,
|
|
helpersRunning: p.helpersRunning,
|
|
panels: p.panels.map((x) => ({
|
|
kind: x.kind,
|
|
ok: x.ok ?? null,
|
|
story: x.caption.story,
|
|
director: x.caption.director,
|
|
balloon: x.balloon?.text ?? null,
|
|
tests: x.tests ?? null,
|
|
events: x.events.length,
|
|
})),
|
|
};
|
|
});
|
|
}
|
|
|
|
describe('Story Mode on real sessions', () => {
|
|
it('has the corpus', () => {
|
|
expect(NAMES.length).toBeGreaterThanOrEqual(9);
|
|
});
|
|
|
|
it.each(NAMES)('%s draws as it did when it was checked by hand', (name) => {
|
|
const pages = pagesOf(name);
|
|
const file = join(DIR, `${name}.expected.json`);
|
|
if (process.env.STORY_CORPUS_UPDATE === '1') writeFileSync(file, `${JSON.stringify(pages, null, 2)}\n`);
|
|
expect(pages).toEqual(JSON.parse(readFileSync(file, 'utf8')));
|
|
});
|
|
|
|
// The beats the review rounds were about, stated outright (the expected files hold everything else).
|
|
const panel = (name: string, turn: number, kind: string) => pagesOf(name)[turn].panels.find((p) => p.kind === kind);
|
|
|
|
it('a fix, a failing test (exit code hidden by a pipe), a fix and a pass fold into one loop', () => {
|
|
expect(panel('python-cli-fix-loop', 0, 'loop')?.story).toBe('Fix and test, 2 rounds. It passes in the end.');
|
|
});
|
|
|
|
it('a prompt typed mid-turn (a queued_command attachment) is drawn where Claude took it in', () => {
|
|
expect(
|
|
panel('long-turn-prompt-added', 0, 'mission') && pagesOf('long-turn-prompt-added')[0].panels[4]
|
|
).toMatchObject({
|
|
kind: 'mission',
|
|
story: 'You add something while Claude works.',
|
|
});
|
|
});
|
|
|
|
it("teammates' reports are their own turns, and every turn ends without turn_duration", () => {
|
|
const pages = pagesOf('teammates-then-interrupt');
|
|
expect(pages.map((p) => [p.origin, p.complete || p.interrupted])).toEqual([
|
|
['human', true],
|
|
['task', true],
|
|
['task', true],
|
|
['human', true],
|
|
['human', true],
|
|
]);
|
|
expect(pages[3].panels.at(-1)?.story).toBe('You stop Claude here.');
|
|
});
|
|
|
|
it('a commit at the end of a test one-liner is kept, and a background helper reports back in 9 s', () => {
|
|
expect(panel('text-plan-dry-run-helper', 0, 'check')?.story).toBe('The tests pass, and Claude saves the work.');
|
|
expect(panel('text-plan-dry-run-helper', 0, 'plan')?.story).toBe('Claude writes a plan: 4 steps.');
|
|
expect(pagesOf('text-plan-dry-run-helper')[1]).toMatchObject({ origin: 'task', durationMs: 9008 });
|
|
});
|
|
|
|
it('/compact is one turn of its own, and an approved plan says so', () => {
|
|
const pages = pagesOf('compact-then-plan-mode');
|
|
expect(pages[1].panels.map((p) => p.kind)).toEqual(['mission', 'compact', 'complete']);
|
|
expect(pages[1].panels[0].story).toBe('You run /compact.');
|
|
expect(panel('compact-then-plan-mode', 2, 'ask')?.story).toBe('Claude shows you its plan, and you approve it.');
|
|
expect(pages[2].filesChanged).toBe(2);
|
|
});
|
|
|
|
it('a push whose command failed says so', () => {
|
|
expect(panel('question-tests-first-push', 0, 'save')).toMatchObject({
|
|
ok: false,
|
|
story: 'Claude tries to send the work up, and the command fails.',
|
|
});
|
|
});
|
|
});
|