From 803e7c7f852a8b668eca5dc93bba2ea402b737f8 Mon Sep 17 00:00:00 2001 From: Aamer Akhter Date: Sat, 24 Oct 2026 23:00:49 -0400 Subject: [PATCH] refactor(codex): share the rollout format between the two Codex history readers The Past Sessions scanner and the offline history reader each parsed Codex rollouts with their own shape helpers, their own CODEX_HOME resolution and different injected-context rules. Move the store location, the three message shapes and the injected-context rule into codex-rollout-format.ts. The scanner's behaviour is unchanged; the offline reader adopts its rule in the next commit. --- src/codex-rollout-format.ts | 128 ++++++++++++++++++++++++++++++ src/codex-transcript.ts | 69 ++++------------ test/codex-rollout-format.test.ts | 112 ++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 53 deletions(-) create mode 100644 src/codex-rollout-format.ts create mode 100644 test/codex-rollout-format.test.ts diff --git a/src/codex-rollout-format.ts b/src/codex-rollout-format.ts new file mode 100644 index 00000000..068e68bf --- /dev/null +++ b/src/codex-rollout-format.ts @@ -0,0 +1,128 @@ +/** + * @fileoverview Shape helpers for Codex rollout JSONL lines, shared by the two + * readers of `~/.codex/sessions`: the Past Sessions scanner + * (`codex-transcript.ts`, previews only) and the offline history reader + * (`local-history.ts`, whole conversations). + * + * Both readers must agree on three facts, so they live here once: + * + * - where the store is (`$CODEX_HOME/sessions`, else `~/.codex/sessions`); + * - which line shapes carry a user or assistant message; + * - which `response_item` user rows are injected context rather than input. + * + * ## The message shapes + * + * Codex has written conversation text under three shapes: + * + * - `event_msg` / `user_message` and `agent_message`, written by every version + * seen so far. `agent_message` carries a `phase` (`commentary` or + * `final_answer`); `user_message` carries none. + * - `event_msg` / `item_completed` with an `item.type` of `UserMessage` or + * `AgentMessage`, written by newer versions (0.152.1 and later). + * - `response_item` rows with `role: 'user'` or `'assistant'`. These are what + * the model saw, so the user rows mix real input with injected context. + * + * ## The injected-context rule + * + * Codex injects context as `response_item` user rows that start with a Markdown + * heading (`# AGENTS.md instructions for ...`) or an XML-style tag + * (``, ``, ``, + * ``, ``, ``, ``). + * Measured on a store of 2,916 rollouts (codex 0.120 to 0.163), every + * `response_item` user row with either prefix and no event twin was injected; + * the real prompts that start with `#` all have a `user_message` or + * `UserMessage` event twin, which this rule never filters. A fixed list of tag + * names would leak every wrapper Codex adds later (`` + * alone was 1,355 rows on that store), so the rule is the prefix. + * + * @module codex-rollout-format + */ + +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +/** Codex's home directory: `$CODEX_HOME`, else `~/.codex`, exactly as codex resolves it. */ +export function codexHomeDir(): string { + return process.env.CODEX_HOME || join(homedir(), '.codex'); +} + +/** The rollout store under {@link codexHomeDir}. */ +export function codexSessionsRoot(): string { + return join(codexHomeDir(), 'sessions'); +} + +/** + * Flatten Codex message content, which is a string or an array of blocks. + * Only blocks carrying a string `text` contribute (images and skills do not). + * Not trimmed: callers that want a preview trim. + */ +export function codexContentText(content: unknown): string { + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return ''; + return content + .filter( + (b): b is { text: string } => !!b && typeof b === 'object' && typeof (b as { text?: unknown }).text === 'string' + ) + .map((b) => b.text) + .filter(Boolean) + .join('\n'); +} + +/** Injected context rather than typed input. Applies to `response_item` user rows only. */ +export function isCodexInjectedContext(text: string): boolean { + const start = text.trimStart(); + return start.startsWith('#') || start.startsWith('<'); +} + +/** One rollout line's user or assistant message, whichever shape carried it. */ +export interface CodexLineMessage { + role: 'user' | 'assistant'; + /** Untrimmed text, never empty. */ + text: string; + /** `event` = user_message/agent_message, `item` = item_completed, `response` = response_item. */ + shape: 'event' | 'item' | 'response'; + /** Codex's own phase (`commentary` / `final_answer`) when the line carries one. */ + phase?: string; + /** A `response_item` user row: may be injected context, see {@link isCodexInjectedContext}. */ + injectionProne: boolean; +} + +function record(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record) : {}; +} + +function phaseOf(value: unknown): string | undefined { + return typeof value === 'string' && value ? value : undefined; +} + +/** The message a parsed rollout line carries, or null for every other line. */ +export function codexMessageFromLine(entry: unknown): CodexLineMessage | null { + const e = record(entry); + const p = record(e.payload); + let role: CodexLineMessage['role'] | undefined; + let text = ''; + let shape: CodexLineMessage['shape'] = 'event'; + let phase: string | undefined; + if (e.type === 'event_msg') { + if (p.type === 'user_message' || p.type === 'agent_message') { + role = p.type === 'user_message' ? 'user' : 'assistant'; + text = codexContentText(p.message); + phase = phaseOf(p.phase); + } else if (p.type === 'item_completed') { + const item = record(p.item); + if (item.type === 'UserMessage' || item.type === 'AgentMessage') { + role = item.type === 'UserMessage' ? 'user' : 'assistant'; + shape = 'item'; + text = codexContentText(item.content); + phase = phaseOf(item.phase); + } + } + } else if (e.type === 'response_item' && (p.role === 'user' || p.role === 'assistant')) { + role = p.role; + shape = 'response'; + text = codexContentText(p.content); + phase = phaseOf(p.phase); + } + if (!role || !text) return null; + return { role, text, shape, phase, injectionProne: shape === 'response' && role === 'user' }; +} diff --git a/src/codex-transcript.ts b/src/codex-transcript.ts index 9f360ee3..aabc0624 100644 --- a/src/codex-transcript.ts +++ b/src/codex-transcript.ts @@ -37,13 +37,20 @@ * summaries), so they are read only when neither shape above appears, and * the obvious injections are dropped. * + * The shapes, the injected-context rule and the store location live in + * `codex-rollout-format.ts`, shared with the offline reader `local-history.ts`. + * * @module codex-transcript */ import { open, readdir, stat } from 'node:fs/promises'; -import { homedir } from 'node:os'; import { join } from 'node:path'; +import { + codexMessageFromLine, + codexSessionsRoot, + isCodexInjectedContext as isInjectedContext, +} from './codex-rollout-format.js'; import { LRUMap } from './utils/lru-map.js'; /** Covers `session_meta` (~19 KiB) plus the first user message (~69 KiB behind it). */ @@ -134,11 +141,6 @@ function shouldCacheIdentity(identity: RolloutIdentity, fileSize: number): boole return identity.firstPrompt !== undefined || fileSize >= HEAD_BYTES; } -function codexSessionsRoot(): string { - const home = process.env.CODEX_HOME || join(homedir(), '.codex'); - return join(home, 'sessions'); -} - /** Read at most `bytes` from the front of a file. Returns '' when unreadable. */ async function readHead(path: string, bytes: number): Promise { const fh = await open(path, 'r').catch(() => null); @@ -176,55 +178,16 @@ async function readTail(path: string, size: number, bytes: number): Promise !!b && typeof b === 'object' && typeof (b as { text?: unknown }).text === 'string' - ) - .map((b) => b.text) - .join('\n') - .trim(); -} - -/** One line's user-prompt text, whichever of the three shapes it is. */ -function userPromptFromLine(entry: { - type?: string; - payload?: { - type?: string; - role?: string; - content?: unknown; - message?: unknown; - item?: { type?: string; content?: unknown }; - }; -}): { text: string; injectionProne: boolean } | null { - const p = entry.payload; - if (!p) return null; - - if (entry.type === 'event_msg' && p.type === 'item_completed' && p.item?.type === 'UserMessage') { - const text = contentText(p.item.content); - return text ? { text, injectionProne: false } : null; - } - if (entry.type === 'event_msg' && p.type === 'user_message') { - const text = typeof p.message === 'string' ? p.message.trim() : contentText(p.message); - return text ? { text, injectionProne: false } : null; - } - if (entry.type === 'response_item' && p.role === 'user') { - const text = contentText(p.content); - return text ? { text, injectionProne: true } : null; - } - return null; -} - /** - * Injected context rather than something the user typed. Codex prepends the - * repository's AGENTS.md and wraps environment context in a tag, and both arrive - * as `response_item` user rows. + * One line's user-prompt text, whichever of the three shapes it is. The shapes + * and the injected-context rule are shared with the offline history reader + * (`codex-rollout-format.ts`), so the two never disagree about what a prompt is. */ -function isInjectedContext(text: string): boolean { - return text.startsWith('#') || text.startsWith('<'); +function userPromptFromLine(entry: unknown): { text: string; injectionProne: boolean } | null { + const message = codexMessageFromLine(entry); + if (!message || message.role !== 'user') return null; + const text = message.text.trim(); + return text ? { text, injectionProne: message.injectionProne } : null; } /** Collapse to one line and cap, so a row carries a title rather than an essay. */ diff --git a/test/codex-rollout-format.test.ts b/test/codex-rollout-format.test.ts new file mode 100644 index 00000000..761f32b8 --- /dev/null +++ b/test/codex-rollout-format.test.ts @@ -0,0 +1,112 @@ +/** + * @fileoverview The Codex rollout shape helpers shared by the Past Sessions + * scanner (codex-transcript.ts) and the offline history reader + * (local-history.ts): store location, message shapes and the injected-context + * rule. Both readers' own tests cover their use of these; this pins the shared + * contract itself. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { join } from 'node:path'; +import { + codexContentText, + codexHomeDir, + codexMessageFromLine, + codexSessionsRoot, + isCodexInjectedContext, +} from '../src/codex-rollout-format.js'; + +const savedHome = process.env.CODEX_HOME; +afterEach(() => { + if (savedHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = savedHome; +}); + +describe('codex rollout format', () => { + it('resolves the store from CODEX_HOME, else ~/.codex', () => { + process.env.CODEX_HOME = '/custom/codex'; + expect(codexHomeDir()).toBe('/custom/codex'); + expect(codexSessionsRoot()).toBe(join('/custom/codex', 'sessions')); + process.env.CODEX_HOME = ''; + expect(codexSessionsRoot()).toBe(join(process.env.HOME!, '.codex', 'sessions')); + }); + + it('reads all three message shapes with their role, shape and phase', () => { + expect(codexMessageFromLine({ type: 'event_msg', payload: { type: 'user_message', message: 'hi' } })).toEqual({ + role: 'user', + text: 'hi', + shape: 'event', + phase: undefined, + injectionProne: false, + }); + expect( + codexMessageFromLine({ + type: 'event_msg', + payload: { type: 'agent_message', message: 'ok', phase: 'commentary' }, + }) + ).toMatchObject({ role: 'assistant', shape: 'event', phase: 'commentary', injectionProne: false }); + expect( + codexMessageFromLine({ + type: 'event_msg', + payload: { + type: 'item_completed', + item: { type: 'AgentMessage', phase: 'final_answer', content: [{ type: 'Text', text: 'done' }] }, + }, + }) + ).toMatchObject({ role: 'assistant', text: 'done', shape: 'item', phase: 'final_answer' }); + expect( + codexMessageFromLine({ + type: 'response_item', + payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'typed' }] }, + }) + ).toMatchObject({ role: 'user', text: 'typed', shape: 'response', injectionProne: true }); + expect( + codexMessageFromLine({ + type: 'response_item', + payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'a' }] }, + }) + ).toMatchObject({ role: 'assistant', shape: 'response', injectionProne: false }); + }); + + it('ignores every other line, empty text and malformed input', () => { + for (const value of [ + null, + 42, + 'text', + [], + { type: 'session_meta', payload: { id: 'x' } }, + { type: 'response_item', payload: { type: 'message', role: 'developer', content: 'policy' } }, + { type: 'response_item', payload: { type: 'reasoning', summary: [] } }, + { type: 'event_msg', payload: { type: 'user_message', message: '' } }, + { type: 'event_msg', payload: { type: 'item_completed', item: { type: 'CommandExecution' } } }, + ]) + expect(codexMessageFromLine(value)).toBeNull(); + }); + + it('flattens only blocks that carry text', () => { + expect(codexContentText('plain')).toBe('plain'); + expect( + codexContentText([ + { type: 'input_text', text: 'a' }, + { type: 'input_image', image_url: 'x' }, + { text: 'b' }, + null, + ]) + ).toBe('a\nb'); + expect(codexContentText(undefined)).toBe(''); + }); + + it('treats a heading or a tag as injected context, wherever the wrapper name is new', () => { + for (const text of [ + '# AGENTS.md instructions for /repo', + '', + '', + '', + '', + '', + ' ', + ]) + expect(isCodexInjectedContext(text)).toBe(true); + for (const text of ['fix the bug', 'You are a reviewer', '$skill run', 'a tag later']) + expect(isCodexInjectedContext(text)).toBe(false); + }); +});