diff --git a/src/codex-rollout-format.ts b/src/codex-rollout-format.ts new file mode 100644 index 00000000..068e68bf --- /dev/null +++ b/src/codex-rollout-format.ts @@ -0,0 +1,128 @@ +/** + * @fileoverview Shape helpers for Codex rollout JSONL lines, shared by the two + * readers of `~/.codex/sessions`: the Past Sessions scanner + * (`codex-transcript.ts`, previews only) and the offline history reader + * (`local-history.ts`, whole conversations). + * + * Both readers must agree on three facts, so they live here once: + * + * - where the store is (`$CODEX_HOME/sessions`, else `~/.codex/sessions`); + * - which line shapes carry a user or assistant message; + * - which `response_item` user rows are injected context rather than input. + * + * ## The message shapes + * + * Codex has written conversation text under three shapes: + * + * - `event_msg` / `user_message` and `agent_message`, written by every version + * seen so far. `agent_message` carries a `phase` (`commentary` or + * `final_answer`); `user_message` carries none. + * - `event_msg` / `item_completed` with an `item.type` of `UserMessage` or + * `AgentMessage`, written by newer versions (0.152.1 and later). + * - `response_item` rows with `role: 'user'` or `'assistant'`. These are what + * the model saw, so the user rows mix real input with injected context. + * + * ## The injected-context rule + * + * Codex injects context as `response_item` user rows that start with a Markdown + * heading (`# AGENTS.md instructions for ...`) or an XML-style tag + * (``, ``, ``, + * ``, ``, ``, ``). + * Measured on a store of 2,916 rollouts (codex 0.120 to 0.163), every + * `response_item` user row with either prefix and no event twin was injected; + * the real prompts that start with `#` all have a `user_message` or + * `UserMessage` event twin, which this rule never filters. A fixed list of tag + * names would leak every wrapper Codex adds later (`` + * alone was 1,355 rows on that store), so the rule is the prefix. + * + * @module codex-rollout-format + */ + +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +/** Codex's home directory: `$CODEX_HOME`, else `~/.codex`, exactly as codex resolves it. */ +export function codexHomeDir(): string { + return process.env.CODEX_HOME || join(homedir(), '.codex'); +} + +/** The rollout store under {@link codexHomeDir}. */ +export function codexSessionsRoot(): string { + return join(codexHomeDir(), 'sessions'); +} + +/** + * Flatten Codex message content, which is a string or an array of blocks. + * Only blocks carrying a string `text` contribute (images and skills do not). + * Not trimmed: callers that want a preview trim. + */ +export function codexContentText(content: unknown): string { + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return ''; + return content + .filter( + (b): b is { text: string } => !!b && typeof b === 'object' && typeof (b as { text?: unknown }).text === 'string' + ) + .map((b) => b.text) + .filter(Boolean) + .join('\n'); +} + +/** Injected context rather than typed input. Applies to `response_item` user rows only. */ +export function isCodexInjectedContext(text: string): boolean { + const start = text.trimStart(); + return start.startsWith('#') || start.startsWith('<'); +} + +/** One rollout line's user or assistant message, whichever shape carried it. */ +export interface CodexLineMessage { + role: 'user' | 'assistant'; + /** Untrimmed text, never empty. */ + text: string; + /** `event` = user_message/agent_message, `item` = item_completed, `response` = response_item. */ + shape: 'event' | 'item' | 'response'; + /** Codex's own phase (`commentary` / `final_answer`) when the line carries one. */ + phase?: string; + /** A `response_item` user row: may be injected context, see {@link isCodexInjectedContext}. */ + injectionProne: boolean; +} + +function record(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record) : {}; +} + +function phaseOf(value: unknown): string | undefined { + return typeof value === 'string' && value ? value : undefined; +} + +/** The message a parsed rollout line carries, or null for every other line. */ +export function codexMessageFromLine(entry: unknown): CodexLineMessage | null { + const e = record(entry); + const p = record(e.payload); + let role: CodexLineMessage['role'] | undefined; + let text = ''; + let shape: CodexLineMessage['shape'] = 'event'; + let phase: string | undefined; + if (e.type === 'event_msg') { + if (p.type === 'user_message' || p.type === 'agent_message') { + role = p.type === 'user_message' ? 'user' : 'assistant'; + text = codexContentText(p.message); + phase = phaseOf(p.phase); + } else if (p.type === 'item_completed') { + const item = record(p.item); + if (item.type === 'UserMessage' || item.type === 'AgentMessage') { + role = item.type === 'UserMessage' ? 'user' : 'assistant'; + shape = 'item'; + text = codexContentText(item.content); + phase = phaseOf(item.phase); + } + } + } else if (e.type === 'response_item' && (p.role === 'user' || p.role === 'assistant')) { + role = p.role; + shape = 'response'; + text = codexContentText(p.content); + phase = phaseOf(p.phase); + } + if (!role || !text) return null; + return { role, text, shape, phase, injectionProne: shape === 'response' && role === 'user' }; +} diff --git a/src/codex-transcript.ts b/src/codex-transcript.ts index 9f360ee3..aabc0624 100644 --- a/src/codex-transcript.ts +++ b/src/codex-transcript.ts @@ -37,13 +37,20 @@ * summaries), so they are read only when neither shape above appears, and * the obvious injections are dropped. * + * The shapes, the injected-context rule and the store location live in + * `codex-rollout-format.ts`, shared with the offline reader `local-history.ts`. + * * @module codex-transcript */ import { open, readdir, stat } from 'node:fs/promises'; -import { homedir } from 'node:os'; import { join } from 'node:path'; +import { + codexMessageFromLine, + codexSessionsRoot, + isCodexInjectedContext as isInjectedContext, +} from './codex-rollout-format.js'; import { LRUMap } from './utils/lru-map.js'; /** Covers `session_meta` (~19 KiB) plus the first user message (~69 KiB behind it). */ @@ -134,11 +141,6 @@ function shouldCacheIdentity(identity: RolloutIdentity, fileSize: number): boole return identity.firstPrompt !== undefined || fileSize >= HEAD_BYTES; } -function codexSessionsRoot(): string { - const home = process.env.CODEX_HOME || join(homedir(), '.codex'); - return join(home, 'sessions'); -} - /** Read at most `bytes` from the front of a file. Returns '' when unreadable. */ async function readHead(path: string, bytes: number): Promise { const fh = await open(path, 'r').catch(() => null); @@ -176,55 +178,16 @@ async function readTail(path: string, size: number, bytes: number): Promise !!b && typeof b === 'object' && typeof (b as { text?: unknown }).text === 'string' - ) - .map((b) => b.text) - .join('\n') - .trim(); -} - -/** One line's user-prompt text, whichever of the three shapes it is. */ -function userPromptFromLine(entry: { - type?: string; - payload?: { - type?: string; - role?: string; - content?: unknown; - message?: unknown; - item?: { type?: string; content?: unknown }; - }; -}): { text: string; injectionProne: boolean } | null { - const p = entry.payload; - if (!p) return null; - - if (entry.type === 'event_msg' && p.type === 'item_completed' && p.item?.type === 'UserMessage') { - const text = contentText(p.item.content); - return text ? { text, injectionProne: false } : null; - } - if (entry.type === 'event_msg' && p.type === 'user_message') { - const text = typeof p.message === 'string' ? p.message.trim() : contentText(p.message); - return text ? { text, injectionProne: false } : null; - } - if (entry.type === 'response_item' && p.role === 'user') { - const text = contentText(p.content); - return text ? { text, injectionProne: true } : null; - } - return null; -} - /** - * Injected context rather than something the user typed. Codex prepends the - * repository's AGENTS.md and wraps environment context in a tag, and both arrive - * as `response_item` user rows. + * One line's user-prompt text, whichever of the three shapes it is. The shapes + * and the injected-context rule are shared with the offline history reader + * (`codex-rollout-format.ts`), so the two never disagree about what a prompt is. */ -function isInjectedContext(text: string): boolean { - return text.startsWith('#') || text.startsWith('<'); +function userPromptFromLine(entry: unknown): { text: string; injectionProne: boolean } | null { + const message = codexMessageFromLine(entry); + if (!message || message.role !== 'user') return null; + const text = message.text.trim(); + return text ? { text, injectionProne: message.injectionProne } : null; } /** Collapse to one line and cap, so a row carries a title rather than an essay. */ diff --git a/test/codex-rollout-format.test.ts b/test/codex-rollout-format.test.ts new file mode 100644 index 00000000..761f32b8 --- /dev/null +++ b/test/codex-rollout-format.test.ts @@ -0,0 +1,112 @@ +/** + * @fileoverview The Codex rollout shape helpers shared by the Past Sessions + * scanner (codex-transcript.ts) and the offline history reader + * (local-history.ts): store location, message shapes and the injected-context + * rule. Both readers' own tests cover their use of these; this pins the shared + * contract itself. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { join } from 'node:path'; +import { + codexContentText, + codexHomeDir, + codexMessageFromLine, + codexSessionsRoot, + isCodexInjectedContext, +} from '../src/codex-rollout-format.js'; + +const savedHome = process.env.CODEX_HOME; +afterEach(() => { + if (savedHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = savedHome; +}); + +describe('codex rollout format', () => { + it('resolves the store from CODEX_HOME, else ~/.codex', () => { + process.env.CODEX_HOME = '/custom/codex'; + expect(codexHomeDir()).toBe('/custom/codex'); + expect(codexSessionsRoot()).toBe(join('/custom/codex', 'sessions')); + process.env.CODEX_HOME = ''; + expect(codexSessionsRoot()).toBe(join(process.env.HOME!, '.codex', 'sessions')); + }); + + it('reads all three message shapes with their role, shape and phase', () => { + expect(codexMessageFromLine({ type: 'event_msg', payload: { type: 'user_message', message: 'hi' } })).toEqual({ + role: 'user', + text: 'hi', + shape: 'event', + phase: undefined, + injectionProne: false, + }); + expect( + codexMessageFromLine({ + type: 'event_msg', + payload: { type: 'agent_message', message: 'ok', phase: 'commentary' }, + }) + ).toMatchObject({ role: 'assistant', shape: 'event', phase: 'commentary', injectionProne: false }); + expect( + codexMessageFromLine({ + type: 'event_msg', + payload: { + type: 'item_completed', + item: { type: 'AgentMessage', phase: 'final_answer', content: [{ type: 'Text', text: 'done' }] }, + }, + }) + ).toMatchObject({ role: 'assistant', text: 'done', shape: 'item', phase: 'final_answer' }); + expect( + codexMessageFromLine({ + type: 'response_item', + payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'typed' }] }, + }) + ).toMatchObject({ role: 'user', text: 'typed', shape: 'response', injectionProne: true }); + expect( + codexMessageFromLine({ + type: 'response_item', + payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'a' }] }, + }) + ).toMatchObject({ role: 'assistant', shape: 'response', injectionProne: false }); + }); + + it('ignores every other line, empty text and malformed input', () => { + for (const value of [ + null, + 42, + 'text', + [], + { type: 'session_meta', payload: { id: 'x' } }, + { type: 'response_item', payload: { type: 'message', role: 'developer', content: 'policy' } }, + { type: 'response_item', payload: { type: 'reasoning', summary: [] } }, + { type: 'event_msg', payload: { type: 'user_message', message: '' } }, + { type: 'event_msg', payload: { type: 'item_completed', item: { type: 'CommandExecution' } } }, + ]) + expect(codexMessageFromLine(value)).toBeNull(); + }); + + it('flattens only blocks that carry text', () => { + expect(codexContentText('plain')).toBe('plain'); + expect( + codexContentText([ + { type: 'input_text', text: 'a' }, + { type: 'input_image', image_url: 'x' }, + { text: 'b' }, + null, + ]) + ).toBe('a\nb'); + expect(codexContentText(undefined)).toBe(''); + }); + + it('treats a heading or a tag as injected context, wherever the wrapper name is new', () => { + for (const text of [ + '# AGENTS.md instructions for /repo', + '', + '', + '', + '', + '', + ' ', + ]) + expect(isCodexInjectedContext(text)).toBe(true); + for (const text of ['fix the bug', 'You are a reviewer', '$skill run', 'a tag later']) + expect(isCodexInjectedContext(text)).toBe(false); + }); +});