From e8ea8b0715f854ccdcee6c2d42377a68af47a8fb Mon Sep 17 00:00:00 2001 From: Aamer Akhter Date: Sat, 24 Oct 2026 20:43:17 -0400 Subject: [PATCH 1/5] feat(history): COD-549 read saved Codex conversations offline --- .changeset/offline-codex-history.md | 5 + docs/offline-history.md | 52 ++ plugins/codeman/skills/codeman/SKILL.md | 15 +- .../skills/codeman/reference/history.md | 51 ++ skills/codeman/SKILL.md | 15 +- skills/codeman/reference/history.md | 51 ++ src/cli-history.ts | 47 ++ src/cli.ts | 2 + src/local-history.ts | 484 ++++++++++++++++++ test/local-history.test.ts | 246 +++++++++ 10 files changed, 960 insertions(+), 8 deletions(-) create mode 100644 .changeset/offline-codex-history.md create mode 100644 docs/offline-history.md create mode 100644 plugins/codeman/skills/codeman/reference/history.md create mode 100644 skills/codeman/reference/history.md create mode 100644 src/cli-history.ts create mode 100644 src/local-history.ts create mode 100644 test/local-history.test.ts diff --git a/.changeset/offline-codex-history.md b/.changeset/offline-codex-history.md new file mode 100644 index 00000000..57090231 --- /dev/null +++ b/.changeset/offline-codex-history.md @@ -0,0 +1,5 @@ +--- +'aicodeman': minor +--- + +Add offline `codeman history read` for saved Codex transcripts by file, native session ID, or conversation family, with whole-turn topic/date filters and bounded local reads and output. diff --git a/docs/offline-history.md b/docs/offline-history.md new file mode 100644 index 00000000..219aabc1 --- /dev/null +++ b/docs/offline-history.md @@ -0,0 +1,52 @@ +# Saved Codex history (offline) + +Use `codeman history read` for retained local Codex JSONL transcripts. This command +works outside a managed session and never calls the HTTP API. Do not bootstrap +API credentials or require `CODEMAN_MUX` for this read-only job. + +Choose exactly one selector: + +```bash +codeman history read --file /path/to/rollout.jsonl +codeman history read --session-id aaaaaaaa --sessions-root /path/to/sessions +codeman history read --root-session-id aaaaaaaa-1111-4111-8111-111111111111 \ + --topic 'authentication' --after 2026-10-01 --before 2026-10-24 \ + --max-bytes 65536 +``` + +`--session-id` accepts a native conversation UUID or unique prefix of at least +8 characters; identity comes from metadata, not filenames. `--root-session-id` +requires a full native UUID and includes transitive descendants found in metadata. +The default store is `$CODEX_HOME/sessions`, or `~/.codex/sessions` when unset. +`--file` selects one file and does not search for its ancestors or children. + +Topic matching is a case-insensitive literal substring. Dates are inclusive +`YYYY-MM-DD` bounds. Each filter selects whole user/assistant turns; a turn is +retained if it has a date match and a topic match, which may be different messages +in that turn. Tool calls/results and injected context are omitted. Adjacent duplicate +native representations are collapsed; repeated messages from the same +representation are retained. + +JSON is the default (`--json` or `--format json`); `--format text` is available. +Results identify source paths, native session/root IDs, timestamps, roles and phases, +plus scan counters and `complete`. A root ID is `null` when ancestry cannot be proven. +Report that uncertainty; never relabel the queried child as its own root. +`complete: false` reports malformed or partial records, skipped links, unattributed +files or missing ancestors. It does not prove the store includes every conversation +ever created, and a filtered result is only evidence for its selected scope. + +Reads require regular UTF8 files and reject final-component symlinks, conflicting +metadata, ambiguous IDs, cyclic ancestry, invalid selectors and changed selected +files. Directory traversal skips symlinks. Parent directories of an explicitly +supplied path must be trusted; this is not a filesystem sandbox. + +The default total metadata/transcript scan budget is 16 MiB; `--scan-bytes` allows +1–67108864 bytes. Directory scans are capped at 8192 entries and depth 8. The output +budget is 8192 bytes by default; `--max-bytes` allows 1–1048576 bytes, including the +final newline. Exhausted budgets fail with an error: output is never silently +truncated or split into partial JSON or turns. Narrow the store/filters or explicitly +raise the relevant budget. Missing evidence is an error, not an empty success. + +This command does not inspect live panes, stage prompts, submit input or manage +sessions. Live session operations use the separate `codeman agent` commands and +require the managed-session environment and running server. diff --git a/plugins/codeman/skills/codeman/SKILL.md b/plugins/codeman/skills/codeman/SKILL.md index 50133e8c..a1328e4b 100644 --- a/plugins/codeman/skills/codeman/SKILL.md +++ b/plugins/codeman/skills/codeman/SKILL.md @@ -8,12 +8,19 @@ description: >- read their output, and clean up; where available, message claude workers directly (Claude Code cross-session messaging). Use when asked to orchestrate or parallelize work across Codeman sessions, get a second opinion from another CLI or model, watch - another session, or start and manage workers. Only usable inside a Codeman-managed - session (CODEMAN_MUX=1); refuse to act otherwise. + another session, start and manage workers, or retrieve saved local Codex history + by file/native ID or root and descendants with topic/date filters. Live API + operations require a Codeman-managed session (CODEMAN_MUX=1). Offline history + reads do not require a server or managed session. --- # Driving Codeman from inside a session +For **saved Codex conversation history**, use `codeman history read` and +[reference/history.md](reference/history.md). This is a read-only local command: +route to it **before §0**, including outside a managed session. It needs no API +URL, credentials or running server. All live API operations below retain §0. + You are an agent running inside a Codeman-managed terminal session. Codeman is the server that spawned you; its HTTP API can start, prompt, watch, and delete other sessions. @@ -43,8 +50,8 @@ machine has no `jq` for the preamble below. ## 0. Guard and bootstrap -If `CODEMAN_MUX` is not `1`, **stop and say so**. Do not guess an API URL; a server -you are not part of is not yours to drive. +For live API operations, if `CODEMAN_MUX` is not `1`, **stop and say so**. +Do not guess an API URL; a server you are not part of is not yours to drive. Two failures come from your CLI, not from Codeman: name them to the user instead of retrying. diff --git a/plugins/codeman/skills/codeman/reference/history.md b/plugins/codeman/skills/codeman/reference/history.md new file mode 100644 index 00000000..0fa87205 --- /dev/null +++ b/plugins/codeman/skills/codeman/reference/history.md @@ -0,0 +1,51 @@ +# Saved Codex history (offline) + +Use `codeman history read` for retained local Codex JSONL transcripts. This command +works outside a managed session and never calls the HTTP API. Do not bootstrap +API credentials or require `CODEMAN_MUX` for this read-only job. + +Choose exactly one selector: + +```bash +codeman history read --file /path/to/rollout.jsonl +codeman history read --session-id aaaaaaaa --sessions-root /path/to/sessions +codeman history read --root-session-id aaaaaaaa-1111-4111-8111-111111111111 \ + --topic 'authentication' --after 2026-10-01 --before 2026-10-24 \ + --max-bytes 65536 +``` + +`--session-id` accepts a native conversation UUID or unique prefix of at least +8 characters; identity comes from metadata, not filenames. `--root-session-id` +requires a full native UUID and includes transitive descendants found in metadata. +The default store is `$CODEX_HOME/sessions`, or `~/.codex/sessions` when unset. +`--file` selects one file and does not search for its ancestors or children. + +Topic matching is a case-insensitive literal substring. Dates are inclusive +`YYYY-MM-DD` bounds. Each filter selects whole user/assistant turns; a turn is +retained if it has a date match and a topic match, which may be different messages +in that turn. Tool calls/results and injected context are omitted. Adjacent duplicate +native representations are collapsed; repeated messages from the same +representation are retained. + +JSON is the default (`--json` or `--format json`); `--format text` is available. +Results identify source paths, native session/root IDs, timestamps, roles and phases, +plus scan counters and `complete`. A root ID is `null` when ancestry cannot be proven. +Report that uncertainty; never relabel the queried child as its own root. +`complete: false` reports malformed or partial records, skipped links, unattributed +files or missing ancestors. It does not prove the store includes every conversation +ever created, and a filtered result is only evidence for its selected scope. + +Reads require regular UTF8 files and reject final-component symlinks, conflicting +metadata, ambiguous IDs, cyclic ancestry, invalid selectors and changed selected +files. Directory traversal skips symlinks. Parent directories of an explicitly +supplied path must be trusted; this is not a filesystem sandbox. + +The default total metadata/transcript scan budget is 16 MiB; `--scan-bytes` allows +1–67108864 bytes. Directory scans are capped at 8192 entries and depth 8. The output +budget is 8192 bytes by default; `--max-bytes` allows 1–1048576 bytes, including the +final newline. Exhausted budgets fail with an error: output is never silently +truncated or split into partial JSON or turns. Narrow the store/filters or explicitly +raise the relevant budget. Missing evidence is an error, not an empty success. + +This command does not inspect live panes, stage prompts, submit input or manage +sessions. For live operations return to the managed-session guard in the skill. diff --git a/skills/codeman/SKILL.md b/skills/codeman/SKILL.md index 50133e8c..a1328e4b 100644 --- a/skills/codeman/SKILL.md +++ b/skills/codeman/SKILL.md @@ -8,12 +8,19 @@ description: >- read their output, and clean up; where available, message claude workers directly (Claude Code cross-session messaging). Use when asked to orchestrate or parallelize work across Codeman sessions, get a second opinion from another CLI or model, watch - another session, or start and manage workers. Only usable inside a Codeman-managed - session (CODEMAN_MUX=1); refuse to act otherwise. + another session, start and manage workers, or retrieve saved local Codex history + by file/native ID or root and descendants with topic/date filters. Live API + operations require a Codeman-managed session (CODEMAN_MUX=1). Offline history + reads do not require a server or managed session. --- # Driving Codeman from inside a session +For **saved Codex conversation history**, use `codeman history read` and +[reference/history.md](reference/history.md). This is a read-only local command: +route to it **before §0**, including outside a managed session. It needs no API +URL, credentials or running server. All live API operations below retain §0. + You are an agent running inside a Codeman-managed terminal session. Codeman is the server that spawned you; its HTTP API can start, prompt, watch, and delete other sessions. @@ -43,8 +50,8 @@ machine has no `jq` for the preamble below. ## 0. Guard and bootstrap -If `CODEMAN_MUX` is not `1`, **stop and say so**. Do not guess an API URL; a server -you are not part of is not yours to drive. +For live API operations, if `CODEMAN_MUX` is not `1`, **stop and say so**. +Do not guess an API URL; a server you are not part of is not yours to drive. Two failures come from your CLI, not from Codeman: name them to the user instead of retrying. diff --git a/skills/codeman/reference/history.md b/skills/codeman/reference/history.md new file mode 100644 index 00000000..0fa87205 --- /dev/null +++ b/skills/codeman/reference/history.md @@ -0,0 +1,51 @@ +# Saved Codex history (offline) + +Use `codeman history read` for retained local Codex JSONL transcripts. This command +works outside a managed session and never calls the HTTP API. Do not bootstrap +API credentials or require `CODEMAN_MUX` for this read-only job. + +Choose exactly one selector: + +```bash +codeman history read --file /path/to/rollout.jsonl +codeman history read --session-id aaaaaaaa --sessions-root /path/to/sessions +codeman history read --root-session-id aaaaaaaa-1111-4111-8111-111111111111 \ + --topic 'authentication' --after 2026-10-01 --before 2026-10-24 \ + --max-bytes 65536 +``` + +`--session-id` accepts a native conversation UUID or unique prefix of at least +8 characters; identity comes from metadata, not filenames. `--root-session-id` +requires a full native UUID and includes transitive descendants found in metadata. +The default store is `$CODEX_HOME/sessions`, or `~/.codex/sessions` when unset. +`--file` selects one file and does not search for its ancestors or children. + +Topic matching is a case-insensitive literal substring. Dates are inclusive +`YYYY-MM-DD` bounds. Each filter selects whole user/assistant turns; a turn is +retained if it has a date match and a topic match, which may be different messages +in that turn. Tool calls/results and injected context are omitted. Adjacent duplicate +native representations are collapsed; repeated messages from the same +representation are retained. + +JSON is the default (`--json` or `--format json`); `--format text` is available. +Results identify source paths, native session/root IDs, timestamps, roles and phases, +plus scan counters and `complete`. A root ID is `null` when ancestry cannot be proven. +Report that uncertainty; never relabel the queried child as its own root. +`complete: false` reports malformed or partial records, skipped links, unattributed +files or missing ancestors. It does not prove the store includes every conversation +ever created, and a filtered result is only evidence for its selected scope. + +Reads require regular UTF8 files and reject final-component symlinks, conflicting +metadata, ambiguous IDs, cyclic ancestry, invalid selectors and changed selected +files. Directory traversal skips symlinks. Parent directories of an explicitly +supplied path must be trusted; this is not a filesystem sandbox. + +The default total metadata/transcript scan budget is 16 MiB; `--scan-bytes` allows +1–67108864 bytes. Directory scans are capped at 8192 entries and depth 8. The output +budget is 8192 bytes by default; `--max-bytes` allows 1–1048576 bytes, including the +final newline. Exhausted budgets fail with an error: output is never silently +truncated or split into partial JSON or turns. Narrow the store/filters or explicitly +raise the relevant budget. Missing evidence is an error, not an empty success. + +This command does not inspect live panes, stage prompts, submit input or manage +sessions. For live operations return to the managed-session guard in the skill. diff --git a/src/cli-history.ts b/src/cli-history.ts new file mode 100644 index 00000000..614a416e --- /dev/null +++ b/src/cli-history.ts @@ -0,0 +1,47 @@ +/** Local offline transcript commands. Deliberately separate from the agent API guard. */ +import type { Command } from 'commander'; +import { formatLocalHistory, HISTORY_DEFAULT_SCAN_BYTES, readLocalHistory } from './local-history.js'; + +export function registerHistoryCommands(program: Command): Command { + const history = program + .command('history') + .description('Read local saved native transcripts without a running server'); + history + .command('read') + .description('Read Codex history by file, native id, or root conversation and descendants') + .option('--file ', 'Explicit local JSONL transcript (regular file only)') + .option('--session-id ', 'Native Codex conversation UUID or unique prefix (at least 8 characters)') + .option('--root-session-id ', 'Native root UUID, including transitive child conversations') + .option('--sessions-root ', 'Local Codex sessions directory (default: CODEX_HOME/sessions)') + .option('--topic ', 'Case-insensitive literal match, preserving whole matching turns') + .option('--after ', 'Inclusive YYYY-MM-DD lower bound; preserves whole matching turns') + .option('--before ', 'Inclusive YYYY-MM-DD upper bound; preserves whole matching turns') + .option('--max-bytes ', 'Maximum complete output bytes (1–1048576)', '8192') + .option( + '--scan-bytes ', + 'Total metadata and selected transcript read budget (1–67108864)', + String(HISTORY_DEFAULT_SCAN_BYTES) + ) + .option('--format ', 'json or text', 'json') + .option('--json', 'JSON output (also the default)') + .action(async (options: Record) => { + try { + const format = options.json ? 'json' : options.format; + if (format !== 'json' && format !== 'text') throw new Error('--format must be json or text'); + const result = await readLocalHistory({ + file: options.file as string | undefined, + sessionId: options.sessionId as string | undefined, + rootSessionId: options.rootSessionId as string | undefined, + sessionsRoot: options.sessionsRoot as string | undefined, + topic: options.topic as string | undefined, + after: options.after as string | undefined, + before: options.before as string | undefined, + scanBytes: Number(options.scanBytes), + }); + program.configureOutput().writeOut?.(`${formatLocalHistory(result, format, Number(options.maxBytes))}\n`); + } catch (error) { + program.error(error instanceof Error ? error.message : String(error), { exitCode: 1, code: 'history.read' }); + } + }); + return history; +} diff --git a/src/cli.ts b/src/cli.ts index ba07793b..c8a4e5e5 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -29,6 +29,7 @@ import { import type { AgentSkillDir } from './config/cli-registry/types.js'; import { getCli, listClis } from './config/cli-registry/registry.js'; import { registerAgentCommands } from './cli-agent.js'; +import { registerHistoryCommands } from './cli-history.js'; import { resolveMcpConfig, runMcpStdio } from './mcp-server.js'; import { getSessionManager } from './session-manager.js'; import { getTaskQueue } from './task-queue.js'; @@ -324,6 +325,7 @@ skillCmd // ============ Agent Commands (session-to-session, any CLI mode) ============ registerAgentCommands(program); +registerHistoryCommands(program); // ============ MCP server (any MCP client drives Codeman) ============ diff --git a/src/local-history.ts b/src/local-history.ts new file mode 100644 index 00000000..ac7fb04f --- /dev/null +++ b/src/local-history.ts @@ -0,0 +1,484 @@ +/** Offline, local-operator Codex history. No HTTP routes or running session required. */ +import { constants } from 'node:fs'; +import { isUtf8 } from 'node:buffer'; +import { lstat, open, opendir, type FileHandle } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { join, resolve } from 'node:path'; + +export const HISTORY_DEFAULT_SCAN_BYTES = 16 * 1024 * 1024; +export const HISTORY_MAX_SCAN_BYTES = 64 * 1024 * 1024; +export const HISTORY_MAX_OUTPUT_BYTES = 1024 * 1024; +const ID = /^[0-9a-f]{8}(?:[0-9a-f-]{0,28})$/i; +const FULL_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const METADATA_BYTES = 128 * 1024; +type Json = Record; +export interface LocalHistoryOptions { + file?: string; + sessionId?: string; + rootSessionId?: string; + sessionsRoot?: string; + topic?: string; + after?: string; + before?: string; + scanBytes?: number; + maxEntries?: number; +} +export interface HistoryRecord { + rootSessionId: string | null; + sessionId: string; + timestamp: string; + role: 'user' | 'assistant'; + phase: string; + message: string; +} +interface Source { + path: string; + sessionId: string; + parentId?: string; + rootId?: string; + timestamp: string; + fingerprint?: string; +} +export interface LocalHistoryResult { + source: 'codex-jsonl'; + complete: boolean; + sessions: Array<{ sessionId: string; rootSessionId: string | null; parentSessionId?: string; path: string }>; + records: HistoryRecord[]; + scan: { + bytesRead: number; + entries: number; + malformedRecords: number; + partialRecords: number; + skippedSymlinks: number; + unattributedFiles: number; + }; +} +interface Budget { + remaining: number; + entriesRemaining: number; + scan: LocalHistoryResult['scan']; +} +function object(value: unknown): Json { + return value && typeof value === 'object' && !Array.isArray(value) ? (value as Json) : {}; +} +function string(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} +function validDate(value: string | undefined): boolean { + return ( + value === undefined || + (/^\d{4}-\d{2}-\d{2}$/.test(value) && + Number.isFinite(Date.parse(value)) && + new Date(value).toISOString().slice(0, 10) === value) + ); +} +function validate(options: LocalHistoryOptions): void { + if ([options.file, options.sessionId, options.rootSessionId].filter((v) => v !== undefined).length !== 1) + throw new Error('Select exactly one of --file, --session-id, or --root-session-id'); + if (options.file !== undefined && !options.file) throw new Error('A nonempty file is required'); + if (options.sessionId !== undefined && !ID.test(options.sessionId)) throw new Error('Invalid native session id'); + if (options.rootSessionId !== undefined && !FULL_ID.test(options.rootSessionId)) + throw new Error('Invalid root session id'); + if ( + !validDate(options.after) || + !validDate(options.before) || + (options.after && options.before && options.after > options.before) + ) + throw new Error('Invalid date or date range (use YYYY-MM-DD)'); + if (options.topic !== undefined && !options.topic.trim()) throw new Error('Invalid empty topic'); + for (const [value, max] of [ + [options.scanBytes ?? HISTORY_DEFAULT_SCAN_BYTES, HISTORY_MAX_SCAN_BYTES], + [options.maxEntries ?? 8192, 65536], + ]) + if (!Number.isSafeInteger(value) || value < 1 || value > max) throw new Error('Invalid scan budget'); +} +async function regularFile(path: string): Promise { + // O_NOFOLLOW prevents a final-component link swap between lstat and open. fstat + // validates the opened descriptor, not a previously inspected pathname. + if (!(await lstat(path)).isFile()) throw new Error('History source must be a regular file, not a symlink'); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + if (!(await handle.stat()).isFile()) { + await handle.close(); + throw new Error('History source must be a regular file'); + } + return handle; +} +async function readChunk(handle: FileHandle, position: number, size: number, budget: Budget): Promise { + if (size > budget.remaining) + throw new Error('History scan byte budget exhausted; narrow the store or raise --scan-bytes'); + const buffer = Buffer.alloc(size); + const { bytesRead } = await handle.read(buffer, 0, size, position); + budget.remaining -= bytesRead; + budget.scan.bytesRead += bytesRead; + return buffer.subarray(0, bytesRead); +} +function metadata(value: unknown, path: string): Source | undefined { + const entry = object(value); + if (entry.type !== 'session_meta') return undefined; + const p = object(entry.payload); + const sessionId = string(p.id) ?? string(p.session_id); + if (!sessionId || !FULL_ID.test(sessionId)) return undefined; + const source = object(p.source); + const spawn = object(object(source.subagent).thread_spawn); + const parentId = string(p.parent_thread_id) ?? string(p.parent_session_id) ?? string(spawn.parent_thread_id); + const declaredRoot = string(p.root_session_id) ?? string(p.session_id); + return { + path, + sessionId: sessionId.toLowerCase(), + parentId: parentId && FULL_ID.test(parentId) ? parentId.toLowerCase() : undefined, + rootId: + declaredRoot && declaredRoot !== sessionId && FULL_ID.test(declaredRoot) ? declaredRoot.toLowerCase() : undefined, + timestamp: string(p.timestamp) ?? string(entry.timestamp) ?? '', + }; +} +async function readMetadata(path: string, budget: Budget): Promise { + const handle = await regularFile(path); + try { + const before = await handle.stat(); + const parts: Buffer[] = []; + let offset = 0; + let parsed = 0; + while (offset < before.size && offset < METADATA_BYTES) { + const chunk = await readChunk( + handle, + offset, + Math.min(4096, before.size - offset, METADATA_BYTES - offset), + budget + ); + if (!chunk.length) break; + parts.push(chunk); + offset += chunk.length; + const buffer = Buffer.concat(parts); + let end = buffer.indexOf(10, parsed); + while (end >= 0) { + const line = buffer.subarray(parsed, end); + if (!isUtf8(line)) throw new Error('History metadata contains invalid UTF8'); + try { + const found = metadata(JSON.parse(line.toString('utf8')), path); + if (found) { + const after = await handle.stat(); + const fingerprint = (value: typeof before) => + `${value.dev}:${value.ino}:${value.size}:${value.mtimeMs}:${value.ctimeMs}`; + if (fingerprint(before) !== fingerprint(after)) + throw new Error('History source changed during metadata scan'); + return { ...found, fingerprint: fingerprint(before) }; + } + } catch (error) { + if (!(error instanceof SyntaxError)) throw error; + } + parsed = end + 1; + end = buffer.indexOf(10, parsed); + } + } + if (offset < before.size) + throw new Error('History metadata scan budget exhausted; cannot prove family completeness'); + return undefined; + } finally { + await handle.close(); + } +} +async function findSources(root: string, budget: Budget): Promise { + if (!(await lstat(root)).isDirectory()) throw new Error('Sessions root must be a real directory, not a symlink'); + const sources: Source[] = []; + const visit = async (dir: string, depth: number): Promise => { + if (depth > 8) throw new Error('History directory scan depth budget exhausted'); + const entries = await opendir(dir); + for await (const entry of entries) { + if (--budget.entriesRemaining < 0) throw new Error('History directory scan entry budget exhausted'); + budget.scan.entries++; + const path = join(dir, entry.name); + if (entry.isSymbolicLink()) { + budget.scan.skippedSymlinks++; + continue; + } + if (entry.isDirectory()) { + // Recheck immediately before descending; never intentionally follow links. + if (!(await lstat(path)).isDirectory()) throw new Error('History directory changed during scan'); + await visit(path, depth + 1); + } else if (entry.isFile() && entry.name.endsWith('.jsonl')) { + const source = await readMetadata(path, budget); + if (source) sources.push(source); + else budget.scan.unattributedFiles++; + } + } + }; + await visit(root, 0); + return sources; +} +function contentText(value: unknown): string { + if (typeof value === 'string') return value; + if (!Array.isArray(value)) return ''; + return value + .map((block) => { + const item = object(block); + return typeof item.type === 'string' && ['text', 'input_text', 'output_text'].includes(item.type.toLowerCase()) + ? (string(item.text) ?? '') + : ''; + }) + .filter(Boolean) + .join('\n'); +} +interface Message { + representation: 'event' | 'modern' | 'response'; + role: 'user' | 'assistant'; + message: string; + phase: string; + timestamp: string; +} +function message(value: unknown): Message | undefined { + const entry = object(value); + const p = object(entry.payload); + if (entry.isSidechain || p.isSidechain) return undefined; + let role: Message['role'] | undefined; + let representation: Message['representation'] = 'event'; + let text = ''; + let phase = string(p.phase) ?? 'progress'; + if (entry.type === 'event_msg') { + if (p.type === 'user_message' || p.type === 'agent_message') { + role = p.type === 'user_message' ? 'user' : 'assistant'; + text = contentText(p.message); + } else if (p.type === 'item_completed') { + representation = 'modern'; + const item = object(p.item); + if (item.type === 'UserMessage' || item.type === 'AgentMessage') { + role = item.type === 'UserMessage' ? 'user' : 'assistant'; + text = contentText(item.content); + phase = string(item.phase) ?? phase; + } + } + } else if ( + entry.type === 'response_item' && + (p.type === 'message' || p.type === undefined) && + (p.role === 'user' || p.role === 'assistant') + ) { + role = p.role; + representation = 'response'; + text = contentText(p.content); + // Codex context is not a human turn. Do not omit arbitrary markdown prompts. + if ( + role === 'user' && + /^(?:# AGENTS\.md instructions|||)/.test(text) + ) + return undefined; + } + if (!role || !text) return undefined; + return { + role, + message: text, + phase, + representation, + timestamp: string(entry.timestamp) ?? string(p.timestamp) ?? '', + }; +} +async function readMessages(source: Source, budget: Budget): Promise { + const handle = await regularFile(source.path); + try { + const before = await handle.stat(); + if (`${before.dev}:${before.ino}:${before.size}:${before.mtimeMs}:${before.ctimeMs}` !== source.fingerprint) + throw new Error('History source changed after attribution; retry explicitly'); + const buffer = await readChunk(handle, 0, before.size, budget); + const after = await handle.stat(); + if ( + buffer.length !== before.size || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) + throw new Error('History source changed while reading; retry explicitly'); + if (!isUtf8(buffer)) throw new Error('History source contains invalid UTF8'); + const lines = buffer.toString('utf8').split('\n'); + const partial = lines.pop(); + if (partial) budget.scan.partialRecords++; + const records: Message[] = []; + let representations = new Set(); + for (const line of lines) { + if (!line.trim()) continue; + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + budget.scan.malformedRecords++; + continue; + } + if (object(parsed).type === 'session_meta') { + const declared = metadata(parsed, source.path); + if ( + !declared || + declared.sessionId !== source.sessionId || + declared.parentId !== source.parentId || + declared.rootId !== source.rootId + ) + throw new Error('History source contains conflicting native metadata; attribution refused'); + continue; + } + const item = message(parsed); + if (!item) continue; + const previous = records.at(-1); + // Native events and response_item commonly duplicate one accepted message. + if ( + previous?.role === item.role && + previous.message === item.message && + !representations.has(item.representation) + ) { + representations.add(item.representation); + if (!previous.timestamp) previous.timestamp = item.timestamp; + if (item.phase === 'final_answer') previous.phase = item.phase; + continue; + } + records.push(item); + representations = new Set([item.representation]); + } + return records; + } finally { + await handle.close(); + } +} +function selectedTurns(messages: Message[], options: LocalHistoryOptions): Message[] { + const turns: Message[][] = []; + for (const item of messages) { + if (item.role === 'user' || !turns.length) turns.push([]); + turns.at(-1)!.push(item); + } + const needle = options.topic?.trim().toLocaleLowerCase(); + return turns + .filter((turn) => { + const dateMatches = + (!options.after && !options.before) || + turn.some((item) => { + const date = item.timestamp.slice(0, 10); + return ( + validDate(date) && (!options.after || date >= options.after) && (!options.before || date <= options.before) + ); + }); + return dateMatches && (!needle || turn.some((item) => item.message.toLocaleLowerCase().includes(needle))); + }) + .flat(); +} + +/** Native ancestry is metadata evidence, not the conversation selected by a query. */ +function nativeRoot(source: Source, sources: Source[]): string | null { + const seen = new Set(); + let current = source; + while (true) { + if (seen.has(current.sessionId)) throw new Error('Native history contains cyclic ancestry'); + seen.add(current.sessionId); + if (current.rootId) return current.rootId; + if (!current.parentId) return current.sessionId; + const parents = sources.filter((candidate) => candidate.sessionId === current.parentId); + if (parents.length === 0) return null; + if (parents.length > 1) throw new Error('Native ancestor id is ambiguous'); + current = parents[0]; + } +} + +export async function readLocalHistory(options: LocalHistoryOptions): Promise { + validate(options); + const scan: LocalHistoryResult['scan'] = { + bytesRead: 0, + entries: 0, + malformedRecords: 0, + partialRecords: 0, + skippedSymlinks: 0, + unattributedFiles: 0, + }; + const budget: Budget = { + remaining: options.scanBytes ?? HISTORY_DEFAULT_SCAN_BYTES, + entriesRemaining: options.maxEntries ?? 8192, + scan, + }; + let sources: Source[]; + let attributionSources: Source[]; + if (options.file !== undefined) { + const source = await readMetadata(resolve(options.file), budget); + if (!source) throw new Error('History source is missing valid native session metadata'); + sources = [source]; + attributionSources = sources; + } else { + const root = options.sessionsRoot ?? join(process.env.CODEX_HOME || join(homedir(), '.codex'), 'sessions'); + const all = await findSources(resolve(root), budget); + const wanted = (options.sessionId ?? options.rootSessionId)!.toLowerCase(); + const matches = all.filter((source) => source.sessionId.toLowerCase().startsWith(wanted)); + if (!matches.length) throw new Error('Native session missing from history store'); + if (matches.length !== 1) throw new Error('Native session id is ambiguous'); + const target = matches[0]; + attributionSources = all; + const queryRootId = target.sessionId; + sources = [target]; + if (options.rootSessionId) { + const selected = new Set([target.sessionId]); + let changed = true; + while (changed) { + changed = false; + for (const source of all) { + if ( + !selected.has(source.sessionId) && + (source.rootId === queryRootId || (source.parentId && selected.has(source.parentId))) + ) { + selected.add(source.sessionId); + sources.push(source); + changed = true; + } + } + } + for (const source of sources) { + if (all.filter((other) => other.sessionId === source.sessionId).length !== 1) + throw new Error('Native family contains ambiguous duplicated session ids'); + } + } + } + sources.sort((a, b) => a.timestamp.localeCompare(b.timestamp) || a.sessionId.localeCompare(b.sessionId)); + const roots = new Map(sources.map((source) => [source.sessionId, nativeRoot(source, attributionSources)])); + const records: HistoryRecord[] = []; + for (const source of sources) { + records.push( + ...selectedTurns(await readMessages(source, budget), options).map((item) => ({ + rootSessionId: roots.get(source.sessionId) ?? null, + sessionId: source.sessionId, + role: item.role, + message: item.message, + phase: item.phase, + timestamp: item.timestamp, + })) + ); + } + if (!records.length) throw new Error('No matching user/assistant evidence found'); + return { + source: 'codex-jsonl', + complete: + scan.malformedRecords === 0 && + scan.partialRecords === 0 && + scan.skippedSymlinks === 0 && + scan.unattributedFiles === 0 && + [...roots.values()].every((root) => root !== null), + sessions: sources.map((source) => ({ + sessionId: source.sessionId, + rootSessionId: roots.get(source.sessionId) ?? null, + ...(source.parentId ? { parentSessionId: source.parentId } : {}), + path: source.path, + })), + records, + scan, + }; +} +/** Output never splits a JSON document, UTF8 character, record, or selected turn. */ +export function formatLocalHistory( + result: LocalHistoryResult, + format: 'json' | 'text' = 'json', + maxBytes = 8192 +): string { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > HISTORY_MAX_OUTPUT_BYTES) + throw new Error('Invalid output byte budget'); + const body = + format === 'json' + ? JSON.stringify(result) + : [ + `source=${result.source} complete=${result.complete} scannedBytes=${result.scan.bytesRead}`, + ...result.records.map( + (row) => + `[root=${row.rootSessionId} session=${row.sessionId} ${row.timestamp} ${row.role} ${row.phase}]\n${row.message}` + ), + ].join('\n\n'); + if (Buffer.byteLength(body) + 1 > maxBytes) + throw new Error('History output byte budget exceeded; narrow filters or raise --max-bytes'); + return body; +} diff --git a/test/local-history.test.ts b/test/local-history.test.ts new file mode 100644 index 00000000..7ffa4c4b --- /dev/null +++ b/test/local-history.test.ts @@ -0,0 +1,246 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import * as history from '../src/local-history.js'; +import { Command } from 'commander'; +import { registerHistoryCommands } from '../src/cli-history.js'; + +const ROOT = 'aaaaaaaa-1111-4111-8111-111111111111'; +const CHILD = 'bbbbbbbb-1111-4111-8111-111111111111'; +const GRANDCHILD = 'cccccccc-1111-4111-8111-111111111111'; +let dir: string; +const meta = (id: string, parent?: string) => ({ + type: 'session_meta', + payload: { + id, + timestamp: '2026-10-24T01:00:00Z', + ...(parent ? { source: { subagent: { thread_spawn: { parent_thread_id: parent } } } } : {}), + }, +}); +const row = (role: string, text: string, timestamp = '2026-10-24T01:00:00Z') => ({ + type: 'response_item', + timestamp, + payload: { type: 'message', role, content: [{ type: role === 'user' ? 'input_text' : 'output_text', text }] }, +}); +async function fixture(id: string, records: unknown[], parent?: string) { + const path = join(dir, `rollout-${id}.jsonl`); + await writeFile(path, [meta(id, parent), ...records].map((r) => JSON.stringify(r)).join('\n') + '\n'); + return path; +} +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'local-history-')); +}); +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +describe('offline native history', () => { + it('reads a native transcript without a server and preserves identity and timestamps', async () => { + const file = await fixture(ROOT, [row('user', 'question'), row('assistant', 'answer')]); + const result = await history.readLocalHistory({ file }); + expect(result.records.map((r) => r.message)).toEqual(['question', 'answer']); + expect(result.records[0]).toMatchObject({ + sessionId: ROOT, + rootSessionId: ROOT, + timestamp: '2026-10-24T01:00:00Z', + }); + expect(result.complete).toBe(true); + }); + it('resolves metadata identities rather than filenames', async () => { + await fixture(ROOT, [row('assistant', 'yes')]); + const result = await history.readLocalHistory({ sessionId: ROOT.slice(0, 8), sessionsRoot: dir }); + expect(result.records[0].sessionId).toBe(ROOT); + }); + it('rejects ambiguous native prefixes', async () => { + await fixture(ROOT, [row('assistant', 'one')]); + await fixture('aaaaaaaa-2222-4222-8222-222222222222', [row('assistant', 'two')]); + await expect(history.readLocalHistory({ sessionId: 'aaaaaaaa', sessionsRoot: dir })).rejects.toThrow(/ambiguous/); + }); + it('gathers transitive children and retains a whole topic/date matching turn', async () => { + await fixture(ROOT, [row('user', 'other'), row('assistant', 'unrelated')]); + await fixture( + CHILD, + [row('user', 'topic question'), row('assistant', 'answer after midnight', '2026-10-25T00:01:00Z')], + ROOT + ); + await fixture(GRANDCHILD, [row('user', 'topic child'), row('assistant', 'child answer')], CHILD); + const result = await history.readLocalHistory({ + rootSessionId: ROOT, + sessionsRoot: dir, + topic: 'TOPIC', + before: '2026-10-24', + }); + expect(result.sessions).toHaveLength(3); + expect(result.records.map((r) => r.message)).toEqual([ + 'topic question', + 'answer after midnight', + 'topic child', + 'child answer', + ]); + expect(result.records.every((r) => r.rootSessionId === ROOT)).toBe(true); + }); + it('resolves the real native root when querying a child by id', async () => { + await fixture(ROOT, [row('assistant', 'root')]); + await fixture(CHILD, [row('assistant', 'child')], ROOT); + await fixture(GRANDCHILD, [row('assistant', 'grandchild')], CHILD); + const result = await history.readLocalHistory({ sessionId: GRANDCHILD, sessionsRoot: dir }); + expect(result.records[0].rootSessionId).toBe(ROOT); + expect(result.sessions[0]).toMatchObject({ parentSessionId: CHILD, rootSessionId: ROOT }); + }); + it('labels a file-only child root unknown rather than inventing an ancestor', async () => { + const file = await fixture(CHILD, [row('assistant', 'child')], ROOT); + const result = await history.readLocalHistory({ file }); + expect(result.records[0].rootSessionId).toBeNull(); + expect(result.sessions[0]).toMatchObject({ parentSessionId: ROOT, rootSessionId: null }); + }); + it('labels missing ancestors unknown and refuses cyclic ancestry', async () => { + await fixture(CHILD, [row('assistant', 'child')], ROOT); + expect( + (await history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir })).records[0].rootSessionId + ).toBeNull(); + await fixture(ROOT, [row('assistant', 'root')], CHILD); + await expect(history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir })).rejects.toThrow(/cyclic/); + }); + it('supports legacy shared session_id genealogy', async () => { + await fixture(ROOT, [row('assistant', 'root')]); + await writeFile( + join(dir, 'legacy.jsonl'), + [{ type: 'session_meta', payload: { id: CHILD, session_id: ROOT } }, row('assistant', 'legacy child')] + .map((r) => JSON.stringify(r)) + .join('\n') + '\n' + ); + expect((await history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir })).sessions).toHaveLength(2); + }); + it('parses modern events and removes adjacent duplicate representations without erasing repeated turns', async () => { + const file = await fixture(ROOT, [ + row('user', 'question'), + { + type: 'event_msg', + payload: { + type: 'item_completed', + item: { type: 'UserMessage', content: [{ type: 'Text', text: 'question' }] }, + }, + }, + row('assistant', 'answer'), + { + type: 'event_msg', + payload: { + type: 'item_completed', + item: { type: 'AgentMessage', phase: 'final_answer', content: [{ type: 'Text', text: 'answer' }] }, + }, + }, + { type: 'event_msg', payload: { type: 'user_message', message: 'question' } }, + { type: 'event_msg', payload: { type: 'agent_message', message: 'answer' } }, + ]); + expect((await history.readLocalHistory({ file })).records.map((r) => r.message)).toEqual([ + 'question', + 'answer', + 'question', + 'answer', + ]); + }); + it('preserves genuinely repeated identical messages from the same representation', async () => { + const file = await fixture(ROOT, [row('user', 'repeat'), row('user', 'repeat'), row('assistant', 'answer')]); + expect((await history.readLocalHistory({ file })).records.map((r) => r.message)).toEqual([ + 'repeat', + 'repeat', + 'answer', + ]); + }); + it('labels unreadable metadata as incomplete rather than hiding possible children', async () => { + await fixture(ROOT, [row('assistant', 'root')]); + await writeFile(join(dir, 'unattributed.jsonl'), '{bad metadata}\n'); + const result = await history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir }); + expect(result.complete).toBe(false); + expect(result.scan.unattributedFiles).toBe(1); + }); + it('refuses conflicting session metadata instead of attributing another conversation answer', async () => { + const file = await fixture(ROOT, [row('user', 'root question'), meta(CHILD), row('assistant', 'child answer')]); + await expect(history.readLocalHistory({ file })).rejects.toThrow(/conflicting.*metadata/); + }); + it('accepts repeated metadata only when native identity and genealogy agree', async () => { + const file = await fixture(ROOT, [row('user', 'question'), meta(ROOT), row('assistant', 'answer')]); + expect((await history.readLocalHistory({ file })).records.map((record) => record.message)).toEqual([ + 'question', + 'answer', + ]); + await fixture(CHILD, [meta(CHILD, GRANDCHILD), row('assistant', 'child answer')], ROOT); + await expect(history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir })).rejects.toThrow( + /conflicting.*metadata/ + ); + }); + it('labels skipped malformed and pending partial records', async () => { + const file = await fixture(ROOT, [row('assistant', 'complete')]); + const { appendFile } = await import('node:fs/promises'); + await appendFile(file, 'not json\n{"type":'); + const result = await history.readLocalHistory({ file }); + expect(result.complete).toBe(false); + expect(result.scan.malformedRecords).toBe(1); + expect(result.scan.partialRecords).toBe(1); + expect(result.records[0].message).toBe('complete'); + }); + it('rejects invalid UTF8, nonregular inputs and symlinks', async () => { + const file = await fixture(ROOT, [row('assistant', 'answer')]); + const linked = join(dir, 'linked.jsonl'); + await symlink(file, linked); + await expect(history.readLocalHistory({ file: linked })).rejects.toThrow(/regular|symlink/); + await expect(history.readLocalHistory({ file: dir })).rejects.toThrow(/regular/); + await writeFile(file, Buffer.from([0xff, 0x0a])); + await expect(history.readLocalHistory({ file })).rejects.toThrow(/UTF8/); + }); + it('fails closed for byte budgets and preserves valid JSON for successful output', async () => { + const file = await fixture(ROOT, [row('assistant', '😀'.repeat(80))]); + await expect(history.readLocalHistory({ file, scanBytes: 32 })).rejects.toThrow(/scan.*budget/); + const result = await history.readLocalHistory({ file }); + expect(() => history.formatLocalHistory(result, 'json', 40)).toThrow(/output.*budget/); + const output = history.formatLocalHistory(result, 'json', 8192); + expect(Buffer.byteLength(output)).toBeLessThanOrEqual(8192); + expect(JSON.parse(output).records[0].message).toBe('😀'.repeat(80)); + }); + it.each([{ after: '2026-02-30' }, { before: 'bad' }, { after: '2026-10-25', before: '2026-10-24' }, { topic: ' ' }])( + 'rejects invalid filters %j', + async (filter) => { + const file = await fixture(ROOT, [row('assistant', 'answer')]); + await expect(history.readLocalHistory({ file, ...filter })).rejects.toThrow(/date|topic/); + } + ); + it('rejects missing selectors, multiple selectors, empty matches and absent native ids', async () => { + const file = await fixture(ROOT, [row('assistant', 'answer')]); + await expect(history.readLocalHistory({})).rejects.toThrow(/exactly one/); + await expect(history.readLocalHistory({ file, sessionId: ROOT })).rejects.toThrow(/exactly one/); + await expect(history.readLocalHistory({ file, topic: 'absent' })).rejects.toThrow(/evidence/); + await expect(history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir })).rejects.toThrow(/missing/); + }); + it('rejects exhausted directory traversal rather than returning an incomplete family', async () => { + await fixture(ROOT, [row('assistant', 'root')]); + await mkdir(join(dir, 'nested')); + await fixture(CHILD, [row('assistant', 'child')], ROOT); + await expect(history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir, maxEntries: 1 })).rejects.toThrow( + /scan.*budget/ + ); + }); + it('executes the offline CLI with complete JSON and no internal parser fields', async () => { + const file = await fixture(ROOT, [row('assistant', 'offline')]); + let output = ''; + const program = new Command().exitOverride().configureOutput({ + writeOut: (text) => { + output += text; + }, + writeErr: () => {}, + }); + registerHistoryCommands(program); + await program.parseAsync(['history', 'read', '--file', file, '--json'], { from: 'user' }); + const result = JSON.parse(output); + expect(result.records[0].message).toBe('offline'); + expect(result.records[0]).not.toHaveProperty('representation'); + await expect( + program.parseAsync(['history', 'read', '--file', file, '--session-id', ROOT], { from: 'user' }) + ).rejects.toThrow(/exactly one/); + }); + it('registers offline commands and reports invalid selectors without process exit', async () => { + const program = new Command().exitOverride(); + registerHistoryCommands(program); + expect(program.commands.find((c) => c.name() === 'history')?.commands.map((c) => c.name())).toEqual(['read']); + }); +}); From 803e7c7f852a8b668eca5dc93bba2ea402b737f8 Mon Sep 17 00:00:00 2001 From: Aamer Akhter Date: Sat, 24 Oct 2026 23:00:49 -0400 Subject: [PATCH 2/5] refactor(codex): share the rollout format between the two Codex history readers The Past Sessions scanner and the offline history reader each parsed Codex rollouts with their own shape helpers, their own CODEX_HOME resolution and different injected-context rules. Move the store location, the three message shapes and the injected-context rule into codex-rollout-format.ts. The scanner's behaviour is unchanged; the offline reader adopts its rule in the next commit. --- src/codex-rollout-format.ts | 128 ++++++++++++++++++++++++++++++ src/codex-transcript.ts | 69 ++++------------ test/codex-rollout-format.test.ts | 112 ++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 53 deletions(-) create mode 100644 src/codex-rollout-format.ts create mode 100644 test/codex-rollout-format.test.ts diff --git a/src/codex-rollout-format.ts b/src/codex-rollout-format.ts new file mode 100644 index 00000000..068e68bf --- /dev/null +++ b/src/codex-rollout-format.ts @@ -0,0 +1,128 @@ +/** + * @fileoverview Shape helpers for Codex rollout JSONL lines, shared by the two + * readers of `~/.codex/sessions`: the Past Sessions scanner + * (`codex-transcript.ts`, previews only) and the offline history reader + * (`local-history.ts`, whole conversations). + * + * Both readers must agree on three facts, so they live here once: + * + * - where the store is (`$CODEX_HOME/sessions`, else `~/.codex/sessions`); + * - which line shapes carry a user or assistant message; + * - which `response_item` user rows are injected context rather than input. + * + * ## The message shapes + * + * Codex has written conversation text under three shapes: + * + * - `event_msg` / `user_message` and `agent_message`, written by every version + * seen so far. `agent_message` carries a `phase` (`commentary` or + * `final_answer`); `user_message` carries none. + * - `event_msg` / `item_completed` with an `item.type` of `UserMessage` or + * `AgentMessage`, written by newer versions (0.152.1 and later). + * - `response_item` rows with `role: 'user'` or `'assistant'`. These are what + * the model saw, so the user rows mix real input with injected context. + * + * ## The injected-context rule + * + * Codex injects context as `response_item` user rows that start with a Markdown + * heading (`# AGENTS.md instructions for ...`) or an XML-style tag + * (``, ``, ``, + * ``, ``, ``, ``). + * Measured on a store of 2,916 rollouts (codex 0.120 to 0.163), every + * `response_item` user row with either prefix and no event twin was injected; + * the real prompts that start with `#` all have a `user_message` or + * `UserMessage` event twin, which this rule never filters. A fixed list of tag + * names would leak every wrapper Codex adds later (`` + * alone was 1,355 rows on that store), so the rule is the prefix. + * + * @module codex-rollout-format + */ + +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +/** Codex's home directory: `$CODEX_HOME`, else `~/.codex`, exactly as codex resolves it. */ +export function codexHomeDir(): string { + return process.env.CODEX_HOME || join(homedir(), '.codex'); +} + +/** The rollout store under {@link codexHomeDir}. */ +export function codexSessionsRoot(): string { + return join(codexHomeDir(), 'sessions'); +} + +/** + * Flatten Codex message content, which is a string or an array of blocks. + * Only blocks carrying a string `text` contribute (images and skills do not). + * Not trimmed: callers that want a preview trim. + */ +export function codexContentText(content: unknown): string { + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return ''; + return content + .filter( + (b): b is { text: string } => !!b && typeof b === 'object' && typeof (b as { text?: unknown }).text === 'string' + ) + .map((b) => b.text) + .filter(Boolean) + .join('\n'); +} + +/** Injected context rather than typed input. Applies to `response_item` user rows only. */ +export function isCodexInjectedContext(text: string): boolean { + const start = text.trimStart(); + return start.startsWith('#') || start.startsWith('<'); +} + +/** One rollout line's user or assistant message, whichever shape carried it. */ +export interface CodexLineMessage { + role: 'user' | 'assistant'; + /** Untrimmed text, never empty. */ + text: string; + /** `event` = user_message/agent_message, `item` = item_completed, `response` = response_item. */ + shape: 'event' | 'item' | 'response'; + /** Codex's own phase (`commentary` / `final_answer`) when the line carries one. */ + phase?: string; + /** A `response_item` user row: may be injected context, see {@link isCodexInjectedContext}. */ + injectionProne: boolean; +} + +function record(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record) : {}; +} + +function phaseOf(value: unknown): string | undefined { + return typeof value === 'string' && value ? value : undefined; +} + +/** The message a parsed rollout line carries, or null for every other line. */ +export function codexMessageFromLine(entry: unknown): CodexLineMessage | null { + const e = record(entry); + const p = record(e.payload); + let role: CodexLineMessage['role'] | undefined; + let text = ''; + let shape: CodexLineMessage['shape'] = 'event'; + let phase: string | undefined; + if (e.type === 'event_msg') { + if (p.type === 'user_message' || p.type === 'agent_message') { + role = p.type === 'user_message' ? 'user' : 'assistant'; + text = codexContentText(p.message); + phase = phaseOf(p.phase); + } else if (p.type === 'item_completed') { + const item = record(p.item); + if (item.type === 'UserMessage' || item.type === 'AgentMessage') { + role = item.type === 'UserMessage' ? 'user' : 'assistant'; + shape = 'item'; + text = codexContentText(item.content); + phase = phaseOf(item.phase); + } + } + } else if (e.type === 'response_item' && (p.role === 'user' || p.role === 'assistant')) { + role = p.role; + shape = 'response'; + text = codexContentText(p.content); + phase = phaseOf(p.phase); + } + if (!role || !text) return null; + return { role, text, shape, phase, injectionProne: shape === 'response' && role === 'user' }; +} diff --git a/src/codex-transcript.ts b/src/codex-transcript.ts index 9f360ee3..aabc0624 100644 --- a/src/codex-transcript.ts +++ b/src/codex-transcript.ts @@ -37,13 +37,20 @@ * summaries), so they are read only when neither shape above appears, and * the obvious injections are dropped. * + * The shapes, the injected-context rule and the store location live in + * `codex-rollout-format.ts`, shared with the offline reader `local-history.ts`. + * * @module codex-transcript */ import { open, readdir, stat } from 'node:fs/promises'; -import { homedir } from 'node:os'; import { join } from 'node:path'; +import { + codexMessageFromLine, + codexSessionsRoot, + isCodexInjectedContext as isInjectedContext, +} from './codex-rollout-format.js'; import { LRUMap } from './utils/lru-map.js'; /** Covers `session_meta` (~19 KiB) plus the first user message (~69 KiB behind it). */ @@ -134,11 +141,6 @@ function shouldCacheIdentity(identity: RolloutIdentity, fileSize: number): boole return identity.firstPrompt !== undefined || fileSize >= HEAD_BYTES; } -function codexSessionsRoot(): string { - const home = process.env.CODEX_HOME || join(homedir(), '.codex'); - return join(home, 'sessions'); -} - /** Read at most `bytes` from the front of a file. Returns '' when unreadable. */ async function readHead(path: string, bytes: number): Promise { const fh = await open(path, 'r').catch(() => null); @@ -176,55 +178,16 @@ async function readTail(path: string, size: number, bytes: number): Promise !!b && typeof b === 'object' && typeof (b as { text?: unknown }).text === 'string' - ) - .map((b) => b.text) - .join('\n') - .trim(); -} - -/** One line's user-prompt text, whichever of the three shapes it is. */ -function userPromptFromLine(entry: { - type?: string; - payload?: { - type?: string; - role?: string; - content?: unknown; - message?: unknown; - item?: { type?: string; content?: unknown }; - }; -}): { text: string; injectionProne: boolean } | null { - const p = entry.payload; - if (!p) return null; - - if (entry.type === 'event_msg' && p.type === 'item_completed' && p.item?.type === 'UserMessage') { - const text = contentText(p.item.content); - return text ? { text, injectionProne: false } : null; - } - if (entry.type === 'event_msg' && p.type === 'user_message') { - const text = typeof p.message === 'string' ? p.message.trim() : contentText(p.message); - return text ? { text, injectionProne: false } : null; - } - if (entry.type === 'response_item' && p.role === 'user') { - const text = contentText(p.content); - return text ? { text, injectionProne: true } : null; - } - return null; -} - /** - * Injected context rather than something the user typed. Codex prepends the - * repository's AGENTS.md and wraps environment context in a tag, and both arrive - * as `response_item` user rows. + * One line's user-prompt text, whichever of the three shapes it is. The shapes + * and the injected-context rule are shared with the offline history reader + * (`codex-rollout-format.ts`), so the two never disagree about what a prompt is. */ -function isInjectedContext(text: string): boolean { - return text.startsWith('#') || text.startsWith('<'); +function userPromptFromLine(entry: unknown): { text: string; injectionProne: boolean } | null { + const message = codexMessageFromLine(entry); + if (!message || message.role !== 'user') return null; + const text = message.text.trim(); + return text ? { text, injectionProne: message.injectionProne } : null; } /** Collapse to one line and cap, so a row carries a title rather than an essay. */ diff --git a/test/codex-rollout-format.test.ts b/test/codex-rollout-format.test.ts new file mode 100644 index 00000000..761f32b8 --- /dev/null +++ b/test/codex-rollout-format.test.ts @@ -0,0 +1,112 @@ +/** + * @fileoverview The Codex rollout shape helpers shared by the Past Sessions + * scanner (codex-transcript.ts) and the offline history reader + * (local-history.ts): store location, message shapes and the injected-context + * rule. Both readers' own tests cover their use of these; this pins the shared + * contract itself. + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { join } from 'node:path'; +import { + codexContentText, + codexHomeDir, + codexMessageFromLine, + codexSessionsRoot, + isCodexInjectedContext, +} from '../src/codex-rollout-format.js'; + +const savedHome = process.env.CODEX_HOME; +afterEach(() => { + if (savedHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = savedHome; +}); + +describe('codex rollout format', () => { + it('resolves the store from CODEX_HOME, else ~/.codex', () => { + process.env.CODEX_HOME = '/custom/codex'; + expect(codexHomeDir()).toBe('/custom/codex'); + expect(codexSessionsRoot()).toBe(join('/custom/codex', 'sessions')); + process.env.CODEX_HOME = ''; + expect(codexSessionsRoot()).toBe(join(process.env.HOME!, '.codex', 'sessions')); + }); + + it('reads all three message shapes with their role, shape and phase', () => { + expect(codexMessageFromLine({ type: 'event_msg', payload: { type: 'user_message', message: 'hi' } })).toEqual({ + role: 'user', + text: 'hi', + shape: 'event', + phase: undefined, + injectionProne: false, + }); + expect( + codexMessageFromLine({ + type: 'event_msg', + payload: { type: 'agent_message', message: 'ok', phase: 'commentary' }, + }) + ).toMatchObject({ role: 'assistant', shape: 'event', phase: 'commentary', injectionProne: false }); + expect( + codexMessageFromLine({ + type: 'event_msg', + payload: { + type: 'item_completed', + item: { type: 'AgentMessage', phase: 'final_answer', content: [{ type: 'Text', text: 'done' }] }, + }, + }) + ).toMatchObject({ role: 'assistant', text: 'done', shape: 'item', phase: 'final_answer' }); + expect( + codexMessageFromLine({ + type: 'response_item', + payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'typed' }] }, + }) + ).toMatchObject({ role: 'user', text: 'typed', shape: 'response', injectionProne: true }); + expect( + codexMessageFromLine({ + type: 'response_item', + payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'a' }] }, + }) + ).toMatchObject({ role: 'assistant', shape: 'response', injectionProne: false }); + }); + + it('ignores every other line, empty text and malformed input', () => { + for (const value of [ + null, + 42, + 'text', + [], + { type: 'session_meta', payload: { id: 'x' } }, + { type: 'response_item', payload: { type: 'message', role: 'developer', content: 'policy' } }, + { type: 'response_item', payload: { type: 'reasoning', summary: [] } }, + { type: 'event_msg', payload: { type: 'user_message', message: '' } }, + { type: 'event_msg', payload: { type: 'item_completed', item: { type: 'CommandExecution' } } }, + ]) + expect(codexMessageFromLine(value)).toBeNull(); + }); + + it('flattens only blocks that carry text', () => { + expect(codexContentText('plain')).toBe('plain'); + expect( + codexContentText([ + { type: 'input_text', text: 'a' }, + { type: 'input_image', image_url: 'x' }, + { text: 'b' }, + null, + ]) + ).toBe('a\nb'); + expect(codexContentText(undefined)).toBe(''); + }); + + it('treats a heading or a tag as injected context, wherever the wrapper name is new', () => { + for (const text of [ + '# AGENTS.md instructions for /repo', + '', + '', + '', + '', + '', + ' ', + ]) + expect(isCodexInjectedContext(text)).toBe(true); + for (const text of ['fix the bug', 'You are a reviewer', '$skill run', 'a tag later']) + expect(isCodexInjectedContext(text)).toBe(false); + }); +}); From 002d876556d89967e1c37072de03e1f740502361 Mon Sep 17 00:00:00 2001 From: Aamer Akhter Date: Sat, 24 Oct 2026 23:00:49 -0400 Subject: [PATCH 3/5] fix(history): read forked threads, large stores and live rollouts A forked or spawned thread's rollout carries its own session_meta followed by its parent's and a copy of the parent's history. A later session_meta for a declared ancestor now opens an inherited prefix that is dropped: by ordinal below subagent_history_start_ordinal in paginated history, and up to the thread_settings_applied event Codex appends after the copy in legacy history, checked against the parent's own messages when the copy holds more than one candidate. An unrelated id still refuses the file. Store attribution now has its own budget (--metadata-bytes) separate from the transcript budget (--scan-bytes), reads each head in 32 KiB steps, and --max-entries is a flag, so id lookups work on stores of thousands of rollouts. An unattributable store file or unreadable directory is counted instead of aborting the lookup, and a rollout that a running Codex appends to is read as a snapshot up to its last complete line. User records carry no phase, date bounds are UTC calendar dates, the injected-context rule is the shared one, a symlinked sessions root is followed as the Past Sessions scanner follows it, and --json with --format text is refused. --- src/cli-history.ts | 70 +++- src/local-history.ts | 638 ++++++++++++++++++++++---------- test/fixtures/codex-rollout.ts | 125 +++++++ test/local-history-live.test.ts | 129 +++++++ test/local-history.test.ts | 472 ++++++++++++++++++++++- 5 files changed, 1234 insertions(+), 200 deletions(-) create mode 100644 test/fixtures/codex-rollout.ts create mode 100644 test/local-history-live.test.ts diff --git a/src/cli-history.ts b/src/cli-history.ts index 614a416e..11be6088 100644 --- a/src/cli-history.ts +++ b/src/cli-history.ts @@ -1,6 +1,33 @@ -/** Local offline transcript commands. Deliberately separate from the agent API guard. */ +/** + * @fileoverview `codeman history read`: the command-line front of the offline + * Codex history reader (`local-history.ts`). + * + * Deliberately separate from `codeman agent` (`cli-agent.ts`): that command is + * a client of a running Codeman and refuses outside a managed session, while + * this one reads rollout files from disk and needs neither a server nor + * `CODEMAN_MUX`. It never writes, never inspects a live pane and never sends + * input. + * + * Every bound the reader enforces has a flag here with the reader's own + * maximum, so a large store is a matter of raising a number rather than an + * impossible lookup. Output is JSON unless `--format text` is given; `--json` + * is the JSON spelling and refuses to combine with `--format text`. + * + * @module cli-history + */ import type { Command } from 'commander'; -import { formatLocalHistory, HISTORY_DEFAULT_SCAN_BYTES, readLocalHistory } from './local-history.js'; +import { + formatLocalHistory, + HISTORY_DEFAULT_MAX_ENTRIES, + HISTORY_DEFAULT_METADATA_BYTES, + HISTORY_DEFAULT_OUTPUT_BYTES, + HISTORY_DEFAULT_SCAN_BYTES, + HISTORY_MAX_ENTRIES, + HISTORY_MAX_METADATA_BYTES, + HISTORY_MAX_OUTPUT_BYTES, + HISTORY_MAX_SCAN_BYTES, + readLocalHistory, +} from './local-history.js'; export function registerHistoryCommands(program: Command): Command { const history = program @@ -12,22 +39,41 @@ export function registerHistoryCommands(program: Command): Command { .option('--file ', 'Explicit local JSONL transcript (regular file only)') .option('--session-id ', 'Native Codex conversation UUID or unique prefix (at least 8 characters)') .option('--root-session-id ', 'Native root UUID, including transitive child conversations') - .option('--sessions-root ', 'Local Codex sessions directory (default: CODEX_HOME/sessions)') + .option( + '--sessions-root ', + 'Local Codex sessions directory (default: $CODEX_HOME/sessions, else ~/.codex/sessions)' + ) .option('--topic ', 'Case-insensitive literal match, preserving whole matching turns') - .option('--after ', 'Inclusive YYYY-MM-DD lower bound; preserves whole matching turns') - .option('--before ', 'Inclusive YYYY-MM-DD upper bound; preserves whole matching turns') - .option('--max-bytes ', 'Maximum complete output bytes (1–1048576)', '8192') + .option('--after ', 'Inclusive YYYY-MM-DD lower bound, as a UTC date; preserves whole matching turns') + .option('--before ', 'Inclusive YYYY-MM-DD upper bound, as a UTC date; preserves whole matching turns') + .option( + '--max-bytes ', + `Maximum complete output bytes (1-${HISTORY_MAX_OUTPUT_BYTES})`, + String(HISTORY_DEFAULT_OUTPUT_BYTES) + ) .option( '--scan-bytes ', - 'Total metadata and selected transcript read budget (1–67108864)', + `Read budget for the selected transcripts (1-${HISTORY_MAX_SCAN_BYTES})`, String(HISTORY_DEFAULT_SCAN_BYTES) ) + .option( + '--metadata-bytes ', + `Read budget for attributing every rollout in the store (1-${HISTORY_MAX_METADATA_BYTES})`, + String(HISTORY_DEFAULT_METADATA_BYTES) + ) + .option( + '--max-entries ', + `Directory entries visited in the store (1-${HISTORY_MAX_ENTRIES})`, + String(HISTORY_DEFAULT_MAX_ENTRIES) + ) .option('--format ', 'json or text', 'json') - .option('--json', 'JSON output (also the default)') - .action(async (options: Record) => { + .option('--json', 'JSON output (the default; conflicts with --format text)') + .action(async (options: Record, command: Command) => { try { - const format = options.json ? 'json' : options.format; - if (format !== 'json' && format !== 'text') throw new Error('--format must be json or text'); + if (options.format !== 'json' && options.format !== 'text') throw new Error('--format must be json or text'); + if (options.json && options.format === 'text' && command.getOptionValueSource('format') === 'cli') + throw new Error('--json conflicts with --format text; pass one of them'); + const format = options.format; const result = await readLocalHistory({ file: options.file as string | undefined, sessionId: options.sessionId as string | undefined, @@ -37,6 +83,8 @@ export function registerHistoryCommands(program: Command): Command { after: options.after as string | undefined, before: options.before as string | undefined, scanBytes: Number(options.scanBytes), + metadataBytes: Number(options.metadataBytes), + maxEntries: Number(options.maxEntries), }); program.configureOutput().writeOut?.(`${formatLocalHistory(result, format, Number(options.maxBytes))}\n`); } catch (error) { diff --git a/src/local-history.ts b/src/local-history.ts index ac7fb04f..e6162c0d 100644 --- a/src/local-history.ts +++ b/src/local-history.ts @@ -1,26 +1,117 @@ -/** Offline, local-operator Codex history. No HTTP routes or running session required. */ +/** + * @fileoverview Offline reader for saved Codex conversations, behind + * `codeman history read` (`cli-history.ts`). It reads rollout JSONL files from + * disk directly: no HTTP route, no running server and no managed session. + * + * ## Selecting conversations + * + * Exactly one selector: `file` (one rollout, no store search), `sessionId` (a + * native thread id or unique prefix of at least 8 characters) or + * `rootSessionId` (a full id plus every transitive descendant). Identity always + * comes from each file's `session_meta`, never from its filename. + * + * ## Budgets + * + * Three separate bounds, each raisable from the CLI up to a fixed maximum, and + * each failing closed with an error rather than returning a partial answer: + * + * - `metadataBytes`: the head reads that attribute every rollout in the store. + * A `session_meta` line embeds Codex's base instructions (about 20 KiB on + * every rollout measured), so this scales with the store, not the query, + * and is kept apart from the transcript budget: a lookup on a store of + * thousands of rollouts must not spend the budget meant for the transcripts + * it returns. Each file's head read is also capped at 128 KiB. + * - `scanBytes`: the selected transcripts, read whole. + * - `maxEntries`: directory entries visited. + * + * ## Forks and sub-agents + * + * A thread Codex forks or spawns as a sub-agent can carry a copy of its + * parent's history in its own file (codex-rs `ForkPersistence::Copied`, written + * by `record_initial_history` in `core/src/session/mod.rs`; the copy keeps the + * parent's `session_meta`, see `keep_forked_rollout_item` in + * `core/src/agent/control/spawn.rs`). Such a file starts with its own + * `session_meta`, then the parent's, then the copied records, then one + * `thread_settings_applied` event, then the child's own records. The copy + * already lives in the parent's own file, so it is dropped here: + * + * - Paginated history (`history_mode: "paginated"`) numbers every record with + * an `ordinal` and declares `subagent_history_start_ordinal`; records below + * it are inherited, exactly as Codex's own projection treats them + * (`thread-store/src/local/thread_history_materialization.rs`). A paginated + * thread with a `history_base` references its prefix in another file and has + * nothing inherited to drop. + * - Legacy history has no ordinals. The prefix runs from the inherited + * `session_meta` to the `thread_settings_applied` event Codex appends after + * the copy. A legacy copy keeps the parent's own `thread_settings_applied` + * events too, so when more than one candidate follows, the boundary is the + * last one whose preceding messages are still a prefix of the parent's own + * messages, read from the parent's file. Without the parent's file that + * cannot be proven: the cut then falls at the LAST candidate, so parent text + * is never attributed to the child, and the result says so + * (`unverifiedPrefixes`, `complete: false`). + * + * A later `session_meta` must be the file's own id or a declared ancestor + * (`parent_thread_id`, `forked_from_id`, or the shared root `session_id`, and + * transitively the ancestors those declare); any other id still refuses the + * whole file rather than attribute one conversation's answer to another. + * + * ## Live rollouts + * + * A running Codex appends to its rollout. Reads take a snapshot: the file's + * identity (device and inode) must match the attributed one and its size may + * only have grown; everything up to the last complete line is read, and an + * unterminated tail is reported as a partial record. + * + * ## Format + * + * The line shapes and the injected-context rule are shared with the Past + * Sessions scanner through `codex-rollout-format.ts`. + * + * @module local-history + */ import { constants } from 'node:fs'; import { isUtf8 } from 'node:buffer'; -import { lstat, open, opendir, type FileHandle } from 'node:fs/promises'; -import { homedir } from 'node:os'; +import { lstat, open, opendir, stat, type FileHandle } from 'node:fs/promises'; import { join, resolve } from 'node:path'; -export const HISTORY_DEFAULT_SCAN_BYTES = 16 * 1024 * 1024; -export const HISTORY_MAX_SCAN_BYTES = 64 * 1024 * 1024; -export const HISTORY_MAX_OUTPUT_BYTES = 1024 * 1024; +import { codexMessageFromLine, codexSessionsRoot, isCodexInjectedContext } from './codex-rollout-format.js'; + +const KIB = 1024; +const MIB = 1024 * KIB; +const GIB = 1024 * MIB; + +export const HISTORY_DEFAULT_SCAN_BYTES = 16 * MIB; +export const HISTORY_MAX_SCAN_BYTES = 256 * MIB; +export const HISTORY_DEFAULT_METADATA_BYTES = 1 * GIB; +export const HISTORY_MAX_METADATA_BYTES = 16 * GIB; +export const HISTORY_DEFAULT_MAX_ENTRIES = 65536; +export const HISTORY_MAX_ENTRIES = 1048576; +export const HISTORY_DEFAULT_OUTPUT_BYTES = 8192; +export const HISTORY_MAX_OUTPUT_BYTES = 1 * MIB; +/** Per-file head window for `session_meta`. The line itself runs to about 20 KiB. */ +const METADATA_HEAD_BYTES = 128 * KIB; +/** Read the head in steps, so a typical rollout costs one step past its session_meta line. */ +const METADATA_CHUNK_BYTES = 32 * KIB; +/** Codex writes `sessions/YYYY/MM/DD`; this leaves five levels of headroom. */ +const MAX_DIRECTORY_DEPTH = 8; + const ID = /^[0-9a-f]{8}(?:[0-9a-f-]{0,28})$/i; const FULL_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -const METADATA_BYTES = 128 * 1024; type Json = Record; + export interface LocalHistoryOptions { file?: string; sessionId?: string; rootSessionId?: string; sessionsRoot?: string; topic?: string; + /** Inclusive UTC date, `YYYY-MM-DD`. */ after?: string; + /** Inclusive UTC date, `YYYY-MM-DD`. */ before?: string; scanBytes?: number; + metadataBytes?: number; maxEntries?: number; } export interface HistoryRecord { @@ -28,16 +119,23 @@ export interface HistoryRecord { sessionId: string; timestamp: string; role: 'user' | 'assistant'; - phase: string; + /** Codex's own phase for assistant messages (`commentary`, `final_answer`); null when none was recorded. */ + phase: string | null; message: string; } interface Source { path: string; sessionId: string; parentId?: string; + forkedFromId?: string; rootId?: string; + /** `subagent_history_start_ordinal`: records below it are inherited. */ + historyStart?: number; timestamp: string; - fingerprint?: string; + /** `dev:ino` of the attributed file. */ + identity: string; + /** Size when attributed; a later read may only see it grow. */ + size: number; } export interface LocalHistoryResult { source: 'codex-jsonl'; @@ -45,25 +143,49 @@ export interface LocalHistoryResult { sessions: Array<{ sessionId: string; rootSessionId: string | null; parentSessionId?: string; path: string }>; records: HistoryRecord[]; scan: { + /** Transcript bytes read (`--scan-bytes`). */ bytesRead: number; + /** Head bytes read to attribute rollouts (`--metadata-bytes`). */ + metadataBytesRead: number; entries: number; malformedRecords: number; partialRecords: number; skippedSymlinks: number; unattributedFiles: number; + unreadableDirectories: number; + /** Inherited fork or sub-agent prefixes dropped (informational). */ + inheritedPrefixes: number; + /** Inherited prefixes that never reached their end marker. */ + incompletePrefixes: number; + /** Legacy prefixes whose end could not be checked against the parent's own file. */ + unverifiedPrefixes: number; }; } interface Budget { - remaining: number; + transcriptRemaining: number; + metadataRemaining: number; entriesRemaining: number; scan: LocalHistoryResult['scan']; } + +/** A budget ran out: always fatal, never counted. */ +class HistoryBudgetError extends Error {} +/** One store file could not be attributed: fatal for `--file`, counted for a store scan. */ +class UnattributableFileError extends Error {} + function object(value: unknown): Json { return value && typeof value === 'object' && !Array.isArray(value) ? (value as Json) : {}; } function string(value: unknown): string | undefined { return typeof value === 'string' ? value : undefined; } +function fullId(value: unknown): string | undefined { + const id = string(value); + return id && FULL_ID.test(id) ? id.toLowerCase() : undefined; +} +function fileIdentity(value: { dev: number; ino: number }): string { + return `${value.dev}:${value.ino}`; +} function validDate(value: string | undefined): boolean { return ( value === undefined || @@ -72,6 +194,14 @@ function validDate(value: string | undefined): boolean { new Date(value).toISOString().slice(0, 10) === value) ); } +/** The UTC calendar date of a native timestamp, or undefined when it does not parse. */ +function utcDate(timestamp: string): string | undefined { + const ms = Date.parse(timestamp); + return Number.isFinite(ms) ? new Date(ms).toISOString().slice(0, 10) : undefined; +} +function checkBudget(value: number, max: number, flag: string): void { + if (!Number.isSafeInteger(value) || value < 1 || value > max) throw new Error(`Invalid ${flag} (1-${max})`); +} function validate(options: LocalHistoryOptions): void { if ([options.file, options.sessionId, options.rootSessionId].filter((v) => v !== undefined).length !== 1) throw new Error('Select exactly one of --file, --session-id, or --root-session-id'); @@ -84,13 +214,11 @@ function validate(options: LocalHistoryOptions): void { !validDate(options.before) || (options.after && options.before && options.after > options.before) ) - throw new Error('Invalid date or date range (use YYYY-MM-DD)'); + throw new Error('Invalid date or date range (use YYYY-MM-DD, UTC)'); if (options.topic !== undefined && !options.topic.trim()) throw new Error('Invalid empty topic'); - for (const [value, max] of [ - [options.scanBytes ?? HISTORY_DEFAULT_SCAN_BYTES, HISTORY_MAX_SCAN_BYTES], - [options.maxEntries ?? 8192, 65536], - ]) - if (!Number.isSafeInteger(value) || value < 1 || value > max) throw new Error('Invalid scan budget'); + checkBudget(options.scanBytes ?? HISTORY_DEFAULT_SCAN_BYTES, HISTORY_MAX_SCAN_BYTES, '--scan-bytes'); + checkBudget(options.metadataBytes ?? HISTORY_DEFAULT_METADATA_BYTES, HISTORY_MAX_METADATA_BYTES, '--metadata-bytes'); + checkBudget(options.maxEntries ?? HISTORY_DEFAULT_MAX_ENTRIES, HISTORY_MAX_ENTRIES, '--max-entries'); } async function regularFile(path: string): Promise { // O_NOFOLLOW prevents a final-component link swap between lstat and open. fstat @@ -103,88 +231,148 @@ async function regularFile(path: string): Promise { } return handle; } -async function readChunk(handle: FileHandle, position: number, size: number, budget: Budget): Promise { - if (size > budget.remaining) - throw new Error('History scan byte budget exhausted; narrow the store or raise --scan-bytes'); +async function readAt(handle: FileHandle, position: number, size: number): Promise { const buffer = Buffer.alloc(size); - const { bytesRead } = await handle.read(buffer, 0, size, position); - budget.remaining -= bytesRead; - budget.scan.bytesRead += bytesRead; - return buffer.subarray(0, bytesRead); + let filled = 0; + while (filled < size) { + const { bytesRead } = await handle.read(buffer, filled, size - filled, position + filled); + if (!bytesRead) break; + filled += bytesRead; + } + return buffer.subarray(0, filled); } -function metadata(value: unknown, path: string): Source | undefined { +async function readMetadataChunk(handle: FileHandle, position: number, size: number, budget: Budget) { + if (size > budget.metadataRemaining) + throw new HistoryBudgetError('History metadata byte budget exhausted; raise --metadata-bytes'); + const chunk = await readAt(handle, position, size); + budget.metadataRemaining -= chunk.length; + budget.scan.metadataBytesRead += chunk.length; + return chunk; +} +async function readTranscript(handle: FileHandle, size: number, budget: Budget): Promise { + if (size > budget.transcriptRemaining) + throw new HistoryBudgetError('History scan byte budget exhausted; narrow the selection or raise --scan-bytes'); + const buffer = await readAt(handle, 0, size); + budget.transcriptRemaining -= buffer.length; + budget.scan.bytesRead += buffer.length; + return buffer; +} + +type Identity = Omit; +function metadata(value: unknown, path: string): Identity | undefined { const entry = object(value); if (entry.type !== 'session_meta') return undefined; const p = object(entry.payload); - const sessionId = string(p.id) ?? string(p.session_id); - if (!sessionId || !FULL_ID.test(sessionId)) return undefined; - const source = object(p.source); - const spawn = object(object(source.subagent).thread_spawn); - const parentId = string(p.parent_thread_id) ?? string(p.parent_session_id) ?? string(spawn.parent_thread_id); - const declaredRoot = string(p.root_session_id) ?? string(p.session_id); + const sessionId = fullId(p.id) ?? fullId(p.session_id); + if (!sessionId) return undefined; + const spawn = object(object(object(p.source).subagent).thread_spawn); + const forkedFromId = fullId(p.forked_from_id); + // A fork with no spawn parent (a `/fork` in the TUI) still descends from its source thread. + const parentId = + fullId(p.parent_thread_id) ?? fullId(p.parent_session_id) ?? fullId(spawn.parent_thread_id) ?? forkedFromId; + // Codex: "session_id is equal to the root thread's ID". + const declaredRoot = fullId(p.root_session_id) ?? fullId(p.session_id); + const start = p.subagent_history_start_ordinal; return { path, - sessionId: sessionId.toLowerCase(), - parentId: parentId && FULL_ID.test(parentId) ? parentId.toLowerCase() : undefined, - rootId: - declaredRoot && declaredRoot !== sessionId && FULL_ID.test(declaredRoot) ? declaredRoot.toLowerCase() : undefined, + sessionId, + parentId, + forkedFromId, + rootId: declaredRoot && declaredRoot !== sessionId ? declaredRoot : undefined, + historyStart: Number.isSafeInteger(start) && (start as number) >= 0 ? (start as number) : undefined, timestamp: string(p.timestamp) ?? string(entry.timestamp) ?? '', }; } +function sameIdentity(a: Identity, b: Identity): boolean { + return ( + a.sessionId === b.sessionId && + a.parentId === b.parentId && + a.forkedFromId === b.forkedFromId && + a.rootId === b.rootId && + a.historyStart === b.historyStart + ); +} + +/** + * The first `session_meta` within the file's head. Problems with the file + * itself throw {@link UnattributableFileError}; a budget throws + * {@link HistoryBudgetError}; a short file with no metadata yet returns undefined. + */ async function readMetadata(path: string, budget: Budget): Promise { - const handle = await regularFile(path); + let handle: FileHandle; + try { + handle = await regularFile(path); + } catch (error) { + throw new UnattributableFileError(error instanceof Error ? error.message : String(error)); + } try { const before = await handle.stat(); - const parts: Buffer[] = []; - let offset = 0; + const limit = Math.min(before.size, METADATA_HEAD_BYTES); + let buffer = Buffer.alloc(0); let parsed = 0; - while (offset < before.size && offset < METADATA_BYTES) { - const chunk = await readChunk( + while (buffer.length < limit) { + const chunk = await readMetadataChunk( handle, - offset, - Math.min(4096, before.size - offset, METADATA_BYTES - offset), + buffer.length, + Math.min(METADATA_CHUNK_BYTES, limit - buffer.length), budget ); if (!chunk.length) break; - parts.push(chunk); - offset += chunk.length; - const buffer = Buffer.concat(parts); + buffer = Buffer.concat([buffer, chunk]); let end = buffer.indexOf(10, parsed); while (end >= 0) { const line = buffer.subarray(parsed, end); - if (!isUtf8(line)) throw new Error('History metadata contains invalid UTF8'); - try { - const found = metadata(JSON.parse(line.toString('utf8')), path); - if (found) { - const after = await handle.stat(); - const fingerprint = (value: typeof before) => - `${value.dev}:${value.ino}:${value.size}:${value.mtimeMs}:${value.ctimeMs}`; - if (fingerprint(before) !== fingerprint(after)) - throw new Error('History source changed during metadata scan'); - return { ...found, fingerprint: fingerprint(before) }; - } - } catch (error) { - if (!(error instanceof SyntaxError)) throw error; - } parsed = end + 1; end = buffer.indexOf(10, parsed); + if (!isUtf8(line)) throw new UnattributableFileError('History metadata contains invalid UTF8'); + let value: unknown; + try { + value = JSON.parse(line.toString('utf8')); + } catch { + continue; + } + const found = metadata(value, path); + if (!found) continue; + // session_meta is written once; a growing file is a live rollout, not a change. + const after = await handle.stat(); + if (fileIdentity(after) !== fileIdentity(before) || after.size < before.size) + throw new UnattributableFileError('History source changed during metadata scan'); + return { ...found, identity: fileIdentity(before), size: before.size }; } } - if (offset < before.size) - throw new Error('History metadata scan budget exhausted; cannot prove family completeness'); + if (buffer.length < limit) throw new UnattributableFileError('History source changed during metadata scan'); + if (before.size > METADATA_HEAD_BYTES) + throw new UnattributableFileError('No session metadata within the first 128 KiB'); return undefined; } finally { await handle.close(); } } + async function findSources(root: string, budget: Budget): Promise { - if (!(await lstat(root)).isDirectory()) throw new Error('Sessions root must be a real directory, not a symlink'); + // The root may itself be a symlink, as the Past Sessions scanner allows + // (`codex-transcript.ts` reads it with readdir, which follows it); links + // BELOW the root are never followed by either reader. + let rootStat; + try { + rootStat = await stat(root); + } catch { + throw new Error(`Sessions root not found: ${root}`); + } + if (!rootStat.isDirectory()) throw new Error('Sessions root must be a directory'); const sources: Source[] = []; const visit = async (dir: string, depth: number): Promise => { - if (depth > 8) throw new Error('History directory scan depth budget exhausted'); - const entries = await opendir(dir); + if (depth > MAX_DIRECTORY_DEPTH) throw new HistoryBudgetError('History directory scan depth budget exhausted'); + let entries; + try { + entries = await opendir(dir); + } catch { + budget.scan.unreadableDirectories++; + return; + } for await (const entry of entries) { - if (--budget.entriesRemaining < 0) throw new Error('History directory scan entry budget exhausted'); + if (--budget.entriesRemaining < 0) + throw new HistoryBudgetError('History directory scan entry budget exhausted; raise --max-entries'); budget.scan.entries++; const path = join(dir, entry.name); if (entry.isSymbolicLink()) { @@ -193,10 +381,19 @@ async function findSources(root: string, budget: Budget): Promise { } if (entry.isDirectory()) { // Recheck immediately before descending; never intentionally follow links. - if (!(await lstat(path)).isDirectory()) throw new Error('History directory changed during scan'); + const now = await lstat(path).catch(() => undefined); + if (!now?.isDirectory()) { + budget.scan.unreadableDirectories++; + continue; + } await visit(path, depth + 1); } else if (entry.isFile() && entry.name.endsWith('.jsonl')) { - const source = await readMetadata(path, budget); + let source: Source | undefined; + try { + source = await readMetadata(path, budget); + } catch (error) { + if (!(error instanceof UnattributableFileError)) throw error; + } if (source) sources.push(source); else budget.scan.unattributedFiles++; } @@ -205,134 +402,167 @@ async function findSources(root: string, budget: Budget): Promise { await visit(root, 0); return sources; } -function contentText(value: unknown): string { - if (typeof value === 'string') return value; - if (!Array.isArray(value)) return ''; - return value - .map((block) => { - const item = object(block); - return typeof item.type === 'string' && ['text', 'input_text', 'output_text'].includes(item.type.toLowerCase()) - ? (string(item.text) ?? '') - : ''; - }) - .filter(Boolean) - .join('\n'); -} + interface Message { representation: 'event' | 'modern' | 'response'; role: 'user' | 'assistant'; message: string; - phase: string; + phase: string | null; timestamp: string; } function message(value: unknown): Message | undefined { const entry = object(value); const p = object(entry.payload); if (entry.isSidechain || p.isSidechain) return undefined; - let role: Message['role'] | undefined; - let representation: Message['representation'] = 'event'; - let text = ''; - let phase = string(p.phase) ?? 'progress'; - if (entry.type === 'event_msg') { - if (p.type === 'user_message' || p.type === 'agent_message') { - role = p.type === 'user_message' ? 'user' : 'assistant'; - text = contentText(p.message); - } else if (p.type === 'item_completed') { - representation = 'modern'; - const item = object(p.item); - if (item.type === 'UserMessage' || item.type === 'AgentMessage') { - role = item.type === 'UserMessage' ? 'user' : 'assistant'; - text = contentText(item.content); - phase = string(item.phase) ?? phase; - } - } - } else if ( - entry.type === 'response_item' && - (p.type === 'message' || p.type === undefined) && - (p.role === 'user' || p.role === 'assistant') - ) { - role = p.role; - representation = 'response'; - text = contentText(p.content); - // Codex context is not a human turn. Do not omit arbitrary markdown prompts. - if ( - role === 'user' && - /^(?:# AGENTS\.md instructions|||)/.test(text) - ) - return undefined; - } - if (!role || !text) return undefined; + const found = codexMessageFromLine(value); + if (!found) return undefined; + // Codex context is not a human turn. Real prompts also arrive as events, which are never filtered. + if (found.injectionProne && isCodexInjectedContext(found.text)) return undefined; return { - role, - message: text, - phase, - representation, + role: found.role, + message: found.text, + // Codex records a phase on assistant messages only; a user turn has none. + phase: found.role === 'assistant' ? (found.phase ?? null) : null, + representation: found.shape === 'item' ? 'modern' : found.shape, timestamp: string(entry.timestamp) ?? string(p.timestamp) ?? '', }; } -async function readMessages(source: Source, budget: Budget): Promise { - const handle = await regularFile(source.path); - try { - const before = await handle.stat(); - if (`${before.dev}:${before.ino}:${before.size}:${before.mtimeMs}:${before.ctimeMs}` !== source.fingerprint) - throw new Error('History source changed after attribution; retry explicitly'); - const buffer = await readChunk(handle, 0, before.size, budget); - const after = await handle.stat(); +/** Native events and response_item commonly duplicate one accepted message. */ +function collapse(items: Message[]): Message[] { + const records: Message[] = []; + let representations = new Set(); + for (const raw of items) { + const item = { ...raw }; + const previous = records.at(-1); if ( - buffer.length !== before.size || - before.size !== after.size || - before.mtimeMs !== after.mtimeMs || - before.ctimeMs !== after.ctimeMs - ) - throw new Error('History source changed while reading; retry explicitly'); - if (!isUtf8(buffer)) throw new Error('History source contains invalid UTF8'); - const lines = buffer.toString('utf8').split('\n'); - const partial = lines.pop(); - if (partial) budget.scan.partialRecords++; - const records: Message[] = []; - let representations = new Set(); - for (const line of lines) { - if (!line.trim()) continue; - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch { - budget.scan.malformedRecords++; - continue; - } - if (object(parsed).type === 'session_meta') { - const declared = metadata(parsed, source.path); - if ( - !declared || - declared.sessionId !== source.sessionId || - declared.parentId !== source.parentId || - declared.rootId !== source.rootId - ) - throw new Error('History source contains conflicting native metadata; attribution refused'); - continue; - } - const item = message(parsed); - if (!item) continue; - const previous = records.at(-1); - // Native events and response_item commonly duplicate one accepted message. - if ( - previous?.role === item.role && - previous.message === item.message && - !representations.has(item.representation) - ) { - representations.add(item.representation); - if (!previous.timestamp) previous.timestamp = item.timestamp; - if (item.phase === 'final_answer') previous.phase = item.phase; - continue; - } - records.push(item); - representations = new Set([item.representation]); + previous?.role === item.role && + previous.message === item.message && + !representations.has(item.representation) + ) { + representations.add(item.representation); + if (!previous.timestamp) previous.timestamp = item.timestamp; + if (item.phase === 'final_answer' || previous.phase === null) previous.phase = item.phase ?? previous.phase; + continue; } - return records; + records.push(item); + representations = new Set([item.representation]); + } + return records; +} +function isPrefixOf(prefix: Message[], whole: Message[]): boolean { + return ( + prefix.length <= whole.length && + prefix.every((item, i) => item.role === whole[i].role && item.message === whole[i].message) + ); +} + +interface ReadContext { + /** The ancestor's own messages, uncut, or undefined when its file is not in the store. */ + ancestorMessages(id: string): Promise; +} +type Piece = Message | { boundary: true }; +interface LegacyPrefix { + ancestorId: string; + /** Messages and `thread_settings_applied` boundaries after the inherited session_meta. */ + pieces: Piece[]; +} + +async function readMessages(source: Source, budget: Budget, context: ReadContext, raw = false): Promise { + const handle = await regularFile(source.path); + let buffer: Buffer; + try { + const now = await handle.stat(); + // A live rollout only grows. A different file or a shorter one is a rewrite. + if (fileIdentity(now) !== source.identity || now.size < source.size) + throw new Error('History source changed after attribution; retry explicitly'); + buffer = await readTranscript(handle, now.size, budget); + if (buffer.length !== now.size) throw new Error('History source shrank while reading; retry explicitly'); } finally { await handle.close(); } + // Snapshot up to the last complete line: a running Codex may be mid-append. + const end = buffer.lastIndexOf(10) + 1; + const complete = buffer.subarray(0, end); + if (buffer.subarray(end).toString('latin1').trim()) budget.scan.partialRecords++; + if (!isUtf8(complete)) throw new Error('History source contains invalid UTF8'); + + const ancestors = new Set([source.parentId, source.forkedFromId, source.rootId].filter((id): id is string => !!id)); + const own: Message[] = []; + let legacy: LegacyPrefix | undefined; + let lastOrdinal = -1; + for (const line of complete.toString('utf8').split('\n')) { + if (!line.trim()) continue; + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + budget.scan.malformedRecords++; + continue; + } + const entry = object(parsed); + const ordinal = typeof entry.ordinal === 'number' ? entry.ordinal : undefined; + if (ordinal !== undefined) lastOrdinal = Math.max(lastOrdinal, ordinal); + if (source.historyStart !== undefined && ordinal === undefined) { + // Codex's projection skips a paginated record without an ordinal; so does this, visibly. + budget.scan.malformedRecords++; + continue; + } + const inherited = source.historyStart !== undefined && ordinal! < source.historyStart; + if (entry.type === 'session_meta') { + const declared = metadata(parsed, source.path); + if (declared && declared.sessionId === source.sessionId && sameIdentity(declared, source)) continue; + if (!declared || declared.sessionId === source.sessionId || !ancestors.has(declared.sessionId)) + throw new Error('History source contains conflicting native metadata; attribution refused'); + // An ancestor that was itself a fork carries its own ancestors' metadata in the copy. + for (const id of [declared.parentId, declared.forkedFromId, declared.rootId]) if (id) ancestors.add(id); + if (raw || inherited || legacy) continue; + budget.scan.inheritedPrefixes++; + legacy = { ancestorId: declared.sessionId, pieces: [] }; + continue; + } + if (inherited && !raw) continue; + if (legacy && entry.type === 'event_msg' && object(entry.payload).type === 'thread_settings_applied') { + legacy.pieces.push({ boundary: true }); + continue; + } + const item = message(parsed); + if (!item) continue; + if (legacy) legacy.pieces.push(item); + else own.push(item); + } + if (!raw && source.historyStart !== undefined && source.historyStart > 0) { + budget.scan.inheritedPrefixes++; + // Initialization died while copying the prefix: Codex refuses to resume such a file. + if (lastOrdinal < source.historyStart - 1) budget.scan.incompletePrefixes++; + } + if (legacy) own.push(...(await afterLegacyPrefix(legacy, budget, context))); + return collapse(own); } + +/** The child's own messages after a legacy inherited prefix. */ +async function afterLegacyPrefix(prefix: LegacyPrefix, budget: Budget, context: ReadContext): Promise { + const boundaries = prefix.pieces.flatMap((piece, i) => ('boundary' in piece ? [i] : [])); + if (!boundaries.length) { + budget.scan.incompletePrefixes++; + return []; + } + const messagesBefore = (index: number) => + collapse(prefix.pieces.slice(0, index).filter((piece): piece is Message => !('boundary' in piece))); + let cut = boundaries[0]; + if (boundaries.length > 1) { + // The last candidate whose preceding messages the parent's own file also holds. + const parent = await context.ancestorMessages(prefix.ancestorId); + const proven = parent ? boundaries.filter((candidate) => isPrefixOf(messagesBefore(candidate), parent)) : []; + if (proven.length) cut = proven.at(-1)!; + else { + // Unprovable: drop up to the last candidate, so parent text is never attributed to the child. + budget.scan.unverifiedPrefixes++; + cut = boundaries.at(-1)!; + } + } + return prefix.pieces.slice(cut + 1).filter((piece): piece is Message => !('boundary' in piece)); +} + function selectedTurns(messages: Message[], options: LocalHistoryOptions): Message[] { const turns: Message[][] = []; for (const item of messages) { @@ -345,9 +575,11 @@ function selectedTurns(messages: Message[], options: LocalHistoryOptions): Messa const dateMatches = (!options.after && !options.before) || turn.some((item) => { - const date = item.timestamp.slice(0, 10); + const date = utcDate(item.timestamp); return ( - validDate(date) && (!options.after || date >= options.after) && (!options.before || date <= options.before) + date !== undefined && + (!options.after || date >= options.after) && + (!options.before || date <= options.before) ); }); return dateMatches && (!needle || turn.some((item) => item.message.toLocaleLowerCase().includes(needle))); @@ -375,30 +607,46 @@ export async function readLocalHistory(options: LocalHistoryOptions): Promise source.sessionId.toLowerCase().startsWith(wanted)); - if (!matches.length) throw new Error('Native session missing from history store'); + const matches = all.filter((source) => source.sessionId.startsWith(wanted)); + if (!matches.length) + throw new Error( + scan.unattributedFiles + ? `Native session missing from history store (${scan.unattributedFiles} file(s) could not be attributed)` + : 'Native session missing from history store' + ); if (matches.length !== 1) throw new Error('Native session id is ambiguous'); const target = matches[0]; attributionSources = all; @@ -428,10 +676,23 @@ export async function readLocalHistory(options: LocalHistoryOptions): Promise a.timestamp.localeCompare(b.timestamp) || a.sessionId.localeCompare(b.sessionId)); const roots = new Map(sources.map((source) => [source.sessionId, nativeRoot(source, attributionSources)])); + const ancestorCache = new Map>(); + const context: ReadContext = { + // A raw read never opens a legacy prefix, so this never recurses. + ancestorMessages(id) { + const cached = ancestorCache.get(id); + if (cached) return cached; + const candidates = attributionSources.filter((source) => source.sessionId === id); + if (candidates.length !== 1) return Promise.resolve(undefined); + const read = readMessages(candidates[0], budget, context, true); + ancestorCache.set(id, read); + return read; + }, + }; const records: HistoryRecord[] = []; for (const source of sources) { records.push( - ...selectedTurns(await readMessages(source, budget), options).map((item) => ({ + ...selectedTurns(await readMessages(source, budget, context), options).map((item) => ({ rootSessionId: roots.get(source.sessionId) ?? null, sessionId: source.sessionId, role: item.role, @@ -449,6 +710,9 @@ export async function readLocalHistory(options: LocalHistoryOptions): Promise root !== null), sessions: sources.map((source) => ({ sessionId: source.sessionId, @@ -464,18 +728,18 @@ export async function readLocalHistory(options: LocalHistoryOptions): Promise HISTORY_MAX_OUTPUT_BYTES) - throw new Error('Invalid output byte budget'); + throw new Error(`Invalid --max-bytes (1-${HISTORY_MAX_OUTPUT_BYTES})`); const body = format === 'json' ? JSON.stringify(result) : [ - `source=${result.source} complete=${result.complete} scannedBytes=${result.scan.bytesRead}`, + `source=${result.source} complete=${result.complete} scannedBytes=${result.scan.bytesRead} metadataBytes=${result.scan.metadataBytesRead}`, ...result.records.map( (row) => - `[root=${row.rootSessionId} session=${row.sessionId} ${row.timestamp} ${row.role} ${row.phase}]\n${row.message}` + `[root=${row.rootSessionId} session=${row.sessionId} ${row.timestamp} ${row.role}${row.phase ? ` ${row.phase}` : ''}]\n${row.message}` ), ].join('\n\n'); if (Buffer.byteLength(body) + 1 > maxBytes) diff --git a/test/fixtures/codex-rollout.ts b/test/fixtures/codex-rollout.ts new file mode 100644 index 00000000..9f90f5c5 --- /dev/null +++ b/test/fixtures/codex-rollout.ts @@ -0,0 +1,125 @@ +/** + * @fileoverview Codex rollout lines shaped like real ones, for the offline + * history tests. The structure (keys, their order, the event types around a + * fork boundary) was taken from real rollouts written by codex 0.120 to 0.163 + * and from codex-rs `SessionMeta` (`protocol/src/protocol.rs`); no + * conversation content was copied. + * + * A forked or spawned thread's file, as `record_initial_history` writes it for + * `ForkPersistence::Copied`: + * + * own session_meta, parent session_meta, copied parent records, + * thread_settings_applied, the child's own records + * + * In paginated history every line also carries an `ordinal`, and the child's + * own `session_meta` declares `subagent_history_start_ordinal`. + */ +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +/** Codex's base instructions make every real session_meta line about 20 KiB. */ +export const BASE_INSTRUCTIONS_BYTES = 19 * 1024; + +export interface MetaOptions { + id: string; + /** Spawn parent: sets parent_thread_id, forked_from_id and source.subagent.thread_spawn. */ + parent?: string; + /** Shared root (`session_id`); defaults to `parent ?? id`, as Codex writes it. */ + root?: string; + mode?: 'legacy' | 'paginated'; + historyStart?: number; + timestamp?: string; + baseInstructionsBytes?: number; +} + +export function realMeta(options: MetaOptions): Record { + const { id, parent, mode = 'legacy', timestamp = '2026-10-24T01:00:00.000Z' } = options; + const payload: Record = { + session_id: options.root ?? parent ?? id, + id, + ...(parent ? { forked_from_id: parent, parent_thread_id: parent } : {}), + timestamp, + cwd: '/work/repo', + originator: 'codex_exec', + cli_version: '0.163.0', + source: parent + ? { + subagent: { + thread_spawn: { + parent_thread_id: parent, + depth: 1, + agent_path: null, + agent_nickname: 'Kepler', + agent_role: null, + }, + }, + } + : 'exec', + thread_source: parent ? 'subagent' : 'user', + ...(parent ? { agent_nickname: 'Kepler' } : {}), + model_provider: 'openai', + base_instructions: { text: 'b'.repeat(options.baseInstructionsBytes ?? BASE_INSTRUCTIONS_BYTES) }, + history_mode: mode, + ...(options.historyStart !== undefined ? { subagent_history_start_ordinal: options.historyStart } : {}), + ...(parent ? { multi_agent_version: 'v1' } : {}), + context_window: { window_id: `${id.slice(0, 8)}-window` }, + }; + return { timestamp, type: 'session_meta', payload }; +} + +export const taskStarted = () => ({ + timestamp: '2026-10-24T01:00:00.000Z', + type: 'event_msg', + payload: { type: 'task_started' }, +}); +export const settingsApplied = () => ({ + timestamp: '2026-10-24T01:00:00.000Z', + type: 'event_msg', + payload: { type: 'thread_settings_applied', thread_settings: { model: 'gpt-5.5', cwd: '/work/repo' } }, +}); +export const developer = () => ({ + timestamp: '2026-10-24T01:00:00.000Z', + type: 'response_item', + payload: { type: 'message', role: 'developer', content: [{ type: 'input_text', text: 'policy' }] }, +}); +export const injected = (tag = 'environment_context') => ({ + timestamp: '2026-10-24T01:00:00.000Z', + type: 'response_item', + payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: `<${tag}>\n ctx\n` }] }, +}); +/** A real user turn: the response_item Codex sent plus its user_message event twin. */ +export const userTurn = (text: string, timestamp = '2026-10-24T01:00:00.000Z') => [ + { + timestamp, + type: 'response_item', + payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text }] }, + }, + { timestamp, type: 'event_msg', payload: { type: 'user_message', message: text, images: [] } }, +]; +/** An assistant message: the event and the response_item, both carrying Codex's phase. */ +export const assistantTurn = (text: string, phase: 'commentary' | 'final_answer' = 'final_answer') => [ + { + timestamp: '2026-10-24T01:00:01.000Z', + type: 'event_msg', + payload: { type: 'agent_message', message: text, phase }, + }, + { + timestamp: '2026-10-24T01:00:01.000Z', + type: 'response_item', + payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text }], phase }, + }, +]; + +/** Number every line, as paginated history does. */ +export function withOrdinals(lines: Array>): Array> { + return lines.map((line, ordinal) => { + const { timestamp, type, payload, ...rest } = line; + return { timestamp, ordinal, type, payload, ...rest }; + }); +} + +export async function writeRollout(dir: string, name: string, lines: unknown[]): Promise { + const path = join(dir, name); + await writeFile(path, lines.map((line) => JSON.stringify(line)).join('\n') + '\n'); + return path; +} diff --git a/test/local-history-live.test.ts b/test/local-history-live.test.ts new file mode 100644 index 00000000..aa0348a6 --- /dev/null +++ b/test/local-history-live.test.ts @@ -0,0 +1,129 @@ +/** + * @fileoverview The offline history reader against rollouts that change while + * it reads them: a running Codex appending to its own rollout (or a child's), + * an unrelated rollout rewritten mid-scan, and a selected rollout replaced + * outright. `node:fs/promises` is wrapped so a test can act at an exact point + * of the read (after the open, after a read call) without timing races. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { appendFile, mkdtemp, rename, rm, truncate, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +type Hook = (path: string, openCount: number, handle: import('node:fs/promises').FileHandle) => Promise | void; +const hooks: { onOpen?: Hook } = {}; +const opens = new Map(); + +vi.mock('node:fs/promises', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + open: async (...args: Parameters) => { + const handle = await actual.open(...args); + const path = String(args[0]); + const count = (opens.get(path) ?? 0) + 1; + opens.set(path, count); + await hooks.onOpen?.(path, count, handle); + return handle; + }, + }; +}); + +const history = await import('../src/local-history.js'); +const { assistantTurn, realMeta, userTurn, writeRollout } = await import('./fixtures/codex-rollout.js'); + +const ROOT = 'aaaaaaaa-1111-4111-8111-111111111111'; +const CHILD = 'bbbbbbbb-1111-4111-8111-111111111111'; +let dir: string; +const line = (value: unknown) => `${JSON.stringify(value)}\n`; + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'local-history-live-')); + opens.clear(); + hooks.onOpen = undefined; +}); +afterEach(async () => { + hooks.onOpen = undefined; + await rm(dir, { recursive: true, force: true }); +}); + +describe('rollouts that change while they are read', () => { + it('reads a snapshot of a rollout a running Codex appends to after attribution', async () => { + const path = await writeRollout(dir, 'rollout-root.jsonl', [realMeta({ id: ROOT }), ...userTurn('question')]); + hooks.onOpen = async (opened, count) => { + // The second open is the transcript read; Codex writes a whole record and starts the next. + if (opened === path && count === 2) { + await appendFile(path, assistantTurn('answer').map(line).join('')); + await appendFile(path, '{"timestamp":"2026-10-24T01:00:02.000Z","type":"event_'); + } + }; + const result = await history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir }); + expect(result.records.map((r) => r.message)).toEqual(['question', 'answer']); + // The record cut mid-write is reported, not parsed and not fatal. + expect(result.scan.partialRecords).toBe(1); + expect(result.scan.malformedRecords).toBe(0); + expect(result.complete).toBe(false); + }); + + it('reads a family whose child is still running', async () => { + await writeRollout(dir, 'rollout-root.jsonl', [realMeta({ id: ROOT }), ...userTurn('root question')]); + const child = await writeRollout(dir, 'rollout-child.jsonl', [ + realMeta({ id: CHILD, parent: ROOT, mode: 'legacy' }), + ...userTurn('child task'), + ]); + hooks.onOpen = async (opened, count) => { + // Grows during the metadata scan and again before its transcript read. + if (opened === child) + await appendFile(child, assistantTurn(`child progress ${count}`, 'commentary').map(line).join('')); + }; + const result = await history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir }); + expect(result.records.map((r) => r.message)).toEqual([ + 'root question', + 'child task', + 'child progress 1', + 'child progress 2', + ]); + expect(result.complete).toBe(true); + }); + + it('counts an unrelated rollout that shrinks during its head read as unattributed', async () => { + await writeRollout(dir, 'rollout-root.jsonl', [realMeta({ id: ROOT }), ...userTurn('question')]); + const other = await writeRollout(dir, 'rollout-other.jsonl', [realMeta({ id: CHILD }), ...userTurn('other')]); + hooks.onOpen = async (opened, count, handle) => { + if (opened !== other || count !== 1) return; + const read = handle.read.bind(handle) as (...args: unknown[]) => Promise; + (handle as unknown as { read: (...args: unknown[]) => Promise }).read = async (...args) => { + const result = await read(...args); + await truncate(other, 10); + return result; + }; + }; + const result = await history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir }); + expect(result.records.map((r) => r.message)).toEqual(['question']); + expect(result.scan.unattributedFiles).toBe(1); + expect(result.complete).toBe(false); + }); + + it('still refuses a selected rollout replaced by another file after attribution', async () => { + const path = await writeRollout(dir, 'rollout-root.jsonl', [realMeta({ id: ROOT }), ...userTurn('question')]); + hooks.onOpen = async (opened, count) => { + if (opened !== path || count !== 1) return; + const replacement = join(dir, 'replacement.tmp'); + await writeFile(replacement, [realMeta({ id: ROOT }), ...userTurn('rewritten')].map(line).join('')); + await rename(replacement, path); + }; + await expect(history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir })).rejects.toThrow( + /changed after attribution/ + ); + }); + + it('still refuses a selected rollout truncated after attribution', async () => { + const path = await writeRollout(dir, 'rollout-root.jsonl', [realMeta({ id: ROOT }), ...userTurn('question')]); + hooks.onOpen = async (opened, count) => { + if (opened === path && count === 2) await truncate(path, 100); + }; + await expect(history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir })).rejects.toThrow( + /changed after attribution/ + ); + }); +}); diff --git a/test/local-history.test.ts b/test/local-history.test.ts index 7ffa4c4b..0a7cf6dc 100644 --- a/test/local-history.test.ts +++ b/test/local-history.test.ts @@ -1,10 +1,21 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises'; +import { chmod, mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import * as history from '../src/local-history.js'; import { Command } from 'commander'; import { registerHistoryCommands } from '../src/cli-history.js'; +import { + assistantTurn, + developer, + injected, + realMeta, + settingsApplied, + taskStarted, + userTurn, + withOrdinals, + writeRollout, +} from './fixtures/codex-rollout.js'; const ROOT = 'aaaaaaaa-1111-4111-8111-111111111111'; const CHILD = 'bbbbbbbb-1111-4111-8111-111111111111'; @@ -155,9 +166,19 @@ describe('offline native history', () => { expect(result.complete).toBe(false); expect(result.scan.unattributedFiles).toBe(1); }); - it('refuses conflicting session metadata instead of attributing another conversation answer', async () => { + it('refuses a later session_meta whose id is not a declared ancestor', async () => { + // ROOT declares no parent, fork source or shared root, so CHILD can only be another conversation. const file = await fixture(ROOT, [row('user', 'root question'), meta(CHILD), row('assistant', 'child answer')]); await expect(history.readLocalHistory({ file })).rejects.toThrow(/conflicting.*metadata/); + // A fork child is refused too when the inherited metadata names a thread it does not descend from. + const forked = await writeRollout(dir, 'unrelated-fork.jsonl', [ + realMeta({ id: CHILD, parent: ROOT, mode: 'legacy' }), + realMeta({ id: GRANDCHILD, mode: 'legacy' }), + row('assistant', 'not ours'), + settingsApplied(), + row('assistant', 'child answer'), + ]); + await expect(history.readLocalHistory({ file: forked })).rejects.toThrow(/conflicting.*metadata/); }); it('accepts repeated metadata only when native identity and genealogy agree', async () => { const file = await fixture(ROOT, [row('user', 'question'), meta(ROOT), row('assistant', 'answer')]); @@ -244,3 +265,450 @@ describe('offline native history', () => { expect(program.commands.find((c) => c.name() === 'history')?.commands.map((c) => c.name())).toEqual(['read']); }); }); + +const messages = (result: history.LocalHistoryResult) => + result.records.map((r) => `${r.sessionId.slice(0, 1)}:${r.message}`); + +/** A parent R, and a legacy child C whose file copies R's history before its own, as Codex writes it. */ +async function legacyFamily(root: string, parentExtra: unknown[] = [], copiedExtra: unknown[] = []) { + await mkdir(root, { recursive: true }); + const parentHistory = [ + taskStarted(), + developer(), + injected(), + ...userTurn('parent question'), + ...assistantTurn('parent commentary', 'commentary'), + ...assistantTurn('parent answer'), + ]; + await writeRollout(root, `rollout-${ROOT}.jsonl`, [ + realMeta({ id: ROOT }), + ...parentHistory, + ...parentExtra, + ...userTurn('parent later question'), + ...assistantTurn('parent later answer'), + ]); + return writeRollout(root, `rollout-${CHILD}.jsonl`, [ + realMeta({ id: CHILD, parent: ROOT }), + realMeta({ id: ROOT }), + ...parentHistory, + ...copiedExtra, + settingsApplied(), + taskStarted(), + ...userTurn('child task'), + ...assistantTurn('child commentary', 'commentary'), + ...assistantTurn('child answer'), + ]); +} + +describe('forked and spawned threads', () => { + it('drops the parent history a legacy fork copies, so the child reads as itself', async () => { + const childFile = await legacyFamily(dir); + const child = await history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir }); + expect(messages(child)).toEqual(['b:child task', 'b:child commentary', 'b:child answer']); + expect(child.complete).toBe(true); + expect(child.scan.inheritedPrefixes).toBe(1); + expect(child.sessions[0]).toMatchObject({ sessionId: CHILD, parentSessionId: ROOT, rootSessionId: ROOT }); + // The whole family: every message once, the copy not repeated under the child. + const family = await history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir }); + expect(messages(family)).toEqual([ + 'a:parent question', + 'a:parent commentary', + 'a:parent answer', + 'a:parent later question', + 'a:parent later answer', + 'b:child task', + 'b:child commentary', + 'b:child answer', + ]); + expect(family.complete).toBe(true); + // An explicit file needs no store: the copy ends at the one thread_settings_applied. + const file = await history.readLocalHistory({ file: childFile }); + expect(messages(file)).toEqual(['b:child task', 'b:child commentary', 'b:child answer']); + expect(file.complete).toBe(true); + }); + + it('finds the end of a legacy copy that carries the parent own thread_settings_applied events', async () => { + // The parent compacted (or checkpointed its first turn) before the fork, so the copy holds that event too. + const parentSettings = [ + settingsApplied(), + ...userTurn('parent second question'), + ...assistantTurn('parent second answer'), + ]; + const childFile = await legacyFamily(dir, parentSettings, parentSettings); + const child = await history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir }); + expect(messages(child)).toEqual(['b:child task', 'b:child commentary', 'b:child answer']); + expect(child.complete).toBe(true); + // Without the parent's file the boundary cannot be proven: nothing of the parent is attributed to the child. + const alone = await history.readLocalHistory({ file: childFile }); + expect(messages(alone)).toEqual(['b:child task', 'b:child commentary', 'b:child answer']); + expect(alone.scan.unverifiedPrefixes).toBe(1); + expect(alone.complete).toBe(false); + }); + + it('keeps the child own later thread_settings_applied events on its side of the cut', async () => { + await legacyFamily(dir); + const path = join(dir, `rollout-${CHILD}.jsonl`); + const { appendFile } = await import('node:fs/promises'); + await appendFile( + path, + [settingsApplied(), ...userTurn('child second task'), ...assistantTurn('child second answer')] + .map((line) => JSON.stringify(line)) + .join('\n') + '\n' + ); + const child = await history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir }); + expect(messages(child)).toEqual([ + 'b:child task', + 'b:child commentary', + 'b:child answer', + 'b:child second task', + 'b:child second answer', + ]); + expect(child.complete).toBe(true); + }); + + it('accepts the copied metadata of an ancestor that was itself a fork', async () => { + const file = await writeRollout(dir, 'grandchild.jsonl', [ + realMeta({ id: GRANDCHILD, parent: CHILD, root: ROOT }), + realMeta({ id: CHILD, parent: ROOT }), + realMeta({ id: ROOT }), + ...userTurn('root question'), + settingsApplied(), + ...userTurn('child question'), + settingsApplied(), + ...userTurn('grandchild task'), + ...assistantTurn('grandchild answer'), + ]); + const result = await history.readLocalHistory({ file }); + expect(messages(result)).toEqual(['c:grandchild task', 'c:grandchild answer']); + expect(result.records[0].rootSessionId).toBe(ROOT); + }); + + it('marks a legacy copy that never reached its end marker incomplete and keeps none of it', async () => { + const file = await writeRollout(dir, 'cut.jsonl', [ + realMeta({ id: CHILD, parent: ROOT }), + realMeta({ id: ROOT }), + ...userTurn('parent question'), + ]); + await expect(history.readLocalHistory({ file })).rejects.toThrow(/evidence/); + await writeRollout(dir, 'cut.jsonl', [ + realMeta({ id: CHILD, parent: ROOT }), + ...userTurn('child before copy'), + realMeta({ id: ROOT }), + ...userTurn('parent question'), + ]); + const result = await history.readLocalHistory({ file }); + expect(messages(result)).toEqual(['b:child before copy']); + expect(result.scan.incompletePrefixes).toBe(1); + expect(result.complete).toBe(false); + }); + + it('drops the inherited records of a paginated sub-agent by ordinal, as Codex projects them', async () => { + const inherited = [ + realMeta({ id: CHILD, parent: ROOT, mode: 'paginated', historyStart: 7 }), + realMeta({ id: ROOT, mode: 'paginated' }), + taskStarted(), + injected(), + ...userTurn('parent question'), + ...assistantTurn('parent answer').slice(1), + ]; + expect(inherited).toHaveLength(7); + const lines = withOrdinals([ + ...inherited, + settingsApplied(), + taskStarted(), + ...assistantTurn('child commentary', 'commentary'), + ...assistantTurn('child answer'), + ]); + await writeRollout( + dir, + `rollout-${ROOT}.jsonl`, + withOrdinals([realMeta({ id: ROOT, mode: 'paginated' }), ...userTurn('parent question')]) + ); + await writeRollout(dir, `rollout-${CHILD}.jsonl`, lines); + const child = await history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir }); + expect(messages(child)).toEqual(['b:child commentary', 'b:child answer']); + expect(child.records.map((r) => r.phase)).toEqual(['commentary', 'final_answer']); + expect(child.complete).toBe(true); + const family = await history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir }); + expect(messages(family)).toEqual(['a:parent question', 'b:child commentary', 'b:child answer']); + }); + + it('reports a paginated copy that ends before its declared start ordinal', async () => { + const file = await writeRollout( + dir, + 'paginated-cut.jsonl', + withOrdinals([ + realMeta({ id: CHILD, parent: ROOT, mode: 'paginated', historyStart: 9 }), + realMeta({ id: ROOT, mode: 'paginated' }), + ...userTurn('parent question'), + ...assistantTurn('parent answer'), + ]) + ); + await expect(history.readLocalHistory({ file })).rejects.toThrow(/evidence/); + // In a family read the parent still answers, and the cut child makes the result incomplete. + await writeRollout( + dir, + `rollout-${ROOT}.jsonl`, + withOrdinals([realMeta({ id: ROOT, mode: 'paginated' }), ...userTurn('parent question')]) + ); + const family = await history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir }); + expect(messages(family)).toEqual(['a:parent question']); + expect(family.scan.incompletePrefixes).toBe(1); + expect(family.complete).toBe(false); + await rm(join(dir, `rollout-${ROOT}.jsonl`)); + // A paginated record without an ordinal is skipped, visibly, as Codex's projection skips it. + const lines = withOrdinals([ + realMeta({ id: CHILD, parent: ROOT, mode: 'paginated', historyStart: 2 }), + realMeta({ id: ROOT, mode: 'paginated' }), + settingsApplied(), + ...assistantTurn('child answer'), + ]); + await writeRollout(dir, 'paginated-cut.jsonl', [...lines, assistantTurn('stray')[0]]); + const result = await history.readLocalHistory({ file }); + expect(messages(result)).toEqual(['b:child answer']); + expect(result.scan.malformedRecords).toBe(1); + expect(result.complete).toBe(false); + }); +}); + +describe('large and untidy stores', () => { + it('finds one id among thousands of rollouts with real-sized session_meta at the default budgets', async () => { + const count = 3500; + const base = realMeta({ id: ROOT }); + const pad = JSON.stringify(base).length; + expect(pad).toBeGreaterThan(19 * 1024); + const writes: Array> = []; + for (let i = 0; i < count; i++) { + const day = join(dir, '2026', String(1 + (i % 12)).padStart(2, '0'), String(1 + (i % 28)).padStart(2, '0')); + const id = `${i.toString(16).padStart(8, '0')}-2222-4222-8222-222222222222`; + writes.push( + mkdir(day, { recursive: true }).then(() => + writeFile( + join(day, `rollout-${id}.jsonl`), + [realMeta({ id }), ...userTurn(`question ${i}`), ...assistantTurn(`answer ${i}`)] + .map((line) => JSON.stringify(line)) + .join('\n') + '\n' + ) + ) + ); + } + await Promise.all(writes); + const target = `${(count - 1).toString(16).padStart(8, '0')}-2222-4222-8222-222222222222`; + const result = await history.readLocalHistory({ sessionId: target, sessionsRoot: dir }); + expect(messages(result)).toEqual([`0:question ${count - 1}`, `0:answer ${count - 1}`]); + expect(result.complete).toBe(true); + // Metadata is charged to its own budget; the transcript budget only paid for the one rollout. + expect(result.scan.metadataBytesRead).toBeGreaterThan(history.HISTORY_MAX_SCAN_BYTES / 4); + expect(result.scan.bytesRead).toBeLessThan(64 * 1024); + // The store-wide caps fail closed and are raisable. + await expect( + history.readLocalHistory({ sessionId: target, sessionsRoot: dir, metadataBytes: 1024 * 1024 }) + ).rejects.toThrow(/metadata.*--metadata-bytes/); + await expect(history.readLocalHistory({ sessionId: target, sessionsRoot: dir, maxEntries: 100 })).rejects.toThrow( + /entry budget.*--max-entries/ + ); + }, 60_000); + + it('reads only the head of each rollout to attribute it, not its whole window', async () => { + await writeRollout(dir, 'rollout-big.jsonl', [ + realMeta({ id: ROOT }), + ...Array.from({ length: 400 }, (_, i) => assistantTurn(`${i} ${'x'.repeat(2048)}`)).flat(), + ]); + const result = await history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir, scanBytes: 8 * 1024 * 1024 }); + expect(result.records).toHaveLength(400); + // One 32 KiB step holds the ~20 KiB session_meta line; the 128 KiB window is only the ceiling. + expect(result.scan.metadataBytesRead).toBe(32 * 1024); + }); + + it.each([ + ['invalid UTF8 in its head', Buffer.from([0x7b, 0xff, 0xfe, 0x7d, 0x0a])], + [ + 'no session_meta within 128 KiB', + Buffer.from(`${JSON.stringify({ type: 'event_msg', payload: {} })}\n`.repeat(6000)), + ], + ])('counts one unrelated file with %s as unattributed instead of failing the lookup', async (_label, bytes) => { + await fixture(ROOT, [row('user', 'question'), row('assistant', 'answer')]); + await writeFile(join(dir, 'rollout-broken.jsonl'), bytes); + const result = await history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir }); + expect(result.records.map((r) => r.message)).toEqual(['question', 'answer']); + expect(result.scan.unattributedFiles).toBe(1); + expect(result.complete).toBe(false); + // The same file named explicitly is still an error. + await expect(history.readLocalHistory({ file: join(dir, 'rollout-broken.jsonl') })).rejects.toThrow(/UTF8|128 KiB/); + }); + + it('counts an unreadable directory instead of failing the lookup', async () => { + await fixture(ROOT, [row('assistant', 'answer')]); + const locked = join(dir, 'locked'); + await mkdir(locked); + await chmod(locked, 0); + try { + const result = await history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir }); + expect(result.records[0].message).toBe('answer'); + expect(result.scan.unreadableDirectories).toBe(process.getuid?.() === 0 ? 0 : 1); + } finally { + await chmod(locked, 0o700); + } + }); + + it('follows a sessions root that is itself a symlink, and still skips links below it', async () => { + const real = join(dir, 'real-sessions'); + await mkdir(real); + await writeRollout(real, 'rollout-root.jsonl', [ + realMeta({ id: ROOT }), + ...userTurn('question'), + ...assistantTurn('answer'), + ]); + const linkedRoot = join(dir, 'sessions-link'); + await symlink(real, linkedRoot); + const result = await history.readLocalHistory({ sessionId: ROOT, sessionsRoot: linkedRoot }); + expect(result.records.map((r) => r.message)).toEqual(['question', 'answer']); + expect(result.complete).toBe(true); + await symlink(join(real, 'rollout-root.jsonl'), join(real, 'rollout-link.jsonl')); + const withLink = await history.readLocalHistory({ sessionId: ROOT, sessionsRoot: linkedRoot }); + expect(withLink.scan.skippedSymlinks).toBe(1); + await expect( + history.readLocalHistory({ sessionId: ROOT, sessionsRoot: join(real, 'rollout-root.jsonl') }) + ).rejects.toThrow(/must be a directory/); + }); +}); + +describe('records, filters and output', () => { + it('gives user records no phase and keeps the phase Codex recorded on assistant messages', async () => { + const file = await writeRollout(dir, 'phases.jsonl', [ + realMeta({ id: ROOT }), + ...userTurn('question'), + ...assistantTurn('working', 'commentary'), + ...assistantTurn('done'), + row('assistant', 'legacy answer without a phase'), + ]); + const result = await history.readLocalHistory({ file }); + expect(result.records.map((r) => [r.role, r.phase])).toEqual([ + ['user', null], + ['assistant', 'commentary'], + ['assistant', 'final_answer'], + ['assistant', null], + ]); + }); + + it('drops every injected-context wrapper but never a typed prompt that starts with #', async () => { + const file = await writeRollout(dir, 'injected.jsonl', [ + realMeta({ id: ROOT }), + injected('environment_context'), + injected('recommended_plugins'), + injected('subagent_notification'), + { + type: 'response_item', + payload: { + type: 'message', + role: 'user', + content: [{ type: 'input_text', text: '# AGENTS.md instructions for /x' }], + }, + }, + ...userTurn('# a heading the user typed'), + ...assistantTurn('answer'), + ]); + const result = await history.readLocalHistory({ file }); + expect(result.records.map((r) => r.message)).toEqual(['# a heading the user typed', 'answer']); + }); + + it('compares date bounds as UTC calendar dates', async () => { + const file = await writeRollout(dir, 'dates.jsonl', [ + realMeta({ id: ROOT }), + // 23:30 at UTC-2 is 01:30 UTC the next day. + ...userTurn('late evening west of UTC', '2026-10-24T23:30:00.000-02:00'), + ...userTurn('plain UTC', '2026-10-24T12:00:00.000Z'), + ]); + const after = await history.readLocalHistory({ file, after: '2026-10-25' }); + expect(after.records.map((r) => r.message)).toEqual(['late evening west of UTC']); + const on = await history.readLocalHistory({ file, before: '2026-10-24' }); + expect(on.records.map((r) => r.message)).toEqual(['plain UTC']); + }); + + it('reads the default store at $CODEX_HOME/sessions, else ~/.codex/sessions', async () => { + const saved = { CODEX_HOME: process.env.CODEX_HOME, HOME: process.env.HOME }; + try { + process.env.CODEX_HOME = join(dir, 'codex-home'); + await mkdir(join(dir, 'codex-home', 'sessions', '2026', '10', '24'), { recursive: true }); + await writeRollout(join(dir, 'codex-home', 'sessions', '2026', '10', '24'), 'rollout-a.jsonl', [ + realMeta({ id: ROOT }), + ...userTurn('from CODEX_HOME'), + ]); + expect((await history.readLocalHistory({ sessionId: ROOT })).records[0].message).toBe('from CODEX_HOME'); + delete process.env.CODEX_HOME; + process.env.HOME = join(dir, 'home'); + await mkdir(join(dir, 'home', '.codex', 'sessions'), { recursive: true }); + await writeRollout(join(dir, 'home', '.codex', 'sessions'), 'rollout-b.jsonl', [ + realMeta({ id: ROOT }), + ...userTurn('from ~/.codex'), + ]); + const result = await history.readLocalHistory({ sessionId: ROOT }); + expect(result.records[0].message).toBe('from ~/.codex'); + expect(result.sessions[0].path).toBe(join(dir, 'home', '.codex', 'sessions', 'rollout-b.jsonl')); + } finally { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + }); +}); + +describe('history CLI flags', () => { + const run = async (args: string[]) => { + let output = ''; + const program = new Command().exitOverride().configureOutput({ + writeOut: (text) => { + output += text; + }, + writeErr: () => {}, + }); + registerHistoryCommands(program); + await program.parseAsync(['history', 'read', ...args], { from: 'user' }); + return output; + }; + + it('prints text with --format text, a header line and one block per record', async () => { + const file = await writeRollout(dir, 'text.jsonl', [ + realMeta({ id: ROOT }), + ...userTurn('question'), + ...assistantTurn('answer'), + ]); + const output = await run(['--file', file, '--format', 'text']); + const blocks = output.trimEnd().split('\n\n'); + expect(blocks[0]).toMatch(/^source=codex-jsonl complete=true scannedBytes=\d+ metadataBytes=\d+$/); + expect(blocks[1]).toBe(`[root=${ROOT} session=${ROOT} 2026-10-24T01:00:00.000Z user]\nquestion`); + expect(blocks[2]).toBe(`[root=${ROOT} session=${ROOT} 2026-10-24T01:00:01.000Z assistant final_answer]\nanswer`); + expect(() => JSON.parse(output)).toThrow(); + }); + + it('refuses --json together with --format text, and accepts either alone', async () => { + const file = await writeRollout(dir, 'flags.jsonl', [realMeta({ id: ROOT }), ...assistantTurn('answer')]); + await expect(run(['--file', file, '--json', '--format', 'text'])).rejects.toThrow( + /--json conflicts with --format text/ + ); + expect(JSON.parse(await run(['--file', file, '--json'])).records[0].message).toBe('answer'); + expect(JSON.parse(await run(['--file', file, '--json', '--format', 'json'])).records[0].message).toBe('answer'); + expect(await run(['--file', file, '--format', 'text'])).toMatch(/^source=codex-jsonl/); + }); + + it('passes the store caps through and validates them', async () => { + await fixture(ROOT, [row('assistant', 'answer')]); + await fixture(CHILD, [row('assistant', 'child')], ROOT); + await expect(run(['--session-id', ROOT, '--sessions-root', dir, '--max-entries', '1'])).rejects.toThrow( + /--max-entries/ + ); + expect(JSON.parse(await run(['--session-id', ROOT, '--sessions-root', dir, '--max-entries', '2'])).complete).toBe( + true + ); + await expect(run(['--session-id', ROOT, '--sessions-root', dir, '--metadata-bytes', '16'])).rejects.toThrow( + /--metadata-bytes/ + ); + await expect(run(['--session-id', ROOT, '--sessions-root', dir, '--metadata-bytes', 'lots'])).rejects.toThrow( + /Invalid --metadata-bytes/ + ); + await expect(run(['--session-id', ROOT, '--sessions-root', dir, '--max-entries', '0'])).rejects.toThrow( + /Invalid --max-entries/ + ); + }); +}); From 44fa2d47572dd243d1d0783402d7c35cc74ac4af Mon Sep 17 00:00:00 2001 From: Aamer Akhter Date: Sat, 24 Oct 2026 23:00:50 -0400 Subject: [PATCH 4/5] docs(history): document the offline reader next to codeman agent Replace docs/offline-history.md, which repeated the skill reference and was linked from nowhere, with a README section and a wiki section beside the codeman agent commands. Bring the skill reference, its plugin mirror, the Codeman Skill wiki page and the changeset in line with the reader: forks, the separate budgets and their flags, UTC dates, phases and complete:false. --- .changeset/offline-codex-history.md | 2 +- README.md | 12 ++++ docs/offline-history.md | 52 -------------- docs/wiki/Codeman-Skill.md | 8 ++- docs/wiki/Driving-Codeman-From-An-Agent.md | 26 +++++++ docs/wiki/Home.md | 2 +- .../skills/codeman/reference/history.md | 70 +++++++++++++------ skills/codeman/reference/history.md | 70 +++++++++++++------ 8 files changed, 142 insertions(+), 100 deletions(-) delete mode 100644 docs/offline-history.md diff --git a/.changeset/offline-codex-history.md b/.changeset/offline-codex-history.md index 57090231..9ed54d37 100644 --- a/.changeset/offline-codex-history.md +++ b/.changeset/offline-codex-history.md @@ -2,4 +2,4 @@ 'aicodeman': minor --- -Add offline `codeman history read` for saved Codex transcripts by file, native session ID, or conversation family, with whole-turn topic/date filters and bounded local reads and output. +Add offline `codeman history read` for saved Codex transcripts by file, native session ID, or conversation family (spawned sub-agents and forks included), with whole-turn topic and UTC date filters. A forked or spawned thread's copy of its parent's history is dropped, so every record belongs to the thread that wrote it. Store attribution has its own budget (`--metadata-bytes`), separate from the transcript budget (`--scan-bytes`), and every bound is raisable (`--max-entries` too), so id lookups keep working on stores of thousands of rollouts. A store file that cannot be attributed, an unreadable directory, or a rollout a running Codex is still appending to no longer fails the lookup; each is reported through `complete: false`. Assistant records carry Codex's own phase and user records none. The Past Sessions scanner and the offline reader now share one Codex rollout format module, including the injected-context rule. diff --git a/README.md b/README.md index bb3c142c..3fc1a495 100644 --- a/README.md +++ b/README.md @@ -996,6 +996,18 @@ codeman agent rm "$SID" # any session except t Rules the commands enforce rather than document: they refuse outside a Codeman session and never guess a URL; `send` transmits printable text plus Enter only (a control byte such as `Ctrl+C` is `app_exit` in opencode — ESC exists solely as `interrupt`, which never appends Enter); `rm` refuses an empty id, an unprovable self id and a prefix match in either direction. Ids may be the 8-character prefixes `ls` prints (resolved through the list; an ambiguous prefix refuses, anything shorter than 8 characters refuses on every verb). A prompt that starts with `-` goes after `--` (`send "$SID" -- "- fix the bug"`). The echo of the prompt you sent is output too: a `--match` marker that appears verbatim in the prompt matches at once, before the worker has done anything, so the prompt asks for it in halves. A remote session whose host is asleep answers a fire-and-forget `send` with `buffered` (Codeman wakes the host and types the prompt once the pane is back) or `dropped` (over the wake buffer's cap, nothing will be typed: exit `1`). Exit codes: `0` delivered/matched/signal, `1` error, `2` timeout, `3` the worker exited or the wait ended without an answer (`delivered:false`, `ended:true`), `4` refused. `spawn` prints the id alone on stdout (prose goes to stderr), so `SID=$(…)` captures exactly the id. `--json` prints the envelope's `data` for every verb. `--until stop` on a mode without hook signals is the server's 400, passed through — the marker path (`--match`) is the answer there, exactly as for the skill. +### `codeman history read`: saved Codex conversations, offline + +`codeman history read` reads Codex's own saved rollouts (`$CODEX_HOME/sessions`, else `~/.codex/sessions`) straight from disk: no server, no managed session, nothing written. Unlike `codeman agent` it works anywhere, including on a machine where Codeman is not running. + +```bash +codeman history read --session-id 019fc604 # one conversation, by native id or 8+ character prefix +codeman history read --root-session-id --topic auth # a conversation and every sub-agent or fork under it +codeman history read --file /path/to/rollout.jsonl --format text +``` + +Filters keep whole user/assistant turns (`--topic` is a literal, case-insensitive match; `--after`/`--before` are inclusive UTC dates). A forked or spawned thread's copy of its parent's history is dropped, so every record belongs to the thread that wrote it. Output is JSON by default and carries `complete: false` whenever the evidence is partial (a file that could not be attributed, a record a running Codex is still writing, an unknown ancestor). Every bound fails closed and is raisable: `--metadata-bytes` (attributing the store, default 1 GiB), `--max-entries` (default 65536), `--scan-bytes` (the selected transcripts, default 16 MiB) and `--max-bytes` (output, default 8192). The full reference is [`skills/codeman/reference/history.md`](skills/codeman/reference/history.md). + ### MCP server (`codeman mcp`): drive Codeman from any MCP client `codeman mcp` is a [Model Context Protocol](https://modelcontextprotocol.io) server on stdio, so any MCP client (Claude Code, Claude Desktop, Codex, Gemini CLI, Cursor, ...) can start Codeman sessions in any installed agent CLI, prompt them, wait for them, read their answers and clean up, from inside Codeman or from outside it. Like `codeman agent`, it is a client of the HTTP API: no new route, auth and ownership unchanged. diff --git a/docs/offline-history.md b/docs/offline-history.md deleted file mode 100644 index 219aabc1..00000000 --- a/docs/offline-history.md +++ /dev/null @@ -1,52 +0,0 @@ -# Saved Codex history (offline) - -Use `codeman history read` for retained local Codex JSONL transcripts. This command -works outside a managed session and never calls the HTTP API. Do not bootstrap -API credentials or require `CODEMAN_MUX` for this read-only job. - -Choose exactly one selector: - -```bash -codeman history read --file /path/to/rollout.jsonl -codeman history read --session-id aaaaaaaa --sessions-root /path/to/sessions -codeman history read --root-session-id aaaaaaaa-1111-4111-8111-111111111111 \ - --topic 'authentication' --after 2026-10-01 --before 2026-10-24 \ - --max-bytes 65536 -``` - -`--session-id` accepts a native conversation UUID or unique prefix of at least -8 characters; identity comes from metadata, not filenames. `--root-session-id` -requires a full native UUID and includes transitive descendants found in metadata. -The default store is `$CODEX_HOME/sessions`, or `~/.codex/sessions` when unset. -`--file` selects one file and does not search for its ancestors or children. - -Topic matching is a case-insensitive literal substring. Dates are inclusive -`YYYY-MM-DD` bounds. Each filter selects whole user/assistant turns; a turn is -retained if it has a date match and a topic match, which may be different messages -in that turn. Tool calls/results and injected context are omitted. Adjacent duplicate -native representations are collapsed; repeated messages from the same -representation are retained. - -JSON is the default (`--json` or `--format json`); `--format text` is available. -Results identify source paths, native session/root IDs, timestamps, roles and phases, -plus scan counters and `complete`. A root ID is `null` when ancestry cannot be proven. -Report that uncertainty; never relabel the queried child as its own root. -`complete: false` reports malformed or partial records, skipped links, unattributed -files or missing ancestors. It does not prove the store includes every conversation -ever created, and a filtered result is only evidence for its selected scope. - -Reads require regular UTF8 files and reject final-component symlinks, conflicting -metadata, ambiguous IDs, cyclic ancestry, invalid selectors and changed selected -files. Directory traversal skips symlinks. Parent directories of an explicitly -supplied path must be trusted; this is not a filesystem sandbox. - -The default total metadata/transcript scan budget is 16 MiB; `--scan-bytes` allows -1–67108864 bytes. Directory scans are capped at 8192 entries and depth 8. The output -budget is 8192 bytes by default; `--max-bytes` allows 1–1048576 bytes, including the -final newline. Exhausted budgets fail with an error: output is never silently -truncated or split into partial JSON or turns. Narrow the store/filters or explicitly -raise the relevant budget. Missing evidence is an error, not an empty success. - -This command does not inspect live panes, stage prompts, submit input or manage -sessions. Live session operations use the separate `codeman agent` commands and -require the managed-session environment and running server. diff --git a/docs/wiki/Codeman-Skill.md b/docs/wiki/Codeman-Skill.md index dcd57f08..6417606e 100644 --- a/docs/wiki/Codeman-Skill.md +++ b/docs/wiki/Codeman-Skill.md @@ -123,8 +123,11 @@ message a worker directly instead of typing into its terminal. ## What it will and will not do -- **It self-gates.** Outside a Codeman session (`CODEMAN_MUX` unset) it refuses to act and - never guesses an API URL, so a global install costs an unrelated agent session nothing. +- **It self-gates.** Outside a Codeman session (`CODEMAN_MUX` unset) it refuses every live + API operation and never guesses an API URL, so a global install cannot drive a server it + is not part of. The one exception is reading saved Codex conversations with + `codeman history read` (see [Driving Codeman From An Agent](Driving-Codeman-From-An-Agent#reading-saved-codex-conversations)), + a local, read-only command that needs no server or session. - **Unprompted, it may only** start sessions, prompt them, and delete the ones **it created in that conversation, by exact id**, behind a guard that refuses to delete the agent's own session. @@ -153,6 +156,7 @@ The skill checks for both and says so instead of acting. | [`reference/recipes.md`](https://github.com/Ark0N/Codeman/blob/master/skills/codeman/reference/recipes.md) | Worked flows: claude, DeepSeek Harness and shell workers, fan-out, watching a blocked worker, messaging. Loaded on demand. | | [`reference/endpoints.md`](https://github.com/Ark0N/Codeman/blob/master/skills/codeman/reference/endpoints.md) | Endpoint tables, error codes, the signals each mode emits, capacity limits. Loaded on demand. | | [`reference/messaging.md`](https://github.com/Ark0N/Codeman/blob/master/skills/codeman/reference/messaging.md) | Talking to claude workers directly through Claude Code's cross-session messaging. Loaded on demand. | +| [`reference/history.md`](https://github.com/Ark0N/Codeman/blob/master/skills/codeman/reference/history.md) | Reading saved Codex conversations offline with `codeman history read`: selectors, filters, forks, budgets. Loaded on demand. | Only `SKILL.md` sits in the agent's context; the references load when a job needs them, so a quick run stays quick. diff --git a/docs/wiki/Driving-Codeman-From-An-Agent.md b/docs/wiki/Driving-Codeman-From-An-Agent.md index 6b2b337f..6e7ce85c 100644 --- a/docs/wiki/Driving-Codeman-From-An-Agent.md +++ b/docs/wiki/Driving-Codeman-From-An-Agent.md @@ -70,6 +70,32 @@ codeman agent rm "$SID" # any session except t - **Exit codes** are the same for every verb: `0` done, `1` error, `2` timeout, `3` the worker exited, `4` refused. `--json` prints the response's `data`. +## Reading saved Codex conversations + +`codeman history read` is the one command here that needs no server and no Codeman +session: it reads Codex's own saved rollouts (`$CODEX_HOME/sessions`, else +`~/.codex/sessions`) from disk and writes nothing. + +```bash +codeman history read --session-id 019fc604 # one conversation, by id or 8+ character prefix +codeman history read --root-session-id --topic auth # it and every sub-agent or fork under it +codeman history read --file /path/to/rollout.jsonl --format text +``` + +- **Filters** keep whole user/assistant turns: `--topic` is a literal, case-insensitive + match, and `--after`/`--before` are inclusive UTC dates. +- **Forks and sub-agents**: a child's copy of its parent's history is dropped, so every + record belongs to the thread that wrote it. +- **`complete: false`** in the JSON means the evidence is partial: a store file that could + not be attributed, a record a running Codex is still writing, an ancestor not found. +- **Budgets** fail closed and are raisable: `--metadata-bytes` (attributing the store, + default 1 GiB), `--max-entries` (65536), `--scan-bytes` (the selected transcripts, + 16 MiB) and `--max-bytes` (output, 8192). `--json` and `--format text` together are an + error. + +The full reference ships with the skill as +[`reference/history.md`](https://github.com/Ark0N/Codeman/blob/master/skills/codeman/reference/history.md). + ## The MCP server `codeman mcp` is a [Model Context Protocol](https://modelcontextprotocol.io) server, so any MCP diff --git a/docs/wiki/Home.md b/docs/wiki/Home.md index 50c108bb..3a0cd566 100644 --- a/docs/wiki/Home.md +++ b/docs/wiki/Home.md @@ -42,7 +42,7 @@ codeman web # then open http://localhost:3000 **Driving it from code** - [The Codeman Skill](Codeman-Skill) - one prompt, a whole crew: an agent starting and supervising worker sessions. -- [Driving Codeman From An Agent](Driving-Codeman-From-An-Agent) - the `codeman agent` commands, the MCP server, wait primitives. +- [Driving Codeman From An Agent](Driving-Codeman-From-An-Agent) - the `codeman agent` commands, the MCP server, wait primitives, reading saved Codex conversations offline. - [HTTP API](HTTP-API) - the envelope, auth, the endpoint map, SSE events. - [Hooks And Integrations](Hooks-And-Integrations) - events flowing back into Codeman. diff --git a/plugins/codeman/skills/codeman/reference/history.md b/plugins/codeman/skills/codeman/reference/history.md index 0fa87205..bcdfc92c 100644 --- a/plugins/codeman/skills/codeman/reference/history.md +++ b/plugins/codeman/skills/codeman/reference/history.md @@ -16,36 +16,62 @@ codeman history read --root-session-id aaaaaaaa-1111-4111-8111-111111111111 \ `--session-id` accepts a native conversation UUID or unique prefix of at least 8 characters; identity comes from metadata, not filenames. `--root-session-id` -requires a full native UUID and includes transitive descendants found in metadata. -The default store is `$CODEX_HOME/sessions`, or `~/.codex/sessions` when unset. -`--file` selects one file and does not search for its ancestors or children. +requires a full native UUID and includes transitive descendants found in metadata +(spawned sub-agents and forks). The default store is `$CODEX_HOME/sessions`, or +`~/.codex/sessions` when unset; a store directory that is itself a symlink is +followed, links inside it are skipped. `--file` selects one file and does not +search for its ancestors or children. Topic matching is a case-insensitive literal substring. Dates are inclusive -`YYYY-MM-DD` bounds. Each filter selects whole user/assistant turns; a turn is -retained if it has a date match and a topic match, which may be different messages -in that turn. Tool calls/results and injected context are omitted. Adjacent duplicate -native representations are collapsed; repeated messages from the same -representation are retained. +`YYYY-MM-DD` bounds compared as UTC calendar dates. Each filter selects whole +user/assistant turns; a turn is retained if it has a date match and a topic match, +which may be different messages in that turn. Tool calls/results and injected +context (rows Codex adds that start with `#` or `<`, such as AGENTS.md and +``) are omitted. Adjacent duplicate native representations +are collapsed; repeated messages from the same representation are retained. -JSON is the default (`--json` or `--format json`); `--format text` is available. +A forked or spawned thread's file starts with a copy of its parent's history. That +copy is dropped (it is in the parent's own file), so each record belongs to the +thread that wrote it. A later `session_meta` for any thread that is not a declared +ancestor still refuses the file. + +JSON is the default (`--json` or `--format json`); `--format text` prints one block +per record. `--json` together with `--format text` is an error. Results identify source paths, native session/root IDs, timestamps, roles and phases, -plus scan counters and `complete`. A root ID is `null` when ancestry cannot be proven. +plus scan counters and `complete`. `phase` is Codex's own (`commentary` or +`final_answer`) on assistant records and `null` on user records or when none was +recorded. A root ID is `null` when ancestry cannot be proven. Report that uncertainty; never relabel the queried child as its own root. -`complete: false` reports malformed or partial records, skipped links, unattributed -files or missing ancestors. It does not prove the store includes every conversation -ever created, and a filtered result is only evidence for its selected scope. +`complete: false` reports malformed or partial records (a running Codex may be +mid-write), skipped links, unattributed files, unreadable directories, missing +ancestors, or an inherited copy whose end could not be found or proven. It does not +prove the store includes every conversation ever created, and a filtered result is +only evidence for its selected scope. Reads require regular UTF8 files and reject final-component symlinks, conflicting -metadata, ambiguous IDs, cyclic ancestry, invalid selectors and changed selected -files. Directory traversal skips symlinks. Parent directories of an explicitly -supplied path must be trusted; this is not a filesystem sandbox. +metadata, ambiguous IDs, cyclic ancestry and invalid selectors. A store file that +cannot be attributed (invalid UTF8 in its head, no metadata in its first 128 KiB, +changing while its head is read) is counted in `unattributedFiles` instead of +failing the lookup. A selected rollout that a running Codex is appending to is read +as a snapshot up to its last complete line; one that was replaced or truncated +fails. Parent directories of an explicitly supplied path must be trusted; this is +not a filesystem sandbox. -The default total metadata/transcript scan budget is 16 MiB; `--scan-bytes` allows -1–67108864 bytes. Directory scans are capped at 8192 entries and depth 8. The output -budget is 8192 bytes by default; `--max-bytes` allows 1–1048576 bytes, including the -final newline. Exhausted budgets fail with an error: output is never silently -truncated or split into partial JSON or turns. Narrow the store/filters or explicitly -raise the relevant budget. Missing evidence is an error, not an empty success. +Budgets, each failing with an error rather than a partial answer: + +| Flag | Bounds | Default | Maximum | +| ---- | ------ | ------- | ------- | +| `--metadata-bytes` | head reads attributing every rollout in the store | 1 GiB | 16 GiB | +| `--max-entries` | directory entries visited | 65536 | 1048576 | +| `--scan-bytes` | the selected transcripts | 16 MiB | 256 MiB | +| `--max-bytes` | output, including the final newline | 8192 | 1048576 | + +Each rollout's `session_meta` carries Codex's base instructions (about 20 KiB), so +attributing a store costs about that much per rollout, charged to +`--metadata-bytes`, never to `--scan-bytes`. Directory depth is capped at 8 (Codex +writes `YYYY/MM/DD`). Output is never silently truncated or split into partial +JSON or turns. Narrow the store/filters or explicitly raise the relevant budget. +Missing evidence is an error, not an empty success. This command does not inspect live panes, stage prompts, submit input or manage sessions. For live operations return to the managed-session guard in the skill. diff --git a/skills/codeman/reference/history.md b/skills/codeman/reference/history.md index 0fa87205..bcdfc92c 100644 --- a/skills/codeman/reference/history.md +++ b/skills/codeman/reference/history.md @@ -16,36 +16,62 @@ codeman history read --root-session-id aaaaaaaa-1111-4111-8111-111111111111 \ `--session-id` accepts a native conversation UUID or unique prefix of at least 8 characters; identity comes from metadata, not filenames. `--root-session-id` -requires a full native UUID and includes transitive descendants found in metadata. -The default store is `$CODEX_HOME/sessions`, or `~/.codex/sessions` when unset. -`--file` selects one file and does not search for its ancestors or children. +requires a full native UUID and includes transitive descendants found in metadata +(spawned sub-agents and forks). The default store is `$CODEX_HOME/sessions`, or +`~/.codex/sessions` when unset; a store directory that is itself a symlink is +followed, links inside it are skipped. `--file` selects one file and does not +search for its ancestors or children. Topic matching is a case-insensitive literal substring. Dates are inclusive -`YYYY-MM-DD` bounds. Each filter selects whole user/assistant turns; a turn is -retained if it has a date match and a topic match, which may be different messages -in that turn. Tool calls/results and injected context are omitted. Adjacent duplicate -native representations are collapsed; repeated messages from the same -representation are retained. +`YYYY-MM-DD` bounds compared as UTC calendar dates. Each filter selects whole +user/assistant turns; a turn is retained if it has a date match and a topic match, +which may be different messages in that turn. Tool calls/results and injected +context (rows Codex adds that start with `#` or `<`, such as AGENTS.md and +``) are omitted. Adjacent duplicate native representations +are collapsed; repeated messages from the same representation are retained. -JSON is the default (`--json` or `--format json`); `--format text` is available. +A forked or spawned thread's file starts with a copy of its parent's history. That +copy is dropped (it is in the parent's own file), so each record belongs to the +thread that wrote it. A later `session_meta` for any thread that is not a declared +ancestor still refuses the file. + +JSON is the default (`--json` or `--format json`); `--format text` prints one block +per record. `--json` together with `--format text` is an error. Results identify source paths, native session/root IDs, timestamps, roles and phases, -plus scan counters and `complete`. A root ID is `null` when ancestry cannot be proven. +plus scan counters and `complete`. `phase` is Codex's own (`commentary` or +`final_answer`) on assistant records and `null` on user records or when none was +recorded. A root ID is `null` when ancestry cannot be proven. Report that uncertainty; never relabel the queried child as its own root. -`complete: false` reports malformed or partial records, skipped links, unattributed -files or missing ancestors. It does not prove the store includes every conversation -ever created, and a filtered result is only evidence for its selected scope. +`complete: false` reports malformed or partial records (a running Codex may be +mid-write), skipped links, unattributed files, unreadable directories, missing +ancestors, or an inherited copy whose end could not be found or proven. It does not +prove the store includes every conversation ever created, and a filtered result is +only evidence for its selected scope. Reads require regular UTF8 files and reject final-component symlinks, conflicting -metadata, ambiguous IDs, cyclic ancestry, invalid selectors and changed selected -files. Directory traversal skips symlinks. Parent directories of an explicitly -supplied path must be trusted; this is not a filesystem sandbox. +metadata, ambiguous IDs, cyclic ancestry and invalid selectors. A store file that +cannot be attributed (invalid UTF8 in its head, no metadata in its first 128 KiB, +changing while its head is read) is counted in `unattributedFiles` instead of +failing the lookup. A selected rollout that a running Codex is appending to is read +as a snapshot up to its last complete line; one that was replaced or truncated +fails. Parent directories of an explicitly supplied path must be trusted; this is +not a filesystem sandbox. -The default total metadata/transcript scan budget is 16 MiB; `--scan-bytes` allows -1–67108864 bytes. Directory scans are capped at 8192 entries and depth 8. The output -budget is 8192 bytes by default; `--max-bytes` allows 1–1048576 bytes, including the -final newline. Exhausted budgets fail with an error: output is never silently -truncated or split into partial JSON or turns. Narrow the store/filters or explicitly -raise the relevant budget. Missing evidence is an error, not an empty success. +Budgets, each failing with an error rather than a partial answer: + +| Flag | Bounds | Default | Maximum | +| ---- | ------ | ------- | ------- | +| `--metadata-bytes` | head reads attributing every rollout in the store | 1 GiB | 16 GiB | +| `--max-entries` | directory entries visited | 65536 | 1048576 | +| `--scan-bytes` | the selected transcripts | 16 MiB | 256 MiB | +| `--max-bytes` | output, including the final newline | 8192 | 1048576 | + +Each rollout's `session_meta` carries Codex's base instructions (about 20 KiB), so +attributing a store costs about that much per rollout, charged to +`--metadata-bytes`, never to `--scan-bytes`. Directory depth is capped at 8 (Codex +writes `YYYY/MM/DD`). Output is never silently truncated or split into partial +JSON or turns. Narrow the store/filters or explicitly raise the relevant budget. +Missing evidence is an error, not an empty success. This command does not inspect live panes, stage prompts, submit input or manage sessions. For live operations return to the managed-session guard in the skill. From 17b2edce2203fdea33cb72b15236d5d73f9466de Mon Sep 17 00:00:00 2001 From: Aamer Akhter Date: Sun, 25 Oct 2026 09:08:54 -0400 Subject: [PATCH 5/5] fix(history): count compressed rollouts, read each file once, degrade an unreadable parent Codex's local_thread_store_compression rewrites cold rollouts to *.jsonl.zst, which the store scan skipped without a trace. Each one is now counted in compressedFiles (complete: false), skipped only when its plain .jsonl sibling exists, as Codex's own discovery does, and a lookup that misses names the compressed file whose name carries the wanted id. A Zstandard file given to --file, or sitting in the store under a .jsonl name, is recognised by its magic number. Each selected rollout is parsed once into every view later needed of it, so a legacy copy's boundary proof reuses the parent's own read instead of charging it to --scan-bytes a second time (and counting its malformed or inherited records twice). A parent read that fails for any reason other than a budget now leaves the boundary unproven (cut at the last candidate, unverifiedPrefixes, complete: false) instead of failing a child whose own file is intact. A budget error still fails the read. --max-bytes is validated before the first file is read, next to the other budgets, and every budget flag accepts plain decimal integers only. --- .changeset/offline-codex-history.md | 2 +- README.md | 2 +- docs/wiki/Driving-Codeman-From-An-Agent.md | 3 +- .../skills/codeman/reference/history.md | 25 ++- skills/codeman/reference/history.md | 25 ++- src/cli-history.ts | 22 ++- src/local-history.ts | 154 ++++++++++++++---- test/local-history-live.test.ts | 30 +++- test/local-history.test.ts | 139 ++++++++++++++++ 9 files changed, 346 insertions(+), 56 deletions(-) diff --git a/.changeset/offline-codex-history.md b/.changeset/offline-codex-history.md index 9ed54d37..f0a2eb0c 100644 --- a/.changeset/offline-codex-history.md +++ b/.changeset/offline-codex-history.md @@ -2,4 +2,4 @@ 'aicodeman': minor --- -Add offline `codeman history read` for saved Codex transcripts by file, native session ID, or conversation family (spawned sub-agents and forks included), with whole-turn topic and UTC date filters. A forked or spawned thread's copy of its parent's history is dropped, so every record belongs to the thread that wrote it. Store attribution has its own budget (`--metadata-bytes`), separate from the transcript budget (`--scan-bytes`), and every bound is raisable (`--max-entries` too), so id lookups keep working on stores of thousands of rollouts. A store file that cannot be attributed, an unreadable directory, or a rollout a running Codex is still appending to no longer fails the lookup; each is reported through `complete: false`. Assistant records carry Codex's own phase and user records none. The Past Sessions scanner and the offline reader now share one Codex rollout format module, including the injected-context rule. +Add offline `codeman history read` for saved Codex transcripts by file, native session ID, or conversation family (spawned sub-agents and forks included), with whole-turn topic and UTC date filters. A forked or spawned thread's copy of its parent's history is dropped, so every record belongs to the thread that wrote it. Store attribution has its own budget (`--metadata-bytes`), separate from the transcript budget (`--scan-bytes`), and every bound is raisable (`--max-entries` too), so id lookups keep working on stores of thousands of rollouts. A store file that cannot be attributed, an unreadable directory, or a rollout a running Codex is still appending to no longer fails the lookup; each is reported through `complete: false`, as is a Codex rollout compressed to `.jsonl.zst` (counted in `compressedFiles` and named when a lookup misses). A parent file read to prove where a forked copy ends is charged to `--scan-bytes` once, and a parent that cannot be read leaves that boundary unproven instead of failing the child. Assistant records carry Codex's own phase and user records none. The Past Sessions scanner and the offline reader now share one Codex rollout format module, including the injected-context rule. diff --git a/README.md b/README.md index 3fc1a495..90412d5c 100644 --- a/README.md +++ b/README.md @@ -1006,7 +1006,7 @@ codeman history read --root-session-id --topic auth # a conversa codeman history read --file /path/to/rollout.jsonl --format text ``` -Filters keep whole user/assistant turns (`--topic` is a literal, case-insensitive match; `--after`/`--before` are inclusive UTC dates). A forked or spawned thread's copy of its parent's history is dropped, so every record belongs to the thread that wrote it. Output is JSON by default and carries `complete: false` whenever the evidence is partial (a file that could not be attributed, a record a running Codex is still writing, an unknown ancestor). Every bound fails closed and is raisable: `--metadata-bytes` (attributing the store, default 1 GiB), `--max-entries` (default 65536), `--scan-bytes` (the selected transcripts, default 16 MiB) and `--max-bytes` (output, default 8192). The full reference is [`skills/codeman/reference/history.md`](skills/codeman/reference/history.md). +Filters keep whole user/assistant turns (`--topic` is a literal, case-insensitive match; `--after`/`--before` are inclusive UTC dates). A forked or spawned thread's copy of its parent's history is dropped, so every record belongs to the thread that wrote it. Output is JSON by default and carries `complete: false` whenever the evidence is partial (a file that could not be attributed, a compressed `.jsonl.zst` rollout, a record a running Codex is still writing, an unknown ancestor). Every bound fails closed and is raisable: `--metadata-bytes` (attributing the store, default 1 GiB), `--max-entries` (default 65536), `--scan-bytes` (the selected transcripts, default 16 MiB) and `--max-bytes` (output, default 8192). The full reference is [`skills/codeman/reference/history.md`](skills/codeman/reference/history.md). ### MCP server (`codeman mcp`): drive Codeman from any MCP client diff --git a/docs/wiki/Driving-Codeman-From-An-Agent.md b/docs/wiki/Driving-Codeman-From-An-Agent.md index 6e7ce85c..1a594682 100644 --- a/docs/wiki/Driving-Codeman-From-An-Agent.md +++ b/docs/wiki/Driving-Codeman-From-An-Agent.md @@ -87,7 +87,8 @@ codeman history read --file /path/to/rollout.jsonl --format text - **Forks and sub-agents**: a child's copy of its parent's history is dropped, so every record belongs to the thread that wrote it. - **`complete: false`** in the JSON means the evidence is partial: a store file that could - not be attributed, a record a running Codex is still writing, an ancestor not found. + not be attributed, a compressed `.jsonl.zst` rollout (not read), a record a running + Codex is still writing, an ancestor not found. - **Budgets** fail closed and are raisable: `--metadata-bytes` (attributing the store, default 1 GiB), `--max-entries` (65536), `--scan-bytes` (the selected transcripts, 16 MiB) and `--max-bytes` (output, 8192). `--json` and `--format text` together are an diff --git a/plugins/codeman/skills/codeman/reference/history.md b/plugins/codeman/skills/codeman/reference/history.md index bcdfc92c..ae087fb6 100644 --- a/plugins/codeman/skills/codeman/reference/history.md +++ b/plugins/codeman/skills/codeman/reference/history.md @@ -43,27 +43,34 @@ plus scan counters and `complete`. `phase` is Codex's own (`commentary` or recorded. A root ID is `null` when ancestry cannot be proven. Report that uncertainty; never relabel the queried child as its own root. `complete: false` reports malformed or partial records (a running Codex may be -mid-write), skipped links, unattributed files, unreadable directories, missing -ancestors, or an inherited copy whose end could not be found or proven. It does not -prove the store includes every conversation ever created, and a filtered result is -only evidence for its selected scope. +mid-write), skipped links, unattributed files, compressed rollouts, unreadable +directories, missing ancestors, or an inherited copy whose end could not be found +or proven (including when the parent's file is present but cannot be read). It +does not prove the store includes every conversation ever created, and a filtered +result is only evidence for its selected scope. Reads require regular UTF8 files and reject final-component symlinks, conflicting metadata, ambiguous IDs, cyclic ancestry and invalid selectors. A store file that cannot be attributed (invalid UTF8 in its head, no metadata in its first 128 KiB, changing while its head is read) is counted in `unattributedFiles` instead of -failing the lookup. A selected rollout that a running Codex is appending to is read -as a snapshot up to its last complete line; one that was replaced or truncated -fails. Parent directories of an explicitly supplied path must be trusted; this is +failing the lookup. Codex's optional rollout compression rewrites cold rollouts to +`*.jsonl.zst`; those are not read. Each is counted in `compressedFiles` (one whose +plain `.jsonl` sibling exists is skipped, as Codex skips it), and a lookup that +misses names them. A compressed file passed to `--file` is refused; read a +decompressed copy instead (`zstd -dc > copy.jsonl`). A selected rollout +that a running Codex is appending to is read as a snapshot up to its last complete +line; one that was replaced or truncated fails. Parent directories of an explicitly supplied path must be trusted; this is not a filesystem sandbox. -Budgets, each failing with an error rather than a partial answer: +Budgets, each failing with an error rather than a partial answer. Every flag is +validated before the first file is read, and only plain decimal integers are +accepted: | Flag | Bounds | Default | Maximum | | ---- | ------ | ------- | ------- | | `--metadata-bytes` | head reads attributing every rollout in the store | 1 GiB | 16 GiB | | `--max-entries` | directory entries visited | 65536 | 1048576 | -| `--scan-bytes` | the selected transcripts | 16 MiB | 256 MiB | +| `--scan-bytes` | the selected transcripts, plus any parent file read to prove where a copy ends (each file charged once) | 16 MiB | 256 MiB | | `--max-bytes` | output, including the final newline | 8192 | 1048576 | Each rollout's `session_meta` carries Codex's base instructions (about 20 KiB), so diff --git a/skills/codeman/reference/history.md b/skills/codeman/reference/history.md index bcdfc92c..ae087fb6 100644 --- a/skills/codeman/reference/history.md +++ b/skills/codeman/reference/history.md @@ -43,27 +43,34 @@ plus scan counters and `complete`. `phase` is Codex's own (`commentary` or recorded. A root ID is `null` when ancestry cannot be proven. Report that uncertainty; never relabel the queried child as its own root. `complete: false` reports malformed or partial records (a running Codex may be -mid-write), skipped links, unattributed files, unreadable directories, missing -ancestors, or an inherited copy whose end could not be found or proven. It does not -prove the store includes every conversation ever created, and a filtered result is -only evidence for its selected scope. +mid-write), skipped links, unattributed files, compressed rollouts, unreadable +directories, missing ancestors, or an inherited copy whose end could not be found +or proven (including when the parent's file is present but cannot be read). It +does not prove the store includes every conversation ever created, and a filtered +result is only evidence for its selected scope. Reads require regular UTF8 files and reject final-component symlinks, conflicting metadata, ambiguous IDs, cyclic ancestry and invalid selectors. A store file that cannot be attributed (invalid UTF8 in its head, no metadata in its first 128 KiB, changing while its head is read) is counted in `unattributedFiles` instead of -failing the lookup. A selected rollout that a running Codex is appending to is read -as a snapshot up to its last complete line; one that was replaced or truncated -fails. Parent directories of an explicitly supplied path must be trusted; this is +failing the lookup. Codex's optional rollout compression rewrites cold rollouts to +`*.jsonl.zst`; those are not read. Each is counted in `compressedFiles` (one whose +plain `.jsonl` sibling exists is skipped, as Codex skips it), and a lookup that +misses names them. A compressed file passed to `--file` is refused; read a +decompressed copy instead (`zstd -dc > copy.jsonl`). A selected rollout +that a running Codex is appending to is read as a snapshot up to its last complete +line; one that was replaced or truncated fails. Parent directories of an explicitly supplied path must be trusted; this is not a filesystem sandbox. -Budgets, each failing with an error rather than a partial answer: +Budgets, each failing with an error rather than a partial answer. Every flag is +validated before the first file is read, and only plain decimal integers are +accepted: | Flag | Bounds | Default | Maximum | | ---- | ------ | ------- | ------- | | `--metadata-bytes` | head reads attributing every rollout in the store | 1 GiB | 16 GiB | | `--max-entries` | directory entries visited | 65536 | 1048576 | -| `--scan-bytes` | the selected transcripts | 16 MiB | 256 MiB | +| `--scan-bytes` | the selected transcripts, plus any parent file read to prove where a copy ends (each file charged once) | 16 MiB | 256 MiB | | `--max-bytes` | output, including the final newline | 8192 | 1048576 | Each rollout's `session_meta` carries Codex's base instructions (about 20 KiB), so diff --git a/src/cli-history.ts b/src/cli-history.ts index 11be6088..238224a2 100644 --- a/src/cli-history.ts +++ b/src/cli-history.ts @@ -27,8 +27,18 @@ import { HISTORY_MAX_OUTPUT_BYTES, HISTORY_MAX_SCAN_BYTES, readLocalHistory, + validateOutputBytes, } from './local-history.js'; +/** + * A budget flag's value as a number. Only plain decimal digits count, so `0x10`, + * `1e3` and ` 5 ` are refused rather than read as 16, 1000 and 5; anything else + * becomes NaN, which the reader's range check rejects with the flag's name. + */ +function budgetFlag(value: string | boolean | undefined): number { + return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : NaN; +} + export function registerHistoryCommands(program: Command): Command { const history = program .command('history') @@ -74,6 +84,10 @@ export function registerHistoryCommands(program: Command): Command { if (options.json && options.format === 'text' && command.getOptionValueSource('format') === 'cli') throw new Error('--json conflicts with --format text; pass one of them'); const format = options.format; + // Every flag is checked before the first file is read: --max-bytes would otherwise + // only be judged after the whole store had been scanned. + const maxBytes = budgetFlag(options.maxBytes); + validateOutputBytes(maxBytes); const result = await readLocalHistory({ file: options.file as string | undefined, sessionId: options.sessionId as string | undefined, @@ -82,11 +96,11 @@ export function registerHistoryCommands(program: Command): Command { topic: options.topic as string | undefined, after: options.after as string | undefined, before: options.before as string | undefined, - scanBytes: Number(options.scanBytes), - metadataBytes: Number(options.metadataBytes), - maxEntries: Number(options.maxEntries), + scanBytes: budgetFlag(options.scanBytes), + metadataBytes: budgetFlag(options.metadataBytes), + maxEntries: budgetFlag(options.maxEntries), }); - program.configureOutput().writeOut?.(`${formatLocalHistory(result, format, Number(options.maxBytes))}\n`); + program.configureOutput().writeOut?.(`${formatLocalHistory(result, format, maxBytes)}\n`); } catch (error) { program.error(error instanceof Error ? error.message : String(error), { exitCode: 1, code: 'history.read' }); } diff --git a/src/local-history.ts b/src/local-history.ts index e6162c0d..4ff244bf 100644 --- a/src/local-history.ts +++ b/src/local-history.ts @@ -46,16 +46,28 @@ * the copy. A legacy copy keeps the parent's own `thread_settings_applied` * events too, so when more than one candidate follows, the boundary is the * last one whose preceding messages are still a prefix of the parent's own - * messages, read from the parent's file. Without the parent's file that - * cannot be proven: the cut then falls at the LAST candidate, so parent text - * is never attributed to the child, and the result says so - * (`unverifiedPrefixes`, `complete: false`). + * messages, read from the parent's file (once: a family read that selects + * the parent reuses that same read, so no file is charged to `scanBytes` + * twice). Without a readable parent file that cannot be proven: the cut then + * falls at the LAST candidate, so parent text is never attributed to the + * child, and the result says so (`unverifiedPrefixes`, `complete: false`). + * Any failure of the parent's read other than a budget leads here, rather + * than failing the child whose own file is intact. * * A later `session_meta` must be the file's own id or a declared ancestor * (`parent_thread_id`, `forked_from_id`, or the shared root `session_id`, and * transitively the ancestors those declare); any other id still refuses the * whole file rather than attribute one conversation's answer to another. * + * ## Compressed rollouts + * + * Codex's `local_thread_store_compression` feature rewrites cold rollouts to + * `*.jsonl.zst` in place. This reader does not decompress them: a store scan + * counts each one in `compressedFiles` (so `complete` is false and a not-found + * error names them), skipping one whose plain `.jsonl` sibling exists, as + * Codex's own discovery does. A Zstandard file given to `file` is refused by + * its magic number, whatever its name. + * * ## Live rollouts * * A running Codex appends to its rollout. Reads take a snapshot: the file's @@ -73,7 +85,7 @@ import { constants } from 'node:fs'; import { isUtf8 } from 'node:buffer'; import { lstat, open, opendir, stat, type FileHandle } from 'node:fs/promises'; -import { join, resolve } from 'node:path'; +import { basename, join, resolve } from 'node:path'; import { codexMessageFromLine, codexSessionsRoot, isCodexInjectedContext } from './codex-rollout-format.js'; @@ -93,6 +105,13 @@ export const HISTORY_MAX_OUTPUT_BYTES = 1 * MIB; const METADATA_HEAD_BYTES = 128 * KIB; /** Read the head in steps, so a typical rollout costs one step past its session_meta line. */ const METADATA_CHUNK_BYTES = 32 * KIB; +/** + * Codex's compressed rollout suffix (codex-rs `rollout/src/compression.rs`: + * `is_compressed_rollout_path` matches a name ending in `.jsonl.zst`). + */ +const COMPRESSED_ROLLOUT_SUFFIX = '.jsonl.zst'; +/** The Zstandard frame magic number, little-endian on disk. */ +const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]); /** Codex writes `sessions/YYYY/MM/DD`; this leaves five levels of headroom. */ const MAX_DIRECTORY_DEPTH = 8; @@ -152,6 +171,11 @@ export interface LocalHistoryResult { partialRecords: number; skippedSymlinks: number; unattributedFiles: number; + /** + * Zstandard-compressed rollouts (`*.jsonl.zst`, Codex's `local_thread_store_compression`) + * this reader cannot open. Their threads are missing from the result. + */ + compressedFiles: number; unreadableDirectories: number; /** Inherited fork or sub-agent prefixes dropped (informational). */ inheritedPrefixes: number; @@ -166,6 +190,8 @@ interface Budget { metadataRemaining: number; entriesRemaining: number; scan: LocalHistoryResult['scan']; + /** Paths behind `scan.compressedFiles`, for the not-found error. */ + compressedPaths: string[]; } /** A budget ran out: always fatal, never counted. */ @@ -318,6 +344,10 @@ async function readMetadata(path: string, budget: Budget): Promise= 0) { @@ -396,6 +426,13 @@ async function findSources(root: string, budget: Budget): Promise { } if (source) sources.push(source); else budget.scan.unattributedFiles++; + } else if (entry.isFile() && entry.name.endsWith(COMPRESSED_ROLLOUT_SUFFIX)) { + // Codex reads the plain sibling instead when both exist (`should_skip_compressed_sibling`), + // and that sibling is visited (or counted) on its own. + const plain = await lstat(path.slice(0, -'.zst'.length)).catch(() => undefined); + if (plain) continue; + budget.scan.compressedFiles++; + budget.compressedPaths.push(path); } } }; @@ -457,7 +494,10 @@ function isPrefixOf(prefix: Message[], whole: Message[]): boolean { } interface ReadContext { - /** The ancestor's own messages, uncut, or undefined when its file is not in the store. */ + /** + * The ancestor's own messages, uncut, or undefined when its file is not in the + * store or could not be read for any reason other than a budget. + */ ancestorMessages(id: string): Promise; } type Piece = Message | { boundary: true }; @@ -466,8 +506,16 @@ interface LegacyPrefix { /** Messages and `thread_settings_applied` boundaries after the inherited session_meta. */ pieces: Piece[]; } +/** One read of one rollout: every view later needed of it, so no file is read (or charged) twice. */ +interface ParsedSource { + /** Every message in file order, inherited copies included: what a child's copy of this thread holds. */ + raw: Message[]; + /** Messages before any legacy inherited prefix, paginated inherited records already dropped. */ + own: Message[]; + legacy?: LegacyPrefix; +} -async function readMessages(source: Source, budget: Budget, context: ReadContext, raw = false): Promise { +async function parseSource(source: Source, budget: Budget): Promise { const handle = await regularFile(source.path); let buffer: Buffer; try { @@ -487,6 +535,7 @@ async function readMessages(source: Source, budget: Budget, context: ReadContext if (!isUtf8(complete)) throw new Error('History source contains invalid UTF8'); const ancestors = new Set([source.parentId, source.forkedFromId, source.rootId].filter((id): id is string => !!id)); + const raw: Message[] = []; const own: Message[] = []; let legacy: LegacyPrefix | undefined; let lastOrdinal = -1; @@ -515,27 +564,34 @@ async function readMessages(source: Source, budget: Budget, context: ReadContext throw new Error('History source contains conflicting native metadata; attribution refused'); // An ancestor that was itself a fork carries its own ancestors' metadata in the copy. for (const id of [declared.parentId, declared.forkedFromId, declared.rootId]) if (id) ancestors.add(id); - if (raw || inherited || legacy) continue; + if (inherited || legacy) continue; budget.scan.inheritedPrefixes++; legacy = { ancestorId: declared.sessionId, pieces: [] }; continue; } - if (inherited && !raw) continue; if (legacy && entry.type === 'event_msg' && object(entry.payload).type === 'thread_settings_applied') { - legacy.pieces.push({ boundary: true }); + if (!inherited) legacy.pieces.push({ boundary: true }); continue; } const item = message(parsed); if (!item) continue; + raw.push(item); + if (inherited) continue; if (legacy) legacy.pieces.push(item); else own.push(item); } - if (!raw && source.historyStart !== undefined && source.historyStart > 0) { + if (source.historyStart !== undefined && source.historyStart > 0) { budget.scan.inheritedPrefixes++; // Initialization died while copying the prefix: Codex refuses to resume such a file. if (lastOrdinal < source.historyStart - 1) budget.scan.incompletePrefixes++; } - if (legacy) own.push(...(await afterLegacyPrefix(legacy, budget, context))); + return { raw: collapse(raw), own, legacy }; +} + +/** The thread's own messages: its inherited prefix dropped, proven against the parent where one is needed. */ +async function readMessages(parsed: ParsedSource, budget: Budget, context: ReadContext): Promise { + const own = [...parsed.own]; + if (parsed.legacy) own.push(...(await afterLegacyPrefix(parsed.legacy, budget, context))); return collapse(own); } @@ -603,6 +659,32 @@ function nativeRoot(source: Source, sources: Source[]): string | null { } } +/** + * The not-found error, naming what the store held but could not attribute. + * Codex names a rollout `rollout--.jsonl[.zst]`, so a + * compressed file whose name carries the wanted id is named outright; the name + * is only a hint here, never identity. + */ +function missingSessionMessage(wanted: string, budget: Budget): string { + const { scan } = budget; + const notes: string[] = []; + if (scan.compressedFiles) { + // `rollout-` (8) + `YYYY-MM-DDTHH-MM-SS` (19) + `-` (1), then the thread id. + const named = budget.compressedPaths.filter((path) => { + const name = basename(path).toLowerCase(); + return name.startsWith('rollout-') && name.slice(28).startsWith(wanted); + }); + const shown = named.slice(0, 3).join(', ') + (named.length > 3 ? ` and ${named.length - 3} more` : ''); + notes.push( + named.length + ? `it may be in compressed rollout ${shown}, which this reader cannot open; read a decompressed copy with --file (zstd -dc > copy.jsonl)` + : `${scan.compressedFiles} compressed .jsonl.zst rollout(s) could not be read` + ); + } + if (scan.unattributedFiles) notes.push(`${scan.unattributedFiles} file(s) could not be attributed`); + return `Native session missing from history store${notes.length ? ` (${notes.join('; ')})` : ''}`; +} + export async function readLocalHistory(options: LocalHistoryOptions): Promise { validate(options); const scan: LocalHistoryResult['scan'] = { @@ -613,6 +695,7 @@ export async function readLocalHistory(options: LocalHistoryOptions): Promise source.sessionId.startsWith(wanted)); - if (!matches.length) - throw new Error( - scan.unattributedFiles - ? `Native session missing from history store (${scan.unattributedFiles} file(s) could not be attributed)` - : 'Native session missing from history store' - ); + if (!matches.length) throw new Error(missingSessionMessage(wanted, budget)); if (matches.length !== 1) throw new Error('Native session id is ambiguous'); const target = matches[0]; attributionSources = all; @@ -676,23 +755,34 @@ export async function readLocalHistory(options: LocalHistoryOptions): Promise a.timestamp.localeCompare(b.timestamp) || a.sessionId.localeCompare(b.sessionId)); const roots = new Map(sources.map((source) => [source.sessionId, nativeRoot(source, attributionSources)])); - const ancestorCache = new Map>(); + // Each selected file is read once, whether the main loop or a child's boundary proof asks first. + const parsedCache = new Map>(); + const parse = (source: Source) => { + let parsed = parsedCache.get(source); + if (!parsed) { + parsed = parseSource(source, budget); + parsedCache.set(source, parsed); + } + return parsed; + }; const context: ReadContext = { - // A raw read never opens a legacy prefix, so this never recurses. - ancestorMessages(id) { - const cached = ancestorCache.get(id); - if (cached) return cached; + // Raw messages need no boundary proof of their own, so this never recurses. + async ancestorMessages(id) { const candidates = attributionSources.filter((source) => source.sessionId === id); - if (candidates.length !== 1) return Promise.resolve(undefined); - const read = readMessages(candidates[0], budget, context, true); - ancestorCache.set(id, read); - return read; + if (candidates.length !== 1) return undefined; + try { + return (await parse(candidates[0])).raw; + } catch (error) { + // A budget always fails the read; any other failure only leaves the boundary unproven. + if (error instanceof HistoryBudgetError) throw error; + return undefined; + } }, }; const records: HistoryRecord[] = []; for (const source of sources) { records.push( - ...selectedTurns(await readMessages(source, budget, context), options).map((item) => ({ + ...selectedTurns(await readMessages(await parse(source), budget, context), options).map((item) => ({ rootSessionId: roots.get(source.sessionId) ?? null, sessionId: source.sessionId, role: item.role, @@ -710,6 +800,7 @@ export async function readLocalHistory(options: LocalHistoryOptions): Promise HISTORY_MAX_OUTPUT_BYTES) - throw new Error(`Invalid --max-bytes (1-${HISTORY_MAX_OUTPUT_BYTES})`); + validateOutputBytes(maxBytes); const body = format === 'json' ? JSON.stringify(result) diff --git a/test/local-history-live.test.ts b/test/local-history-live.test.ts index aa0348a6..850afdab 100644 --- a/test/local-history-live.test.ts +++ b/test/local-history-live.test.ts @@ -30,7 +30,8 @@ vi.mock('node:fs/promises', async (importOriginal) => { }); const history = await import('../src/local-history.js'); -const { assistantTurn, realMeta, userTurn, writeRollout } = await import('./fixtures/codex-rollout.js'); +const { assistantTurn, realMeta, settingsApplied, userTurn, writeRollout } = + await import('./fixtures/codex-rollout.js'); const ROOT = 'aaaaaaaa-1111-4111-8111-111111111111'; const CHILD = 'bbbbbbbb-1111-4111-8111-111111111111'; @@ -126,4 +127,31 @@ describe('rollouts that change while they are read', () => { /changed after attribution/ ); }); + it('leaves a child readable, with an unproven cut, when its parent is replaced after attribution', async () => { + const parentSettings = [settingsApplied(), ...userTurn('parent second question')]; + const parent = await writeRollout(dir, 'rollout-root.jsonl', [ + realMeta({ id: ROOT }), + ...userTurn('parent question'), + ...parentSettings, + ]); + const child = await writeRollout(dir, 'rollout-child.jsonl', [ + realMeta({ id: CHILD, parent: ROOT }), + realMeta({ id: ROOT }), + ...userTurn('parent question'), + ...parentSettings, + settingsApplied(), + ...userTurn('child task'), + ]); + hooks.onOpen = async (opened, count) => { + // The child's transcript read comes before the parent's proof read: replace the parent in between. + if (opened !== child || count !== 2) return; + const replacement = join(dir, 'replacement.tmp'); + await writeFile(replacement, [realMeta({ id: ROOT }), ...userTurn('rewritten')].map(line).join('')); + await rename(replacement, parent); + }; + const result = await history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir }); + expect(result.records.map((r) => r.message)).toEqual(['child task']); + expect(result.scan.unverifiedPrefixes).toBe(1); + expect(result.complete).toBe(false); + }); }); diff --git a/test/local-history.test.ts b/test/local-history.test.ts index 0a7cf6dc..15989c6c 100644 --- a/test/local-history.test.ts +++ b/test/local-history.test.ts @@ -471,6 +471,120 @@ describe('forked and spawned threads', () => { }); }); +describe('one read per file, and a parent that cannot be read', () => { + const parentSettings = [ + settingsApplied(), + ...userTurn('parent second question'), + ...assistantTurn('parent second answer'), + ]; + const sizes = async () => { + const { stat } = await import('node:fs/promises'); + const parent = (await stat(join(dir, `rollout-${ROOT}.jsonl`))).size; + const child = (await stat(join(dir, `rollout-${CHILD}.jsonl`))).size; + return { parent, child }; + }; + + it('charges the parent of a legacy copy to --scan-bytes once, even when it proves the boundary', async () => { + await legacyFamily(dir, parentSettings, parentSettings); + const { parent, child } = await sizes(); + // A budget of exactly the two files: the family read selects the parent and proves the child's cut with it. + const family = await history.readLocalHistory({ + rootSessionId: ROOT, + sessionsRoot: dir, + scanBytes: parent + child, + }); + expect(family.scan.bytesRead).toBe(parent + child); + expect(family.complete).toBe(true); + expect(messages(family).filter((m) => m.startsWith('b:'))).toEqual([ + 'b:child task', + 'b:child commentary', + 'b:child answer', + ]); + // One byte less fails closed rather than returning less. + await expect( + history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir, scanBytes: parent + child - 1 }) + ).rejects.toThrow(/scan byte budget/); + // A child read pays for the parent it proves against, once. + const alone = await history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir, scanBytes: parent + child }); + expect(alone.scan.bytesRead).toBe(parent + child); + expect(alone.complete).toBe(true); + }); + + it('counts a parent record once when the parent is both selected and the proof of a child', async () => { + await legacyFamily(dir, parentSettings, parentSettings); + const { appendFile } = await import('node:fs/promises'); + await appendFile(join(dir, `rollout-${ROOT}.jsonl`), '{not json\n'); + const family = await history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir }); + expect(family.scan.malformedRecords).toBe(1); + expect(family.scan.inheritedPrefixes).toBe(1); + }); + + it('falls back to the unproven cut when the parent file cannot be read, instead of failing the child', async () => { + await legacyFamily(dir, parentSettings, parentSettings); + const { appendFile } = await import('node:fs/promises'); + await appendFile(join(dir, `rollout-${ROOT}.jsonl`), Buffer.from([0xff, 0x0a])); + const child = await history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir }); + expect(messages(child)).toEqual(['b:child task', 'b:child commentary', 'b:child answer']); + expect(child.scan.unverifiedPrefixes).toBe(1); + expect(child.complete).toBe(false); + // The parent's own read still fails: its evidence is what is broken. + await expect(history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir })).rejects.toThrow(/invalid UTF8/); + }); + + it('still fails when the parent read runs out of budget', async () => { + await legacyFamily(dir, parentSettings, parentSettings); + const { child } = await sizes(); + await expect( + history.readLocalHistory({ sessionId: CHILD, sessionsRoot: dir, scanBytes: child + 1 }) + ).rejects.toThrow(/scan byte budget/); + }); +}); + +describe('compressed rollouts', () => { + const ZSTD = Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x04, 0x00, 0x00, 0x00]); + const compressedName = (id: string) => `rollout-2026-10-24T01-00-00-${id}.jsonl.zst`; + + it('counts a .jsonl.zst rollout, so a family with a compressed member is incomplete', async () => { + await legacyFamily(dir); + await mkdir(join(dir, '2026', '10', '24'), { recursive: true }); + await writeFile(join(dir, '2026', '10', '24', compressedName(GRANDCHILD)), ZSTD); + const family = await history.readLocalHistory({ rootSessionId: ROOT, sessionsRoot: dir }); + expect(family.scan.compressedFiles).toBe(1); + expect(family.scan.unattributedFiles).toBe(0); + expect(family.complete).toBe(false); + }); + + it('names the compressed rollout when the wanted thread is not otherwise found', async () => { + await fixture(ROOT, [row('assistant', 'root')]); + const path = join(dir, compressedName(CHILD)); + await writeFile(path, ZSTD); + const error = await history.readLocalHistory({ sessionId: CHILD.slice(0, 8), sessionsRoot: dir }).catch((e) => e); + expect(error.message).toContain(path); + expect(error.message).toMatch(/compressed rollout/); + // A compressed file whose name does not carry the id is still reported, by count. + const other = await history.readLocalHistory({ sessionId: GRANDCHILD, sessionsRoot: dir }).catch((e) => e); + expect(other.message).toMatch(/1 compressed \.jsonl\.zst rollout\(s\) could not be read/); + }); + + it('skips a compressed rollout whose plain sibling exists, as Codex does', async () => { + const plain = await fixture(ROOT, [row('assistant', 'root')]); + await writeFile(`${plain}.zst`, ZSTD); + const result = await history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir }); + expect(result.scan.compressedFiles).toBe(0); + expect(result.complete).toBe(true); + }); + + it('refuses a Zstandard file given to --file, and counts one in the store whatever its name', async () => { + const file = join(dir, 'renamed.jsonl'); + await writeFile(file, ZSTD); + await expect(history.readLocalHistory({ file })).rejects.toThrow(/Zstandard-compressed/); + await fixture(ROOT, [row('assistant', 'root')]); + const result = await history.readLocalHistory({ sessionId: ROOT, sessionsRoot: dir }); + expect(result.scan.unattributedFiles).toBe(1); + expect(result.complete).toBe(false); + }); +}); + describe('large and untidy stores', () => { it('finds one id among thousands of rollouts with real-sized session_meta at the default budgets', async () => { const count = 3500; @@ -692,6 +806,31 @@ describe('history CLI flags', () => { expect(await run(['--file', file, '--format', 'text'])).toMatch(/^source=codex-jsonl/); }); + it('validates --max-bytes before any file is read', async () => { + // A missing store would be the first thing a read touches; an invalid --max-bytes must be refused first. + const missing = join(dir, 'no-such-store'); + for (const value of ['0', String(history.HISTORY_MAX_OUTPUT_BYTES + 1), 'lots', '0x10', '1e3', ' 5']) + await expect(run(['--session-id', ROOT, '--sessions-root', missing, '--max-bytes', value])).rejects.toThrow( + /Invalid --max-bytes/ + ); + await expect(run(['--session-id', ROOT, '--sessions-root', missing, '--max-bytes', '65536'])).rejects.toThrow( + /Sessions root not found/ + ); + }); + + it('reads budget flags as plain decimal integers only', async () => { + await fixture(ROOT, [row('assistant', 'answer')]); + await expect(run(['--session-id', ROOT, '--sessions-root', dir, '--scan-bytes', '0x100000'])).rejects.toThrow( + /Invalid --scan-bytes/ + ); + await expect(run(['--session-id', ROOT, '--sessions-root', dir, '--max-entries', '1e3'])).rejects.toThrow( + /Invalid --max-entries/ + ); + expect( + JSON.parse(await run(['--session-id', ROOT, '--sessions-root', dir, '--scan-bytes', '1048576'])).complete + ).toBe(true); + }); + it('passes the store caps through and validates them', async () => { await fixture(ROOT, [row('assistant', 'answer')]); await fixture(CHILD, [row('assistant', 'child')], ROOT);