diff --git a/docs/remote-sessions.md b/docs/remote-sessions.md index c10c6f07..3ca63ec9 100644 --- a/docs/remote-sessions.md +++ b/docs/remote-sessions.md @@ -251,6 +251,18 @@ unreachable host answers "unknown", which also means do not revive. The answer is cached per session and cleared whenever the pane is next seen alive, so a stale `true` from one transport drop can never revive the NEXT clean exit. +## Terminal history from the remote tmux + +The local pane runs the ssh attach, so it only sees the remote tmux client's +full-screen repaints and keeps a fraction of the scrollback (none after a +reattach). A full-history load (`GET /api/sessions/:id/terminal?full=1`) of a +remote session therefore reads the scrollback rows from the REMOTE pane in one +ssh round trip (`src/remote-pane-history.ts`) and splices them above the local +visible frame and caret. Bounded (5 s, the shared ssh limiter) and fail-soft: +any failure keeps the local capture and skips that host for 30 s. How much +history exists is the remote tmux's own `history-limit`, which the launch +command does not set (tmux's default is 2000 lines). + ## File access over SSH A remote case's `workingDir` is an absolute path on the **remote** host diff --git a/src/mux-interface.ts b/src/mux-interface.ts index 56214633..9dcd2a1e 100644 --- a/src/mux-interface.ts +++ b/src/mux-interface.ts @@ -189,6 +189,13 @@ export interface PaneCaptureOptions { fullHistory?: boolean; /** Bound the full-history capture to this many scrollback lines (`-S -`). */ historyLimitLines?: number; + /** + * Full-history only: scrollback rows read from somewhere other than this pane + * (raw `capture-pane -p -e -J` text, oldest first). The pane then contributes + * only its visible frame and caret, and its own history is not read. A remote + * session's real history lives in the REMOTE tmux (`remote-pane-history.ts`). + */ + scrollbackOverride?: string; /** * Byte cap the consumer will keep from the capture. Sizes the child-process * stdout buffer (with slack) so multi-MB scrollback dumps aren't killed by diff --git a/src/remote-pane-history.ts b/src/remote-pane-history.ts new file mode 100644 index 00000000..88b445e3 --- /dev/null +++ b/src/remote-pane-history.ts @@ -0,0 +1,182 @@ +/** + * @fileoverview Scrollback of a remote SSH session, read from the REMOTE tmux. + * + * A remote session's local pane runs `ssh -t … tmux new-session -A` (or `attach`), + * so the local tmux only ever sees what the remote tmux CLIENT draws: full-screen + * repaints, not scrolling output. The local pane therefore keeps a fraction of the + * history (measured: 48 of 279 lines while output streamed, none after a reattach), + * while the remote pane, where the shell really runs, keeps all of it. A full-history + * load of a remote session reads the scrollback from there instead, and the caller + * splices it above the LOCAL visible frame (`PaneCaptureOptions.scrollbackOverride`): + * the local pane is what the live stream keeps drawing into, so its frame and caret + * position are the ones the browser must end up with. + * + * ONE ssh round trip prints the remote pane's `#{history_size}` and its scrollback + * rows (`capture-pane -S - -E -1`). The size is load-bearing: with no history at + * all, tmux clamps `-E -1` to the first VISIBLE row and would return it as history. + * + * Bounded and fail-soft. Any failure (unreachable host, auth, missing session, odd + * output) returns null and the caller keeps the local capture, so a remote load is + * never worse than before. A failure also starts a short per-host back-off, so a + * host that is asleep costs one timeout, not one per tab switch. The child runs under + * the shared remote-ssh limiter (`remote-ssh-limiter.ts`) like every other short-lived + * remote ssh, but gives up on a slot it waited too long for, and the connection + * options come from `buildSshConnectionArgs()`. + * + * How much history exists is the REMOTE tmux's `history-limit` (2000 lines unless + * the remote user's tmux.conf raises it); the launch command does not set it. + */ + +import { exec } from 'node:child_process'; +import { promisify } from 'node:util'; +import type { SessionRemote } from './types/session.js'; +import { buildSshConnectionArgs, remoteSshTarget, shellescape } from './remote-hosts.js'; +import { runWithRemoteSshLimit } from './remote-ssh-limiter.js'; +import { remoteTmuxLocation } from './tmux-manager.js'; + +const execAsync = promisify(exec); + +/** + * Bound on the whole round trip. A full load waits on it, so it is far below the + * 10 s `ConnectTimeout` the shared ssh options carry: past this the local capture + * is the better answer. + */ +const REMOTE_HISTORY_TIMEOUT_MS = 5_000; + +/** + * Longest a load waits for a slot in the shared ssh pool, which file previews and + * attachment probes (20 s each) also hold. Past it the local capture is used. + */ +const REMOTE_HISTORY_SLOT_WAIT_MS = 1_500; + +/** After a failed read, skip that host for this long and use the local capture. */ +const REMOTE_HISTORY_BACKOFF_MS = 30_000; + +/** Headroom over the remote `tail -c` bound for the size line and markers. */ +const REMOTE_HISTORY_SLACK_BYTES = 64 * 1024; + +/** Read ceiling when the configured byte cap is 0 (unbounded) and no `tail -c` runs. */ +const UNBOUNDED_READ_MAX_BYTES = 64 * 1024 * 1024; + +/** + * Host id → time the back-off ends. Keyed by HOST, so a sleeping host with five + * sessions costs one timeout, and the map is bounded by the configured hosts. + */ +const backoffUntil = new Map(); + +/** + * The remote shell script. Its first byte is a NUL so a login banner or an rc-file + * `echo` printed before it runs is separated without guessing (the same marker + * `buildRemoteProbeCommand` uses); a second NUL ends the history size. + */ +export function buildRemoteScrollbackScript( + socket: string, + sessionName: string, + historyLines: number, + maxBytes: number +): string { + const tmux = `tmux -L ${shellescape(socket)}`; + // `=name:` is an EXACT session match, then its active pane. A bare name is + // read as a pane index or window name first, so a discovered session named + // `0` would capture some other session's pane. + const target = shellescape(`=${sessionName}:`); + const lines = Math.max(1, Math.trunc(historyLines)); + // The newest rows are the ones worth keeping, so the byte bound is applied on + // the remote side (`tail -c`) rather than by killing the read at maxBuffer. + // A cap of 0 means unbounded, as in the route. + const bound = maxBytes > 0 ? ` | tail -c ${Math.trunc(maxBytes)}` : ''; + return ( + `printf '\\0' && ${tmux} display-message -p -t ${target} '#{history_size}' && printf '\\0' && ` + + `${tmux} capture-pane -p -e -J -S -${lines} -E -1 -t ${target}${bound}` + ); +} + +/** The full ssh command line for {@link buildRemoteScrollbackScript}. */ +export function buildRemoteScrollbackCommand( + remote: SessionRemote, + sessionId: string, + historyLines: number, + maxBytes: number +): string { + const { socket, sessionName } = remoteTmuxLocation(remote, sessionId); + const script = buildRemoteScrollbackScript(socket, sessionName, historyLines, maxBytes); + // `exec` so a timeout kill reaches ssh itself, not only the local `sh -c`. + return ['exec', ...buildSshConnectionArgs(remote), remoteSshTarget(remote), shellescape(script)].join(' '); +} + +/** + * A `tail -c` cut lands mid-row (possibly mid-escape) whenever it bit, so the + * first, partial row is dropped. Under the bound, the rows are whole. + */ +export function dropPartialFirstRow(scrollback: string, maxBytes: number): string { + if (maxBytes <= 0 || Buffer.byteLength(scrollback, 'utf-8') < maxBytes) return scrollback; + const newline = scrollback.indexOf('\n'); + return newline === -1 ? '' : scrollback.slice(newline + 1); +} + +/** + * The scrollback rows from the script's stdout, '' when the pane has no history, + * or null when the output is not the script's. + */ +export function parseRemoteScrollbackOutput(stdout: string): string | null { + const start = stdout.indexOf('\0'); + if (start === -1) return null; + const sizeEnd = stdout.indexOf('\0', start + 1); + if (sizeEnd === -1) return null; + const sizeText = stdout.slice(start + 1, sizeEnd).trim(); + if (!/^\d+$/.test(sizeText)) return null; + if (Number(sizeText) === 0) return ''; + return stdout.slice(sizeEnd + 1); +} + +/** + * Read a remote session's scrollback from the remote tmux, or null to keep the + * local capture. Never throws. + */ +export async function fetchRemoteScrollback( + remote: SessionRemote, + sessionId: string, + historyLines: number, + maxBytes: number +): Promise { + // No real ssh under test (mirrors remote-files.ts / remote-hosts.ts). + if (process.env.VITEST) return null; + const queuedAt = Date.now(); + const until = backoffUntil.get(remote.hostId); + if (until !== undefined) { + if (queuedAt < until) return null; + backoffUntil.delete(remote.hostId); + } + const command = buildRemoteScrollbackCommand(remote, sessionId, historyLines, maxBytes); + try { + const stdout = await runWithRemoteSshLimit(async () => { + // The pool was busy past what a load should wait: give the slot back. + if (Date.now() - queuedAt > REMOTE_HISTORY_SLOT_WAIT_MS) return null; + const result = await execAsync(command, { + encoding: 'utf-8', + timeout: REMOTE_HISTORY_TIMEOUT_MS, + maxBuffer: (maxBytes > 0 ? maxBytes : UNBOUNDED_READ_MAX_BYTES) + REMOTE_HISTORY_SLACK_BYTES, + }); + return result.stdout; + }); + if (stdout === null) return null; + const scrollback = parseRemoteScrollbackOutput(stdout); + if (scrollback === null) { + backoffUntil.set(remote.hostId, Date.now() + REMOTE_HISTORY_BACKOFF_MS); + console.warn(`[remote-history] ${remote.label}: unexpected output; using the local capture`); + return null; + } + return dropPartialFirstRow(scrollback, maxBytes); + } catch (err) { + backoffUntil.set(remote.hostId, Date.now() + REMOTE_HISTORY_BACKOFF_MS); + const e = err as { killed?: boolean; code?: unknown }; + const reason = e.killed ? 'timed out' : typeof e.code === 'number' ? `exit ${e.code}` : 'failed'; + console.warn(`[remote-history] ${remote.label}: remote scrollback read ${reason}; using the local capture`); + return null; + } +} + +/** Test hook: forget every host back-off. */ +export function resetRemoteScrollbackBackoff(): void { + backoffUntil.clear(); +} diff --git a/src/tmux-manager.ts b/src/tmux-manager.ts index d7974a75..548a9e45 100644 --- a/src/tmux-manager.ts +++ b/src/tmux-manager.ts @@ -694,6 +694,16 @@ export function normalizeScrollbackEol(buffer: string): string { return buffer.replace(/\r?\n/g, '\r\n'); } +/** + * Put scrollback rows read elsewhere (`PaneCaptureOptions.scrollbackOverride`) + * above a visible-frame capture, as one capture in tmux's own shape: rows joined + * by a bare `\n`. Undefined or empty scrollback leaves the frame as it is. + */ +export function joinScrollbackAndFrame(scrollback: string | undefined, frame: string): string { + if (!scrollback) return frame; + return `${scrollback.replace(/\n$/, '')}\n${frame}`; +} + /** Pane geometry and caret position, as `display-message` reports them. */ interface PaneCursorGeometry { cols: number; @@ -962,6 +972,19 @@ export function remoteTmuxSessionName(sessionId: string): string { return `codeman-ssh-${sessionId.slice(0, 8)}`; } +/** + * The remote tmux socket and session a remote Codeman session's pane is attached + * to, matching the command that attached it: an owned session is the one + * `buildRemoteLaunchCommand` created on the private socket, a non-owned one is the + * discovered session `buildRemoteAttachCommand` joined on `-L codeman`. + */ +export function remoteTmuxLocation(remote: SessionRemote, sessionId: string): { socket: string; sessionName: string } { + if (remote.owned === false) { + return { socket: 'codeman', sessionName: remote.remoteSessionName || remoteTmuxSessionName(sessionId) }; + } + return { socket: REMOTE_TMUX_SOCKET, sessionName: remoteTmuxSessionName(sessionId) }; +} + /** * COD-104 — build the SSH command that launches (or reattaches) a remote * session INSIDE a tmux server on the remote host, so the remote agent survives @@ -3870,7 +3893,14 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { typeof requestedLines === 'number' && Number.isFinite(requestedLines) && requestedLines > 0 ? Math.trunc(requestedLines) : DEFAULT_TMUX_HISTORY_LIMIT; - const captureFlags = fullHistory ? `capture-pane -p -e -J -S -${historyLines}` : 'capture-pane -p -e'; + // With a scrollback override this pane's own history is not read at all: + // `-S 0` starts at the top of the visible frame. + const scrollbackOverride = fullHistory ? opts?.scrollbackOverride : undefined; + const captureFlags = !fullHistory + ? 'capture-pane -p -e' + : scrollbackOverride !== undefined + ? 'capture-pane -p -e -J -S 0' + : `capture-pane -p -e -J -S -${historyLines}`; // execSync's default maxBuffer (1MB) kills multi-MB scrollback dumps // (ENOBUFS) and would silently degrade full-history capture to the byte // buffer for exactly the long sessions it exists for — size it from the @@ -3883,7 +3913,10 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { execOpts.maxBuffer = (opts?.maxCaptureBytes ?? DEFAULT_TERMINAL_BUFFER_MAX_BYTES) + FULL_HISTORY_CAPTURE_SLACK_BYTES; } - const rawCapture = execSync(`${this.tmux()} ${captureFlags} -t ${shellescape(target)}`, execOpts); + const rawCapture = joinScrollbackAndFrame( + scrollbackOverride, + execSync(`${this.tmux()} ${captureFlags} -t ${shellescape(target)}`, execOpts) + ); // Query the cursor BEFORE deciding anything else. On the full-history path // it settles both how the capture is trimmed and whether a cursor move is // appended, and those two have to agree: trailing blank rows are only safe diff --git a/src/web/routes/session-routes.ts b/src/web/routes/session-routes.ts index 5960d9e6..06f1407d 100644 --- a/src/web/routes/session-routes.ts +++ b/src/web/routes/session-routes.ts @@ -163,6 +163,7 @@ import { toAttachedSessionRemote, toSessionRemote, } from '../../remote-hosts.js'; +import { fetchRemoteScrollback } from '../../remote-pane-history.js'; import { checkDockerAvailable, checkDockerConfigDrift, @@ -3116,6 +3117,19 @@ export function registerSessionRoutes( maxCaptureBytes: terminalBufferMaxBytes, } : {}; + // A remote session's local pane only sees the remote tmux client's repaints, + // so its own scrollback is a fraction of the real one: read that from the + // remote tmux and splice it above the local frame. Null (host unreachable, + // no remote session, back-off) keeps the local capture as before. + if (isFullReload && session.remote && muxName && captureOpts.historyLimitLines !== undefined) { + const remoteScrollback = await fetchRemoteScrollback( + session.remote, + session.id, + captureOpts.historyLimitLines, + terminalBufferMaxBytes + ); + if (remoteScrollback !== null) captureOpts.scrollbackOverride = remoteScrollback; + } const liveMuxBuffer = muxName && typeof ctx.mux.captureActivePaneBuffer === 'function' ? ctx.mux.captureActivePaneBuffer(muxName, captureOpts) diff --git a/test/remote-pane-history.test.ts b/test/remote-pane-history.test.ts new file mode 100644 index 00000000..c516c274 --- /dev/null +++ b/test/remote-pane-history.test.ts @@ -0,0 +1,156 @@ +/** + * @fileoverview Tests for reading a remote session's scrollback from the remote + * tmux (`src/remote-pane-history.ts`) and splicing it above the local frame. + * + * The remote script is also run by a real `/bin/sh`, with a stub `tmux` first on + * PATH: the script IS the remote command, so `sh -c