diff --git a/src/config/foreign-tmux.ts b/src/config/foreign-tmux.ts index e4c838c2..d60b5fd8 100644 --- a/src/config/foreign-tmux.ts +++ b/src/config/foreign-tmux.ts @@ -3,9 +3,12 @@ * outside Codeman). * * Two facts drive every number here. First, the number of tmux sockets and panes - * on a machine is NOT under Codeman's control — a discovery walk with no ceiling - * is an unbounded loop over data someone else produces, so sockets and panes are - * both hard-capped. Second, an ssh handshake is an order of magnitude slower than + * on a machine is NOT under Codeman's control, so a discovery walk with no ceiling + * is an unbounded loop over data someone else produces. Parsed rows are capped by + * count and the socket sweep by time: a sweep capped by count would stop in glob + * order, where `codeman*` sorts before `default` and a machine running many + * instances would never reach the sessions this feature is for. Second, an ssh + * handshake is an order of magnitude slower than * a local `exec`; reusing the shared 5s `EXEC_TIMEOUT_MS` would classify every * remote host as unreachable, so the probe gets its own timeout. * @@ -25,8 +28,15 @@ export const FOREIGN_CACHE_TTL_MS = 5000; /** Timeout for one probe invocation (local exec, `docker exec`, or one ssh). */ export const FOREIGN_PROBE_TIMEOUT_MS = 12000; -/** Max tmux sockets inspected per location, oldest-first by directory order. */ -export const FOREIGN_MAX_SOCKETS = 16; +/** + * How long one tmux server may take to answer the probe's ping, in the script's own + * `sleep`, so whole seconds. A healthy server takes milliseconds. One that misses this + * costs only its own rows, and the sweep moves on to the next socket. + */ +export const FOREIGN_SOCKET_TIMEOUT_SECONDS = 2; + +/** How long a timed-out probe's process group gets on SIGTERM before SIGKILL. */ +export const FOREIGN_PROBE_KILL_GRACE_MS = 2000; /** Max pane rows parsed from one probe. Panes past this are dropped, not errors. */ export const FOREIGN_MAX_PANES = 400; diff --git a/src/foreign-tmux-discovery.ts b/src/foreign-tmux-discovery.ts index f3bd5d87..d94e057f 100644 --- a/src/foreign-tmux-discovery.ts +++ b/src/foreign-tmux-discovery.ts @@ -15,8 +15,8 @@ * ## ⚠️ The probe must not cross an extra shell * * The script contains `$TMUX_TMPDIR` and `$(id -u)`, which the INNER shell has to - * expand. Local and docker therefore spawn with an argv array (`execFile`), where - * no outer shell exists to eat them first. Remote is the exception: the ssh + * expand. Local and docker therefore spawn with an argv array, where no outer + * shell exists to eat them first. Remote is the exception: the ssh * command line is a string like every other ssh call in this codebase, so the * script is wrapped in SINGLE quotes there — which is exactly why * `buildForeignProbeScript()` may not contain one. @@ -26,11 +26,19 @@ * inner loop iterating over the literal `/tmux-/*` and reporting zero sessions on * a machine that had three. * + * ## A probe that goes wrong still answers + * + * All three transports go through `runForeignProbe`, which never rejects and hands + * back what it read. A timeout has to reach the whole process tree, not just the + * `sh` on top (the same rule as `runFetch` in git-fetch.ts): signalling only `sh` + * left its tmux child stuck on a stopped server, one more per scan, for as long as + * the home screen stayed open. And a scan cut short keeps the rows it already has, + * since one bad server must not empty the list of a machine with good ones. + * * @module foreign-tmux-discovery */ -import { execFile, exec } from 'node:child_process'; -import { promisify } from 'node:util'; +import { spawn, type ChildProcess } from 'node:child_process'; import { buildForeignProbeScript, classifyForeignPaneMode, @@ -45,20 +53,93 @@ import { dockerEngineArgv, readDockerCases, readDockerHosts } from './docker-hos import { buildSshConnectionArgs, readRemoteHosts, remoteSshTarget, shellescape } from './remote-hosts.js'; import { runWithRemoteSshLimit } from './remote-ssh-limiter.js'; import { resolveTmuxSocketName, getDataDir } from './config/instance.js'; -import { FOREIGN_CACHE_TTL_MS, FOREIGN_PROBE_MAX_BYTES, FOREIGN_PROBE_TIMEOUT_MS } from './config/foreign-tmux.js'; +import { + FOREIGN_CACHE_TTL_MS, + FOREIGN_PROBE_KILL_GRACE_MS, + FOREIGN_PROBE_MAX_BYTES, + FOREIGN_PROBE_TIMEOUT_MS, +} from './config/foreign-tmux.js'; import type { ForeignDiscoveryResult, ForeignTmuxSession } from './types/foreign-tmux.js'; import type { DockerCase, DockerHost, RemoteHost, SessionDocker } from './types/session.js'; -const execFileAsync = promisify(execFile); -const execAsync = promisify(exec); - const IS_TEST_MODE = process.env.VITEST === 'true' || process.env.NODE_ENV === 'test'; -const PROBE_OPTS = { - encoding: 'utf-8' as const, - timeout: FOREIGN_PROBE_TIMEOUT_MS, - maxBuffer: FOREIGN_PROBE_MAX_BYTES, -}; +/** What one probe produced, however it ended. */ +export interface ForeignProbeRun { + /** Everything read before it finished or was stopped. */ + stdout: string; + /** False when it timed out, flooded, exited non-zero or could not start. */ + ok: boolean; +} + +/** + * Run one probe in its own process group. Never rejects (see @fileoverview). + * + * The caller is answered AT the deadline, while the group is still dying. + */ +export function runForeignProbe( + argv: readonly string[], + opts: { timeoutMs?: number; maxBytes?: number } = {} +): Promise { + const { timeoutMs = FOREIGN_PROBE_TIMEOUT_MS, maxBytes = FOREIGN_PROBE_MAX_BYTES } = opts; + return new Promise((resolve) => { + const [file, ...args] = argv; + const chunks: Buffer[] = []; + let bytes = 0; + let settled = false; + let child: ChildProcess | undefined; + let escalation: NodeJS.Timeout | undefined; + + const finish = (ok: boolean) => { + if (settled) return; + settled = true; + clearTimeout(deadline); + resolve({ stdout: Buffer.concat(chunks).toString('utf-8'), ok }); + }; + const signalGroup = (signal: NodeJS.Signals) => { + try { + if (child?.pid) process.kill(-child.pid, signal); + } catch { + /* already gone */ + } + }; + const stop = () => { + signalGroup('SIGTERM'); + escalation = setTimeout(() => signalGroup('SIGKILL'), FOREIGN_PROBE_KILL_GRACE_MS); + escalation.unref(); + // ⚠️ Let go of the pipe by hand. A stopped tmux server can hold it open with + // everything in this group dead (see foreign-tmux.ts), so waiting for its EOF + // would keep one handle per scan until that server is resumed. + child?.stdout?.destroy(); + finish(false); + }; + const deadline = setTimeout(stop, timeoutMs); + + try { + child = spawn(file ?? '', args, { stdio: ['ignore', 'pipe', 'ignore'], detached: true }); + } catch { + finish(false); + return; + } + child.stdout?.on('data', (chunk: Buffer) => { + if (settled) return; + chunks.push(chunk.subarray(0, Math.max(0, maxBytes - bytes))); + bytes += chunk.length; + if (bytes > maxBytes) stop(); + }); + child.on('error', () => finish(false)); + child.on('close', (code) => { + // The group is gone, so a late SIGKILL could only reach whoever reused its id. + clearTimeout(escalation); + finish(code === 0); + }); + }); +} + +/** The note for a probe that ended badly. Rows already read stay, so say which case this is. */ +function probeFailureNote(where: string, run: ForeignProbeRun, nothing: string): string { + return run.stdout ? `${where}: the scan did not finish, this list may be incomplete` : `${where}: ${nothing}`; +} /** * Turn one probe's stdout into candidates. Shared by all three transports, so a @@ -68,7 +149,7 @@ const PROBE_OPTS = { *