diff --git a/src/host-sensors.ts b/src/host-sensors.ts index a972be07..26704f15 100644 --- a/src/host-sensors.ts +++ b/src/host-sensors.ts @@ -166,11 +166,20 @@ export function parseTypeperf(output: string): WindowsCounters { return result; } +/** + * Run a reader, answering undefined on failure or once `timeout` has passed. execFile's own `timeout` only + * sends the child a signal, and its callback waits for the child to exit: an nvidia-smi stuck on a hung + * driver (uninterruptible sleep) never does, and every stats request would wait on it with CPU and MEM. So + * the deadline here answers by itself; the child is left to the signal. + */ const run = (cmd: string, args: string[], timeout = EXEC_TIMEOUT_MS) => new Promise((resolve) => { - execFile(cmd, args, { timeout, windowsHide: true, maxBuffer: 4 * MB }, (err, stdout) => - resolve(err ? undefined : stdout) - ); + const deadline = setTimeout(() => resolve(undefined), timeout); + deadline.unref(); + execFile(cmd, args, { timeout, windowsHide: true, maxBuffer: 4 * MB }, (err, stdout) => { + clearTimeout(deadline); + resolve(err ? undefined : stdout); + }); }); const readText = (path: string) => diff --git a/test/host-sensors-readers.test.ts b/test/host-sensors-readers.test.ts index ed1fb79c..8b599182 100644 --- a/test/host-sensors-readers.test.ts +++ b/test/host-sensors-readers.test.ts @@ -15,6 +15,9 @@ const state = vi.hoisted(() => ({ /** When set, PowerShell does not answer until the test calls the callbacks kept here. */ holdPowershell: false, heldPowershell: [] as Array<() => void>, + /** When set, nvidia-smi never answers (a child stuck on a hung driver) until the test calls these. */ + holdNvidia: false, + heldNvidia: [] as Array<() => void>, execCalls: [] as Array<{ cmd: string; args: string[] }>, spawnCalls: [] as Array<{ cmd: string; args: string[]; stdout: unknown }>, })); @@ -22,6 +25,10 @@ const state = vi.hoisted(() => ({ vi.mock('node:child_process', () => ({ execFile: (cmd: string, args: string[], _opts: unknown, cb: (err: Error | null, stdout: string) => void) => { state.execCalls.push({ cmd, args }); + if (state.holdNvidia && cmd === 'nvidia-smi') { + state.heldNvidia.push(() => cb(null, NVIDIA)); + return; + } if (state.holdPowershell && cmd.endsWith('powershell.exe')) { state.heldPowershell.push(() => { const held = state.powershell; @@ -69,6 +76,8 @@ beforeEach(() => { state.powershell = new Error('powershell failed'); state.holdPowershell = false; state.heldPowershell = []; + state.holdNvidia = false; + state.heldNvidia = []; state.execCalls = []; state.spawnCalls = []; process.env.WSL_INTEROP = '1'; @@ -151,6 +160,32 @@ describe('a failing reader is left alone for a minute', () => { }); }); +describe('a hung nvidia-smi', () => { + it('answers at its own deadline instead of waiting for a child that never exits, then backs off', async () => { + state.holdNvidia = true; + const { readHostSensors, stopHostSensors } = await load(); + const started = performance.now(); + // execFile's timeout only signals the child; one that never exits never calls back + const first = await Promise.race([ + readHostSensors({ gpu: true }), + new Promise<'hung'>((r) => setTimeout(() => r('hung'), 4000)), + ]); + expect(first).not.toBe('hung'); + expect(performance.now() - started).toBeLessThan(3500); + expect((first as { gpu?: object }).gpu).toBeUndefined(); + expect(counted('nvidia-smi')).toBe(1); + // the reader that timed out is left alone like a failing one: the next poll does not start a second + vi.setSystemTime(Date.now() + 2500); + await readHostSensors({ gpu: true }); + expect(counted('nvidia-smi')).toBe(1); + // the child finally exiting changes nothing + state.heldNvidia.forEach((answer) => answer()); + await tick(); + expect(counted('nvidia-smi')).toBe(1); + stopHostSensors(); + }); +}); + describe('the PowerShell RAM probe', () => { it('is never waited for: the memory in use shows at once, and the total once Windows has answered', async () => { state.nvidia = new Error('no nvidia');