mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-14 02:29:33 +01:00
fix: give the host-sensor readers a hard deadline so a hung nvidia-smi cannot hold the stats route (release review)
`run()` relied on execFile's `timeout`, which only sends the child a signal: execFile's callback, and so the promise, settles when the child exits. An nvidia-smi stuck on a hung driver (uninterruptible sleep) never does, so `readNativeGpu()` stayed pending, every later `?gpu=1` request joined that promise through `inFlight`, and since the route awaits the sensors before `getSystemStats()`, CPU and MEM froze too for every device with GPU Stats on. The backoff (`nvidiaSkipUntil`) only armed once the child finally exited. `run()` now races its own unref'd timer and answers undefined at the deadline, clearing the timer when execFile settles first. The existing failure path then arms the 60 s backoff as for any failing reader. The PowerShell RAM probe shares `run()` and gets the same bound. Test: an nvidia-smi that never calls back answers within the deadline, the next poll does not run it again, and its late exit changes nothing (the test fails on the previous `run()`). Checked against a real TERM-ignoring child on PATH: the read settles at 1.5 s instead of when the child exits. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
fb83a5c8df
commit
40158ce1ef
2 files changed
+47
-3
No files matched your search
@@ -15,6 +15,9 @@ const state = vi.hoisted(() => ({
|
||||
/** When set, PowerShell does not answer until the test calls the callbacks kept here. */
|
||||
holdPowershell: false,
|
||||
heldPowershell: [] as Array<() => void>,
|
||||
/** When set, nvidia-smi never answers (a child stuck on a hung driver) until the test calls these. */
|
||||
holdNvidia: false,
|
||||
heldNvidia: [] as Array<() => void>,
|
||||
execCalls: [] as Array<{ cmd: string; args: string[] }>,
|
||||
spawnCalls: [] as Array<{ cmd: string; args: string[]; stdout: unknown }>,
|
||||
}));
|
||||
@@ -22,6 +25,10 @@ const state = vi.hoisted(() => ({
|
||||
vi.mock('node:child_process', () => ({
|
||||
execFile: (cmd: string, args: string[], _opts: unknown, cb: (err: Error | null, stdout: string) => void) => {
|
||||
state.execCalls.push({ cmd, args });
|
||||
if (state.holdNvidia && cmd === 'nvidia-smi') {
|
||||
state.heldNvidia.push(() => cb(null, NVIDIA));
|
||||
return;
|
||||
}
|
||||
if (state.holdPowershell && cmd.endsWith('powershell.exe')) {
|
||||
state.heldPowershell.push(() => {
|
||||
const held = state.powershell;
|
||||
@@ -69,6 +76,8 @@ beforeEach(() => {
|
||||
state.powershell = new Error('powershell failed');
|
||||
state.holdPowershell = false;
|
||||
state.heldPowershell = [];
|
||||
state.holdNvidia = false;
|
||||
state.heldNvidia = [];
|
||||
state.execCalls = [];
|
||||
state.spawnCalls = [];
|
||||
process.env.WSL_INTEROP = '1';
|
||||
@@ -151,6 +160,32 @@ describe('a failing reader is left alone for a minute', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('a hung nvidia-smi', () => {
|
||||
it('answers at its own deadline instead of waiting for a child that never exits, then backs off', async () => {
|
||||
state.holdNvidia = true;
|
||||
const { readHostSensors, stopHostSensors } = await load();
|
||||
const started = performance.now();
|
||||
// execFile's timeout only signals the child; one that never exits never calls back
|
||||
const first = await Promise.race([
|
||||
readHostSensors({ gpu: true }),
|
||||
new Promise<'hung'>((r) => setTimeout(() => r('hung'), 4000)),
|
||||
]);
|
||||
expect(first).not.toBe('hung');
|
||||
expect(performance.now() - started).toBeLessThan(3500);
|
||||
expect((first as { gpu?: object }).gpu).toBeUndefined();
|
||||
expect(counted('nvidia-smi')).toBe(1);
|
||||
// the reader that timed out is left alone like a failing one: the next poll does not start a second
|
||||
vi.setSystemTime(Date.now() + 2500);
|
||||
await readHostSensors({ gpu: true });
|
||||
expect(counted('nvidia-smi')).toBe(1);
|
||||
// the child finally exiting changes nothing
|
||||
state.heldNvidia.forEach((answer) => answer());
|
||||
await tick();
|
||||
expect(counted('nvidia-smi')).toBe(1);
|
||||
stopHostSensors();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the PowerShell RAM probe', () => {
|
||||
it('is never waited for: the memory in use shows at once, and the total once Windows has answered', async () => {
|
||||
state.nvidia = new Error('no nvidia');
|
||||
|
||||
Reference in new issue
Block a user