fix(host-sensors): never wait on a cold sampler, back off failing readers, one stats request at a time

This commit is contained in:
Devvyn committed 2026-10-24 10:17:20 +08:00
1 parent f15afe1de3
commit 23a9d78cdd
7 files changed
+270 -28

No files matched your search

+1 -1
View File
@@ -1029,7 +1029,7 @@ With `classic` nothing is left behind: no headings, no breaks, no inline order,
### Header stats styles
**Header stats styles** (`headerStatsStyle`, per-device, desktop only, default `compact`; Discussion #426 option G): how the WS readout, CPU, MEM and the plan-usage windows are drawn. `tiles` gives each its own label-over-value tile with a bar along the bottom edge; `compact` is two pills, WS / CPU / MEM and the plan windows, with a ring beside every value (stat rings accent, red past 80%; plan rings green, yellow or red); `classic` is the bars and `5H · 7D` chip as before. `data-header-stats` on `<html>` (pre-paint, then `applyHeaderStatsStyle()` from `applyHeaderVisibilitySettings()`) drives every rule; narrower than 768px and in a solo window it resolves to `classic`, because the stats are hidden there anyway and the moves below would otherwise reach the phone header.
**Header stats styles** (`headerStatsStyle`, per-device, desktop only, default `compact`; Discussion #426 option G): how the WS readout, CPU, MEM and the plan-usage windows are drawn. `tiles` gives each its own label-over-value tile with a bar along the bottom edge; `compact` is two pills, WS / CPU / MEM and the plan windows, with a ring beside every value (stat rings accent, red past 80%; plan rings green, yellow or red); `classic` is the bars and `5H · 7D` chip as before. `data-header-stats` on `<html>` (pre-paint, then `applyHeaderStatsStyle()` from `applyHeaderVisibilitySettings()`) drives every rule; narrower than 768px and in a solo window it resolves to `classic`, because the stats are hidden there anyway and the moves below would otherwise reach the phone header. The opt-in host readings after MEM (GPU, VRAM, GPU MHz, CPU MHz: `.stat-item` elements that start `hidden` and are shown by `_updateHostSensors` in panels-ui.js) rely on `html[data-header-stats] .header-system-stats .stat-item[hidden]` in styles.css outranking the Tiles style's own `display` on `.stat-item`; keep that selector's specificity above any style rule that sets `display` on a stat item, or the hidden placeholders (`--%`) reappear.
⚠️ The clustered styles need the three elements contiguous, and the template keeps the classic order, so `applyHeaderStatsStyle()` MOVES `#connectionIndicator` into `#headerSystemStats` (first child) and `#planUsageChip` right after it, and `classic` moves them back to comment anchors it left at the template positions. Ids are unchanged, so every writer still finds them. ⚠️ The indicator joins the pill only while System Stats is shown: the pill is hidden with `display: none`, and the WS readout must not disappear with it.
+54 -21
View File
@@ -48,6 +48,10 @@ export interface SensorRequest {
const CACHE_MS = 2000;
/** A counter sample older than this (a few sampling intervals) is no longer a live reading. */
const FRESH_MS = 10000;
/** After a reader fails, leave it alone this long: nvidia-smi absent or timing out, PowerShell failing. */
const READER_BACKOFF_MS = 60000;
/** The GPU counters stay in the sampler this long after the last GPU read that needed them. */
const GPU_COUNTERS_HOLD_MS = 120000;
const EXEC_TIMEOUT_MS = 1500;
const SYS_DRM = '/sys/class/drm';
const SYS_CPU = '/sys/devices/system/cpu';
@@ -191,16 +195,24 @@ function inWsl(): boolean {
}
let windowsRamMB: Promise<number | undefined> | undefined;
let ramRetryAt = 0;
/** Windows' installed RAM, asked once (PowerShell takes about a second) and remembered when it answers. */
/**
* Windows' installed RAM, asked once (PowerShell takes about a second) and remembered when it answers. A
* failure is not retried on every GPU poll (8 s timeout each): it waits READER_BACKOFF_MS first.
*/
function readWindowsRamMB(): Promise<number | undefined> {
if (!windowsRamMB && Date.now() < ramRetryAt) return Promise.resolve(undefined);
windowsRamMB ??= run(
POWERSHELL,
['-NoProfile', '-NonInteractive', '-Command', '(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory'],
8000
).then((out) => {
const bytes = out ? finite(Number(out.trim())) : undefined;
if (!bytes || bytes <= 0) windowsRamMB = undefined;
if (!bytes || bytes <= 0) {
windowsRamMB = undefined;
ramRetryAt = Date.now() + READER_BACKOFF_MS;
}
return bytes && bytes > 0 ? Math.round(bytes / MB) : undefined;
});
return windowsRamMB;
@@ -212,7 +224,11 @@ export type SamplerChild = Pick<ChildProcess, 'stdout' | 'kill' | 'once'>;
export interface CounterStreamOptions {
start: (counters: string[]) => SamplerChild;
now?: () => number;
/** How long the first read waits for the first sample. */
/**
* How long a read waits for a sample when it has none fresh (0, the default, never waits: the read
* answers undefined and the next poll shows the reading). The stats route awaits the sensors before it
* answers CPU and MEM, so a cold sampler must not hold it up.
*/
firstSampleMs?: number;
/** A sampler older than this is replaced, so counters of processes that started since are picked up. */
recycleMs?: number;
@@ -230,7 +246,7 @@ export interface CounterStreamOptions {
*/
export function createCounterStream(options: CounterStreamOptions) {
const now = options.now ?? Date.now;
const firstSampleMs = options.firstSampleMs ?? 8000;
const firstSampleMs = options.firstSampleMs ?? 0;
const recycleMs = options.recycleMs ?? 30000;
const idleMs = options.idleMs ?? 30000;
const backoffMs = options.backoffMs ?? 60000;
@@ -307,7 +323,7 @@ export function createCounterStream(options: CounterStreamOptions) {
};
return {
/** The newest sample for these counters; the first read waits briefly for one. */
/** The newest fresh sample for these counters, or undefined (a cold sampler is not waited for unless `firstSampleMs` says so). */
async read(counters: string[]): Promise<WindowsCounters | undefined> {
lastRead = now();
if (now() < blockedUntil) return undefined;
@@ -315,10 +331,12 @@ export function createCounterStream(options: CounterStreamOptions) {
if (!child || key !== wanted || now() - startedAt > recycleMs) begin(counters);
if (!latest || key !== wanted || now() - latest.at >= FRESH_MS) {
if (!child) return undefined;
await new Promise<void>((resolve) => {
waiting.push(resolve);
setTimeout(resolve, firstSampleMs).unref();
});
if (firstSampleMs > 0) {
await new Promise<void>((resolve) => {
waiting.push(resolve);
setTimeout(resolve, firstSampleMs).unref();
});
}
}
return latest && now() - latest.at < FRESH_MS ? latest.value : undefined;
},
@@ -342,21 +360,28 @@ const CPU_COUNTERS = ['\\Processor Information(_Total)\\Processor Frequency'];
/**
* The counters one sampler reads. They do not depend on which reading a browser asked for (those are per
* device, so two browsers can differ): the CPU clock is always sampled, and the GPU counters are added, once
* and for good, on a machine with no native GPU reader. A request then only picks its fields from the sample.
* device, so two browsers can differ): the CPU clock is always sampled, and the GPU counters are added on a
* machine with no native GPU reader (and dropped again once nothing has needed them for a while). A request then only picks its fields from the sample.
*/
export function windowsCounterList(needGpu: boolean): string[] {
return needGpu ? [...GPU_COUNTERS, ...CPU_COUNTERS] : [...CPU_COUNTERS];
}
export function createWindowsSource(stream: Pick<ReturnType<typeof createCounterStream>, 'read' | 'stop'>) {
let needGpu = false;
export function createWindowsSource(
stream: Pick<ReturnType<typeof createCounterStream>, 'read' | 'stop'>,
now: () => number = Date.now
) {
let gpuUntil = 0;
return {
/** A GPU reading is wanted and no native reader answered: sample the GPU counters too. */
/**
* A GPU reading is wanted and no native reader answered: sample the GPU counters too, for
* GPU_COUNTERS_HOLD_MS after the last such read. A single failed nvidia-smi therefore costs a sampler
* restart, not a GPU sampler for the life of the server.
*/
needGpuCounters() {
needGpu = true;
gpuUntil = now() + GPU_COUNTERS_HOLD_MS;
},
read: () => stream.read(windowsCounterList(needGpu)),
read: () => stream.read(windowsCounterList(now() < gpuUntil)),
stop: () => stream.stop(),
};
}
@@ -399,12 +424,20 @@ async function readAmdGpu(): Promise<GpuReading | undefined> {
}
/** NVIDIA, then AMD: the readers that need nothing from Windows. */
let nvidiaSkipUntil = 0;
/** NVIDIA, then AMD. A host without a working nvidia-smi is not asked again on every poll. */
async function readNativeGpu(): Promise<GpuReading | undefined> {
const out = await run('nvidia-smi', [
'--query-gpu=utilization.gpu,memory.used,memory.total,clocks.gr',
'--format=csv,noheader,nounits',
]);
return (out && parseNvidiaSmi(out)) || readAmdGpu();
if (Date.now() >= nvidiaSkipUntil) {
const out = await run('nvidia-smi', [
'--query-gpu=utilization.gpu,memory.used,memory.total,clocks.gr',
'--format=csv,noheader,nounits',
]);
const reading = out ? parseNvidiaSmi(out) : undefined;
if (reading) return reading;
nvidiaSkipUntil = Date.now() + READER_BACKOFF_MS;
}
return readAmdGpu();
}
/** A GPU reading from the Windows counters (WSL only): load and memory in use, but no clock. */
+2 -2
View File
@@ -194,11 +194,11 @@
</div>
<span class="stat-value" id="statVram">--%</span>
</div>
<div class="stat-item stat-mhz" id="statGpuMhzItem" hidden>
<div class="stat-item" id="statGpuMhzItem" hidden>
<span class="stat-label">GPU</span>
<span class="stat-value" id="statGpuMhz">-- MHz</span>
</div>
<div class="stat-item stat-mhz" id="statCpuMhzItem" hidden>
<div class="stat-item" id="statCpuMhzItem" hidden>
<span class="stat-label">CPU</span>
<span class="stat-value" id="statCpuMhz">-- MHz</span>
</div>
+12 -1
View File
@@ -6716,6 +6716,10 @@ Object.assign(CodemanApp.prototype, {
// Skip polling when system stats display is hidden
const statsEl = document.getElementById('headerSystemStats');
if (!statsEl || statsEl.style.display === 'none') return;
// One request at a time: the poll fires every 2 s, and a slow answer (a cold WSL sampler, a hung
// server) must not pile up requests behind it.
if (this._systemStatsPending) return;
this._systemStatsPending = true;
try {
// Opt-in readings are only requested while their header setting is on (default off).
@@ -6723,11 +6727,18 @@ Object.assign(CodemanApp.prototype, {
const wanted = ['gpu', settings.showGpuStats, 'cpumhz', settings.showCpuMhz];
const query = [];
for (let i = 0; i < wanted.length; i += 2) if (wanted[i + 1] === true) query.push(`${wanted[i]}=1`);
const res = await fetch('/api/system/stats' + (query.length ? '?' + query.join('&') : ''));
const res = await fetch('/api/system/stats' + (query.length ? '?' + query.join('&') : ''), {
signal:
typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function'
? AbortSignal.timeout(8000)
: undefined,
});
const stats = await res.json();
this.updateSystemStatsDisplay(stats.data);
} catch (err) {
// Silently fail - system stats are not critical
} finally {
this._systemStatsPending = false;
}
},
+19 -3
View File
@@ -30,9 +30,11 @@ const classList = () => {
};
};
function harness(settings: Record<string, unknown>) {
function harness(settings: Record<string, unknown>, options: { hold?: boolean } = {}) {
const els = new Map<string, El>();
const fetched: string[] = [];
let release: () => void = () => undefined;
const gate = options.hold ? new Promise<void>((resolve) => (release = resolve)) : null;
class CodemanApp {}
const context = vm.createContext({
CodemanApp,
@@ -40,6 +42,7 @@ function harness(settings: Record<string, unknown>) {
document: { getElementById: () => ({ style: {} }) },
fetch: async (url: string) => {
fetched.push(url);
if (gate) await gate;
return { json: async () => ({ data: { cpu: 1, memory: { usedMB: 1, totalMB: 2, percent: 50 } } }) };
},
localStorage: { getItem: () => null },
@@ -62,7 +65,7 @@ function harness(settings: Record<string, unknown>) {
return els.get(id);
},
});
return { app, els, fetched };
return { app, els, fetched, release };
}
describe('header host sensors', () => {
@@ -131,6 +134,17 @@ describe('header host sensors', () => {
expect(appJs).not.toMatch(/\} else \{\s*this\.stopSystemStatsPolling\(\);/);
});
it('runs one stats request at a time, so a slow answer cannot pile requests up behind it', async () => {
const { app, fetched, release } = harness({}, { hold: true });
const polls = [app.fetchSystemStats.call(app), app.fetchSystemStats.call(app), app.fetchSystemStats.call(app)];
expect(fetched).toHaveLength(1);
release();
await Promise.all(polls);
// answered: the next poll goes out again
await app.fetchSystemStats.call(app);
expect(fetched).toHaveLength(2);
});
it('is opt-in in the settings page: unchecked chips, off in every default, stripped from the settings PUT', () => {
const html = read('index.html');
for (const id of ['appSettingsShowGpuStats', 'appSettingsShowCpuMhz']) {
@@ -142,6 +156,8 @@ describe('header host sensors', () => {
// per-device, so stripped from the PUT body like the other header toggles (the schema is .strict())
expect(settingsUi).toContain('showGpuStats: _sgp,');
expect(settingsUi).toContain('showCpuMhz: _scm,');
expect(readFileSync(join(process.cwd(), 'src/web/schemas.ts'), 'utf-8')).not.toContain('showCpuMhz');
const schemas = readFileSync(join(process.cwd(), 'src/web/schemas.ts'), 'utf-8');
expect(schemas).not.toContain('showCpuMhz');
expect(schemas).not.toContain('showGpuStats');
});
});
+162
View File
@@ -0,0 +1,162 @@
/**
* The order and the back-offs of readHostSensors, with `child_process` and the sysfs reads mocked so no
* hardware is needed: native GPU readers go first (a machine with a working nvidia-smi never starts the
* Windows GPU counters), a failing reader is left alone for a minute, a cold Windows sampler is not
* waited for, and the GPU counters leave the sampler again once nothing needs them.
*/
import { EventEmitter } from 'node:events';
import { PassThrough } from 'node:stream';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
type ExecCall = { cmd: string; args: string[] };
const state = vi.hoisted(() => ({
nvidia: '' as string | Error,
powershell: '' as string | Error,
execCalls: [] as Array<{ cmd: string; args: string[] }>,
spawnCalls: [] as Array<{ cmd: string; args: string[]; stdout: unknown }>,
}));
vi.mock('node:child_process', () => ({
execFile: (cmd: string, args: string[], _opts: unknown, cb: (err: Error | null, stdout: string) => void) => {
state.execCalls.push({ cmd, args });
const answer = cmd === 'nvidia-smi' ? state.nvidia : cmd.endsWith('powershell.exe') ? state.powershell : '';
if (answer instanceof Error) cb(answer, '');
else cb(null, answer);
},
spawn: (cmd: string, args: string[]) => {
const stdout = new PassThrough();
const child = Object.assign(new EventEmitter(), { stdout, kill: () => true });
state.spawnCalls.push({ cmd, args, stdout });
return child;
},
}));
vi.mock('node:fs/promises', () => ({
readdir: async () => [],
readFile: async () => {
throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' });
},
}));
const NVIDIA = '37, 2048, 8192, 1815\n';
const HEADER =
'"(PDH-CSV 4.0)",' +
'"\\\\H\\GPU Engine(pid_1_luid_0x00000000_0x000139EC_phys_0_eng_0_engtype_3D)\\Utilization Percentage",' +
'"\\\\H\\GPU Adapter Memory(luid_0x00000000_0x000139EC_phys_0)\\Shared Usage",' +
'"\\\\H\\GPU Adapter Memory(luid_0x00000000_0x000139EC_phys_0)\\Dedicated Usage",' +
'"\\\\H\\Processor Information(_Total)\\Processor Frequency"\r\n';
const ROW = '"t","20","1073741824","0","1650"\r\n';
async function load() {
vi.resetModules();
return import('../src/host-sensors.js');
}
const counted = (cmd: string) => state.execCalls.filter((c: ExecCall) => c.cmd.endsWith(cmd)).length;
const tick = () => new Promise((r) => setImmediate(r));
beforeEach(() => {
state.nvidia = NVIDIA;
state.powershell = new Error('powershell failed');
state.execCalls = [];
state.spawnCalls = [];
process.env.WSL_INTEROP = '1';
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(new Date('2026-10-24T00:00:00Z'));
});
afterEach(() => {
delete process.env.WSL_INTEROP;
vi.useRealTimers();
});
describe('native readers first', () => {
it('answers from nvidia-smi on WSL and never starts the Windows GPU counters', async () => {
const { readHostSensors, stopHostSensors } = await load();
const out = await readHostSensors({ gpu: true });
expect(out.gpu).toMatchObject({ percent: 37, memUsedMB: 2048, memPercent: 25, mhz: 1815 });
expect(state.spawnCalls).toEqual([]);
stopHostSensors();
});
it('starts only the CPU clock counter for a CPU MHz request, and never waits for it', async () => {
const { readHostSensors, stopHostSensors } = await load();
const started = Date.now();
const out = await readHostSensors({ cpuMhz: true });
expect(Date.now() - started).toBeLessThan(1000);
expect(out).toEqual({});
expect(state.spawnCalls).toHaveLength(1);
expect(state.spawnCalls[0].args.join(' ')).toContain('Processor Frequency');
expect(state.spawnCalls[0].args.join(' ')).not.toContain('GPU Engine');
stopHostSensors();
});
it('falls back to the Windows counters, GPU ones included, when nvidia-smi gives nothing', async () => {
state.nvidia = new Error('nvidia-smi: command not found');
const { readHostSensors, stopHostSensors } = await load();
await readHostSensors({ gpu: true });
expect(state.spawnCalls).toHaveLength(1);
expect(state.spawnCalls[0].args.join(' ')).toContain('GPU Engine');
stopHostSensors();
});
});
describe('a failing reader is left alone for a minute', () => {
it('does not run nvidia-smi on every poll of a host without one, and tries again after the back-off', async () => {
state.nvidia = new Error('nvidia-smi: command not found');
const { readHostSensors, stopHostSensors } = await load();
for (let i = 0; i < 5; i++) {
await readHostSensors({ gpu: true });
vi.setSystemTime(Date.now() + 2500); // past the 2 s cache each time
}
expect(counted('nvidia-smi')).toBe(1);
vi.setSystemTime(Date.now() + 61_000);
await readHostSensors({ gpu: true });
expect(counted('nvidia-smi')).toBe(2);
stopHostSensors();
});
it('does not retry a failing PowerShell on every GPU poll', async () => {
state.nvidia = new Error('no nvidia');
const { readHostSensors, stopHostSensors } = await load();
await readHostSensors({ gpu: true }); // starts the sampler
(state.spawnCalls[0].stdout as PassThrough).write(HEADER + ROW);
await tick();
for (let i = 0; i < 4; i++) {
vi.setSystemTime(Date.now() + 2500);
// the real sampler emits a row every 2 s
(state.spawnCalls[0].stdout as PassThrough).write(ROW);
await tick();
const out = await readHostSensors({ gpu: true });
expect(out.gpu).toMatchObject({ percent: 20, memUsedMB: 1024 });
expect(out.gpu?.memPercent).toBeUndefined(); // no RAM total without PowerShell
}
expect(counted('powershell.exe')).toBe(1);
vi.setSystemTime(Date.now() + 61_000);
(state.spawnCalls[0].stdout as PassThrough).write(ROW);
await tick();
await readHostSensors({ gpu: true });
expect(counted('powershell.exe')).toBe(2);
stopHostSensors();
});
});
describe('the GPU counters do not stay in the sampler for good', () => {
it('holds them for a while after the last read that needed them, then drops them', async () => {
const { createWindowsSource, windowsCounterList } = await load();
let clock = 1_000;
const started: string[][] = [];
const stream = {
read: async (counters: string[]) => {
started.push(counters);
return undefined;
},
stop: () => undefined,
};
const source = createWindowsSource(stream, () => clock);
await source.read();
source.needGpuCounters();
clock += 60_000;
await source.read(); // still inside the hold
clock += 61_000; // 121 s after the last need
await source.read();
expect(started).toEqual([windowsCounterList(false), windowsCounterList(true), windowsCounterList(false)]);
});
});
+20
View File
@@ -171,6 +171,26 @@ describe('createCounterStream', () => {
expect(children).toHaveLength(2);
});
it('does not wait at all by default: a cold sampler answers nothing and the next read has the sample', async () => {
const children: PassThrough[] = [];
const stream = createCounterStream({
start: () => {
const stdout = new PassThrough();
children.push(stdout);
return { stdout, kill: () => true, once: () => undefined } as never;
},
});
const counters = ['\\Processor Information(_Total)\\Processor Frequency'];
const header = '"(PDH-CSV 4.0)","\\\\H\\Processor Information(_Total)\\Processor Frequency"\r\n';
const started = Date.now();
expect(await stream.read(counters)).toBeUndefined();
expect(Date.now() - started).toBeLessThan(500);
children[0].write(header + '"t","1700"\r\n');
await new Promise((r) => setImmediate(r));
expect(await stream.read(counters)).toEqual({ cpuMhz: 1700 });
expect(children).toHaveLength(1);
});
it('answers nothing, within the wait, when no sample ever arrives', async () => {
const { stream } = fake();
expect(await stream.read(counters)).toBeUndefined();