mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-20 04:59:34 +01:00
fix(watchers): stop a chokidar watcher whose fs.watch fails instead of retrying forever
chokidar 3 with `persistent: false` has no guard for an `fs.watch` that throws (EMFILE once the inotify instance limit is used up, ENOSPC once the watch limit is). It emits the error, then a TypeError, then retries on the parent directory, and ends up re-adding `/` forever. TeamWatcher only logged its `'error'` events, so one failed watch became an endless stream of stack traces. WorkflowRunWatcher already closed its watcher on error, but `close()` removes every listener, so the TypeError from the same pass became an unhandled rejection, one per 10 s poll, and the server exits after five of those in a minute. `closeWatcherOnError()` closes the watcher on its first error, keeps a no-op listener on it, and turns its `add()` into a no-op, since a callback chokidar still has in flight can otherwise re-add a path and reopen it. TeamWatcher logs one line per watcher and relies on its 30 s poll; WorkflowRunWatcher still drops the watcher and re-arms it on the next poll.
This commit is contained in:
1 parent
e99b42c387
commit
10e443a309
5 files changed
+465
-8
No files matched your search
@@ -0,0 +1,416 @@
|
||||
/**
|
||||
* @fileoverview Pins bounded fs.watch failure handling and polling fallback after a production chokidar retry loop flooded logs and synchronous close caused unhandled rejections.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { TeamWatcher } from '../src/team-watcher.js';
|
||||
import { WorkflowRunWatcher } from '../src/workflow-run-watcher.js';
|
||||
import { TEAM_POLL_INTERVAL_MS } from '../src/config/team-config.js';
|
||||
import { WORKFLOW_RUN_POLL_INTERVAL_MS } from '../src/config/workflow-config.js';
|
||||
import type { TeamConfig } from '../src/types.js';
|
||||
import type { WorkflowRunInfo } from '../src/types/workflow-run.js';
|
||||
|
||||
const FAILURES = [
|
||||
{ code: 'EMFILE', errno: -24, text: 'too many open files' },
|
||||
{ code: 'ENOSPC', errno: -28, text: 'no space left on device' },
|
||||
] as const;
|
||||
type WatchFailure = (typeof FAILURES)[number];
|
||||
|
||||
const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
async function waitUntil(predicate: () => boolean, message: string): Promise<void> {
|
||||
const deadline = Date.now() + 1500;
|
||||
while (!predicate()) {
|
||||
if (Date.now() >= deadline) throw new Error(message);
|
||||
await delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
function watchError(failure: WatchFailure, path: string): NodeJS.ErrnoException {
|
||||
return Object.assign(new Error(`${failure.code}: ${failure.text}, watch '${path}'`), {
|
||||
code: failure.code,
|
||||
errno: failure.errno,
|
||||
syscall: 'watch',
|
||||
path,
|
||||
});
|
||||
}
|
||||
|
||||
function failEveryWatch(failure: WatchFailure) {
|
||||
return vi.spyOn(fs, 'watch').mockImplementation((path) => {
|
||||
throw watchError(failure, String(path));
|
||||
});
|
||||
}
|
||||
|
||||
let tempDir: string;
|
||||
let watcher: TeamWatcher | WorkflowRunWatcher | undefined;
|
||||
let warn: ReturnType<typeof vi.spyOn<typeof console, 'warn'>>;
|
||||
let error: ReturnType<typeof vi.spyOn<typeof console, 'error'>>;
|
||||
let unhandled: unknown[];
|
||||
let onUnhandled: (reason: unknown) => void;
|
||||
|
||||
beforeEach(async () => {
|
||||
watcher = undefined;
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'codeman-watch-failure-'));
|
||||
warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
error = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
unhandled = [];
|
||||
onUnhandled = (reason) => unhandled.push(reason);
|
||||
process.on('unhandledRejection', onUnhandled);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
watcher?.stop();
|
||||
// Keep mocks and the rejection listener installed while asynchronous close settles.
|
||||
await delay(100);
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
} finally {
|
||||
process.off('unhandledRejection', onUnhandled);
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
}
|
||||
});
|
||||
|
||||
function createTeamConfig(): TeamConfig {
|
||||
return {
|
||||
name: 'test-team',
|
||||
createdAt: 1_700_000_000_000,
|
||||
leadAgentId: 'team-lead@test-team',
|
||||
leadSessionId: 'session-123',
|
||||
members: [
|
||||
{
|
||||
agentId: 'team-lead@test-team',
|
||||
name: 'team-lead',
|
||||
agentType: 'team-lead',
|
||||
joinedAt: 1_700_000_000_000,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function writeTeam(teamsDir: string, config: TeamConfig): Promise<void> {
|
||||
const dir = join(teamsDir, config.name);
|
||||
await mkdir(dir);
|
||||
await writeFile(join(dir, 'config.json'), JSON.stringify(config));
|
||||
}
|
||||
|
||||
describe('TeamWatcher fs.watch failure', () => {
|
||||
let teamsDir: string;
|
||||
let tasksDir: string;
|
||||
let teamWatcher: TeamWatcher;
|
||||
|
||||
beforeEach(async () => {
|
||||
teamsDir = join(tempDir, 'teams');
|
||||
tasksDir = join(tempDir, 'tasks');
|
||||
await mkdir(teamsDir);
|
||||
await mkdir(tasksDir);
|
||||
watcher = teamWatcher = new TeamWatcher(teamsDir, tasksDir);
|
||||
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
|
||||
});
|
||||
|
||||
it.each(FAILURES)('$code stops both watchers with two warnings and keeps polling', async (failure) => {
|
||||
const watch = failEveryWatch(failure);
|
||||
try {
|
||||
teamWatcher.start();
|
||||
await waitUntil(() => warn.mock.calls.length >= 2, 'timed out waiting for both watcher warnings');
|
||||
const attempts = watch.mock.calls.length;
|
||||
await delay(500);
|
||||
|
||||
// Check counts before arguments so the unfixed log storm cannot flood failure output.
|
||||
expect(warn).toHaveBeenCalledTimes(2);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
for (const call of warn.mock.calls) expect(call.length).toBe(1);
|
||||
expect(warn.mock.calls.map(([message]) => message).sort()).toEqual(
|
||||
[
|
||||
`[TeamWatcher] teams watcher stopped, relying on polling: ${watchError(failure, teamsDir).message}`,
|
||||
`[TeamWatcher] tasks watcher stopped, relying on polling: ${watchError(failure, tasksDir).message}`,
|
||||
].sort()
|
||||
);
|
||||
expect(watch.mock.calls.length).toBe(attempts);
|
||||
expect(attempts).toBeGreaterThanOrEqual(2);
|
||||
// Allow a handful of in-flight attempts, but never an ancestor retry loop.
|
||||
expect(attempts).toBeLessThanOrEqual(6);
|
||||
expect(unhandled.length).toBe(0);
|
||||
|
||||
const config = createTeamConfig();
|
||||
await writeTeam(teamsDir, config);
|
||||
expect(teamWatcher.getTeams()).toEqual([]);
|
||||
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
|
||||
await waitUntil(() => teamWatcher.getTeams().length > 0, 'polling did not discover the new team');
|
||||
expect(teamWatcher.getTeams()).toEqual([config]);
|
||||
expect(watch.mock.calls.length).toBe(attempts);
|
||||
expect(warn).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(() => teamWatcher.stop()).not.toThrow();
|
||||
await delay(500);
|
||||
expect(watch.mock.calls.length).toBe(attempts);
|
||||
expect(warn).toHaveBeenCalledTimes(2);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
expect(unhandled.length).toBe(0);
|
||||
} finally {
|
||||
teamWatcher.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps a healthy watcher active and discovers a new team before the poll interval', async () => {
|
||||
try {
|
||||
teamWatcher.start();
|
||||
await delay(200);
|
||||
const config = createTeamConfig();
|
||||
await writeTeam(teamsDir, config);
|
||||
// No interval is advanced: discovery must come from the real filesystem watcher.
|
||||
await waitUntil(() => teamWatcher.getTeams().length > 0, 'healthy watcher did not discover the new team');
|
||||
expect(teamWatcher.getTeams()).toEqual([config]);
|
||||
teamWatcher.stop();
|
||||
await delay(100);
|
||||
expect(warn).toHaveBeenCalledTimes(0);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
expect(unhandled.length).toBe(0);
|
||||
} finally {
|
||||
teamWatcher.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('TeamWatcher in-flight fs.watch failure', () => {
|
||||
it('stays stopped when file replacement and deletion race with ENOSPC', async () => {
|
||||
const teamsDir = join(tempDir, 'teams');
|
||||
const tasksDir = join(tempDir, 'tasks');
|
||||
const teamDir = join(teamsDir, 't');
|
||||
await mkdir(teamDir, { recursive: true });
|
||||
await mkdir(tasksDir);
|
||||
await writeFile(join(teamDir, 'a.json'), '{}');
|
||||
await writeFile(join(teamDir, 'b.json'), '{}');
|
||||
const teamWatcher = new TeamWatcher(teamsDir, tasksDir);
|
||||
watcher = teamWatcher;
|
||||
const realWatch = fs.watch;
|
||||
const listeners = new Map<string, fs.WatchListener<string | Buffer>>();
|
||||
const watch = vi.spyOn(fs, 'watch').mockImplementation((...args) => {
|
||||
const handle = realWatch(...args);
|
||||
if (args[2]) listeners.set(String(args[0]), args[2]);
|
||||
return handle;
|
||||
});
|
||||
let firstFailure: NodeJS.ErrnoException | undefined;
|
||||
let failingCalls = 0;
|
||||
|
||||
try {
|
||||
teamWatcher.start();
|
||||
await waitUntil(
|
||||
() => listeners.has(join(teamDir, 'a.json')) && listeners.has(join(teamDir, 'b.json')),
|
||||
'timed out waiting for both file watches'
|
||||
);
|
||||
expect(warn).toHaveBeenCalledTimes(0);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
watch.mockImplementation((...args) => {
|
||||
// Bound the unfixed retry loop so it can recover and the test can finish.
|
||||
if (failingCalls >= 2000) return realWatch(...args);
|
||||
failingCalls++;
|
||||
const failure = watchError(FAILURES[1], String(args[0]));
|
||||
firstFailure ??= failure;
|
||||
throw failure;
|
||||
});
|
||||
|
||||
fs.writeFileSync(join(teamDir, 'b.json.tmp'), '{"replaced":true}');
|
||||
fs.renameSync(join(teamDir, 'b.json.tmp'), join(teamDir, 'b.json'));
|
||||
await waitUntil(() => warn.mock.calls.length > 0, 'timed out waiting for the teams watcher warning');
|
||||
// Deliver the queued deletion callback only after the replacement failure stops the watcher.
|
||||
fs.unlinkSync(join(teamDir, 'a.json'));
|
||||
listeners.get(join(teamDir, 'a.json'))!('rename', 'a.json');
|
||||
await delay(1000);
|
||||
const attempts = watch.mock.calls.length;
|
||||
const failures = failingCalls;
|
||||
await delay(500);
|
||||
const settledAttempts = watch.mock.calls.length;
|
||||
const settledFailures = failingCalls;
|
||||
const warnings = warn.mock.calls.length;
|
||||
|
||||
teamWatcher.stop();
|
||||
await delay(500);
|
||||
|
||||
// Check counts before arguments so a retry storm cannot flood failure output.
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
expect(warnings).toBe(1);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
expect(unhandled.length).toBe(0);
|
||||
expect(watch.mock.calls.length).toBe(settledAttempts);
|
||||
expect(failingCalls).toBe(settledFailures);
|
||||
expect(settledAttempts).toBe(attempts);
|
||||
expect(settledFailures).toBe(failures);
|
||||
expect(failures).toBeGreaterThanOrEqual(1);
|
||||
expect(failures).toBeLessThanOrEqual(6);
|
||||
expect(firstFailure?.code).toBe('ENOSPC');
|
||||
expect(firstFailure?.path?.startsWith(`${teamDir}/`)).toBe(true);
|
||||
expect(warn.mock.calls).toEqual([
|
||||
[`[TeamWatcher] teams watcher stopped, relying on polling: ${firstFailure?.message}`],
|
||||
]);
|
||||
} finally {
|
||||
teamWatcher.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const PROJECT_HASH = '-home-arkon-default-claudeman';
|
||||
const SESSION_UUID = '388113c8-cd01-4e80-93a8-3be66ab1519b';
|
||||
const RUN_ID = 'wf_test1234-abc';
|
||||
const RUN_START = 1_700_000_000_000;
|
||||
|
||||
function sampleRunJson() {
|
||||
return {
|
||||
runId: RUN_ID,
|
||||
timestamp: new Date(RUN_START).toISOString(),
|
||||
workflowName: 'watch-failure-test',
|
||||
summary: 'Discover a run despite exhausted filesystem watches',
|
||||
status: 'completed',
|
||||
startTime: RUN_START,
|
||||
agentCount: 0,
|
||||
durationMs: 1234,
|
||||
totalTokens: 0,
|
||||
totalToolCalls: 0,
|
||||
phases: [],
|
||||
workflowProgress: [],
|
||||
};
|
||||
}
|
||||
|
||||
const { workflowProgress: _progress, ...runFields } = sampleRunJson();
|
||||
const EXPECTED_RUN: WorkflowRunInfo = {
|
||||
...runFields,
|
||||
agents: [],
|
||||
projectHash: PROJECT_HASH,
|
||||
sessionUuid: SESSION_UUID,
|
||||
lastActivityAt: RUN_START,
|
||||
};
|
||||
|
||||
describe('WorkflowRunWatcher fs.watch failure', () => {
|
||||
let runWatcher: WorkflowRunWatcher;
|
||||
let workflowsDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
const projectsDir = join(tempDir, 'projects');
|
||||
workflowsDir = join(projectsDir, PROJECT_HASH, SESSION_UUID, 'workflows');
|
||||
await mkdir(workflowsDir, { recursive: true });
|
||||
await writeFile(join(workflowsDir, `${RUN_ID}.json`), JSON.stringify(sampleRunJson()));
|
||||
watcher = runWatcher = new WorkflowRunWatcher(projectsDir);
|
||||
// Later interval polls run only when a test explicitly advances the clock.
|
||||
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
|
||||
});
|
||||
|
||||
function firstRun(): Promise<WorkflowRunInfo> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
runWatcher.off('run_discovered', onRun);
|
||||
reject(new Error('timed out waiting for run_discovered'));
|
||||
}, 1500);
|
||||
function onRun(info: WorkflowRunInfo) {
|
||||
clearTimeout(timer);
|
||||
resolve(info);
|
||||
}
|
||||
runWatcher.once('run_discovered', onRun);
|
||||
runWatcher.start();
|
||||
});
|
||||
}
|
||||
|
||||
it.each(FAILURES)('$code discovers the run without retries, logging, or unhandled rejections', async (failure) => {
|
||||
const watch = failEveryWatch(failure);
|
||||
try {
|
||||
const discovered = await firstRun();
|
||||
await delay(500);
|
||||
const attempts = watch.mock.calls.length;
|
||||
await delay(500);
|
||||
|
||||
expect(unhandled.length).toBe(0);
|
||||
expect(warn).toHaveBeenCalledTimes(0);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
expect(watch.mock.calls.length).toBe(attempts);
|
||||
expect(attempts).toBeGreaterThanOrEqual(1);
|
||||
expect(attempts).toBeLessThanOrEqual(3);
|
||||
expect(discovered).toEqual(EXPECTED_RUN);
|
||||
expect(runWatcher.getRun(RUN_ID)).toEqual(EXPECTED_RUN);
|
||||
expect(runWatcher.getAllRuns()).toEqual([EXPECTED_RUN]);
|
||||
expect(runWatcher.isRunning()).toBe(true);
|
||||
runWatcher.stop();
|
||||
expect(runWatcher.isRunning()).toBe(false);
|
||||
await delay(100);
|
||||
expect(unhandled.length).toBe(0);
|
||||
expect(watch.mock.calls.length).toBe(attempts);
|
||||
expect(warn).toHaveBeenCalledTimes(0);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
} finally {
|
||||
runWatcher.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it('retries a failed directory once per poll and resumes live discovery when fs.watch recovers', async () => {
|
||||
const watch = failEveryWatch(FAILURES[0]);
|
||||
const discovered = vi.fn();
|
||||
try {
|
||||
expect(await firstRun()).toEqual(EXPECTED_RUN);
|
||||
await delay(500);
|
||||
const attemptsPerPoll = watch.mock.calls.length;
|
||||
expect(attemptsPerPoll).toBeGreaterThanOrEqual(1);
|
||||
expect(attemptsPerPoll).toBeLessThanOrEqual(3);
|
||||
await delay(500);
|
||||
expect(watch).toHaveBeenCalledTimes(attemptsPerPoll);
|
||||
expect(warn).toHaveBeenCalledTimes(0);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
expect(unhandled.length).toBe(0);
|
||||
|
||||
for (const pollCount of [2, 3]) {
|
||||
vi.advanceTimersByTime(WORKFLOW_RUN_POLL_INTERVAL_MS);
|
||||
await waitUntil(
|
||||
() => watch.mock.calls.length > attemptsPerPoll * (pollCount - 1),
|
||||
'later poll did not retry the failed workflows directory'
|
||||
);
|
||||
await delay(500);
|
||||
expect(watch).toHaveBeenCalledTimes(attemptsPerPoll * pollCount);
|
||||
expect(warn).toHaveBeenCalledTimes(0);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
expect(unhandled.length).toBe(0);
|
||||
}
|
||||
|
||||
watch.mockRestore();
|
||||
const recoveredWatch = vi.spyOn(fs, 'watch');
|
||||
vi.advanceTimersByTime(WORKFLOW_RUN_POLL_INTERVAL_MS);
|
||||
await waitUntil(() => recoveredWatch.mock.calls.length > 0, 'recovery poll did not install a watcher');
|
||||
// Let watcher setup and the recovery poll settle before creating the new completion file.
|
||||
await delay(500);
|
||||
expect(runWatcher.getAllRuns()).toEqual([EXPECTED_RUN]);
|
||||
const otherId = 'wf_recovered5678-def';
|
||||
runWatcher.on('run_discovered', discovered);
|
||||
await writeFile(join(workflowsDir, `${otherId}.json`), JSON.stringify({ ...sampleRunJson(), runId: otherId }));
|
||||
// Do not advance the interval again: only the recovered watcher can discover this file.
|
||||
await waitUntil(() => discovered.mock.calls.length > 0, 'recovered watcher did not discover the new run');
|
||||
expect(discovered).toHaveBeenCalledTimes(1);
|
||||
const expectedRun = { ...EXPECTED_RUN, runId: otherId };
|
||||
expect(discovered).toHaveBeenCalledWith(expectedRun);
|
||||
expect(runWatcher.getRun(otherId)).toEqual(expectedRun);
|
||||
runWatcher.stop();
|
||||
await delay(100);
|
||||
expect(warn).toHaveBeenCalledTimes(0);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
expect(unhandled.length).toBe(0);
|
||||
} finally {
|
||||
runWatcher.stop();
|
||||
runWatcher.off('run_discovered', discovered);
|
||||
}
|
||||
});
|
||||
|
||||
it('discovers a run with a healthy watcher without logging or unhandled rejections', async () => {
|
||||
try {
|
||||
const discovered = await firstRun();
|
||||
await delay(1000);
|
||||
expect(discovered).toEqual(EXPECTED_RUN);
|
||||
expect(runWatcher.getRun(RUN_ID)).toEqual(EXPECTED_RUN);
|
||||
expect(runWatcher.getAllRuns()).toEqual([EXPECTED_RUN]);
|
||||
runWatcher.stop();
|
||||
await delay(100);
|
||||
expect(warn).toHaveBeenCalledTimes(0);
|
||||
expect(error).toHaveBeenCalledTimes(0);
|
||||
expect(unhandled.length).toBe(0);
|
||||
} finally {
|
||||
runWatcher.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user