Files
Codeman/test/watcher-fs-watch-failure.test.ts
T
Codeman maintainer 8202e07286 fix(watchers): the two review items of #818
The fs.watch failure tests skip on macOS, where chokidar 3 never calls
fs.watch for these watchers (fsevents, or polling without it), so the stub
never fired and the waits timed out. TeamWatcher.stop() clears its outage
set, so a stop() and start() on the same instance warns again about a
watcher that still fails; a new test pins it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 04:02:07 +01:00

696 lines
28 KiB
TypeScript

/**
* @fileoverview Pins bounded fs.watch failure handling and polling fallback after a production chokidar retry loop flooded logs and synchronous close caused unhandled rejections.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import fs from 'node:fs';
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { TeamWatcher } from '../src/team-watcher.js';
import { WorkflowRunWatcher } from '../src/workflow-run-watcher.js';
import { TEAM_POLL_INTERVAL_MS } from '../src/config/team-config.js';
import { WORKFLOW_RUN_POLL_INTERVAL_MS } from '../src/config/workflow-config.js';
import type { TeamConfig } from '../src/types.js';
import type { WorkflowRunInfo } from '../src/types/workflow-run.js';
const FAILURES = [
{ code: 'EMFILE', errno: -24, text: 'too many open files' },
{ code: 'ENOSPC', errno: -28, text: 'no space left on device' },
] as const;
type WatchFailure = { code: string; errno: number; text: string };
// On macOS chokidar 3 never calls fs.watch for these watchers: it uses fsevents,
// or polling when fsevents is missing, so the retry loop these tests pin cannot
// happen there and the stubbed fs.watch never fires.
const describeLinuxWatch = describe.skipIf(process.platform === 'darwin');
const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
async function waitUntil(predicate: () => boolean, message: string): Promise<void> {
const deadline = Date.now() + 1500;
while (!predicate()) {
if (Date.now() >= deadline) throw new Error(message);
await delay(10);
}
}
function watchError(failure: WatchFailure, path: string): NodeJS.ErrnoException {
return Object.assign(new Error(`${failure.code}: ${failure.text}, watch '${path}'`), {
code: failure.code,
errno: failure.errno,
syscall: 'watch',
path,
});
}
function failEveryWatch(failure: WatchFailure) {
return vi.spyOn(fs, 'watch').mockImplementation((path) => {
throw watchError(failure, String(path));
});
}
let tempDir: string;
let watcher: TeamWatcher | WorkflowRunWatcher | undefined;
let warn: ReturnType<typeof vi.spyOn<typeof console, 'warn'>>;
let error: ReturnType<typeof vi.spyOn<typeof console, 'error'>>;
let unhandled: unknown[];
let onUnhandled: (reason: unknown) => void;
beforeEach(async () => {
watcher = undefined;
tempDir = await mkdtemp(join(tmpdir(), 'codeman-watch-failure-'));
warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
error = vi.spyOn(console, 'error').mockImplementation(() => {});
unhandled = [];
onUnhandled = (reason) => unhandled.push(reason);
process.on('unhandledRejection', onUnhandled);
});
afterEach(async () => {
try {
watcher?.stop();
// Keep mocks and the rejection listener installed while asynchronous close settles.
await delay(100);
await rm(tempDir, { recursive: true, force: true });
} finally {
process.off('unhandledRejection', onUnhandled);
vi.useRealTimers();
vi.restoreAllMocks();
}
});
function createTeamConfig(): TeamConfig {
return {
name: 'test-team',
createdAt: 1_700_000_000_000,
leadAgentId: 'team-lead@test-team',
leadSessionId: 'session-123',
members: [
{
agentId: 'team-lead@test-team',
name: 'team-lead',
agentType: 'team-lead',
joinedAt: 1_700_000_000_000,
},
],
};
}
async function writeTeam(teamsDir: string, config: TeamConfig): Promise<void> {
const dir = join(teamsDir, config.name);
await mkdir(dir);
await writeFile(join(dir, 'config.json'), JSON.stringify(config));
}
describeLinuxWatch('TeamWatcher fs.watch failure', () => {
let teamsDir: string;
let tasksDir: string;
let teamWatcher: TeamWatcher;
beforeEach(async () => {
teamsDir = join(tempDir, 'teams');
tasksDir = join(tempDir, 'tasks');
await mkdir(teamsDir);
await mkdir(tasksDir);
watcher = teamWatcher = new TeamWatcher(teamsDir, tasksDir);
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
});
function expectWarnings(messages: string[]): void {
// Check counts before arguments so a retry storm cannot flood failure output.
expect(warn).toHaveBeenCalledTimes(messages.length);
expect(error).toHaveBeenCalledTimes(0);
expect(unhandled.length).toBe(0);
expect(warn.mock.calls.map((call) => [...call]).sort()).toEqual(messages.map((message) => [message]).sort());
}
it.each(FAILURES)('$code stops both watchers with two warnings and keeps polling', async (failure) => {
const watch = failEveryWatch(failure);
try {
teamWatcher.start();
await waitUntil(() => warn.mock.calls.length >= 2, 'timed out waiting for both watcher warnings');
const attempts = watch.mock.calls.length;
await delay(500);
// Check counts before arguments so the unfixed log storm cannot flood failure output.
expect(warn).toHaveBeenCalledTimes(2);
expect(error).toHaveBeenCalledTimes(0);
for (const call of warn.mock.calls) expect(call.length).toBe(1);
expect(warn.mock.calls.map(([message]) => message).sort()).toEqual(
[
`[TeamWatcher] teams watcher stopped, relying on polling: ${watchError(failure, teamsDir).message}`,
`[TeamWatcher] tasks watcher stopped, relying on polling: ${watchError(failure, tasksDir).message}`,
].sort()
);
expect(watch.mock.calls.length).toBe(attempts);
expect(attempts).toBeGreaterThanOrEqual(2);
// Allow a handful of in-flight attempts, but never an ancestor retry loop.
expect(attempts).toBeLessThanOrEqual(6);
expect(unhandled.length).toBe(0);
const expectedWarnings = [
`[TeamWatcher] teams watcher stopped, relying on polling: ${watchError(failure, teamsDir).message}`,
`[TeamWatcher] tasks watcher stopped, relying on polling: ${watchError(failure, tasksDir).message}`,
];
for (const pollCount of [2, 3, 4]) {
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
await delay(500);
// The same empty roots must cost the same bounded number of attempts on every tick.
expect(watch).toHaveBeenCalledTimes(attempts * pollCount);
expectWarnings(expectedWarnings);
expect(teamWatcher.getTeams()).toEqual([]);
await delay(200);
expect(watch).toHaveBeenCalledTimes(attempts * pollCount);
}
const config = createTeamConfig();
await writeTeam(teamsDir, config);
expect(teamWatcher.getTeams()).toEqual([]);
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
await waitUntil(() => teamWatcher.getTeams().length > 0, 'polling did not discover the new team');
expect(teamWatcher.getTeams()).toEqual([config]);
await delay(500);
const finalAttempts = watch.mock.calls.length;
expect(finalAttempts - attempts * 4).toBeGreaterThanOrEqual(2);
expect(finalAttempts - attempts * 4).toBeLessThanOrEqual(6);
expectWarnings(expectedWarnings);
expect(() => teamWatcher.stop()).not.toThrow();
expect(() => teamWatcher.stop()).not.toThrow();
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS * 3);
await delay(500);
expect(watch).toHaveBeenCalledTimes(finalAttempts);
expectWarnings(expectedWarnings);
} finally {
teamWatcher.stop();
}
});
it('warns again about a watcher that still fails after stop() and start() on the same instance', async () => {
const failure = FAILURES[0];
const watch = failEveryWatch(failure);
const expectedWarnings = [
`[TeamWatcher] teams watcher stopped, relying on polling: ${watchError(failure, teamsDir).message}`,
`[TeamWatcher] tasks watcher stopped, relying on polling: ${watchError(failure, tasksDir).message}`,
];
try {
teamWatcher.start();
await waitUntil(() => warn.mock.calls.length >= 2, 'timed out waiting for both watcher warnings');
await delay(200);
expectWarnings(expectedWarnings);
teamWatcher.stop();
teamWatcher.start();
await waitUntil(() => warn.mock.calls.length >= 4, 'a restarted watcher stayed silent about the same outage');
await delay(200);
expectWarnings([...expectedWarnings, ...expectedWarnings]);
} finally {
teamWatcher.stop();
watch.mockRestore();
}
});
it('keeps a healthy watcher active and discovers a new team before the poll interval', async () => {
const watch = vi.spyOn(fs, 'watch');
try {
teamWatcher.start();
await waitUntil(() => watch.mock.calls.length >= 2, 'healthy watchers did not start');
await delay(200);
const attempts = watch.mock.calls.length;
for (let tick = 0; tick < 3; tick++) {
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
await delay(200);
expect(watch).toHaveBeenCalledTimes(attempts);
expect(teamWatcher.getTeams()).toEqual([]);
expectWarnings([]);
}
const config = createTeamConfig();
await writeTeam(teamsDir, config);
// No interval is advanced: discovery must come from the real filesystem watcher.
await waitUntil(() => teamWatcher.getTeams().length > 0, 'healthy watcher did not discover the new team');
expect(teamWatcher.getTeams()).toEqual([config]);
teamWatcher.stop();
await delay(100);
expect(warn).toHaveBeenCalledTimes(0);
expect(error).toHaveBeenCalledTimes(0);
expect(unhandled.length).toBe(0);
} finally {
teamWatcher.stop();
}
});
it.each(FAILURES)('$code resumes live discovery when both watchers recover on the next poll', async (failure) => {
const watch = failEveryWatch(failure);
const created = vi.fn();
const expectedWarnings = [
`[TeamWatcher] teams watcher stopped, relying on polling: ${watchError(failure, teamsDir).message}`,
`[TeamWatcher] tasks watcher stopped, relying on polling: ${watchError(failure, tasksDir).message}`,
];
try {
teamWatcher.on('teamCreated', created);
teamWatcher.start();
await waitUntil(() => warn.mock.calls.length >= 2, 'timed out waiting for both watcher warnings');
await delay(200);
expectWarnings(expectedWarnings);
watch.mockRestore();
const recoveredWatch = vi.spyOn(fs, 'watch');
await delay(200);
expect(recoveredWatch).toHaveBeenCalledTimes(0);
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
await delay(500);
expect(recoveredWatch).toHaveBeenCalledTimes(2);
expect(recoveredWatch.mock.calls.map(([path]) => String(path)).sort()).toEqual([teamsDir, tasksDir].sort());
expect(teamWatcher.getTeams()).toEqual([]);
const config = createTeamConfig();
await writeTeam(teamsDir, config);
// No further interval tick: the recovered teams watcher must discover this team.
await waitUntil(() => created.mock.calls.length > 0, 'recovered watcher did not discover the new team');
expect(created).toHaveBeenCalledTimes(1);
expect(created.mock.calls).toEqual([[config]]);
expect(teamWatcher.getTeams()).toEqual([config]);
expectWarnings(expectedWarnings);
} finally {
teamWatcher.stop();
teamWatcher.off('teamCreated', created);
}
});
it('re-arms after one new file throws ENOENT and resumes live discovery without another poll', async () => {
const realWatch = fs.watch;
const vanishedPath = join(teamsDir, 'vanished.json');
const failure = watchError({ code: 'ENOENT', errno: -2, text: 'no such file or directory' }, vanishedPath);
let failedCalls = 0;
const watch = vi.spyOn(fs, 'watch').mockImplementation((...args) => {
if (String(args[0]) === vanishedPath && failedCalls === 0) {
failedCalls++;
throw failure;
}
return realWatch(...args);
});
const created = vi.fn();
let closeErrorMessage = '';
try {
(undefined as unknown as fs.FSWatcher).close();
} catch (error) {
closeErrorMessage = (error as TypeError).message;
}
const expectedWarnings = [`[TeamWatcher] teams watcher stopped, relying on polling: ${closeErrorMessage}`];
try {
teamWatcher.on('teamCreated', created);
teamWatcher.start();
await waitUntil(() => watch.mock.calls.length >= 2, 'healthy watchers did not start');
await delay(200);
await writeFile(vanishedPath, '{}');
await waitUntil(() => warn.mock.calls.length > 0, 'the transient failure did not stop the teams watcher');
await delay(200);
expect(failedCalls).toBe(1);
expectWarnings(expectedWarnings);
await rm(vanishedPath);
const stoppedAttempts = watch.mock.calls.length;
await delay(200);
expect(watch).toHaveBeenCalledTimes(stoppedAttempts);
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
await delay(500);
expect(watch).toHaveBeenCalledTimes(stoppedAttempts + 1);
expect(watch.mock.calls.slice(stoppedAttempts).map(([path]) => String(path))).toEqual([teamsDir]);
expect(teamWatcher.getTeams()).toEqual([]);
const config = createTeamConfig();
await writeTeam(teamsDir, config);
await waitUntil(() => created.mock.calls.length > 0, 're-armed watcher did not discover the new team');
expect(created).toHaveBeenCalledTimes(1);
expect(created.mock.calls).toEqual([[config]]);
expect(teamWatcher.getTeams()).toEqual([config]);
expect(failedCalls).toBe(1);
expectWarnings(expectedWarnings);
} finally {
teamWatcher.stop();
teamWatcher.off('teamCreated', created);
}
});
it.each(FAILURES.flatMap((failure) => ['teams', 'tasks'].map((kind) => ({ ...failure, kind }))))(
'$code re-arms only the $kind watcher and warns again only after a healthy poll',
async ({ kind, ...failure }) => {
const root = kind === 'teams' ? teamsDir : tasksDir;
const healthyRoot = kind === 'teams' ? tasksDir : teamsDir;
const realWatch = fs.watch;
let failingPath: string | undefined = root;
const watch = vi.spyOn(fs, 'watch').mockImplementation((...args) => {
if (String(args[0]) === failingPath) throw watchError(failure, failingPath);
return realWatch(...args);
});
const expectedWarnings = [
`[TeamWatcher] ${kind} watcher stopped, relying on polling: ${watchError(failure, root).message}`,
];
const attemptsAt = (path: string) =>
watch.mock.calls.filter(([watchedPath]) => String(watchedPath) === path).length;
try {
teamWatcher.start();
await waitUntil(() => warn.mock.calls.length > 0, 'timed out waiting for the failed watcher warning');
await delay(200);
expectWarnings(expectedWarnings);
expect(attemptsAt(root)).toBe(1);
expect(attemptsAt(healthyRoot)).toBe(1);
for (const expectedAttempts of [2, 3]) {
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
await delay(500);
expect(watch).toHaveBeenCalledTimes(expectedAttempts + 1);
expect(attemptsAt(root)).toBe(expectedAttempts);
expect(attemptsAt(healthyRoot)).toBe(1);
expectWarnings(expectedWarnings);
await delay(200);
expect(watch).toHaveBeenCalledTimes(expectedAttempts + 1);
}
failingPath = undefined;
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
await delay(500);
expect(watch).toHaveBeenCalledTimes(5);
expect(attemptsAt(root)).toBe(4);
expect(attemptsAt(healthyRoot)).toBe(1);
expectWarnings(expectedWarnings);
// A successful re-arm alone does not end the outage before the following poll.
failingPath = join(root, 'before-healthy-poll.json');
await writeFile(failingPath, '{}');
await waitUntil(() => attemptsAt(failingPath!) > 0, 'recreated watcher did not watch the first new file');
await delay(200);
expect(watch).toHaveBeenCalledTimes(6);
expectWarnings(expectedWarnings);
await rm(failingPath);
failingPath = undefined;
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
await delay(500);
expect(watch).toHaveBeenCalledTimes(7);
expect(attemptsAt(root)).toBe(5);
expect(attemptsAt(healthyRoot)).toBe(1);
expectWarnings(expectedWarnings);
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS);
await delay(200);
expect(watch).toHaveBeenCalledTimes(7);
expectWarnings(expectedWarnings);
failingPath = join(root, 'after-healthy-poll.json');
await writeFile(failingPath, '{}');
await waitUntil(() => attemptsAt(failingPath!) > 0, 'healthy watcher did not watch the second new file');
await delay(200);
expect(watch).toHaveBeenCalledTimes(8);
expectWarnings([
...expectedWarnings,
`[TeamWatcher] ${kind} watcher stopped, relying on polling: ${watchError(failure, failingPath).message}`,
]);
expect(teamWatcher.getTeams()).toEqual([]);
} finally {
teamWatcher.stop();
}
}
);
it.each([{ code: 'healthy', failure: undefined }, ...FAILURES.map((failure) => ({ code: failure.code, failure }))])(
'$code stop closes both watchers and the interval and is idempotent',
async ({ failure }) => {
const watch = failure ? failEveryWatch(failure) : vi.spyOn(fs, 'watch');
const expectedWarnings = failure
? [
`[TeamWatcher] teams watcher stopped, relying on polling: ${watchError(failure, teamsDir).message}`,
`[TeamWatcher] tasks watcher stopped, relying on polling: ${watchError(failure, tasksDir).message}`,
]
: [];
try {
teamWatcher.start();
await waitUntil(() => watch.mock.calls.length >= 2, 'watchers did not attempt to start');
await delay(200);
expectWarnings(expectedWarnings);
const attempts = watch.mock.calls.length;
const closed: string[] = [];
if (!failure) {
expect(watch).toHaveBeenCalledTimes(2);
watch.mock.results.forEach((result, index) => {
expect(result.type).toBe('return');
(result.value as fs.FSWatcher).on('close', () => closed.push(String(watch.mock.calls[index][0])));
});
}
expect(() => teamWatcher.stop()).not.toThrow();
expect(() => teamWatcher.stop()).not.toThrow();
await delay(200);
if (!failure) expect(closed.sort()).toEqual([teamsDir, tasksDir].sort());
expect(vi.getTimerCount()).toBe(0);
await writeTeam(teamsDir, createTeamConfig());
await writeFile(join(tasksDir, 'after-stop.json'), '{}');
vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS * 3);
await delay(500);
expect(watch).toHaveBeenCalledTimes(attempts);
expect(teamWatcher.getTeams()).toEqual([]);
expectWarnings(expectedWarnings);
} finally {
teamWatcher.stop();
}
}
);
});
describeLinuxWatch('TeamWatcher in-flight fs.watch failure', () => {
it('stays stopped when file replacement and deletion race with ENOSPC', async () => {
const teamsDir = join(tempDir, 'teams');
const tasksDir = join(tempDir, 'tasks');
const teamDir = join(teamsDir, 't');
await mkdir(teamDir, { recursive: true });
await mkdir(tasksDir);
await writeFile(join(teamDir, 'a.json'), '{}');
await writeFile(join(teamDir, 'b.json'), '{}');
const teamWatcher = new TeamWatcher(teamsDir, tasksDir);
watcher = teamWatcher;
const realWatch = fs.watch;
const listeners = new Map<string, fs.WatchListener<string | Buffer>>();
const watch = vi.spyOn(fs, 'watch').mockImplementation((...args) => {
const handle = realWatch(...args);
if (args[2]) listeners.set(String(args[0]), args[2]);
return handle;
});
let firstFailure: NodeJS.ErrnoException | undefined;
let failingCalls = 0;
try {
teamWatcher.start();
await waitUntil(
() => listeners.has(join(teamDir, 'a.json')) && listeners.has(join(teamDir, 'b.json')),
'timed out waiting for both file watches'
);
expect(warn).toHaveBeenCalledTimes(0);
expect(error).toHaveBeenCalledTimes(0);
watch.mockImplementation((...args) => {
// Bound the unfixed retry loop so it can recover and the test can finish.
if (failingCalls >= 2000) return realWatch(...args);
failingCalls++;
const failure = watchError(FAILURES[1], String(args[0]));
firstFailure ??= failure;
throw failure;
});
fs.writeFileSync(join(teamDir, 'b.json.tmp'), '{"replaced":true}');
fs.renameSync(join(teamDir, 'b.json.tmp'), join(teamDir, 'b.json'));
await waitUntil(() => warn.mock.calls.length > 0, 'timed out waiting for the teams watcher warning');
// Deliver the queued deletion callback only after the replacement failure stops the watcher.
fs.unlinkSync(join(teamDir, 'a.json'));
listeners.get(join(teamDir, 'a.json'))!('rename', 'a.json');
await delay(1000);
const attempts = watch.mock.calls.length;
const failures = failingCalls;
await delay(500);
const settledAttempts = watch.mock.calls.length;
const settledFailures = failingCalls;
const warnings = warn.mock.calls.length;
teamWatcher.stop();
await delay(500);
// Check counts before arguments so a retry storm cannot flood failure output.
expect(warn).toHaveBeenCalledTimes(1);
expect(warnings).toBe(1);
expect(error).toHaveBeenCalledTimes(0);
expect(unhandled.length).toBe(0);
expect(watch.mock.calls.length).toBe(settledAttempts);
expect(failingCalls).toBe(settledFailures);
expect(settledAttempts).toBe(attempts);
expect(settledFailures).toBe(failures);
expect(failures).toBeGreaterThanOrEqual(1);
expect(failures).toBeLessThanOrEqual(6);
expect(firstFailure?.code).toBe('ENOSPC');
expect(firstFailure?.path?.startsWith(`${teamDir}/`)).toBe(true);
expect(warn.mock.calls).toEqual([
[`[TeamWatcher] teams watcher stopped, relying on polling: ${firstFailure?.message}`],
]);
} finally {
teamWatcher.stop();
}
});
});
const PROJECT_HASH = '-home-arkon-default-claudeman';
const SESSION_UUID = '388113c8-cd01-4e80-93a8-3be66ab1519b';
const RUN_ID = 'wf_test1234-abc';
const RUN_START = 1_700_000_000_000;
function sampleRunJson() {
return {
runId: RUN_ID,
timestamp: new Date(RUN_START).toISOString(),
workflowName: 'watch-failure-test',
summary: 'Discover a run despite exhausted filesystem watches',
status: 'completed',
startTime: RUN_START,
agentCount: 0,
durationMs: 1234,
totalTokens: 0,
totalToolCalls: 0,
phases: [],
workflowProgress: [],
};
}
const { workflowProgress: _progress, ...runFields } = sampleRunJson();
const EXPECTED_RUN: WorkflowRunInfo = {
...runFields,
agents: [],
projectHash: PROJECT_HASH,
sessionUuid: SESSION_UUID,
lastActivityAt: RUN_START,
};
describeLinuxWatch('WorkflowRunWatcher fs.watch failure', () => {
let runWatcher: WorkflowRunWatcher;
let workflowsDir: string;
beforeEach(async () => {
const projectsDir = join(tempDir, 'projects');
workflowsDir = join(projectsDir, PROJECT_HASH, SESSION_UUID, 'workflows');
await mkdir(workflowsDir, { recursive: true });
await writeFile(join(workflowsDir, `${RUN_ID}.json`), JSON.stringify(sampleRunJson()));
watcher = runWatcher = new WorkflowRunWatcher(projectsDir);
// Later interval polls run only when a test explicitly advances the clock.
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
});
function firstRun(): Promise<WorkflowRunInfo> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
runWatcher.off('run_discovered', onRun);
reject(new Error('timed out waiting for run_discovered'));
}, 1500);
function onRun(info: WorkflowRunInfo) {
clearTimeout(timer);
resolve(info);
}
runWatcher.once('run_discovered', onRun);
runWatcher.start();
});
}
it.each(FAILURES)('$code discovers the run without retries, logging, or unhandled rejections', async (failure) => {
const watch = failEveryWatch(failure);
try {
const discovered = await firstRun();
await delay(500);
const attempts = watch.mock.calls.length;
await delay(500);
expect(unhandled.length).toBe(0);
expect(warn).toHaveBeenCalledTimes(0);
expect(error).toHaveBeenCalledTimes(0);
expect(watch.mock.calls.length).toBe(attempts);
expect(attempts).toBeGreaterThanOrEqual(1);
expect(attempts).toBeLessThanOrEqual(3);
expect(discovered).toEqual(EXPECTED_RUN);
expect(runWatcher.getRun(RUN_ID)).toEqual(EXPECTED_RUN);
expect(runWatcher.getAllRuns()).toEqual([EXPECTED_RUN]);
expect(runWatcher.isRunning()).toBe(true);
runWatcher.stop();
expect(runWatcher.isRunning()).toBe(false);
await delay(100);
expect(unhandled.length).toBe(0);
expect(watch.mock.calls.length).toBe(attempts);
expect(warn).toHaveBeenCalledTimes(0);
expect(error).toHaveBeenCalledTimes(0);
} finally {
runWatcher.stop();
}
});
it('retries a failed directory once per poll and resumes live discovery when fs.watch recovers', async () => {
const watch = failEveryWatch(FAILURES[0]);
const discovered = vi.fn();
try {
expect(await firstRun()).toEqual(EXPECTED_RUN);
await delay(500);
const attemptsPerPoll = watch.mock.calls.length;
expect(attemptsPerPoll).toBeGreaterThanOrEqual(1);
expect(attemptsPerPoll).toBeLessThanOrEqual(3);
await delay(500);
expect(watch).toHaveBeenCalledTimes(attemptsPerPoll);
expect(warn).toHaveBeenCalledTimes(0);
expect(error).toHaveBeenCalledTimes(0);
expect(unhandled.length).toBe(0);
for (const pollCount of [2, 3]) {
vi.advanceTimersByTime(WORKFLOW_RUN_POLL_INTERVAL_MS);
await waitUntil(
() => watch.mock.calls.length > attemptsPerPoll * (pollCount - 1),
'later poll did not retry the failed workflows directory'
);
await delay(500);
expect(watch).toHaveBeenCalledTimes(attemptsPerPoll * pollCount);
expect(warn).toHaveBeenCalledTimes(0);
expect(error).toHaveBeenCalledTimes(0);
expect(unhandled.length).toBe(0);
}
watch.mockRestore();
const recoveredWatch = vi.spyOn(fs, 'watch');
vi.advanceTimersByTime(WORKFLOW_RUN_POLL_INTERVAL_MS);
await waitUntil(() => recoveredWatch.mock.calls.length > 0, 'recovery poll did not install a watcher');
// Let watcher setup and the recovery poll settle before creating the new completion file.
await delay(500);
expect(runWatcher.getAllRuns()).toEqual([EXPECTED_RUN]);
const otherId = 'wf_recovered5678-def';
runWatcher.on('run_discovered', discovered);
await writeFile(join(workflowsDir, `${otherId}.json`), JSON.stringify({ ...sampleRunJson(), runId: otherId }));
// Do not advance the interval again: only the recovered watcher can discover this file.
await waitUntil(() => discovered.mock.calls.length > 0, 'recovered watcher did not discover the new run');
expect(discovered).toHaveBeenCalledTimes(1);
const expectedRun = { ...EXPECTED_RUN, runId: otherId };
expect(discovered).toHaveBeenCalledWith(expectedRun);
expect(runWatcher.getRun(otherId)).toEqual(expectedRun);
runWatcher.stop();
await delay(100);
expect(warn).toHaveBeenCalledTimes(0);
expect(error).toHaveBeenCalledTimes(0);
expect(unhandled.length).toBe(0);
} finally {
runWatcher.stop();
runWatcher.off('run_discovered', discovered);
}
});
it('discovers a run with a healthy watcher without logging or unhandled rejections', async () => {
try {
const discovered = await firstRun();
await delay(1000);
expect(discovered).toEqual(EXPECTED_RUN);
expect(runWatcher.getRun(RUN_ID)).toEqual(EXPECTED_RUN);
expect(runWatcher.getAllRuns()).toEqual([EXPECTED_RUN]);
runWatcher.stop();
await delay(100);
expect(warn).toHaveBeenCalledTimes(0);
expect(error).toHaveBeenCalledTimes(0);
expect(unhandled.length).toBe(0);
} finally {
runWatcher.stop();
}
});
});