From 10e443a30983412084e8ade9bca3e67a3898b7a4 Mon Sep 17 00:00:00 2001 From: JD Date: Sat, 14 Nov 2026 14:31:41 -0500 Subject: [PATCH] fix(watchers): stop a chokidar watcher whose fs.watch fails instead of retrying forever chokidar 3 with `persistent: false` has no guard for an `fs.watch` that throws (EMFILE once the inotify instance limit is used up, ENOSPC once the watch limit is). It emits the error, then a TypeError, then retries on the parent directory, and ends up re-adding `/` forever. TeamWatcher only logged its `'error'` events, so one failed watch became an endless stream of stack traces. WorkflowRunWatcher already closed its watcher on error, but `close()` removes every listener, so the TypeError from the same pass became an unhandled rejection, one per 10 s poll, and the server exits after five of those in a minute. `closeWatcherOnError()` closes the watcher on its first error, keeps a no-op listener on it, and turns its `add()` into a no-op, since a callback chokidar still has in flight can otherwise re-add a path and reopen it. TeamWatcher logs one line per watcher and relies on its 30 s poll; WorkflowRunWatcher still drops the watcher and re-arms it on the next poll. --- CLAUDE.md | 2 + src/team-watcher.ts | 10 +- src/utils/close-watcher-on-error.ts | 38 +++ src/workflow-run-watcher.ts | 7 +- test/watcher-fs-watch-failure.test.ts | 416 ++++++++++++++++++++++++++ 5 files changed, 465 insertions(+), 8 deletions(-) create mode 100644 src/utils/close-watcher-on-error.ts create mode 100644 test/watcher-fs-watch-failure.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 9472b727..9f21bc7e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -160,6 +160,8 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph **Process-tree walks are bounded** (`proc-tree.ts`, pure): `collectDescendants(pid, byParent)` is the ONE descendant traversal, fed by one cached `ps -eo pid=,ppid=` snapshot (`refreshProcSnapshot()` in tmux-manager.ts: async, in-flight-shared, and ANY error discards the result rather than caching a truncated `ps`). ⚠️ Never walk a process tree with per-node `pgrep` or unbounded recursion (the unbounded version took a machine down): the walk must terminate on cycles, cap depth (`PROC_WALK_MAX_DEPTH`) and node count (`PROC_WALK_MAX_NODES`), and never spawn anything. ⚠️ Keep it in its own module so the test exercises the shipped code, and report truncation through `onTruncated` naming both caps, never silently. → [architecture-invariants#process-tree-walks-are-bounded](docs/architecture-invariants.md#process-tree-walks-are-bounded) +**A `persistent: false` chokidar watcher stops on its first error**: ⚠️ its `'error'` handler goes through `closeWatcherOnError()` (`utils/close-watcher-on-error.ts`), never a bare log or `close()`. chokidar 3 retries a failed `fs.watch` (EMFILE once the inotify instances are used up, ENOSPC once the watches are) on the parent directory up to `/` forever, logging every pass (one server wrote 174 GiB of log that way); a bare `close()` turns the follow-up TypeError into an unhandled rejection, and a callback already in flight can reopen it through `add()`. The watcher's own poll must cover what it watched. Tests: `test/watcher-fs-watch-failure.test.ts`. + **Auto-resume on usage limit** (opt-in per session, top of the Respawn tab): when Claude halts on a subscription limit, `usage-limit-patterns.ts` (pure, unit-tested) parses the reset time and `SessionAutoOps` arms a timer for reset+2min, then sends Esc + `continue`. ⚠️ Respawn cycles are blocked while paused (`isLimitPaused` guard in `onIdleDetected`), which is what prevents `/clear` from wiping the paused conversation. Claude-mode only. → [architecture-invariants#auto-resume-on-usage-limit](docs/architecture-invariants.md#auto-resume-on-usage-limit) **Auto-compact and auto-clear** (opt-in per session, Respawn tab; `session-auto-ops.ts`, pure decision `decideAutoThreshold()`): for a Claude session the size compared with the threshold is the context Claude Code reports on its statusline (`context_window.total_input_tokens`, cache reads included, handed over by `POST /api/status-telemetry` through `Session.noteContextReport()`); the session's token counter never sees the cached context and is only the fallback for a launch without the exporter. Thresholds go up to the reported window (`contextWindowSize`, 1M while unknown). ⚠️ After a `/compact` or `/clear` the report from before the command is ignored by VALUE, never for a length of time (a 601k compact took 39 s). ⚠️ The decision is made AGAIN when the command goes out (`_decideAtSend()`): a report fires the trigger in the middle of a turn, and the wait for that turn to end can outlast a compaction by Claude Code itself, so a `/clear` sent on the old answer would wipe a conversation that is small again. A skipped command stamps nothing. ⚠️ Nothing is sent into a usage-limit pause (`_heldByLimit()`, until auto-resume's `continue` went out), or that `continue` is typed into an empty conversation. → [architecture-invariants#auto-compact-and-auto-clear-thresholds](docs/architecture-invariants.md#auto-compact-and-auto-clear-thresholds) diff --git a/src/team-watcher.ts b/src/team-watcher.ts index d54d1c69..74b821e6 100644 --- a/src/team-watcher.ts +++ b/src/team-watcher.ts @@ -16,6 +16,8 @@ import { join } from 'node:path'; import { watch as chokidarWatch, type FSWatcher as ChokidarWatcher } from 'chokidar'; import type { TeamConfig, TeamTask, InboxMessage } from './types.js'; +import { getErrorMessage } from './types/api.js'; +import { closeWatcherOnError } from './utils/close-watcher-on-error.js'; import { LRUMap } from './utils/lru-map.js'; import { TEAM_POLL_INTERVAL_MS, MAX_CACHED_TEAMS, MAX_CACHED_TASKS } from './config/team-config.js'; @@ -66,8 +68,8 @@ export class TeamWatcher extends EventEmitter { this.teamsWatcher.on('change', teamsHandler); this.teamsWatcher.on('unlink', teamsHandler); this.teamsWatcher.on('unlinkDir', teamsHandler); - this.teamsWatcher.on('error', (err) => { - console.warn('[TeamWatcher] chokidar teams watcher error:', err); + closeWatcherOnError(this.teamsWatcher, (err) => { + console.warn(`[TeamWatcher] teams watcher stopped, relying on polling: ${getErrorMessage(err)}`); }); } catch (err) { console.warn('[TeamWatcher] Failed to set up teams chokidar watcher, relying on polling:', err); @@ -84,8 +86,8 @@ export class TeamWatcher extends EventEmitter { this.tasksWatcher.on('add', () => this.pollTasks().catch(() => {})); // Ignore - poll errors are non-fatal, next poll will retry this.tasksWatcher.on('change', () => this.pollTasks().catch(() => {})); // Ignore - poll errors are non-fatal, next poll will retry - this.tasksWatcher.on('error', (err) => { - console.warn('[TeamWatcher] chokidar tasks watcher error:', err); + closeWatcherOnError(this.tasksWatcher, (err) => { + console.warn(`[TeamWatcher] tasks watcher stopped, relying on polling: ${getErrorMessage(err)}`); }); } catch (err) { console.warn('[TeamWatcher] Failed to set up tasks chokidar watcher, relying on polling:', err); diff --git a/src/utils/close-watcher-on-error.ts b/src/utils/close-watcher-on-error.ts new file mode 100644 index 00000000..7e27f9c7 --- /dev/null +++ b/src/utils/close-watcher-on-error.ts @@ -0,0 +1,38 @@ +/** + * @fileoverview Close a chokidar watcher on its first error, safely. + * + * chokidar 3 with `persistent: false` has no guard for an `fs.watch` that + * throws (EMFILE once the inotify instance limit is used up, ENOSPC once the + * watch limit is): it emits the error, then a TypeError from closing the + * watcher it never got, then retries on the parent directory, one level per + * pass, and re-adds `/` forever, since the parent of `/` is `/`. Only a closed + * watcher ends that loop, so a watcher whose `'error'` handler just logs logs + * forever. + * + * Closing from the handler is not enough on its own: + * - `close()` removes every listener, and the TypeError of the same pass is + * emitted after it. With no `'error'` listener EventEmitter throws it, and it + * reaches the process as an unhandled rejection. So a no-op listener stays + * on the closed watcher. + * - callbacks chokidar already has in flight can still call `add()` (a deleted + * file's `_remove()` re-adds the file when chokidar thinks it was the only + * watched path), and `add()` sets `closed = false`, which starts the climb + * again, silently now. So `add()` becomes a no-op on the closed watcher. + * + * @module utils/close-watcher-on-error + */ + +import type { FSWatcher } from 'chokidar'; + +/** + * Close `watcher` on its first `'error'` and pass that error to `onClosed`, + * once. The caller's polling has to cover what the watcher watched. + */ +export function closeWatcherOnError(watcher: FSWatcher, onClosed: (err: unknown) => void): void { + watcher.once('error', (err: unknown) => { + watcher.close().catch(() => {}); // best-effort, like the stop() paths + watcher.on('error', () => {}); // see the fileoverview + watcher.add = () => watcher; // likewise + onClosed(err); + }); +} diff --git a/src/workflow-run-watcher.ts b/src/workflow-run-watcher.ts index ee7c9144..675dc046 100644 --- a/src/workflow-run-watcher.ts +++ b/src/workflow-run-watcher.ts @@ -40,6 +40,7 @@ import { join } from 'node:path'; import { watch as chokidarWatch, type FSWatcher as ChokidarWatcher } from 'chokidar'; import type { WorkflowRunInfo, WorkflowRunSummary, WorkflowAgentInfo, WorkflowRunPhase } from './types/workflow-run.js'; +import { closeWatcherOnError } from './utils/close-watcher-on-error.js'; import { LRUMap } from './utils/lru-map.js'; import { WORKFLOW_RUN_POLL_INTERVAL_MS, @@ -865,10 +866,8 @@ export class WorkflowRunWatcher extends EventEmitter { // subagents/workflows/ children are wf_/ DIRS — catch their add/remove too. watcher.on('addDir', handler); watcher.on('unlinkDir', handler); - watcher.on('error', () => { - // chokidar surfaced an error for this dir — drop the watcher; poll still covers it. - this.removeDirWatcher(workflowsDir); - }); + // chokidar surfaced an error for this dir — drop the watcher; poll still covers it. + closeWatcherOnError(watcher, () => this.dirWatchers.delete(workflowsDir)); this.dirWatchers.set(workflowsDir, watcher); } catch { // Watch setup failed — periodic poll still discovers changes. diff --git a/test/watcher-fs-watch-failure.test.ts b/test/watcher-fs-watch-failure.test.ts new file mode 100644 index 00000000..bf74e847 --- /dev/null +++ b/test/watcher-fs-watch-failure.test.ts @@ -0,0 +1,416 @@ +/** + * @fileoverview Pins bounded fs.watch failure handling and polling fallback after a production chokidar retry loop flooded logs and synchronous close caused unhandled rejections. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { TeamWatcher } from '../src/team-watcher.js'; +import { WorkflowRunWatcher } from '../src/workflow-run-watcher.js'; +import { TEAM_POLL_INTERVAL_MS } from '../src/config/team-config.js'; +import { WORKFLOW_RUN_POLL_INTERVAL_MS } from '../src/config/workflow-config.js'; +import type { TeamConfig } from '../src/types.js'; +import type { WorkflowRunInfo } from '../src/types/workflow-run.js'; + +const FAILURES = [ + { code: 'EMFILE', errno: -24, text: 'too many open files' }, + { code: 'ENOSPC', errno: -28, text: 'no space left on device' }, +] as const; +type WatchFailure = (typeof FAILURES)[number]; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function waitUntil(predicate: () => boolean, message: string): Promise { + const deadline = Date.now() + 1500; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error(message); + await delay(10); + } +} + +function watchError(failure: WatchFailure, path: string): NodeJS.ErrnoException { + return Object.assign(new Error(`${failure.code}: ${failure.text}, watch '${path}'`), { + code: failure.code, + errno: failure.errno, + syscall: 'watch', + path, + }); +} + +function failEveryWatch(failure: WatchFailure) { + return vi.spyOn(fs, 'watch').mockImplementation((path) => { + throw watchError(failure, String(path)); + }); +} + +let tempDir: string; +let watcher: TeamWatcher | WorkflowRunWatcher | undefined; +let warn: ReturnType>; +let error: ReturnType>; +let unhandled: unknown[]; +let onUnhandled: (reason: unknown) => void; + +beforeEach(async () => { + watcher = undefined; + tempDir = await mkdtemp(join(tmpdir(), 'codeman-watch-failure-')); + warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + error = vi.spyOn(console, 'error').mockImplementation(() => {}); + unhandled = []; + onUnhandled = (reason) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); +}); + +afterEach(async () => { + try { + watcher?.stop(); + // Keep mocks and the rejection listener installed while asynchronous close settles. + await delay(100); + await rm(tempDir, { recursive: true, force: true }); + } finally { + process.off('unhandledRejection', onUnhandled); + vi.useRealTimers(); + vi.restoreAllMocks(); + } +}); + +function createTeamConfig(): TeamConfig { + return { + name: 'test-team', + createdAt: 1_700_000_000_000, + leadAgentId: 'team-lead@test-team', + leadSessionId: 'session-123', + members: [ + { + agentId: 'team-lead@test-team', + name: 'team-lead', + agentType: 'team-lead', + joinedAt: 1_700_000_000_000, + }, + ], + }; +} + +async function writeTeam(teamsDir: string, config: TeamConfig): Promise { + const dir = join(teamsDir, config.name); + await mkdir(dir); + await writeFile(join(dir, 'config.json'), JSON.stringify(config)); +} + +describe('TeamWatcher fs.watch failure', () => { + let teamsDir: string; + let tasksDir: string; + let teamWatcher: TeamWatcher; + + beforeEach(async () => { + teamsDir = join(tempDir, 'teams'); + tasksDir = join(tempDir, 'tasks'); + await mkdir(teamsDir); + await mkdir(tasksDir); + watcher = teamWatcher = new TeamWatcher(teamsDir, tasksDir); + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] }); + }); + + it.each(FAILURES)('$code stops both watchers with two warnings and keeps polling', async (failure) => { + const watch = failEveryWatch(failure); + try { + teamWatcher.start(); + await waitUntil(() => warn.mock.calls.length >= 2, 'timed out waiting for both watcher warnings'); + const attempts = watch.mock.calls.length; + await delay(500); + + // Check counts before arguments so the unfixed log storm cannot flood failure output. + expect(warn).toHaveBeenCalledTimes(2); + expect(error).toHaveBeenCalledTimes(0); + for (const call of warn.mock.calls) expect(call.length).toBe(1); + expect(warn.mock.calls.map(([message]) => message).sort()).toEqual( + [ + `[TeamWatcher] teams watcher stopped, relying on polling: ${watchError(failure, teamsDir).message}`, + `[TeamWatcher] tasks watcher stopped, relying on polling: ${watchError(failure, tasksDir).message}`, + ].sort() + ); + expect(watch.mock.calls.length).toBe(attempts); + expect(attempts).toBeGreaterThanOrEqual(2); + // Allow a handful of in-flight attempts, but never an ancestor retry loop. + expect(attempts).toBeLessThanOrEqual(6); + expect(unhandled.length).toBe(0); + + const config = createTeamConfig(); + await writeTeam(teamsDir, config); + expect(teamWatcher.getTeams()).toEqual([]); + vi.advanceTimersByTime(TEAM_POLL_INTERVAL_MS); + await waitUntil(() => teamWatcher.getTeams().length > 0, 'polling did not discover the new team'); + expect(teamWatcher.getTeams()).toEqual([config]); + expect(watch.mock.calls.length).toBe(attempts); + expect(warn).toHaveBeenCalledTimes(2); + + expect(() => teamWatcher.stop()).not.toThrow(); + await delay(500); + expect(watch.mock.calls.length).toBe(attempts); + expect(warn).toHaveBeenCalledTimes(2); + expect(error).toHaveBeenCalledTimes(0); + expect(unhandled.length).toBe(0); + } finally { + teamWatcher.stop(); + } + }); + + it('keeps a healthy watcher active and discovers a new team before the poll interval', async () => { + try { + teamWatcher.start(); + await delay(200); + const config = createTeamConfig(); + await writeTeam(teamsDir, config); + // No interval is advanced: discovery must come from the real filesystem watcher. + await waitUntil(() => teamWatcher.getTeams().length > 0, 'healthy watcher did not discover the new team'); + expect(teamWatcher.getTeams()).toEqual([config]); + teamWatcher.stop(); + await delay(100); + expect(warn).toHaveBeenCalledTimes(0); + expect(error).toHaveBeenCalledTimes(0); + expect(unhandled.length).toBe(0); + } finally { + teamWatcher.stop(); + } + }); +}); + +describe('TeamWatcher in-flight fs.watch failure', () => { + it('stays stopped when file replacement and deletion race with ENOSPC', async () => { + const teamsDir = join(tempDir, 'teams'); + const tasksDir = join(tempDir, 'tasks'); + const teamDir = join(teamsDir, 't'); + await mkdir(teamDir, { recursive: true }); + await mkdir(tasksDir); + await writeFile(join(teamDir, 'a.json'), '{}'); + await writeFile(join(teamDir, 'b.json'), '{}'); + const teamWatcher = new TeamWatcher(teamsDir, tasksDir); + watcher = teamWatcher; + const realWatch = fs.watch; + const listeners = new Map>(); + const watch = vi.spyOn(fs, 'watch').mockImplementation((...args) => { + const handle = realWatch(...args); + if (args[2]) listeners.set(String(args[0]), args[2]); + return handle; + }); + let firstFailure: NodeJS.ErrnoException | undefined; + let failingCalls = 0; + + try { + teamWatcher.start(); + await waitUntil( + () => listeners.has(join(teamDir, 'a.json')) && listeners.has(join(teamDir, 'b.json')), + 'timed out waiting for both file watches' + ); + expect(warn).toHaveBeenCalledTimes(0); + expect(error).toHaveBeenCalledTimes(0); + watch.mockImplementation((...args) => { + // Bound the unfixed retry loop so it can recover and the test can finish. + if (failingCalls >= 2000) return realWatch(...args); + failingCalls++; + const failure = watchError(FAILURES[1], String(args[0])); + firstFailure ??= failure; + throw failure; + }); + + fs.writeFileSync(join(teamDir, 'b.json.tmp'), '{"replaced":true}'); + fs.renameSync(join(teamDir, 'b.json.tmp'), join(teamDir, 'b.json')); + await waitUntil(() => warn.mock.calls.length > 0, 'timed out waiting for the teams watcher warning'); + // Deliver the queued deletion callback only after the replacement failure stops the watcher. + fs.unlinkSync(join(teamDir, 'a.json')); + listeners.get(join(teamDir, 'a.json'))!('rename', 'a.json'); + await delay(1000); + const attempts = watch.mock.calls.length; + const failures = failingCalls; + await delay(500); + const settledAttempts = watch.mock.calls.length; + const settledFailures = failingCalls; + const warnings = warn.mock.calls.length; + + teamWatcher.stop(); + await delay(500); + + // Check counts before arguments so a retry storm cannot flood failure output. + expect(warn).toHaveBeenCalledTimes(1); + expect(warnings).toBe(1); + expect(error).toHaveBeenCalledTimes(0); + expect(unhandled.length).toBe(0); + expect(watch.mock.calls.length).toBe(settledAttempts); + expect(failingCalls).toBe(settledFailures); + expect(settledAttempts).toBe(attempts); + expect(settledFailures).toBe(failures); + expect(failures).toBeGreaterThanOrEqual(1); + expect(failures).toBeLessThanOrEqual(6); + expect(firstFailure?.code).toBe('ENOSPC'); + expect(firstFailure?.path?.startsWith(`${teamDir}/`)).toBe(true); + expect(warn.mock.calls).toEqual([ + [`[TeamWatcher] teams watcher stopped, relying on polling: ${firstFailure?.message}`], + ]); + } finally { + teamWatcher.stop(); + } + }); +}); + +const PROJECT_HASH = '-home-arkon-default-claudeman'; +const SESSION_UUID = '388113c8-cd01-4e80-93a8-3be66ab1519b'; +const RUN_ID = 'wf_test1234-abc'; +const RUN_START = 1_700_000_000_000; + +function sampleRunJson() { + return { + runId: RUN_ID, + timestamp: new Date(RUN_START).toISOString(), + workflowName: 'watch-failure-test', + summary: 'Discover a run despite exhausted filesystem watches', + status: 'completed', + startTime: RUN_START, + agentCount: 0, + durationMs: 1234, + totalTokens: 0, + totalToolCalls: 0, + phases: [], + workflowProgress: [], + }; +} + +const { workflowProgress: _progress, ...runFields } = sampleRunJson(); +const EXPECTED_RUN: WorkflowRunInfo = { + ...runFields, + agents: [], + projectHash: PROJECT_HASH, + sessionUuid: SESSION_UUID, + lastActivityAt: RUN_START, +}; + +describe('WorkflowRunWatcher fs.watch failure', () => { + let runWatcher: WorkflowRunWatcher; + let workflowsDir: string; + + beforeEach(async () => { + const projectsDir = join(tempDir, 'projects'); + workflowsDir = join(projectsDir, PROJECT_HASH, SESSION_UUID, 'workflows'); + await mkdir(workflowsDir, { recursive: true }); + await writeFile(join(workflowsDir, `${RUN_ID}.json`), JSON.stringify(sampleRunJson())); + watcher = runWatcher = new WorkflowRunWatcher(projectsDir); + // Later interval polls run only when a test explicitly advances the clock. + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] }); + }); + + function firstRun(): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + runWatcher.off('run_discovered', onRun); + reject(new Error('timed out waiting for run_discovered')); + }, 1500); + function onRun(info: WorkflowRunInfo) { + clearTimeout(timer); + resolve(info); + } + runWatcher.once('run_discovered', onRun); + runWatcher.start(); + }); + } + + it.each(FAILURES)('$code discovers the run without retries, logging, or unhandled rejections', async (failure) => { + const watch = failEveryWatch(failure); + try { + const discovered = await firstRun(); + await delay(500); + const attempts = watch.mock.calls.length; + await delay(500); + + expect(unhandled.length).toBe(0); + expect(warn).toHaveBeenCalledTimes(0); + expect(error).toHaveBeenCalledTimes(0); + expect(watch.mock.calls.length).toBe(attempts); + expect(attempts).toBeGreaterThanOrEqual(1); + expect(attempts).toBeLessThanOrEqual(3); + expect(discovered).toEqual(EXPECTED_RUN); + expect(runWatcher.getRun(RUN_ID)).toEqual(EXPECTED_RUN); + expect(runWatcher.getAllRuns()).toEqual([EXPECTED_RUN]); + expect(runWatcher.isRunning()).toBe(true); + runWatcher.stop(); + expect(runWatcher.isRunning()).toBe(false); + await delay(100); + expect(unhandled.length).toBe(0); + expect(watch.mock.calls.length).toBe(attempts); + expect(warn).toHaveBeenCalledTimes(0); + expect(error).toHaveBeenCalledTimes(0); + } finally { + runWatcher.stop(); + } + }); + + it('retries a failed directory once per poll and resumes live discovery when fs.watch recovers', async () => { + const watch = failEveryWatch(FAILURES[0]); + const discovered = vi.fn(); + try { + expect(await firstRun()).toEqual(EXPECTED_RUN); + await delay(500); + const attemptsPerPoll = watch.mock.calls.length; + expect(attemptsPerPoll).toBeGreaterThanOrEqual(1); + expect(attemptsPerPoll).toBeLessThanOrEqual(3); + await delay(500); + expect(watch).toHaveBeenCalledTimes(attemptsPerPoll); + expect(warn).toHaveBeenCalledTimes(0); + expect(error).toHaveBeenCalledTimes(0); + expect(unhandled.length).toBe(0); + + for (const pollCount of [2, 3]) { + vi.advanceTimersByTime(WORKFLOW_RUN_POLL_INTERVAL_MS); + await waitUntil( + () => watch.mock.calls.length > attemptsPerPoll * (pollCount - 1), + 'later poll did not retry the failed workflows directory' + ); + await delay(500); + expect(watch).toHaveBeenCalledTimes(attemptsPerPoll * pollCount); + expect(warn).toHaveBeenCalledTimes(0); + expect(error).toHaveBeenCalledTimes(0); + expect(unhandled.length).toBe(0); + } + + watch.mockRestore(); + const recoveredWatch = vi.spyOn(fs, 'watch'); + vi.advanceTimersByTime(WORKFLOW_RUN_POLL_INTERVAL_MS); + await waitUntil(() => recoveredWatch.mock.calls.length > 0, 'recovery poll did not install a watcher'); + // Let watcher setup and the recovery poll settle before creating the new completion file. + await delay(500); + expect(runWatcher.getAllRuns()).toEqual([EXPECTED_RUN]); + const otherId = 'wf_recovered5678-def'; + runWatcher.on('run_discovered', discovered); + await writeFile(join(workflowsDir, `${otherId}.json`), JSON.stringify({ ...sampleRunJson(), runId: otherId })); + // Do not advance the interval again: only the recovered watcher can discover this file. + await waitUntil(() => discovered.mock.calls.length > 0, 'recovered watcher did not discover the new run'); + expect(discovered).toHaveBeenCalledTimes(1); + const expectedRun = { ...EXPECTED_RUN, runId: otherId }; + expect(discovered).toHaveBeenCalledWith(expectedRun); + expect(runWatcher.getRun(otherId)).toEqual(expectedRun); + runWatcher.stop(); + await delay(100); + expect(warn).toHaveBeenCalledTimes(0); + expect(error).toHaveBeenCalledTimes(0); + expect(unhandled.length).toBe(0); + } finally { + runWatcher.stop(); + runWatcher.off('run_discovered', discovered); + } + }); + + it('discovers a run with a healthy watcher without logging or unhandled rejections', async () => { + try { + const discovered = await firstRun(); + await delay(1000); + expect(discovered).toEqual(EXPECTED_RUN); + expect(runWatcher.getRun(RUN_ID)).toEqual(EXPECTED_RUN); + expect(runWatcher.getAllRuns()).toEqual([EXPECTED_RUN]); + runWatcher.stop(); + await delay(100); + expect(warn).toHaveBeenCalledTimes(0); + expect(error).toHaveBeenCalledTimes(0); + expect(unhandled.length).toBe(0); + } finally { + runWatcher.stop(); + } + }); +});