mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-20 04:59:34 +01:00
fix(watchers): stop a chokidar watcher whose fs.watch fails instead of retrying forever
chokidar 3 with `persistent: false` has no guard for an `fs.watch` that throws (EMFILE once the inotify instance limit is used up, ENOSPC once the watch limit is). It emits the error, then a TypeError, then retries on the parent directory, and ends up re-adding `/` forever. TeamWatcher only logged its `'error'` events, so one failed watch became an endless stream of stack traces. WorkflowRunWatcher already closed its watcher on error, but `close()` removes every listener, so the TypeError from the same pass became an unhandled rejection, one per 10 s poll, and the server exits after five of those in a minute. `closeWatcherOnError()` closes the watcher on its first error, keeps a no-op listener on it, and turns its `add()` into a no-op, since a callback chokidar still has in flight can otherwise re-add a path and reopen it. TeamWatcher logs one line per watcher and relies on its 30 s poll; WorkflowRunWatcher still drops the watcher and re-arms it on the next poll.
This commit is contained in:
1 parent
e99b42c387
commit
10e443a309
5 files changed
+465
-8
No files matched your search
+6
-4
@@ -16,6 +16,8 @@ import { join } from 'node:path';
|
||||
import { watch as chokidarWatch, type FSWatcher as ChokidarWatcher } from 'chokidar';
|
||||
|
||||
import type { TeamConfig, TeamTask, InboxMessage } from './types.js';
|
||||
import { getErrorMessage } from './types/api.js';
|
||||
import { closeWatcherOnError } from './utils/close-watcher-on-error.js';
|
||||
import { LRUMap } from './utils/lru-map.js';
|
||||
import { TEAM_POLL_INTERVAL_MS, MAX_CACHED_TEAMS, MAX_CACHED_TASKS } from './config/team-config.js';
|
||||
|
||||
@@ -66,8 +68,8 @@ export class TeamWatcher extends EventEmitter {
|
||||
this.teamsWatcher.on('change', teamsHandler);
|
||||
this.teamsWatcher.on('unlink', teamsHandler);
|
||||
this.teamsWatcher.on('unlinkDir', teamsHandler);
|
||||
this.teamsWatcher.on('error', (err) => {
|
||||
console.warn('[TeamWatcher] chokidar teams watcher error:', err);
|
||||
closeWatcherOnError(this.teamsWatcher, (err) => {
|
||||
console.warn(`[TeamWatcher] teams watcher stopped, relying on polling: ${getErrorMessage(err)}`);
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn('[TeamWatcher] Failed to set up teams chokidar watcher, relying on polling:', err);
|
||||
@@ -84,8 +86,8 @@ export class TeamWatcher extends EventEmitter {
|
||||
|
||||
this.tasksWatcher.on('add', () => this.pollTasks().catch(() => {})); // Ignore - poll errors are non-fatal, next poll will retry
|
||||
this.tasksWatcher.on('change', () => this.pollTasks().catch(() => {})); // Ignore - poll errors are non-fatal, next poll will retry
|
||||
this.tasksWatcher.on('error', (err) => {
|
||||
console.warn('[TeamWatcher] chokidar tasks watcher error:', err);
|
||||
closeWatcherOnError(this.tasksWatcher, (err) => {
|
||||
console.warn(`[TeamWatcher] tasks watcher stopped, relying on polling: ${getErrorMessage(err)}`);
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn('[TeamWatcher] Failed to set up tasks chokidar watcher, relying on polling:', err);
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* @fileoverview Close a chokidar watcher on its first error, safely.
|
||||
*
|
||||
* chokidar 3 with `persistent: false` has no guard for an `fs.watch` that
|
||||
* throws (EMFILE once the inotify instance limit is used up, ENOSPC once the
|
||||
* watch limit is): it emits the error, then a TypeError from closing the
|
||||
* watcher it never got, then retries on the parent directory, one level per
|
||||
* pass, and re-adds `/` forever, since the parent of `/` is `/`. Only a closed
|
||||
* watcher ends that loop, so a watcher whose `'error'` handler just logs logs
|
||||
* forever.
|
||||
*
|
||||
* Closing from the handler is not enough on its own:
|
||||
* - `close()` removes every listener, and the TypeError of the same pass is
|
||||
* emitted after it. With no `'error'` listener EventEmitter throws it, and it
|
||||
* reaches the process as an unhandled rejection. So a no-op listener stays
|
||||
* on the closed watcher.
|
||||
* - callbacks chokidar already has in flight can still call `add()` (a deleted
|
||||
* file's `_remove()` re-adds the file when chokidar thinks it was the only
|
||||
* watched path), and `add()` sets `closed = false`, which starts the climb
|
||||
* again, silently now. So `add()` becomes a no-op on the closed watcher.
|
||||
*
|
||||
* @module utils/close-watcher-on-error
|
||||
*/
|
||||
|
||||
import type { FSWatcher } from 'chokidar';
|
||||
|
||||
/**
|
||||
* Close `watcher` on its first `'error'` and pass that error to `onClosed`,
|
||||
* once. The caller's polling has to cover what the watcher watched.
|
||||
*/
|
||||
export function closeWatcherOnError(watcher: FSWatcher, onClosed: (err: unknown) => void): void {
|
||||
watcher.once('error', (err: unknown) => {
|
||||
watcher.close().catch(() => {}); // best-effort, like the stop() paths
|
||||
watcher.on('error', () => {}); // see the fileoverview
|
||||
watcher.add = () => watcher; // likewise
|
||||
onClosed(err);
|
||||
});
|
||||
}
|
||||
@@ -40,6 +40,7 @@ import { join } from 'node:path';
|
||||
import { watch as chokidarWatch, type FSWatcher as ChokidarWatcher } from 'chokidar';
|
||||
|
||||
import type { WorkflowRunInfo, WorkflowRunSummary, WorkflowAgentInfo, WorkflowRunPhase } from './types/workflow-run.js';
|
||||
import { closeWatcherOnError } from './utils/close-watcher-on-error.js';
|
||||
import { LRUMap } from './utils/lru-map.js';
|
||||
import {
|
||||
WORKFLOW_RUN_POLL_INTERVAL_MS,
|
||||
@@ -865,10 +866,8 @@ export class WorkflowRunWatcher extends EventEmitter {
|
||||
// subagents/workflows/ children are wf_<id>/ DIRS — catch their add/remove too.
|
||||
watcher.on('addDir', handler);
|
||||
watcher.on('unlinkDir', handler);
|
||||
watcher.on('error', () => {
|
||||
// chokidar surfaced an error for this dir — drop the watcher; poll still covers it.
|
||||
this.removeDirWatcher(workflowsDir);
|
||||
});
|
||||
// chokidar surfaced an error for this dir — drop the watcher; poll still covers it.
|
||||
closeWatcherOnError(watcher, () => this.dirWatchers.delete(workflowsDir));
|
||||
this.dirWatchers.set(workflowsDir, watcher);
|
||||
} catch {
|
||||
// Watch setup failed — periodic poll still discovers changes.
|
||||
|
||||
Reference in new issue
Block a user