diff --git a/.changeset/quiet-folders-share.md b/.changeset/quiet-folders-share.md
index 677c2c83..59ae2dd7 100644
--- a/.changeset/quiet-folders-share.md
+++ b/.changeset/quiet-folders-share.md
@@ -2,4 +2,4 @@
'aicodeman': patch
---
-Files panel: one large folder no longer uses up the 5,000-entry listing limit before the folders after it, or the ones inside it, get a turn. The tree and its search now walk folders in a per-directory round-robin (breadth-first), so search results come back shallow-first rather than depth-first. This is fair per directory, not per subtree: a folder made of thousands of small subfolders can still use most of the limit one level down. One listing also stops opening folders after reading 250,000 entries, so folders that add up to that can leave later ones unread, and a folder below the workspace that does not answer is skipped rather than waited on. A folder a limit stopped short of no longer looks empty: `GET /api/sessions/:id/files` marks it `incomplete: true` (additive) and, when expanded, it shows a dim "Not fully scanned (file limit reached)" row.
+Files panel: one large folder no longer uses up the 5,000-entry listing limit before the folders after it, or the ones inside it, get a turn. The tree and its search now walk folders in a per-directory round-robin (breadth-first), so search results come back shallow-first rather than depth-first. This is fair per directory, not per subtree: a folder made of thousands of small subfolders can still use most of the limit one level down. One listing also stops opening folders after reading 250,000 entries, so folders that add up to that can leave later ones unread, and a folder below the workspace that does not answer is skipped rather than waited on. A folder the scan stopped short of for any of these reasons no longer looks empty: `GET /api/sessions/:id/files` marks it `incomplete: true` (additive), sets `truncated`, and, when expanded, it shows a dim "Not fully scanned" row.
diff --git a/docs/api-reference.md b/docs/api-reference.md
index f484247c..d8de8665 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -1024,11 +1024,11 @@ as one more share beside the subfolders it has just found, so one huge folder ca
listing limit before its later siblings or its own subfolders are read, and the `matches` of a `q`
search come back shallow-first. A folder made of thousands of small subfolders can still use most
of the limit one level down, and folders that add up to the read limit can leave later ones
-unread. In the tree, a directory node either limit stopped short of also carries
-`incomplete: true`: its `children` hold what was listed before the limit, which can be nothing.
-The key is absent on every other node. A folder below the start of the walk that does not answer a
-bounded probe (a mount that stopped responding) is skipped like an unreadable one: it is listed
-with empty `children` and no `incomplete`.
+unread. A folder below the start of the walk that does not answer a bounded probe (a mount that
+stopped responding) is not read either, and counts as left over the same way. In the tree, a
+directory node the walk stopped short of for either reason also carries `incomplete: true`: its
+`children` hold what was listed before it stopped, which can be nothing. The key is absent on every
+other node.
## Session model (`displayModel`)
diff --git a/docs/architecture-invariants.md b/docs/architecture-invariants.md
index 4673753b..2c29b215 100644
--- a/docs/architecture-invariants.md
+++ b/docs/architecture-invariants.md
@@ -762,9 +762,9 @@ Frontend `readmymind-ui.js`: header 🧠 marker-hidden (`btn-readmymind--hidden`
⚠️ **The budget caps what a listing returns, never what it reads.** Taking turns means opening every folder found so far, where the depth-first walk stopped reading soon after its budget ran out, and a share of 16 entries costs the same read as a share of 5,000. So `readWalkFolder()` keeps only what the listing can use (the entries it lists, sorted, no more than the budget has left, with `more` recording that it dropped some: such a cursor stays queued and is flagged), and `MAX_WALK_ENTRIES_READ` (250,000) stops opening folders. The folder that crosses that line is read whole, as any folder always was, the ones not opened yet wait in `unread` and are flagged, and what was already read keeps its turns. Without the two, 100 folders of 50,000 files cost one GET 5 million entries and a 400 MB heap died (which ends the server, not the request). The price is that folders adding up to the ceiling leave later ones unread: flagged, not hidden, but not the fairness the budget gets, and a depth-first walk over the same tree showed less.
-⚠️ **A folder below the walk's root is probed before it is read** (rule #516), because the round-robin reaches folders the depth-first walk stopped short of, each with an unbounded `readdir`. It is a bulk scan, so default options and the stall cap apply. Anything but `directory` is skipped like an unreadable folder, with no flag, since the row says "file limit". With the cap engaged (two unrelated paths stalled, by default) that is every folder below the root, until one answers. The root is not probed: a refused probe there would blank a healthy workspace.
+⚠️ **A folder below the walk's root is probed before it is read** (rule #516), because the round-robin reaches folders the depth-first walk stopped short of, each with an unbounded `readdir`. It is a bulk scan, so default options and the stall cap apply. `unknown` is never `absent`, so a folder the probe could not answer for goes on `unread` (`cursor.unanswered`), which flags it `incomplete` and sets `truncated`: a healthy mount that merely took longer than `PATH_PROBE_TIMEOUT_MS` on first touch would otherwise list as empty, and a search would miss it in silence. It stays unread until a later listing asks again. Only `absent` or `file`, a folder that left while the walk ran, drops out with no flag. With the cap engaged (two unrelated paths stalled, by default) every folder below the root is flagged, until one answers, which is the honest outcome: none of them was read. The root is not probed: a refused probe there would blank a healthy workspace. A `readdir` that fails after a good probe (permission denied) is still treated as an empty folder, as it always was.
-⚠️ A directory gets its node, with `children: []`, when it is counted and is read in a later round, so every cursor still queued after the loop, and every one in `unread`, is a folder that is unread or half read: it must carry `incomplete: true` or it reads as an empty folder, the failure that went from 0 to 635 such folders before the flag existed. `renderFileBrowserTree` draws one dim, inert row under it while it is expanded, and the row is not a `.file-tree-item`, so the click wiring leaves it alone. The budget check inside the round runs BEFORE each `shift()`, so a directory the round skips stays queued and keeps its flag. Hidden and excluded entries are dropped at the read, so they never leave a folder looking unfinished, but a folder found as the budget ended is flagged unread even if it would have turned out to be empty: only reading it could tell. The walk's root has no node, so `truncated` is its only signal, and a folder dropped at the depth cap is not flagged, since the row says "file limit" and it looks the same as it always has.
+⚠️ A directory gets its node, with `children: []`, when it is counted and is read in a later round, so every cursor still queued after the loop, and every one in `unread`, is a folder that is unread or half read: it must carry `incomplete: true` or it reads as an empty folder, the failure that went from 0 to 635 such folders before the flag existed. `renderFileBrowserTree` draws one dim, inert row under it while it is expanded, worded to fit either reason, and the row is not a `.file-tree-item`, so the click wiring leaves it alone. The budget check inside the round runs BEFORE each `shift()`, so a directory the round skips stays queued and keeps its flag. Hidden and excluded entries are dropped at the read, so they never leave a folder looking unfinished, but a folder found as the budget ended is flagged unread even if it would have turned out to be empty: only reading it could tell. The walk's root has no node, so `truncated` is its only signal, and a folder dropped at the depth cap is not flagged: it looks the same as it always has.
⚠️ **Globs are never compiled into a RegExp**: `*a*a*a…` translated to `^.*a.*a.*a…$` is a classic backtracking blowup evaluated synchronously against every walked path, so one pathological query would freeze the event loop for the whole server (the same reason `search-service.ts` is regex-free). `globMatch()` is a two-pointer wildcard walk instead, O(text · pattern) with both operands short by construction, and an overlong query (`MAX_QUERY_LENGTH`, 256) also compiles to `null` rather than running.
diff --git a/docs/wiki/Working-With-Files.md b/docs/wiki/Working-With-Files.md
index 4b99f892..3d9d02d8 100644
--- a/docs/wiki/Working-With-Files.md
+++ b/docs/wiki/Working-With-Files.md
@@ -15,9 +15,9 @@ taking turns between folders, so one huge folder cannot use up that limit before
or the folders inside it, get a turn. They also stop opening folders after reading 250,000 entries,
so a workspace with several enormous folders can still leave later ones unread. When a workspace is
bigger than either limit the status line says `(truncated)`, and an expanded folder the scan did
-not finish shows **Not fully scanned (file limit reached)** rather than looking empty. A folder that
-does not answer, such as a network mount that went away, is skipped rather than waited on, and
-looks empty.
+not finish shows **Not fully scanned** rather than looking empty. A folder that does not answer,
+such as a network mount that went away, is skipped rather than waited on and is marked the same way,
+so a search can miss what is inside it. The panel's Refresh button tries it again.
It renders what it can:
diff --git a/src/web/public/i18n.js b/src/web/public/i18n.js
index 34c7a1fb..a3730c91 100644
--- a/src/web/public/i18n.js
+++ b/src/web/public/i18n.js
@@ -1144,7 +1144,7 @@
Worktree: '工作树',
'Session folder': '会话文件夹',
'Back to worktree': '返回工作树',
- 'Not fully scanned (file limit reached)': '未完全扫描（已达到文件数量上限）',
+ 'Not fully scanned': '未完全扫描',
'Any status': '任意状态',
'Any time': '任意时间',
'Last hour': '最近一小时',
diff --git a/src/web/public/panels-ui.js b/src/web/public/panels-ui.js
index 482d4fbf..a1522d45 100644
--- a/src/web/public/panels-ui.js
+++ b/src/web/public/panels-ui.js
@@ -3795,10 +3795,10 @@ Object.assign(CodemanApp.prototype, {
}
}
- // The scan's file limit stopped short of this folder, so what is listed above is not all of
- // it. Not a .file-tree-item: the click wiring below would try to open it as a file.
+ // The scan stopped at a limit or the folder did not answer, so what is listed above is not all
+ // of it. Not a .file-tree-item: the click wiring below would try to open it as a file.
if (isDir && isExpanded && node.incomplete === true) {
- html.push(`${escapeHtml('Not fully scanned (file limit reached)')}
`);
+ html.push(`${escapeHtml('Not fully scanned')}
`);
}
};
diff --git a/src/web/routes/file-routes.ts b/src/web/routes/file-routes.ts
index 5024b94b..f04c0412 100644
--- a/src/web/routes/file-routes.ts
+++ b/src/web/routes/file-routes.ts
@@ -800,12 +800,15 @@ interface WalkCursor {
index: number;
/** The read kept fewer entries than the folder lists, so running out of `entries` does not finish it. */
more?: boolean;
+ /** Its probe answered `unknown`: it may hold anything, so it must not pass for an empty folder. */
+ unanswered?: boolean;
}
/**
* First turn of a folder: reads it and sets `cursor.entries` to the part the listing could use,
* which is the entries it lists, sorted, up to what is left of the budget. Leaves `entries` unset
- * for a folder it could not read. Answers the number of entries read, for the walk's read ceiling.
+ * for a folder it could not read, and sets `unanswered` when that was a probe that established
+ * nothing. Answers the number of entries read, for the walk's read ceiling.
*/
async function readWalkFolder(
cursor: WalkCursor,
@@ -814,8 +817,13 @@ async function readWalkFolder(
): Promise {
// The walk's root is the session's own folder. Below it any folder can be a mount that stopped
// answering, and a bulk scan keeps the stall cap (#516): an unbounded readdir there would hold a
- // threadpool worker for good. Skipped like an unreadable one: it is not the file limit's doing.
- if (cursor.depth > 1 && (await probePathKind(cursor.dirPath)) !== 'directory') return 0;
+ // threadpool worker for good. `unknown` is never `absent`, so only a folder the filesystem says
+ // is gone drops out quietly: one that did not answer goes on the walk's unfinished list.
+ if (cursor.depth > 1) {
+ const kind = await probePathKind(cursor.dirPath);
+ cursor.unanswered = kind === 'unknown';
+ if (kind !== 'directory') return 0;
+ }
let read: Dirent[];
try {
@@ -1734,7 +1742,7 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even
size?: number;
extension?: string;
children?: FileTreeNode[];
- /** Set only on a folder the walk's limits stopped it short of, so `children` is not all of it. */
+ /** Set only on a folder the walk stopped short of or could not reach, so `children` is not all of it. */
incomplete?: boolean;
}
@@ -1757,7 +1765,7 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even
const maxMatches = 1000;
const queue: WalkCursor[] = [{ dirPath: walkRoot, depth: 1, index: 0 }];
- // Folders the read ceiling kept shut. What was already read still gets its turns.
+ // Folders the read ceiling kept shut or that did not answer. What was already read still gets its turns.
const unread: WalkCursor[] = [];
while (queue.length > 0 && totalFiles + totalDirectories < maxFiles && matches.length < maxMatches) {
// Per-directory round-robin (breadth-first): each queued directory gets an equal share of what
@@ -1786,6 +1794,7 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even
continue;
}
entriesRead += await readWalkFolder(cursor, maxFiles - totalFiles - totalDirectories, isListed);
+ if (cursor.unanswered) unread.push(cursor);
if (!cursor.entries) continue;
}
@@ -1876,6 +1885,7 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even
continue;
}
entriesRead += await readWalkFolder(cursor, maxFiles - totalFiles - totalDirectories, isListed);
+ if (cursor.unanswered) unread.push(cursor);
if (!cursor.entries) continue;
}
diff --git a/test/file-browser-incomplete.test.ts b/test/file-browser-incomplete.test.ts
index c31d6fe3..abf3cd3f 100644
--- a/test/file-browser-incomplete.test.ts
+++ b/test/file-browser-incomplete.test.ts
@@ -1,8 +1,9 @@
/**
- * @fileoverview The Files panel's row for a folder the scan's file limit stopped short of.
+ * @fileoverview The Files panel's row for a folder the scan stopped short of.
*
* `GET .../files` gives a directory its node before reading it, so a folder still queued when the
- * budget ends has `children: []` or a partial list (test/routes/file-search-mode.test.ts). Without
+ * budget ends, or one that did not answer, has `children: []` or a partial list
+ * (test/routes/file-search-mode.test.ts, test/routes/file-walk-bounds.test.ts). Without
* `incomplete` that reads as an empty folder. What can go wrong here is the client: the flag lost
* on the way in, the row drawn for a collapsed or finished folder, or the click wiring treating it
* as a file.
@@ -76,7 +77,7 @@ const render = (app: Record, tree: unknown[]) => {
return [...treeEl().querySelectorAll('.file-tree-incomplete')] as HTMLElement[];
};
-describe('Files panel, a folder the file limit stopped short of', () => {
+describe('Files panel, a folder the scan stopped short of', () => {
it('keeps the flag through the envelope check, which ignores what it does not know', () => {
const app = loadPanel();
const data = app._validateFileBrowserTreeEnvelope(envelope([bulk, later]));
@@ -89,7 +90,7 @@ describe('Files panel, a folder the file limit stopped short of', () => {
const rows = render(loadPanel(['a-bulk']), [bulk, later]);
expect(rows).toHaveLength(1);
- expect(rows[0].textContent).toBe('Not fully scanned (file limit reached)');
+ expect(rows[0].textContent).toBe('Not fully scanned');
expect(rows[0].dataset.depth).toBe('1');
const items = [...treeEl().children].map((el) => (el as HTMLElement).dataset.path ?? 'ROW');
expect(items).toEqual(['a-bulk', 'a-bulk/one.txt', 'ROW', 'z-later']);
diff --git a/test/routes/file-walk-bounds.test.ts b/test/routes/file-walk-bounds.test.ts
index 82b52df8..8957f84b 100644
--- a/test/routes/file-walk-bounds.test.ts
+++ b/test/routes/file-walk-bounds.test.ts
@@ -143,9 +143,11 @@ describe('files endpoint read bounds', () => {
it('answers without it, and costs it one stuck stat however often the panel asks', async () => {
const first = await deadline(get(''));
+ // Nothing was read from it, so `children: []` alone would pass it off as an empty folder (#516:
+ // unknown is never absent).
const dead = first.tree.find((node: { name: string }) => node.name === 'z-nas');
- expect(dead).toEqual({ name: 'z-nas', path: 'z-nas', type: 'directory', children: [] });
- expect(first.truncated).toBe(false);
+ expect(dead).toEqual({ name: 'z-nas', path: 'z-nas', type: 'directory', children: [], incomplete: true });
+ expect(first.truncated).toBe(true);
expect(first.tree[0].children.map((node: { name: string }) => node.name)).toEqual(['needle.ts']);
expect(deadReads).toBe(0);
@@ -153,8 +155,24 @@ describe('files endpoint read bounds', () => {
const stats = statCalls();
const search = await deadline(get('?q=needle'));
expect(search.matches.map((match: { path: string }) => match.path)).toEqual([join('a-fine', 'needle.ts')]);
+ // A hit could be behind it, which is what the search's own `truncated` is for.
+ expect(search.truncated).toBe(true);
expect(statCalls()).toBe(stats);
expect(deadReads).toBe(0);
});
+
+ it('leaves a folder alone once the filesystem says it is gone: that is not an unfinished scan', async () => {
+ vi.mocked(fs.stat).mockImplementation((async (path: string) => {
+ if (path === deadFolder) throw Object.assign(new Error('gone'), { code: 'ENOENT' });
+ return { size: 1, isDirectory: () => true };
+ }) as never);
+
+ const data = await deadline(get(''));
+
+ const gone = data.tree.find((node: { name: string }) => node.name === 'z-nas');
+ expect(gone).toEqual({ name: 'z-nas', path: 'z-nas', type: 'directory', children: [] });
+ expect(data.truncated).toBe(false);
+ expect(deadReads).toBe(0);
+ });
});
});