Merge #614 fix(files): prevent large folders starving File Viewer results

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/web/public/i18n.js

@@ -1139,6 +1139,7 @@
     Worktree: '工作树',
     'Session folder': '会话文件夹',
     'Back to worktree': '返回工作树',
+    'Not fully scanned (file limit reached)': '未完全扫描（已达到文件数量上限）',
     'Any status': '任意状态',
     'Any time': '任意时间',
     'Last hour': '最近一小时',

src/web/public/panels-ui.js

@@ -3794,6 +3794,12 @@ Object.assign(CodemanApp.prototype, {
           renderNode(child, depth + 1);
         }
       }
+
+      // The scan's file limit stopped short of this folder, so what is listed above is not all of
+      // it. Not a .file-tree-item: the click wiring below would try to open it as a file.
+      if (isDir && isExpanded && node.incomplete === true) {
+        html.push(`<div class="file-tree-incomplete" data-depth="${depth + 1}">${escapeHtml('Not fully scanned (file limit reached)')}</div>`);
+      }
     };
 
     for (const node of tree) {

src/web/public/styles.css

@@ -12338,6 +12338,16 @@ html:is([data-skin="paper-gray"], [data-skin="solarized-light"], [data-skin="cat
   display: none;
 }
 
+/* Under an expanded folder the scan's file limit stopped short of. Inert: no hover, no click. */
+.file-tree-incomplete {
+  padding: 0.2rem 0.5rem;
+  font-size: 0.7rem;
+  font-style: italic;
+  color: var(--text-muted);
+  white-space: nowrap;
+  cursor: default;
+}
+
 .file-tree-icon {
   width: 1rem;
   text-align: center;
@@ -12394,15 +12404,15 @@ html:is([data-skin="paper-gray"], [data-skin="solarized-light"], [data-skin="cat
 
 /* File tree indent levels */
 .file-tree-item[data-depth="0"] { padding-left: 0.5rem; }
-.file-tree-item[data-depth="1"] { padding-left: 1.25rem; }
-.file-tree-item[data-depth="2"] { padding-left: 2rem; }
-.file-tree-item[data-depth="3"] { padding-left: 2.75rem; }
-.file-tree-item[data-depth="4"] { padding-left: 3.5rem; }
-.file-tree-item[data-depth="5"] { padding-left: 4.25rem; }
-.file-tree-item[data-depth="6"] { padding-left: 5rem; }
-.file-tree-item[data-depth="7"] { padding-left: 5.75rem; }
-.file-tree-item[data-depth="8"] { padding-left: 6.5rem; }
-.file-tree-item[data-depth="9"] { padding-left: 7.25rem; }
+.file-tree-item[data-depth="1"], .file-tree-incomplete[data-depth="1"] { padding-left: 1.25rem; }
+.file-tree-item[data-depth="2"], .file-tree-incomplete[data-depth="2"] { padding-left: 2rem; }
+.file-tree-item[data-depth="3"], .file-tree-incomplete[data-depth="3"] { padding-left: 2.75rem; }
+.file-tree-item[data-depth="4"], .file-tree-incomplete[data-depth="4"] { padding-left: 3.5rem; }
+.file-tree-item[data-depth="5"], .file-tree-incomplete[data-depth="5"] { padding-left: 4.25rem; }
+.file-tree-item[data-depth="6"], .file-tree-incomplete[data-depth="6"] { padding-left: 5rem; }
+.file-tree-item[data-depth="7"], .file-tree-incomplete[data-depth="7"] { padding-left: 5.75rem; }
+.file-tree-item[data-depth="8"], .file-tree-incomplete[data-depth="8"] { padding-left: 6.5rem; }
+.file-tree-item[data-depth="9"], .file-tree-incomplete[data-depth="9"] { padding-left: 7.25rem; }
 
 /* ========== File Preview Overlay ========== */
 

src/web/routes/file-routes.ts

@@ -8,7 +8,7 @@
 
 import { FastifyInstance, type FastifyReply } from 'fastify';
 import { basename as pathBasename, dirname, extname, isAbsolute, join, relative, resolve, sep } from 'node:path';
-import { createReadStream, realpathSync } from 'node:fs';
+import { createReadStream, realpathSync, type Dirent } from 'node:fs';
 import type { Readable } from 'node:stream';
 import fs from 'node:fs/promises';
 import { createHash, randomBytes } from 'node:crypto';
@@ -782,6 +782,72 @@ export function isPathWithinRoot(root: string, candidate: string): boolean {
   return rel === '' || (!isAbsolute(rel) && rel !== '..' && !rel.startsWith(`..${sep}`));
 }
 
+/**
+ * Folder entries one Files panel listing may read, summed over the folders it opens. The budget of
+ * the walk caps what a listing RETURNS, but taking turns means opening every folder found so far,
+ * so this is what caps the reading: past it, folders not opened yet stay unread and are reported as
+ * unfinished. It also bounds what the walk holds, since a folder keeps only entries it has read. A
+ * folder is read whole, so the one that crosses this line overshoots by its own size.
+ */
+const MAX_WALK_ENTRIES_READ = 250_000;
+
+/** One folder of a Files panel walk, tree or search. */
+interface WalkCursor {
+  dirPath: string;
+  depth: number;
+  /** What the listing can use of this folder, folders first. Unset until its first turn. */
+  entries?: Dirent[];
+  index: number;
+  /** The read kept fewer entries than the folder lists, so running out of `entries` does not finish it. */
+  more?: boolean;
+}
+
+/**
+ * First turn of a folder: reads it and sets `cursor.entries` to the part the listing could use,
+ * which is the entries it lists, sorted, up to what is left of the budget. Leaves `entries` unset
+ * for a folder it could not read. Answers the number of entries read, for the walk's read ceiling.
+ */
+async function readWalkFolder(
+  cursor: WalkCursor,
+  budgetLeft: number,
+  isListed: (entry: Dirent) => boolean
+): Promise<number> {
+  // The walk's root is the session's own folder. Below it any folder can be a mount that stopped
+  // answering, and a bulk scan keeps the stall cap (#516): an unbounded readdir there would hold a
+  // threadpool worker for good. Skipped like an unreadable one: it is not the file limit's doing.
+  if (cursor.depth > 1 && (await probePathKind(cursor.dirPath)) !== 'directory') return 0;
+
+  let read: Dirent[];
+  try {
+    read = await fs.readdir(cursor.dirPath, { withFileTypes: true });
+  } catch {
+    // Can't read directory (permission denied, etc.).
+    return 0;
+  }
+  const listed = read.filter(isListed).sort((a, b) => {
+    if (a.isDirectory() && !b.isDirectory()) return -1;
+    if (!a.isDirectory() && b.isDirectory()) return 1;
+    return a.name.localeCompare(b.name);
+  });
+  cursor.more = listed.length > budgetLeft;
+  cursor.entries = cursor.more ? listed.slice(0, budgetLeft) : listed;
+  return read.length;
+}
+
+/**
+ * Where a folder's turn ends, given a `share` of the budget. Folders sort first, and each one found
+ * here waits for a later round, by which time a folder that spent the rest of its share on its own
+ * files has left nothing to read it with. So those files count as one more folder: they take an
+ * equal part of what the subfolders left and the remainder stays in the budget. A turn that found
+ * no subfolder has nothing to hold back for, and a turn always covers at least one entry.
+ */
+function walkTurnEnd(entries: Dirent[], start: number, share: number): number {
+  const end = Math.min(start + share, entries.length);
+  let firstFile = start;
+  while (firstFile < end && entries[firstFile].isDirectory()) firstFile++;
+  return firstFile + Math.floor((end - firstFile) / (firstFile - start + 1));
+}
+
 /**
  * Whether a `root=live` listing may start at a session's `liveWorktree` (#765). `findLiveWorktree()`
  * judged that path lexically when the pane moved, and a symlink can take its place, or a parent's,
@@ -1667,12 +1733,17 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even
       size?: number;
       extension?: string;
       children?: FileTreeNode[];
+      /** Set only on a folder the walk's limits stopped it short of, so `children` is not all of it. */
+      incomplete?: boolean;
     }
 
     let totalFiles = 0;
     let totalDirectories = 0;
     let truncated = false;
     const maxFiles = 5000;
+    let entriesRead = 0;
+    const isListed = (entry: Dirent): boolean =>
+      (includeHidden || !entry.name.startsWith('.')) && !(entry.isDirectory() && excludeDirs.has(entry.name));
 
     // ===== Search mode =====
     // A query turns this endpoint into a FLAT match list rather than a nested
@@ -1684,66 +1755,76 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even
       const matches: FileTreeNode[] = [];
       const maxMatches = 1000;
 
-      const searchDirectory = async (dirPath: string, currentDepth: number): Promise<void> => {
-        if (currentDepth > maxDepth || totalFiles + totalDirectories > maxFiles || matches.length >= maxMatches) {
-          truncated = true;
-          return;
-        }
+      const queue: WalkCursor[] = [{ dirPath: walkRoot, depth: 1, index: 0 }];
+      // Folders the read ceiling kept shut. What was already read still gets its turns.
+      const unread: WalkCursor[] = [];
+      while (queue.length > 0 && totalFiles + totalDirectories < maxFiles && matches.length < maxMatches) {
+        // Per-directory round-robin (breadth-first): each queued directory gets an equal share of what
+        // is left before any of them gets more, so results arrive shallow-first. Fair per directory,
+        // not per subtree: a folder of thousands of small subfolders queues thousands of directories
+        // and can still take most of the budget one level down. Reading a folder costs the same
+        // whatever its share, so this never bounds the reading: MAX_WALK_ENTRIES_READ does.
+        const directoriesInRound = queue.length;
+        const nodesPerDirectory = Math.max(
+          1,
+          Math.floor((maxFiles - totalFiles - totalDirectories) / directoriesInRound)
+        );
 
-        let entries: import('node:fs').Dirent[];
-        try {
-          entries = await fs.readdir(dirPath, { withFileTypes: true });
-        } catch {
-          // Can't read directory (permission denied, etc.)
-          return;
-        }
-        entries.sort((a, b) => {
-          if (a.isDirectory() && !b.isDirectory()) return -1;
-          if (!a.isDirectory() && b.isDirectory()) return 1;
-          return a.name.localeCompare(b.name);
-        });
-
-        for (const entry of entries) {
-          if (totalFiles + totalDirectories > maxFiles || matches.length >= maxMatches) {
+        for (let directoryIndex = 0; directoryIndex < directoriesInRound; directoryIndex++) {
+          // Spent part way through a round: the rest stay queued unread rather than sorted for nothing.
+          if (totalFiles + totalDirectories >= maxFiles || matches.length >= maxMatches) break;
+          const cursor = queue.shift();
+          if (!cursor) break;
+          if (cursor.depth > maxDepth) {
             truncated = true;
-            break;
+            continue;
           }
-          if (!includeHidden && entry.name.startsWith('.')) continue;
-          if (entry.isDirectory() && excludeDirs.has(entry.name)) continue;
-
-          const fullPath = join(dirPath, entry.name);
-          const relativePath = relative(workingDir, fullPath);
-          const listedPath = relative(walkRoot, fullPath);
-
-          if (entry.isDirectory()) {
-            totalDirectories++;
-            if (matcher(entry.name, listedPath)) {
-              matches.push({ name: entry.name, path: relativePath, type: 'directory' });
+          if (!cursor.entries) {
+            if (entriesRead >= MAX_WALK_ENTRIES_READ) {
+              unread.push(cursor);
+              continue;
             }
-            // Always recurse, even when this directory does not match.
-            await searchDirectory(fullPath, currentDepth + 1);
-          } else {
-            totalFiles++;
-            if (matcher(entry.name, listedPath)) {
-              let size: number | undefined;
-              try {
-                size = (await fs.stat(fullPath)).size;
-              } catch {
-                // Skip size if we can't stat the match.
+            entriesRead += await readWalkFolder(cursor, maxFiles - totalFiles - totalDirectories, isListed);
+            if (!cursor.entries) continue;
+          }
+
+          const endIndex = walkTurnEnd(cursor.entries, cursor.index, nodesPerDirectory);
+          while (cursor.index < endIndex && totalFiles + totalDirectories < maxFiles && matches.length < maxMatches) {
+            const entry = cursor.entries[cursor.index++];
+            const fullPath = join(cursor.dirPath, entry.name);
+            const relativePath = relative(workingDir, fullPath);
+            const listedPath = relative(walkRoot, fullPath);
+
+            if (entry.isDirectory()) {
+              totalDirectories++;
+              if (matcher(entry.name, listedPath)) {
+                matches.push({ name: entry.name, path: relativePath, type: 'directory' });
+              }
+              // Queued, not entered: it waits for a later round with every other directory found.
+              queue.push({ dirPath: fullPath, depth: cursor.depth + 1, index: 0 });
+            } else {
+              totalFiles++;
+              if (matcher(entry.name, listedPath)) {
+                let size: number | undefined;
+                try {
+                  size = (await fs.stat(fullPath)).size;
+                } catch {
+                  // Skip size if we can't stat the match.
+                }
+                matches.push({
+                  name: entry.name,
+                  path: relativePath,
+                  type: 'file',
+                  size,
+                  extension: entry.name.includes('.') ? entry.name.split('.').pop()?.toLowerCase() : undefined,
+                });
               }
-              matches.push({
-                name: entry.name,
-                path: relativePath,
-                type: 'file',
-                size,
-                extension: entry.name.includes('.') ? entry.name.split('.').pop()?.toLowerCase() : undefined,
-              });
             }
           }
+          if (cursor.index < cursor.entries.length || cursor.more) queue.push(cursor);
         }
-      };
-
-      await searchDirectory(walkRoot, 1);
+      }
+      if (queue.length > 0 || unread.length > 0) truncated = true;
 
       return {
         success: true,
@@ -1762,47 +1843,56 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even
       };
     }
 
-    const scanDirectory = async (dirPath: string, currentDepth: number): Promise<FileTreeNode[]> => {
-      if (currentDepth > maxDepth || totalFiles + totalDirectories > maxFiles) {
-        truncated = true;
-        return [];
-      }
+    interface TreeDirectoryCursor extends WalkCursor {
+      /** This directory's node. The root of the walk has none: `tree` is its list of children. */
+      node?: FileTreeNode;
+      nodes: FileTreeNode[];
+    }
 
-      try {
-        const entries = await fs.readdir(dirPath, { withFileTypes: true });
-        const nodes: FileTreeNode[] = [];
+    const tree: FileTreeNode[] = [];
+    const queue: TreeDirectoryCursor[] = [{ dirPath: walkRoot, depth: 1, nodes: tree, index: 0 }];
+    const unread: TreeDirectoryCursor[] = [];
+    while (queue.length > 0 && totalFiles + totalDirectories < maxFiles) {
+      // The same per-directory round-robin as the search walk above, with the same limits.
+      const directoriesInRound = queue.length;
+      const nodesPerDirectory = Math.max(
+        1,
+        Math.floor((maxFiles - totalFiles - totalDirectories) / directoriesInRound)
+      );
 
-        // Sort: directories first, then alphabetically
-        entries.sort((a, b) => {
-          if (a.isDirectory() && !b.isDirectory()) return -1;
-          if (!a.isDirectory() && b.isDirectory()) return 1;
-          return a.name.localeCompare(b.name);
-        });
-
-        for (const entry of entries) {
-          if (totalFiles + totalDirectories > maxFiles) {
-            truncated = true;
-            break;
+      for (let directoryIndex = 0; directoryIndex < directoriesInRound; directoryIndex++) {
+        // Before the shift, so a cursor this skips stays queued and is flagged below.
+        if (totalFiles + totalDirectories >= maxFiles) break;
+        const cursor = queue.shift();
+        if (!cursor) break;
+        if (cursor.depth > maxDepth) {
+          truncated = true;
+          continue;
+        }
+        if (!cursor.entries) {
+          if (entriesRead >= MAX_WALK_ENTRIES_READ) {
+            unread.push(cursor);
+            continue;
           }
+          entriesRead += await readWalkFolder(cursor, maxFiles - totalFiles - totalDirectories, isListed);
+          if (!cursor.entries) continue;
+        }
 
-          // Skip hidden files unless requested
-          if (!includeHidden && entry.name.startsWith('.')) continue;
-
-          // Skip excluded directories
-          if (entry.isDirectory() && excludeDirs.has(entry.name)) continue;
-
-          const fullPath = join(dirPath, entry.name);
+        const endIndex = walkTurnEnd(cursor.entries, cursor.index, nodesPerDirectory);
+        while (cursor.index < endIndex && totalFiles + totalDirectories < maxFiles) {
+          const entry = cursor.entries[cursor.index++];
+          const fullPath = join(cursor.dirPath, entry.name);
           const relativePath = fullPath.slice(workingDir.length + 1);
-
           if (entry.isDirectory()) {
             totalDirectories++;
-            const children = await scanDirectory(fullPath, currentDepth + 1);
-            nodes.push({
+            const node: FileTreeNode = {
               name: entry.name,
               path: relativePath,
               type: 'directory',
-              children,
-            });
+              children: [],
+            };
+            cursor.nodes.push(node);
+            queue.push({ dirPath: fullPath, depth: cursor.depth + 1, node, nodes: node.children!, index: 0 });
           } else {
             totalFiles++;
             const ext = entry.name.includes('.') ? entry.name.split('.').pop()?.toLowerCase() : undefined;
@@ -1811,26 +1901,20 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even
               const stat = await fs.stat(fullPath);
               size = stat.size;
             } catch {
-              // Skip if can't stat
+              // Skip if can't stat.
             }
-            nodes.push({
-              name: entry.name,
-              path: relativePath,
-              type: 'file',
-              size,
-              extension: ext,
-            });
+            cursor.nodes.push({ name: entry.name, path: relativePath, type: 'file', size, extension: ext });
           }
         }
-
-        return nodes;
-      } catch {
-        // Can't read directory (permission denied, etc.)
-        return [];
+        if (cursor.index < cursor.entries.length || cursor.more) queue.push(cursor);
       }
-    };
-
-    const tree = await scanDirectory(walkRoot, 1);
+    }
+    // Every directory still queued is unread or half read, and an empty `children` would read as an
+    // empty folder. The root has no node to flag: `truncated` says the same for it.
+    for (const cursor of [...queue, ...unread]) {
+      if (cursor.node) cursor.node.incomplete = true;
+    }
+    if (queue.length > 0 || unread.length > 0) truncated = true;
 
     return {
       success: true,