diff --git a/.changeset/quiet-folders-share.md b/.changeset/quiet-folders-share.md new file mode 100644 index 00000000..677c2c83 --- /dev/null +++ b/.changeset/quiet-folders-share.md @@ -0,0 +1,5 @@ +--- +'aicodeman': patch +--- + +Files panel: one large folder no longer uses up the 5,000-entry listing limit before the folders after it, or the ones inside it, get a turn. The tree and its search now walk folders in a per-directory round-robin (breadth-first), so search results come back shallow-first rather than depth-first. This is fair per directory, not per subtree: a folder made of thousands of small subfolders can still use most of the limit one level down. One listing also stops opening folders after reading 250,000 entries, so folders that add up to that can leave later ones unread, and a folder below the workspace that does not answer is skipped rather than waited on. A folder a limit stopped short of no longer looks empty: `GET /api/sessions/:id/files` marks it `incomplete: true` (additive) and, when expanded, it shows a dim "Not fully scanned (file limit reached)" row. diff --git a/CLAUDE.md b/CLAUDE.md index 6b431c0e..696cb319 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -236,7 +236,7 @@ The terminal provider scans `terminalLinkLine()`, which extends `terminalPathLin **File Viewer edit mode** (issue #212): the file-preview overlay edits workspace text files in place: `GET .../file-content?edit=1` + `PUT /api/sessions/:id/file-content`, policy in `src/config/file-editing.ts`. This is a **third file surface and the only one that writes file contents** (the path picker's New folder creates folders only): read-path confinement (realpath + workspace + ownership) plus sensitive/blocked/`.git` denies and an extension **allowlist**; writes are `wx`-temp + rename (no `O_CREAT` anywhere = edit-in-place is structural); optimistic concurrency via sha256 `baseHash` → 409. ⚠️ `edit=1` never truncates and the client must never save a plain-preview buffer (the 500-line truncation would silently delete the rest). ⚠️ CRLF/UTF-8 guards: EOL re-applied server-side, non-UTF-8 refused via round-trip compare. → [architecture-invariants#file-viewer-edit-mode](docs/architecture-invariants.md#file-viewer-edit-mode), `docs/file-viewer-edit-plan.md` -**Files panel search** (COD-236, the `q` param on `GET /api/sessions/:id/files`): `compileFileQuery()` (`utils/file-query.ts`, pure) compiles the query into a predicate the server-side walk prunes with; a query returns a FLAT match list and the walk recurses past non-matching directories. An empty, whitespace-only or overlong (`MAX_QUERY_LENGTH`, 256) query compiles to `null`, keeping the default tree response byte-identical. ⚠️ **Never compile a glob into a RegExp** (`*a*a*a…` backtracks and freezes the event loop for the whole server): `globMatch()` is a two-pointer wildcard walk. → [architecture-invariants#files-panel-search](docs/architecture-invariants.md#files-panel-search) +**Files panel search** (COD-236, the `q` param on `GET /api/sessions/:id/files`): `compileFileQuery()` (`utils/file-query.ts`, pure) compiles the query into a predicate the server-side walk prunes with; a query returns a FLAT match list and the walk recurses past non-matching directories. An empty, whitespace-only or overlong (`MAX_QUERY_LENGTH`, 256) query compiles to `null`, keeping the default tree response byte-identical. ⚠️ **Never compile a glob into a RegExp** (`*a*a*a…` backtracks and freezes the event loop for the whole server): `globMatch()` is a two-pointer wildcard walk. Tree and search share one walk shape: a per-directory round-robin (breadth-first, so fair per directory, not per subtree) over one 5,000-node budget, where a folder's own files count as one share beside the subfolders it just found (`walkTurnEnd()`). ⚠️ That budget caps what is LISTED, never what is read: a read keeps only what the listing can use (`readWalkFolder()`) and `MAX_WALK_ENTRIES_READ` (250,000) stops opening folders, or one GET reads the whole workspace into memory. ⚠️ A folder below the walk's root is probed (#516) before its `readdir`, and anything but `directory` is skipped unflagged. ⚠️ A directory has its node before it is read, so every cursor still queued or unread when the walk ends must get `incomplete: true` or it reads as an empty folder. → [architecture-invariants#files-panel-search](docs/architecture-invariants.md#files-panel-search) **Files panel in a worktree** (#765, `src/live-worktree.ts`, `root=live` on `GET /api/sessions/:id/files`): Claude's EnterWorktree moves the process, and `paneCwd` follows it (tmux reads the process cwd, measured on 2.1.318, back out on ExitWorktree too), while `workingDir` stays the session's identity. `refreshLiveWorktree()` (server.ts) turns a changed `paneCwd` into `SessionState.liveWorktree` by reading the `.git` FILE (never spawning git), walking up and stopping at `workingDir`, a repository or submodule of its own, or anything unreadable. ⚠️ The security argument has two halves. The lookup only ever answers a folder strictly below `workingDir`, but that is a LEXICAL judgement made when the pane moves, and a symlink can replace the folder or a parent later without moving the pane, so `root=live` re-checks on every request (`canWalkLiveWorktree()`, file-routes.ts: walk the worktree only when it resolves strictly inside the resolved `workingDir`, else walk `workingDir`). The walk starts there, returned `path`s stay relative to `workingDir`, and the routes keep confining to `workingDir`, so never make `liveWorktree` a confinement root (a worktree outside the folder stays unfollowed). ⚠️ Only that check uses resolved paths: the walk keeps the session's spelling, or a workspace reached through a symlink stops producing paths relative to `workingDir`. ⚠️ A plain subfolder is deliberately no answer, or a `cd` in a shell would re-root the panel. ⚠️ A listing reports `worktree` whichever folder it walked, even when the worktree has gone or the check refused it: `_followFileBrowserWorktree()` compares that with the session's state, and an absent report would reload forever. Tests: `test/live-worktree.test.ts`, `test/routes/file-routes-live-worktree.test.ts`, `test/file-browser-worktree.test.ts`. → [architecture-invariants#files-panel-in-a-worktree](docs/architecture-invariants.md#files-panel-in-a-worktree) diff --git a/docs/api-reference.md b/docs/api-reference.md index 023ce288..beba55dc 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1014,6 +1014,21 @@ while a `q` containing `/` is matched against paths relative to the folder walke also carries `worktree`, the same `{ name, path }`, whenever the session has a `liveWorktree`, whichever folder was walked and with or without `root=live`. +The walk lists at most 5,000 files and folders and opens no more folders once it has read 250,000 +directory entries (the folder that crosses that line is read whole). It sets `truncated` when it +stops with anything left, which includes a folder it never opened, even if that folder would have +turned out to be empty. It is a per-directory round-robin (breadth-first): each folder found so far +gets an equal share of what is left before any of them gets more, and a folder's own files count +as one more share beside the subfolders it has just found, so one huge folder cannot use up the +listing limit before its later siblings or its own subfolders are read, and the `matches` of a `q` +search come back shallow-first. A folder made of thousands of small subfolders can still use most +of the limit one level down, and folders that add up to the read limit can leave later ones +unread. In the tree, a directory node either limit stopped short of also carries +`incomplete: true`: its `children` hold what was listed before the limit, which can be nothing. +The key is absent on every other node. A folder below the start of the walk that does not answer a +bounded probe (a mount that stopped responding) is skipped like an unreadable one: it is listed +with empty `children` and no `incomplete`. + ## Session model (`displayModel`) Session state (`GET /api/v1/sessions`, the `session:updated` event) carries the model a diff --git a/docs/architecture-invariants.md b/docs/architecture-invariants.md index b3a22c9c..c004fd0a 100644 --- a/docs/architecture-invariants.md +++ b/docs/architecture-invariants.md @@ -740,6 +740,14 @@ Frontend `readmymind-ui.js`: header 🧠 marker-hidden (`btn-readmymind--hidden` ⚠️ **A query turns that endpoint into a FLAT match list rather than a nested tree**, and the walk deliberately recurses past non-matching directories, since the whole point of searching is to reach a file whose ancestors do not match. An empty or whitespace-only query compiles to `null`, which is what keeps the default tree response byte-identical when no search is requested. +⚠️ **Both walks are a per-directory round-robin (breadth-first) over ONE 5,000-node budget.** They used to recurse depth-first through name-sorted folders, so an early folder could spend the whole budget: 4 of 57 top-level folders were shown in one real workspace. Each round divides what is left equally among the directories queued so far, and a folder's own files count as one more share beside the subfolders it has just found (`walkTurnEnd()`). Those subfolders are read in a later round, so a folder that spent all of its share on its own files left them nothing, alone in its round or beside a sibling, and search hits inside them were lost. A turn that found no subfolder holds nothing back. That is fair per directory, not per subtree: a folder of thousands of one-file subfolders still crowds out its neighbours one level down (`a/b/c/d/needle.ts` beside such a `z/` is found depth-first and missed here), and that trade is accepted, since a file browser wants the shallow levels first. Search results therefore arrive shallow-first. + +⚠️ **The budget caps what a listing returns, never what it reads.** Taking turns means opening every folder found so far, where the depth-first walk stopped reading soon after its budget ran out, and a share of 16 entries costs the same read as a share of 5,000. So `readWalkFolder()` keeps only what the listing can use (the entries it lists, sorted, no more than the budget has left, with `more` recording that it dropped some: such a cursor stays queued and is flagged), and `MAX_WALK_ENTRIES_READ` (250,000) stops opening folders. The folder that crosses that line is read whole, as any folder always was, the ones not opened yet wait in `unread` and are flagged, and what was already read keeps its turns. Without the two, 100 folders of 50,000 files cost one GET 5 million entries and a 400 MB heap died (which ends the server, not the request). The price is that folders adding up to the ceiling leave later ones unread: flagged, not hidden, but not the fairness the budget gets, and a depth-first walk over the same tree showed less. + +⚠️ **A folder below the walk's root is probed before it is read** (rule #516), because the round-robin reaches folders the depth-first walk stopped short of, each with an unbounded `readdir`. It is a bulk scan, so default options and the stall cap apply. Anything but `directory` is skipped like an unreadable folder, with no flag, since the row says "file limit". With the cap engaged (two unrelated paths stalled, by default) that is every folder below the root, until one answers. The root is not probed: a refused probe there would blank a healthy workspace. + +⚠️ A directory gets its node, with `children: []`, when it is counted and is read in a later round, so every cursor still queued after the loop, and every one in `unread`, is a folder that is unread or half read: it must carry `incomplete: true` or it reads as an empty folder, the failure that went from 0 to 635 such folders before the flag existed. `renderFileBrowserTree` draws one dim, inert row under it while it is expanded, and the row is not a `.file-tree-item`, so the click wiring leaves it alone. The budget check inside the round runs BEFORE each `shift()`, so a directory the round skips stays queued and keeps its flag. Hidden and excluded entries are dropped at the read, so they never leave a folder looking unfinished, but a folder found as the budget ended is flagged unread even if it would have turned out to be empty: only reading it could tell. The walk's root has no node, so `truncated` is its only signal, and a folder dropped at the depth cap is not flagged, since the row says "file limit" and it looks the same as it always has. + ⚠️ **Globs are never compiled into a RegExp**: `*a*a*a…` translated to `^.*a.*a.*a…$` is a classic backtracking blowup evaluated synchronously against every walked path, so one pathological query would freeze the event loop for the whole server (the same reason `search-service.ts` is regex-free). `globMatch()` is a two-pointer wildcard walk instead, O(text · pattern) with both operands short by construction, and an overlong query (`MAX_QUERY_LENGTH`, 256) also compiles to `null` rather than running. Matching semantics: a query containing `/` matches the path relative to the folder being walked (`workingDir`, or the worktree of a `root=live` listing; the returned `path`s stay relative to `workingDir` either way), otherwise the bare entry name; globs match anchored and case-insensitively (`*` spans any run, slashes included, `?` exactly one character), everything else is a plain case-insensitive substring. diff --git a/docs/wiki/Working-With-Files.md b/docs/wiki/Working-With-Files.md index eebc3370..13598f1d 100644 --- a/docs/wiki/Working-With-Files.md +++ b/docs/wiki/Working-With-Files.md @@ -10,6 +10,15 @@ A panel that browses the active session's working directory, or the git worktree moved into ([Git Worktrees](Git-Worktrees#when-an-agent-moves-into-a-worktree)). Its header button is on by default; if it is missing, re-enable it in **App Settings → Header & Panels**. +The panel and its search list up to 5,000 files and folders per load, shallow levels first and +taking turns between folders, so one huge folder cannot use up that limit before the ones after it, +or the folders inside it, get a turn. They also stop opening folders after reading 250,000 entries, +so a workspace with several enormous folders can still leave later ones unread. When a workspace is +bigger than either limit the status line says `(truncated)`, and an expanded folder the scan did +not finish shows **Not fully scanned (file limit reached)** rather than looking empty. A folder that +does not answer, such as a network mount that went away, is skipped rather than waited on, and +looks empty. + It renders what it can: | Kind | Behaviour | diff --git a/src/web/public/i18n.js b/src/web/public/i18n.js index 6310cf86..c203dd48 100644 --- a/src/web/public/i18n.js +++ b/src/web/public/i18n.js @@ -1139,6 +1139,7 @@ Worktree: '工作树', 'Session folder': '会话文件夹', 'Back to worktree': '返回工作树', + 'Not fully scanned (file limit reached)': '未完全扫描（已达到文件数量上限）', 'Any status': '任意状态', 'Any time': '任意时间', 'Last hour': '最近一小时', diff --git a/src/web/public/panels-ui.js b/src/web/public/panels-ui.js index 79158528..482d4fbf 100644 --- a/src/web/public/panels-ui.js +++ b/src/web/public/panels-ui.js @@ -3794,6 +3794,12 @@ Object.assign(CodemanApp.prototype, { renderNode(child, depth + 1); } } + + // The scan's file limit stopped short of this folder, so what is listed above is not all of + // it. Not a .file-tree-item: the click wiring below would try to open it as a file. + if (isDir && isExpanded && node.incomplete === true) { + html.push(`
${escapeHtml('Not fully scanned (file limit reached)')}
`); + } }; for (const node of tree) { diff --git a/src/web/public/styles.css b/src/web/public/styles.css index 932984a5..176d6c78 100644 --- a/src/web/public/styles.css +++ b/src/web/public/styles.css @@ -12338,6 +12338,16 @@ html:is([data-skin="paper-gray"], [data-skin="solarized-light"], [data-skin="cat display: none; } +/* Under an expanded folder the scan's file limit stopped short of. Inert: no hover, no click. */ +.file-tree-incomplete { + padding: 0.2rem 0.5rem; + font-size: 0.7rem; + font-style: italic; + color: var(--text-muted); + white-space: nowrap; + cursor: default; +} + .file-tree-icon { width: 1rem; text-align: center; @@ -12394,15 +12404,15 @@ html:is([data-skin="paper-gray"], [data-skin="solarized-light"], [data-skin="cat /* File tree indent levels */ .file-tree-item[data-depth="0"] { padding-left: 0.5rem; } -.file-tree-item[data-depth="1"] { padding-left: 1.25rem; } -.file-tree-item[data-depth="2"] { padding-left: 2rem; } -.file-tree-item[data-depth="3"] { padding-left: 2.75rem; } -.file-tree-item[data-depth="4"] { padding-left: 3.5rem; } -.file-tree-item[data-depth="5"] { padding-left: 4.25rem; } -.file-tree-item[data-depth="6"] { padding-left: 5rem; } -.file-tree-item[data-depth="7"] { padding-left: 5.75rem; } -.file-tree-item[data-depth="8"] { padding-left: 6.5rem; } -.file-tree-item[data-depth="9"] { padding-left: 7.25rem; } +.file-tree-item[data-depth="1"], .file-tree-incomplete[data-depth="1"] { padding-left: 1.25rem; } +.file-tree-item[data-depth="2"], .file-tree-incomplete[data-depth="2"] { padding-left: 2rem; } +.file-tree-item[data-depth="3"], .file-tree-incomplete[data-depth="3"] { padding-left: 2.75rem; } +.file-tree-item[data-depth="4"], .file-tree-incomplete[data-depth="4"] { padding-left: 3.5rem; } +.file-tree-item[data-depth="5"], .file-tree-incomplete[data-depth="5"] { padding-left: 4.25rem; } +.file-tree-item[data-depth="6"], .file-tree-incomplete[data-depth="6"] { padding-left: 5rem; } +.file-tree-item[data-depth="7"], .file-tree-incomplete[data-depth="7"] { padding-left: 5.75rem; } +.file-tree-item[data-depth="8"], .file-tree-incomplete[data-depth="8"] { padding-left: 6.5rem; } +.file-tree-item[data-depth="9"], .file-tree-incomplete[data-depth="9"] { padding-left: 7.25rem; } /* ========== File Preview Overlay ========== */ diff --git a/src/web/routes/file-routes.ts b/src/web/routes/file-routes.ts index 0b018eda..bcc420d6 100644 --- a/src/web/routes/file-routes.ts +++ b/src/web/routes/file-routes.ts @@ -8,7 +8,7 @@ import { FastifyInstance, type FastifyReply } from 'fastify'; import { basename as pathBasename, dirname, extname, isAbsolute, join, relative, resolve, sep } from 'node:path'; -import { createReadStream, realpathSync } from 'node:fs'; +import { createReadStream, realpathSync, type Dirent } from 'node:fs'; import type { Readable } from 'node:stream'; import fs from 'node:fs/promises'; import { createHash, randomBytes } from 'node:crypto'; @@ -782,6 +782,72 @@ export function isPathWithinRoot(root: string, candidate: string): boolean { return rel === '' || (!isAbsolute(rel) && rel !== '..' && !rel.startsWith(`..${sep}`)); } +/** + * Folder entries one Files panel listing may read, summed over the folders it opens. The budget of + * the walk caps what a listing RETURNS, but taking turns means opening every folder found so far, + * so this is what caps the reading: past it, folders not opened yet stay unread and are reported as + * unfinished. It also bounds what the walk holds, since a folder keeps only entries it has read. A + * folder is read whole, so the one that crosses this line overshoots by its own size. + */ +const MAX_WALK_ENTRIES_READ = 250_000; + +/** One folder of a Files panel walk, tree or search. */ +interface WalkCursor { + dirPath: string; + depth: number; + /** What the listing can use of this folder, folders first. Unset until its first turn. */ + entries?: Dirent[]; + index: number; + /** The read kept fewer entries than the folder lists, so running out of `entries` does not finish it. */ + more?: boolean; +} + +/** + * First turn of a folder: reads it and sets `cursor.entries` to the part the listing could use, + * which is the entries it lists, sorted, up to what is left of the budget. Leaves `entries` unset + * for a folder it could not read. Answers the number of entries read, for the walk's read ceiling. + */ +async function readWalkFolder( + cursor: WalkCursor, + budgetLeft: number, + isListed: (entry: Dirent) => boolean +): Promise { + // The walk's root is the session's own folder. Below it any folder can be a mount that stopped + // answering, and a bulk scan keeps the stall cap (#516): an unbounded readdir there would hold a + // threadpool worker for good. Skipped like an unreadable one: it is not the file limit's doing. + if (cursor.depth > 1 && (await probePathKind(cursor.dirPath)) !== 'directory') return 0; + + let read: Dirent[]; + try { + read = await fs.readdir(cursor.dirPath, { withFileTypes: true }); + } catch { + // Can't read directory (permission denied, etc.). + return 0; + } + const listed = read.filter(isListed).sort((a, b) => { + if (a.isDirectory() && !b.isDirectory()) return -1; + if (!a.isDirectory() && b.isDirectory()) return 1; + return a.name.localeCompare(b.name); + }); + cursor.more = listed.length > budgetLeft; + cursor.entries = cursor.more ? listed.slice(0, budgetLeft) : listed; + return read.length; +} + +/** + * Where a folder's turn ends, given a `share` of the budget. Folders sort first, and each one found + * here waits for a later round, by which time a folder that spent the rest of its share on its own + * files has left nothing to read it with. So those files count as one more folder: they take an + * equal part of what the subfolders left and the remainder stays in the budget. A turn that found + * no subfolder has nothing to hold back for, and a turn always covers at least one entry. + */ +function walkTurnEnd(entries: Dirent[], start: number, share: number): number { + const end = Math.min(start + share, entries.length); + let firstFile = start; + while (firstFile < end && entries[firstFile].isDirectory()) firstFile++; + return firstFile + Math.floor((end - firstFile) / (firstFile - start + 1)); +} + /** * Whether a `root=live` listing may start at a session's `liveWorktree` (#765). `findLiveWorktree()` * judged that path lexically when the pane moved, and a symlink can take its place, or a parent's, @@ -1667,12 +1733,17 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even size?: number; extension?: string; children?: FileTreeNode[]; + /** Set only on a folder the walk's limits stopped it short of, so `children` is not all of it. */ + incomplete?: boolean; } let totalFiles = 0; let totalDirectories = 0; let truncated = false; const maxFiles = 5000; + let entriesRead = 0; + const isListed = (entry: Dirent): boolean => + (includeHidden || !entry.name.startsWith('.')) && !(entry.isDirectory() && excludeDirs.has(entry.name)); // ===== Search mode ===== // A query turns this endpoint into a FLAT match list rather than a nested @@ -1684,66 +1755,76 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even const matches: FileTreeNode[] = []; const maxMatches = 1000; - const searchDirectory = async (dirPath: string, currentDepth: number): Promise => { - if (currentDepth > maxDepth || totalFiles + totalDirectories > maxFiles || matches.length >= maxMatches) { - truncated = true; - return; - } + const queue: WalkCursor[] = [{ dirPath: walkRoot, depth: 1, index: 0 }]; + // Folders the read ceiling kept shut. What was already read still gets its turns. + const unread: WalkCursor[] = []; + while (queue.length > 0 && totalFiles + totalDirectories < maxFiles && matches.length < maxMatches) { + // Per-directory round-robin (breadth-first): each queued directory gets an equal share of what + // is left before any of them gets more, so results arrive shallow-first. Fair per directory, + // not per subtree: a folder of thousands of small subfolders queues thousands of directories + // and can still take most of the budget one level down. Reading a folder costs the same + // whatever its share, so this never bounds the reading: MAX_WALK_ENTRIES_READ does. + const directoriesInRound = queue.length; + const nodesPerDirectory = Math.max( + 1, + Math.floor((maxFiles - totalFiles - totalDirectories) / directoriesInRound) + ); - let entries: import('node:fs').Dirent[]; - try { - entries = await fs.readdir(dirPath, { withFileTypes: true }); - } catch { - // Can't read directory (permission denied, etc.) - return; - } - entries.sort((a, b) => { - if (a.isDirectory() && !b.isDirectory()) return -1; - if (!a.isDirectory() && b.isDirectory()) return 1; - return a.name.localeCompare(b.name); - }); - - for (const entry of entries) { - if (totalFiles + totalDirectories > maxFiles || matches.length >= maxMatches) { + for (let directoryIndex = 0; directoryIndex < directoriesInRound; directoryIndex++) { + // Spent part way through a round: the rest stay queued unread rather than sorted for nothing. + if (totalFiles + totalDirectories >= maxFiles || matches.length >= maxMatches) break; + const cursor = queue.shift(); + if (!cursor) break; + if (cursor.depth > maxDepth) { truncated = true; - break; + continue; } - if (!includeHidden && entry.name.startsWith('.')) continue; - if (entry.isDirectory() && excludeDirs.has(entry.name)) continue; - - const fullPath = join(dirPath, entry.name); - const relativePath = relative(workingDir, fullPath); - const listedPath = relative(walkRoot, fullPath); - - if (entry.isDirectory()) { - totalDirectories++; - if (matcher(entry.name, listedPath)) { - matches.push({ name: entry.name, path: relativePath, type: 'directory' }); + if (!cursor.entries) { + if (entriesRead >= MAX_WALK_ENTRIES_READ) { + unread.push(cursor); + continue; } - // Always recurse, even when this directory does not match. - await searchDirectory(fullPath, currentDepth + 1); - } else { - totalFiles++; - if (matcher(entry.name, listedPath)) { - let size: number | undefined; - try { - size = (await fs.stat(fullPath)).size; - } catch { - // Skip size if we can't stat the match. + entriesRead += await readWalkFolder(cursor, maxFiles - totalFiles - totalDirectories, isListed); + if (!cursor.entries) continue; + } + + const endIndex = walkTurnEnd(cursor.entries, cursor.index, nodesPerDirectory); + while (cursor.index < endIndex && totalFiles + totalDirectories < maxFiles && matches.length < maxMatches) { + const entry = cursor.entries[cursor.index++]; + const fullPath = join(cursor.dirPath, entry.name); + const relativePath = relative(workingDir, fullPath); + const listedPath = relative(walkRoot, fullPath); + + if (entry.isDirectory()) { + totalDirectories++; + if (matcher(entry.name, listedPath)) { + matches.push({ name: entry.name, path: relativePath, type: 'directory' }); + } + // Queued, not entered: it waits for a later round with every other directory found. + queue.push({ dirPath: fullPath, depth: cursor.depth + 1, index: 0 }); + } else { + totalFiles++; + if (matcher(entry.name, listedPath)) { + let size: number | undefined; + try { + size = (await fs.stat(fullPath)).size; + } catch { + // Skip size if we can't stat the match. + } + matches.push({ + name: entry.name, + path: relativePath, + type: 'file', + size, + extension: entry.name.includes('.') ? entry.name.split('.').pop()?.toLowerCase() : undefined, + }); } - matches.push({ - name: entry.name, - path: relativePath, - type: 'file', - size, - extension: entry.name.includes('.') ? entry.name.split('.').pop()?.toLowerCase() : undefined, - }); } } + if (cursor.index < cursor.entries.length || cursor.more) queue.push(cursor); } - }; - - await searchDirectory(walkRoot, 1); + } + if (queue.length > 0 || unread.length > 0) truncated = true; return { success: true, @@ -1762,47 +1843,56 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even }; } - const scanDirectory = async (dirPath: string, currentDepth: number): Promise => { - if (currentDepth > maxDepth || totalFiles + totalDirectories > maxFiles) { - truncated = true; - return []; - } + interface TreeDirectoryCursor extends WalkCursor { + /** This directory's node. The root of the walk has none: `tree` is its list of children. */ + node?: FileTreeNode; + nodes: FileTreeNode[]; + } - try { - const entries = await fs.readdir(dirPath, { withFileTypes: true }); - const nodes: FileTreeNode[] = []; + const tree: FileTreeNode[] = []; + const queue: TreeDirectoryCursor[] = [{ dirPath: walkRoot, depth: 1, nodes: tree, index: 0 }]; + const unread: TreeDirectoryCursor[] = []; + while (queue.length > 0 && totalFiles + totalDirectories < maxFiles) { + // The same per-directory round-robin as the search walk above, with the same limits. + const directoriesInRound = queue.length; + const nodesPerDirectory = Math.max( + 1, + Math.floor((maxFiles - totalFiles - totalDirectories) / directoriesInRound) + ); - // Sort: directories first, then alphabetically - entries.sort((a, b) => { - if (a.isDirectory() && !b.isDirectory()) return -1; - if (!a.isDirectory() && b.isDirectory()) return 1; - return a.name.localeCompare(b.name); - }); - - for (const entry of entries) { - if (totalFiles + totalDirectories > maxFiles) { - truncated = true; - break; + for (let directoryIndex = 0; directoryIndex < directoriesInRound; directoryIndex++) { + // Before the shift, so a cursor this skips stays queued and is flagged below. + if (totalFiles + totalDirectories >= maxFiles) break; + const cursor = queue.shift(); + if (!cursor) break; + if (cursor.depth > maxDepth) { + truncated = true; + continue; + } + if (!cursor.entries) { + if (entriesRead >= MAX_WALK_ENTRIES_READ) { + unread.push(cursor); + continue; } + entriesRead += await readWalkFolder(cursor, maxFiles - totalFiles - totalDirectories, isListed); + if (!cursor.entries) continue; + } - // Skip hidden files unless requested - if (!includeHidden && entry.name.startsWith('.')) continue; - - // Skip excluded directories - if (entry.isDirectory() && excludeDirs.has(entry.name)) continue; - - const fullPath = join(dirPath, entry.name); + const endIndex = walkTurnEnd(cursor.entries, cursor.index, nodesPerDirectory); + while (cursor.index < endIndex && totalFiles + totalDirectories < maxFiles) { + const entry = cursor.entries[cursor.index++]; + const fullPath = join(cursor.dirPath, entry.name); const relativePath = fullPath.slice(workingDir.length + 1); - if (entry.isDirectory()) { totalDirectories++; - const children = await scanDirectory(fullPath, currentDepth + 1); - nodes.push({ + const node: FileTreeNode = { name: entry.name, path: relativePath, type: 'directory', - children, - }); + children: [], + }; + cursor.nodes.push(node); + queue.push({ dirPath: fullPath, depth: cursor.depth + 1, node, nodes: node.children!, index: 0 }); } else { totalFiles++; const ext = entry.name.includes('.') ? entry.name.split('.').pop()?.toLowerCase() : undefined; @@ -1811,26 +1901,20 @@ export function registerFileRoutes(app: FastifyInstance, ctx: SessionPort & Even const stat = await fs.stat(fullPath); size = stat.size; } catch { - // Skip if can't stat + // Skip if can't stat. } - nodes.push({ - name: entry.name, - path: relativePath, - type: 'file', - size, - extension: ext, - }); + cursor.nodes.push({ name: entry.name, path: relativePath, type: 'file', size, extension: ext }); } } - - return nodes; - } catch { - // Can't read directory (permission denied, etc.) - return []; + if (cursor.index < cursor.entries.length || cursor.more) queue.push(cursor); } - }; - - const tree = await scanDirectory(walkRoot, 1); + } + // Every directory still queued is unread or half read, and an empty `children` would read as an + // empty folder. The root has no node to flag: `truncated` says the same for it. + for (const cursor of [...queue, ...unread]) { + if (cursor.node) cursor.node.incomplete = true; + } + if (queue.length > 0 || unread.length > 0) truncated = true; return { success: true, diff --git a/test/file-browser-incomplete.test.ts b/test/file-browser-incomplete.test.ts new file mode 100644 index 00000000..c31d6fe3 --- /dev/null +++ b/test/file-browser-incomplete.test.ts @@ -0,0 +1,167 @@ +/** + * @fileoverview The Files panel's row for a folder the scan's file limit stopped short of. + * + * `GET .../files` gives a directory its node before reading it, so a folder still queued when the + * budget ends has `children: []` or a partial list (test/routes/file-search-mode.test.ts). Without + * `incomplete` that reads as an empty folder. What can go wrong here is the client: the flag lost + * on the way in, the row drawn for a collapsed or finished folder, or the click wiring treating it + * as a file. + * + * panels-ui.js loads via `vm` as in file-browser-worktree.test.ts, over a real DOM, since the row + * has to survive `innerHTML` and the `.file-tree-item` selector. Translation goes through the real + * i18n.js. Port: N/A. + */ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import vm from 'node:vm'; +import { JSDOM } from 'jsdom'; +import { afterAll, describe, expect, it, vi } from 'vitest'; + +const PUBLIC = resolve(import.meta.dirname, '../src/web/public'); +const PANELS = readFileSync(resolve(PUBLIC, 'panels-ui.js'), 'utf8'); +const I18N = readFileSync(resolve(PUBLIC, 'i18n.js'), 'utf8'); +const STYLES = readFileSync(resolve(PUBLIC, 'styles.css'), 'utf8'); + +const dom = new JSDOM('
'); +afterAll(() => dom.window.close()); +const document = dom.window.document; +const treeEl = () => document.getElementById('fileBrowserTree') as HTMLElement; + +function loadPanel(expanded: string[] = []) { + const CodemanApp = function CodemanApp(this: unknown) {} as unknown as new () => Record; + const context = vm.createContext({ + CodemanApp, + console, + localStorage: { getItem: () => null, setItem: () => {}, removeItem: () => {} }, + escapeHtml: (s: string) => String(s), + document: { getElementById: () => null, addEventListener: vi.fn() }, + window: { addEventListener: vi.fn() }, + setTimeout, + clearTimeout, + CodemanBase: { url: (p: string) => p }, + }); + vm.runInContext(PANELS, context, { filename: 'panels-ui.js' }); + + const app = new CodemanApp(); + app.$ = (id: string) => document.getElementById(id); + app.activeSessionId = 'sess-1'; + app.fileBrowserExpandedDirs = new Set(expanded); + app.getFileIcon = () => ''; + app.formatFileSize = () => ''; + app.openFilePreview = vi.fn(); + return app; +} + +const bulk = { + name: 'a-bulk', + path: 'a-bulk', + type: 'directory', + incomplete: true, + children: [{ name: 'one.txt', path: 'a-bulk/one.txt', type: 'file', size: 1, extension: 'txt' }], +}; +const later = { + name: 'z-later', + path: 'z-later', + type: 'directory', + children: [{ name: 'needle.ts', path: 'z-later/needle.ts', type: 'file', size: 1, extension: 'ts' }], +}; +const envelope = (tree: unknown[]) => ({ + success: true, + data: { root: '/repo', tree, totalFiles: 2, totalDirectories: 2, truncated: true }, +}); + +const render = (app: Record, tree: unknown[]) => { + app.fileBrowserData = app._validateFileBrowserTreeEnvelope(envelope(tree)); + app.renderFileBrowserTree('sess-1'); + return [...treeEl().querySelectorAll('.file-tree-incomplete')] as HTMLElement[]; +}; + +describe('Files panel, a folder the file limit stopped short of', () => { + it('keeps the flag through the envelope check, which ignores what it does not know', () => { + const app = loadPanel(); + const data = app._validateFileBrowserTreeEnvelope(envelope([bulk, later])); + expect(data.tree[0].incomplete).toBe(true); + // An old tab on a new server, and a new tab on an old server: neither is refused. + expect(app._validateFileBrowserTreeEnvelope(envelope([later]))).not.toBeNull(); + }); + + it('draws one row under an expanded incomplete folder, after what was read', () => { + const rows = render(loadPanel(['a-bulk']), [bulk, later]); + + expect(rows).toHaveLength(1); + expect(rows[0].textContent).toBe('Not fully scanned (file limit reached)'); + expect(rows[0].dataset.depth).toBe('1'); + const items = [...treeEl().children].map((el) => (el as HTMLElement).dataset.path ?? 'ROW'); + expect(items).toEqual(['a-bulk', 'a-bulk/one.txt', 'ROW', 'z-later']); + }); + + it('draws it for a folder nothing was read from, which would otherwise look empty', () => { + const unread = { name: 'b-unread', path: 'b-unread', type: 'directory', incomplete: true, children: [] }; + const rows = render(loadPanel(['b-unread']), [unread]); + + expect(rows).toHaveLength(1); + expect(treeEl().querySelectorAll('.file-tree-item')).toHaveLength(1); + }); + + it('indents the row one level under a nested folder', () => { + const nested = { name: 'outer', path: 'outer', type: 'directory', children: [bulk] }; + const rows = render(loadPanel(['outer', 'a-bulk']), [nested]); + + expect(rows[0].dataset.depth).toBe('2'); + }); + + it('draws nothing while the folder is collapsed', () => { + expect(render(loadPanel(), [bulk, later])).toHaveLength(0); + }); + + it('draws nothing under an expanded folder that was finished', () => { + expect(render(loadPanel(['z-later']), [bulk, later])).toHaveLength(0); + }); + + it('accepts only a real true, so a stray truthy value draws nothing', () => { + const odd = { ...bulk, incomplete: 'false' }; + expect(render(loadPanel(['a-bulk']), [odd])).toHaveLength(0); + }); + + it('is inert: a click neither opens nor toggles anything', () => { + const app = loadPanel(['a-bulk']); + const toggle = vi.spyOn(app, 'toggleFileBrowserFolder'); + const [row] = render(app, [bulk, later]); + + row.click(); + + expect(row.matches('.file-tree-item')).toBe(false); + expect(row.dataset.path).toBeUndefined(); + expect(app.openFilePreview).not.toHaveBeenCalled(); + expect(toggle).not.toHaveBeenCalled(); + // The file beside it still opens, so the click wiring did run. + (treeEl().querySelector('[data-path="a-bulk/one.txt"]') as HTMLElement).click(); + expect(app.openFilePreview).toHaveBeenCalledWith('a-bulk/one.txt', 'sess-1'); + }); + + it('is indented by the same steps as the rows around it', () => { + for (let depth = 1; depth <= 9; depth++) { + expect(STYLES).toContain( + `.file-tree-item[data-depth="${depth}"], .file-tree-incomplete[data-depth="${depth}"] {` + ); + } + }); + + it('reads in Chinese, through the string the row really shows', () => { + const [row] = render(loadPanel(['a-bulk']), [bulk]); + const i18n = new JSDOM('', { runScripts: 'outside-only', url: 'http://localhost/' }); + vm.runInContext(I18N, i18n.getInternalVMContext(), { filename: 'i18n.js' }); + const api = ( + i18n.window as unknown as { + CodemanI18n: { t(s: string): string; configure(o: Record): void; start(): void }; + } + ).CodemanI18n; + api.start(); + api.configure({ language: 'zh-CN' }); + + const translated = api.t(row.textContent as string); + expect(translated).not.toBe(row.textContent); + expect(translated).not.toMatch(/[A-Za-z]/); + i18n.window.close(); + }); +}); diff --git a/test/routes/file-routes-live-worktree.test.ts b/test/routes/file-routes-live-worktree.test.ts index a5b8a86e..392c8530 100644 --- a/test/routes/file-routes-live-worktree.test.ts +++ b/test/routes/file-routes-live-worktree.test.ts @@ -163,6 +163,35 @@ describe('Files panel listing of a live worktree', () => { expect(data.worktree).toEqual({ name: 'probe', path: worktree }); }); + it('walks a large worktree in fair rounds, and the paths stay relative to the session folder', async () => { + mkdirSync(join(worktree, 'a-bulk')); + mkdirSync(join(worktree, 'z-later')); + for (let index = 0; index < 5100; index++) { + writeFileSync(join(worktree, 'a-bulk', `item-${index}.txt`), ''); + } + writeFileSync(join(worktree, 'z-later', 'needle.ts'), 'x'); + // The main checkout must play no part: this is the folder a walk seeded wrongly would list. + writeFileSync(join(repo, 'src', 'needle.ts'), 'main'); + + const data = await list('&root=live'); + expect(data.root).toBe(worktree); + expect(data.truncated).toBe(true); + expect(names(data.tree)).toEqual(['a-bulk', 'src', 'z-later']); + const [bulk, , later] = data.tree; + expect(bulk).toMatchObject({ path: `${PREFIX}/a-bulk`, incomplete: true }); + expect(bulk.children[0].path.startsWith(`${PREFIX}/a-bulk/`)).toBe(true); + expect(later).not.toHaveProperty('incomplete'); + expect(later.children[0].path).toBe(`${PREFIX}/z-later/needle.ts`); + + // A `/` in the query reads the listing's paths, and only the answer is in workingDir's terms. + const found = await list(`&root=live&q=${encodeURIComponent('z-later/*.ts')}`); + expect(found.root).toBe(worktree); + expect(found.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]); + expect(found.truncated).toBe(true); + const anywhere = await list('&root=live&q=needle'); + expect(anywhere.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]); + }); + it('serves a listed path through the unchanged preview route', async () => { const path = `${PREFIX}/src/worktree-only.ts`; const res = await harness.app.inject({ diff --git a/test/routes/file-search-mode.test.ts b/test/routes/file-search-mode.test.ts index 52a0b37e..4a9b1784 100644 --- a/test/routes/file-search-mode.test.ts +++ b/test/routes/file-search-mode.test.ts @@ -91,4 +91,231 @@ describe('files endpoint search mode', () => { expect(body.data.matches).toHaveLength(1); expect(body.data.matches[0]).toMatchObject({ name: 'nested', type: 'directory' }); }); + + it('shares the scan budget across sibling folders in both tree and search modes', async () => { + const bulk = join(root, 'a-bulk'); + const middle = join(root, 'm-middle'); + const later = join(root, 'z-later'); + mkdirSync(bulk); + mkdirSync(middle); + mkdirSync(later); + for (let index = 0; index < 5100; index++) { + writeFileSync(join(bulk, `item-${index}.txt`), ''); + } + writeFileSync(join(middle, 'middle.ts'), 'x\n'); + writeFileSync(join(later, 'needle.ts'), 'x\n'); + + try { + const tree = await get(''); + const treePaths = tree.data.tree.map((entry: { path: string }) => entry.path); + expect(treePaths).toContain('a-bulk'); + expect(treePaths).toContain('m-middle'); + expect(treePaths).toContain('src'); + expect(treePaths).toContain('z-later'); + expect(tree.data.truncated).toBe(true); + expect(tree.data.totalFiles + tree.data.totalDirectories).toBeLessThanOrEqual(5000); + + const node = (path: string) => tree.data.tree.find((entry: { path: string }) => entry.path === path); + expect(node('a-bulk').incomplete).toBe(true); + expect(node('a-bulk').children.length).toBeGreaterThan(0); + // Finished folders do not carry the key at all, not even as false. + expect(node('z-later')).not.toHaveProperty('incomplete'); + expect(node('z-later').children.map((entry: { path: string }) => entry.path)).toEqual([ + join('z-later', 'needle.ts'), + ]); + expect(node('m-middle')).not.toHaveProperty('incomplete'); + + const search = await get('?q=needle'); + expect(search.data.matches.map((match: { path: string }) => match.path)).toContain(join('z-later', 'needle.ts')); + expect(search.data.truncated).toBe(true); + } finally { + rmSync(bulk, { recursive: true, force: true }); + rmSync(middle, { recursive: true, force: true }); + rmSync(later, { recursive: true, force: true }); + } + }); + + it('adds nothing to a listing that fits the budget', async () => { + const { data } = await get(''); + + expect(data.truncated).toBe(false); + expect(JSON.stringify(data)).not.toContain('incomplete'); + const dir = data.tree.find((entry: { name: string }) => entry.name === 'src'); + expect(Object.keys(dir)).toEqual(['name', 'path', 'type', 'children']); + const file = data.tree.find((entry: { name: string }) => entry.name === 'README.md'); + expect(Object.keys(file)).toEqual(['name', 'path', 'type', 'size', 'extension']); + }); + + it('flags folders the budget ran out on part way through a round, read or not', async () => { + // 3,000 folders use 3,000 of the 5,000 in the first round. The 2,000 left then buy one file + // each, so the round ends with 1,000 folders never read: the last of those is the one a budget + // check after the shift would drop from the queue, and with it the flag. + const wide = mkdtempSync(join(tmpdir(), 'codeman-file-wide-')); + for (let index = 0; index < 3000; index++) { + const dir = join(wide, `dir-${String(index).padStart(4, '0')}`); + mkdirSync(dir); + writeFileSync(join(dir, 'one.txt'), ''); + writeFileSync(join(dir, 'two.txt'), ''); + } + harness.ctx._session.workingDir = wide; + + try { + const { data } = await get(''); + expect(data.truncated).toBe(true); + expect(data.tree).toHaveLength(3000); + expect(data.tree.filter((node: { incomplete?: boolean }) => node.incomplete === true)).toHaveLength(3000); + const sizes = data.tree.map((node: { children: unknown[] }) => node.children.length); + expect(sizes.filter((size: number) => size === 1)).toHaveLength(2000); + expect(sizes.filter((size: number) => size === 0)).toHaveLength(1000); + } finally { + rmSync(wide, { recursive: true, force: true }); + } + }); + + it('copes with the root itself being unfinished, and its subfolders still get read', async () => { + // The root has no node to flag. Its files stop the walk, but only after the folder it found. + const wide = mkdtempSync(join(tmpdir(), 'codeman-file-root-')); + mkdirSync(join(wide, 'first')); + writeFileSync(join(wide, 'first', 'inside.txt'), ''); + for (let index = 0; index < 5100; index++) { + writeFileSync(join(wide, `item-${index}.txt`), ''); + } + harness.ctx._session.workingDir = wide; + + try { + const { data } = await get(''); + expect(data.truncated).toBe(true); + expect(data.totalFiles + data.totalDirectories).toBe(5000); + expect(data.tree[0]).toMatchObject({ name: 'first', children: [{ name: 'inside.txt' }] }); + expect(data.tree[0]).not.toHaveProperty('incomplete'); + + const search = await get('?q=inside'); + expect(search.data.matches.map((match: { path: string }) => match.path)).toEqual([join('first', 'inside.txt')]); + expect(search.data.truncated).toBe(true); + } finally { + rmSync(wide, { recursive: true, force: true }); + } + }); + + it('does not let a large folder alone in its round starve the subfolders it holds', async () => { + // One level below the walk root, with a second level of subfolder to reach. + const wide = mkdtempSync(join(tmpdir(), 'codeman-file-sole-')); + mkdirSync(join(wide, 'repo', 'src', 'lib'), { recursive: true }); + writeFileSync(join(wide, 'repo', 'src', 'lib', 'needle.ts'), ''); + for (let index = 0; index < 5100; index++) { + writeFileSync(join(wide, 'repo', `item-${index}.txt`), ''); + } + harness.ctx._session.workingDir = wide; + + try { + const { data } = await get('?q=needle'); + expect(data.matches.map((match: { path: string }) => match.path)).toEqual([ + join('repo', 'src', 'lib', 'needle.ts'), + ]); + expect(data.truncated).toBe(true); + } finally { + rmSync(wide, { recursive: true, force: true }); + } + }); + + it('does not let large sibling folders starve the subfolders they hold', async () => { + // Two in a round, so "alone in its round" is not what makes this go wrong: each folder's + // equal share used to be spent on its own files, leaving nothing for what it had just found. + const wide = mkdtempSync(join(tmpdir(), 'codeman-file-pair-')); + for (const name of ['a-big', 'b-big']) { + mkdirSync(join(wide, name, 'src'), { recursive: true }); + writeFileSync(join(wide, name, 'src', 'needle.ts'), ''); + for (let index = 0; index < 5100; index++) { + writeFileSync(join(wide, name, `item-${index}.txt`), ''); + } + } + harness.ctx._session.workingDir = wide; + + try { + const { data } = await get('?q=needle'); + expect(data.matches.map((match: { path: string }) => match.path).sort()).toEqual([ + join('a-big', 'src', 'needle.ts'), + join('b-big', 'src', 'needle.ts'), + ]); + } finally { + rmSync(wide, { recursive: true, force: true }); + } + }); + + it('calls a workspace of exactly 5,000 nodes finished', async () => { + const exact = mkdtempSync(join(tmpdir(), 'codeman-file-exact-')); + mkdirSync(join(exact, 'empty')); + for (let index = 0; index < 4999; index++) { + writeFileSync(join(exact, `item-${index}.txt`), ''); + } + harness.ctx._session.workingDir = exact; + + try { + const { data } = await get(''); + expect(data.totalFiles + data.totalDirectories).toBe(5000); + expect(data.truncated).toBe(false); + expect(JSON.stringify(data)).not.toContain('incomplete'); + } finally { + rmSync(exact, { recursive: true, force: true }); + } + }); + + it('does not flag a folder whose only unread entries are ones the listing leaves out', async () => { + // 3,000 folders leave 2,000 of the budget, which buys each of the first 2,000 its `src`. All + // that is left unread in them is `vendor` and `.hidden`, which would never be listed anyway. + const wide = mkdtempSync(join(tmpdir(), 'codeman-file-excluded-')); + for (let index = 0; index < 3000; index++) { + const dir = join(wide, `dir-${String(index).padStart(4, '0')}`); + mkdirSync(join(dir, 'src'), { recursive: true }); + mkdirSync(join(dir, 'vendor')); + writeFileSync(join(dir, '.hidden'), ''); + } + harness.ctx._session.workingDir = wide; + + try { + const { data } = await get(''); + expect(data.truncated).toBe(true); + expect(data.tree[0].children.map((node: { name: string }) => node.name)).toEqual(['src']); + expect(data.tree[0]).not.toHaveProperty('incomplete'); + expect(data.tree[2999]).toMatchObject({ incomplete: true, children: [] }); + } finally { + rmSync(wide, { recursive: true, force: true }); + } + }); + + it('returns search results shallow-first', async () => { + const deep = join(root, 'a-deep', 'b', 'c'); + mkdirSync(deep, { recursive: true }); + mkdirSync(join(root, 'z-shallow')); + writeFileSync(join(deep, 'order-probe.ts'), 'x\n'); + writeFileSync(join(root, 'z-shallow', 'order-probe.ts'), 'x\n'); + + try { + const { data } = await get('?q=order-probe'); + expect(data.matches.map((match: { path: string }) => match.path)).toEqual([ + join('z-shallow', 'order-probe.ts'), + join('a-deep', 'b', 'c', 'order-probe.ts'), + ]); + } finally { + rmSync(join(root, 'a-deep'), { recursive: true, force: true }); + rmSync(join(root, 'z-shallow'), { recursive: true, force: true }); + } + }); + + it('stops at the match limit and says so', async () => { + const many = mkdtempSync(join(tmpdir(), 'codeman-file-many-')); + for (let index = 0; index < 1100; index++) { + writeFileSync(join(many, `hit-${index}.txt`), ''); + } + harness.ctx._session.workingDir = many; + + try { + const { data } = await get('?q=hit-'); + expect(data.matches).toHaveLength(1000); + expect(data.matchCount).toBe(1000); + expect(data.truncated).toBe(true); + } finally { + rmSync(many, { recursive: true, force: true }); + } + }); }); diff --git a/test/routes/file-walk-bounds.test.ts b/test/routes/file-walk-bounds.test.ts new file mode 100644 index 00000000..82b52df8 --- /dev/null +++ b/test/routes/file-walk-bounds.test.ts @@ -0,0 +1,160 @@ +/** + * GET /api/sessions/:id/files: what one listing costs the server. + * + * test/routes/file-search-mode.test.ts pins what the walk returns, on real folders. This file pins + * what it reads, which is where a real tree cannot go: 100 folders of 50,000 files is minutes of + * setup, and a dead network mount cannot be made in a test. So `fs.readdir` and `fs.stat` are + * stubbed here, on the one object the route and the bounded probe both call through. + * + * Port: none (app.inject). + */ +import { Dirent, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createRouteTestHarness, type RouteTestHarness } from './_route-test-utils.js'; +import { registerFileRoutes } from '../../src/web/routes/file-routes.js'; + +const UV_DIRENT_FILE = 1; +const UV_DIRENT_DIR = 2; +const dirent = (name: string, type: number, parent: string): Dirent => + new (Dirent as unknown as new (name: string, type: number, path: string) => Dirent)(name, type, parent); + +describe('files endpoint read bounds', () => { + let harness: RouteTestHarness; + + beforeEach(async () => { + harness = await createRouteTestHarness(registerFileRoutes); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + const get = async (query: string) => { + const res = await harness.app.inject({ + method: 'GET', + url: `/api/sessions/${harness.ctx._sessionId}/files${query}`, + }); + expect(res.statusCode).toBe(200); + return JSON.parse(res.body).data; + }; + + describe('over a workspace far bigger than the budget', () => { + const FOLDERS = 100; + const FILES_EACH = 50_000; + const WORKSPACE = '/virtual/ws'; + let entriesServed = 0; + + beforeEach(() => { + entriesServed = 0; + const folders = Array.from({ length: FOLDERS }, (_, index) => + dirent(`class-${String(index).padStart(5, '0')}`, UV_DIRENT_DIR, WORKSPACE) + ); + const files = Array.from({ length: FILES_EACH }, (_, index) => + dirent(`image-${String(index).padStart(8, '0')}.jpg`, UV_DIRENT_FILE, 'x') + ); + vi.spyOn(fs, 'readdir').mockImplementation((async (path: string) => { + const entries = path === WORKSPACE ? folders : files; + entriesServed += entries.length; + return [...entries]; + }) as never); + vi.spyOn(fs, 'stat').mockImplementation((async (path: string) => ({ + size: 1, + isDirectory: () => !String(path).endsWith('.jpg'), + })) as never); + harness.ctx._session.workingDir = WORKSPACE; + }); + + it('stops reading once it has read 250,000 entries, not at the end of the workspace', async () => { + const data = await get(''); + + // Past the ceiling by at most the folder that crossed it. + expect(entriesServed).toBeLessThanOrEqual(250_000 + FILES_EACH); + // What was read is still used, up to the budget, and nothing unread passes as empty. + expect(data.totalFiles + data.totalDirectories).toBe(5000); + expect(data.truncated).toBe(true); + expect(data.tree).toHaveLength(FOLDERS); + for (const folder of data.tree) expect(folder.incomplete).toBe(true); + expect(data.tree.filter((folder: { children: unknown[] }) => folder.children.length === 0)).toHaveLength( + FOLDERS - 5 + ); + }); + + it('stops a search the same way', async () => { + const data = await get('?q=image-00000000'); + + expect(entriesServed).toBeLessThanOrEqual(250_000 + FILES_EACH); + expect(data.matches.length).toBeGreaterThan(0); + expect(data.truncated).toBe(true); + }); + }); + + describe('with a dead mount below the workspace', () => { + let workspace: string; + let deadFolder: string; + let deadReads: number; + let releaseStat: (() => void) | undefined; + + beforeEach(() => { + workspace = mkdtempSync(join(tmpdir(), 'codeman-file-dead-')); + deadFolder = join(workspace, 'z-nas'); + mkdirSync(deadFolder); + mkdirSync(join(workspace, 'a-fine')); + writeFileSync(join(workspace, 'a-fine', 'needle.ts'), ''); + harness.ctx._session.workingDir = workspace; + + deadReads = 0; + const never = new Promise(() => {}); + const realReaddir = fs.readdir.bind(fs); + const realStat = fs.stat.bind(fs); + vi.spyOn(fs, 'readdir').mockImplementation(((path: string, options: never) => { + if (path !== deadFolder) return realReaddir(path, options); + deadReads++; + return never; + }) as never); + vi.spyOn(fs, 'stat').mockImplementation(((path: string) => { + if (path !== deadFolder) return realStat(path); + // Settled in afterEach: the probe calls a path stalled for as long as its stat is pending. + return new Promise((_resolve, reject) => { + releaseStat = () => reject(Object.assign(new Error('gone'), { code: 'EIO' })); + }); + }) as never); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + afterEach(async () => { + releaseStat?.(); + releaseStat = undefined; + await new Promise((settled) => setTimeout(settled, 0)); + rmSync(workspace, { recursive: true, force: true }); + }); + + const deadline = (work: Promise): Promise => + Promise.race([ + work, + new Promise((_resolve, reject) => { + setTimeout(() => reject(new Error('the listing never answered')), 5000).unref(); + }), + ]); + + it('answers without it, and costs it one stuck stat however often the panel asks', async () => { + const first = await deadline(get('')); + + const dead = first.tree.find((node: { name: string }) => node.name === 'z-nas'); + expect(dead).toEqual({ name: 'z-nas', path: 'z-nas', type: 'directory', children: [] }); + expect(first.truncated).toBe(false); + expect(first.tree[0].children.map((node: { name: string }) => node.name)).toEqual(['needle.ts']); + expect(deadReads).toBe(0); + + const statCalls = () => vi.mocked(fs.stat).mock.calls.filter(([path]) => path === deadFolder).length; + const stats = statCalls(); + const search = await deadline(get('?q=needle')); + expect(search.matches.map((match: { path: string }) => match.path)).toEqual([join('a-fine', 'needle.ts')]); + expect(statCalls()).toBe(stats); + expect(deadReads).toBe(0); + }); + }); +});