Merge #614 fix(files): prevent large folders starving File Viewer results

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codeman maintainer committed 2026-11-06 05:04:44 +01:00
commit aacf105cea
13 files changed
+831 -110

No files matched your search

@@ -163,6 +163,35 @@ describe('Files panel listing of a live worktree', () => {
expect(data.worktree).toEqual({ name: 'probe', path: worktree });
});
it('walks a large worktree in fair rounds, and the paths stay relative to the session folder', async () => {
mkdirSync(join(worktree, 'a-bulk'));
mkdirSync(join(worktree, 'z-later'));
for (let index = 0; index < 5100; index++) {
writeFileSync(join(worktree, 'a-bulk', `item-${index}.txt`), '');
}
writeFileSync(join(worktree, 'z-later', 'needle.ts'), 'x');
// The main checkout must play no part: this is the folder a walk seeded wrongly would list.
writeFileSync(join(repo, 'src', 'needle.ts'), 'main');
const data = await list('&root=live');
expect(data.root).toBe(worktree);
expect(data.truncated).toBe(true);
expect(names(data.tree)).toEqual(['a-bulk', 'src', 'z-later']);
const [bulk, , later] = data.tree;
expect(bulk).toMatchObject({ path: `${PREFIX}/a-bulk`, incomplete: true });
expect(bulk.children[0].path.startsWith(`${PREFIX}/a-bulk/`)).toBe(true);
expect(later).not.toHaveProperty('incomplete');
expect(later.children[0].path).toBe(`${PREFIX}/z-later/needle.ts`);
// A `/` in the query reads the listing's paths, and only the answer is in workingDir's terms.
const found = await list(`&root=live&q=${encodeURIComponent('z-later/*.ts')}`);
expect(found.root).toBe(worktree);
expect(found.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]);
expect(found.truncated).toBe(true);
const anywhere = await list('&root=live&q=needle');
expect(anywhere.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]);
});
it('serves a listed path through the unchanged preview route', async () => {
const path = `${PREFIX}/src/worktree-only.ts`;
const res = await harness.app.inject({
+227
View File
@@ -91,4 +91,231 @@ describe('files endpoint search mode', () => {
expect(body.data.matches).toHaveLength(1);
expect(body.data.matches[0]).toMatchObject({ name: 'nested', type: 'directory' });
});
it('shares the scan budget across sibling folders in both tree and search modes', async () => {
const bulk = join(root, 'a-bulk');
const middle = join(root, 'm-middle');
const later = join(root, 'z-later');
mkdirSync(bulk);
mkdirSync(middle);
mkdirSync(later);
for (let index = 0; index < 5100; index++) {
writeFileSync(join(bulk, `item-${index}.txt`), '');
}
writeFileSync(join(middle, 'middle.ts'), 'x\n');
writeFileSync(join(later, 'needle.ts'), 'x\n');
try {
const tree = await get('');
const treePaths = tree.data.tree.map((entry: { path: string }) => entry.path);
expect(treePaths).toContain('a-bulk');
expect(treePaths).toContain('m-middle');
expect(treePaths).toContain('src');
expect(treePaths).toContain('z-later');
expect(tree.data.truncated).toBe(true);
expect(tree.data.totalFiles + tree.data.totalDirectories).toBeLessThanOrEqual(5000);
const node = (path: string) => tree.data.tree.find((entry: { path: string }) => entry.path === path);
expect(node('a-bulk').incomplete).toBe(true);
expect(node('a-bulk').children.length).toBeGreaterThan(0);
// Finished folders do not carry the key at all, not even as false.
expect(node('z-later')).not.toHaveProperty('incomplete');
expect(node('z-later').children.map((entry: { path: string }) => entry.path)).toEqual([
join('z-later', 'needle.ts'),
]);
expect(node('m-middle')).not.toHaveProperty('incomplete');
const search = await get('?q=needle');
expect(search.data.matches.map((match: { path: string }) => match.path)).toContain(join('z-later', 'needle.ts'));
expect(search.data.truncated).toBe(true);
} finally {
rmSync(bulk, { recursive: true, force: true });
rmSync(middle, { recursive: true, force: true });
rmSync(later, { recursive: true, force: true });
}
});
it('adds nothing to a listing that fits the budget', async () => {
const { data } = await get('');
expect(data.truncated).toBe(false);
expect(JSON.stringify(data)).not.toContain('incomplete');
const dir = data.tree.find((entry: { name: string }) => entry.name === 'src');
expect(Object.keys(dir)).toEqual(['name', 'path', 'type', 'children']);
const file = data.tree.find((entry: { name: string }) => entry.name === 'README.md');
expect(Object.keys(file)).toEqual(['name', 'path', 'type', 'size', 'extension']);
});
it('flags folders the budget ran out on part way through a round, read or not', async () => {
// 3,000 folders use 3,000 of the 5,000 in the first round. The 2,000 left then buy one file
// each, so the round ends with 1,000 folders never read: the last of those is the one a budget
// check after the shift would drop from the queue, and with it the flag.
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-wide-'));
for (let index = 0; index < 3000; index++) {
const dir = join(wide, `dir-${String(index).padStart(4, '0')}`);
mkdirSync(dir);
writeFileSync(join(dir, 'one.txt'), '');
writeFileSync(join(dir, 'two.txt'), '');
}
harness.ctx._session.workingDir = wide;
try {
const { data } = await get('');
expect(data.truncated).toBe(true);
expect(data.tree).toHaveLength(3000);
expect(data.tree.filter((node: { incomplete?: boolean }) => node.incomplete === true)).toHaveLength(3000);
const sizes = data.tree.map((node: { children: unknown[] }) => node.children.length);
expect(sizes.filter((size: number) => size === 1)).toHaveLength(2000);
expect(sizes.filter((size: number) => size === 0)).toHaveLength(1000);
} finally {
rmSync(wide, { recursive: true, force: true });
}
});
it('copes with the root itself being unfinished, and its subfolders still get read', async () => {
// The root has no node to flag. Its files stop the walk, but only after the folder it found.
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-root-'));
mkdirSync(join(wide, 'first'));
writeFileSync(join(wide, 'first', 'inside.txt'), '');
for (let index = 0; index < 5100; index++) {
writeFileSync(join(wide, `item-${index}.txt`), '');
}
harness.ctx._session.workingDir = wide;
try {
const { data } = await get('');
expect(data.truncated).toBe(true);
expect(data.totalFiles + data.totalDirectories).toBe(5000);
expect(data.tree[0]).toMatchObject({ name: 'first', children: [{ name: 'inside.txt' }] });
expect(data.tree[0]).not.toHaveProperty('incomplete');
const search = await get('?q=inside');
expect(search.data.matches.map((match: { path: string }) => match.path)).toEqual([join('first', 'inside.txt')]);
expect(search.data.truncated).toBe(true);
} finally {
rmSync(wide, { recursive: true, force: true });
}
});
it('does not let a large folder alone in its round starve the subfolders it holds', async () => {
// One level below the walk root, with a second level of subfolder to reach.
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-sole-'));
mkdirSync(join(wide, 'repo', 'src', 'lib'), { recursive: true });
writeFileSync(join(wide, 'repo', 'src', 'lib', 'needle.ts'), '');
for (let index = 0; index < 5100; index++) {
writeFileSync(join(wide, 'repo', `item-${index}.txt`), '');
}
harness.ctx._session.workingDir = wide;
try {
const { data } = await get('?q=needle');
expect(data.matches.map((match: { path: string }) => match.path)).toEqual([
join('repo', 'src', 'lib', 'needle.ts'),
]);
expect(data.truncated).toBe(true);
} finally {
rmSync(wide, { recursive: true, force: true });
}
});
it('does not let large sibling folders starve the subfolders they hold', async () => {
// Two in a round, so "alone in its round" is not what makes this go wrong: each folder's
// equal share used to be spent on its own files, leaving nothing for what it had just found.
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-pair-'));
for (const name of ['a-big', 'b-big']) {
mkdirSync(join(wide, name, 'src'), { recursive: true });
writeFileSync(join(wide, name, 'src', 'needle.ts'), '');
for (let index = 0; index < 5100; index++) {
writeFileSync(join(wide, name, `item-${index}.txt`), '');
}
}
harness.ctx._session.workingDir = wide;
try {
const { data } = await get('?q=needle');
expect(data.matches.map((match: { path: string }) => match.path).sort()).toEqual([
join('a-big', 'src', 'needle.ts'),
join('b-big', 'src', 'needle.ts'),
]);
} finally {
rmSync(wide, { recursive: true, force: true });
}
});
it('calls a workspace of exactly 5,000 nodes finished', async () => {
const exact = mkdtempSync(join(tmpdir(), 'codeman-file-exact-'));
mkdirSync(join(exact, 'empty'));
for (let index = 0; index < 4999; index++) {
writeFileSync(join(exact, `item-${index}.txt`), '');
}
harness.ctx._session.workingDir = exact;
try {
const { data } = await get('');
expect(data.totalFiles + data.totalDirectories).toBe(5000);
expect(data.truncated).toBe(false);
expect(JSON.stringify(data)).not.toContain('incomplete');
} finally {
rmSync(exact, { recursive: true, force: true });
}
});
it('does not flag a folder whose only unread entries are ones the listing leaves out', async () => {
// 3,000 folders leave 2,000 of the budget, which buys each of the first 2,000 its `src`. All
// that is left unread in them is `vendor` and `.hidden`, which would never be listed anyway.
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-excluded-'));
for (let index = 0; index < 3000; index++) {
const dir = join(wide, `dir-${String(index).padStart(4, '0')}`);
mkdirSync(join(dir, 'src'), { recursive: true });
mkdirSync(join(dir, 'vendor'));
writeFileSync(join(dir, '.hidden'), '');
}
harness.ctx._session.workingDir = wide;
try {
const { data } = await get('');
expect(data.truncated).toBe(true);
expect(data.tree[0].children.map((node: { name: string }) => node.name)).toEqual(['src']);
expect(data.tree[0]).not.toHaveProperty('incomplete');
expect(data.tree[2999]).toMatchObject({ incomplete: true, children: [] });
} finally {
rmSync(wide, { recursive: true, force: true });
}
});
it('returns search results shallow-first', async () => {
const deep = join(root, 'a-deep', 'b', 'c');
mkdirSync(deep, { recursive: true });
mkdirSync(join(root, 'z-shallow'));
writeFileSync(join(deep, 'order-probe.ts'), 'x\n');
writeFileSync(join(root, 'z-shallow', 'order-probe.ts'), 'x\n');
try {
const { data } = await get('?q=order-probe');
expect(data.matches.map((match: { path: string }) => match.path)).toEqual([
join('z-shallow', 'order-probe.ts'),
join('a-deep', 'b', 'c', 'order-probe.ts'),
]);
} finally {
rmSync(join(root, 'a-deep'), { recursive: true, force: true });
rmSync(join(root, 'z-shallow'), { recursive: true, force: true });
}
});
it('stops at the match limit and says so', async () => {
const many = mkdtempSync(join(tmpdir(), 'codeman-file-many-'));
for (let index = 0; index < 1100; index++) {
writeFileSync(join(many, `hit-${index}.txt`), '');
}
harness.ctx._session.workingDir = many;
try {
const { data } = await get('?q=hit-');
expect(data.matches).toHaveLength(1000);
expect(data.matchCount).toBe(1000);
expect(data.truncated).toBe(true);
} finally {
rmSync(many, { recursive: true, force: true });
}
});
});
+160
View File
@@ -0,0 +1,160 @@
/**
* GET /api/sessions/:id/files: what one listing costs the server.
*
* test/routes/file-search-mode.test.ts pins what the walk returns, on real folders. This file pins
* what it reads, which is where a real tree cannot go: 100 folders of 50,000 files is minutes of
* setup, and a dead network mount cannot be made in a test. So `fs.readdir` and `fs.stat` are
* stubbed here, on the one object the route and the bounded probe both call through.
*
* Port: none (app.inject).
*/
import { Dirent, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import fs from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { createRouteTestHarness, type RouteTestHarness } from './_route-test-utils.js';
import { registerFileRoutes } from '../../src/web/routes/file-routes.js';
const UV_DIRENT_FILE = 1;
const UV_DIRENT_DIR = 2;
const dirent = (name: string, type: number, parent: string): Dirent =>
new (Dirent as unknown as new (name: string, type: number, path: string) => Dirent)(name, type, parent);
describe('files endpoint read bounds', () => {
let harness: RouteTestHarness;
beforeEach(async () => {
harness = await createRouteTestHarness(registerFileRoutes);
});
afterEach(() => {
vi.restoreAllMocks();
});
const get = async (query: string) => {
const res = await harness.app.inject({
method: 'GET',
url: `/api/sessions/${harness.ctx._sessionId}/files${query}`,
});
expect(res.statusCode).toBe(200);
return JSON.parse(res.body).data;
};
describe('over a workspace far bigger than the budget', () => {
const FOLDERS = 100;
const FILES_EACH = 50_000;
const WORKSPACE = '/virtual/ws';
let entriesServed = 0;
beforeEach(() => {
entriesServed = 0;
const folders = Array.from({ length: FOLDERS }, (_, index) =>
dirent(`class-${String(index).padStart(5, '0')}`, UV_DIRENT_DIR, WORKSPACE)
);
const files = Array.from({ length: FILES_EACH }, (_, index) =>
dirent(`image-${String(index).padStart(8, '0')}.jpg`, UV_DIRENT_FILE, 'x')
);
vi.spyOn(fs, 'readdir').mockImplementation((async (path: string) => {
const entries = path === WORKSPACE ? folders : files;
entriesServed += entries.length;
return [...entries];
}) as never);
vi.spyOn(fs, 'stat').mockImplementation((async (path: string) => ({
size: 1,
isDirectory: () => !String(path).endsWith('.jpg'),
})) as never);
harness.ctx._session.workingDir = WORKSPACE;
});
it('stops reading once it has read 250,000 entries, not at the end of the workspace', async () => {
const data = await get('');
// Past the ceiling by at most the folder that crossed it.
expect(entriesServed).toBeLessThanOrEqual(250_000 + FILES_EACH);
// What was read is still used, up to the budget, and nothing unread passes as empty.
expect(data.totalFiles + data.totalDirectories).toBe(5000);
expect(data.truncated).toBe(true);
expect(data.tree).toHaveLength(FOLDERS);
for (const folder of data.tree) expect(folder.incomplete).toBe(true);
expect(data.tree.filter((folder: { children: unknown[] }) => folder.children.length === 0)).toHaveLength(
FOLDERS - 5
);
});
it('stops a search the same way', async () => {
const data = await get('?q=image-00000000');
expect(entriesServed).toBeLessThanOrEqual(250_000 + FILES_EACH);
expect(data.matches.length).toBeGreaterThan(0);
expect(data.truncated).toBe(true);
});
});
describe('with a dead mount below the workspace', () => {
let workspace: string;
let deadFolder: string;
let deadReads: number;
let releaseStat: (() => void) | undefined;
beforeEach(() => {
workspace = mkdtempSync(join(tmpdir(), 'codeman-file-dead-'));
deadFolder = join(workspace, 'z-nas');
mkdirSync(deadFolder);
mkdirSync(join(workspace, 'a-fine'));
writeFileSync(join(workspace, 'a-fine', 'needle.ts'), '');
harness.ctx._session.workingDir = workspace;
deadReads = 0;
const never = new Promise<never>(() => {});
const realReaddir = fs.readdir.bind(fs);
const realStat = fs.stat.bind(fs);
vi.spyOn(fs, 'readdir').mockImplementation(((path: string, options: never) => {
if (path !== deadFolder) return realReaddir(path, options);
deadReads++;
return never;
}) as never);
vi.spyOn(fs, 'stat').mockImplementation(((path: string) => {
if (path !== deadFolder) return realStat(path);
// Settled in afterEach: the probe calls a path stalled for as long as its stat is pending.
return new Promise((_resolve, reject) => {
releaseStat = () => reject(Object.assign(new Error('gone'), { code: 'EIO' }));
});
}) as never);
vi.spyOn(console, 'warn').mockImplementation(() => {});
});
afterEach(async () => {
releaseStat?.();
releaseStat = undefined;
await new Promise((settled) => setTimeout(settled, 0));
rmSync(workspace, { recursive: true, force: true });
});
const deadline = <T>(work: Promise<T>): Promise<T> =>
Promise.race([
work,
new Promise<T>((_resolve, reject) => {
setTimeout(() => reject(new Error('the listing never answered')), 5000).unref();
}),
]);
it('answers without it, and costs it one stuck stat however often the panel asks', async () => {
const first = await deadline(get(''));
const dead = first.tree.find((node: { name: string }) => node.name === 'z-nas');
expect(dead).toEqual({ name: 'z-nas', path: 'z-nas', type: 'directory', children: [] });
expect(first.truncated).toBe(false);
expect(first.tree[0].children.map((node: { name: string }) => node.name)).toEqual(['needle.ts']);
expect(deadReads).toBe(0);
const statCalls = () => vi.mocked(fs.stat).mock.calls.filter(([path]) => path === deadFolder).length;
const stats = statCalls();
const search = await deadline(get('?q=needle'));
expect(search.matches.map((match: { path: string }) => match.path)).toEqual([join('a-fine', 'needle.ts')]);
expect(statCalls()).toBe(stats);
expect(deadReads).toBe(0);
});
});
});