mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-18 04:09:33 +01:00
Merge #614 fix(files): prevent large folders starving File Viewer results
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
aacf105cea
13 files changed
+831
-110
No files matched your search
@@ -163,6 +163,35 @@ describe('Files panel listing of a live worktree', () => {
|
||||
expect(data.worktree).toEqual({ name: 'probe', path: worktree });
|
||||
});
|
||||
|
||||
it('walks a large worktree in fair rounds, and the paths stay relative to the session folder', async () => {
|
||||
mkdirSync(join(worktree, 'a-bulk'));
|
||||
mkdirSync(join(worktree, 'z-later'));
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(worktree, 'a-bulk', `item-${index}.txt`), '');
|
||||
}
|
||||
writeFileSync(join(worktree, 'z-later', 'needle.ts'), 'x');
|
||||
// The main checkout must play no part: this is the folder a walk seeded wrongly would list.
|
||||
writeFileSync(join(repo, 'src', 'needle.ts'), 'main');
|
||||
|
||||
const data = await list('&root=live');
|
||||
expect(data.root).toBe(worktree);
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(names(data.tree)).toEqual(['a-bulk', 'src', 'z-later']);
|
||||
const [bulk, , later] = data.tree;
|
||||
expect(bulk).toMatchObject({ path: `${PREFIX}/a-bulk`, incomplete: true });
|
||||
expect(bulk.children[0].path.startsWith(`${PREFIX}/a-bulk/`)).toBe(true);
|
||||
expect(later).not.toHaveProperty('incomplete');
|
||||
expect(later.children[0].path).toBe(`${PREFIX}/z-later/needle.ts`);
|
||||
|
||||
// A `/` in the query reads the listing's paths, and only the answer is in workingDir's terms.
|
||||
const found = await list(`&root=live&q=${encodeURIComponent('z-later/*.ts')}`);
|
||||
expect(found.root).toBe(worktree);
|
||||
expect(found.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]);
|
||||
expect(found.truncated).toBe(true);
|
||||
const anywhere = await list('&root=live&q=needle');
|
||||
expect(anywhere.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]);
|
||||
});
|
||||
|
||||
it('serves a listed path through the unchanged preview route', async () => {
|
||||
const path = `${PREFIX}/src/worktree-only.ts`;
|
||||
const res = await harness.app.inject({
|
||||
|
||||
@@ -91,4 +91,231 @@ describe('files endpoint search mode', () => {
|
||||
expect(body.data.matches).toHaveLength(1);
|
||||
expect(body.data.matches[0]).toMatchObject({ name: 'nested', type: 'directory' });
|
||||
});
|
||||
|
||||
it('shares the scan budget across sibling folders in both tree and search modes', async () => {
|
||||
const bulk = join(root, 'a-bulk');
|
||||
const middle = join(root, 'm-middle');
|
||||
const later = join(root, 'z-later');
|
||||
mkdirSync(bulk);
|
||||
mkdirSync(middle);
|
||||
mkdirSync(later);
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(bulk, `item-${index}.txt`), '');
|
||||
}
|
||||
writeFileSync(join(middle, 'middle.ts'), 'x\n');
|
||||
writeFileSync(join(later, 'needle.ts'), 'x\n');
|
||||
|
||||
try {
|
||||
const tree = await get('');
|
||||
const treePaths = tree.data.tree.map((entry: { path: string }) => entry.path);
|
||||
expect(treePaths).toContain('a-bulk');
|
||||
expect(treePaths).toContain('m-middle');
|
||||
expect(treePaths).toContain('src');
|
||||
expect(treePaths).toContain('z-later');
|
||||
expect(tree.data.truncated).toBe(true);
|
||||
expect(tree.data.totalFiles + tree.data.totalDirectories).toBeLessThanOrEqual(5000);
|
||||
|
||||
const node = (path: string) => tree.data.tree.find((entry: { path: string }) => entry.path === path);
|
||||
expect(node('a-bulk').incomplete).toBe(true);
|
||||
expect(node('a-bulk').children.length).toBeGreaterThan(0);
|
||||
// Finished folders do not carry the key at all, not even as false.
|
||||
expect(node('z-later')).not.toHaveProperty('incomplete');
|
||||
expect(node('z-later').children.map((entry: { path: string }) => entry.path)).toEqual([
|
||||
join('z-later', 'needle.ts'),
|
||||
]);
|
||||
expect(node('m-middle')).not.toHaveProperty('incomplete');
|
||||
|
||||
const search = await get('?q=needle');
|
||||
expect(search.data.matches.map((match: { path: string }) => match.path)).toContain(join('z-later', 'needle.ts'));
|
||||
expect(search.data.truncated).toBe(true);
|
||||
} finally {
|
||||
rmSync(bulk, { recursive: true, force: true });
|
||||
rmSync(middle, { recursive: true, force: true });
|
||||
rmSync(later, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('adds nothing to a listing that fits the budget', async () => {
|
||||
const { data } = await get('');
|
||||
|
||||
expect(data.truncated).toBe(false);
|
||||
expect(JSON.stringify(data)).not.toContain('incomplete');
|
||||
const dir = data.tree.find((entry: { name: string }) => entry.name === 'src');
|
||||
expect(Object.keys(dir)).toEqual(['name', 'path', 'type', 'children']);
|
||||
const file = data.tree.find((entry: { name: string }) => entry.name === 'README.md');
|
||||
expect(Object.keys(file)).toEqual(['name', 'path', 'type', 'size', 'extension']);
|
||||
});
|
||||
|
||||
it('flags folders the budget ran out on part way through a round, read or not', async () => {
|
||||
// 3,000 folders use 3,000 of the 5,000 in the first round. The 2,000 left then buy one file
|
||||
// each, so the round ends with 1,000 folders never read: the last of those is the one a budget
|
||||
// check after the shift would drop from the queue, and with it the flag.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-wide-'));
|
||||
for (let index = 0; index < 3000; index++) {
|
||||
const dir = join(wide, `dir-${String(index).padStart(4, '0')}`);
|
||||
mkdirSync(dir);
|
||||
writeFileSync(join(dir, 'one.txt'), '');
|
||||
writeFileSync(join(dir, 'two.txt'), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('');
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(data.tree).toHaveLength(3000);
|
||||
expect(data.tree.filter((node: { incomplete?: boolean }) => node.incomplete === true)).toHaveLength(3000);
|
||||
const sizes = data.tree.map((node: { children: unknown[] }) => node.children.length);
|
||||
expect(sizes.filter((size: number) => size === 1)).toHaveLength(2000);
|
||||
expect(sizes.filter((size: number) => size === 0)).toHaveLength(1000);
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('copes with the root itself being unfinished, and its subfolders still get read', async () => {
|
||||
// The root has no node to flag. Its files stop the walk, but only after the folder it found.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-root-'));
|
||||
mkdirSync(join(wide, 'first'));
|
||||
writeFileSync(join(wide, 'first', 'inside.txt'), '');
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(wide, `item-${index}.txt`), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('');
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(data.totalFiles + data.totalDirectories).toBe(5000);
|
||||
expect(data.tree[0]).toMatchObject({ name: 'first', children: [{ name: 'inside.txt' }] });
|
||||
expect(data.tree[0]).not.toHaveProperty('incomplete');
|
||||
|
||||
const search = await get('?q=inside');
|
||||
expect(search.data.matches.map((match: { path: string }) => match.path)).toEqual([join('first', 'inside.txt')]);
|
||||
expect(search.data.truncated).toBe(true);
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('does not let a large folder alone in its round starve the subfolders it holds', async () => {
|
||||
// One level below the walk root, with a second level of subfolder to reach.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-sole-'));
|
||||
mkdirSync(join(wide, 'repo', 'src', 'lib'), { recursive: true });
|
||||
writeFileSync(join(wide, 'repo', 'src', 'lib', 'needle.ts'), '');
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(wide, 'repo', `item-${index}.txt`), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('?q=needle');
|
||||
expect(data.matches.map((match: { path: string }) => match.path)).toEqual([
|
||||
join('repo', 'src', 'lib', 'needle.ts'),
|
||||
]);
|
||||
expect(data.truncated).toBe(true);
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('does not let large sibling folders starve the subfolders they hold', async () => {
|
||||
// Two in a round, so "alone in its round" is not what makes this go wrong: each folder's
|
||||
// equal share used to be spent on its own files, leaving nothing for what it had just found.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-pair-'));
|
||||
for (const name of ['a-big', 'b-big']) {
|
||||
mkdirSync(join(wide, name, 'src'), { recursive: true });
|
||||
writeFileSync(join(wide, name, 'src', 'needle.ts'), '');
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(wide, name, `item-${index}.txt`), '');
|
||||
}
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('?q=needle');
|
||||
expect(data.matches.map((match: { path: string }) => match.path).sort()).toEqual([
|
||||
join('a-big', 'src', 'needle.ts'),
|
||||
join('b-big', 'src', 'needle.ts'),
|
||||
]);
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('calls a workspace of exactly 5,000 nodes finished', async () => {
|
||||
const exact = mkdtempSync(join(tmpdir(), 'codeman-file-exact-'));
|
||||
mkdirSync(join(exact, 'empty'));
|
||||
for (let index = 0; index < 4999; index++) {
|
||||
writeFileSync(join(exact, `item-${index}.txt`), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = exact;
|
||||
|
||||
try {
|
||||
const { data } = await get('');
|
||||
expect(data.totalFiles + data.totalDirectories).toBe(5000);
|
||||
expect(data.truncated).toBe(false);
|
||||
expect(JSON.stringify(data)).not.toContain('incomplete');
|
||||
} finally {
|
||||
rmSync(exact, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('does not flag a folder whose only unread entries are ones the listing leaves out', async () => {
|
||||
// 3,000 folders leave 2,000 of the budget, which buys each of the first 2,000 its `src`. All
|
||||
// that is left unread in them is `vendor` and `.hidden`, which would never be listed anyway.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-excluded-'));
|
||||
for (let index = 0; index < 3000; index++) {
|
||||
const dir = join(wide, `dir-${String(index).padStart(4, '0')}`);
|
||||
mkdirSync(join(dir, 'src'), { recursive: true });
|
||||
mkdirSync(join(dir, 'vendor'));
|
||||
writeFileSync(join(dir, '.hidden'), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('');
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(data.tree[0].children.map((node: { name: string }) => node.name)).toEqual(['src']);
|
||||
expect(data.tree[0]).not.toHaveProperty('incomplete');
|
||||
expect(data.tree[2999]).toMatchObject({ incomplete: true, children: [] });
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('returns search results shallow-first', async () => {
|
||||
const deep = join(root, 'a-deep', 'b', 'c');
|
||||
mkdirSync(deep, { recursive: true });
|
||||
mkdirSync(join(root, 'z-shallow'));
|
||||
writeFileSync(join(deep, 'order-probe.ts'), 'x\n');
|
||||
writeFileSync(join(root, 'z-shallow', 'order-probe.ts'), 'x\n');
|
||||
|
||||
try {
|
||||
const { data } = await get('?q=order-probe');
|
||||
expect(data.matches.map((match: { path: string }) => match.path)).toEqual([
|
||||
join('z-shallow', 'order-probe.ts'),
|
||||
join('a-deep', 'b', 'c', 'order-probe.ts'),
|
||||
]);
|
||||
} finally {
|
||||
rmSync(join(root, 'a-deep'), { recursive: true, force: true });
|
||||
rmSync(join(root, 'z-shallow'), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('stops at the match limit and says so', async () => {
|
||||
const many = mkdtempSync(join(tmpdir(), 'codeman-file-many-'));
|
||||
for (let index = 0; index < 1100; index++) {
|
||||
writeFileSync(join(many, `hit-${index}.txt`), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = many;
|
||||
|
||||
try {
|
||||
const { data } = await get('?q=hit-');
|
||||
expect(data.matches).toHaveLength(1000);
|
||||
expect(data.matchCount).toBe(1000);
|
||||
expect(data.truncated).toBe(true);
|
||||
} finally {
|
||||
rmSync(many, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* GET /api/sessions/:id/files: what one listing costs the server.
|
||||
*
|
||||
* test/routes/file-search-mode.test.ts pins what the walk returns, on real folders. This file pins
|
||||
* what it reads, which is where a real tree cannot go: 100 folders of 50,000 files is minutes of
|
||||
* setup, and a dead network mount cannot be made in a test. So `fs.readdir` and `fs.stat` are
|
||||
* stubbed here, on the one object the route and the bounded probe both call through.
|
||||
*
|
||||
* Port: none (app.inject).
|
||||
*/
|
||||
import { Dirent, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { createRouteTestHarness, type RouteTestHarness } from './_route-test-utils.js';
|
||||
import { registerFileRoutes } from '../../src/web/routes/file-routes.js';
|
||||
|
||||
const UV_DIRENT_FILE = 1;
|
||||
const UV_DIRENT_DIR = 2;
|
||||
const dirent = (name: string, type: number, parent: string): Dirent =>
|
||||
new (Dirent as unknown as new (name: string, type: number, path: string) => Dirent)(name, type, parent);
|
||||
|
||||
describe('files endpoint read bounds', () => {
|
||||
let harness: RouteTestHarness;
|
||||
|
||||
beforeEach(async () => {
|
||||
harness = await createRouteTestHarness(registerFileRoutes);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
const get = async (query: string) => {
|
||||
const res = await harness.app.inject({
|
||||
method: 'GET',
|
||||
url: `/api/sessions/${harness.ctx._sessionId}/files${query}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
return JSON.parse(res.body).data;
|
||||
};
|
||||
|
||||
describe('over a workspace far bigger than the budget', () => {
|
||||
const FOLDERS = 100;
|
||||
const FILES_EACH = 50_000;
|
||||
const WORKSPACE = '/virtual/ws';
|
||||
let entriesServed = 0;
|
||||
|
||||
beforeEach(() => {
|
||||
entriesServed = 0;
|
||||
const folders = Array.from({ length: FOLDERS }, (_, index) =>
|
||||
dirent(`class-${String(index).padStart(5, '0')}`, UV_DIRENT_DIR, WORKSPACE)
|
||||
);
|
||||
const files = Array.from({ length: FILES_EACH }, (_, index) =>
|
||||
dirent(`image-${String(index).padStart(8, '0')}.jpg`, UV_DIRENT_FILE, 'x')
|
||||
);
|
||||
vi.spyOn(fs, 'readdir').mockImplementation((async (path: string) => {
|
||||
const entries = path === WORKSPACE ? folders : files;
|
||||
entriesServed += entries.length;
|
||||
return [...entries];
|
||||
}) as never);
|
||||
vi.spyOn(fs, 'stat').mockImplementation((async (path: string) => ({
|
||||
size: 1,
|
||||
isDirectory: () => !String(path).endsWith('.jpg'),
|
||||
})) as never);
|
||||
harness.ctx._session.workingDir = WORKSPACE;
|
||||
});
|
||||
|
||||
it('stops reading once it has read 250,000 entries, not at the end of the workspace', async () => {
|
||||
const data = await get('');
|
||||
|
||||
// Past the ceiling by at most the folder that crossed it.
|
||||
expect(entriesServed).toBeLessThanOrEqual(250_000 + FILES_EACH);
|
||||
// What was read is still used, up to the budget, and nothing unread passes as empty.
|
||||
expect(data.totalFiles + data.totalDirectories).toBe(5000);
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(data.tree).toHaveLength(FOLDERS);
|
||||
for (const folder of data.tree) expect(folder.incomplete).toBe(true);
|
||||
expect(data.tree.filter((folder: { children: unknown[] }) => folder.children.length === 0)).toHaveLength(
|
||||
FOLDERS - 5
|
||||
);
|
||||
});
|
||||
|
||||
it('stops a search the same way', async () => {
|
||||
const data = await get('?q=image-00000000');
|
||||
|
||||
expect(entriesServed).toBeLessThanOrEqual(250_000 + FILES_EACH);
|
||||
expect(data.matches.length).toBeGreaterThan(0);
|
||||
expect(data.truncated).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with a dead mount below the workspace', () => {
|
||||
let workspace: string;
|
||||
let deadFolder: string;
|
||||
let deadReads: number;
|
||||
let releaseStat: (() => void) | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
workspace = mkdtempSync(join(tmpdir(), 'codeman-file-dead-'));
|
||||
deadFolder = join(workspace, 'z-nas');
|
||||
mkdirSync(deadFolder);
|
||||
mkdirSync(join(workspace, 'a-fine'));
|
||||
writeFileSync(join(workspace, 'a-fine', 'needle.ts'), '');
|
||||
harness.ctx._session.workingDir = workspace;
|
||||
|
||||
deadReads = 0;
|
||||
const never = new Promise<never>(() => {});
|
||||
const realReaddir = fs.readdir.bind(fs);
|
||||
const realStat = fs.stat.bind(fs);
|
||||
vi.spyOn(fs, 'readdir').mockImplementation(((path: string, options: never) => {
|
||||
if (path !== deadFolder) return realReaddir(path, options);
|
||||
deadReads++;
|
||||
return never;
|
||||
}) as never);
|
||||
vi.spyOn(fs, 'stat').mockImplementation(((path: string) => {
|
||||
if (path !== deadFolder) return realStat(path);
|
||||
// Settled in afterEach: the probe calls a path stalled for as long as its stat is pending.
|
||||
return new Promise((_resolve, reject) => {
|
||||
releaseStat = () => reject(Object.assign(new Error('gone'), { code: 'EIO' }));
|
||||
});
|
||||
}) as never);
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
releaseStat?.();
|
||||
releaseStat = undefined;
|
||||
await new Promise((settled) => setTimeout(settled, 0));
|
||||
rmSync(workspace, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const deadline = <T>(work: Promise<T>): Promise<T> =>
|
||||
Promise.race([
|
||||
work,
|
||||
new Promise<T>((_resolve, reject) => {
|
||||
setTimeout(() => reject(new Error('the listing never answered')), 5000).unref();
|
||||
}),
|
||||
]);
|
||||
|
||||
it('answers without it, and costs it one stuck stat however often the panel asks', async () => {
|
||||
const first = await deadline(get(''));
|
||||
|
||||
const dead = first.tree.find((node: { name: string }) => node.name === 'z-nas');
|
||||
expect(dead).toEqual({ name: 'z-nas', path: 'z-nas', type: 'directory', children: [] });
|
||||
expect(first.truncated).toBe(false);
|
||||
expect(first.tree[0].children.map((node: { name: string }) => node.name)).toEqual(['needle.ts']);
|
||||
expect(deadReads).toBe(0);
|
||||
|
||||
const statCalls = () => vi.mocked(fs.stat).mock.calls.filter(([path]) => path === deadFolder).length;
|
||||
const stats = statCalls();
|
||||
const search = await deadline(get('?q=needle'));
|
||||
expect(search.matches.map((match: { path: string }) => match.path)).toEqual([join('a-fine', 'needle.ts')]);
|
||||
expect(statCalls()).toBe(stats);
|
||||
expect(deadReads).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user