mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-15 02:49:33 +01:00
Merge #614 fix(files): prevent large folders starving File Viewer results
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
aacf105cea
13 files changed
+831
-110
No files matched your search
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* @fileoverview The Files panel's row for a folder the scan's file limit stopped short of.
|
||||
*
|
||||
* `GET .../files` gives a directory its node before reading it, so a folder still queued when the
|
||||
* budget ends has `children: []` or a partial list (test/routes/file-search-mode.test.ts). Without
|
||||
* `incomplete` that reads as an empty folder. What can go wrong here is the client: the flag lost
|
||||
* on the way in, the row drawn for a collapsed or finished folder, or the click wiring treating it
|
||||
* as a file.
|
||||
*
|
||||
* panels-ui.js loads via `vm` as in file-browser-worktree.test.ts, over a real DOM, since the row
|
||||
* has to survive `innerHTML` and the `.file-tree-item` selector. Translation goes through the real
|
||||
* i18n.js. Port: N/A.
|
||||
*/
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import vm from 'node:vm';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { afterAll, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const PUBLIC = resolve(import.meta.dirname, '../src/web/public');
|
||||
const PANELS = readFileSync(resolve(PUBLIC, 'panels-ui.js'), 'utf8');
|
||||
const I18N = readFileSync(resolve(PUBLIC, 'i18n.js'), 'utf8');
|
||||
const STYLES = readFileSync(resolve(PUBLIC, 'styles.css'), 'utf8');
|
||||
|
||||
const dom = new JSDOM('<div id="fileBrowserTree"></div><div id="fileBrowserPanel" class="visible"></div>');
|
||||
afterAll(() => dom.window.close());
|
||||
const document = dom.window.document;
|
||||
const treeEl = () => document.getElementById('fileBrowserTree') as HTMLElement;
|
||||
|
||||
function loadPanel(expanded: string[] = []) {
|
||||
const CodemanApp = function CodemanApp(this: unknown) {} as unknown as new () => Record<string, any>;
|
||||
const context = vm.createContext({
|
||||
CodemanApp,
|
||||
console,
|
||||
localStorage: { getItem: () => null, setItem: () => {}, removeItem: () => {} },
|
||||
escapeHtml: (s: string) => String(s),
|
||||
document: { getElementById: () => null, addEventListener: vi.fn() },
|
||||
window: { addEventListener: vi.fn() },
|
||||
setTimeout,
|
||||
clearTimeout,
|
||||
CodemanBase: { url: (p: string) => p },
|
||||
});
|
||||
vm.runInContext(PANELS, context, { filename: 'panels-ui.js' });
|
||||
|
||||
const app = new CodemanApp();
|
||||
app.$ = (id: string) => document.getElementById(id);
|
||||
app.activeSessionId = 'sess-1';
|
||||
app.fileBrowserExpandedDirs = new Set(expanded);
|
||||
app.getFileIcon = () => '';
|
||||
app.formatFileSize = () => '';
|
||||
app.openFilePreview = vi.fn();
|
||||
return app;
|
||||
}
|
||||
|
||||
const bulk = {
|
||||
name: 'a-bulk',
|
||||
path: 'a-bulk',
|
||||
type: 'directory',
|
||||
incomplete: true,
|
||||
children: [{ name: 'one.txt', path: 'a-bulk/one.txt', type: 'file', size: 1, extension: 'txt' }],
|
||||
};
|
||||
const later = {
|
||||
name: 'z-later',
|
||||
path: 'z-later',
|
||||
type: 'directory',
|
||||
children: [{ name: 'needle.ts', path: 'z-later/needle.ts', type: 'file', size: 1, extension: 'ts' }],
|
||||
};
|
||||
const envelope = (tree: unknown[]) => ({
|
||||
success: true,
|
||||
data: { root: '/repo', tree, totalFiles: 2, totalDirectories: 2, truncated: true },
|
||||
});
|
||||
|
||||
const render = (app: Record<string, any>, tree: unknown[]) => {
|
||||
app.fileBrowserData = app._validateFileBrowserTreeEnvelope(envelope(tree));
|
||||
app.renderFileBrowserTree('sess-1');
|
||||
return [...treeEl().querySelectorAll('.file-tree-incomplete')] as HTMLElement[];
|
||||
};
|
||||
|
||||
describe('Files panel, a folder the file limit stopped short of', () => {
|
||||
it('keeps the flag through the envelope check, which ignores what it does not know', () => {
|
||||
const app = loadPanel();
|
||||
const data = app._validateFileBrowserTreeEnvelope(envelope([bulk, later]));
|
||||
expect(data.tree[0].incomplete).toBe(true);
|
||||
// An old tab on a new server, and a new tab on an old server: neither is refused.
|
||||
expect(app._validateFileBrowserTreeEnvelope(envelope([later]))).not.toBeNull();
|
||||
});
|
||||
|
||||
it('draws one row under an expanded incomplete folder, after what was read', () => {
|
||||
const rows = render(loadPanel(['a-bulk']), [bulk, later]);
|
||||
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].textContent).toBe('Not fully scanned (file limit reached)');
|
||||
expect(rows[0].dataset.depth).toBe('1');
|
||||
const items = [...treeEl().children].map((el) => (el as HTMLElement).dataset.path ?? 'ROW');
|
||||
expect(items).toEqual(['a-bulk', 'a-bulk/one.txt', 'ROW', 'z-later']);
|
||||
});
|
||||
|
||||
it('draws it for a folder nothing was read from, which would otherwise look empty', () => {
|
||||
const unread = { name: 'b-unread', path: 'b-unread', type: 'directory', incomplete: true, children: [] };
|
||||
const rows = render(loadPanel(['b-unread']), [unread]);
|
||||
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(treeEl().querySelectorAll('.file-tree-item')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('indents the row one level under a nested folder', () => {
|
||||
const nested = { name: 'outer', path: 'outer', type: 'directory', children: [bulk] };
|
||||
const rows = render(loadPanel(['outer', 'a-bulk']), [nested]);
|
||||
|
||||
expect(rows[0].dataset.depth).toBe('2');
|
||||
});
|
||||
|
||||
it('draws nothing while the folder is collapsed', () => {
|
||||
expect(render(loadPanel(), [bulk, later])).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('draws nothing under an expanded folder that was finished', () => {
|
||||
expect(render(loadPanel(['z-later']), [bulk, later])).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('accepts only a real true, so a stray truthy value draws nothing', () => {
|
||||
const odd = { ...bulk, incomplete: 'false' };
|
||||
expect(render(loadPanel(['a-bulk']), [odd])).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('is inert: a click neither opens nor toggles anything', () => {
|
||||
const app = loadPanel(['a-bulk']);
|
||||
const toggle = vi.spyOn(app, 'toggleFileBrowserFolder');
|
||||
const [row] = render(app, [bulk, later]);
|
||||
|
||||
row.click();
|
||||
|
||||
expect(row.matches('.file-tree-item')).toBe(false);
|
||||
expect(row.dataset.path).toBeUndefined();
|
||||
expect(app.openFilePreview).not.toHaveBeenCalled();
|
||||
expect(toggle).not.toHaveBeenCalled();
|
||||
// The file beside it still opens, so the click wiring did run.
|
||||
(treeEl().querySelector('[data-path="a-bulk/one.txt"]') as HTMLElement).click();
|
||||
expect(app.openFilePreview).toHaveBeenCalledWith('a-bulk/one.txt', 'sess-1');
|
||||
});
|
||||
|
||||
it('is indented by the same steps as the rows around it', () => {
|
||||
for (let depth = 1; depth <= 9; depth++) {
|
||||
expect(STYLES).toContain(
|
||||
`.file-tree-item[data-depth="${depth}"], .file-tree-incomplete[data-depth="${depth}"] {`
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('reads in Chinese, through the string the row really shows', () => {
|
||||
const [row] = render(loadPanel(['a-bulk']), [bulk]);
|
||||
const i18n = new JSDOM('<html></html>', { runScripts: 'outside-only', url: 'http://localhost/' });
|
||||
vm.runInContext(I18N, i18n.getInternalVMContext(), { filename: 'i18n.js' });
|
||||
const api = (
|
||||
i18n.window as unknown as {
|
||||
CodemanI18n: { t(s: string): string; configure(o: Record<string, string>): void; start(): void };
|
||||
}
|
||||
).CodemanI18n;
|
||||
api.start();
|
||||
api.configure({ language: 'zh-CN' });
|
||||
|
||||
const translated = api.t(row.textContent as string);
|
||||
expect(translated).not.toBe(row.textContent);
|
||||
expect(translated).not.toMatch(/[A-Za-z]/);
|
||||
i18n.window.close();
|
||||
});
|
||||
});
|
||||
@@ -163,6 +163,35 @@ describe('Files panel listing of a live worktree', () => {
|
||||
expect(data.worktree).toEqual({ name: 'probe', path: worktree });
|
||||
});
|
||||
|
||||
it('walks a large worktree in fair rounds, and the paths stay relative to the session folder', async () => {
|
||||
mkdirSync(join(worktree, 'a-bulk'));
|
||||
mkdirSync(join(worktree, 'z-later'));
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(worktree, 'a-bulk', `item-${index}.txt`), '');
|
||||
}
|
||||
writeFileSync(join(worktree, 'z-later', 'needle.ts'), 'x');
|
||||
// The main checkout must play no part: this is the folder a walk seeded wrongly would list.
|
||||
writeFileSync(join(repo, 'src', 'needle.ts'), 'main');
|
||||
|
||||
const data = await list('&root=live');
|
||||
expect(data.root).toBe(worktree);
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(names(data.tree)).toEqual(['a-bulk', 'src', 'z-later']);
|
||||
const [bulk, , later] = data.tree;
|
||||
expect(bulk).toMatchObject({ path: `${PREFIX}/a-bulk`, incomplete: true });
|
||||
expect(bulk.children[0].path.startsWith(`${PREFIX}/a-bulk/`)).toBe(true);
|
||||
expect(later).not.toHaveProperty('incomplete');
|
||||
expect(later.children[0].path).toBe(`${PREFIX}/z-later/needle.ts`);
|
||||
|
||||
// A `/` in the query reads the listing's paths, and only the answer is in workingDir's terms.
|
||||
const found = await list(`&root=live&q=${encodeURIComponent('z-later/*.ts')}`);
|
||||
expect(found.root).toBe(worktree);
|
||||
expect(found.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]);
|
||||
expect(found.truncated).toBe(true);
|
||||
const anywhere = await list('&root=live&q=needle');
|
||||
expect(anywhere.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]);
|
||||
});
|
||||
|
||||
it('serves a listed path through the unchanged preview route', async () => {
|
||||
const path = `${PREFIX}/src/worktree-only.ts`;
|
||||
const res = await harness.app.inject({
|
||||
|
||||
@@ -91,4 +91,231 @@ describe('files endpoint search mode', () => {
|
||||
expect(body.data.matches).toHaveLength(1);
|
||||
expect(body.data.matches[0]).toMatchObject({ name: 'nested', type: 'directory' });
|
||||
});
|
||||
|
||||
it('shares the scan budget across sibling folders in both tree and search modes', async () => {
|
||||
const bulk = join(root, 'a-bulk');
|
||||
const middle = join(root, 'm-middle');
|
||||
const later = join(root, 'z-later');
|
||||
mkdirSync(bulk);
|
||||
mkdirSync(middle);
|
||||
mkdirSync(later);
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(bulk, `item-${index}.txt`), '');
|
||||
}
|
||||
writeFileSync(join(middle, 'middle.ts'), 'x\n');
|
||||
writeFileSync(join(later, 'needle.ts'), 'x\n');
|
||||
|
||||
try {
|
||||
const tree = await get('');
|
||||
const treePaths = tree.data.tree.map((entry: { path: string }) => entry.path);
|
||||
expect(treePaths).toContain('a-bulk');
|
||||
expect(treePaths).toContain('m-middle');
|
||||
expect(treePaths).toContain('src');
|
||||
expect(treePaths).toContain('z-later');
|
||||
expect(tree.data.truncated).toBe(true);
|
||||
expect(tree.data.totalFiles + tree.data.totalDirectories).toBeLessThanOrEqual(5000);
|
||||
|
||||
const node = (path: string) => tree.data.tree.find((entry: { path: string }) => entry.path === path);
|
||||
expect(node('a-bulk').incomplete).toBe(true);
|
||||
expect(node('a-bulk').children.length).toBeGreaterThan(0);
|
||||
// Finished folders do not carry the key at all, not even as false.
|
||||
expect(node('z-later')).not.toHaveProperty('incomplete');
|
||||
expect(node('z-later').children.map((entry: { path: string }) => entry.path)).toEqual([
|
||||
join('z-later', 'needle.ts'),
|
||||
]);
|
||||
expect(node('m-middle')).not.toHaveProperty('incomplete');
|
||||
|
||||
const search = await get('?q=needle');
|
||||
expect(search.data.matches.map((match: { path: string }) => match.path)).toContain(join('z-later', 'needle.ts'));
|
||||
expect(search.data.truncated).toBe(true);
|
||||
} finally {
|
||||
rmSync(bulk, { recursive: true, force: true });
|
||||
rmSync(middle, { recursive: true, force: true });
|
||||
rmSync(later, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('adds nothing to a listing that fits the budget', async () => {
|
||||
const { data } = await get('');
|
||||
|
||||
expect(data.truncated).toBe(false);
|
||||
expect(JSON.stringify(data)).not.toContain('incomplete');
|
||||
const dir = data.tree.find((entry: { name: string }) => entry.name === 'src');
|
||||
expect(Object.keys(dir)).toEqual(['name', 'path', 'type', 'children']);
|
||||
const file = data.tree.find((entry: { name: string }) => entry.name === 'README.md');
|
||||
expect(Object.keys(file)).toEqual(['name', 'path', 'type', 'size', 'extension']);
|
||||
});
|
||||
|
||||
it('flags folders the budget ran out on part way through a round, read or not', async () => {
|
||||
// 3,000 folders use 3,000 of the 5,000 in the first round. The 2,000 left then buy one file
|
||||
// each, so the round ends with 1,000 folders never read: the last of those is the one a budget
|
||||
// check after the shift would drop from the queue, and with it the flag.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-wide-'));
|
||||
for (let index = 0; index < 3000; index++) {
|
||||
const dir = join(wide, `dir-${String(index).padStart(4, '0')}`);
|
||||
mkdirSync(dir);
|
||||
writeFileSync(join(dir, 'one.txt'), '');
|
||||
writeFileSync(join(dir, 'two.txt'), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('');
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(data.tree).toHaveLength(3000);
|
||||
expect(data.tree.filter((node: { incomplete?: boolean }) => node.incomplete === true)).toHaveLength(3000);
|
||||
const sizes = data.tree.map((node: { children: unknown[] }) => node.children.length);
|
||||
expect(sizes.filter((size: number) => size === 1)).toHaveLength(2000);
|
||||
expect(sizes.filter((size: number) => size === 0)).toHaveLength(1000);
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('copes with the root itself being unfinished, and its subfolders still get read', async () => {
|
||||
// The root has no node to flag. Its files stop the walk, but only after the folder it found.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-root-'));
|
||||
mkdirSync(join(wide, 'first'));
|
||||
writeFileSync(join(wide, 'first', 'inside.txt'), '');
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(wide, `item-${index}.txt`), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('');
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(data.totalFiles + data.totalDirectories).toBe(5000);
|
||||
expect(data.tree[0]).toMatchObject({ name: 'first', children: [{ name: 'inside.txt' }] });
|
||||
expect(data.tree[0]).not.toHaveProperty('incomplete');
|
||||
|
||||
const search = await get('?q=inside');
|
||||
expect(search.data.matches.map((match: { path: string }) => match.path)).toEqual([join('first', 'inside.txt')]);
|
||||
expect(search.data.truncated).toBe(true);
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('does not let a large folder alone in its round starve the subfolders it holds', async () => {
|
||||
// One level below the walk root, with a second level of subfolder to reach.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-sole-'));
|
||||
mkdirSync(join(wide, 'repo', 'src', 'lib'), { recursive: true });
|
||||
writeFileSync(join(wide, 'repo', 'src', 'lib', 'needle.ts'), '');
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(wide, 'repo', `item-${index}.txt`), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('?q=needle');
|
||||
expect(data.matches.map((match: { path: string }) => match.path)).toEqual([
|
||||
join('repo', 'src', 'lib', 'needle.ts'),
|
||||
]);
|
||||
expect(data.truncated).toBe(true);
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('does not let large sibling folders starve the subfolders they hold', async () => {
|
||||
// Two in a round, so "alone in its round" is not what makes this go wrong: each folder's
|
||||
// equal share used to be spent on its own files, leaving nothing for what it had just found.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-pair-'));
|
||||
for (const name of ['a-big', 'b-big']) {
|
||||
mkdirSync(join(wide, name, 'src'), { recursive: true });
|
||||
writeFileSync(join(wide, name, 'src', 'needle.ts'), '');
|
||||
for (let index = 0; index < 5100; index++) {
|
||||
writeFileSync(join(wide, name, `item-${index}.txt`), '');
|
||||
}
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('?q=needle');
|
||||
expect(data.matches.map((match: { path: string }) => match.path).sort()).toEqual([
|
||||
join('a-big', 'src', 'needle.ts'),
|
||||
join('b-big', 'src', 'needle.ts'),
|
||||
]);
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('calls a workspace of exactly 5,000 nodes finished', async () => {
|
||||
const exact = mkdtempSync(join(tmpdir(), 'codeman-file-exact-'));
|
||||
mkdirSync(join(exact, 'empty'));
|
||||
for (let index = 0; index < 4999; index++) {
|
||||
writeFileSync(join(exact, `item-${index}.txt`), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = exact;
|
||||
|
||||
try {
|
||||
const { data } = await get('');
|
||||
expect(data.totalFiles + data.totalDirectories).toBe(5000);
|
||||
expect(data.truncated).toBe(false);
|
||||
expect(JSON.stringify(data)).not.toContain('incomplete');
|
||||
} finally {
|
||||
rmSync(exact, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('does not flag a folder whose only unread entries are ones the listing leaves out', async () => {
|
||||
// 3,000 folders leave 2,000 of the budget, which buys each of the first 2,000 its `src`. All
|
||||
// that is left unread in them is `vendor` and `.hidden`, which would never be listed anyway.
|
||||
const wide = mkdtempSync(join(tmpdir(), 'codeman-file-excluded-'));
|
||||
for (let index = 0; index < 3000; index++) {
|
||||
const dir = join(wide, `dir-${String(index).padStart(4, '0')}`);
|
||||
mkdirSync(join(dir, 'src'), { recursive: true });
|
||||
mkdirSync(join(dir, 'vendor'));
|
||||
writeFileSync(join(dir, '.hidden'), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = wide;
|
||||
|
||||
try {
|
||||
const { data } = await get('');
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(data.tree[0].children.map((node: { name: string }) => node.name)).toEqual(['src']);
|
||||
expect(data.tree[0]).not.toHaveProperty('incomplete');
|
||||
expect(data.tree[2999]).toMatchObject({ incomplete: true, children: [] });
|
||||
} finally {
|
||||
rmSync(wide, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('returns search results shallow-first', async () => {
|
||||
const deep = join(root, 'a-deep', 'b', 'c');
|
||||
mkdirSync(deep, { recursive: true });
|
||||
mkdirSync(join(root, 'z-shallow'));
|
||||
writeFileSync(join(deep, 'order-probe.ts'), 'x\n');
|
||||
writeFileSync(join(root, 'z-shallow', 'order-probe.ts'), 'x\n');
|
||||
|
||||
try {
|
||||
const { data } = await get('?q=order-probe');
|
||||
expect(data.matches.map((match: { path: string }) => match.path)).toEqual([
|
||||
join('z-shallow', 'order-probe.ts'),
|
||||
join('a-deep', 'b', 'c', 'order-probe.ts'),
|
||||
]);
|
||||
} finally {
|
||||
rmSync(join(root, 'a-deep'), { recursive: true, force: true });
|
||||
rmSync(join(root, 'z-shallow'), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('stops at the match limit and says so', async () => {
|
||||
const many = mkdtempSync(join(tmpdir(), 'codeman-file-many-'));
|
||||
for (let index = 0; index < 1100; index++) {
|
||||
writeFileSync(join(many, `hit-${index}.txt`), '');
|
||||
}
|
||||
harness.ctx._session.workingDir = many;
|
||||
|
||||
try {
|
||||
const { data } = await get('?q=hit-');
|
||||
expect(data.matches).toHaveLength(1000);
|
||||
expect(data.matchCount).toBe(1000);
|
||||
expect(data.truncated).toBe(true);
|
||||
} finally {
|
||||
rmSync(many, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* GET /api/sessions/:id/files: what one listing costs the server.
|
||||
*
|
||||
* test/routes/file-search-mode.test.ts pins what the walk returns, on real folders. This file pins
|
||||
* what it reads, which is where a real tree cannot go: 100 folders of 50,000 files is minutes of
|
||||
* setup, and a dead network mount cannot be made in a test. So `fs.readdir` and `fs.stat` are
|
||||
* stubbed here, on the one object the route and the bounded probe both call through.
|
||||
*
|
||||
* Port: none (app.inject).
|
||||
*/
|
||||
import { Dirent, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { createRouteTestHarness, type RouteTestHarness } from './_route-test-utils.js';
|
||||
import { registerFileRoutes } from '../../src/web/routes/file-routes.js';
|
||||
|
||||
const UV_DIRENT_FILE = 1;
|
||||
const UV_DIRENT_DIR = 2;
|
||||
const dirent = (name: string, type: number, parent: string): Dirent =>
|
||||
new (Dirent as unknown as new (name: string, type: number, path: string) => Dirent)(name, type, parent);
|
||||
|
||||
describe('files endpoint read bounds', () => {
|
||||
let harness: RouteTestHarness;
|
||||
|
||||
beforeEach(async () => {
|
||||
harness = await createRouteTestHarness(registerFileRoutes);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
const get = async (query: string) => {
|
||||
const res = await harness.app.inject({
|
||||
method: 'GET',
|
||||
url: `/api/sessions/${harness.ctx._sessionId}/files${query}`,
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
return JSON.parse(res.body).data;
|
||||
};
|
||||
|
||||
describe('over a workspace far bigger than the budget', () => {
|
||||
const FOLDERS = 100;
|
||||
const FILES_EACH = 50_000;
|
||||
const WORKSPACE = '/virtual/ws';
|
||||
let entriesServed = 0;
|
||||
|
||||
beforeEach(() => {
|
||||
entriesServed = 0;
|
||||
const folders = Array.from({ length: FOLDERS }, (_, index) =>
|
||||
dirent(`class-${String(index).padStart(5, '0')}`, UV_DIRENT_DIR, WORKSPACE)
|
||||
);
|
||||
const files = Array.from({ length: FILES_EACH }, (_, index) =>
|
||||
dirent(`image-${String(index).padStart(8, '0')}.jpg`, UV_DIRENT_FILE, 'x')
|
||||
);
|
||||
vi.spyOn(fs, 'readdir').mockImplementation((async (path: string) => {
|
||||
const entries = path === WORKSPACE ? folders : files;
|
||||
entriesServed += entries.length;
|
||||
return [...entries];
|
||||
}) as never);
|
||||
vi.spyOn(fs, 'stat').mockImplementation((async (path: string) => ({
|
||||
size: 1,
|
||||
isDirectory: () => !String(path).endsWith('.jpg'),
|
||||
})) as never);
|
||||
harness.ctx._session.workingDir = WORKSPACE;
|
||||
});
|
||||
|
||||
it('stops reading once it has read 250,000 entries, not at the end of the workspace', async () => {
|
||||
const data = await get('');
|
||||
|
||||
// Past the ceiling by at most the folder that crossed it.
|
||||
expect(entriesServed).toBeLessThanOrEqual(250_000 + FILES_EACH);
|
||||
// What was read is still used, up to the budget, and nothing unread passes as empty.
|
||||
expect(data.totalFiles + data.totalDirectories).toBe(5000);
|
||||
expect(data.truncated).toBe(true);
|
||||
expect(data.tree).toHaveLength(FOLDERS);
|
||||
for (const folder of data.tree) expect(folder.incomplete).toBe(true);
|
||||
expect(data.tree.filter((folder: { children: unknown[] }) => folder.children.length === 0)).toHaveLength(
|
||||
FOLDERS - 5
|
||||
);
|
||||
});
|
||||
|
||||
it('stops a search the same way', async () => {
|
||||
const data = await get('?q=image-00000000');
|
||||
|
||||
expect(entriesServed).toBeLessThanOrEqual(250_000 + FILES_EACH);
|
||||
expect(data.matches.length).toBeGreaterThan(0);
|
||||
expect(data.truncated).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with a dead mount below the workspace', () => {
|
||||
let workspace: string;
|
||||
let deadFolder: string;
|
||||
let deadReads: number;
|
||||
let releaseStat: (() => void) | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
workspace = mkdtempSync(join(tmpdir(), 'codeman-file-dead-'));
|
||||
deadFolder = join(workspace, 'z-nas');
|
||||
mkdirSync(deadFolder);
|
||||
mkdirSync(join(workspace, 'a-fine'));
|
||||
writeFileSync(join(workspace, 'a-fine', 'needle.ts'), '');
|
||||
harness.ctx._session.workingDir = workspace;
|
||||
|
||||
deadReads = 0;
|
||||
const never = new Promise<never>(() => {});
|
||||
const realReaddir = fs.readdir.bind(fs);
|
||||
const realStat = fs.stat.bind(fs);
|
||||
vi.spyOn(fs, 'readdir').mockImplementation(((path: string, options: never) => {
|
||||
if (path !== deadFolder) return realReaddir(path, options);
|
||||
deadReads++;
|
||||
return never;
|
||||
}) as never);
|
||||
vi.spyOn(fs, 'stat').mockImplementation(((path: string) => {
|
||||
if (path !== deadFolder) return realStat(path);
|
||||
// Settled in afterEach: the probe calls a path stalled for as long as its stat is pending.
|
||||
return new Promise((_resolve, reject) => {
|
||||
releaseStat = () => reject(Object.assign(new Error('gone'), { code: 'EIO' }));
|
||||
});
|
||||
}) as never);
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
releaseStat?.();
|
||||
releaseStat = undefined;
|
||||
await new Promise((settled) => setTimeout(settled, 0));
|
||||
rmSync(workspace, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const deadline = <T>(work: Promise<T>): Promise<T> =>
|
||||
Promise.race([
|
||||
work,
|
||||
new Promise<T>((_resolve, reject) => {
|
||||
setTimeout(() => reject(new Error('the listing never answered')), 5000).unref();
|
||||
}),
|
||||
]);
|
||||
|
||||
it('answers without it, and costs it one stuck stat however often the panel asks', async () => {
|
||||
const first = await deadline(get(''));
|
||||
|
||||
const dead = first.tree.find((node: { name: string }) => node.name === 'z-nas');
|
||||
expect(dead).toEqual({ name: 'z-nas', path: 'z-nas', type: 'directory', children: [] });
|
||||
expect(first.truncated).toBe(false);
|
||||
expect(first.tree[0].children.map((node: { name: string }) => node.name)).toEqual(['needle.ts']);
|
||||
expect(deadReads).toBe(0);
|
||||
|
||||
const statCalls = () => vi.mocked(fs.stat).mock.calls.filter(([path]) => path === deadFolder).length;
|
||||
const stats = statCalls();
|
||||
const search = await deadline(get('?q=needle'));
|
||||
expect(search.matches.map((match: { path: string }) => match.path)).toEqual([join('a-fine', 'needle.ts')]);
|
||||
expect(statCalls()).toBe(stats);
|
||||
expect(deadReads).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user