fix(files): mark folders the scan limit stopped short of

The fair rounds give a directory its node before reading it, so every folder
still queued when the 5,000-node budget ends sat in the tree with empty or
partial children and read as an empty folder. Keep the node on the tree cursor
and set an additive incomplete: true on every cursor left in the queue, then
draw one dim, inert row under such a folder while it is expanded. The envelope
check ignores keys it does not know, so old and new clients stay compatible.

Stop each round at the top once the budget is spent, before the shift: the
rest stay queued unread, and flagged, rather than read and sorted for nothing.

Say what the walk is: a per-directory round-robin (breadth-first), fair per
directory and not per subtree, with search results now shallow-first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

test/file-browser-incomplete.test.ts

@@ -0,0 +1,167 @@
+/**
+ * @fileoverview The Files panel's row for a folder the scan's file limit stopped short of.
+ *
+ * `GET .../files` gives a directory its node before reading it, so a folder still queued when the
+ * budget ends has `children: []` or a partial list (test/routes/file-search-mode.test.ts). Without
+ * `incomplete` that reads as an empty folder. What can go wrong here is the client: the flag lost
+ * on the way in, the row drawn for a collapsed or finished folder, or the click wiring treating it
+ * as a file.
+ *
+ * panels-ui.js loads via `vm` as in file-browser-worktree.test.ts, over a real DOM, since the row
+ * has to survive `innerHTML` and the `.file-tree-item` selector. Translation goes through the real
+ * i18n.js. Port: N/A.
+ */
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import vm from 'node:vm';
+import { JSDOM } from 'jsdom';
+import { afterAll, describe, expect, it, vi } from 'vitest';
+
+const PUBLIC = resolve(import.meta.dirname, '../src/web/public');
+const PANELS = readFileSync(resolve(PUBLIC, 'panels-ui.js'), 'utf8');
+const I18N = readFileSync(resolve(PUBLIC, 'i18n.js'), 'utf8');
+const STYLES = readFileSync(resolve(PUBLIC, 'styles.css'), 'utf8');
+
+const dom = new JSDOM('<div id="fileBrowserTree"></div><div id="fileBrowserPanel" class="visible"></div>');
+afterAll(() => dom.window.close());
+const document = dom.window.document;
+const treeEl = () => document.getElementById('fileBrowserTree') as HTMLElement;
+
+function loadPanel(expanded: string[] = []) {
+  const CodemanApp = function CodemanApp(this: unknown) {} as unknown as new () => Record<string, any>;
+  const context = vm.createContext({
+    CodemanApp,
+    console,
+    localStorage: { getItem: () => null, setItem: () => {}, removeItem: () => {} },
+    escapeHtml: (s: string) => String(s),
+    document: { getElementById: () => null, addEventListener: vi.fn() },
+    window: { addEventListener: vi.fn() },
+    setTimeout,
+    clearTimeout,
+    CodemanBase: { url: (p: string) => p },
+  });
+  vm.runInContext(PANELS, context, { filename: 'panels-ui.js' });
+
+  const app = new CodemanApp();
+  app.$ = (id: string) => document.getElementById(id);
+  app.activeSessionId = 'sess-1';
+  app.fileBrowserExpandedDirs = new Set(expanded);
+  app.getFileIcon = () => '';
+  app.formatFileSize = () => '';
+  app.openFilePreview = vi.fn();
+  return app;
+}
+
+const bulk = {
+  name: 'a-bulk',
+  path: 'a-bulk',
+  type: 'directory',
+  incomplete: true,
+  children: [{ name: 'one.txt', path: 'a-bulk/one.txt', type: 'file', size: 1, extension: 'txt' }],
+};
+const later = {
+  name: 'z-later',
+  path: 'z-later',
+  type: 'directory',
+  children: [{ name: 'needle.ts', path: 'z-later/needle.ts', type: 'file', size: 1, extension: 'ts' }],
+};
+const envelope = (tree: unknown[]) => ({
+  success: true,
+  data: { root: '/repo', tree, totalFiles: 2, totalDirectories: 2, truncated: true },
+});
+
+const render = (app: Record<string, any>, tree: unknown[]) => {
+  app.fileBrowserData = app._validateFileBrowserTreeEnvelope(envelope(tree));
+  app.renderFileBrowserTree('sess-1');
+  return [...treeEl().querySelectorAll('.file-tree-incomplete')] as HTMLElement[];
+};
+
+describe('Files panel, a folder the file limit stopped short of', () => {
+  it('keeps the flag through the envelope check, which ignores what it does not know', () => {
+    const app = loadPanel();
+    const data = app._validateFileBrowserTreeEnvelope(envelope([bulk, later]));
+    expect(data.tree[0].incomplete).toBe(true);
+    // An old tab on a new server, and a new tab on an old server: neither is refused.
+    expect(app._validateFileBrowserTreeEnvelope(envelope([later]))).not.toBeNull();
+  });
+
+  it('draws one row under an expanded incomplete folder, after what was read', () => {
+    const rows = render(loadPanel(['a-bulk']), [bulk, later]);
+
+    expect(rows).toHaveLength(1);
+    expect(rows[0].textContent).toBe('Not fully scanned (file limit reached)');
+    expect(rows[0].dataset.depth).toBe('1');
+    const items = [...treeEl().children].map((el) => (el as HTMLElement).dataset.path ?? 'ROW');
+    expect(items).toEqual(['a-bulk', 'a-bulk/one.txt', 'ROW', 'z-later']);
+  });
+
+  it('draws it for a folder nothing was read from, which would otherwise look empty', () => {
+    const unread = { name: 'b-unread', path: 'b-unread', type: 'directory', incomplete: true, children: [] };
+    const rows = render(loadPanel(['b-unread']), [unread]);
+
+    expect(rows).toHaveLength(1);
+    expect(treeEl().querySelectorAll('.file-tree-item')).toHaveLength(1);
+  });
+
+  it('indents the row one level under a nested folder', () => {
+    const nested = { name: 'outer', path: 'outer', type: 'directory', children: [bulk] };
+    const rows = render(loadPanel(['outer', 'a-bulk']), [nested]);
+
+    expect(rows[0].dataset.depth).toBe('2');
+  });
+
+  it('draws nothing while the folder is collapsed', () => {
+    expect(render(loadPanel(), [bulk, later])).toHaveLength(0);
+  });
+
+  it('draws nothing under an expanded folder that was finished', () => {
+    expect(render(loadPanel(['z-later']), [bulk, later])).toHaveLength(0);
+  });
+
+  it('accepts only a real true, so a stray truthy value draws nothing', () => {
+    const odd = { ...bulk, incomplete: 'false' };
+    expect(render(loadPanel(['a-bulk']), [odd])).toHaveLength(0);
+  });
+
+  it('is inert: a click neither opens nor toggles anything', () => {
+    const app = loadPanel(['a-bulk']);
+    const toggle = vi.spyOn(app, 'toggleFileBrowserFolder');
+    const [row] = render(app, [bulk, later]);
+
+    row.click();
+
+    expect(row.matches('.file-tree-item')).toBe(false);
+    expect(row.dataset.path).toBeUndefined();
+    expect(app.openFilePreview).not.toHaveBeenCalled();
+    expect(toggle).not.toHaveBeenCalled();
+    // The file beside it still opens, so the click wiring did run.
+    (treeEl().querySelector('[data-path="a-bulk/one.txt"]') as HTMLElement).click();
+    expect(app.openFilePreview).toHaveBeenCalledWith('a-bulk/one.txt', 'sess-1');
+  });
+
+  it('is indented by the same steps as the rows around it', () => {
+    for (let depth = 1; depth <= 9; depth++) {
+      expect(STYLES).toContain(
+        `.file-tree-item[data-depth="${depth}"], .file-tree-incomplete[data-depth="${depth}"] {`
+      );
+    }
+  });
+
+  it('reads in Chinese, through the string the row really shows', () => {
+    const [row] = render(loadPanel(['a-bulk']), [bulk]);
+    const i18n = new JSDOM('<html></html>', { runScripts: 'outside-only', url: 'http://localhost/' });
+    vm.runInContext(I18N, i18n.getInternalVMContext(), { filename: 'i18n.js' });
+    const api = (
+      i18n.window as unknown as {
+        CodemanI18n: { t(s: string): string; configure(o: Record<string, string>): void; start(): void };
+      }
+    ).CodemanI18n;
+    api.start();
+    api.configure({ language: 'zh-CN' });
+
+    const translated = api.t(row.textContent as string);
+    expect(translated).not.toBe(row.textContent);
+    expect(translated).not.toMatch(/[A-Za-z]/);
+    i18n.window.close();
+  });
+});

test/routes/file-routes-live-worktree.test.ts

@@ -163,6 +163,35 @@ describe('Files panel listing of a live worktree', () => {
     expect(data.worktree).toEqual({ name: 'probe', path: worktree });
   });
 
+  it('walks a large worktree in fair rounds, and the paths stay relative to the session folder', async () => {
+    mkdirSync(join(worktree, 'a-bulk'));
+    mkdirSync(join(worktree, 'z-later'));
+    for (let index = 0; index < 5100; index++) {
+      writeFileSync(join(worktree, 'a-bulk', `item-${index}.txt`), '');
+    }
+    writeFileSync(join(worktree, 'z-later', 'needle.ts'), 'x');
+    // The main checkout must play no part: this is the folder a walk seeded wrongly would list.
+    writeFileSync(join(repo, 'src', 'needle.ts'), 'main');
+
+    const data = await list('&root=live');
+    expect(data.root).toBe(worktree);
+    expect(data.truncated).toBe(true);
+    expect(names(data.tree)).toEqual(['a-bulk', 'src', 'z-later']);
+    const [bulk, , later] = data.tree;
+    expect(bulk).toMatchObject({ path: `${PREFIX}/a-bulk`, incomplete: true });
+    expect(bulk.children[0].path.startsWith(`${PREFIX}/a-bulk/`)).toBe(true);
+    expect(later).not.toHaveProperty('incomplete');
+    expect(later.children[0].path).toBe(`${PREFIX}/z-later/needle.ts`);
+
+    // A `/` in the query reads the listing's paths, and only the answer is in workingDir's terms.
+    const found = await list(`&root=live&q=${encodeURIComponent('z-later/*.ts')}`);
+    expect(found.root).toBe(worktree);
+    expect(found.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]);
+    expect(found.truncated).toBe(true);
+    const anywhere = await list('&root=live&q=needle');
+    expect(anywhere.matches.map((match: { path: string }) => match.path)).toEqual([`${PREFIX}/z-later/needle.ts`]);
+  });
+
   it('serves a listed path through the unchanged preview route', async () => {
     const path = `${PREFIX}/src/worktree-only.ts`;
     const res = await harness.app.inject({

test/routes/file-search-mode.test.ts

@@ -94,27 +94,138 @@ describe('files endpoint search mode', () => {
 
   it('shares the scan budget across sibling folders in both tree and search modes', async () => {
     const bulk = join(root, 'a-bulk');
+    const middle = join(root, 'm-middle');
     const later = join(root, 'z-later');
     mkdirSync(bulk);
+    mkdirSync(middle);
     mkdirSync(later);
     for (let index = 0; index < 5100; index++) {
       writeFileSync(join(bulk, `item-${index}.txt`), '');
     }
+    writeFileSync(join(middle, 'middle.ts'), 'x\n');
     writeFileSync(join(later, 'needle.ts'), 'x\n');
 
     try {
       const tree = await get('');
       const treePaths = tree.data.tree.map((entry: { path: string }) => entry.path);
       expect(treePaths).toContain('a-bulk');
+      expect(treePaths).toContain('m-middle');
+      expect(treePaths).toContain('src');
       expect(treePaths).toContain('z-later');
       expect(tree.data.truncated).toBe(true);
+      expect(tree.data.totalFiles + tree.data.totalDirectories).toBeLessThanOrEqual(5000);
+
+      const node = (path: string) => tree.data.tree.find((entry: { path: string }) => entry.path === path);
+      expect(node('a-bulk').incomplete).toBe(true);
+      expect(node('a-bulk').children.length).toBeGreaterThan(0);
+      // Finished folders do not carry the key at all, not even as false.
+      expect(node('z-later')).not.toHaveProperty('incomplete');
+      expect(node('z-later').children.map((entry: { path: string }) => entry.path)).toEqual([
+        join('z-later', 'needle.ts'),
+      ]);
+      expect(node('m-middle')).not.toHaveProperty('incomplete');
 
       const search = await get('?q=needle');
       expect(search.data.matches.map((match: { path: string }) => match.path)).toContain(join('z-later', 'needle.ts'));
       expect(search.data.truncated).toBe(true);
     } finally {
       rmSync(bulk, { recursive: true, force: true });
+      rmSync(middle, { recursive: true, force: true });
       rmSync(later, { recursive: true, force: true });
     }
   });
+
+  it('adds nothing to a listing that fits the budget', async () => {
+    const { data } = await get('');
+
+    expect(data.truncated).toBe(false);
+    expect(JSON.stringify(data)).not.toContain('incomplete');
+    const dir = data.tree.find((entry: { name: string }) => entry.name === 'src');
+    expect(Object.keys(dir)).toEqual(['name', 'path', 'type', 'children']);
+    const file = data.tree.find((entry: { name: string }) => entry.name === 'README.md');
+    expect(Object.keys(file)).toEqual(['name', 'path', 'type', 'size', 'extension']);
+  });
+
+  it('flags folders the budget ran out on part way through a round, read or not', async () => {
+    // 3,000 folders use 3,000 of the 5,000 in the first round. The 2,000 left then buy one file
+    // each, so the round ends with 1,000 folders never read: the last of those is the one a budget
+    // check after the shift would drop from the queue, and with it the flag.
+    const wide = mkdtempSync(join(tmpdir(), 'codeman-file-wide-'));
+    for (let index = 0; index < 3000; index++) {
+      const dir = join(wide, `dir-${String(index).padStart(4, '0')}`);
+      mkdirSync(dir);
+      writeFileSync(join(dir, 'one.txt'), '');
+      writeFileSync(join(dir, 'two.txt'), '');
+    }
+    harness.ctx._session.workingDir = wide;
+
+    try {
+      const { data } = await get('');
+      expect(data.truncated).toBe(true);
+      expect(data.tree).toHaveLength(3000);
+      expect(data.tree.filter((node: { incomplete?: boolean }) => node.incomplete === true)).toHaveLength(3000);
+      const sizes = data.tree.map((node: { children: unknown[] }) => node.children.length);
+      expect(sizes.filter((size: number) => size === 1)).toHaveLength(2000);
+      expect(sizes.filter((size: number) => size === 0)).toHaveLength(1000);
+    } finally {
+      rmSync(wide, { recursive: true, force: true });
+    }
+  });
+
+  it('copes with the root itself being unfinished', async () => {
+    // The root has no node to flag. Its folders are listed before its files, and none is read.
+    const wide = mkdtempSync(join(tmpdir(), 'codeman-file-root-'));
+    mkdirSync(join(wide, 'first'));
+    writeFileSync(join(wide, 'first', 'inside.txt'), '');
+    for (let index = 0; index < 5100; index++) {
+      writeFileSync(join(wide, `item-${index}.txt`), '');
+    }
+    harness.ctx._session.workingDir = wide;
+
+    try {
+      const { data } = await get('');
+      expect(data.truncated).toBe(true);
+      expect(data.tree).toHaveLength(5000);
+      expect(data.tree[0]).toMatchObject({ name: 'first', incomplete: true, children: [] });
+      expect(data.tree.filter((node: { incomplete?: boolean }) => node.incomplete)).toHaveLength(1);
+    } finally {
+      rmSync(wide, { recursive: true, force: true });
+    }
+  });
+
+  it('returns search results shallow-first', async () => {
+    const deep = join(root, 'a-deep', 'b', 'c');
+    mkdirSync(deep, { recursive: true });
+    mkdirSync(join(root, 'z-shallow'));
+    writeFileSync(join(deep, 'order-probe.ts'), 'x\n');
+    writeFileSync(join(root, 'z-shallow', 'order-probe.ts'), 'x\n');
+
+    try {
+      const { data } = await get('?q=order-probe');
+      expect(data.matches.map((match: { path: string }) => match.path)).toEqual([
+        join('z-shallow', 'order-probe.ts'),
+        join('a-deep', 'b', 'c', 'order-probe.ts'),
+      ]);
+    } finally {
+      rmSync(join(root, 'a-deep'), { recursive: true, force: true });
+      rmSync(join(root, 'z-shallow'), { recursive: true, force: true });
+    }
+  });
+
+  it('stops at the match limit and says so', async () => {
+    const many = mkdtempSync(join(tmpdir(), 'codeman-file-many-'));
+    for (let index = 0; index < 1100; index++) {
+      writeFileSync(join(many, `hit-${index}.txt`), '');
+    }
+    harness.ctx._session.workingDir = many;
+
+    try {
+      const { data } = await get('?q=hit-');
+      expect(data.matches).toHaveLength(1000);
+      expect(data.matchCount).toBe(1000);
+      expect(data.truncated).toBe(true);
+    } finally {
+      rmSync(many, { recursive: true, force: true });
+    }
+  });
 });