mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-13 18:29:32 +01:00
Codeman maintainer
f62c1f7e2f
The install window polls GET /api/local-model every second. A read of the
state file that was still out while the job wrote its last state and let
go of this.job came back with the old `installing`, and settling that with
no job in the process read it as an interrupted job: `paused` with Resume
for one tick before `failed`. Under CI load this made
test/local-model-install-e2e.test.ts fail on a different failure case per
run (master 86502b69, PR #790).
currentState() now reads again when the job ended during the read. The job
writes before it lets go, so the second read has the final state.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2642 lines
112 KiB
TypeScript
2642 lines
112 KiB
TypeScript
/**
|
|
* @fileoverview The service that drives the local model install job
|
|
* (`createLocalModelService()` in src/local-model/install-job.ts).
|
|
*
|
|
* Every dependency is a fake: no runtime is downloaded or started, no model is pulled, no
|
|
* installer is run and no port is bound. The runtime modules are replaced through the
|
|
* service's own dependency object, so these tests pin what the JOB does with their answers,
|
|
* not what they answer. The state lives in memory and is passed through the real parser on
|
|
* every write, so a state the state file would not read back fails here.
|
|
*
|
|
* What is pinned: the whole job from the click to "installed"; one job at a time (409); the
|
|
* consent bound to the size shown (409); what is checked before anything is downloaded;
|
|
* Cancel in every step; Resume; a job a server restart interrupted comes back paused and
|
|
* waits for a click; every failure is a state with a code and a next step; the coding agent
|
|
* install (the stock command, the audit line, the resolver caches forgotten, still missing is
|
|
* a failure); the digest that refuses a changed model; the runtime verbs; Remove.
|
|
*
|
|
* Port: N/A (nothing listens).
|
|
*/
|
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
import { dirname } from 'node:path';
|
|
import { getDataDir } from '../src/config/instance.js';
|
|
import { customModelHostsPath, readCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js';
|
|
import { connectLocalModel, disconnectLocalModel, inspectLocalEndpoint } from '../src/local-model/connect.js';
|
|
import {
|
|
createLocalModelService,
|
|
type LocalModelActor,
|
|
type LocalModelService,
|
|
type LocalModelServiceDeps,
|
|
} from '../src/local-model/install-job.js';
|
|
import type { PullOptions } from '../src/local-model/ollama-client.js';
|
|
import type { LocalModelPaths } from '../src/local-model/paths.js';
|
|
import {
|
|
OLLAMA_RUNTIME_PIN,
|
|
type RuntimeStartOptions,
|
|
type RuntimeStartResult,
|
|
type RuntimeStatus,
|
|
} from '../src/local-model/runtime.js';
|
|
import { IDLE_LOCAL_MODEL_STATE, parseLocalModelState } from '../src/local-model/state-store.js';
|
|
import { SETTINGS_PATH } from '../src/web/route-helpers.js';
|
|
import {
|
|
LOCAL_MODEL_INSTALL_LABELS,
|
|
type Candidate,
|
|
type HardwareFacts,
|
|
type HarnessFact,
|
|
type LocalModelCatalog,
|
|
type LocalModelInstallRequest,
|
|
type LocalModelInstallStep,
|
|
type LocalModelStateFile,
|
|
type Recommendation,
|
|
} from '../src/types.js';
|
|
|
|
// ---------- Fixtures ----------
|
|
|
|
const DIGEST = '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba';
|
|
const PULL_BYTES = 23_105_798_956;
|
|
const RUNTIME_BYTES = OLLAMA_RUNTIME_PIN.bytes;
|
|
const MEASURED = 40_157_944_217;
|
|
const INSTALL_COMMAND = 'curl -fsSL https://opencode.ai/install | bash';
|
|
const ADMIN: LocalModelActor = { username: 'root', ip: '127.0.0.1' };
|
|
|
|
const MAC: HardwareFacts = {
|
|
platform: 'darwin',
|
|
arch: 'arm64',
|
|
arm64: true,
|
|
memBytes: 51539607552,
|
|
memGb: '51.5',
|
|
memClassGb: 48,
|
|
chip: 'Apple M5 Pro',
|
|
source: 'sysctl',
|
|
};
|
|
|
|
/** Only what the service reads of a catalogue: the harness rows and the models' ids and labels. */
|
|
const CATALOG = {
|
|
schema: 1,
|
|
updated: '2026-11-06',
|
|
minMacosMajor: 14,
|
|
models: [
|
|
{ id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B' },
|
|
{ id: 'gpt-oss:20b', label: 'GPT-OSS 20B' },
|
|
{ id: 'qwen3.5:9b', label: 'Qwen3.5 9B' },
|
|
],
|
|
harnesses: [
|
|
{ cli: 'opencode', minContext: 65536, rank: 3 },
|
|
{ cli: 'claude', minContext: 65536, rank: 2 },
|
|
],
|
|
chips: [],
|
|
} as unknown as LocalModelCatalog;
|
|
|
|
const FACTS: HarnessFact[] = [
|
|
{
|
|
cli: 'opencode',
|
|
label: 'OpenCode',
|
|
installed: false,
|
|
installable: true,
|
|
installCommand: INSTALL_COMMAND,
|
|
acceptsModelId: () => true,
|
|
},
|
|
];
|
|
|
|
function candidate(
|
|
modelId: string,
|
|
modelLabel: string,
|
|
context: number,
|
|
bytes: number,
|
|
installed: boolean,
|
|
needBytes = 36_000_000_000
|
|
): Candidate {
|
|
return {
|
|
modelId,
|
|
modelLabel,
|
|
license: 'Apache-2.0',
|
|
paramsB: 20,
|
|
context,
|
|
cli: 'opencode',
|
|
cliLabel: 'OpenCode',
|
|
cliInstalled: installed,
|
|
needBytes,
|
|
needGb: '0.0',
|
|
downloadBytes: bytes,
|
|
downloadGb: '0.0',
|
|
budgetShare: 0.8,
|
|
fit: 'good',
|
|
fitLabel: 'Good',
|
|
speed: 'unknown',
|
|
tested: false,
|
|
harnesses: [
|
|
{
|
|
cli: 'opencode',
|
|
cliLabel: 'OpenCode',
|
|
installed,
|
|
...(installed ? {} : { installCommand: INSTALL_COMMAND }),
|
|
},
|
|
],
|
|
installBytes: bytes + RUNTIME_BYTES,
|
|
installGb: '0.0',
|
|
} as unknown as Candidate;
|
|
}
|
|
|
|
function recommendation(agentInstalled = false): Recommendation {
|
|
return {
|
|
verdict: 'ok',
|
|
headline: { text: 'Best fit for this Mac' },
|
|
notes: [],
|
|
pick: candidate('qwen3.6:35b-a3b', 'Qwen3.6 35B-A3B', 131072, PULL_BYTES, agentInstalled),
|
|
// Needs less memory than the pick, so it is what "does not fit" offers next.
|
|
alternatives: [candidate('gpt-oss:20b', 'GPT-OSS 20B', 131072, 13_793_441_427, agentInstalled, 26_600_000_000)],
|
|
budgetBytes: 37_580_963_840,
|
|
budgetGb: '37.6',
|
|
budgetSource: 'estimated',
|
|
} as unknown as Recommendation;
|
|
}
|
|
|
|
const REQUEST: LocalModelInstallRequest = {
|
|
modelId: 'qwen3.6:35b-a3b',
|
|
cli: 'opencode',
|
|
context: 131072,
|
|
acknowledgedBytes: PULL_BYTES + RUNTIME_BYTES,
|
|
};
|
|
|
|
const PATHS: LocalModelPaths = {
|
|
root: '/data/local-model',
|
|
runtimeRoot: '/data/local-model/runtime',
|
|
runtimeDir: '/data/local-model/runtime/ollama-0.40.15',
|
|
binary: '/data/local-model/runtime/ollama-0.40.15/ollama',
|
|
archivePartial: '/data/local-model/runtime/ollama-0.40.15.tgz.partial',
|
|
unpackDir: '/data/local-model/runtime/ollama-0.40.15.unpack',
|
|
modelsDir: '/data/local-model/models',
|
|
homeDir: '/data/local-model/home',
|
|
stateFile: '/data/local-model.json',
|
|
};
|
|
|
|
const INSTALLED: LocalModelStateFile = {
|
|
schema: 1,
|
|
state: 'installed',
|
|
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072, pullBytes: PULL_BYTES, digest: DIGEST },
|
|
cli: { id: 'opencode', label: 'OpenCode' },
|
|
runtime: { version: '0.40.15', port: 11435, pid: 4711, enabled: true, measuredTotalBytes: MEASURED },
|
|
loaded: { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 },
|
|
connection: { endpointId: 'local-ollama', flagWasEnabled: false },
|
|
};
|
|
|
|
function jobAt(step: LocalModelInstallStep, state: 'installing' | 'paused' = 'installing'): LocalModelStateFile {
|
|
return {
|
|
schema: 1,
|
|
state,
|
|
step,
|
|
steps: ['runtime', 'model', 'verify', 'agent', 'connect'],
|
|
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 },
|
|
cli: { id: 'opencode', label: 'OpenCode' },
|
|
...(step === 'runtime' ? {} : { runtime: { version: '0.40.15', port: 11435, pid: 5000, enabled: true } }),
|
|
};
|
|
}
|
|
|
|
// ---------- The harness ----------
|
|
|
|
/** What a write to a full disk rejects with. */
|
|
function noSpace(): Error {
|
|
return Object.assign(new Error('ENOSPC: no space left on device, write'), { code: 'ENOSPC' });
|
|
}
|
|
|
|
/**
|
|
* The state file in memory. Every write goes through the real parser, like a write to disk and
|
|
* a read back. `refuse` makes the disk full: while it answers true for a write, that write
|
|
* rejects with ENOSPC and the file keeps what it had, like the real temp file plus rename.
|
|
*/
|
|
function memoryStore(initial: LocalModelStateFile = IDLE_LOCAL_MODEL_STATE) {
|
|
let state = initial;
|
|
let chain: Promise<unknown> = Promise.resolve();
|
|
const history: LocalModelStateFile[] = [];
|
|
const disk = {
|
|
refuse: undefined as ((next: LocalModelStateFile, before: LocalModelStateFile) => boolean) | undefined,
|
|
};
|
|
return {
|
|
get: () => state,
|
|
history,
|
|
disk,
|
|
readState: async () => state,
|
|
mutateState: (fn: (current: LocalModelStateFile) => LocalModelStateFile | undefined) => {
|
|
const run = chain.then(async () => {
|
|
const before = state;
|
|
const next = fn(before);
|
|
if (!next) return { before, after: undefined };
|
|
if (disk.refuse?.(next, before)) throw noSpace();
|
|
state = parseLocalModelState(JSON.stringify(next));
|
|
history.push(state);
|
|
return { before, after: state };
|
|
});
|
|
chain = run.catch(() => {});
|
|
return run;
|
|
},
|
|
};
|
|
}
|
|
|
|
/** Resolves when `signal` aborts. What a step in flight waits on in a Cancel test. */
|
|
function untilAborted(signal?: AbortSignal): Promise<void> {
|
|
return new Promise((resolve) => {
|
|
if (!signal || signal.aborted) return resolve();
|
|
signal.addEventListener('abort', () => resolve(), { once: true });
|
|
});
|
|
}
|
|
|
|
function setup(
|
|
options: {
|
|
state?: LocalModelStateFile;
|
|
agentInstalled?: boolean;
|
|
runtimeOnDisk?: boolean;
|
|
overrides?: Partial<LocalModelServiceDeps>;
|
|
} = {}
|
|
) {
|
|
const store = memoryStore(options.state);
|
|
const world = {
|
|
agentInstalled: options.agentInstalled ?? false,
|
|
runtimeOnDisk: options.runtimeOnDisk ?? false,
|
|
clock: 1_000_000,
|
|
startResult: undefined as RuntimeStartResult | undefined,
|
|
measured: MEASURED as number | undefined,
|
|
running: false,
|
|
gaveUp: false,
|
|
pid: 5000,
|
|
};
|
|
const calls: string[] = [];
|
|
|
|
const supervisor = {
|
|
start: vi.fn(async (start: RuntimeStartOptions): Promise<RuntimeStartResult> => {
|
|
calls.push('supervisor.start');
|
|
if (world.startResult && !world.startResult.ok) {
|
|
world.running = false;
|
|
return world.startResult;
|
|
}
|
|
world.running = true;
|
|
return {
|
|
ok: true,
|
|
pid: world.pid,
|
|
port: start.port,
|
|
orphan: 'none',
|
|
...(world.measured !== undefined ? { measuredTotalBytes: world.measured } : {}),
|
|
};
|
|
}),
|
|
stop: vi.fn(async (_options?: { previousPid?: number }) => {
|
|
calls.push('supervisor.stop');
|
|
world.running = false;
|
|
}),
|
|
status: (): RuntimeStatus => ({ running: world.running, restarts: 0, gaveUp: world.gaveUp }),
|
|
};
|
|
|
|
const deps = {
|
|
readHardware: vi.fn(async () => MAC),
|
|
getCatalog: vi.fn(() => CATALOG),
|
|
buildHarnessFacts: vi.fn(async () => FACTS),
|
|
recommend: vi.fn(() => recommendation(world.agentInstalled)),
|
|
modelFitsBudget: vi.fn(() => true),
|
|
isRuntimeInstalled: vi.fn(async () => world.runtimeOnDisk),
|
|
ensureRuntimeInstalled: vi.fn(
|
|
async (install?: { onProgress?: (p: { completedBytes: number; totalBytes: number }) => void }) => {
|
|
calls.push('ensureRuntimeInstalled');
|
|
install?.onProgress?.({ completedBytes: RUNTIME_BYTES, totalBytes: RUNTIME_BYTES });
|
|
world.runtimeOnDisk = true;
|
|
return { ok: true as const, binaryPath: PATHS.binary, alreadyInstalled: false };
|
|
}
|
|
),
|
|
chooseRuntimePort: vi.fn(async () => 11435 as number | undefined),
|
|
removeLocalModelFiles: vi.fn(async () => {
|
|
calls.push('removeLocalModelFiles');
|
|
world.runtimeOnDisk = false;
|
|
}),
|
|
supervisor,
|
|
pullModel: vi.fn(async (_target: unknown, _id: string, pull?: { onProgress?: (p: unknown) => void }) => {
|
|
calls.push('pullModel');
|
|
pull?.onProgress?.({ completedBytes: PULL_BYTES, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 });
|
|
return { ok: true as const, totalBytes: PULL_BYTES };
|
|
}),
|
|
verifyModel: vi.fn(async () => {
|
|
calls.push('verifyModel');
|
|
return {
|
|
ok: true as const,
|
|
loaded: { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 },
|
|
};
|
|
}),
|
|
readModelDigest: vi.fn(async () => DIGEST as string | undefined),
|
|
// The runtime lists what was pulled: the model of the job.
|
|
listOpenAiModels: vi.fn(async () => [store.get().model?.id ?? '']),
|
|
listRunning: vi.fn(async () => [
|
|
{ name: 'qwen3.6:35b-a3b', size: 26_300_000_000, sizeVram: 26_300_000_000, contextLength: 32768 },
|
|
]),
|
|
getRuntimeVersion: vi.fn(async () => '0.40.15' as string | undefined),
|
|
deleteModel: vi.fn(async () => {
|
|
calls.push('deleteModel');
|
|
return true;
|
|
}),
|
|
removePartialModelBlobs: vi.fn(async () => {
|
|
calls.push('removePartialModelBlobs');
|
|
return 0;
|
|
}),
|
|
checkDeveloperTools: vi.fn(async () => 'missing' as const),
|
|
stockInstallCommand: vi.fn((id: string) => (id === FACTS[0].cli ? INSTALL_COMMAND : undefined)),
|
|
tryBeginCliInstall: vi.fn((): (() => void) | null => () => {
|
|
calls.push('release');
|
|
}),
|
|
runInstallCommand: vi.fn(async () => {
|
|
calls.push('runInstallCommand');
|
|
world.agentInstalled = true;
|
|
return { code: 0 as number | null, output: 'opencode installed\n', timedOut: false, aborted: false };
|
|
}),
|
|
recheckCliInstalled: vi.fn(async () => {
|
|
calls.push('recheckCliInstalled');
|
|
return world.agentInstalled;
|
|
}),
|
|
audit: vi.fn(async () => {}),
|
|
// Nothing saved under the reserved id, and the endpoints setting is off: a first install.
|
|
inspectLocalEndpoint: vi.fn(async () => ({ record: 'absent' as 'absent' | 'same' | 'other', flagEnabled: false })),
|
|
connectLocalModel: vi.fn(async () => {
|
|
calls.push('connectLocalModel');
|
|
return { ok: true as const, flagWasEnabled: false };
|
|
}),
|
|
moveLocalEndpoint: vi.fn(async () => {}),
|
|
ensureLocalEndpoint: vi.fn(async () => false),
|
|
disconnectLocalModel: vi.fn(async () => {
|
|
calls.push('disconnectLocalModel');
|
|
}),
|
|
isEndpointIdTaken: vi.fn(async () => false),
|
|
readState: store.readState,
|
|
mutateState: store.mutateState,
|
|
pin: OLLAMA_RUNTIME_PIN,
|
|
paths: PATHS,
|
|
now: () => world.clock,
|
|
...options.overrides,
|
|
};
|
|
|
|
const service = createLocalModelService(deps as unknown as Partial<LocalModelServiceDeps>);
|
|
return { service, store, world, deps, supervisor, calls };
|
|
}
|
|
|
|
/** Starts the job and waits for its end. */
|
|
async function installAndWait(service: LocalModelService, request = REQUEST) {
|
|
const result = await service.install(request, ADMIN);
|
|
await service.idle();
|
|
return result;
|
|
}
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
// ---------- The whole job ----------
|
|
|
|
describe('a job from the click to installed', () => {
|
|
it('runs the five steps in order and ends installed, with every fact a later start needs', async () => {
|
|
const { service, store, calls, deps } = setup();
|
|
const started = await service.install(REQUEST, ADMIN);
|
|
// The verb answers once the job is started: the work goes on in the server.
|
|
expect(started.ok && started.view.state).toBe('installing');
|
|
await service.idle();
|
|
|
|
expect(calls).toEqual([
|
|
'ensureRuntimeInstalled',
|
|
'supervisor.start',
|
|
'supervisor.start',
|
|
'pullModel',
|
|
'supervisor.start',
|
|
'verifyModel',
|
|
'recheckCliInstalled',
|
|
'runInstallCommand',
|
|
'release',
|
|
'recheckCliInstalled',
|
|
'supervisor.start',
|
|
'connectLocalModel',
|
|
]);
|
|
expect(store.get()).toMatchObject({
|
|
schema: 1,
|
|
state: 'installed',
|
|
model: {
|
|
id: 'qwen3.6:35b-a3b',
|
|
label: 'Qwen3.6 35B-A3B',
|
|
context: 131072,
|
|
pullBytes: PULL_BYTES,
|
|
digest: DIGEST,
|
|
},
|
|
cli: { id: 'opencode', label: 'OpenCode' },
|
|
runtime: { version: '0.40.15', port: 11435, pid: 5000, enabled: true, measuredTotalBytes: MEASURED },
|
|
loaded: { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 },
|
|
connection: { endpointId: 'local-ollama', flagWasEnabled: false },
|
|
});
|
|
// A job's own members are gone once it is installed.
|
|
for (const key of ['step', 'steps', 'progress', 'error']) expect(store.get()).not.toHaveProperty(key);
|
|
|
|
// The runtime was started on the chosen port with the chosen context, every time.
|
|
for (const [start] of deps.supervisor.start.mock.calls) {
|
|
expect(start).toMatchObject({ port: 11435, contextLength: 131072 });
|
|
}
|
|
// The connection names the model as the runtime lists it, at the chosen context. It was
|
|
// claimed in the state file first, so the record is written as ours.
|
|
expect(deps.connectLocalModel).toHaveBeenCalledWith({
|
|
port: 11435,
|
|
modelId: 'qwen3.6:35b-a3b',
|
|
context: 131072,
|
|
ours: true,
|
|
flagWasEnabled: false,
|
|
});
|
|
});
|
|
|
|
it('walks the steps one at a time in the state file', async () => {
|
|
const { service, store } = setup();
|
|
await installAndWait(service);
|
|
const steps = store.history.map((state) => `${state.state}:${state.step ?? '-'}`);
|
|
const order = steps.filter((entry, index) => entry !== steps[index - 1]);
|
|
expect(order).toEqual([
|
|
'installing:runtime',
|
|
'installing:model',
|
|
'installing:verify',
|
|
'installing:agent',
|
|
'installing:connect',
|
|
'installed:-',
|
|
]);
|
|
});
|
|
|
|
it('the view of the installed model: the Done facts, the endpoint, and a session can be started', async () => {
|
|
const { service } = setup();
|
|
await installAndWait(service);
|
|
const view = await service.view();
|
|
expect(view).toMatchObject({
|
|
state: 'installed',
|
|
running: true,
|
|
developerTools: 'missing',
|
|
model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B' },
|
|
cli: 'opencode',
|
|
cliLabel: 'OpenCode',
|
|
context: 131072,
|
|
port: 11435,
|
|
loaded: { context: 131072, memoryBytes: 26_300_000_000, memoryGb: '26.3', onGpu: true },
|
|
endpointId: 'local-ollama',
|
|
actions: ['start-session', 'stop', 'remove'],
|
|
folder: '/data/local-model',
|
|
runtime: { version: '0.40.15', installed: true, downloadBytes: 0 },
|
|
});
|
|
expect(view).not.toHaveProperty('error');
|
|
expect(await service.launchRefusal('local-ollama')).toBeUndefined();
|
|
});
|
|
|
|
it('a coding agent that is installed is not part of the job: nothing is run for it', async () => {
|
|
const { service, store, deps } = setup({ agentInstalled: true });
|
|
await installAndWait(service);
|
|
expect(store.get().state).toBe('installed');
|
|
expect(store.history.some((state) => state.step === 'agent')).toBe(false);
|
|
expect(deps.runInstallCommand).not.toHaveBeenCalled();
|
|
expect(deps.recheckCliInstalled).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('judges the model against the MEASURED budget right after the runtime started, before the pull', async () => {
|
|
const { service, deps, calls } = setup();
|
|
await installAndWait(service);
|
|
expect(deps.modelFitsBudget).toHaveBeenCalledTimes(1);
|
|
expect(deps.modelFitsBudget).toHaveBeenCalledWith(MAC, CATALOG, 'qwen3.6:35b-a3b', 131072, {
|
|
measuredTotalBytes: MEASURED,
|
|
runtimeDownloadBytes: 0,
|
|
});
|
|
expect(deps.modelFitsBudget.mock.invocationCallOrder[0]).toBeLessThan(deps.pullModel.mock.invocationCallOrder[0]);
|
|
expect(calls.indexOf('pullModel')).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('from then on the GET recommends with the measured budget and no runtime download', async () => {
|
|
const { service } = setup();
|
|
expect(await service.recommendOptions()).toEqual({ runtimeDownloadBytes: RUNTIME_BYTES });
|
|
await installAndWait(service);
|
|
expect(await service.recommendOptions()).toEqual({ measuredTotalBytes: MEASURED, runtimeDownloadBytes: 0 });
|
|
});
|
|
|
|
it('every size shown for the model comes from the pull once it is known', async () => {
|
|
const pull = vi.fn(async () => ({ ok: true as const, totalBytes: 23_999_000_111 }));
|
|
const { service, store } = setup({ overrides: { pullModel: pull } });
|
|
await installAndWait(service);
|
|
expect(store.get().model?.pullBytes).toBe(23_999_000_111);
|
|
});
|
|
|
|
it('writes an audit line for the install and one for the coding agent, with the literal command', async () => {
|
|
const { service, deps } = setup();
|
|
await installAndWait(service);
|
|
expect(deps.audit).toHaveBeenCalledWith({
|
|
admin: 'root',
|
|
action: 'local_model_install',
|
|
target: 'qwen3.6:35b-a3b',
|
|
ip: '127.0.0.1',
|
|
detail: { cli: 'opencode', context: 131072, bytes: PULL_BYTES + RUNTIME_BYTES },
|
|
});
|
|
expect(deps.audit).toHaveBeenCalledWith({
|
|
admin: 'root',
|
|
action: 'cli_install',
|
|
target: 'opencode',
|
|
ip: '127.0.0.1',
|
|
detail: { command: INSTALL_COMMAND, exitCode: 0, timedOut: false, aborted: false, via: 'local-model' },
|
|
});
|
|
});
|
|
|
|
it('the port is chosen once: a job that already has one keeps it', async () => {
|
|
const { service, deps, store } = setup({
|
|
state: { ...jobAt('runtime', 'paused'), runtime: { version: '0.40.15', port: 11440, enabled: true } },
|
|
});
|
|
await service.resume({}, ADMIN);
|
|
await service.idle();
|
|
expect(deps.chooseRuntimePort).not.toHaveBeenCalled();
|
|
expect(store.get().runtime?.port).toBe(11440);
|
|
expect(deps.connectLocalModel).toHaveBeenCalledWith(expect.objectContaining({ port: 11440 }));
|
|
});
|
|
});
|
|
|
|
// ---------- What install checks first ----------
|
|
|
|
describe('install is checked before anything is downloaded', () => {
|
|
const nothingStarted = (h: ReturnType<typeof setup>) => {
|
|
expect(h.store.get()).toBe(IDLE_LOCAL_MODEL_STATE);
|
|
expect(h.deps.ensureRuntimeInstalled).not.toHaveBeenCalled();
|
|
expect(h.deps.pullModel).not.toHaveBeenCalled();
|
|
expect(h.supervisor.start).not.toHaveBeenCalled();
|
|
expect(h.deps.audit).not.toHaveBeenCalled();
|
|
};
|
|
|
|
it('consent is bound to the size shown: other bytes are 409 size-changed with the total', async () => {
|
|
const h = setup();
|
|
const result = await h.service.install({ ...REQUEST, acknowledgedBytes: PULL_BYTES }, ADMIN);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
status: 409,
|
|
conflict: {
|
|
reason: 'size-changed',
|
|
message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSizeChanged, vars: { size: '23.3' } },
|
|
expectedBytes: PULL_BYTES + RUNTIME_BYTES,
|
|
expectedGb: '23.3',
|
|
},
|
|
});
|
|
nothingStarted(h);
|
|
});
|
|
|
|
it('once the runtime is on disk the total is the model alone, so the earlier figure is refused', async () => {
|
|
const h = setup({ runtimeOnDisk: true });
|
|
h.deps.recommend.mockImplementation(() => {
|
|
const rec = recommendation();
|
|
for (const row of [rec.pick!, ...rec.alternatives]) row.installBytes = row.downloadBytes;
|
|
return rec;
|
|
});
|
|
const stale = await h.service.install(REQUEST, ADMIN);
|
|
expect(stale).toMatchObject({
|
|
ok: false,
|
|
status: 409,
|
|
conflict: { reason: 'size-changed', expectedBytes: PULL_BYTES },
|
|
});
|
|
nothingStarted(h);
|
|
const fresh = await h.service.install({ ...REQUEST, acknowledgedBytes: PULL_BYTES }, ADMIN);
|
|
expect(fresh.ok).toBe(true);
|
|
await h.service.idle();
|
|
});
|
|
|
|
it('a row without its own total is the model download plus the runtime download', async () => {
|
|
const h = setup();
|
|
h.deps.recommend.mockImplementation(() => {
|
|
const rec = recommendation();
|
|
delete (rec.pick as Partial<Candidate>).installBytes;
|
|
delete (rec.pick as Partial<Candidate>).harnesses;
|
|
return rec;
|
|
});
|
|
const wrong = await h.service.install({ ...REQUEST, acknowledgedBytes: PULL_BYTES }, ADMIN);
|
|
expect(wrong).toMatchObject({ ok: false, status: 409, conflict: { expectedBytes: PULL_BYTES + RUNTIME_BYTES } });
|
|
const right = await h.service.install(REQUEST, ADMIN);
|
|
expect(right.ok).toBe(true);
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('installed');
|
|
});
|
|
|
|
it.each([
|
|
['a model the server does not offer', { modelId: 'llama9:900b' }],
|
|
['an offered model at another context', { context: 65536 }],
|
|
['a coding agent the row does not list', { cli: 'claude' }],
|
|
['a coding agent that is no CLI at all', { cli: 'rm' }],
|
|
])('%s is a 400', async (_label, change) => {
|
|
const h = setup();
|
|
const result = await h.service.install({ ...REQUEST, ...change }, ADMIN);
|
|
expect(result).toMatchObject({ ok: false, status: 400 });
|
|
nothingStarted(h);
|
|
});
|
|
|
|
it('a coding agent the row lists but the catalogue does not have is a 400', async () => {
|
|
const h = setup();
|
|
h.deps.getCatalog.mockReturnValue({ ...CATALOG, harnesses: [] } as unknown as LocalModelCatalog);
|
|
expect(await h.service.install(REQUEST, ADMIN)).toMatchObject({ ok: false, status: 400 });
|
|
nothingStarted(h);
|
|
});
|
|
|
|
it('a missing coding agent without a stock install command is a 400', async () => {
|
|
const h = setup();
|
|
h.deps.stockInstallCommand.mockReturnValue(undefined);
|
|
const result = await h.service.install(REQUEST, ADMIN);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
status: 400,
|
|
message: { text: '{cli} is not installed and cannot be installed from here.', vars: { cli: 'OpenCode' } },
|
|
});
|
|
nothingStarted(h);
|
|
});
|
|
|
|
it('a saved endpoint that already has our id is said now, not after the download', async () => {
|
|
const h = setup();
|
|
h.deps.isEndpointIdTaken.mockResolvedValue(true);
|
|
const result = await h.service.install(REQUEST, ADMIN);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
status: 400,
|
|
message: {
|
|
text: 'A saved model connection named {id} already exists and was not created here. Rename or delete it in App Settings, then try again.',
|
|
vars: { id: 'local-ollama' },
|
|
},
|
|
});
|
|
nothingStarted(h);
|
|
});
|
|
|
|
it('when no model is recommended nothing can be installed', async () => {
|
|
const h = setup();
|
|
h.deps.recommend.mockReturnValue({
|
|
verdict: 'none',
|
|
reason: 'not-macos',
|
|
headline: { text: 'x' },
|
|
notes: [],
|
|
alternatives: [],
|
|
budgetBytes: 0,
|
|
budgetGb: '0.0',
|
|
budgetSource: 'estimated',
|
|
} as unknown as Recommendation);
|
|
expect(await h.service.install(REQUEST, ADMIN)).toMatchObject({ ok: false, status: 400 });
|
|
nothingStarted(h);
|
|
});
|
|
|
|
it('an alternative row installs like the pick', async () => {
|
|
const h = setup();
|
|
const result = await installAndWait(h.service, {
|
|
modelId: 'gpt-oss:20b',
|
|
cli: 'opencode',
|
|
context: 131072,
|
|
acknowledgedBytes: 13_793_441_427 + RUNTIME_BYTES,
|
|
});
|
|
expect(result.ok).toBe(true);
|
|
expect(h.store.get()).toMatchObject({ state: 'installed', model: { id: 'gpt-oss:20b', label: 'GPT-OSS 20B' } });
|
|
});
|
|
});
|
|
|
|
// ---------- One job at a time ----------
|
|
|
|
describe('one job at a time', () => {
|
|
it('while a job runs, a second install, a resume, a runtime action and a remove are 409 job-running', async () => {
|
|
const h = setup({
|
|
overrides: {
|
|
pullModel: vi.fn(async (_t: unknown, _id: string, pull?: { signal?: AbortSignal }) => {
|
|
await untilAborted(pull?.signal);
|
|
return { ok: false as const, code: 'cancelled' as const };
|
|
}),
|
|
},
|
|
});
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(h.store.get().step).toBe('model'));
|
|
|
|
const jobRunning = { ok: false, status: 409, conflict: { reason: 'job-running' } };
|
|
expect(await h.service.install(REQUEST, ADMIN)).toMatchObject(jobRunning);
|
|
expect(await h.service.resume({}, ADMIN)).toMatchObject(jobRunning);
|
|
expect(await h.service.runtime({ action: 'stop' })).toMatchObject(jobRunning);
|
|
expect(await h.service.remove({}, [])).toMatchObject(jobRunning);
|
|
expect(h.deps.ensureRuntimeInstalled).toHaveBeenCalledTimes(1);
|
|
expect(h.supervisor.stop).not.toHaveBeenCalled();
|
|
expect(h.deps.removeLocalModelFiles).not.toHaveBeenCalled();
|
|
|
|
await h.service.cancel();
|
|
});
|
|
|
|
it('two installs sent at the same moment: one starts, the other is refused', async () => {
|
|
const h = setup();
|
|
const [first, second] = await Promise.all([h.service.install(REQUEST, ADMIN), h.service.install(REQUEST, ADMIN)]);
|
|
expect(first.ok).toBe(true);
|
|
// The second finds the first's job running, or (it ran to its end already) a model installed.
|
|
expect(second).toMatchObject({ ok: false, status: 409 });
|
|
await h.service.idle();
|
|
expect(h.deps.ensureRuntimeInstalled).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('install while a model is installed is 409 already-installed: remove it first', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
expect(await h.service.install(REQUEST, ADMIN)).toEqual({
|
|
ok: false,
|
|
status: 409,
|
|
conflict: { reason: 'already-installed', message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictAlreadyInstalled } },
|
|
});
|
|
});
|
|
|
|
it.each([
|
|
['cancel with no job', (s: LocalModelService) => s.cancel(), IDLE_LOCAL_MODEL_STATE],
|
|
['cancel while paused', (s: LocalModelService) => s.cancel(), jobAt('model', 'paused')],
|
|
['cancel while installed', (s: LocalModelService) => s.cancel(), INSTALLED],
|
|
['resume with no job', (s: LocalModelService) => s.resume({}, ADMIN), IDLE_LOCAL_MODEL_STATE],
|
|
['resume while installed', (s: LocalModelService) => s.resume({}, ADMIN), INSTALLED],
|
|
[
|
|
'a runtime action with nothing installed',
|
|
(s: LocalModelService) => s.runtime({ action: 'start' }),
|
|
IDLE_LOCAL_MODEL_STATE,
|
|
],
|
|
[
|
|
'a runtime action on a paused job',
|
|
(s: LocalModelService) => s.runtime({ action: 'stop' }),
|
|
jobAt('model', 'paused'),
|
|
],
|
|
])('%s is 409 wrong-state and changes nothing', async (_label, verb, state) => {
|
|
const h = setup({ state });
|
|
expect(await verb(h.service)).toEqual({
|
|
ok: false,
|
|
status: 409,
|
|
conflict: { reason: 'wrong-state', message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictWrongState } },
|
|
});
|
|
expect(h.store.history).toEqual([]);
|
|
});
|
|
});
|
|
|
|
// ---------- Cancel ----------
|
|
|
|
describe('Cancel in every step', () => {
|
|
/** A fake that waits for the abort signal at position `at` of its arguments' options, then says cancelled. */
|
|
const blocked: Record<Exclude<LocalModelInstallStep, 'connect'>, Partial<LocalModelServiceDeps>> = {
|
|
runtime: {
|
|
ensureRuntimeInstalled: vi.fn(async (install?: { signal?: AbortSignal }) => {
|
|
await untilAborted(install?.signal);
|
|
return { ok: false as const, code: 'cancelled' as const };
|
|
}),
|
|
},
|
|
model: {
|
|
pullModel: vi.fn(async (_t: unknown, _id: string, pull?: { signal?: AbortSignal }) => {
|
|
await untilAborted(pull?.signal);
|
|
return { ok: false as const, code: 'cancelled' as const };
|
|
}),
|
|
},
|
|
verify: {
|
|
verifyModel: vi.fn(async (_t: unknown, _id: string, _ctx: number, signal?: AbortSignal) => {
|
|
await untilAborted(signal);
|
|
return { ok: false as const, code: 'cancelled' as const };
|
|
}),
|
|
},
|
|
agent: {
|
|
runInstallCommand: vi.fn(async (_command: string, run?: { signal?: AbortSignal }) => {
|
|
await untilAborted(run?.signal);
|
|
return { code: null, output: '', timedOut: false, aborted: true };
|
|
}),
|
|
},
|
|
} as unknown as Record<Exclude<LocalModelInstallStep, 'connect'>, Partial<LocalModelServiceDeps>>;
|
|
|
|
it.each(['runtime', 'model', 'verify', 'agent'] as const)(
|
|
'in the %s step: the step in flight is interrupted and the job is paused there',
|
|
async (step) => {
|
|
const h = setup({ overrides: blocked[step] });
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(h.store.get().step).toBe(step));
|
|
await vi.waitFor(() => expect(Object.values(blocked[step])[0]).toHaveBeenCalled());
|
|
|
|
const result = await h.service.cancel();
|
|
expect(result.ok && result.view).toMatchObject({
|
|
state: 'paused',
|
|
step,
|
|
actions: ['resume', 'install', 'remove'],
|
|
});
|
|
expect(h.store.get()).toMatchObject({ state: 'paused', step });
|
|
// Paused is not failed: no error was stored, and nothing went on after the step.
|
|
expect(h.store.get()).not.toHaveProperty('error');
|
|
expect(h.store.history.some((state) => state.state === 'failed')).toBe(false);
|
|
expect(h.deps.connectLocalModel).not.toHaveBeenCalled();
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('paused');
|
|
}
|
|
);
|
|
|
|
it('an interrupted step that answers with an error instead of "cancelled" is still a pause, not a failure', async () => {
|
|
const h = setup({
|
|
overrides: {
|
|
pullModel: vi.fn(async (_t: unknown, _id: string, pull?: { signal?: AbortSignal }) => {
|
|
await untilAborted(pull?.signal);
|
|
return { ok: false as const, code: 'pull-failed' as const, detail: 'socket hang up' };
|
|
}),
|
|
},
|
|
});
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(h.store.get().step).toBe('model'));
|
|
await vi.waitFor(() => expect(h.deps.pullModel).toHaveBeenCalled());
|
|
await h.service.cancel();
|
|
expect(h.store.get()).toMatchObject({ state: 'paused', step: 'model' });
|
|
expect(h.store.get()).not.toHaveProperty('error');
|
|
});
|
|
|
|
it('in the connect step, before the first write: honoured, and nothing was written', async () => {
|
|
const h = setup({
|
|
overrides: {
|
|
listOpenAiModels: vi.fn(async (_t: unknown, signal?: AbortSignal) => {
|
|
await untilAborted(signal);
|
|
return ['qwen3.6:35b-a3b'];
|
|
}),
|
|
},
|
|
});
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(h.deps.listOpenAiModels).toHaveBeenCalled());
|
|
const result = await h.service.cancel();
|
|
expect(result.ok && result.view.state).toBe('paused');
|
|
expect(h.store.get()).toMatchObject({ state: 'paused', step: 'connect' });
|
|
expect(h.deps.connectLocalModel).not.toHaveBeenCalled();
|
|
expect(h.store.get()).not.toHaveProperty('connection');
|
|
});
|
|
|
|
it('in the connect step, once the writes began: they finish and the job ends installed', async () => {
|
|
let release: () => void = () => {};
|
|
const writing = new Promise<void>((resolve) => (release = resolve));
|
|
const h = setup({
|
|
overrides: {
|
|
connectLocalModel: vi.fn(async () => {
|
|
await writing;
|
|
return { ok: true as const, flagWasEnabled: true };
|
|
}),
|
|
},
|
|
});
|
|
// What the claim keeps: the setting was on before this feature touched it.
|
|
h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'absent', flagEnabled: true });
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(h.deps.connectLocalModel).toHaveBeenCalled());
|
|
const cancelling = h.service.cancel();
|
|
release();
|
|
const result = await cancelling;
|
|
// The pair is never left half done: the answer is the view after the verb, installed.
|
|
expect(result.ok && result.view.state).toBe('installed');
|
|
expect(h.store.get()).toMatchObject({
|
|
state: 'installed',
|
|
connection: { endpointId: 'local-ollama', flagWasEnabled: true },
|
|
});
|
|
});
|
|
|
|
it('keeps the download as it stood, without a speed', async () => {
|
|
const h = setup({
|
|
overrides: {
|
|
pullModel: vi.fn(async (_t: unknown, _id: string, pull?: PullOptions) => {
|
|
pull?.onProgress?.({ completedBytes: 6_200_000_000, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 });
|
|
await untilAborted(pull?.signal);
|
|
return { ok: false as const, code: 'cancelled' as const };
|
|
}),
|
|
},
|
|
});
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(async () => expect((await h.service.view()).progress?.completedBytes).toBe(6_200_000_000));
|
|
// While it runs the window sees the live figure and the speed.
|
|
expect((await h.service.view()).progress).toEqual({
|
|
completedBytes: 6_200_000_000,
|
|
totalBytes: PULL_BYTES,
|
|
completedGb: '6.2',
|
|
totalGb: '23.1',
|
|
bytesPerSecond: 58_000_000,
|
|
mbPerSecond: '58',
|
|
secondsLeft: 292,
|
|
});
|
|
const result = await h.service.cancel();
|
|
expect(result.ok && result.view.progress).toEqual({
|
|
completedBytes: 6_200_000_000,
|
|
totalBytes: PULL_BYTES,
|
|
completedGb: '6.2',
|
|
totalGb: '23.1',
|
|
});
|
|
expect(h.store.get().progress).toEqual({ completedBytes: 6_200_000_000, totalBytes: PULL_BYTES });
|
|
});
|
|
|
|
it('a runtime that already runs keeps running after a Cancel', async () => {
|
|
const h = setup({ overrides: blocked.model });
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(h.store.get().step).toBe('model'));
|
|
await h.service.cancel();
|
|
expect(h.supervisor.stop).not.toHaveBeenCalled();
|
|
expect((await h.service.view()).running).toBe(true);
|
|
});
|
|
});
|
|
|
|
// ---------- Resume ----------
|
|
|
|
describe('Resume', () => {
|
|
it('re-enters the step the job stopped in and does not repeat the steps before it', async () => {
|
|
const h = setup({ state: jobAt('verify', 'paused'), runtimeOnDisk: true });
|
|
const result = await h.service.resume({}, ADMIN);
|
|
expect(result.ok && result.view.state).toBe('installing');
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('installed');
|
|
expect(h.deps.ensureRuntimeInstalled).not.toHaveBeenCalled();
|
|
expect(h.deps.pullModel).not.toHaveBeenCalled();
|
|
expect(h.deps.verifyModel).toHaveBeenCalledTimes(1);
|
|
// The step needs the runtime: it is started first, with the stored port and the old pid.
|
|
expect(h.supervisor.start.mock.calls[0][0]).toEqual({ port: 11435, contextLength: 131072, previousPid: 5000 });
|
|
});
|
|
|
|
it('"Try again" after a failure clears the error and runs the same step again', async () => {
|
|
const h = setup();
|
|
h.deps.verifyModel.mockResolvedValueOnce({
|
|
ok: false,
|
|
code: 'probe-unfinished',
|
|
detail: 'finish_reason length',
|
|
} as never);
|
|
await installAndWait(h.service);
|
|
expect(h.store.get()).toMatchObject({ state: 'failed', step: 'verify', error: { code: 'probe-unfinished' } });
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('installed');
|
|
expect(h.deps.pullModel).toHaveBeenCalledTimes(1);
|
|
expect(h.deps.verifyModel).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('a resumed agent step looks for the binary first and runs no installer when it is there', async () => {
|
|
const h = setup({ state: jobAt('agent', 'paused'), agentInstalled: true });
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('installed');
|
|
expect(h.deps.recheckCliInstalled).toHaveBeenCalledTimes(1);
|
|
expect(h.deps.runInstallCommand).not.toHaveBeenCalled();
|
|
expect(h.deps.audit).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('"Use another port" chooses a port that is not the stored one before it resumes', async () => {
|
|
const h = setup({
|
|
state: { ...jobAt('model', 'paused'), state: 'failed', error: { code: 'port-in-use', vars: { port: '11435' } } },
|
|
});
|
|
h.deps.chooseRuntimePort.mockResolvedValue(11436);
|
|
await h.service.resume({ newPort: true }, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.deps.chooseRuntimePort).toHaveBeenCalledWith({ avoid: [11435] });
|
|
expect(h.store.get()).toMatchObject({ state: 'installed', runtime: { port: 11436 } });
|
|
// The stored pid goes along: when it is still our own runtime on the OLD port, the start
|
|
// ends it instead of leaving a second runtime behind.
|
|
expect(h.supervisor.start.mock.calls[0][0]).toEqual({ port: 11436, contextLength: 131072, previousPid: 5000 });
|
|
expect(h.deps.connectLocalModel).toHaveBeenCalledWith(expect.objectContaining({ port: 11436 }));
|
|
});
|
|
});
|
|
|
|
// ---------- A server restart ----------
|
|
|
|
describe('a job a server restart interrupted', () => {
|
|
it('comes back paused at the same step, and nothing runs without a click', async () => {
|
|
const h = setup({ state: jobAt('model') });
|
|
const view = await h.service.view();
|
|
expect(view).toMatchObject({ state: 'paused', step: 'model', actions: ['resume', 'install', 'remove'] });
|
|
expect(h.store.get()).toMatchObject({ state: 'paused', step: 'model' });
|
|
await h.service.idle();
|
|
expect(h.calls).toEqual([]);
|
|
});
|
|
|
|
it('boot() does the same, and starts no runtime for a job', async () => {
|
|
const h = setup({ state: jobAt('verify') });
|
|
await h.service.boot();
|
|
expect(h.store.get()).toMatchObject({ state: 'paused', step: 'verify' });
|
|
expect(h.supervisor.start).not.toHaveBeenCalled();
|
|
// The runtime the killed server left behind is ended: nothing would use it until a click.
|
|
expect(h.calls).toEqual(['supervisor.stop']);
|
|
expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 5000 });
|
|
});
|
|
|
|
it('boot() ends nothing for a job that never started a runtime', async () => {
|
|
const h = setup({ state: jobAt('runtime') });
|
|
await h.service.boot();
|
|
expect(h.store.get()).toMatchObject({ state: 'paused', step: 'runtime' });
|
|
expect(h.calls).toEqual([]);
|
|
});
|
|
|
|
it.each(['install', 'cancel', 'resume', 'remove'] as const)(
|
|
'a %s sent before anything else finds it paused, not running',
|
|
async (verb) => {
|
|
const h = setup({ state: jobAt('model') });
|
|
const result =
|
|
verb === 'install'
|
|
? await h.service.install(REQUEST, ADMIN)
|
|
: verb === 'cancel'
|
|
? await h.service.cancel()
|
|
: verb === 'resume'
|
|
? await h.service.resume({}, ADMIN)
|
|
: await h.service.remove({}, []);
|
|
// A paused job accepts install, resume and remove; Cancel has nothing to cancel.
|
|
if (verb === 'cancel') expect(result).toMatchObject({ ok: false, conflict: { reason: 'wrong-state' } });
|
|
else expect(result.ok).toBe(true);
|
|
expect(h.store.history[0]).toMatchObject({ state: 'paused', step: 'model' });
|
|
await h.service.idle();
|
|
}
|
|
);
|
|
|
|
it('shutdown interrupts the job without a word in the state file, so the next start reads it as paused', async () => {
|
|
const pull = vi.fn(async (_t: unknown, _id: string, options?: { signal?: AbortSignal }) => {
|
|
await untilAborted(options?.signal);
|
|
return { ok: false as const, code: 'pull-failed' as const, detail: 'aborted' };
|
|
});
|
|
const h = setup({ overrides: { pullModel: pull } });
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(pull).toHaveBeenCalled());
|
|
|
|
await h.service.shutdown();
|
|
expect(h.supervisor.stop).toHaveBeenCalledTimes(1);
|
|
// Neither paused nor failed was written: the file still says installing.
|
|
expect(h.store.get()).toMatchObject({ state: 'installing', step: 'model' });
|
|
|
|
// The next server process: a new service over the same file.
|
|
const next = setup({ state: h.store.get() });
|
|
expect((await next.service.view()).state).toBe('paused');
|
|
expect(next.calls).toEqual([]);
|
|
});
|
|
|
|
it('paused, failed and installed stay what they were across a restart', async () => {
|
|
const failed: LocalModelStateFile = {
|
|
...jobAt('verify', 'paused'),
|
|
state: 'failed',
|
|
error: { code: 'no-tool-call' },
|
|
};
|
|
for (const state of [
|
|
jobAt('model', 'paused'),
|
|
failed,
|
|
{ ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } },
|
|
]) {
|
|
const h = setup({ state });
|
|
await h.service.boot();
|
|
expect(h.store.get()).toEqual(state);
|
|
expect(h.store.history).toEqual([]);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ---------- Boot of an installed model ----------
|
|
|
|
describe('boot with a model installed', () => {
|
|
it('starts the runtime on the stored port, with the stored pid to tell an orphan of ours, and reads the digest again', async () => {
|
|
const h = setup({ state: INSTALLED, runtimeOnDisk: true });
|
|
await h.service.boot();
|
|
expect(h.supervisor.start).toHaveBeenCalledWith({ port: 11435, contextLength: 131072, previousPid: 4711 });
|
|
expect(h.deps.readModelDigest).toHaveBeenCalledWith(
|
|
{ baseUrl: 'http://127.0.0.1:11435' },
|
|
'qwen3.6:35b-a3b',
|
|
expect.anything()
|
|
);
|
|
// The pid of the child that runs now replaces the stored one.
|
|
expect(h.store.get().runtime?.pid).toBe(5000);
|
|
const view = await h.service.view();
|
|
expect(view).toMatchObject({ state: 'installed', running: true, actions: ['start-session', 'stop', 'remove'] });
|
|
expect(view).not.toHaveProperty('error');
|
|
});
|
|
|
|
it('puts the saved endpoint back when it is gone (a config-backup restore), also for a stopped runtime', async () => {
|
|
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } });
|
|
h.deps.ensureLocalEndpoint.mockResolvedValue(true);
|
|
await h.service.boot();
|
|
expect(h.deps.ensureLocalEndpoint).toHaveBeenCalledWith({
|
|
port: 11435,
|
|
modelId: 'qwen3.6:35b-a3b',
|
|
context: 131072,
|
|
});
|
|
expect(warn.mock.calls.flat().join(' ')).toContain('was put back');
|
|
// The settings flag is not touched at boot: only the record.
|
|
expect(h.deps.connectLocalModel).not.toHaveBeenCalled();
|
|
expect(h.supervisor.start).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('writes no endpoint for an installation that never connected, nor for a job', async () => {
|
|
const { connection: _connection, ...unconnected } = INSTALLED;
|
|
for (const state of [unconnected, jobAt('connect', 'paused')]) {
|
|
const h = setup({ state });
|
|
await h.service.boot();
|
|
expect(h.deps.ensureLocalEndpoint).not.toHaveBeenCalled();
|
|
}
|
|
});
|
|
|
|
it('does not start a runtime the person stopped', async () => {
|
|
const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } });
|
|
await h.service.boot();
|
|
expect(h.supervisor.start).not.toHaveBeenCalled();
|
|
const view = await h.service.view();
|
|
expect(view).toMatchObject({ state: 'installed', running: false, actions: ['start', 'remove'] });
|
|
// A stopped runtime is not an error.
|
|
expect(view).not.toHaveProperty('error');
|
|
});
|
|
|
|
it('a model whose digest changed: the state stays installed, the view says why, and no session is offered', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
h.deps.readModelDigest.mockResolvedValue('ffff00000000');
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
await h.service.boot();
|
|
const view = await h.service.view();
|
|
expect(view.state).toBe('installed');
|
|
expect(view.error).toEqual({
|
|
code: 'model-changed',
|
|
message: { text: 'The model on this Mac changed since it was installed. Remove it and install it again.' },
|
|
next: 'remove',
|
|
});
|
|
expect(view.actions).toEqual(['stop', 'remove']);
|
|
expect(h.store.get().state).toBe('installed');
|
|
expect(h.store.get()).not.toHaveProperty('error');
|
|
});
|
|
|
|
it('another program on the port: port-in-use on the view, "Use another port" as the next step', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
h.world.startResult = { ok: false, code: 'port-in-use' };
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
await h.service.boot();
|
|
const view = await h.service.view();
|
|
expect(view.error).toEqual({
|
|
code: 'port-in-use',
|
|
message: { text: 'Port {port} is in use by another program.', vars: { port: '11435' } },
|
|
next: 'other-port',
|
|
});
|
|
expect(view.actions).toEqual(['start', 'remove']);
|
|
});
|
|
|
|
it('never throws, whatever the runtime does', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
h.supervisor.start.mockRejectedValue(new Error('spawn EACCES'));
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
await expect(h.service.boot()).resolves.toBeUndefined();
|
|
expect((await h.service.view()).running).toBe(false);
|
|
});
|
|
|
|
it('a new pid the supervisor reports after a crash and restart is stored', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
h.service.onRuntimeChange({ running: true, pid: 6001, restarts: 1, gaveUp: false });
|
|
await vi.waitFor(() => expect(h.store.get().runtime?.pid).toBe(6001));
|
|
// A report without a pid, or with what is already stored, writes nothing.
|
|
const writes = h.store.history.length;
|
|
h.service.onRuntimeChange({ running: false, restarts: 3, gaveUp: true });
|
|
h.service.onRuntimeChange({ running: true, pid: 6001, restarts: 1, gaveUp: false });
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
expect(h.store.history.length).toBe(writes);
|
|
});
|
|
});
|
|
|
|
// ---------- Failures ----------
|
|
|
|
describe('every failure is a state with a code and a next step', () => {
|
|
async function failed(h: ReturnType<typeof setup>) {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
await installAndWait(h.service);
|
|
const view = await h.service.view();
|
|
expect(view.state).toBe('failed');
|
|
expect(view.actions).toEqual(['resume', 'install', 'remove']);
|
|
return view;
|
|
}
|
|
|
|
it('a damaged runtime download: runtime-damaged, try again', async () => {
|
|
const h = setup();
|
|
h.deps.ensureRuntimeInstalled.mockResolvedValue({
|
|
ok: false,
|
|
code: 'runtime-damaged',
|
|
detail: 'sha256 mismatch',
|
|
} as never);
|
|
const view = await failed(h);
|
|
expect(view.step).toBe('runtime');
|
|
expect(view.error).toEqual({
|
|
code: 'runtime-damaged',
|
|
message: { text: 'The runtime download was damaged and has been deleted. Try again.' },
|
|
next: 'retry',
|
|
detail: 'sha256 mismatch',
|
|
});
|
|
expect(h.supervisor.start).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('no free port in the span: port-in-use with the first port, before a runtime is started', async () => {
|
|
const h = setup();
|
|
h.deps.chooseRuntimePort.mockResolvedValue(undefined);
|
|
const view = await failed(h);
|
|
expect(view.error).toMatchObject({ code: 'port-in-use', next: 'other-port', message: { vars: { port: '11435' } } });
|
|
expect(h.supervisor.start).not.toHaveBeenCalled();
|
|
expect(view).not.toHaveProperty('port');
|
|
});
|
|
|
|
it('another program on the chosen port: port-in-use with that port', async () => {
|
|
const h = setup();
|
|
h.world.startResult = { ok: false, code: 'port-in-use', detail: 'answers but is not ours' };
|
|
const view = await failed(h);
|
|
expect(view.step).toBe('runtime');
|
|
expect(view.port).toBe(11435);
|
|
expect(view.error).toMatchObject({ code: 'port-in-use', next: 'other-port', message: { vars: { port: '11435' } } });
|
|
});
|
|
|
|
it('a runtime that does not start: runtime-start-failed', async () => {
|
|
const h = setup();
|
|
h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'exited with code 1' };
|
|
const view = await failed(h);
|
|
expect(view.error).toMatchObject({ code: 'runtime-start-failed', next: 'retry', detail: 'exited with code 1' });
|
|
expect(h.deps.pullModel).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('the measured budget rules the model out: does-not-fit BEFORE the model is downloaded, with the next model', async () => {
|
|
const h = setup();
|
|
h.deps.modelFitsBudget.mockReturnValue(false);
|
|
const view = await failed(h);
|
|
expect(view.step).toBe('runtime');
|
|
expect(view.error).toEqual({
|
|
code: 'does-not-fit',
|
|
message: {
|
|
text: '{model} does not fit in the memory of this Mac after all.',
|
|
vars: { model: 'Qwen3.6 35B-A3B' },
|
|
},
|
|
next: 'next-model',
|
|
nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' },
|
|
});
|
|
expect(h.deps.pullModel).not.toHaveBeenCalled();
|
|
// The next model was asked for with the measured budget, not the estimate.
|
|
expect(h.deps.recommend.mock.calls.at(-1)?.[3]).toEqual({ measuredTotalBytes: MEASURED, runtimeDownloadBytes: 0 });
|
|
});
|
|
|
|
it('without a measured total (an adopted runtime printed nothing) the estimate stands and the job goes on', async () => {
|
|
const h = setup();
|
|
h.world.measured = undefined;
|
|
h.deps.modelFitsBudget.mockReturnValue(false);
|
|
await installAndWait(h.service);
|
|
expect(h.deps.modelFitsBudget).not.toHaveBeenCalled();
|
|
expect(h.store.get().state).toBe('installed');
|
|
});
|
|
|
|
it('a pull that keeps failing: pull-failed with the runtime own last error', async () => {
|
|
const h = setup();
|
|
h.deps.pullModel.mockResolvedValue({ ok: false, code: 'pull-failed', detail: 'tls: bad record MAC' } as never);
|
|
const view = await failed(h);
|
|
expect(view.step).toBe('model');
|
|
expect(view.error).toEqual({
|
|
code: 'pull-failed',
|
|
message: { text: LOCAL_MODEL_INSTALL_LABELS.errPullFailed },
|
|
next: 'retry',
|
|
detail: 'tls: bad record MAC',
|
|
});
|
|
});
|
|
|
|
it('a model the registry no longer has: model-not-found, and the next model to try', async () => {
|
|
const h = setup();
|
|
h.deps.pullModel.mockResolvedValue({ ok: false, code: 'model-not-found' } as never);
|
|
const view = await failed(h);
|
|
expect(view.error).toMatchObject({
|
|
code: 'model-not-found',
|
|
next: 'next-model',
|
|
message: { vars: { model: 'Qwen3.6 35B-A3B' } },
|
|
nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' },
|
|
});
|
|
});
|
|
|
|
it('a full disk: disk-full', async () => {
|
|
const h = setup();
|
|
h.deps.pullModel.mockResolvedValue({ ok: false, code: 'disk-full' } as never);
|
|
expect((await failed(h)).error).toMatchObject({ code: 'disk-full', next: 'retry' });
|
|
});
|
|
|
|
it.each([
|
|
['load-failed', 'retry', false],
|
|
['does-not-fit', 'next-model', true],
|
|
['no-tool-call', 'next-model', true],
|
|
['probe-unfinished', 'retry', false],
|
|
] as const)('the check answers %s: next step %s', async (code, next, offersModel) => {
|
|
const h = setup();
|
|
h.deps.verifyModel.mockResolvedValue({ ok: false, code, detail: 'detail' } as never);
|
|
const view = await failed(h);
|
|
expect(view.step).toBe('verify');
|
|
expect(view.error).toMatchObject({ code, next, message: { vars: { model: 'Qwen3.6 35B-A3B' } }, detail: 'detail' });
|
|
expect(view.error?.nextModel).toEqual(
|
|
offersModel ? { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' } : undefined
|
|
);
|
|
// Nothing after the check ran: no coding agent, no connection.
|
|
expect(h.deps.runInstallCommand).not.toHaveBeenCalled();
|
|
expect(h.deps.connectLocalModel).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('a model loaded at another context: the sentence carries what the runtime reports and what was chosen', async () => {
|
|
const h = setup();
|
|
h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'context-not-applied' } as never);
|
|
const view = await failed(h);
|
|
expect(view.error?.message).toEqual({
|
|
text: '{model} loaded with a context of {actual} tokens instead of {wanted}.',
|
|
vars: { model: 'Qwen3.6 35B-A3B', actual: '32768', wanted: '131072' },
|
|
});
|
|
});
|
|
|
|
it('a model the runtime reports no digest for cannot be installed: there would be nothing to compare later', async () => {
|
|
const h = setup();
|
|
h.deps.readModelDigest.mockResolvedValue(undefined);
|
|
const view = await failed(h);
|
|
expect(view.step).toBe('verify');
|
|
expect(view.error).toMatchObject({ code: 'load-failed', detail: 'The runtime reports no digest for the model.' });
|
|
});
|
|
|
|
it('a runtime that does not list the model for a coding agent: connect-failed, nothing written', async () => {
|
|
const h = setup();
|
|
h.deps.listOpenAiModels.mockResolvedValue(['some-other:1b']);
|
|
const view = await failed(h);
|
|
expect(view.step).toBe('connect');
|
|
expect(view.error).toMatchObject({ code: 'connect-failed', detail: 'The runtime does not list qwen3.6:35b-a3b.' });
|
|
expect(h.deps.connectLocalModel).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('a saved endpoint with our id that is not ours: endpoint-id-taken with the id, and nothing is claimed or written', async () => {
|
|
const h = setup();
|
|
h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'other', flagEnabled: true });
|
|
const view = await failed(h);
|
|
expect(view.error).toMatchObject({
|
|
code: 'endpoint-id-taken',
|
|
next: 'retry',
|
|
message: { vars: { id: 'local-ollama' } },
|
|
});
|
|
expect(h.store.get()).not.toHaveProperty('connection');
|
|
expect(h.deps.connectLocalModel).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('a refused write of the connection: connect-failed with the reason', async () => {
|
|
const h = setup();
|
|
h.deps.connectLocalModel.mockResolvedValue({
|
|
ok: false,
|
|
code: 'connect-failed',
|
|
detail: 'settings.json is not valid',
|
|
} as never);
|
|
expect((await failed(h)).error).toMatchObject({ code: 'connect-failed', detail: 'settings.json is not valid' });
|
|
});
|
|
|
|
it('an exception in a step is the code internal with its text, never an unhandled rejection', async () => {
|
|
const h = setup();
|
|
h.deps.pullModel.mockRejectedValue(new Error('socket exploded'));
|
|
const view = await failed(h);
|
|
expect(view.error).toEqual({
|
|
code: 'internal',
|
|
message: { text: 'The install stopped on an unexpected error.' },
|
|
next: 'retry',
|
|
detail: 'socket exploded',
|
|
});
|
|
});
|
|
|
|
it('a failed job can be replaced by another choice, and the model it left is deleted before the new pull', async () => {
|
|
const h = setup();
|
|
h.deps.verifyModel.mockResolvedValueOnce({ ok: false, code: 'no-tool-call' } as never);
|
|
await installAndWait(h.service);
|
|
expect(h.store.get().state).toBe('failed');
|
|
h.calls.length = 0;
|
|
|
|
// The runtime is on disk now, so the total is the model alone.
|
|
h.deps.recommend.mockImplementation(() => {
|
|
const rec = recommendation();
|
|
for (const row of [rec.pick!, ...rec.alternatives]) row.installBytes = row.downloadBytes;
|
|
return rec;
|
|
});
|
|
const result = await installAndWait(h.service, {
|
|
modelId: 'gpt-oss:20b',
|
|
cli: 'opencode',
|
|
context: 131072,
|
|
acknowledgedBytes: 13_793_441_427,
|
|
});
|
|
expect(result.ok).toBe(true);
|
|
expect(h.store.get()).toMatchObject({ state: 'installed', model: { id: 'gpt-oss:20b' } });
|
|
expect(h.deps.deleteModel).toHaveBeenCalledWith(
|
|
{ baseUrl: 'http://127.0.0.1:11435' },
|
|
'qwen3.6:35b-a3b',
|
|
expect.anything()
|
|
);
|
|
expect(h.calls.indexOf('deleteModel')).toBeLessThan(h.calls.indexOf('pullModel'));
|
|
expect(h.store.get()).not.toHaveProperty('replacedModelId');
|
|
// The port was chosen once, by the first job.
|
|
expect(h.deps.chooseRuntimePort).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|
|
|
|
// ---------- The coding agent ----------
|
|
|
|
describe('the coding agent step', () => {
|
|
it('forgets the cached lookups and looks for the binary again after the installer ran', async () => {
|
|
const h = setup();
|
|
await installAndWait(h.service);
|
|
// Once before (a resumed job may find it), once after: the second is what a session start relies on.
|
|
expect(h.deps.recheckCliInstalled).toHaveBeenCalledTimes(2);
|
|
expect(h.deps.recheckCliInstalled).toHaveBeenCalledWith('opencode');
|
|
const order = h.calls.filter((call) => call === 'recheckCliInstalled' || call === 'runInstallCommand');
|
|
expect(order).toEqual(['recheckCliInstalled', 'runInstallCommand', 'recheckCliInstalled']);
|
|
});
|
|
|
|
it('runs exactly the stock command, with the job signal, inside the shared install slot', async () => {
|
|
const h = setup();
|
|
await installAndWait(h.service);
|
|
expect(h.deps.stockInstallCommand).toHaveBeenCalledWith('opencode');
|
|
expect(h.deps.tryBeginCliInstall).toHaveBeenCalledWith('opencode');
|
|
expect(h.deps.runInstallCommand).toHaveBeenCalledTimes(1);
|
|
const [command, options] = h.deps.runInstallCommand.mock.calls[0] as unknown as [string, { signal: AbortSignal }];
|
|
expect(command).toBe(INSTALL_COMMAND);
|
|
expect(options.signal).toBeInstanceOf(AbortSignal);
|
|
// The slot is given back.
|
|
expect(h.calls).toContain('release');
|
|
});
|
|
|
|
it('an installer that ends well but leaves no binary is harness-failed: still missing is a failure', async () => {
|
|
const h = setup();
|
|
h.deps.runInstallCommand.mockResolvedValue({ code: 0, output: '', timedOut: false, aborted: false });
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
await installAndWait(h.service);
|
|
const view = await h.service.view();
|
|
expect(view).toMatchObject({ state: 'failed', step: 'agent' });
|
|
expect(view.error).toEqual({
|
|
code: 'harness-failed',
|
|
message: { text: '{cli} could not be installed.', vars: { cli: 'OpenCode' } },
|
|
next: 'retry',
|
|
detail: 'The installer finished, but the program was not found afterwards.',
|
|
});
|
|
expect(h.deps.connectLocalModel).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('an installer that fails is harness-failed with its last lines, and the caches are forgotten all the same', async () => {
|
|
const h = setup();
|
|
h.deps.runInstallCommand.mockResolvedValue({
|
|
code: 1,
|
|
output: `${'x'.repeat(2000)}\ncurl: (6) Could not resolve host`,
|
|
timedOut: false,
|
|
aborted: false,
|
|
});
|
|
await installAndWait(h.service);
|
|
const view = await h.service.view();
|
|
expect(view.error?.code).toBe('harness-failed');
|
|
expect(view.error?.detail?.endsWith('curl: (6) Could not resolve host')).toBe(true);
|
|
expect(view.error?.detail?.length).toBeLessThanOrEqual(1000);
|
|
expect(h.deps.recheckCliInstalled).toHaveBeenCalledTimes(2);
|
|
expect(h.deps.audit).toHaveBeenCalledWith(
|
|
expect.objectContaining({ action: 'cli_install', detail: expect.objectContaining({ exitCode: 1 }) })
|
|
);
|
|
});
|
|
|
|
it('an installer that timed out says so', async () => {
|
|
const h = setup();
|
|
h.deps.runInstallCommand.mockResolvedValue({ code: null, output: 'partial', timedOut: true, aborted: false });
|
|
await installAndWait(h.service);
|
|
expect((await h.service.view()).error).toMatchObject({
|
|
code: 'harness-failed',
|
|
detail: 'The installer was stopped after 300 seconds.',
|
|
});
|
|
});
|
|
|
|
it('an installer that left the binary although it ended badly counts as installed', async () => {
|
|
const h = setup();
|
|
h.deps.runInstallCommand.mockImplementation(async () => {
|
|
h.world.agentInstalled = true;
|
|
return { code: 1, output: 'could not edit .zshrc', timedOut: false, aborted: false };
|
|
});
|
|
await installAndWait(h.service);
|
|
expect(h.store.get().state).toBe('installed');
|
|
});
|
|
|
|
it('while the registry route installs the same CLI the step fails instead of running a second installer', async () => {
|
|
const h = setup();
|
|
h.deps.tryBeginCliInstall.mockReturnValue(null);
|
|
await installAndWait(h.service);
|
|
expect((await h.service.view()).error).toMatchObject({
|
|
code: 'harness-failed',
|
|
detail: 'Another install of this coding agent is running.',
|
|
});
|
|
expect(h.deps.runInstallCommand).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('never runs a command for a CLI that is not a coding agent of the catalogue', async () => {
|
|
const state: LocalModelStateFile = { ...jobAt('agent', 'paused'), cli: { id: 'grok', label: 'Grok Build' } };
|
|
const h = setup({ state });
|
|
h.deps.stockInstallCommand.mockReturnValue('curl https://example.invalid | bash');
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.store.get()).toMatchObject({ state: 'failed', error: { code: 'harness-failed' } });
|
|
expect(h.deps.runInstallCommand).not.toHaveBeenCalled();
|
|
expect(h.deps.tryBeginCliInstall).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('the audit line names who resumed the job, not who started it', async () => {
|
|
const h = setup({ state: jobAt('agent', 'paused') });
|
|
await h.service.resume({}, { username: 'second-admin', ip: '10.0.0.2' });
|
|
await h.service.idle();
|
|
expect(h.deps.audit).toHaveBeenCalledWith(
|
|
expect.objectContaining({ admin: 'second-admin', ip: '10.0.0.2', action: 'cli_install' })
|
|
);
|
|
});
|
|
});
|
|
|
|
// ---------- The launch check ----------
|
|
|
|
describe('launchRefusal: the digest and the runtime decide whether a session starts', () => {
|
|
async function installed(overrides: Partial<LocalModelStateFile> = {}) {
|
|
const h = setup({ state: { ...INSTALLED, ...overrides } });
|
|
await h.service.boot();
|
|
return h;
|
|
}
|
|
|
|
it('allows the launch while our runtime runs and the digest is the stored one', async () => {
|
|
const h = await installed();
|
|
h.deps.readModelDigest.mockClear();
|
|
expect(await h.service.launchRefusal('local-ollama')).toBeUndefined();
|
|
// Read again for this launch, not taken from the last look.
|
|
expect(h.deps.readModelDigest).toHaveBeenCalledTimes(1);
|
|
expect(await h.service.launchRefusalMessage('local-ollama', 'qwen3.6:35b-a3b')).toBeUndefined();
|
|
});
|
|
|
|
it('refuses a model whose digest is no longer the stored one, in one plain sentence', async () => {
|
|
const h = await installed();
|
|
h.deps.readModelDigest.mockResolvedValue('1111222233334444');
|
|
expect(await h.service.launchRefusal('local-ollama')).toEqual({
|
|
code: 'model-changed',
|
|
message: { text: 'The model on this Mac changed since it was installed. Remove it and install it again.' },
|
|
next: 'remove',
|
|
});
|
|
expect(await h.service.launchRefusalMessage('local-ollama', 'qwen3.6:35b-a3b')).toBe(
|
|
'The model on this Mac changed since it was installed. Remove it and install it again.'
|
|
);
|
|
// And the window stops offering a session.
|
|
expect((await h.service.view()).actions).toEqual(['stop', 'remove']);
|
|
});
|
|
|
|
it('a digest with the sha256 prefix some listings add is the same digest', async () => {
|
|
const h = await installed();
|
|
h.deps.readModelDigest.mockResolvedValue(`sha256:${DIGEST.toUpperCase()}`);
|
|
expect(await h.service.launchRefusal('local-ollama')).toBeUndefined();
|
|
});
|
|
|
|
it('refuses while the runtime is stopped: whatever answers on the port then is not ours', async () => {
|
|
const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } });
|
|
await h.service.boot();
|
|
expect(await h.service.launchRefusal('local-ollama')).toEqual({
|
|
code: 'runtime-not-running',
|
|
message: { text: 'The model on this Mac is not running. Start it first.' },
|
|
next: 'start',
|
|
});
|
|
expect(h.deps.readModelDigest).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('names another program on the port, and a runtime that keeps stopping', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const taken = setup({ state: INSTALLED });
|
|
taken.world.startResult = { ok: false, code: 'port-in-use' };
|
|
await taken.service.boot();
|
|
expect(await taken.service.launchRefusal('local-ollama')).toMatchObject({
|
|
code: 'port-in-use',
|
|
message: { vars: { port: '11435' } },
|
|
});
|
|
|
|
const crashing = setup({ state: INSTALLED });
|
|
crashing.world.gaveUp = true;
|
|
expect(await crashing.service.launchRefusal('local-ollama')).toMatchObject({
|
|
code: 'runtime-stopped',
|
|
next: 'start',
|
|
});
|
|
});
|
|
|
|
it('a model that is gone from the runtime is a changed model; a runtime that stopped answering is not running', async () => {
|
|
const gone = await installed();
|
|
gone.deps.readModelDigest.mockResolvedValue(undefined);
|
|
expect(await gone.service.launchRefusal('local-ollama')).toMatchObject({ code: 'model-changed' });
|
|
|
|
const silent = await installed();
|
|
silent.deps.readModelDigest.mockResolvedValue(undefined);
|
|
silent.deps.getRuntimeVersion.mockResolvedValue(undefined);
|
|
expect(await silent.service.launchRefusal('local-ollama')).toMatchObject({ code: 'runtime-not-running' });
|
|
});
|
|
|
|
it('is not asked about any other endpoint, nor about a record with our id that is not ours', async () => {
|
|
const h = await installed();
|
|
h.deps.readModelDigest.mockResolvedValue('changed');
|
|
expect(await h.service.launchRefusal('office-gpu')).toBeUndefined();
|
|
expect(await h.service.launchRefusalMessage('office-gpu', 'anything')).toBeUndefined();
|
|
|
|
// Nothing installed: a hand-made endpoint with the reserved id is the person's own.
|
|
const none = setup();
|
|
expect(await none.service.launchRefusal('local-ollama')).toBeUndefined();
|
|
const { connection: _connection, ...unconnected } = INSTALLED;
|
|
const notOurs = setup({ state: unconnected });
|
|
expect(await notOurs.service.launchRefusal('local-ollama')).toBeUndefined();
|
|
});
|
|
|
|
it('only the model that was installed and checked may be started on the managed endpoint', async () => {
|
|
const h = await installed();
|
|
expect(await h.service.launchRefusalMessage('local-ollama', 'someone-else:7b')).toBe(
|
|
'Only Qwen3.6 35B-A3B is installed on this Mac. Remove it and install another model first.'
|
|
);
|
|
});
|
|
|
|
it('never throws: a state that cannot be read refuses the launch', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
await h.service.view();
|
|
h.deps.readState = vi.fn(async () => {
|
|
throw new Error('EIO');
|
|
});
|
|
const broken = createLocalModelService({ ...(h.deps as unknown as Partial<LocalModelServiceDeps>) });
|
|
expect(await broken.launchRefusal('local-ollama')).toMatchObject({ code: 'runtime-not-running' });
|
|
});
|
|
|
|
it('the view re-reads the digest in the background at most every 30 seconds', async () => {
|
|
const h = await installed();
|
|
h.deps.readModelDigest.mockClear();
|
|
await h.service.view();
|
|
await h.service.view();
|
|
expect(h.deps.readModelDigest).not.toHaveBeenCalled();
|
|
h.world.clock += 30_000;
|
|
h.deps.readModelDigest.mockResolvedValue('changed-under-us');
|
|
await h.service.view();
|
|
await vi.waitFor(async () => expect((await h.service.view()).error?.code).toBe('model-changed'));
|
|
expect(h.deps.readModelDigest).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|
|
|
|
// ---------- The runtime verbs ----------
|
|
|
|
describe('the runtime verbs', () => {
|
|
it('Stop stops the child and keeps it from starting at the next boot; Start undoes that', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
await h.service.boot();
|
|
const stopped = await h.service.runtime({ action: 'stop' });
|
|
expect(stopped.ok && stopped.view).toMatchObject({
|
|
state: 'installed',
|
|
running: false,
|
|
actions: ['start', 'remove'],
|
|
});
|
|
expect(h.supervisor.stop).toHaveBeenCalledTimes(1);
|
|
expect(h.store.get().runtime?.enabled).toBe(false);
|
|
|
|
// The next server process does not start it.
|
|
const next = setup({ state: h.store.get() });
|
|
await next.service.boot();
|
|
expect(next.supervisor.start).not.toHaveBeenCalled();
|
|
|
|
const started = await h.service.runtime({ action: 'start' });
|
|
expect(started.ok && started.view).toMatchObject({ running: true, actions: ['start-session', 'stop', 'remove'] });
|
|
expect(h.store.get().runtime?.enabled).toBe(true);
|
|
});
|
|
|
|
it('Start reads the digest again', async () => {
|
|
const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } });
|
|
h.deps.readModelDigest.mockResolvedValue('not-the-stored-one');
|
|
const started = await h.service.runtime({ action: 'start' });
|
|
expect(started.ok && started.view.error?.code).toBe('model-changed');
|
|
});
|
|
|
|
it('"Use another port": a port that is not the old one, and the saved endpoint moves with it', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
h.world.startResult = { ok: false, code: 'port-in-use' };
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
await h.service.boot();
|
|
expect((await h.service.view()).error?.code).toBe('port-in-use');
|
|
|
|
h.world.startResult = undefined;
|
|
h.deps.chooseRuntimePort.mockResolvedValue(11437);
|
|
const result = await h.service.runtime({ action: 'start', newPort: true });
|
|
expect(h.deps.chooseRuntimePort).toHaveBeenCalledWith({ avoid: [11435] });
|
|
expect(h.deps.moveLocalEndpoint).toHaveBeenCalledWith(11437);
|
|
// The stored pid goes along, so our own runtime on the old port could not be left behind.
|
|
expect(h.supervisor.start).toHaveBeenLastCalledWith({ port: 11437, contextLength: 131072, previousPid: 4711 });
|
|
expect(result.ok && result.view).toMatchObject({ port: 11437, running: true });
|
|
expect(result.ok && result.view.error).toBeUndefined();
|
|
expect(h.store.get().runtime).toMatchObject({ port: 11437, enabled: true });
|
|
});
|
|
|
|
it('a refused write of the saved endpoint leaves the old port in the state: the two never disagree', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
h.deps.chooseRuntimePort.mockResolvedValue(11437);
|
|
h.deps.moveLocalEndpoint.mockRejectedValue(new Error('custom-model-hosts.json is not a valid JSON list'));
|
|
await expect(h.service.runtime({ action: 'start', newPort: true })).rejects.toThrow('custom-model-hosts.json');
|
|
expect(h.store.get().runtime?.port).toBe(11435);
|
|
expect(h.supervisor.start).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('when no other port is free the old one is kept and the error stays', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
h.world.startResult = { ok: false, code: 'port-in-use' };
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
await h.service.boot();
|
|
h.deps.chooseRuntimePort.mockResolvedValue(undefined);
|
|
const result = await h.service.runtime({ action: 'start', newPort: true });
|
|
expect(h.deps.moveLocalEndpoint).not.toHaveBeenCalled();
|
|
expect(result.ok && result.view).toMatchObject({ port: 11435, running: false, error: { code: 'port-in-use' } });
|
|
});
|
|
});
|
|
|
|
// ---------- Remove ----------
|
|
|
|
describe('Remove', () => {
|
|
it('while a live session uses the model: 409 with the session names, and nothing is touched', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
await h.service.boot();
|
|
const result = await h.service.remove({ removeModel: true }, ['w1-app', 'w4-lib']);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
status: 409,
|
|
conflict: {
|
|
reason: 'sessions-open',
|
|
message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen, vars: { sessions: 'w1-app, w4-lib' } },
|
|
sessions: ['w1-app', 'w4-lib'],
|
|
},
|
|
});
|
|
expect(h.supervisor.stop).not.toHaveBeenCalled();
|
|
expect(h.deps.disconnectLocalModel).not.toHaveBeenCalled();
|
|
expect(h.deps.removeLocalModelFiles).not.toHaveBeenCalled();
|
|
expect(h.store.get().state).toBe('installed');
|
|
expect((await h.service.view()).running).toBe(true);
|
|
});
|
|
|
|
it('without one: stops the child, undoes the connection, deletes the files, and is idle again', async () => {
|
|
const h = setup({ state: INSTALLED, runtimeOnDisk: true });
|
|
await h.service.boot();
|
|
h.calls.length = 0;
|
|
const result = await h.service.remove({}, [], ADMIN);
|
|
expect(h.calls).toEqual(['supervisor.stop', 'disconnectLocalModel', 'removeLocalModelFiles']);
|
|
expect(h.deps.disconnectLocalModel).toHaveBeenCalledWith({ flagWasEnabled: false });
|
|
// The model stays unless it was asked to go.
|
|
expect(h.deps.removeLocalModelFiles).toHaveBeenCalledWith({ models: false, paths: PATHS });
|
|
expect(result.ok && result.view).toEqual({
|
|
state: 'idle',
|
|
running: false,
|
|
developerTools: 'missing',
|
|
runtime: {
|
|
version: '0.40.15',
|
|
installed: false,
|
|
downloadBytes: RUNTIME_BYTES,
|
|
downloadGb: '0.2',
|
|
unpackedGb: '0.6',
|
|
},
|
|
folder: '/data/local-model',
|
|
actions: ['install'],
|
|
});
|
|
expect(h.store.get()).toMatchObject({ schema: 1, state: 'idle' });
|
|
for (const key of ['model', 'cli', 'runtime', 'connection', 'loaded'])
|
|
expect(h.store.get()).not.toHaveProperty(key);
|
|
expect(h.deps.audit).toHaveBeenCalledWith({
|
|
admin: 'root',
|
|
action: 'local_model_remove',
|
|
target: 'qwen3.6:35b-a3b',
|
|
ip: '127.0.0.1',
|
|
detail: { removeModel: false },
|
|
});
|
|
});
|
|
|
|
it('removeModel deletes the downloaded model too', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
await h.service.remove({ removeModel: true }, []);
|
|
expect(h.deps.removeLocalModelFiles).toHaveBeenCalledWith({ models: true, paths: PATHS });
|
|
});
|
|
|
|
it('hands the flag value from before this feature to the disconnect', async () => {
|
|
const h = setup({ state: { ...INSTALLED, connection: { endpointId: 'local-ollama', flagWasEnabled: true } } });
|
|
await h.service.remove({}, []);
|
|
expect(h.deps.disconnectLocalModel).toHaveBeenCalledWith({ flagWasEnabled: true });
|
|
});
|
|
|
|
it('a paused or failed job has no connection to undo: the files go, sessions on some other endpoint do not matter', async () => {
|
|
const h = setup({ state: jobAt('model', 'paused') });
|
|
// The names belong to sessions on an endpoint a person made with our id; it is not ours.
|
|
const result = await h.service.remove({ removeModel: true }, ['w1-app']);
|
|
expect(result.ok).toBe(true);
|
|
expect(h.deps.disconnectLocalModel).not.toHaveBeenCalled();
|
|
expect(h.deps.removeLocalModelFiles).toHaveBeenCalledWith({ models: true, paths: PATHS });
|
|
expect(h.store.get().state).toBe('idle');
|
|
});
|
|
|
|
it('a connection that cannot be undone stops the removal before any file is deleted', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
h.deps.disconnectLocalModel.mockRejectedValue(new Error('custom-model-hosts.json is not a valid JSON list'));
|
|
await expect(h.service.remove({ removeModel: true }, [])).rejects.toThrow('custom-model-hosts.json');
|
|
expect(h.deps.removeLocalModelFiles).not.toHaveBeenCalled();
|
|
expect(h.store.get().state).toBe('installed');
|
|
// The service still answers afterwards.
|
|
expect((await h.service.view()).state).toBe('installed');
|
|
});
|
|
|
|
it('the coding agent stays: nothing of it is asked for, run or removed', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
await h.service.remove({ removeModel: true }, []);
|
|
expect(h.deps.runInstallCommand).not.toHaveBeenCalled();
|
|
expect(h.deps.stockInstallCommand).not.toHaveBeenCalled();
|
|
expect(h.deps.recheckCliInstalled).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('after a remove the model can be installed again', async () => {
|
|
const h = setup({ state: INSTALLED, agentInstalled: true });
|
|
await h.service.remove({}, []);
|
|
await installAndWait(h.service);
|
|
expect(h.store.get().state).toBe('installed');
|
|
});
|
|
});
|
|
|
|
// ---------- Remove and the real connection files ----------
|
|
|
|
describe('Remove with the real connection: the settings flag', () => {
|
|
const HOSTS = () => customModelHostsPath(getDataDir());
|
|
const OTHER: CustomModelHost = { id: 'office-gpu', label: 'Office GPU', baseUrl: 'http://192.168.1.50:8080' };
|
|
|
|
function writeSettings(settings: Record<string, unknown>): void {
|
|
mkdirSync(dirname(SETTINGS_PATH), { recursive: true });
|
|
writeFileSync(SETTINGS_PATH, JSON.stringify(settings));
|
|
}
|
|
const flag = (): unknown => JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).customModelEndpointsEnabled;
|
|
|
|
afterEach(() => {
|
|
rmSync(HOSTS(), { force: true });
|
|
rmSync(SETTINGS_PATH, { force: true });
|
|
});
|
|
|
|
/** Installs with the real connect step, then removes with the real disconnect. */
|
|
async function installThenRemove() {
|
|
const h = setup({
|
|
agentInstalled: true,
|
|
overrides: { inspectLocalEndpoint, connectLocalModel, disconnectLocalModel },
|
|
});
|
|
await installAndWait(h.service);
|
|
expect(h.store.get().state).toBe('installed');
|
|
expect(flag()).toBe(true);
|
|
expect((await readCustomModelHosts(getDataDir())).map((host) => host.id)).toContain('local-ollama');
|
|
const removed = await h.service.remove({}, []);
|
|
expect(removed.ok).toBe(true);
|
|
expect((await readCustomModelHosts(getDataDir())).map((host) => host.id)).not.toContain('local-ollama');
|
|
return h;
|
|
}
|
|
|
|
it('this feature switched it on and no other endpoint is saved: it is switched off again', async () => {
|
|
writeSettings({ customModelEndpointsEnabled: false, tmuxHistoryLimit: 5000 });
|
|
await installThenRemove();
|
|
expect(flag()).toBe(false);
|
|
expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).tmuxHistoryLimit).toBe(5000);
|
|
});
|
|
|
|
it('it was on before this feature: it stays on', async () => {
|
|
writeSettings({ customModelEndpointsEnabled: true });
|
|
await installThenRemove();
|
|
expect(flag()).toBe(true);
|
|
});
|
|
|
|
it('this feature switched it on but another endpoint is saved: it stays on, and that endpoint is untouched', async () => {
|
|
writeSettings({});
|
|
writeFileSync(HOSTS(), JSON.stringify([OTHER]));
|
|
await installThenRemove();
|
|
expect(flag()).toBe(true);
|
|
expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]);
|
|
});
|
|
|
|
it('a resumed connect step does not forget that the flag was off before', async () => {
|
|
writeSettings({ customModelEndpointsEnabled: false });
|
|
const h = setup({
|
|
agentInstalled: true,
|
|
overrides: { inspectLocalEndpoint, connectLocalModel, disconnectLocalModel },
|
|
});
|
|
await installAndWait(h.service);
|
|
// The connection is written a second time, as a resumed job would: the flag is on by now.
|
|
const again = await connectLocalModel({
|
|
port: 11435,
|
|
modelId: 'qwen3.6:35b-a3b',
|
|
context: 131072,
|
|
ours: true,
|
|
flagWasEnabled: h.store.get().connection!.flagWasEnabled,
|
|
});
|
|
expect(again).toEqual({ ok: true, flagWasEnabled: false });
|
|
await h.service.remove({}, []);
|
|
expect(flag()).toBe(false);
|
|
});
|
|
});
|
|
|
|
// ---------- The view never throws ----------
|
|
|
|
describe('view()', () => {
|
|
it('answers an idle view when a dependency throws', async () => {
|
|
const h = setup();
|
|
h.deps.checkDeveloperTools.mockRejectedValue(new Error('spawn failed'));
|
|
h.deps.isRuntimeInstalled.mockRejectedValue(new Error('EACCES'));
|
|
expect(await h.service.view()).toMatchObject({
|
|
state: 'idle',
|
|
developerTools: 'unknown',
|
|
runtime: { installed: false, downloadBytes: RUNTIME_BYTES },
|
|
actions: ['install'],
|
|
});
|
|
});
|
|
|
|
it('answers the bare idle view when the state itself cannot be read', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const service = createLocalModelService({
|
|
...(setup().deps as unknown as Partial<LocalModelServiceDeps>),
|
|
readState: async () => {
|
|
throw new Error('EIO');
|
|
},
|
|
});
|
|
expect(await service.view()).toMatchObject({ state: 'idle', running: false, actions: ['install'] });
|
|
});
|
|
|
|
it('computes a download speed for the runtime archive, which reports bytes only', async () => {
|
|
let report: (p: { completedBytes: number; totalBytes: number }) => void = () => {};
|
|
const h = setup({
|
|
overrides: {
|
|
ensureRuntimeInstalled: vi.fn(async (install?: { signal?: AbortSignal; onProgress?: typeof report }) => {
|
|
report = install?.onProgress ?? report;
|
|
await untilAborted(install?.signal);
|
|
return { ok: false as const, code: 'cancelled' as const };
|
|
}),
|
|
},
|
|
});
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(h.deps.ensureRuntimeInstalled).toHaveBeenCalled());
|
|
report({ completedBytes: 0, totalBytes: RUNTIME_BYTES });
|
|
expect((await h.service.view()).progress).not.toHaveProperty('bytesPerSecond');
|
|
h.world.clock += 2_000;
|
|
report({ completedBytes: 20_000_000, totalBytes: RUNTIME_BYTES });
|
|
expect((await h.service.view()).progress).toMatchObject({
|
|
completedBytes: 20_000_000,
|
|
totalBytes: RUNTIME_BYTES,
|
|
bytesPerSecond: 10_000_000,
|
|
mbPerSecond: '10',
|
|
// (185,971,671 - 20,000,000) / 10,000,000 = 16.6, rounded up.
|
|
secondsLeft: 17,
|
|
});
|
|
await h.service.cancel();
|
|
});
|
|
|
|
it('a view whose read began while the job ran shows how the job ended, never paused', async () => {
|
|
// The window polls every second, so a read of the state file can be under way while the
|
|
// job writes its last state and lets go. That read returns the old `installing`, and
|
|
// settling it with no job in this process read it as an interrupted job: the window (and
|
|
// the e2e test, under CI load) saw `paused` with Resume for one tick instead of `failed`.
|
|
const slow = {
|
|
read: undefined as undefined | (() => Promise<LocalModelStateFile>),
|
|
hold: undefined as undefined | Promise<void>,
|
|
entered: undefined as undefined | (() => void),
|
|
};
|
|
const h = setup({
|
|
overrides: {
|
|
readState: async () => {
|
|
const snapshot = await slow.read!();
|
|
const hold = slow.hold;
|
|
slow.hold = undefined;
|
|
if (hold) {
|
|
slow.entered?.();
|
|
await hold;
|
|
}
|
|
return snapshot;
|
|
},
|
|
},
|
|
});
|
|
slow.read = h.store.readState;
|
|
let failVerify!: () => void;
|
|
const verifying = new Promise<void>((resolve) => (failVerify = resolve));
|
|
h.deps.verifyModel.mockImplementationOnce(async () => {
|
|
await verifying;
|
|
return { ok: false, code: 'no-tool-call' } as never;
|
|
});
|
|
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(h.deps.verifyModel).toHaveBeenCalled());
|
|
|
|
// A view starts reading while the job is still in the verify step...
|
|
let releaseRead!: () => void;
|
|
slow.hold = new Promise<void>((resolve) => (releaseRead = resolve));
|
|
const entered = new Promise<void>((resolve) => (slow.entered = resolve));
|
|
const viewing = h.service.view();
|
|
await entered;
|
|
// ...the job fails, writes it and ends while that read is still out...
|
|
failVerify();
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('failed');
|
|
// ...and only then does the read come back, with the file as it was before.
|
|
releaseRead();
|
|
const view = await viewing;
|
|
expect(view).toMatchObject({ state: 'failed', step: 'verify', error: { code: 'no-tool-call' } });
|
|
});
|
|
|
|
it('names the model to try next only when the catalogue still has it', async () => {
|
|
const state: LocalModelStateFile = {
|
|
...jobAt('verify', 'paused'),
|
|
state: 'failed',
|
|
error: { code: 'no-tool-call', nextModelId: 'gone:1b' },
|
|
};
|
|
const h = setup({ state });
|
|
const view = await h.service.view();
|
|
expect(view.error?.next).toBe('next-model');
|
|
expect(view.error).not.toHaveProperty('nextModel');
|
|
});
|
|
});
|
|
|
|
// ---------- The connection is claimed before it is written ----------
|
|
|
|
describe('the connect step claims the connection before it writes anything outside the folder', () => {
|
|
it('the state file knows the connection is ours before the endpoint record is written', async () => {
|
|
let claimAtWrite: LocalModelStateFile['connection'];
|
|
const h = setup();
|
|
h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'absent', flagEnabled: true });
|
|
h.deps.connectLocalModel.mockImplementation(async () => {
|
|
claimAtWrite = h.store.get().connection;
|
|
return { ok: true as const, flagWasEnabled: true };
|
|
});
|
|
await installAndWait(h.service);
|
|
// The claim carries the value the setting had BEFORE this feature switched it on.
|
|
expect(claimAtWrite).toEqual({ endpointId: 'local-ollama', flagWasEnabled: true });
|
|
expect(h.deps.inspectLocalEndpoint).toHaveBeenCalledWith({
|
|
port: 11435,
|
|
modelId: 'qwen3.6:35b-a3b',
|
|
context: 131072,
|
|
});
|
|
});
|
|
|
|
it('a claim that cannot be written (a full disk) writes nothing outside, and "Try again" finishes the install', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: jobAt('connect', 'paused') });
|
|
// The claim is refused once; every other write lands.
|
|
let refused = false;
|
|
h.store.disk.refuse = (next, before) => {
|
|
if (refused || !next.connection || before.connection) return false;
|
|
refused = true;
|
|
return true;
|
|
};
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(await h.service.view()).toMatchObject({ state: 'failed', step: 'connect', error: { code: 'disk-full' } });
|
|
// Nothing was written outside our folder: the record and the setting are untouched.
|
|
expect(h.deps.connectLocalModel).not.toHaveBeenCalled();
|
|
expect(h.store.get()).not.toHaveProperty('connection');
|
|
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.store.get()).toMatchObject({
|
|
state: 'installed',
|
|
connection: { endpointId: 'local-ollama', flagWasEnabled: false },
|
|
});
|
|
expect(h.deps.connectLocalModel).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('a record that already says what ours would say is ours, and the setting is then never switched off', async () => {
|
|
const h = setup({ state: jobAt('connect', 'paused') });
|
|
h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'same', flagEnabled: true });
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.store.get()).toMatchObject({
|
|
state: 'installed',
|
|
connection: { endpointId: 'local-ollama', flagWasEnabled: true },
|
|
});
|
|
});
|
|
|
|
it('an endpoint store that cannot be read: connect-failed, before anything is claimed', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: jobAt('connect', 'paused') });
|
|
h.deps.inspectLocalEndpoint.mockRejectedValue(new Error('custom-model-hosts.json is not a valid JSON list'));
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(await h.service.view()).toMatchObject({
|
|
state: 'failed',
|
|
error: { code: 'connect-failed', detail: 'custom-model-hosts.json is not a valid JSON list' },
|
|
});
|
|
expect(h.store.get()).not.toHaveProperty('connection');
|
|
expect(h.deps.connectLocalModel).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('a resumed job does not look again: the claim it has keeps the value from before', async () => {
|
|
const h = setup({
|
|
state: { ...jobAt('connect', 'paused'), connection: { endpointId: 'local-ollama', flagWasEnabled: false } },
|
|
});
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.deps.inspectLocalEndpoint).not.toHaveBeenCalled();
|
|
expect(h.deps.connectLocalModel).toHaveBeenCalledWith(
|
|
expect.objectContaining({ ours: true, flagWasEnabled: false })
|
|
);
|
|
expect(h.store.get().state).toBe('installed');
|
|
});
|
|
});
|
|
|
|
describe('the connect step with the real endpoint store and settings: a write that does not land', () => {
|
|
const HOSTS = () => customModelHostsPath(getDataDir());
|
|
const real = { inspectLocalEndpoint, connectLocalModel, disconnectLocalModel };
|
|
const hostIds = async () => (await readCustomModelHosts(getDataDir())).map((host) => host.id);
|
|
const flag = (): unknown => {
|
|
try {
|
|
return JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).customModelEndpointsEnabled;
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
};
|
|
|
|
afterEach(() => {
|
|
rmSync(HOSTS(), { force: true });
|
|
rmSync(SETTINGS_PATH, { force: true });
|
|
});
|
|
|
|
it('the claim fails once: nothing is left behind, "Try again" installs, and Remove undoes both writes', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: jobAt('connect', 'paused'), overrides: real });
|
|
let refused = false;
|
|
h.store.disk.refuse = (next, before) => {
|
|
if (refused || !next.connection || before.connection) return false;
|
|
refused = true;
|
|
return true;
|
|
};
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect((await h.service.view()).error?.code).toBe('disk-full');
|
|
expect(await hostIds()).toEqual([]);
|
|
expect(flag()).toBeUndefined();
|
|
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('installed');
|
|
expect(await hostIds()).toEqual(['local-ollama']);
|
|
expect(flag()).toBe(true);
|
|
|
|
const removed = await h.service.remove({ removeModel: true }, [], ADMIN);
|
|
expect(removed.ok && removed.view.state).toBe('idle');
|
|
expect(await hostIds()).toEqual([]);
|
|
expect(flag()).toBe(false);
|
|
});
|
|
|
|
it('the server is killed after the claim and before the record: the next server finishes it on Continue', async () => {
|
|
// What the killed server left: the claim in the state file, nothing in the endpoint store.
|
|
const h = setup({
|
|
state: { ...jobAt('connect'), connection: { endpointId: 'local-ollama', flagWasEnabled: false } },
|
|
overrides: real,
|
|
});
|
|
await h.service.boot();
|
|
expect(await h.service.view()).toMatchObject({ state: 'paused', step: 'connect' });
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('installed');
|
|
expect(await hostIds()).toEqual(['local-ollama']);
|
|
await h.service.remove({}, [], ADMIN);
|
|
expect(await hostIds()).toEqual([]);
|
|
expect(flag()).toBe(false);
|
|
});
|
|
|
|
it('the server is killed after both writes and before the last step was recorded: Continue ends installed', async () => {
|
|
const h = setup({
|
|
state: { ...jobAt('connect'), connection: { endpointId: 'local-ollama', flagWasEnabled: false } },
|
|
overrides: real,
|
|
});
|
|
await connectLocalModel({ port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072, ours: true });
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(await h.service.view()).toMatchObject({ state: 'installed' });
|
|
expect(await hostIds()).toEqual(['local-ollama']);
|
|
});
|
|
|
|
it('a record of ours the state file does not know (written by an earlier order of the two) is taken as ours', async () => {
|
|
// The record and the flag exist, the state file has no claim.
|
|
await connectLocalModel({ port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072, ours: false });
|
|
const h = setup({ state: jobAt('connect', 'paused'), overrides: real });
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(await h.service.view()).toMatchObject({ state: 'installed' });
|
|
// Remove takes the record out. The setting stays: what it was before is not known.
|
|
await h.service.remove({}, [], ADMIN);
|
|
expect(await hostIds()).toEqual([]);
|
|
expect(flag()).toBe(true);
|
|
});
|
|
|
|
it('a record with our id that says something else is never overwritten', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
mkdirSync(dirname(HOSTS()), { recursive: true });
|
|
const foreign = { id: 'local-ollama', label: 'My own', baseUrl: 'http://127.0.0.1:11434' };
|
|
writeFileSync(HOSTS(), JSON.stringify([foreign]));
|
|
const h = setup({ state: jobAt('connect', 'paused'), overrides: real });
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect((await h.service.view()).error?.code).toBe('endpoint-id-taken');
|
|
expect(await readCustomModelHosts(getDataDir())).toEqual([foreign]);
|
|
expect(h.store.get()).not.toHaveProperty('connection');
|
|
});
|
|
});
|
|
|
|
// ---------- A job whose end could not be written ----------
|
|
|
|
describe('a state file that cannot be written when a step fails (a full disk)', () => {
|
|
/** A pull that fills the disk: from its failure on, every write of the state file is refused. */
|
|
function diskFillsDuringPull(h: ReturnType<typeof setup>) {
|
|
h.deps.pullModel.mockImplementation(
|
|
async (_target: unknown, _id: string, pull?: { onProgress?: (p: unknown) => void }) => {
|
|
pull?.onProgress?.({ completedBytes: 14_200_000_000, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 });
|
|
h.store.disk.refuse = () => true;
|
|
return { ok: false as const, code: 'disk-full' as const, detail: 'write blob: no space left on device' };
|
|
}
|
|
);
|
|
}
|
|
|
|
it('the job is not lost: the view says disk-full with the download as it stood, and no speed', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: jobAt('model', 'paused') });
|
|
diskFillsDuringPull(h);
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
|
|
// The file could not be told and still says the job runs.
|
|
expect(h.store.get().state).toBe('installing');
|
|
const view = await h.service.view();
|
|
expect(view).toMatchObject({
|
|
state: 'failed',
|
|
step: 'model',
|
|
error: { code: 'disk-full', next: 'retry', detail: 'write blob: no space left on device' },
|
|
actions: ['resume', 'install', 'remove'],
|
|
});
|
|
expect(view.progress).toMatchObject({ completedBytes: 14_200_000_000, totalBytes: PULL_BYTES });
|
|
expect(view.progress).not.toHaveProperty('bytesPerSecond');
|
|
// A GET writes nothing: the file is as it was.
|
|
expect(h.store.get().state).toBe('installing');
|
|
});
|
|
|
|
it('Cancel has nothing to cancel (409, not a 500), and "Try again" on a disk that is still full says so again', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: jobAt('model', 'paused') });
|
|
diskFillsDuringPull(h);
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
|
|
expect(await h.service.cancel()).toMatchObject({ ok: false, status: 409, conflict: { reason: 'wrong-state' } });
|
|
const again = await h.service.resume({}, ADMIN);
|
|
expect(again.ok && again.view).toMatchObject({ state: 'failed', error: { code: 'disk-full' } });
|
|
expect(h.deps.pullModel).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('once there is space again "Try again" writes the outcome, runs the step and installs', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: jobAt('model', 'paused') });
|
|
diskFillsDuringPull(h);
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
|
|
h.store.disk.refuse = undefined;
|
|
h.deps.pullModel.mockImplementation(async () => ({ ok: true as const, totalBytes: PULL_BYTES }));
|
|
const resumed = await h.service.resume({}, ADMIN);
|
|
expect(resumed.ok && resumed.view.state).toBe('installing');
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('installed');
|
|
expect((await h.service.view()).error).toBeUndefined();
|
|
});
|
|
|
|
it('another verb writes what the view already showed: a new install replaces the failed job', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: jobAt('model', 'paused'), runtimeOnDisk: true });
|
|
diskFillsDuringPull(h);
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
h.store.disk.refuse = undefined;
|
|
h.deps.pullModel.mockImplementation(async () => ({ ok: true as const, totalBytes: 13_793_441_427 }));
|
|
h.deps.recommend.mockImplementation(() => {
|
|
const rec = recommendation();
|
|
for (const row of [rec.pick!, ...rec.alternatives]) row.installBytes = row.downloadBytes;
|
|
return rec;
|
|
});
|
|
const result = await installAndWait(h.service, {
|
|
modelId: 'gpt-oss:20b',
|
|
cli: 'opencode',
|
|
context: 131072,
|
|
acknowledgedBytes: 13_793_441_427,
|
|
});
|
|
expect(result.ok).toBe(true);
|
|
expect(h.store.get()).toMatchObject({ state: 'installed', model: { id: 'gpt-oss:20b' } });
|
|
});
|
|
|
|
it('Cancel while the disk is full stops the job and shows it paused, without throwing', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({
|
|
overrides: {
|
|
pullModel: vi.fn(async (_t: unknown, _id: string, pull?: PullOptions) => {
|
|
pull?.onProgress?.({ completedBytes: 6_200_000_000, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 });
|
|
await untilAborted(pull?.signal);
|
|
return { ok: false as const, code: 'cancelled' as const };
|
|
}),
|
|
},
|
|
});
|
|
await h.service.install(REQUEST, ADMIN);
|
|
await vi.waitFor(() => expect(h.deps.pullModel).toHaveBeenCalled());
|
|
h.store.disk.refuse = () => true;
|
|
const result = await h.service.cancel();
|
|
expect(result.ok && result.view).toMatchObject({
|
|
state: 'paused',
|
|
step: 'model',
|
|
progress: { completedBytes: 6_200_000_000 },
|
|
actions: ['resume', 'install', 'remove'],
|
|
});
|
|
// Space again: Continue writes the pause it could not write, and goes on.
|
|
h.store.disk.refuse = undefined;
|
|
h.deps.pullModel.mockImplementation(async () => ({ ok: true as const, totalBytes: PULL_BYTES }));
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('installed');
|
|
});
|
|
|
|
it('a first install on a full disk starts nothing and is answered with the sentence', async () => {
|
|
const h = setup();
|
|
h.store.disk.refuse = () => true;
|
|
const result = await h.service.install(REQUEST, ADMIN);
|
|
expect(result).toEqual({
|
|
ok: false,
|
|
status: 507,
|
|
message: { text: 'The disk is full. Free up space, then try again.' },
|
|
});
|
|
expect(h.calls).toEqual([]);
|
|
expect(h.store.get().state).toBe('idle');
|
|
});
|
|
|
|
it('a step that finished but could not be recorded (a full disk) is disk-full, and the step runs again', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: jobAt('verify', 'paused') });
|
|
// The write that would move the job past `verify` is refused, and every write after it.
|
|
h.store.disk.refuse = (next, before) => before.step === 'verify' && next.step !== 'verify';
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(await h.service.view()).toMatchObject({ state: 'failed', step: 'verify', error: { code: 'disk-full' } });
|
|
h.store.disk.refuse = undefined;
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
expect(h.store.get().state).toBe('installed');
|
|
});
|
|
});
|
|
|
|
// ---------- A runtime that does not start, and one an earlier server left behind ----------
|
|
|
|
describe('an installed model whose runtime does not start', () => {
|
|
it('says so on the view with what the runtime said, and Start tries again', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: INSTALLED, runtimeOnDisk: true });
|
|
h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'spawn ENOENT' };
|
|
await h.service.boot();
|
|
const view = await h.service.view();
|
|
expect(view).toMatchObject({ state: 'installed', running: false, actions: ['start', 'remove'] });
|
|
expect(view.error).toEqual({
|
|
code: 'runtime-start-failed',
|
|
message: { text: 'The runtime did not start on this Mac.' },
|
|
// Start is the verb an installed model has: there is no job to resume.
|
|
next: 'start',
|
|
detail: 'spawn ENOENT',
|
|
});
|
|
// A session start is refused with the same reason, not with "start it first".
|
|
expect(await h.service.launchRefusalMessage('local-ollama', 'qwen3.6:35b-a3b')).toBe(
|
|
'The runtime did not start on this Mac.'
|
|
);
|
|
|
|
// Start again, and it fails again: the view still says why.
|
|
const again = await h.service.runtime({ action: 'start' });
|
|
expect(again.ok && again.view.error?.code).toBe('runtime-start-failed');
|
|
|
|
// Start again, and it works: the error is gone.
|
|
h.world.startResult = undefined;
|
|
const started = await h.service.runtime({ action: 'start' });
|
|
expect(started.ok && started.view).toMatchObject({ running: true });
|
|
expect(started.ok && started.view.error).toBeUndefined();
|
|
});
|
|
|
|
it('Stop clears it: a model the person stopped shows no error', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: INSTALLED, runtimeOnDisk: true });
|
|
h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'spawn ENOENT' };
|
|
await h.service.boot();
|
|
const stopped = await h.service.runtime({ action: 'stop' });
|
|
expect(stopped.ok && stopped.view.error).toBeUndefined();
|
|
});
|
|
});
|
|
|
|
describe('a later build with another runtime pin', () => {
|
|
const OLD = '0.39.2';
|
|
const OLD_STATE: LocalModelStateFile = { ...INSTALLED, runtime: { ...INSTALLED.runtime!, version: OLD } };
|
|
/** Only the old version is unpacked: what an update of Codeman leaves on disk. */
|
|
const onlyOldOnDisk = async (_pin?: unknown, paths?: LocalModelPaths) =>
|
|
paths?.binary === `/data/local-model/runtime/ollama-${OLD}/ollama`;
|
|
|
|
it('an installed model keeps running the version it was installed with', async () => {
|
|
const h = setup({ state: OLD_STATE });
|
|
h.deps.isRuntimeInstalled.mockImplementation(onlyOldOnDisk as never);
|
|
await h.service.boot();
|
|
expect(h.supervisor.start).toHaveBeenCalledWith({
|
|
port: 11435,
|
|
contextLength: 131072,
|
|
previousPid: 4711,
|
|
version: OLD,
|
|
});
|
|
// The stored version stays what runs, so the next start finds it again.
|
|
expect(h.store.get().runtime?.version).toBe(OLD);
|
|
const view = await h.service.view();
|
|
expect(view).toMatchObject({ state: 'installed', running: true, actions: ['start-session', 'stop', 'remove'] });
|
|
expect(view.error).toBeUndefined();
|
|
// A NEW install would still download the pinned build.
|
|
expect(view.runtime).toMatchObject({ version: OLLAMA_RUNTIME_PIN.version, installed: false });
|
|
});
|
|
|
|
it('with the pinned build unpacked that one runs, and the state says so', async () => {
|
|
const h = setup({ state: OLD_STATE, runtimeOnDisk: true });
|
|
await h.service.boot();
|
|
expect(h.supervisor.start).toHaveBeenCalledWith({ port: 11435, contextLength: 131072, previousPid: 4711 });
|
|
expect(h.store.get().runtime?.version).toBe(OLLAMA_RUNTIME_PIN.version);
|
|
});
|
|
|
|
it('with neither on disk the start fails in the open: the view says the runtime did not start', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup({ state: OLD_STATE });
|
|
h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'spawn ENOENT' };
|
|
await h.service.boot();
|
|
expect((await h.service.view()).error).toMatchObject({ code: 'runtime-start-failed', next: 'start' });
|
|
});
|
|
|
|
it('a version the state file could never name is not turned into a path', async () => {
|
|
const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, version: '../../evil' } } });
|
|
await h.service.boot();
|
|
expect(h.supervisor.start).toHaveBeenCalledWith({ port: 11435, contextLength: 131072, previousPid: 4711 });
|
|
});
|
|
});
|
|
|
|
describe('a runtime an earlier server left behind', () => {
|
|
it.each(['paused', 'failed'] as const)('Remove of a %s job ends it before any file is deleted', async (state) => {
|
|
const base = jobAt('model', 'paused');
|
|
const h = setup({
|
|
state: state === 'paused' ? base : { ...base, state: 'failed', error: { code: 'pull-failed' } },
|
|
});
|
|
const removed = await h.service.remove({ removeModel: true }, [], ADMIN);
|
|
expect(removed.ok && removed.view.state).toBe('idle');
|
|
expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 5000 });
|
|
expect(h.calls.indexOf('supervisor.stop')).toBeLessThan(h.calls.indexOf('removeLocalModelFiles'));
|
|
});
|
|
|
|
it('Stop of an installed model ends it too, also when this server never started it', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
await h.service.runtime({ action: 'stop' });
|
|
expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 4711 });
|
|
});
|
|
|
|
it('boot ends it for a model the person had stopped', async () => {
|
|
const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } });
|
|
await h.service.boot();
|
|
expect(h.supervisor.start).not.toHaveBeenCalled();
|
|
expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 4711 });
|
|
});
|
|
|
|
it('the pid is stored the moment the child is spawned, before it answers', async () => {
|
|
const h = setup({
|
|
state: { ...jobAt('model', 'paused'), runtime: { version: '0.40.15', port: 11435, enabled: true } },
|
|
});
|
|
h.service.onRuntimeChange({ running: false, pid: 7777, restarts: 0, gaveUp: false });
|
|
await vi.waitFor(() => expect(h.store.get().runtime?.pid).toBe(7777));
|
|
});
|
|
});
|
|
|
|
// ---------- What a failed download is called, and what is offered next ----------
|
|
|
|
describe('a runtime that is gone for good during the download', () => {
|
|
it('is said as the runtime, not as the network, and "Try again" runs the step again', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup();
|
|
h.deps.pullModel.mockImplementation(async () => {
|
|
h.world.running = false;
|
|
h.world.gaveUp = true;
|
|
return { ok: false as const, code: 'pull-failed' as const, detail: 'connect ECONNREFUSED 127.0.0.1:11435' };
|
|
});
|
|
await installAndWait(h.service);
|
|
const view = await h.service.view();
|
|
expect(view).toMatchObject({ state: 'failed', step: 'model', actions: ['resume', 'install', 'remove'] });
|
|
expect(view.error).toEqual({
|
|
code: 'runtime-stopped',
|
|
message: { text: 'The runtime keeps stopping, so Codeman no longer restarts it.' },
|
|
// A job has no Start verb: the step is run again.
|
|
next: 'retry',
|
|
detail: 'connect ECONNREFUSED 127.0.0.1:11435',
|
|
});
|
|
});
|
|
|
|
it('a pull that fails while the runtime runs is still the download', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup();
|
|
h.deps.pullModel.mockResolvedValue({ ok: false, code: 'pull-failed', detail: 'unexpected EOF' } as never);
|
|
await installAndWait(h.service);
|
|
expect((await h.service.view()).error).toMatchObject({ code: 'pull-failed', next: 'retry' });
|
|
});
|
|
});
|
|
|
|
describe('the model offered after "does not fit"', () => {
|
|
it('needs less memory than the one that failed: a model that needs more is not offered', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup();
|
|
// The only other row needs MORE than the model that failed.
|
|
h.deps.recommend.mockImplementation(() => {
|
|
const rec = recommendation();
|
|
rec.alternatives = [candidate('qwen3.8:27b', 'Qwen3.8 27B', 131072, 17_700_000_000, false, 39_000_000_000)];
|
|
return rec;
|
|
});
|
|
h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'does-not-fit' } as never);
|
|
await installAndWait(h.service);
|
|
const view = await h.service.view();
|
|
expect(view.error).toMatchObject({ code: 'does-not-fit', next: 'next-model' });
|
|
expect(view.error).not.toHaveProperty('nextModel');
|
|
});
|
|
|
|
it('skips a larger row for the first one that needs less', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup();
|
|
h.deps.getCatalog.mockReturnValue({
|
|
...CATALOG,
|
|
models: [...CATALOG.models, { id: 'qwen3.8:27b', label: 'Qwen3.8 27B' }],
|
|
} as unknown as LocalModelCatalog);
|
|
h.deps.recommend.mockImplementation(() => {
|
|
const rec = recommendation();
|
|
rec.alternatives = [
|
|
candidate('qwen3.8:27b', 'Qwen3.8 27B', 131072, 17_700_000_000, false, 39_000_000_000),
|
|
candidate('gpt-oss:20b', 'GPT-OSS 20B', 131072, 13_793_441_427, false, 26_600_000_000),
|
|
];
|
|
return rec;
|
|
});
|
|
h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'does-not-fit' } as never);
|
|
await installAndWait(h.service);
|
|
expect((await h.service.view()).error?.nextModel).toEqual({ modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' });
|
|
});
|
|
|
|
it('after "no tool call" memory is not the question: the best other model is offered', async () => {
|
|
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
const h = setup();
|
|
h.deps.getCatalog.mockReturnValue({
|
|
...CATALOG,
|
|
models: [...CATALOG.models, { id: 'qwen3.8:27b', label: 'Qwen3.8 27B' }],
|
|
} as unknown as LocalModelCatalog);
|
|
h.deps.recommend.mockImplementation(() => {
|
|
const rec = recommendation();
|
|
rec.alternatives = [candidate('qwen3.8:27b', 'Qwen3.8 27B', 131072, 17_700_000_000, false, 39_000_000_000)];
|
|
return rec;
|
|
});
|
|
h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'no-tool-call' } as never);
|
|
await installAndWait(h.service);
|
|
expect((await h.service.view()).error?.nextModel).toEqual({ modelId: 'qwen3.8:27b', modelLabel: 'Qwen3.8 27B' });
|
|
});
|
|
});
|
|
|
|
describe('what a replaced download left behind', () => {
|
|
it('the partial blobs are deleted as files, after the model was asked for and before the new pull', async () => {
|
|
const h = setup({
|
|
state: { ...jobAt('model', 'paused'), replacedModelId: 'gpt-oss:20b' },
|
|
});
|
|
h.deps.removePartialModelBlobs.mockImplementation(async () => {
|
|
h.calls.push('removePartialModelBlobs');
|
|
return 12_708_189_425;
|
|
});
|
|
vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
await h.service.resume({}, ADMIN);
|
|
await h.service.idle();
|
|
const order = h.calls.filter((call) => ['deleteModel', 'removePartialModelBlobs', 'pullModel'].includes(call));
|
|
expect(order).toEqual(['deleteModel', 'removePartialModelBlobs', 'pullModel']);
|
|
expect(h.deps.removePartialModelBlobs).toHaveBeenCalledWith(PATHS);
|
|
expect(h.store.get()).not.toHaveProperty('replacedModelId');
|
|
});
|
|
|
|
it('a job that replaced nothing deletes nothing', async () => {
|
|
const h = setup();
|
|
await installAndWait(h.service);
|
|
expect(h.deps.deleteModel).not.toHaveBeenCalled();
|
|
expect(h.deps.removePartialModelBlobs).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
// ---------- Live sessions keep the address they were started with ----------
|
|
|
|
describe('Stop and "Use another port" while live sessions use the model', () => {
|
|
it('Stop is 409 sessions-open with their names, and the runtime keeps running', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
await h.service.boot();
|
|
h.supervisor.stop.mockClear();
|
|
const result = await h.service.runtime({ action: 'stop' }, ['w1-app', 'w4-lib']);
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
status: 409,
|
|
conflict: {
|
|
reason: 'sessions-open',
|
|
sessions: ['w1-app', 'w4-lib'],
|
|
message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen, vars: { sessions: 'w1-app, w4-lib' } },
|
|
},
|
|
});
|
|
expect(h.supervisor.stop).not.toHaveBeenCalled();
|
|
expect(h.store.get().runtime?.enabled).toBe(true);
|
|
});
|
|
|
|
it('a start on another port is refused the same way, and nothing moves', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
const result = await h.service.runtime({ action: 'start', newPort: true }, ['w1-app']);
|
|
expect(result).toMatchObject({ ok: false, status: 409, conflict: { reason: 'sessions-open' } });
|
|
expect(h.deps.chooseRuntimePort).not.toHaveBeenCalled();
|
|
expect(h.deps.moveLocalEndpoint).not.toHaveBeenCalled();
|
|
expect(h.store.get().runtime?.port).toBe(11435);
|
|
});
|
|
|
|
it('a plain Start is not refused: it changes no address', async () => {
|
|
const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } });
|
|
const result = await h.service.runtime({ action: 'start' }, ['w1-app']);
|
|
expect(result.ok && result.view.running).toBe(true);
|
|
});
|
|
|
|
it('"Use another port" of a job is refused only when a connection of ours exists', async () => {
|
|
const failedOnPort: LocalModelStateFile = {
|
|
...jobAt('connect', 'paused'),
|
|
state: 'failed',
|
|
error: { code: 'port-in-use', vars: { port: '11435' } },
|
|
};
|
|
const connected = setup({
|
|
state: { ...failedOnPort, connection: { endpointId: 'local-ollama', flagWasEnabled: false } },
|
|
});
|
|
expect(await connected.service.resume({ newPort: true }, ADMIN, ['w1-app'])).toMatchObject({
|
|
ok: false,
|
|
conflict: { reason: 'sessions-open', sessions: ['w1-app'] },
|
|
});
|
|
expect(connected.deps.chooseRuntimePort).not.toHaveBeenCalled();
|
|
|
|
// Without a connection of ours no session can be on the model: sessions elsewhere do not matter.
|
|
const fresh = setup({ state: failedOnPort });
|
|
fresh.deps.chooseRuntimePort.mockResolvedValue(11436);
|
|
const result = await fresh.service.resume({ newPort: true }, ADMIN, ['w1-app']);
|
|
expect(result.ok).toBe(true);
|
|
await fresh.service.idle();
|
|
});
|
|
|
|
it('a state write that fails before the saved endpoint moved changes nothing', async () => {
|
|
const h = setup({ state: INSTALLED });
|
|
h.deps.chooseRuntimePort.mockResolvedValue(11437);
|
|
h.store.disk.refuse = (next) => next.runtime?.port === 11437;
|
|
await expect(h.service.runtime({ action: 'start', newPort: true })).rejects.toThrow('ENOSPC');
|
|
expect(h.deps.moveLocalEndpoint).not.toHaveBeenCalled();
|
|
expect(h.store.get().runtime?.port).toBe(11435);
|
|
});
|
|
});
|