From fa978008d2963c457921835180153b6597ad3c66 Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 23:54:45 +0100 Subject: [PATCH] fix(local-model): claim the connection first, keep an outcome the disk refused, say why a start failed The connect step records the connection in the state file BEFORE it writes the endpoint record and the settings flag (the flag's earlier value read by inspectLocalEndpoint()), so a state write that fails or a server killed between the two never leaves a record of ours that the state file does not know; a record under the reserved id that says exactly what ours would say is ours. changePort() writes the state first and puts the old port back when the endpoint move is refused. A job whose end could not be written (a full disk) is kept in memory: the view shows it, Resume and a new install start from it, the next verb writes it, and Cancel never answers a 500. A first install on a full disk is a 507 with the sentence. "installing" in the file while this process runs no job always reads as paused or that failure. An installed model whose runtime does not start says so (runtime-start-failed with Start as the next step, and the launch check says it too); the stored runtime version is started while its folder is unpacked. Stop, Remove and the boot of anything that is not a running installation end a runtime an earlier server left behind before any file is deleted. Stop and "Use another port" answer 409 sessions-open while a live session uses the model. A pull that failed because the runtime is gone is runtime-stopped, not a network problem. After "does not fit" only a model that needs less memory is offered. A replaced job's partial blobs are deleted. Every 400 carries its sentence as data.message, and the six inline sentences are labels with zh-CN entries. Co-Authored-By: Claude Fable 5.1 --- src/local-model/connect.ts | 74 ++- src/local-model/install-job.ts | 621 +++++++++++++++---- src/local-model/ollama-client.ts | 4 +- src/types/local-model-install.ts | 68 ++- src/web/routes/custom-model-routes.ts | 12 +- src/web/routes/local-model-routes.ts | 45 +- test/local-model-connect.test.ts | 72 +++ test/local-model-install-e2e.test.ts | 92 ++- test/local-model-install-job.test.ts | 39 ++ test/local-model-install-service.test.ts | 729 ++++++++++++++++++++++- test/local-model-ollama-client.test.ts | 4 +- test/routes/custom-model-routes.test.ts | 2 +- test/routes/local-model-routes.test.ts | 69 ++- 13 files changed, 1647 insertions(+), 184 deletions(-) diff --git a/src/local-model/connect.ts b/src/local-model/connect.ts index a5001f9e..0af1fb12 100644 --- a/src/local-model/connect.ts +++ b/src/local-model/connect.ts @@ -15,6 +15,15 @@ * Rules: * - The id is reserved. A record with it that this feature did not write is never * overwritten: the connect step fails with `endpoint-id-taken`. + * - The connection is CLAIMED before it is written: the install job records `connection` in + * the state file first (`inspectLocalEndpoint()` tells it what is there and what the flag + * is), then calls `connectLocalModel()` with `ours: true`. A state write that fails, or a + * server that is killed between the two, therefore never leaves a record of ours that the + * state file does not know: the worst case is a claim with nothing behind it, which a + * resumed job fills in and Remove undoes. + * - Second net: a record with the reserved id that already says exactly what ours would say + * (`isSameLocalEndpoint()`) is ours whatever `ours` says. Nobody else can have made it: the + * endpoint routes refuse the reserved id. * - Both writes are idempotent, so a resumed job writes them again. * - Undoing removes our record, and switches the flag off again only when this feature * switched it on AND no other endpoint is saved. @@ -27,7 +36,10 @@ import { mutateCustomModelHosts } from '../custom-model-hosts-writer.js'; import type { CustomModelHost } from '../custom-model-hosts.js'; +import { getDataDir } from '../config/instance.js'; +import { readCustomModelHosts } from '../custom-model-hosts.js'; import { getErrorMessage, type LocalModelErrorCode } from '../types.js'; +import { readJsonConfig, SETTINGS_PATH } from '../web/route-helpers.js'; import { CustomModelHostSchema } from '../web/schemas.js'; import { mutateSettingsFile } from '../web/settings-writer.js'; import { readLocalModelState } from './state-store.js'; @@ -63,10 +75,58 @@ export function buildLocalEndpointRecord(input: LocalEndpointInput): CustomModel }; } +/** + * Whether a saved record says exactly what our record for `input` says: the same address, the + * one model, its context, no key. What discovery may have added to it (context lengths, a + * time stamp) does not count. Pure. + */ +export function isSameLocalEndpoint(host: CustomModelHost, input: LocalEndpointInput): boolean { + const ours = buildLocalEndpointRecord(input); + return ( + host.id === ours.id && + host.label === ours.label && + host.baseUrl === ours.baseUrl && + host.authStyle === ours.authStyle && + host.apiKey === undefined && + Array.isArray(host.models) && + host.models.length === 1 && + host.models[0] === input.modelId && + host.defaultModelId === ours.defaultModelId && + host.defaultContextLength === ours.defaultContextLength + ); +} + +/** What the connect step finds before it claims the connection. */ +export interface LocalEndpointInspection { + /** + * The saved record with the reserved id: there is none, it says what ours would say, or it + * says something else (somebody else's: `endpoint-id-taken`). + */ + record: 'absent' | 'same' | 'other'; + /** `customModelEndpointsEnabled` right now: what the claim keeps as the value before this feature. */ + flagEnabled: boolean; +} + +/** + * Looks at the endpoint store and the settings flag WITHOUT writing either. The install job + * calls it before it claims the connection in the state file. Rejects when the endpoint store + * cannot be read; a settings file that cannot be read counts as "the flag is off". + */ +export async function inspectLocalEndpoint(input: LocalEndpointInput): Promise { + const hosts = await readCustomModelHosts(getDataDir()); + const existing = hosts.find((host) => host.id === LOCAL_MODEL_ENDPOINT_ID); + const settings = await readJsonConfig>(SETTINGS_PATH, 'settings.json', {}); + return { + record: !existing ? 'absent' : isSameLocalEndpoint(existing, input) ? 'same' : 'other', + flagEnabled: settings[ENDPOINTS_FLAG] === true, + }; +} + export interface ConnectLocalModelInput extends LocalEndpointInput { /** * True when the state file already records a connection of ours: an existing record with - * our id is then ours and is replaced. False: such a record is somebody else's. + * our id is then ours and is replaced. False: such a record is somebody else's, unless it + * says exactly what ours would say (`isSameLocalEndpoint()`). */ ours: boolean; /** @@ -106,7 +166,7 @@ export async function connectLocalModel(input: ConnectLocalModelInput): Promise< outcome = await mutateCustomModelHosts<'added' | 'replaced' | 'taken'>((hosts) => { const index = hosts.findIndex((host) => host.id === record.id); if (index === -1) return { hosts: [...hosts, record], result: 'added' }; - if (!input.ours) return { result: 'taken' }; + if (!input.ours && !isSameLocalEndpoint(hosts[index], input)) return { result: 'taken' }; const next = [...hosts]; next[index] = record; return { hosts: next, result: 'replaced' }; @@ -158,13 +218,17 @@ async function removeLocalEndpoint(): Promise { }); } -/** Moves our record to another port (the answer to "Use another port" once connected). A missing record is not an error. */ +/** + * Moves our record to another port (the answer to "Use another port" once connected). A + * missing record is not an error, and a record that is already there is not written again. + */ export async function moveLocalEndpoint(port: number): Promise { + const baseUrl = `http://127.0.0.1:${port}`; await mutateCustomModelHosts((hosts) => { const index = hosts.findIndex((host) => host.id === LOCAL_MODEL_ENDPOINT_ID); - if (index === -1) return { result: undefined }; + if (index === -1 || hosts[index].baseUrl === baseUrl) return { result: undefined }; const next = [...hosts]; - next[index] = { ...next[index], baseUrl: `http://127.0.0.1:${port}` }; + next[index] = { ...next[index], baseUrl }; return { hosts: next, result: undefined }; }); } diff --git a/src/local-model/install-job.ts b/src/local-model/install-job.ts index 7d688f74..cc2ae0d8 100644 --- a/src/local-model/install-job.ts +++ b/src/local-model/install-job.ts @@ -41,7 +41,8 @@ * | | never removed by us | | | * | `connect` | our endpoint record and the settings | honoured before the first write; once | writes both again | * | | flag: the two writes outside our | the writes began they finish and the job | | - * | | folder | ends `installed` | | + * | | folder. The state file claims them | ends `installed` | | + * | | first | | | * * How the memory budget gets measured before a model is downloaded (ONE consent, bound to the * size shown): the person confirms `Candidate.installBytes` once. The `runtime` step then @@ -62,7 +63,9 @@ * - A job read as `installing` at boot was interrupted by the restart: the service applies * the event `restart` before anything else, so it is `paused` and waits for a click. * - A new install from `paused` or `failed` REPLACES the job. When it names another model, - * what the old job left of its model is deleted from the store first. + * what the old job left of its model is deleted from the store first: the model through the + * runtime when it was downloaded whole, and the partial blobs of a download that was + * interrupted (the runtime knows no model for those, so they are deleted as files). * - Install is checked before anything is downloaded: admin only in multi-user mode; the * model, the context and the coding agent must be a row the current recommendation offers; * the coding agent must be a harness row of the catalogue; `acknowledgedBytes` must be the @@ -80,13 +83,29 @@ * - A download's progress is kept in memory, written to the state file every few seconds and * when the job pauses or fails. The view shows the live figure and its speed. * - What blocks a session on an installed model (another program on the port, a runtime that - * keeps stopping, a model whose digest changed) is found live and shown as `error` on the - * view. It is never stored, and the state stays `installed`. - * - `launchRefusalMessage()` is what both session launch paths ask before they start a - * session on the managed endpoint: the digest is read again for every launch. + * keeps stopping or did not start, a model whose digest changed) is found live and shown as + * `error` on the view. It is never stored, and the state stays `installed`. + * - `launchRefusalMessage()` is what every path that STARTS a coding agent on the managed + * endpoint asks first: a new session with the model, a model applied to a session, the + * attach that starts an agent in a session that has none, and the rebuild after a reboot. + * The digest is read again for every one of them. Two things are not launches and are not + * asked: a pane that survived a server restart (its agent is already running), and a + * respawn cycle (it types into the agent that runs). * - With a model installed, `boot()` puts the saved endpoint record back when it is gone (a * config-backup restore can take it away), starts the runtime unless it was stopped, and * reads the digest again. + * - A runtime an earlier server left behind (a crash, a forced quit) is never left running + * under a deleted folder or beside a second one: its pid is stored the moment it is + * spawned, a start adopts or replaces it, and Stop, Remove and the boot of anything that + * is not a running installation end it (`RuntimeSupervisor.stop({ previousPid })`). + * - An installed model keeps the runtime VERSION it was installed with: when a later build + * pins another version, the stored one is started for as long as its folder is unpacked. + * - A job whose end could not be written to the state file (a full disk) is not lost: the + * outcome is kept in memory, the view shows it, Resume and a new install start from it, and + * the next verb writes it. The same holds for any `installing` in the file while this + * process runs no job, which is what a restart leaves too. + * - Stop and "Use another port" are refused (409 `sessions-open`) while live sessions use + * the model: their coding agents were given the address when they started. */ import { appendAdminAudit } from '../web/admin-audit.js'; @@ -124,6 +143,7 @@ import { type LocalModelInstallState, type LocalModelInstallStep, type LocalModelInstallView, + type LocalModelNextStep, type LocalModelProgress, type LocalModelRemoveRequest, type LocalModelResumeRequest, @@ -139,12 +159,14 @@ import { connectLocalModel, disconnectLocalModel, ensureLocalEndpoint, + inspectLocalEndpoint, isLocalModelEndpointId, LOCAL_MODEL_ENDPOINT_ID, moveLocalEndpoint, type ConnectLocalModelInput, type ConnectResult, type LocalEndpointInput, + type LocalEndpointInspection, } from './connect.js'; import { checkDeveloperTools } from './developer-tools.js'; import { formatGb } from './format.js'; @@ -164,8 +186,8 @@ import { type PullResult, type VerifyResult, } from './ollama-client.js'; -import { localModelPaths, type LocalModelPaths } from './paths.js'; -import { modelFitsBudget, recommend } from './recommend.js'; +import { localModelPaths, runtimeBinaryForVersion, type LocalModelPaths } from './paths.js'; +import { modelFitsBudget, needBytes, recommend } from './recommend.js'; import { chooseRuntimePort, createRuntimeSupervisor, @@ -173,6 +195,7 @@ import { isRuntimeInstalled, OLLAMA_RUNTIME_PIN, removeLocalModelFiles, + removePartialModelBlobs, RUNTIME_PORT_FIRST, type ChooseRuntimePortOptions, type RemoveLocalModelFilesOptions, @@ -304,7 +327,15 @@ export const INSTALL_STEP_TABLE: Readonly; detail?: string; nextModel?: { modelId: string; modelLabel: string } } = {} + extra: { + vars?: Record; + detail?: string; + nextModel?: { modelId: string; modelLabel: string }; + /** The next step in the state the error is shown in, when it is not the code's default. */ + next?: LocalModelNextStep; + } = {} ): LocalModelInstallError { - const next = LOCAL_MODEL_ERROR_NEXT[code]; + const next = extra.next ?? LOCAL_MODEL_ERROR_NEXT[code]; return { code, message: { @@ -542,8 +581,8 @@ export function buildInstallError( /** * What blocks a session in state `installed`, found live and never stored: another program - * on the port, a runtime that keeps stopping, a model that changed. `undefined` when nothing - * does, which includes a runtime the person stopped. Pure. + * on the port, a runtime that keeps stopping, a runtime that did not start, a model that + * changed. `undefined` when nothing does, which includes a runtime the person stopped. Pure. */ export function installedBlocker( state: LocalModelStateFile, @@ -554,6 +593,13 @@ export function installedBlocker( return buildInstallError('port-in-use', { vars: { port: String(state.runtime?.port ?? '') } }); } if (!live.running && live.gaveUp) return buildInstallError('runtime-stopped'); + if (!live.running && live.startFailed) { + // Start is the verb an installed model has: it tries the start again. + return buildInstallError('runtime-start-failed', { + next: 'start', + ...(live.startFailed.detail ? { detail: live.startFailed.detail } : {}), + }); + } if (live.digest === 'changed') return buildInstallError('model-changed'); return undefined; } @@ -646,6 +692,8 @@ export function buildInstallView(state: LocalModelStateFile, live: InstallLiveFa vars: state.error.vars, detail: state.error.detail, nextModel: live.nextModel, + // A job has no Start verb: what helps there is to run the step again. + ...(LOCAL_MODEL_ERROR_NEXT[state.error.code] === 'start' ? { next: 'retry' as const } : {}), }); } return view; @@ -667,10 +715,11 @@ export interface LocalModelActor { ip?: string; } -/** How a verb ended: the view after it, a 400 with its message, or a 409 with its conflict. */ +/** How a verb ended: the view after it, a refusal with its sentence, or a 409 with its conflict. */ export type LocalModelVerbResult = | { ok: true; view: LocalModelInstallView } - | { ok: false; status: 400; message: string } + /** 400: the request is not something the server offers. 507: nothing could be started because the disk is full. */ + | { ok: false; status: 400 | 507; message: LocalModelInstallMessage } | { ok: false; status: 409; conflict: LocalModelConflict }; /** @@ -710,6 +759,8 @@ export interface LocalModelServiceDeps { listRunning: (target: OllamaTarget, signal?: AbortSignal) => Promise; getRuntimeVersion: (target: OllamaTarget, signal?: AbortSignal) => Promise; deleteModel: (target: OllamaTarget, modelId: string, signal?: AbortSignal) => Promise; + /** `removePartialModelBlobs()` of src/local-model/runtime.ts: what an interrupted download left in the model store. Resolves with the bytes freed. */ + removePartialModelBlobs: (paths?: LocalModelPaths) => Promise; // src/local-model/developer-tools.ts checkDeveloperTools: () => Promise; // src/cli-install-runner.ts @@ -721,6 +772,8 @@ export interface LocalModelServiceDeps { /** Writes one admin audit line (`appendAdminAudit()`). */ audit: (entry: { admin: string; action: string; target: string; ip?: string; detail?: unknown }) => Promise; // src/local-model/connect.ts + /** What is saved under the reserved id and what the settings flag is, read before the connection is claimed. Writes nothing. */ + inspectLocalEndpoint: (input: LocalEndpointInput) => Promise; connectLocalModel: (input: ConnectLocalModelInput) => Promise; moveLocalEndpoint: (port: number) => Promise; /** Puts our endpoint record back when it is gone (after a config-backup restore). True when it was added. */ @@ -749,9 +802,10 @@ export interface LocalModelService { * yes, which is also the answer for every endpoint that is not this feature's. Otherwise the * error that refuses the launch. While the runtime child Codeman started is not alive * (whatever answers on the port then is not ours), the reason is the one that is known: - * `port-in-use`, `runtime-stopped`, else `runtime-not-running`. While it runs, the digest is - * read again: `model-changed` when it is not the stored one or the model is gone. The session - * launch path asks this before it starts a session with the managed endpoint. Never throws. + * `port-in-use`, `runtime-stopped`, `runtime-start-failed`, else `runtime-not-running`. + * While it runs, the digest is read again: `model-changed` when it is not the stored one or + * the model is gone. Every path that starts a coding agent on the managed endpoint asks + * this first (see the file header for the list). Never throws. */ launchRefusal(endpointId: string): Promise; /** @@ -767,10 +821,21 @@ export interface LocalModelService { install(request: LocalModelInstallRequest, actor: LocalModelActor): Promise; /** `POST /api/local-model/cancel`. Answers once the step in flight was interrupted. */ cancel(): Promise; - /** `POST /api/local-model/resume`. */ - resume(request: LocalModelResumeRequest, actor: LocalModelActor): Promise; - /** `POST /api/local-model/runtime`. */ - runtime(request: LocalModelRuntimeRequest): Promise; + /** + * `POST /api/local-model/resume`. `liveSessionNames` as for `remove()`: with `newPort` and a + * connection of ours, not empty answers 409 `sessions-open`. + */ + resume( + request: LocalModelResumeRequest, + actor: LocalModelActor, + liveSessionNames?: readonly string[] + ): Promise; + /** + * `POST /api/local-model/runtime`. `liveSessionNames` as for `remove()`: Stop and a start on + * another port answer 409 `sessions-open` while one of them uses the model, because their + * coding agents keep calling the address they were started with. + */ + runtime(request: LocalModelRuntimeRequest, liveSessionNames?: readonly string[]): Promise; /** * `DELETE /api/local-model`. `liveSessionNames` is what `sessionNamesUsingEndpoint()` found: * not empty answers 409 `sessions-open` while a connection of ours exists. Else: stop the @@ -788,13 +853,14 @@ export interface LocalModelService { * Called once when the server starts. An interrupted job becomes `paused`. With a model * installed: the saved endpoint record is put back when it is gone, a runtime that was not * stopped is started (an own orphan on the port is adopted or replaced, another listener is - * `port-in-use`) and the digest is read again. Never throws, and never keeps the server - * from starting. + * `port-in-use`) and the digest is read again. In every other case a runtime an earlier + * server left behind is ended: nothing would use it, and it holds the model's memory. + * Never throws, and never keeps the server from starting. */ boot(): Promise; /** Called when the server stops, beside `stopDeepSeekWeb()`: aborts a job in flight (it reads `paused` at the next boot) and stops the runtime. */ shutdown(): Promise; - /** What the runtime supervisor reports through `onChange`: a new pid after a restart, or the budget line. Stored, never thrown. */ + /** What the runtime supervisor reports through `onChange`: the pid of a child the moment it is spawned, a new pid after a restart, the budget line. Stored, never thrown. */ onRuntimeChange(status: RuntimeStatus): void; /** Resolves once no job is in flight. For `shutdown()` and for tests. */ idle(): Promise; @@ -842,13 +908,23 @@ function lazySupervisor(build: () => RuntimeSupervisor): RuntimeSupervisor { let instance: RuntimeSupervisor | undefined; return { start: (options) => (instance ??= build()).start(options), - stop: async () => { - if (instance) await instance.stop(); + stop: async (options) => { + // Built for a stored pid too: a runtime an earlier server left behind has no instance here. + if (instance || options?.previousPid !== undefined) await (instance ??= build()).stop(options); }, status: () => (instance ? instance.status() : { running: false, restarts: 0, gaveUp: false }), }; } +/** Whether a write failed for lack of space. */ +function isDiskFullError(err: unknown): boolean { + const code = typeof err === 'object' && err !== null ? (err as NodeJS.ErrnoException).code : undefined; + return code === 'ENOSPC' || code === 'EDQUOT' || /no space left on device/i.test(getErrorMessage(err)); +} + +/** How a job ended, as the `fail` event carries it. */ +type JobFailure = Omit, 'type'>; + /** Forgets the cached lookups of a CLI's binaries the way the CLI registry routes do, then probes it again. */ async function recheckCliInstalledNow(cli: string): Promise { const entry = getCli(cli); @@ -866,12 +942,22 @@ class LocalModelServiceImpl implements LocalModelService { private verbChain: Promise = Promise.resolve(); private recovery: Promise | undefined; private stopping = false; + /** A verb wrote `installing` and is about to start the job: the view must not take that for a job that is gone. */ + private jobStarting = false; + /** + * How the last job ended when the state file could not be told (a full disk): the failure, + * or nothing for a pause, and the download as it stood. Shown from memory and written by + * the next verb. + */ + private unrecorded: { failure?: JobFailure; progress?: { completedBytes: number; totalBytes: number } } | undefined; /** The download in flight, as last reported. The state file gets it every few seconds. */ private liveProgress: { completedBytes: number; totalBytes: number; bytesPerSecond?: number } | undefined; private lastProgressFlush = 0; private speedSamples: Array<{ at: number; bytes: number }> = []; /** The last start found another program on the stored port. */ private portInUse = false; + /** The last start failed for another reason, with what the runtime said. Cleared by a start that works and by Stop. */ + private startFailed: { detail?: string } | undefined; private digestStatus: 'same' | 'changed' | 'unknown' | undefined; private digestCheckedAt = 0; @@ -897,12 +983,14 @@ class LocalModelServiceImpl implements LocalModelService { listRunning, getRuntimeVersion, deleteModel, + removePartialModelBlobs, checkDeveloperTools: () => checkDeveloperTools(), stockInstallCommand, tryBeginCliInstall, runInstallCommand, recheckCliInstalled: recheckCliInstalledNow, audit: (entry) => appendAdminAudit({ ...entry, detail: entry.detail as Record | undefined }), + inspectLocalEndpoint, connectLocalModel, moveLocalEndpoint, ensureLocalEndpoint, @@ -930,7 +1018,7 @@ class LocalModelServiceImpl implements LocalModelService { return run; } - /** Applies one event to the stored state. False when the reducer refused it. */ + /** Applies one event of the running job to the stored state. False when the reducer refused it. */ private async dispatch(event: InstallJobEvent): Promise { const { after } = await this.deps.mutateState((current) => { const next = reduceInstallJob(current, event); @@ -939,6 +1027,76 @@ class LocalModelServiceImpl implements LocalModelService { return after !== undefined; } + /** + * `state` as it stands once a job that is over is counted as over: `installing` in the file + * while this process runs no job is a job that ended without the file knowing. It is + * `failed` with the outcome kept in memory, or `paused` when there is none (a restart, a + * Cancel whose write was refused). A paused or failed job with an outcome in memory is a + * Resume that could not be written. Pure apart from reading `this.job` and `this.unrecorded`. + */ + private settled(state: LocalModelStateFile): LocalModelStateFile { + if (this.job) return state; + const failure = this.unrecorded?.failure; + let next: LocalModelStateFile; + if (state.state === 'installing') { + next = failure + ? reduceInstallJob(state, { type: 'fail', ...failure }) + : reduceInstallJob(state, { type: 'restart' }); + } else if (failure && (state.state === 'paused' || state.state === 'failed')) { + // A Resume that could not even be written (the disk is full): the job did not start, + // and it says why instead of looking as if the click did nothing. + next = reduceInstallJob({ ...state, state: 'installing' }, { type: 'fail', ...failure }); + } else { + return state; + } + const progress = this.unrecorded?.progress; + return progress ? { ...next, progress } : next; + } + + /** The stored state as a view or a verb must see it: settled, unless a verb is just starting a job. */ + private async currentState(): Promise { + const state = await this.deps.readState(); + return this.jobStarting ? state : this.settled(state); + } + + /** + * Applies one event of a VERB to the stored state, after settling it (`settled()`), in one + * write. False when the reducer refused the event; the settlement is written all the same. + */ + private async verbDispatch(event: InstallJobEvent): Promise { + let applied = false; + let settledNow = false; + await this.deps.mutateState((current) => { + const base = this.settled(current); + settledNow = base !== current; + const next = reduceInstallJob(base, event); + applied = next !== base; + if (applied) return next; + return settledNow ? base : undefined; + }); + if (settledNow || applied) this.unrecorded = undefined; + return applied; + } + + /** + * Writes the settlement alone, so the file says what the view already shows. Best effort: a + * write that is refused (the disk is still full) changes nothing, the outcome stays in + * memory and the next verb tries again. + */ + private async settle(): Promise { + try { + let settledNow = false; + await this.deps.mutateState((current) => { + const base = this.settled(current); + settledNow = base !== current; + return settledNow ? base : undefined; + }); + if (settledNow) this.unrecorded = undefined; + } catch (err) { + console.warn(`[local-model] Could not write the state of the install: ${getErrorMessage(err)}`); + } + } + /** Writes facts a step learned. Never moves the state, and writes nothing once the job's state is gone. */ private async record(change: (current: LocalModelStateFile) => LocalModelStateFile | undefined): Promise { await this.deps.mutateState((current) => (current.state === 'idle' ? undefined : change(current))); @@ -946,16 +1104,12 @@ class LocalModelServiceImpl implements LocalModelService { /** * A job the state file still calls `installing` when this process has none is one a restart - * interrupted: it becomes `paused`. Done once, before the first view or verb, whether or not - * `boot()` ran yet. + * interrupted: it becomes `paused`. Written once per process, before the first view or verb, + * whether or not `boot()` ran yet. This is the one write a GET makes; a job that ends later + * without the file knowing is shown from memory (`settled()`) and written by the next verb. */ private recovered(): Promise { - this.recovery ??= this.dispatch({ type: 'restart' }).then( - () => undefined, - (err: unknown) => { - console.warn(`[local-model] Could not read the state at start: ${getErrorMessage(err)}`); - } - ); + this.recovery ??= this.settle(); return this.recovery; } @@ -967,6 +1121,28 @@ class LocalModelServiceImpl implements LocalModelService { } } + /** + * The runtime version to start: the pinned one when it is unpacked, else the version the + * state file records when THAT is still unpacked (a model installed by an earlier build + * with another pin keeps running), else the pinned one, whose start then says what is + * missing. + */ + private async runtimeVersionToRun(state: LocalModelStateFile): Promise { + const pinned = this.deps.pin.version; + const stored = state.runtime?.version; + if (!stored || stored === pinned || (await this.isRuntimeOnDisk())) return pinned; + try { + const paths = this.paths(); + const binary = runtimeBinaryForVersion(paths, stored); + if (await this.deps.isRuntimeInstalled({ ...this.deps.pin, version: stored }, { ...paths, binary })) { + return stored; + } + } catch { + /* not a version this build can name: the pinned one is tried */ + } + return pinned; + } + private errorVars(code: LocalModelErrorCode, state: LocalModelStateFile): Record | undefined { const model = state.model?.label ?? ''; switch (code) { @@ -987,15 +1163,38 @@ class LocalModelServiceImpl implements LocalModelService { } } - /** The best model the current recommendation offers that is not `failedId`. */ - private async nextModelId(failedId: string, options?: RecommendOptions): Promise { + /** + * The model to offer after `failed` did not work: the best one the current recommendation + * offers that is another model. With `smallerThan` (the memory the failed model needs, for + * `does-not-fit`) only a model that needs LESS is offered: one that needs more would fail + * the same way. `undefined` when there is none. + */ + private async nextModelId( + failed: { id: string; context: number }, + options?: RecommendOptions, + rule: { needsLessMemory?: boolean } = {} + ): Promise { try { const hw = await this.deps.readHardware(); const catalog = this.deps.getCatalog(); const facts = await this.deps.buildHarnessFacts(catalog); const rec = this.deps.recommend(hw, catalog, facts, options ?? (await this.recommendOptions())); - const rows = [rec.pick, ...rec.alternatives].filter((row): row is Candidate => row !== undefined); - return rows.find((row) => row.modelId !== failedId)?.modelId; + const rows = [rec.pick, ...rec.alternatives].filter( + (row): row is Candidate => row !== undefined && row.modelId !== failed.id + ); + if (!rule.needsLessMemory) return rows[0]?.modelId; + // What the failed model needs: its row when the recommendation still has it, else the + // catalogue's figures for it at the context that was chosen. + const failedRow = [rec.pick, ...rec.alternatives].find((row) => row?.modelId === failed.id); + const entry = catalog.models.find((model) => model.id === failed.id); + const limit = + failedRow && failedRow.context === failed.context + ? failedRow.needBytes + : entry + ? needBytes(entry, failed.context) + : undefined; + if (limit === undefined || !Number.isFinite(limit)) return undefined; + return rows.find((row) => row.needBytes < limit)?.modelId; } catch { return undefined; } @@ -1038,6 +1237,12 @@ class LocalModelServiceImpl implements LocalModelService { }); } + /** The download as it stands, for a job that ends: what a paused or failed job keeps. */ + private progressNow(): { completedBytes: number; totalBytes: number } | undefined { + const progress = this.liveProgress; + return progress ? { completedBytes: progress.completedBytes, totalBytes: progress.totalBytes } : undefined; + } + private clearProgress(): void { this.liveProgress = undefined; this.speedSamples = []; @@ -1052,17 +1257,24 @@ class LocalModelServiceImpl implements LocalModelService { return { ok: false, code: 'runtime-start-failed', detail: 'No port was chosen for the runtime yet.' }; } const { port } = state.runtime; + const version = await this.runtimeVersionToRun(state); let result: RuntimeStartResult; try { result = await this.deps.supervisor.start({ port, contextLength: state.model.context, ...(state.runtime.pid !== undefined ? { previousPid: state.runtime.pid } : {}), + ...(version !== this.deps.pin.version ? { version } : {}), }); } catch (err) { result = { ok: false, code: 'runtime-start-failed', detail: getErrorMessage(err) }; } this.portInUse = !result.ok && result.code === 'port-in-use'; + // Kept, so an installed model whose runtime does not start says so instead of only "stopped". + this.startFailed = + !result.ok && result.code === 'runtime-start-failed' + ? { ...(result.detail ? { detail: result.detail } : {}) } + : undefined; if (result.ok) { const { pid, measuredTotalBytes } = result; await this.record((current) => @@ -1071,7 +1283,7 @@ class LocalModelServiceImpl implements LocalModelService { ...current, runtime: { ...current.runtime, - version: this.deps.pin.version, + version, pid, ...(measuredTotalBytes !== undefined ? { measuredTotalBytes } : {}), }, @@ -1111,7 +1323,9 @@ class LocalModelServiceImpl implements LocalModelService { } onRuntimeChange(status: RuntimeStatus): void { - if (!status.running || status.pid === undefined) return; + // The pid is stored from the moment the child is spawned, not only once it answers: a + // server that is killed in between must still find its own runtime at the next start. + if (status.pid === undefined) return; const { pid, measuredTotalBytes } = status; void this.record((current) => { if (!current.runtime) return undefined; @@ -1149,6 +1363,7 @@ class LocalModelServiceImpl implements LocalModelService { running: status.running, gaveUp: status.gaveUp, portInUse: this.portInUse, + ...(this.startFailed ? { startFailed: this.startFailed } : {}), ...(state.state === 'installed' && this.digestStatus ? { digest: this.digestStatus } : {}), developerTools, runtimeInstalled: await this.isRuntimeOnDisk(), @@ -1162,7 +1377,7 @@ class LocalModelServiceImpl implements LocalModelService { async view(): Promise { try { await this.recovered(); - let state = await this.deps.readState(); + let state = await this.currentState(); if (state.state === 'installing' && this.liveProgress) { state = { ...state, @@ -1214,13 +1429,14 @@ class LocalModelServiceImpl implements LocalModelService { if (!isLocalModelEndpointId(endpointId)) return undefined; try { await this.recovered(); - const state = await this.deps.readState(); + const state = await this.currentState(); // A record with the reserved id that this feature did not write is not ours to judge. if (state.connection?.endpointId !== endpointId) return undefined; const status = this.deps.supervisor.status(); if (state.state !== 'installed' || !state.runtime || !state.model || !status.running) { if (this.portInUse) return buildInstallError('port-in-use', { vars: this.errorVars('port-in-use', state) }); if (status.gaveUp) return buildInstallError('runtime-stopped'); + if (this.startFailed) return buildInstallError('runtime-start-failed', { next: 'start' }); return buildInstallError('runtime-not-running'); } const digest = await this.refreshDigest(state); @@ -1253,7 +1469,10 @@ class LocalModelServiceImpl implements LocalModelService { try { const state = await this.deps.readState(); if (state.connection?.endpointId === endpointId && state.model && state.model.id !== modelId) { - return `Only ${state.model.label} is installed on this Mac. Remove it and install another model first.`; + return formatInstallMessage({ + text: LOCAL_MODEL_INSTALL_LABELS.refusedOtherModel, + vars: { model: state.model.label }, + }); } } catch { /* an unreadable state is "nothing of ours": nothing to refuse */ @@ -1269,8 +1488,8 @@ class LocalModelServiceImpl implements LocalModelService { install(request: LocalModelInstallRequest, actor: LocalModelActor): Promise { return this.exclusive(async () => { - await this.recovered(); - const state = await this.deps.readState(); + await this.settle(); + const state = await this.currentState(); if (state.state === 'installing') return conflict('job-running'); if (state.state === 'installed') return conflict('already-installed'); @@ -1284,7 +1503,7 @@ class LocalModelServiceImpl implements LocalModelService { const rows = [rec.pick, ...rec.alternatives].filter((row): row is Candidate => row !== undefined); const candidate = rows.find((row) => row.modelId === request.modelId); if (!candidate || candidate.context !== request.context) { - return { ok: false, status: 400, message: 'This model at this context is not offered for this Mac right now.' }; + return { ok: false, status: 400, message: { text: LOCAL_MODEL_INSTALL_LABELS.refusedModelNotOffered } }; } // A row without the list (an older shape) still names its one coding agent in the flat fields. const harnesses: CandidateHarness[] = candidate.harnesses ?? [ @@ -1292,13 +1511,13 @@ class LocalModelServiceImpl implements LocalModelService { ]; const harness = harnesses.find((row) => row.cli === request.cli); if (!harness || !catalog.harnesses.some((row) => row.cli === request.cli)) { - return { ok: false, status: 400, message: 'This coding agent is not offered for this model.' }; + return { ok: false, status: 400, message: { text: LOCAL_MODEL_INSTALL_LABELS.refusedAgentNotOffered } }; } if (!harness.installed && !this.deps.stockInstallCommand(harness.cli)) { return { ok: false, status: 400, - message: `${harness.cliLabel} is not installed and cannot be installed from here.`, + message: { text: LOCAL_MODEL_INSTALL_LABELS.refusedAgentNotInstallable, vars: { cli: harness.cliLabel } }, }; } const expectedBytes = candidate.installBytes ?? candidate.downloadBytes + (options.runtimeDownloadBytes ?? 0); @@ -1311,19 +1530,17 @@ class LocalModelServiceImpl implements LocalModelService { return { ok: false, status: 400, - message: formatInstallMessage( - buildInstallError('endpoint-id-taken', { vars: { id: LOCAL_MODEL_ENDPOINT_ID } }).message - ), + message: buildInstallError('endpoint-id-taken', { vars: { id: LOCAL_MODEL_ENDPOINT_ID } }).message, }; } - const started = await this.dispatch({ + const refused = await this.beginJob(actor, { type: 'install', model: { id: candidate.modelId, label: candidate.modelLabel, context: candidate.context }, cli: { id: harness.cli, label: harness.cliLabel }, steps: jobSteps({ agentInstalled: harness.installed }), }); - if (!started) return conflict('wrong-state'); + if (refused) return refused; void this.deps .audit({ admin: actor.username, @@ -1333,71 +1550,149 @@ class LocalModelServiceImpl implements LocalModelService { detail: { cli: harness.cli, context: candidate.context, bytes: expectedBytes }, }) .catch(() => {}); - this.startJob(actor); return this.answer(); }); } + /** + * Writes the event that starts a job (`install` or `resume`) and starts the job. `undefined`: + * started. A state file that cannot be written for lack of space is not a 500: the job did + * not start, and the view says `disk-full` with its "Try again" (a first install, which has + * no job to show it on, is answered with that sentence). + */ + private async beginJob( + actor: LocalModelActor, + event: Extract + ): Promise { + this.jobStarting = true; + try { + const started = await this.verbDispatch(event); + if (!started) return conflict('wrong-state'); + this.startJob(actor); + return undefined; + } catch (err) { + if (!isDiskFullError(err)) throw err; + // A first install has no job to show the failure on: the sentence goes back as it is. + if (event.type === 'install' && (await this.currentState().catch(() => undefined))?.state === 'idle') { + return { ok: false, status: 507, message: { text: LOCAL_MODEL_INSTALL_LABELS.errDiskFull } }; + } + const kept = this.unrecorded?.progress; + this.unrecorded = { + failure: { code: 'disk-full', detail: getErrorMessage(err) }, + ...(kept ? { progress: kept } : {}), + }; + return this.answer(); + } finally { + this.jobStarting = false; + } + } + cancel(): Promise { return this.exclusive(async () => { - await this.recovered(); - const state = await this.deps.readState(); - if (state.state !== 'installing') return conflict('wrong-state'); + // Taken before anything is awaited: a Cancel that arrives while a job runs is answered + // with the view after it, also when the job finishes in the same moment. const job = this.job; - if (job) { - job.controller.abort(); - await job.promise.catch(() => {}); + if (!job) { + // No job in this process: a job a restart interrupted is paused, never running. + await this.settle(); + return conflict('wrong-state'); } + job.controller.abort(); + await job.promise.catch(() => {}); // The step may have finished the job while it was being interrupted (the connect step // completes its two writes). Then there is nothing left to pause, and the view says so. - await this.flushProgress().catch(() => {}); - await this.dispatch({ type: 'cancel' }); + // Otherwise the job is paused with its download as it stood, in ONE write. + const progress = this.progressNow(); + try { + await this.deps.mutateState((current) => { + if (current.state !== 'installing') return undefined; + const kept = progress ? reduceInstallJob(current, { type: 'progress', ...progress }) : current; + return reduceInstallJob(kept, { type: 'cancel' }); + }); + } catch (err) { + // The job IS stopped. The file could not be told: the view shows the pause from memory. + console.warn(`[local-model] Could not write the paused install: ${getErrorMessage(err)}`); + this.unrecorded = { ...(progress ? { progress } : {}) }; + } this.clearProgress(); return this.answer(); }); } - resume(request: LocalModelResumeRequest, actor: LocalModelActor): Promise { + resume( + request: LocalModelResumeRequest, + actor: LocalModelActor, + liveSessionNames: readonly string[] = [] + ): Promise { return this.exclusive(async () => { - await this.recovered(); - const state = await this.deps.readState(); + await this.settle(); + const state = await this.currentState(); if (state.state === 'installing') return conflict('job-running'); if (state.state !== 'paused' && state.state !== 'failed') return conflict('wrong-state'); - if (request.newPort === true && state.runtime) await this.changePort(state); - const resumed = await this.dispatch({ type: 'resume' }); - if (!resumed) return conflict('wrong-state'); - this.startJob(actor); - return this.answer(); + if (request.newPort === true && state.runtime) { + const open = this.sessionsOpen(state, liveSessionNames); + if (open) return open; + await this.changePort(state); + } + return (await this.beginJob(actor, { type: 'resume' })) ?? this.answer(); }); } - /** Chooses another port than the stored one and moves the saved endpoint with it. False when no other port is free. */ + /** 409 `sessions-open` while live sessions use a connection of ours, else `undefined`. */ + private sessionsOpen( + state: LocalModelStateFile, + liveSessionNames: readonly string[] + ): LocalModelVerbResult | undefined { + if (!state.connection || liveSessionNames.length === 0) return undefined; + const sessions = [...liveSessionNames]; + return conflict('sessions-open', { sessions }, { sessions: sessions.join(', ') }); + } + + /** + * Chooses another port than the stored one and moves the saved endpoint with it. False when + * no other port is free. The state file first, then the saved endpoint: a state write that + * fails changes nothing, and a refused endpoint write puts the old port back, so the two + * never disagree. The stored pid stays: when it is still our own runtime on the OLD port, + * the next start ends it instead of leaving a second runtime behind. + */ private async changePort(state: LocalModelStateFile): Promise { if (!state.runtime) return false; - const port = await this.deps.chooseRuntimePort({ avoid: [state.runtime.port] }); + const previous = state.runtime.port; + const port = await this.deps.chooseRuntimePort({ avoid: [previous] }); if (port === undefined) return false; - // The saved endpoint first: when that write is refused, nothing has changed yet. - if (state.connection) await this.deps.moveLocalEndpoint(port); - await this.record((current) => { - if (!current.runtime) return undefined; - // The stored pid belonged to the old port: it must not make a listener on the new one "ours". - const { pid: _pid, ...runtime } = current.runtime; - return { ...current, runtime: { ...runtime, port } }; - }); + const setPort = (value: number): Promise => + this.record((current) => + current.runtime ? { ...current, runtime: { ...current.runtime, port: value } } : undefined + ); + await setPort(port); + if (state.connection) { + try { + await this.deps.moveLocalEndpoint(port); + } catch (err) { + await setPort(previous).catch(() => {}); + throw err; + } + } this.portInUse = false; return true; } - runtime(request: LocalModelRuntimeRequest): Promise { + runtime(request: LocalModelRuntimeRequest, liveSessionNames: readonly string[] = []): Promise { return this.exclusive(async () => { - await this.recovered(); - let state = await this.deps.readState(); + await this.settle(); + let state = await this.currentState(); if (state.state === 'installing') return conflict('job-running'); if (state.state !== 'installed' || !state.runtime) return conflict('wrong-state'); if (request.action === 'stop') { - await this.deps.supervisor.stop(); + // Their coding agents keep calling the address they were started with. + const open = this.sessionsOpen(state, liveSessionNames); + if (open) return open; + await this.deps.supervisor.stop({ + ...(state.runtime.pid !== undefined ? { previousPid: state.runtime.pid } : {}), + }); this.portInUse = false; + this.startFailed = undefined; await this.record((current) => current.runtime ? { ...current, runtime: { ...current.runtime, enabled: false } } : undefined ); @@ -1405,8 +1700,12 @@ class LocalModelServiceImpl implements LocalModelService { } if (request.newPort === true) { + const open = this.sessionsOpen(state, liveSessionNames); + if (open) return open; // The old child, when one is still alive, holds the old port: it must go first. - await this.deps.supervisor.stop(); + await this.deps.supervisor.stop({ + ...(state.runtime.pid !== undefined ? { previousPid: state.runtime.pid } : {}), + }); await this.changePort(state); } await this.record((current) => @@ -1425,15 +1724,17 @@ class LocalModelServiceImpl implements LocalModelService { actor?: LocalModelActor ): Promise { return this.exclusive(async () => { - await this.recovered(); - const state = await this.deps.readState(); + await this.settle(); + const state = await this.currentState(); if (state.state === 'installing') return conflict('job-running'); - if (state.connection && liveSessionNames.length > 0) { - const sessions = [...liveSessionNames]; - return conflict('sessions-open', { sessions }, { sessions: sessions.join(', ') }); - } + const open = this.sessionsOpen(state, liveSessionNames); + if (open) return open; - await this.deps.supervisor.stop(); + // Our child, and a runtime an earlier server left behind (known by the stored pid): no + // file is deleted under a process that still runs from it. + await this.deps.supervisor.stop({ + ...(state.runtime?.pid !== undefined ? { previousPid: state.runtime.pid } : {}), + }); // The two writes outside our folder are undone first: when one is refused, nothing was // deleted yet and the person can try again. if (state.connection) { @@ -1441,9 +1742,11 @@ class LocalModelServiceImpl implements LocalModelService { } const removeModel = request.removeModel === true; await this.deps.removeLocalModelFiles({ models: removeModel, paths: this.paths() }); - await this.dispatch({ type: 'remove' }); + await this.verbDispatch({ type: 'remove' }); this.clearProgress(); + this.unrecorded = undefined; this.portInUse = false; + this.startFailed = undefined; this.digestStatus = undefined; if (actor) { void this.deps @@ -1463,11 +1766,11 @@ class LocalModelServiceImpl implements LocalModelService { async boot(): Promise { try { await this.recovered(); - const state = await this.deps.readState(); - if (state.state !== 'installed' || !state.runtime || !state.model) return; + const state = await this.currentState(); + const installed = state.state === 'installed' && state.runtime !== undefined && state.model !== undefined; // The record of the connection can be gone while the model is still installed (a // config-backup restore of an older endpoint store): it is put back, never replaced. - if (state.connection) { + if (installed && state.connection && state.runtime && state.model) { const restored = await this.deps.ensureLocalEndpoint({ port: state.runtime.port, modelId: state.model.id, @@ -1476,7 +1779,15 @@ class LocalModelServiceImpl implements LocalModelService { if (restored) console.warn('[local-model] The saved endpoint of the model on this Mac was missing and was put back.'); } - if (!state.runtime.enabled) return; + if (!installed || !state.runtime?.enabled) { + // Nothing starts the runtime here (a job waits for a click, the person stopped the + // model). One an earlier server left behind is ended: it holds the model's memory, + // and a later Remove must not delete its folder under it. + if (state.runtime?.pid !== undefined) { + await this.deps.supervisor.stop({ previousPid: state.runtime.pid }); + } + return; + } const started = await this.startRuntime(state); if (!started.ok) { console.warn( @@ -1519,6 +1830,7 @@ class LocalModelServiceImpl implements LocalModelService { private startJob(actor: LocalModelActor): void { const controller = new AbortController(); this.clearProgress(); + this.unrecorded = undefined; const job = { controller, promise: this.runJob(actor, controller.signal).finally(() => { @@ -1538,7 +1850,12 @@ class LocalModelServiceImpl implements LocalModelService { try { outcome = await this.runStep(state.step, state, actor, signal); } catch (err) { - outcome = { kind: 'fail', code: 'internal', detail: getErrorMessage(err) }; + // A write of the step itself that found no space is a full disk, not a mystery. + outcome = { + kind: 'fail', + code: isDiskFullError(err) ? 'disk-full' : 'internal', + detail: getErrorMessage(err), + }; } if (outcome.kind === 'done') { // A step that finished stays finished, also when Cancel arrived while it ran. Its @@ -1550,19 +1867,38 @@ class LocalModelServiceImpl implements LocalModelService { // An interrupted step is not a failed one, whatever its last call answered. Cancel // writes `paused` itself; a server that is stopping writes nothing. if (outcome.kind === 'cancelled' || signal.aborted || this.stopping) return; - await this.flushProgress().catch(() => {}); - await this.dispatch({ - type: 'fail', + const failure: JobFailure = { code: outcome.code, ...(outcome.vars ? { vars: outcome.vars } : {}), ...(outcome.detail ? { detail: outcome.detail } : {}), ...(outcome.nextModelId ? { nextModelId: outcome.nextModelId } : {}), - }); + }; + const progress = this.progressNow(); + try { + // The failure and the download as it stood, in ONE write. + await this.deps.mutateState((current) => { + const kept = progress ? reduceInstallJob(current, { type: 'progress', ...progress }) : current; + const next = reduceInstallJob(kept, { type: 'fail', ...failure }); + return next === current ? undefined : next; + }); + } catch (err) { + // The disk that failed the step also refuses the state file. The outcome is not + // lost: the view shows it from memory and the next verb writes it. + console.warn(`[local-model] Could not write why the install stopped: ${getErrorMessage(err)}`); + this.unrecorded = { failure, ...(progress ? { progress } : {}) }; + } + this.clearProgress(); return; } } catch (err) { + // A write of the job itself was refused (the step that finished could not be recorded). console.warn(`[local-model] The install job stopped: ${getErrorMessage(err)}`); - await this.dispatch({ type: 'fail', code: 'internal', detail: getErrorMessage(err) }).catch(() => {}); + const progress = this.progressNow(); + this.unrecorded = { + failure: { code: isDiskFullError(err) ? 'disk-full' : 'internal', detail: getErrorMessage(err) }, + ...(progress ? { progress } : {}), + }; + this.clearProgress(); } } @@ -1629,7 +1965,7 @@ class LocalModelServiceImpl implements LocalModelService { const catalog = this.deps.getCatalog(); const options: RecommendOptions = { measuredTotalBytes, runtimeDownloadBytes: 0 }; if (!this.deps.modelFitsBudget(hw, catalog, state.model.id, state.model.context, options)) { - const nextModelId = await this.nextModelId(state.model.id, options); + const nextModelId = await this.nextModelId(state.model, options, { needsLessMemory: true }); return { kind: 'fail', code: 'does-not-fit', @@ -1657,6 +1993,15 @@ class LocalModelServiceImpl implements LocalModelService { /* a leftover is not a reason to stop: Remove with the models deletes it */ } if (signal.aborted) return CANCELLED; + // A download that was interrupted left partial blobs and no model: the runtime answers + // "not found" for it and deletes nothing. No download is in flight here, so every + // partial file in the store is a dead one. + try { + const freed = await this.deps.removePartialModelBlobs(this.paths()); + if (freed > 0) console.log(`[local-model] Deleted ${formatGb(freed)} GB a replaced download had left.`); + } catch { + /* as above */ + } await this.record((current) => { const { replacedModelId: _replaced, ...rest } = current; return rest; @@ -1669,7 +2014,16 @@ class LocalModelServiceImpl implements LocalModelService { }); if (!pulled.ok) { if (pulled.code === 'cancelled') return CANCELLED; - const nextModelId = pulled.code === 'model-not-found' ? await this.nextModelId(state.model.id) : undefined; + // A runtime that is gone for good looks like a broken network to the pull. It is not. + const status = this.deps.supervisor.status(); + if (pulled.code === 'pull-failed' && (status.gaveUp || !status.running)) { + return { + kind: 'fail', + code: 'runtime-stopped', + ...(pulled.detail ? { detail: pulled.detail } : {}), + }; + } + const nextModelId = pulled.code === 'model-not-found' ? await this.nextModelId(state.model) : undefined; return { kind: 'fail', code: pulled.code, @@ -1711,8 +2065,12 @@ class LocalModelServiceImpl implements LocalModelService { } vars = { model: state.model.label, actual, wanted: String(context) }; } - const offersNext = verified.code === 'does-not-fit' || verified.code === 'no-tool-call'; - const nextModelId = offersNext ? await this.nextModelId(id) : undefined; + const nextModelId = + verified.code === 'does-not-fit' + ? await this.nextModelId(state.model, undefined, { needsLessMemory: true }) + : verified.code === 'no-tool-call' + ? await this.nextModelId(state.model) + : undefined; return { kind: 'fail', code: verified.code, @@ -1815,13 +2173,41 @@ class LocalModelServiceImpl implements LocalModelService { return { kind: 'fail', code: 'connect-failed', detail: `The runtime does not list ${id}.` }; } + // The connection is CLAIMED in the state file before either write, so no record of ours + // can exist that the state file does not know: a state write that fails, or a server that + // is killed between the two, leaves at worst a claim with nothing behind it. + let claim = running.connection; + if (!claim) { + let found: LocalEndpointInspection; + try { + found = await this.deps.inspectLocalEndpoint({ port, modelId: id, context }); + } catch (err) { + return { kind: 'fail', code: 'connect-failed', detail: getErrorMessage(err) }; + } + if (signal.aborted) return CANCELLED; + // A record with our id that says something else is somebody else's: never overwritten. + if (found.record === 'other') { + return { kind: 'fail', code: 'endpoint-id-taken', vars: this.errorVars('endpoint-id-taken', state) }; + } + // A record that already says what ours would say can only be ours (the routes refuse + // the id): the flag was then switched on by us, and its earlier value is not known, so + // it is never switched off again. + claim = { + endpointId: LOCAL_MODEL_ENDPOINT_ID, + flagWasEnabled: found.record === 'same' ? true : found.flagEnabled, + }; + const claimed = claim; + await this.record((current) => ({ ...current, connection: claimed })); + } + if (signal.aborted) return CANCELLED; + // From here on the two writes run to their end: Cancel no longer interrupts. const connected = await this.deps.connectLocalModel({ port, modelId: id, context, - ours: running.connection !== undefined, - ...(running.connection ? { flagWasEnabled: running.connection.flagWasEnabled } : {}), + ours: true, + flagWasEnabled: claim.flagWasEnabled, }); if (!connected.ok) { return { @@ -1831,11 +2217,6 @@ class LocalModelServiceImpl implements LocalModelService { ...(connected.detail ? { detail: connected.detail } : {}), }; } - const flagWasEnabled = connected.flagWasEnabled; - await this.record((current) => ({ - ...current, - connection: { endpointId: LOCAL_MODEL_ENDPOINT_ID, flagWasEnabled }, - })); return DONE; } } diff --git a/src/local-model/ollama-client.ts b/src/local-model/ollama-client.ts index a597c138..90fe3bb7 100644 --- a/src/local-model/ollama-client.ts +++ b/src/local-model/ollama-client.ts @@ -31,6 +31,7 @@ */ import type { LocalModelErrorCode } from '../types.js'; +import { formatGb } from './format.js'; /** How often a pull that ended on an error line or a cut stream is started again. */ export const PULL_RETRIES = 5; @@ -683,7 +684,8 @@ export function judgeLoaded( return { ok: false, code: 'does-not-fit', - detail: `${model.sizeVram} of ${model.size} bytes are on the GPU`, + // A size a person reads: decimal GB, like every other size of this feature. + detail: `${formatGb(model.sizeVram)} of ${formatGb(model.size)} GB of the model are on the GPU`, }; } if (model.contextLength !== context) { diff --git a/src/types/local-model-install.ts b/src/types/local-model-install.ts index 3bd29022..2fdd8f2b 100644 --- a/src/types/local-model-install.ts +++ b/src/types/local-model-install.ts @@ -27,14 +27,19 @@ * | | | `already-installed`, `size-changed` | * | `POST /api/local-model/cancel` | none | 409 `wrong-state` (no job is running) | * | `POST /api/local-model/resume` | `LocalModelResumeRequest` | 409 `job-running`, `wrong-state` (not paused or | - * | | | failed) | - * | `POST /api/local-model/runtime` | `LocalModelRuntimeRequest` | 409 `wrong-state` (not installed), `job-running` | + * | | | failed), `sessions-open` (another port while live | + * | | | sessions use the model) | + * | `POST /api/local-model/runtime` | `LocalModelRuntimeRequest` | 409 `wrong-state` (not installed), `job-running`, | + * | | | `sessions-open` (stop, or another port, while live | + * | | | sessions use the model) | * | `DELETE /api/local-model` | `LocalModelRemoveRequest` | 409 `job-running`, `sessions-open` | * * A 409 is the error envelope plus `data: LocalModelConflict`: * `{ success: false, error, errorCode: 'CONFLICT', data: { reason, message, ... } }`. A 409 * WITHOUT `data` is a write of `settings.json` or of the endpoint store that was refused - * because the file does not parse; `error` names the fix. + * because the file does not parse; `error` names the fix. A 400 the service names carries its + * sentence the same way: `{ success: false, error, errorCode: 'INVALID_INPUT', data: { + * message } }` (`LocalModelRefusal`), so the window can print it in its own language. * * The rules of src/types/local-model.ts hold here too: * 1. A size a person reads travels twice: as bytes and as decimal GB text from `formatGb()` @@ -118,8 +123,10 @@ export const LOCAL_MODEL_INSTALL_STEPS: readonly LocalModelInstallStep[] = [ * | `runtime-download-failed` | the archive download still fails after its retries | * | `runtime-damaged` | the archive's sha256 is not the pinned one (the file was deleted) | * | `runtime-unpack-failed` | `/usr/bin/tar` failed | - * | `runtime-start-failed` | the runtime was started and never answered `/api/version` | - * | `runtime-stopped` | installed: the runtime crashed and was restarted 3 times, then given up | + * | `runtime-start-failed` | the runtime was started and never answered `/api/version`. Installed: | + * | | the last start failed (the detail says why), and Start tries again | + * | `runtime-stopped` | the runtime crashed and was restarted 3 times, then given up: on an | + * | | installed model, or during a job's download | * | `runtime-not-running` | installed: the runtime is not running (the person stopped it). Only a | * | | refused session start says it; the view shows no error for a stopped runtime | * | `port-in-use` | another program listens on the stored port, or no port of the span is free | @@ -170,6 +177,11 @@ export type LocalModelErrorCode = * Without a `nextModel` the window offers the chooser and Remove. * - `start`: "Start". `POST /api/local-model/runtime` with `action: 'start'`. * - `remove`: "Remove". `DELETE /api/local-model`. + * + * `LOCAL_MODEL_ERROR_NEXT` is the default of a code. The view adapts it to the state the + * error is shown in, so the button is always one the server accepts there: a code whose + * default is `start` says `retry` on a failed job (there is no Start verb for a job), and + * `runtime-start-failed` says `start` on an installed model (there is nothing to resume). */ export type LocalModelNextStep = 'retry' | 'other-port' | 'next-model' | 'start' | 'remove'; @@ -212,11 +224,18 @@ export type LocalModelDeveloperTools = * | `errContextNotApplied` | `model`; `actual`: the context the runtime reports; `wanted`: the chosen one | * | `errHarnessFailed` | `cli`: the coding agent's label | * | `errEndpointIdTaken` | `id`: the endpoint id this feature uses | + * | `refusedAgentNotInstallable` | `cli`: the coding agent's label | + * | `refusedOtherModel` | `model`: the label of the model that is installed | * | `conflictSizeChanged` | `size`: the server's own total as decimal GB | * | `conflictSessionsOpen` | `sessions`: the session names, joined with ', ' | * * `developerToolsMissing` is not sent: the view carries `developerTools` and the window prints * this sentence when it is `missing`. It is here so that its wording has one source. + * + * `endpointManaged` and `endpointIdReserved` are what the endpoint routes + * (src/web/routes/custom-model-routes.ts) answer for the record this feature manages and for + * its reserved id. They have no slot, so the App Settings toast that prints them translates + * them as it translates every other whole sentence. */ export const LOCAL_MODEL_INSTALL_LABELS = { errRuntimeDownloadFailed: @@ -246,6 +265,13 @@ export const LOCAL_MODEL_INSTALL_LABELS = { conflictSizeChanged: 'The download is {size} GB now. Check the sizes and confirm again.', conflictSessionsOpen: 'Sessions are still using this model: {sessions}. Close them first.', conflictWrongState: 'That is not possible in the current state.', + refusedModelNotOffered: 'This model at this context is not offered for this Mac right now.', + refusedAgentNotOffered: 'This coding agent is not offered for this model.', + refusedAgentNotInstallable: '{cli} is not installed and cannot be installed from here.', + refusedOtherModel: 'Only {model} is installed on this Mac. Remove it and install another model first.', + endpointManaged: + 'This endpoint belongs to the model Codeman runs on this Mac. Change or remove it in the local model window.', + endpointIdReserved: 'This endpoint id is reserved for the model Codeman runs on this Mac. Pick another id.', developerToolsMissing: 'Coding agents use git and Python. This Mac has no developer tools yet, so macOS will offer to install them the first time an agent needs one.', } as const; @@ -336,11 +362,12 @@ export interface LocalModelInstallError { code: LocalModelErrorCode; /** `LOCAL_MODEL_ERROR_LABELS[code]` with its slots filled. */ message: LocalModelInstallMessage; - /** `LOCAL_MODEL_ERROR_NEXT[code]`. */ + /** `LOCAL_MODEL_ERROR_NEXT[code]`, adapted to the state the error is shown in (see `LocalModelNextStep`). */ next: LocalModelNextStep; /** - * For `next-model`: the best model that still fits and is not the one that failed. Absent - * when there is none. + * For `next-model`: the model to try instead. After `does-not-fit` it is one that needs LESS + * memory than the one that failed; after the other codes the best other model that fits. + * Absent when there is none. */ nextModel?: { modelId: string; modelLabel: string }; /** @@ -484,6 +511,11 @@ export interface LocalModelRemoveRequest { removeModel?: boolean; } +/** The `data` of a 400 the service names: its sentence, for the window to print in its own language. */ +export interface LocalModelRefusal { + message: LocalModelInstallMessage; +} + /** What every verb answers on success: the view after the verb. */ export type LocalModelVerbAnswer = LocalModelInstallView; @@ -493,7 +525,9 @@ export type LocalModelVerbAnswer = LocalModelInstallView; * - `already-installed`: install while a model is installed. Remove it first. * - `size-changed`: `acknowledgedBytes` is not the server's total; `expectedBytes` and * `expectedGb` carry that total, so the window can show it and ask again. - * - `sessions-open`: remove while live sessions use the endpoint; `sessions` names them. + * - `sessions-open`: remove, stop or a change of port while live sessions use the endpoint; + * `sessions` names them. Their coding agents were given the runtime's address when they + * started, so they would go on calling a port nothing of ours holds. * - `wrong-state`: the verb makes no sense in the current state (see the route table above). */ export type LocalModelConflictReason = @@ -567,11 +601,17 @@ export interface LocalModelStateFile { cli?: { id: string; label: string }; /** The runtime, once its port was chosen. */ runtime?: { - /** The pinned version that was unpacked. */ + /** + * The runtime version this installation runs. A later build with another pin keeps + * starting this version for as long as its folder is unpacked. + */ version: string; /** The loopback port, chosen once by binding. */ port: number; - /** The pid of the child Codeman last started. Used at boot to tell our own orphan from another program. */ + /** + * The pid of the child Codeman last started, stored the moment it is spawned. Used to + * tell our own runtime from another program: at boot, and before the folder is deleted. + */ pid?: number; /** False after Stop: the runtime is not started at boot. */ enabled: boolean; @@ -582,7 +622,11 @@ export interface LocalModelStateFile { progress?: { completedBytes: number; totalBytes: number }; /** What the runtime reported when the job checked the loaded model. */ loaded?: { context: number; sizeBytes: number; sizeVramBytes: number }; - /** Present once the `connect` step wrote: what uninstall must undo. */ + /** + * Present once the `connect` step CLAIMED the connection, which it does before its two + * writes: what uninstall must undo. A claim with nothing behind it (the step was interrupted + * right after it) is harmless: a resumed job writes both, and uninstall removes nothing. + */ connection?: { /** The id of the endpoint record this feature wrote. */ endpointId: string; diff --git a/src/web/routes/custom-model-routes.ts b/src/web/routes/custom-model-routes.ts index 4516b516..d718f598 100644 --- a/src/web/routes/custom-model-routes.ts +++ b/src/web/routes/custom-model-routes.ts @@ -30,7 +30,7 @@ */ import type { FastifyInstance, FastifyRequest } from 'fastify'; -import { ApiErrorCode, createErrorResponse, type ApiResponse } from '../../types.js'; +import { ApiErrorCode, createErrorResponse, LOCAL_MODEL_INSTALL_LABELS, type ApiResponse } from '../../types.js'; import { isAdmin, parseBody, readJsonConfig, SETTINGS_PATH } from '../route-helpers.js'; import { isMultiUserMode } from '../../config/multiuser.js'; import { getDataDir } from '../../config/instance.js'; @@ -933,10 +933,7 @@ function refusedHostsWrite(err: unknown, reply: { code: (n: number) => unknown } /** The answer to an edit or a delete of the record the local model install manages. */ function managedEndpointRefusal(): ApiResponse { - return createErrorResponse( - ApiErrorCode.INVALID_INPUT, - 'This endpoint belongs to the model Codeman runs on this Mac. Change or remove it in the local model window.' - ); + return createErrorResponse(ApiErrorCode.INVALID_INPUT, LOCAL_MODEL_INSTALL_LABELS.endpointManaged); } export function registerCustomModelRoutes(app: FastifyInstance): void { @@ -956,10 +953,7 @@ export function registerCustomModelRoutes(app: FastifyInstance): void { const badDefault = invalidDefaultModel(host); if (badDefault) return badDefault; if (isLocalModelEndpointId(host.id)) { - return createErrorResponse( - ApiErrorCode.INVALID_INPUT, - `The endpoint id "${host.id}" is reserved for the model Codeman runs on this Mac. Pick another id.` - ); + return createErrorResponse(ApiErrorCode.INVALID_INPUT, LOCAL_MODEL_INSTALL_LABELS.endpointIdReserved); } try { return await mutateCustomModelHosts>((hosts) => { diff --git a/src/web/routes/local-model-routes.ts b/src/web/routes/local-model-routes.ts index 74c858ef..2cd8bc6a 100644 --- a/src/web/routes/local-model-routes.ts +++ b/src/web/routes/local-model-routes.ts @@ -40,11 +40,14 @@ * - A verb answers the view AFTER the verb. A refusal the service names is a 400 (the request * is not a row the server offers) or a 409 whose `data` is the `LocalModelConflict` * (`reason`, the sentence, and for `size-changed` the server's own total, for - * `sessions-open` the session names). A write of the settings or of the endpoint store - * that was refused (a file that does not parse) is a 409 without `data`, with the fix. + * `sessions-open` the session names). A 400 the service names carries its sentence as + * `data.message` too, so the window prints it in its own language. A write of the settings + * or of the endpoint store that was refused (a file that does not parse) is a 409 without + * `data`, with the fix. * - The verbs do the work in the install service; this file only gates, parses and maps. - * `DELETE` is the one that looks at the live sessions: it hands their names to the service, - * which refuses while one of them still uses the model. + * `DELETE`, `runtime` and `resume` look at the live sessions: they hand their names to the + * service, which refuses a removal, a stop and a change of port while one of them still + * uses the model. * - Every dependency is injected (`LocalModelRouteDeps`), so a route test spawns nothing and * does not need a Mac: it passes its own hardware, harness facts and service. */ @@ -61,6 +64,7 @@ import { type LocalModelCatalog, type LocalModelConflict, type LocalModelInstallView, + type LocalModelRefusal, type LocalModelStatus, type Recommendation, type RecommendOptions, @@ -152,8 +156,20 @@ export interface LocalModelConflictResponse { data: LocalModelConflict; } -/** What a verb answers: the view after it, an error envelope, or the 409 that carries the conflict. */ -export type LocalModelVerbResponse = ApiResponse | LocalModelConflictResponse; +/** The 400 (or 507) of a verb the service refused with a sentence: the error envelope plus that sentence as `data`. */ +export interface LocalModelRefusalResponse { + success: false; + /** The sentence in English, its slots filled. */ + error: string; + errorCode: ApiErrorCode; + data: LocalModelRefusal; +} + +/** What a verb answers: the view after it, an error envelope, or a refusal that carries its sentence. */ +export type LocalModelVerbResponse = + | ApiResponse + | LocalModelConflictResponse + | LocalModelRefusalResponse; /** 403 for a non-admin in multi-user mode, before anything else runs. `null`: go on. */ function adminOnly(req: FastifyRequest, reply: FastifyReply): ApiResponse | null { @@ -170,7 +186,14 @@ function actorOf(req: FastifyRequest): LocalModelActor { function respond(result: LocalModelVerbResult, reply: FastifyReply): LocalModelVerbResponse { if (result.ok) return { success: true, data: result.view }; reply.code(result.status); - if (result.status === 400) return createErrorResponse(ApiErrorCode.INVALID_INPUT, result.message); + if (result.status !== 409) { + return { + success: false, + error: formatInstallMessage(result.message), + errorCode: result.status === 400 ? ApiErrorCode.INVALID_INPUT : ApiErrorCode.OPERATION_FAILED, + data: { message: result.message }, + }; + } return { success: false, error: formatInstallMessage(result.conflict.message), @@ -258,14 +281,18 @@ export function registerLocalModelRoutes( const denied = adminOnly(req, reply); if (denied) return denied; const body = parseBody(LocalModelResumeSchema, req.body ?? {}); - return runVerb(reply, () => resolved.service().resume(body, actorOf(req))); + const sessionNames = sessionNamesUsingEndpoint(ctx.sessions.values()); + return runVerb(reply, () => resolved.service().resume(body, actorOf(req), sessionNames)); }); app.post('/api/local-model/runtime', async (req, reply): Promise => { const denied = adminOnly(req, reply); if (denied) return denied; const body = parseBody(LocalModelRuntimeSchema, req.body); - return runVerb(reply, () => resolved.service().runtime(body)); + // Asked at the moment of the request, like the DELETE: a stop or a change of port leaves + // a live session calling an address nothing of ours holds. + const sessionNames = sessionNamesUsingEndpoint(ctx.sessions.values()); + return runVerb(reply, () => resolved.service().runtime(body, sessionNames)); }); app.delete('/api/local-model', async (req, reply): Promise => { diff --git a/test/local-model-connect.test.ts b/test/local-model-connect.test.ts index c1dd2150..3630baf6 100644 --- a/test/local-model-connect.test.ts +++ b/test/local-model-connect.test.ts @@ -27,8 +27,10 @@ import { connectLocalModel, disconnectLocalModel, ensureLocalEndpoint, + inspectLocalEndpoint, isLocalModelEndpointId, isManagedLocalEndpoint, + isSameLocalEndpoint, LOCAL_MODEL_ENDPOINT_ID, LOCAL_MODEL_ENDPOINT_LABEL, moveLocalEndpoint, @@ -401,6 +403,76 @@ describe('compareModelDigest', () => { }); }); +describe('a record that says what ours would say is ours', () => { + it('isSameLocalEndpoint: the address, the one model, its context and no key', () => { + expect(isSameLocalEndpoint(RECORD, INPUT)).toBe(true); + // What discovery may have added does not make it somebody else's. + expect( + isSameLocalEndpoint( + { ...RECORD, modelContextLengths: { 'qwen3.6:35b-a3b': 131072 }, lastDiscoveredAt: '2026-11-06T10:00:00Z' }, + INPUT + ) + ).toBe(true); + const differs: Partial[] = [ + { baseUrl: 'http://127.0.0.1:11434' }, + { label: 'My own' }, + { models: ['qwen3.6:35b-a3b', 'gpt-oss:20b'] }, + { models: ['gpt-oss:20b'], defaultModelId: 'gpt-oss:20b' }, + { defaultContextLength: 65536 }, + { apiKey: 'sk-not-ours' }, + { authStyle: 'api-key' }, + ]; + for (const change of differs) expect(isSameLocalEndpoint({ ...RECORD, ...change }, INPUT)).toBe(false); + }); + + it('connectLocalModel takes such a record as ours although the caller did not say so', async () => { + writeSettings({}); + writeHosts([OTHER, RECORD]); + expect(await connectLocalModel({ ...INPUT, ours: false })).toEqual({ ok: true, flagWasEnabled: false }); + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER, RECORD]); + }); + + it('a record with our id that says something else is still somebody else: taken, and untouched', async () => { + const foreign = { ...RECORD, baseUrl: 'http://127.0.0.1:11434' }; + writeHosts([foreign]); + expect(await connectLocalModel({ ...INPUT, ours: false })).toEqual({ ok: false, code: 'endpoint-id-taken' }); + expect(await readCustomModelHosts(getDataDir())).toEqual([foreign]); + }); +}); + +describe('inspectLocalEndpoint: what the connect step finds before it claims the connection', () => { + it('nothing saved and no settings file: absent, and the flag is off', async () => { + expect(await inspectLocalEndpoint(INPUT)).toEqual({ record: 'absent', flagEnabled: false }); + }); + + it('reads the flag as it is now, and writes nothing', async () => { + writeSettings({ customModelEndpointsEnabled: true, tmuxHistoryLimit: 5000 }); + writeHosts([OTHER]); + const settingsBefore = readFileSync(SETTINGS_PATH, 'utf8'); + const hostsBefore = readFileSync(HOSTS(), 'utf8'); + expect(await inspectLocalEndpoint(INPUT)).toEqual({ record: 'absent', flagEnabled: true }); + expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe(settingsBefore); + expect(readFileSync(HOSTS(), 'utf8')).toBe(hostsBefore); + }); + + it("tells our record from somebody else's under the same id", async () => { + writeHosts([RECORD]); + expect((await inspectLocalEndpoint(INPUT)).record).toBe('same'); + expect((await inspectLocalEndpoint({ ...INPUT, port: 11436 })).record).toBe('other'); + }); +}); + +describe('moveLocalEndpoint', () => { + it('a record that is already on the port is not written again', async () => { + writeHosts([RECORD]); + const before = readFileSync(HOSTS(), 'utf8'); + await moveLocalEndpoint(11435); + expect(readFileSync(HOSTS(), 'utf8')).toBe(before); + await moveLocalEndpoint(11440); + expect((await readCustomModelHosts(getDataDir()))[0].baseUrl).toBe('http://127.0.0.1:11440'); + }); +}); + describe('the runtime keeps a private key that is never served', () => { it('everything in the private home is on the sensitive list, and so is a runtime key under a real home', () => { const home = join(getDataDir(), 'local-model', 'home'); diff --git a/test/local-model-install-e2e.test.ts b/test/local-model-install-e2e.test.ts index 886c26ce..f87a5261 100644 --- a/test/local-model-install-e2e.test.ts +++ b/test/local-model-install-e2e.test.ts @@ -17,7 +17,8 @@ * * The walks: the whole install and its removal; a damaged runtime download; a model download * that fails twice and then succeeds; one that keeps failing; a model that calls no tool; - * Cancel during the model download; a server restart during the model download. + * Cancel during the model download; a server restart during the model download; a server + * that is KILLED during the model download and leaves its runtime running. * * "A server" here is one install service with its own runtime supervisor (`startServer()`). * A restart is `shutdown()` of the one and a new one over the same data dir. @@ -45,7 +46,11 @@ import { createLocalModelService, type LocalModelService } from '../src/local-mo import { memClassGb, formatGb } from '../src/local-model/format.js'; import { pullModel, PULL_RETRIES } from '../src/local-model/ollama-client.js'; import { localModelPaths, type LocalModelPaths } from '../src/local-model/paths.js'; -import { createRuntimeSupervisor, ensureRuntimeInstalled } from '../src/local-model/runtime.js'; +import { + createRuntimeSupervisor, + ensureRuntimeInstalled, + type RuntimeSupervisorDeps, +} from '../src/local-model/runtime.js'; import { readLocalModelState } from '../src/local-model/state-store.js'; import { registerLocalModelRoutes } from '../src/web/routes/local-model-routes.js'; import { SETTINGS_PATH } from '../src/web/route-helpers.js'; @@ -123,7 +128,7 @@ const archiveFetch: typeof fetch = async (input, init) => { * supervisor, over the data dir, the pin and the port of the test. It becomes `world.service`, * which is the one the routes ask. */ -function startServer(): LocalModelService { +function startServer(processes: Partial> = {}): LocalModelService { const pin = world.archive.pin(); const { paths, port } = world; const supervisor = createRuntimeSupervisor({ @@ -136,6 +141,7 @@ function startServer(): LocalModelService { env: { PATH: process.env.PATH, CODEMAN_PASSWORD: 'must-not-reach-the-runtime', OLLAMA_ORIGINS: '*' }, commandOfPid: async () => undefined, kill: () => {}, + ...processes, sleep: async () => {}, pin, paths, @@ -675,3 +681,83 @@ describe('the install from the first GET to Remove, real modules over a fake run expect(await readCustomModelHosts(getDataDir())).toMatchObject([{ id: 'local-ollama', models: [pick.modelId] }]); }, 60_000); }); + +describe('a server that is killed leaves its runtime running: the next server never deletes under it', () => { + /** + * Installs up to the model download, then "kills" the server: its job stops and it signals + * nothing, as a process that got SIGKILL does. Its runtime child goes on listening. Answers + * what the next server needs to find that process: `ps` and `kill` over the fake child. + */ + async function killedDuringTheDownload() { + const api = await routes(); + const pick = (await api.get()).recommendation.pick!; + const runtime = prepareRuntime(pick); + slowPull(runtime); + await api.install(pick); + await api.pulling(); + + const child = runtime.state.children[0]; + const signal = child.kill.bind(child); + // A killed server sends its child nothing. + child.kill = () => false; + await world.service.shutdown(); + expect(runtime.listening).toBe(true); + // The pid was stored the moment the child was spawned. + expect((await readLocalModelState()).runtime?.pid).toBe(child.pid); + + const signals: Array<{ pid: number; signal: string; folderThere: boolean }> = []; + const processes: Partial> = { + // What `ps -o command= -p ` prints for the runtime while it lives. + commandOfPid: async (pid) => (pid === child.pid && runtime.listening ? `${world.paths.binary} serve` : undefined), + kill: (pid, sig) => { + signals.push({ pid, signal: sig, folderThere: existsSync(world.paths.runtimeDir) }); + if (pid === child.pid) signal(sig); + }, + }; + return { ...api, pick, runtime, child, signals, processes }; + } + + it('Remove of the job it left ends that runtime BEFORE the folder is deleted', async () => { + const { app, runtime, child, signals, processes } = await killedDuringTheDownload(); + startServer(processes); + const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: true } }); + expect(removed.statusCode).toBe(200); + expect(removed.json().data.state).toBe('idle'); + // The process was signalled while its binary was still on disk, and it is gone. + expect(signals).toEqual([{ pid: child.pid, signal: 'SIGTERM', folderThere: true }]); + await vi.waitFor(() => expect(runtime.listening).toBe(false)); + expect(existsSync(world.paths.root)).toBe(false); + expect(await readLocalModelState()).toEqual(expect.objectContaining({ schema: 1, state: 'idle' })); + }, 60_000); + + it('the next start of the server ends it too: nothing would use it until a click', async () => { + const { get, settled, app, runtime, child, signals, processes } = await killedDuringTheDownload(); + const next = startServer(processes); + await next.boot(); + expect(signals.map((entry) => [entry.pid, entry.signal])).toEqual([[child.pid, 'SIGTERM']]); + await vi.waitFor(() => expect(runtime.listening).toBe(false)); + expect((await get()).install).toMatchObject({ state: 'paused', step: 'model', running: false }); + + // Continue starts a runtime of its own on the same port and finishes the install. + quickPull(runtime); + const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} }); + expect(resumed.statusCode).toBe(200); + const install = await settled(); + expect(install).toMatchObject({ state: 'installed', running: true }); + expect(runtime.state.children).toHaveLength(2); + }, 60_000); + + it('a pid that now runs something else is never signalled', async () => { + const { app, child, signals } = await killedDuringTheDownload(); + // The pid was reused by another program since. + startServer({ + commandOfPid: async () => '/Applications/Something.app/Contents/MacOS/Something', + kill: (pid, sig) => signals.push({ pid, signal: sig, folderThere: true }), + }); + const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: {} }); + expect(removed.statusCode).toBe(200); + expect(signals).toEqual([]); + // Left to the test's own cleanup. + child.crash(0); + }, 60_000); +}); diff --git a/test/local-model-install-job.test.ts b/test/local-model-install-job.test.ts index 740c4a81..dec84ee8 100644 --- a/test/local-model-install-job.test.ts +++ b/test/local-model-install-job.test.ts @@ -488,6 +488,30 @@ describe('installedBlocker', () => { }); }); + it('a runtime that did not start: what it said, and "Start" to try again (an installed model has no job to resume)', () => { + expect(installedBlocker(SAMPLE.installed, { ...LIVE, startFailed: { detail: 'spawn ENOENT' } })).toEqual({ + code: 'runtime-start-failed', + message: { text: 'The runtime did not start on this Mac.' }, + next: 'start', + detail: 'spawn ENOENT', + }); + expect(installedBlocker(SAMPLE.installed, { ...LIVE, startFailed: {} })).toEqual({ + code: 'runtime-start-failed', + message: { text: 'The runtime did not start on this Mac.' }, + next: 'start', + }); + // A runtime that runs has started: an older failure says nothing any more. + expect( + installedBlocker(SAMPLE.installed, { ...LIVE, running: true, digest: 'same', startFailed: { detail: 'x' } }) + ).toBeUndefined(); + }); + + it('the port and a runtime that keeps stopping are said before a failed start', () => { + const failed = { startFailed: { detail: 'x' } }; + expect(installedBlocker(SAMPLE.installed, { ...LIVE, ...failed, portInUse: true })?.code).toBe('port-in-use'); + expect(installedBlocker(SAMPLE.installed, { ...LIVE, ...failed, gaveUp: true })?.code).toBe('runtime-stopped'); + }); + it('a model that changed: "Remove"', () => { expect(installedBlocker(SAMPLE.installed, { ...LIVE, running: true, digest: 'changed' })).toEqual({ code: 'model-changed', @@ -617,6 +641,21 @@ describe('buildInstallView', () => { }); }); + it('a failed job never says "Start": a code whose own next step is that says "Try again" there', () => { + const state: LocalModelStateFile = { ...SAMPLE.failed, error: { code: 'runtime-stopped', detail: 'ECONNREFUSED' } }; + expect(buildInstallView(state, LIVE).error).toEqual({ + code: 'runtime-stopped', + message: { text: 'The runtime keeps stopping, so Codeman no longer restarts it.' }, + next: 'retry', + detail: 'ECONNREFUSED', + }); + // Every next step a failed job is shown with is one its actions can carry out. + for (const code of Object.keys(LOCAL_MODEL_ERROR_NEXT) as LocalModelErrorCode[]) { + const next = buildInstallView({ ...SAMPLE.failed, error: { code } }, LIVE).error?.next; + expect(next, code).not.toBe('start'); + } + }); + it('a next model is named only for an error whose next step is another model', () => { const view = buildInstallView(SAMPLE.failed, { ...LIVE, diff --git a/test/local-model-install-service.test.ts b/test/local-model-install-service.test.ts index 67da16a7..bae8ffbe 100644 --- a/test/local-model-install-service.test.ts +++ b/test/local-model-install-service.test.ts @@ -22,7 +22,7 @@ import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { dirname } from 'node:path'; import { getDataDir } from '../src/config/instance.js'; import { customModelHostsPath, readCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js'; -import { connectLocalModel, disconnectLocalModel } from '../src/local-model/connect.js'; +import { connectLocalModel, disconnectLocalModel, inspectLocalEndpoint } from '../src/local-model/connect.js'; import { createLocalModelService, type LocalModelActor, @@ -99,7 +99,14 @@ const FACTS: HarnessFact[] = [ }, ]; -function candidate(modelId: string, modelLabel: string, context: number, bytes: number, installed: boolean): Candidate { +function candidate( + modelId: string, + modelLabel: string, + context: number, + bytes: number, + installed: boolean, + needBytes = 36_000_000_000 +): Candidate { return { modelId, modelLabel, @@ -109,8 +116,8 @@ function candidate(modelId: string, modelLabel: string, context: number, bytes: cli: 'opencode', cliLabel: 'OpenCode', cliInstalled: installed, - needBytes: 30_000_000_000, - needGb: '30.0', + needBytes, + needGb: '0.0', downloadBytes: bytes, downloadGb: '0.0', budgetShare: 0.8, @@ -137,7 +144,8 @@ function recommendation(agentInstalled = false): Recommendation { headline: { text: 'Best fit for this Mac' }, notes: [], pick: candidate('qwen3.6:35b-a3b', 'Qwen3.6 35B-A3B', 131072, PULL_BYTES, agentInstalled), - alternatives: [candidate('gpt-oss:20b', 'GPT-OSS 20B', 131072, 13_793_441_427, agentInstalled)], + // Needs less memory than the pick, so it is what "does not fit" offers next. + alternatives: [candidate('gpt-oss:20b', 'GPT-OSS 20B', 131072, 13_793_441_427, agentInstalled, 26_600_000_000)], budgetBytes: 37_580_963_840, budgetGb: '37.6', budgetSource: 'estimated', @@ -187,20 +195,34 @@ function jobAt(step: LocalModelInstallStep, state: 'installing' | 'paused' = 'in // ---------- The harness ---------- -/** The state file in memory. Every write goes through the real parser, like a write to disk and a read back. */ +/** What a write to a full disk rejects with. */ +function noSpace(): Error { + return Object.assign(new Error('ENOSPC: no space left on device, write'), { code: 'ENOSPC' }); +} + +/** + * The state file in memory. Every write goes through the real parser, like a write to disk and + * a read back. `refuse` makes the disk full: while it answers true for a write, that write + * rejects with ENOSPC and the file keeps what it had, like the real temp file plus rename. + */ function memoryStore(initial: LocalModelStateFile = IDLE_LOCAL_MODEL_STATE) { let state = initial; let chain: Promise = Promise.resolve(); const history: LocalModelStateFile[] = []; + const disk = { + refuse: undefined as ((next: LocalModelStateFile, before: LocalModelStateFile) => boolean) | undefined, + }; return { get: () => state, history, + disk, readState: async () => state, mutateState: (fn: (current: LocalModelStateFile) => LocalModelStateFile | undefined) => { const run = chain.then(async () => { const before = state; const next = fn(before); if (!next) return { before, after: undefined }; + if (disk.refuse?.(next, before)) throw noSpace(); state = parseLocalModelState(JSON.stringify(next)); history.push(state); return { before, after: state }; @@ -256,7 +278,7 @@ function setup( ...(world.measured !== undefined ? { measuredTotalBytes: world.measured } : {}), }; }), - stop: vi.fn(async () => { + stop: vi.fn(async (_options?: { previousPid?: number }) => { calls.push('supervisor.stop'); world.running = false; }), @@ -307,6 +329,10 @@ function setup( calls.push('deleteModel'); return true; }), + removePartialModelBlobs: vi.fn(async () => { + calls.push('removePartialModelBlobs'); + return 0; + }), checkDeveloperTools: vi.fn(async () => 'missing' as const), stockInstallCommand: vi.fn((id: string) => (id === FACTS[0].cli ? INSTALL_COMMAND : undefined)), tryBeginCliInstall: vi.fn((): (() => void) | null => () => { @@ -322,6 +348,8 @@ function setup( return world.agentInstalled; }), audit: vi.fn(async () => {}), + // Nothing saved under the reserved id, and the endpoints setting is off: a first install. + inspectLocalEndpoint: vi.fn(async () => ({ record: 'absent' as 'absent' | 'same' | 'other', flagEnabled: false })), connectLocalModel: vi.fn(async () => { calls.push('connectLocalModel'); return { ok: true as const, flagWasEnabled: false }; @@ -401,12 +429,14 @@ describe('a job from the click to installed', () => { for (const [start] of deps.supervisor.start.mock.calls) { expect(start).toMatchObject({ port: 11435, contextLength: 131072 }); } - // The connection names the model as the runtime lists it, at the chosen context. + // The connection names the model as the runtime lists it, at the chosen context. It was + // claimed in the state file first, so the record is written as ours. expect(deps.connectLocalModel).toHaveBeenCalledWith({ port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072, - ours: false, + ours: true, + flagWasEnabled: false, }); }); @@ -602,7 +632,7 @@ describe('install is checked before anything is downloaded', () => { expect(result).toEqual({ ok: false, status: 400, - message: 'OpenCode is not installed and cannot be installed from here.', + message: { text: '{cli} is not installed and cannot be installed from here.', vars: { cli: 'OpenCode' } }, }); nothingStarted(h); }); @@ -614,8 +644,10 @@ describe('install is checked before anything is downloaded', () => { expect(result).toEqual({ ok: false, status: 400, - message: - 'A saved model connection named local-ollama already exists and was not created here. Rename or delete it in App Settings, then try again.', + message: { + text: 'A saved model connection named {id} already exists and was not created here. Rename or delete it in App Settings, then try again.', + vars: { id: 'local-ollama' }, + }, }); nothingStarted(h); }); @@ -823,6 +855,8 @@ describe('Cancel in every step', () => { }), }, }); + // What the claim keeps: the setting was on before this feature touched it. + h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'absent', flagEnabled: true }); await h.service.install(REQUEST, ADMIN); await vi.waitFor(() => expect(h.deps.connectLocalModel).toHaveBeenCalled()); const cancelling = h.service.cancel(); @@ -929,8 +963,9 @@ describe('Resume', () => { await h.service.idle(); expect(h.deps.chooseRuntimePort).toHaveBeenCalledWith({ avoid: [11435] }); expect(h.store.get()).toMatchObject({ state: 'installed', runtime: { port: 11436 } }); - // The pid that was stored belonged to the old port: it is not passed on as "ours". - expect(h.supervisor.start.mock.calls[0][0]).toEqual({ port: 11436, contextLength: 131072 }); + // The stored pid goes along: when it is still our own runtime on the OLD port, the start + // ends it instead of leaving a second runtime behind. + expect(h.supervisor.start.mock.calls[0][0]).toEqual({ port: 11436, contextLength: 131072, previousPid: 5000 }); expect(h.deps.connectLocalModel).toHaveBeenCalledWith(expect.objectContaining({ port: 11436 })); }); }); @@ -951,6 +986,16 @@ describe('a job a server restart interrupted', () => { const h = setup({ state: jobAt('verify') }); await h.service.boot(); expect(h.store.get()).toMatchObject({ state: 'paused', step: 'verify' }); + expect(h.supervisor.start).not.toHaveBeenCalled(); + // The runtime the killed server left behind is ended: nothing would use it until a click. + expect(h.calls).toEqual(['supervisor.stop']); + expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 5000 }); + }); + + it('boot() ends nothing for a job that never started a runtime', async () => { + const h = setup({ state: jobAt('runtime') }); + await h.service.boot(); + expect(h.store.get()).toMatchObject({ state: 'paused', step: 'runtime' }); expect(h.calls).toEqual([]); }); @@ -1280,9 +1325,9 @@ describe('every failure is a state with a code and a next step', () => { expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); }); - it('a saved endpoint with our id that is not ours: endpoint-id-taken with the id', async () => { + it('a saved endpoint with our id that is not ours: endpoint-id-taken with the id, and nothing is claimed or written', async () => { const h = setup(); - h.deps.connectLocalModel.mockResolvedValue({ ok: false, code: 'endpoint-id-taken' } as never); + h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'other', flagEnabled: true }); const view = await failed(h); expect(view.error).toMatchObject({ code: 'endpoint-id-taken', @@ -1290,6 +1335,7 @@ describe('every failure is a state with a code and a next step', () => { message: { vars: { id: 'local-ollama' } }, }); expect(h.store.get()).not.toHaveProperty('connection'); + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); }); it('a refused write of the connection: connect-failed with the reason', async () => { @@ -1629,7 +1675,8 @@ describe('the runtime verbs', () => { const result = await h.service.runtime({ action: 'start', newPort: true }); expect(h.deps.chooseRuntimePort).toHaveBeenCalledWith({ avoid: [11435] }); expect(h.deps.moveLocalEndpoint).toHaveBeenCalledWith(11437); - expect(h.supervisor.start).toHaveBeenLastCalledWith({ port: 11437, contextLength: 131072 }); + // The stored pid goes along, so our own runtime on the old port could not be left behind. + expect(h.supervisor.start).toHaveBeenLastCalledWith({ port: 11437, contextLength: 131072, previousPid: 4711 }); expect(result.ok && result.view).toMatchObject({ port: 11437, running: true }); expect(result.ok && result.view.error).toBeUndefined(); expect(h.store.get().runtime).toMatchObject({ port: 11437, enabled: true }); @@ -1781,7 +1828,10 @@ describe('Remove with the real connection: the settings flag', () => { /** Installs with the real connect step, then removes with the real disconnect. */ async function installThenRemove() { - const h = setup({ agentInstalled: true, overrides: { connectLocalModel, disconnectLocalModel } }); + const h = setup({ + agentInstalled: true, + overrides: { inspectLocalEndpoint, connectLocalModel, disconnectLocalModel }, + }); await installAndWait(h.service); expect(h.store.get().state).toBe('installed'); expect(flag()).toBe(true); @@ -1815,7 +1865,10 @@ describe('Remove with the real connection: the settings flag', () => { it('a resumed connect step does not forget that the flag was off before', async () => { writeSettings({ customModelEndpointsEnabled: false }); - const h = setup({ agentInstalled: true, overrides: { connectLocalModel, disconnectLocalModel } }); + const h = setup({ + agentInstalled: true, + overrides: { inspectLocalEndpoint, connectLocalModel, disconnectLocalModel }, + }); await installAndWait(h.service); // The connection is written a second time, as a resumed job would: the flag is on by now. const again = await connectLocalModel({ @@ -1897,3 +1950,641 @@ describe('view()', () => { expect(view.error).not.toHaveProperty('nextModel'); }); }); + +// ---------- The connection is claimed before it is written ---------- + +describe('the connect step claims the connection before it writes anything outside the folder', () => { + it('the state file knows the connection is ours before the endpoint record is written', async () => { + let claimAtWrite: LocalModelStateFile['connection']; + const h = setup(); + h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'absent', flagEnabled: true }); + h.deps.connectLocalModel.mockImplementation(async () => { + claimAtWrite = h.store.get().connection; + return { ok: true as const, flagWasEnabled: true }; + }); + await installAndWait(h.service); + // The claim carries the value the setting had BEFORE this feature switched it on. + expect(claimAtWrite).toEqual({ endpointId: 'local-ollama', flagWasEnabled: true }); + expect(h.deps.inspectLocalEndpoint).toHaveBeenCalledWith({ + port: 11435, + modelId: 'qwen3.6:35b-a3b', + context: 131072, + }); + }); + + it('a claim that cannot be written (a full disk) writes nothing outside, and "Try again" finishes the install', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('connect', 'paused') }); + // The claim is refused once; every other write lands. + let refused = false; + h.store.disk.refuse = (next, before) => { + if (refused || !next.connection || before.connection) return false; + refused = true; + return true; + }; + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ state: 'failed', step: 'connect', error: { code: 'disk-full' } }); + // Nothing was written outside our folder: the record and the setting are untouched. + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + expect(h.store.get()).not.toHaveProperty('connection'); + + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get()).toMatchObject({ + state: 'installed', + connection: { endpointId: 'local-ollama', flagWasEnabled: false }, + }); + expect(h.deps.connectLocalModel).toHaveBeenCalledTimes(1); + }); + + it('a record that already says what ours would say is ours, and the setting is then never switched off', async () => { + const h = setup({ state: jobAt('connect', 'paused') }); + h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'same', flagEnabled: true }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get()).toMatchObject({ + state: 'installed', + connection: { endpointId: 'local-ollama', flagWasEnabled: true }, + }); + }); + + it('an endpoint store that cannot be read: connect-failed, before anything is claimed', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('connect', 'paused') }); + h.deps.inspectLocalEndpoint.mockRejectedValue(new Error('custom-model-hosts.json is not a valid JSON list')); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ + state: 'failed', + error: { code: 'connect-failed', detail: 'custom-model-hosts.json is not a valid JSON list' }, + }); + expect(h.store.get()).not.toHaveProperty('connection'); + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + }); + + it('a resumed job does not look again: the claim it has keeps the value from before', async () => { + const h = setup({ + state: { ...jobAt('connect', 'paused'), connection: { endpointId: 'local-ollama', flagWasEnabled: false } }, + }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.deps.inspectLocalEndpoint).not.toHaveBeenCalled(); + expect(h.deps.connectLocalModel).toHaveBeenCalledWith( + expect.objectContaining({ ours: true, flagWasEnabled: false }) + ); + expect(h.store.get().state).toBe('installed'); + }); +}); + +describe('the connect step with the real endpoint store and settings: a write that does not land', () => { + const HOSTS = () => customModelHostsPath(getDataDir()); + const real = { inspectLocalEndpoint, connectLocalModel, disconnectLocalModel }; + const hostIds = async () => (await readCustomModelHosts(getDataDir())).map((host) => host.id); + const flag = (): unknown => { + try { + return JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).customModelEndpointsEnabled; + } catch { + return undefined; + } + }; + + afterEach(() => { + rmSync(HOSTS(), { force: true }); + rmSync(SETTINGS_PATH, { force: true }); + }); + + it('the claim fails once: nothing is left behind, "Try again" installs, and Remove undoes both writes', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('connect', 'paused'), overrides: real }); + let refused = false; + h.store.disk.refuse = (next, before) => { + if (refused || !next.connection || before.connection) return false; + refused = true; + return true; + }; + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect((await h.service.view()).error?.code).toBe('disk-full'); + expect(await hostIds()).toEqual([]); + expect(flag()).toBeUndefined(); + + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + expect(await hostIds()).toEqual(['local-ollama']); + expect(flag()).toBe(true); + + const removed = await h.service.remove({ removeModel: true }, [], ADMIN); + expect(removed.ok && removed.view.state).toBe('idle'); + expect(await hostIds()).toEqual([]); + expect(flag()).toBe(false); + }); + + it('the server is killed after the claim and before the record: the next server finishes it on Continue', async () => { + // What the killed server left: the claim in the state file, nothing in the endpoint store. + const h = setup({ + state: { ...jobAt('connect'), connection: { endpointId: 'local-ollama', flagWasEnabled: false } }, + overrides: real, + }); + await h.service.boot(); + expect(await h.service.view()).toMatchObject({ state: 'paused', step: 'connect' }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + expect(await hostIds()).toEqual(['local-ollama']); + await h.service.remove({}, [], ADMIN); + expect(await hostIds()).toEqual([]); + expect(flag()).toBe(false); + }); + + it('the server is killed after both writes and before the last step was recorded: Continue ends installed', async () => { + const h = setup({ + state: { ...jobAt('connect'), connection: { endpointId: 'local-ollama', flagWasEnabled: false } }, + overrides: real, + }); + await connectLocalModel({ port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072, ours: true }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ state: 'installed' }); + expect(await hostIds()).toEqual(['local-ollama']); + }); + + it('a record of ours the state file does not know (written by an earlier order of the two) is taken as ours', async () => { + // The record and the flag exist, the state file has no claim. + await connectLocalModel({ port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072, ours: false }); + const h = setup({ state: jobAt('connect', 'paused'), overrides: real }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ state: 'installed' }); + // Remove takes the record out. The setting stays: what it was before is not known. + await h.service.remove({}, [], ADMIN); + expect(await hostIds()).toEqual([]); + expect(flag()).toBe(true); + }); + + it('a record with our id that says something else is never overwritten', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + mkdirSync(dirname(HOSTS()), { recursive: true }); + const foreign = { id: 'local-ollama', label: 'My own', baseUrl: 'http://127.0.0.1:11434' }; + writeFileSync(HOSTS(), JSON.stringify([foreign])); + const h = setup({ state: jobAt('connect', 'paused'), overrides: real }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect((await h.service.view()).error?.code).toBe('endpoint-id-taken'); + expect(await readCustomModelHosts(getDataDir())).toEqual([foreign]); + expect(h.store.get()).not.toHaveProperty('connection'); + }); +}); + +// ---------- A job whose end could not be written ---------- + +describe('a state file that cannot be written when a step fails (a full disk)', () => { + /** A pull that fills the disk: from its failure on, every write of the state file is refused. */ + function diskFillsDuringPull(h: ReturnType) { + h.deps.pullModel.mockImplementation( + async (_target: unknown, _id: string, pull?: { onProgress?: (p: unknown) => void }) => { + pull?.onProgress?.({ completedBytes: 14_200_000_000, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 }); + h.store.disk.refuse = () => true; + return { ok: false as const, code: 'disk-full' as const, detail: 'write blob: no space left on device' }; + } + ); + } + + it('the job is not lost: the view says disk-full with the download as it stood, and no speed', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('model', 'paused') }); + diskFillsDuringPull(h); + await h.service.resume({}, ADMIN); + await h.service.idle(); + + // The file could not be told and still says the job runs. + expect(h.store.get().state).toBe('installing'); + const view = await h.service.view(); + expect(view).toMatchObject({ + state: 'failed', + step: 'model', + error: { code: 'disk-full', next: 'retry', detail: 'write blob: no space left on device' }, + actions: ['resume', 'install', 'remove'], + }); + expect(view.progress).toMatchObject({ completedBytes: 14_200_000_000, totalBytes: PULL_BYTES }); + expect(view.progress).not.toHaveProperty('bytesPerSecond'); + // A GET writes nothing: the file is as it was. + expect(h.store.get().state).toBe('installing'); + }); + + it('Cancel has nothing to cancel (409, not a 500), and "Try again" on a disk that is still full says so again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('model', 'paused') }); + diskFillsDuringPull(h); + await h.service.resume({}, ADMIN); + await h.service.idle(); + + expect(await h.service.cancel()).toMatchObject({ ok: false, status: 409, conflict: { reason: 'wrong-state' } }); + const again = await h.service.resume({}, ADMIN); + expect(again.ok && again.view).toMatchObject({ state: 'failed', error: { code: 'disk-full' } }); + expect(h.deps.pullModel).toHaveBeenCalledTimes(1); + }); + + it('once there is space again "Try again" writes the outcome, runs the step and installs', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('model', 'paused') }); + diskFillsDuringPull(h); + await h.service.resume({}, ADMIN); + await h.service.idle(); + + h.store.disk.refuse = undefined; + h.deps.pullModel.mockImplementation(async () => ({ ok: true as const, totalBytes: PULL_BYTES })); + const resumed = await h.service.resume({}, ADMIN); + expect(resumed.ok && resumed.view.state).toBe('installing'); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + expect((await h.service.view()).error).toBeUndefined(); + }); + + it('another verb writes what the view already showed: a new install replaces the failed job', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('model', 'paused'), runtimeOnDisk: true }); + diskFillsDuringPull(h); + await h.service.resume({}, ADMIN); + await h.service.idle(); + h.store.disk.refuse = undefined; + h.deps.pullModel.mockImplementation(async () => ({ ok: true as const, totalBytes: 13_793_441_427 })); + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + for (const row of [rec.pick!, ...rec.alternatives]) row.installBytes = row.downloadBytes; + return rec; + }); + const result = await installAndWait(h.service, { + modelId: 'gpt-oss:20b', + cli: 'opencode', + context: 131072, + acknowledgedBytes: 13_793_441_427, + }); + expect(result.ok).toBe(true); + expect(h.store.get()).toMatchObject({ state: 'installed', model: { id: 'gpt-oss:20b' } }); + }); + + it('Cancel while the disk is full stops the job and shows it paused, without throwing', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ + overrides: { + pullModel: vi.fn(async (_t: unknown, _id: string, pull?: PullOptions) => { + pull?.onProgress?.({ completedBytes: 6_200_000_000, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 }); + await untilAborted(pull?.signal); + return { ok: false as const, code: 'cancelled' as const }; + }), + }, + }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(h.deps.pullModel).toHaveBeenCalled()); + h.store.disk.refuse = () => true; + const result = await h.service.cancel(); + expect(result.ok && result.view).toMatchObject({ + state: 'paused', + step: 'model', + progress: { completedBytes: 6_200_000_000 }, + actions: ['resume', 'install', 'remove'], + }); + // Space again: Continue writes the pause it could not write, and goes on. + h.store.disk.refuse = undefined; + h.deps.pullModel.mockImplementation(async () => ({ ok: true as const, totalBytes: PULL_BYTES })); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + }); + + it('a first install on a full disk starts nothing and is answered with the sentence', async () => { + const h = setup(); + h.store.disk.refuse = () => true; + const result = await h.service.install(REQUEST, ADMIN); + expect(result).toEqual({ + ok: false, + status: 507, + message: { text: 'The disk is full. Free up space, then try again.' }, + }); + expect(h.calls).toEqual([]); + expect(h.store.get().state).toBe('idle'); + }); + + it('a step that finished but could not be recorded (a full disk) is disk-full, and the step runs again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('verify', 'paused') }); + // The write that would move the job past `verify` is refused, and every write after it. + h.store.disk.refuse = (next, before) => before.step === 'verify' && next.step !== 'verify'; + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ state: 'failed', step: 'verify', error: { code: 'disk-full' } }); + h.store.disk.refuse = undefined; + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + }); +}); + +// ---------- A runtime that does not start, and one an earlier server left behind ---------- + +describe('an installed model whose runtime does not start', () => { + it('says so on the view with what the runtime said, and Start tries again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: INSTALLED, runtimeOnDisk: true }); + h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'spawn ENOENT' }; + await h.service.boot(); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'installed', running: false, actions: ['start', 'remove'] }); + expect(view.error).toEqual({ + code: 'runtime-start-failed', + message: { text: 'The runtime did not start on this Mac.' }, + // Start is the verb an installed model has: there is no job to resume. + next: 'start', + detail: 'spawn ENOENT', + }); + // A session start is refused with the same reason, not with "start it first". + expect(await h.service.launchRefusalMessage('local-ollama', 'qwen3.6:35b-a3b')).toBe( + 'The runtime did not start on this Mac.' + ); + + // Start again, and it fails again: the view still says why. + const again = await h.service.runtime({ action: 'start' }); + expect(again.ok && again.view.error?.code).toBe('runtime-start-failed'); + + // Start again, and it works: the error is gone. + h.world.startResult = undefined; + const started = await h.service.runtime({ action: 'start' }); + expect(started.ok && started.view).toMatchObject({ running: true }); + expect(started.ok && started.view.error).toBeUndefined(); + }); + + it('Stop clears it: a model the person stopped shows no error', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: INSTALLED, runtimeOnDisk: true }); + h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'spawn ENOENT' }; + await h.service.boot(); + const stopped = await h.service.runtime({ action: 'stop' }); + expect(stopped.ok && stopped.view.error).toBeUndefined(); + }); +}); + +describe('a later build with another runtime pin', () => { + const OLD = '0.39.2'; + const OLD_STATE: LocalModelStateFile = { ...INSTALLED, runtime: { ...INSTALLED.runtime!, version: OLD } }; + /** Only the old version is unpacked: what an update of Codeman leaves on disk. */ + const onlyOldOnDisk = async (_pin?: unknown, paths?: LocalModelPaths) => + paths?.binary === `/data/local-model/runtime/ollama-${OLD}/ollama`; + + it('an installed model keeps running the version it was installed with', async () => { + const h = setup({ state: OLD_STATE }); + h.deps.isRuntimeInstalled.mockImplementation(onlyOldOnDisk as never); + await h.service.boot(); + expect(h.supervisor.start).toHaveBeenCalledWith({ + port: 11435, + contextLength: 131072, + previousPid: 4711, + version: OLD, + }); + // The stored version stays what runs, so the next start finds it again. + expect(h.store.get().runtime?.version).toBe(OLD); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'installed', running: true, actions: ['start-session', 'stop', 'remove'] }); + expect(view.error).toBeUndefined(); + // A NEW install would still download the pinned build. + expect(view.runtime).toMatchObject({ version: OLLAMA_RUNTIME_PIN.version, installed: false }); + }); + + it('with the pinned build unpacked that one runs, and the state says so', async () => { + const h = setup({ state: OLD_STATE, runtimeOnDisk: true }); + await h.service.boot(); + expect(h.supervisor.start).toHaveBeenCalledWith({ port: 11435, contextLength: 131072, previousPid: 4711 }); + expect(h.store.get().runtime?.version).toBe(OLLAMA_RUNTIME_PIN.version); + }); + + it('with neither on disk the start fails in the open: the view says the runtime did not start', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: OLD_STATE }); + h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'spawn ENOENT' }; + await h.service.boot(); + expect((await h.service.view()).error).toMatchObject({ code: 'runtime-start-failed', next: 'start' }); + }); + + it('a version the state file could never name is not turned into a path', async () => { + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, version: '../../evil' } } }); + await h.service.boot(); + expect(h.supervisor.start).toHaveBeenCalledWith({ port: 11435, contextLength: 131072, previousPid: 4711 }); + }); +}); + +describe('a runtime an earlier server left behind', () => { + it.each(['paused', 'failed'] as const)('Remove of a %s job ends it before any file is deleted', async (state) => { + const base = jobAt('model', 'paused'); + const h = setup({ + state: state === 'paused' ? base : { ...base, state: 'failed', error: { code: 'pull-failed' } }, + }); + const removed = await h.service.remove({ removeModel: true }, [], ADMIN); + expect(removed.ok && removed.view.state).toBe('idle'); + expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 5000 }); + expect(h.calls.indexOf('supervisor.stop')).toBeLessThan(h.calls.indexOf('removeLocalModelFiles')); + }); + + it('Stop of an installed model ends it too, also when this server never started it', async () => { + const h = setup({ state: INSTALLED }); + await h.service.runtime({ action: 'stop' }); + expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 4711 }); + }); + + it('boot ends it for a model the person had stopped', async () => { + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } }); + await h.service.boot(); + expect(h.supervisor.start).not.toHaveBeenCalled(); + expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 4711 }); + }); + + it('the pid is stored the moment the child is spawned, before it answers', async () => { + const h = setup({ + state: { ...jobAt('model', 'paused'), runtime: { version: '0.40.15', port: 11435, enabled: true } }, + }); + h.service.onRuntimeChange({ running: false, pid: 7777, restarts: 0, gaveUp: false }); + await vi.waitFor(() => expect(h.store.get().runtime?.pid).toBe(7777)); + }); +}); + +// ---------- What a failed download is called, and what is offered next ---------- + +describe('a runtime that is gone for good during the download', () => { + it('is said as the runtime, not as the network, and "Try again" runs the step again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + h.deps.pullModel.mockImplementation(async () => { + h.world.running = false; + h.world.gaveUp = true; + return { ok: false as const, code: 'pull-failed' as const, detail: 'connect ECONNREFUSED 127.0.0.1:11435' }; + }); + await installAndWait(h.service); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'failed', step: 'model', actions: ['resume', 'install', 'remove'] }); + expect(view.error).toEqual({ + code: 'runtime-stopped', + message: { text: 'The runtime keeps stopping, so Codeman no longer restarts it.' }, + // A job has no Start verb: the step is run again. + next: 'retry', + detail: 'connect ECONNREFUSED 127.0.0.1:11435', + }); + }); + + it('a pull that fails while the runtime runs is still the download', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + h.deps.pullModel.mockResolvedValue({ ok: false, code: 'pull-failed', detail: 'unexpected EOF' } as never); + await installAndWait(h.service); + expect((await h.service.view()).error).toMatchObject({ code: 'pull-failed', next: 'retry' }); + }); +}); + +describe('the model offered after "does not fit"', () => { + it('needs less memory than the one that failed: a model that needs more is not offered', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + // The only other row needs MORE than the model that failed. + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + rec.alternatives = [candidate('qwen3.8:27b', 'Qwen3.8 27B', 131072, 17_700_000_000, false, 39_000_000_000)]; + return rec; + }); + h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'does-not-fit' } as never); + await installAndWait(h.service); + const view = await h.service.view(); + expect(view.error).toMatchObject({ code: 'does-not-fit', next: 'next-model' }); + expect(view.error).not.toHaveProperty('nextModel'); + }); + + it('skips a larger row for the first one that needs less', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + h.deps.getCatalog.mockReturnValue({ + ...CATALOG, + models: [...CATALOG.models, { id: 'qwen3.8:27b', label: 'Qwen3.8 27B' }], + } as unknown as LocalModelCatalog); + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + rec.alternatives = [ + candidate('qwen3.8:27b', 'Qwen3.8 27B', 131072, 17_700_000_000, false, 39_000_000_000), + candidate('gpt-oss:20b', 'GPT-OSS 20B', 131072, 13_793_441_427, false, 26_600_000_000), + ]; + return rec; + }); + h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'does-not-fit' } as never); + await installAndWait(h.service); + expect((await h.service.view()).error?.nextModel).toEqual({ modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' }); + }); + + it('after "no tool call" memory is not the question: the best other model is offered', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + h.deps.getCatalog.mockReturnValue({ + ...CATALOG, + models: [...CATALOG.models, { id: 'qwen3.8:27b', label: 'Qwen3.8 27B' }], + } as unknown as LocalModelCatalog); + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + rec.alternatives = [candidate('qwen3.8:27b', 'Qwen3.8 27B', 131072, 17_700_000_000, false, 39_000_000_000)]; + return rec; + }); + h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'no-tool-call' } as never); + await installAndWait(h.service); + expect((await h.service.view()).error?.nextModel).toEqual({ modelId: 'qwen3.8:27b', modelLabel: 'Qwen3.8 27B' }); + }); +}); + +describe('what a replaced download left behind', () => { + it('the partial blobs are deleted as files, after the model was asked for and before the new pull', async () => { + const h = setup({ + state: { ...jobAt('model', 'paused'), replacedModelId: 'gpt-oss:20b' }, + }); + h.deps.removePartialModelBlobs.mockImplementation(async () => { + h.calls.push('removePartialModelBlobs'); + return 12_708_189_425; + }); + vi.spyOn(console, 'log').mockImplementation(() => {}); + await h.service.resume({}, ADMIN); + await h.service.idle(); + const order = h.calls.filter((call) => ['deleteModel', 'removePartialModelBlobs', 'pullModel'].includes(call)); + expect(order).toEqual(['deleteModel', 'removePartialModelBlobs', 'pullModel']); + expect(h.deps.removePartialModelBlobs).toHaveBeenCalledWith(PATHS); + expect(h.store.get()).not.toHaveProperty('replacedModelId'); + }); + + it('a job that replaced nothing deletes nothing', async () => { + const h = setup(); + await installAndWait(h.service); + expect(h.deps.deleteModel).not.toHaveBeenCalled(); + expect(h.deps.removePartialModelBlobs).not.toHaveBeenCalled(); + }); +}); + +// ---------- Live sessions keep the address they were started with ---------- + +describe('Stop and "Use another port" while live sessions use the model', () => { + it('Stop is 409 sessions-open with their names, and the runtime keeps running', async () => { + const h = setup({ state: INSTALLED }); + await h.service.boot(); + h.supervisor.stop.mockClear(); + const result = await h.service.runtime({ action: 'stop' }, ['w1-app', 'w4-lib']); + expect(result).toMatchObject({ + ok: false, + status: 409, + conflict: { + reason: 'sessions-open', + sessions: ['w1-app', 'w4-lib'], + message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen, vars: { sessions: 'w1-app, w4-lib' } }, + }, + }); + expect(h.supervisor.stop).not.toHaveBeenCalled(); + expect(h.store.get().runtime?.enabled).toBe(true); + }); + + it('a start on another port is refused the same way, and nothing moves', async () => { + const h = setup({ state: INSTALLED }); + const result = await h.service.runtime({ action: 'start', newPort: true }, ['w1-app']); + expect(result).toMatchObject({ ok: false, status: 409, conflict: { reason: 'sessions-open' } }); + expect(h.deps.chooseRuntimePort).not.toHaveBeenCalled(); + expect(h.deps.moveLocalEndpoint).not.toHaveBeenCalled(); + expect(h.store.get().runtime?.port).toBe(11435); + }); + + it('a plain Start is not refused: it changes no address', async () => { + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } }); + const result = await h.service.runtime({ action: 'start' }, ['w1-app']); + expect(result.ok && result.view.running).toBe(true); + }); + + it('"Use another port" of a job is refused only when a connection of ours exists', async () => { + const failedOnPort: LocalModelStateFile = { + ...jobAt('connect', 'paused'), + state: 'failed', + error: { code: 'port-in-use', vars: { port: '11435' } }, + }; + const connected = setup({ + state: { ...failedOnPort, connection: { endpointId: 'local-ollama', flagWasEnabled: false } }, + }); + expect(await connected.service.resume({ newPort: true }, ADMIN, ['w1-app'])).toMatchObject({ + ok: false, + conflict: { reason: 'sessions-open', sessions: ['w1-app'] }, + }); + expect(connected.deps.chooseRuntimePort).not.toHaveBeenCalled(); + + // Without a connection of ours no session can be on the model: sessions elsewhere do not matter. + const fresh = setup({ state: failedOnPort }); + fresh.deps.chooseRuntimePort.mockResolvedValue(11436); + const result = await fresh.service.resume({ newPort: true }, ADMIN, ['w1-app']); + expect(result.ok).toBe(true); + await fresh.service.idle(); + }); + + it('a state write that fails before the saved endpoint moved changes nothing', async () => { + const h = setup({ state: INSTALLED }); + h.deps.chooseRuntimePort.mockResolvedValue(11437); + h.store.disk.refuse = (next) => next.runtime?.port === 11437; + await expect(h.service.runtime({ action: 'start', newPort: true })).rejects.toThrow('ENOSPC'); + expect(h.deps.moveLocalEndpoint).not.toHaveBeenCalled(); + expect(h.store.get().runtime?.port).toBe(11435); + }); +}); diff --git a/test/local-model-ollama-client.test.ts b/test/local-model-ollama-client.test.ts index 35fd13b0..94fcb670 100644 --- a/test/local-model-ollama-client.test.ts +++ b/test/local-model-ollama-client.test.ts @@ -708,7 +708,7 @@ describe('judgeLoaded', () => { expect(judged).toEqual({ ok: false, code: 'does-not-fit', - detail: '20000000000 of 23205798956 bytes are on the GPU', + detail: '20.0 of 23.2 GB of the model are on the GPU', }); // One byte is enough. expect(judgeLoaded([loaded({ sizeVram: 23_205_798_955 })], MODEL, CONTEXT)).toMatchObject({ code: 'does-not-fit' }); @@ -885,7 +885,7 @@ describe('verifyModel', () => { expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({ ok: false, code: 'does-not-fit', - detail: '18000000000 of 23205798956 bytes are on the GPU', + detail: '18.0 of 23.2 GB of the model are on the GPU', }); expect(server.requests.filter((r) => r.path === '/v1/chat/completions')).toHaveLength(1); }); diff --git a/test/routes/custom-model-routes.test.ts b/test/routes/custom-model-routes.test.ts index 7253c9aa..fc31495a 100644 --- a/test/routes/custom-model-routes.test.ts +++ b/test/routes/custom-model-routes.test.ts @@ -777,7 +777,7 @@ describe('the endpoint the local model install manages', () => { }); expect(res.json()).toEqual({ success: false, - error: 'The endpoint id "local-ollama" is reserved for the model Codeman runs on this Mac. Pick another id.', + error: 'This endpoint id is reserved for the model Codeman runs on this Mac. Pick another id.', errorCode: 'INVALID_INPUT', }); expect(await readCustomModelHosts(getDataDir())).toEqual([]); diff --git a/test/routes/local-model-routes.test.ts b/test/routes/local-model-routes.test.ts index 0ac50443..11ecb354 100644 --- a/test/routes/local-model-routes.test.ts +++ b/test/routes/local-model-routes.test.ts @@ -657,20 +657,43 @@ describe('the install verbs', () => { expect(f.service.install).toHaveBeenCalledWith(INSTALL_BODY, { username: 'alice', ip: '127.0.0.1' }); }); - it('a row the server does not offer is a 400 with the service own sentence', async () => { + it('a row the server does not offer is a 400 with the service own sentence, as text and as data', async () => { const f = fakes(); f.service.install.mockResolvedValue({ ok: false, status: 400, - message: 'This model at this context is not offered for this Mac right now.', + message: { text: LOCAL_MODEL_INSTALL_LABELS.refusedAgentNotInstallable, vars: { cli: 'OpenCode' } }, }); const { app } = await harness(f.deps); const res = await app.inject({ method: 'POST', url: '/api/local-model/install', payload: INSTALL_BODY }); expect(res.statusCode).toBe(400); + // `error` is the English sentence with its slots filled; `data.message` is the sentence + // with its slots apart, which the window prints in its own language. expect(res.json()).toEqual({ success: false, - error: 'This model at this context is not offered for this Mac right now.', + error: 'OpenCode is not installed and cannot be installed from here.', errorCode: 'INVALID_INPUT', + data: { + message: { text: '{cli} is not installed and cannot be installed from here.', vars: { cli: 'OpenCode' } }, + }, + }); + }); + + it('a first install on a full disk is a 507 with the sentence, not a 500', async () => { + const f = fakes(); + f.service.install.mockResolvedValue({ + ok: false, + status: 507, + message: { text: LOCAL_MODEL_INSTALL_LABELS.errDiskFull }, + }); + const { app } = await harness(f.deps); + const res = await app.inject({ method: 'POST', url: '/api/local-model/install', payload: INSTALL_BODY }); + expect(res.statusCode).toBe(507); + expect(res.json()).toEqual({ + success: false, + error: 'The disk is full. Free up space, then try again.', + errorCode: 'OPERATION_FAILED', + data: { message: { text: 'The disk is full. Free up space, then try again.' } }, }); }); @@ -743,6 +766,46 @@ describe('the install verbs', () => { expect(f.service.resume.mock.calls[0][1]).toEqual({ username: 'admin', ip: '127.0.0.1' }); }); + it('resume and runtime hand the service the live sessions on the model, read at the moment of the request', async () => { + const f = fakes(); + const sessions = new Map([ + ['a', { name: 'w1-app', customModel: { endpointId: 'local-ollama' } }], + ['b', { name: 'w2-app', customModel: { endpointId: 'office-gpu' } }], + ]); + const { app } = await createRouteTestHarness((instance) => + registerLocalModelRoutes(instance, { sessions }, f.deps) + ); + await app.inject({ method: 'POST', url: '/api/local-model/runtime', payload: { action: 'stop' } }); + await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: { newPort: true } }); + sessions.delete('a'); + await app.inject({ method: 'POST', url: '/api/local-model/runtime', payload: { action: 'stop' } }); + expect(f.service.runtime.mock.calls.map((args) => args[1])).toEqual([['w1-app'], []]); + expect(f.service.resume.mock.calls[0][2]).toEqual(['w1-app']); + }); + + it('a stop while a live session uses the model: 409 with the session names', async () => { + const f = fakes(); + f.service.runtime.mockImplementation(async (_request, names = []) => ({ + ok: false, + status: 409, + conflict: { + reason: 'sessions-open', + message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen, vars: { sessions: names.join(', ') } }, + sessions: [...names], + }, + })); + const { app } = await harness(f.deps, undefined, [ + { name: 'w1-app', customModel: { endpointId: 'local-ollama' } }, + ]); + const res = await app.inject({ method: 'POST', url: '/api/local-model/runtime', payload: { action: 'stop' } }); + expect(res.statusCode).toBe(409); + expect(res.json()).toMatchObject({ + errorCode: 'CONFLICT', + error: 'Sessions are still using this model: w1-app. Close them first.', + data: { reason: 'sessions-open', sessions: ['w1-app'] }, + }); + }); + it('resume refuses a body it does not know', async () => { const f = fakes(); const { app } = await harness(f.deps);