diff --git a/src/local-model/connect.ts b/src/local-model/connect.ts index a5001f9e..0af1fb12 100644 --- a/src/local-model/connect.ts +++ b/src/local-model/connect.ts @@ -15,6 +15,15 @@ * Rules: * - The id is reserved. A record with it that this feature did not write is never * overwritten: the connect step fails with `endpoint-id-taken`. + * - The connection is CLAIMED before it is written: the install job records `connection` in + * the state file first (`inspectLocalEndpoint()` tells it what is there and what the flag + * is), then calls `connectLocalModel()` with `ours: true`. A state write that fails, or a + * server that is killed between the two, therefore never leaves a record of ours that the + * state file does not know: the worst case is a claim with nothing behind it, which a + * resumed job fills in and Remove undoes. + * - Second net: a record with the reserved id that already says exactly what ours would say + * (`isSameLocalEndpoint()`) is ours whatever `ours` says. Nobody else can have made it: the + * endpoint routes refuse the reserved id. * - Both writes are idempotent, so a resumed job writes them again. * - Undoing removes our record, and switches the flag off again only when this feature * switched it on AND no other endpoint is saved. @@ -27,7 +36,10 @@ import { mutateCustomModelHosts } from '../custom-model-hosts-writer.js'; import type { CustomModelHost } from '../custom-model-hosts.js'; +import { getDataDir } from '../config/instance.js'; +import { readCustomModelHosts } from '../custom-model-hosts.js'; import { getErrorMessage, type LocalModelErrorCode } from '../types.js'; +import { readJsonConfig, SETTINGS_PATH } from '../web/route-helpers.js'; import { CustomModelHostSchema } from '../web/schemas.js'; import { mutateSettingsFile } from '../web/settings-writer.js'; import { readLocalModelState } from './state-store.js'; @@ -63,10 +75,58 @@ export function buildLocalEndpointRecord(input: LocalEndpointInput): CustomModel }; } +/** + * Whether a saved record says exactly what our record for `input` says: the same address, the + * one model, its context, no key. What discovery may have added to it (context lengths, a + * time stamp) does not count. Pure. + */ +export function isSameLocalEndpoint(host: CustomModelHost, input: LocalEndpointInput): boolean { + const ours = buildLocalEndpointRecord(input); + return ( + host.id === ours.id && + host.label === ours.label && + host.baseUrl === ours.baseUrl && + host.authStyle === ours.authStyle && + host.apiKey === undefined && + Array.isArray(host.models) && + host.models.length === 1 && + host.models[0] === input.modelId && + host.defaultModelId === ours.defaultModelId && + host.defaultContextLength === ours.defaultContextLength + ); +} + +/** What the connect step finds before it claims the connection. */ +export interface LocalEndpointInspection { + /** + * The saved record with the reserved id: there is none, it says what ours would say, or it + * says something else (somebody else's: `endpoint-id-taken`). + */ + record: 'absent' | 'same' | 'other'; + /** `customModelEndpointsEnabled` right now: what the claim keeps as the value before this feature. */ + flagEnabled: boolean; +} + +/** + * Looks at the endpoint store and the settings flag WITHOUT writing either. The install job + * calls it before it claims the connection in the state file. Rejects when the endpoint store + * cannot be read; a settings file that cannot be read counts as "the flag is off". + */ +export async function inspectLocalEndpoint(input: LocalEndpointInput): Promise { + const hosts = await readCustomModelHosts(getDataDir()); + const existing = hosts.find((host) => host.id === LOCAL_MODEL_ENDPOINT_ID); + const settings = await readJsonConfig>(SETTINGS_PATH, 'settings.json', {}); + return { + record: !existing ? 'absent' : isSameLocalEndpoint(existing, input) ? 'same' : 'other', + flagEnabled: settings[ENDPOINTS_FLAG] === true, + }; +} + export interface ConnectLocalModelInput extends LocalEndpointInput { /** * True when the state file already records a connection of ours: an existing record with - * our id is then ours and is replaced. False: such a record is somebody else's. + * our id is then ours and is replaced. False: such a record is somebody else's, unless it + * says exactly what ours would say (`isSameLocalEndpoint()`). */ ours: boolean; /** @@ -106,7 +166,7 @@ export async function connectLocalModel(input: ConnectLocalModelInput): Promise< outcome = await mutateCustomModelHosts<'added' | 'replaced' | 'taken'>((hosts) => { const index = hosts.findIndex((host) => host.id === record.id); if (index === -1) return { hosts: [...hosts, record], result: 'added' }; - if (!input.ours) return { result: 'taken' }; + if (!input.ours && !isSameLocalEndpoint(hosts[index], input)) return { result: 'taken' }; const next = [...hosts]; next[index] = record; return { hosts: next, result: 'replaced' }; @@ -158,13 +218,17 @@ async function removeLocalEndpoint(): Promise { }); } -/** Moves our record to another port (the answer to "Use another port" once connected). A missing record is not an error. */ +/** + * Moves our record to another port (the answer to "Use another port" once connected). A + * missing record is not an error, and a record that is already there is not written again. + */ export async function moveLocalEndpoint(port: number): Promise { + const baseUrl = `http://127.0.0.1:${port}`; await mutateCustomModelHosts((hosts) => { const index = hosts.findIndex((host) => host.id === LOCAL_MODEL_ENDPOINT_ID); - if (index === -1) return { result: undefined }; + if (index === -1 || hosts[index].baseUrl === baseUrl) return { result: undefined }; const next = [...hosts]; - next[index] = { ...next[index], baseUrl: `http://127.0.0.1:${port}` }; + next[index] = { ...next[index], baseUrl }; return { hosts: next, result: undefined }; }); } diff --git a/src/local-model/install-job.ts b/src/local-model/install-job.ts index 7d688f74..cc2ae0d8 100644 --- a/src/local-model/install-job.ts +++ b/src/local-model/install-job.ts @@ -41,7 +41,8 @@ * | | never removed by us | | | * | `connect` | our endpoint record and the settings | honoured before the first write; once | writes both again | * | | flag: the two writes outside our | the writes began they finish and the job | | - * | | folder | ends `installed` | | + * | | folder. The state file claims them | ends `installed` | | + * | | first | | | * * How the memory budget gets measured before a model is downloaded (ONE consent, bound to the * size shown): the person confirms `Candidate.installBytes` once. The `runtime` step then @@ -62,7 +63,9 @@ * - A job read as `installing` at boot was interrupted by the restart: the service applies * the event `restart` before anything else, so it is `paused` and waits for a click. * - A new install from `paused` or `failed` REPLACES the job. When it names another model, - * what the old job left of its model is deleted from the store first. + * what the old job left of its model is deleted from the store first: the model through the + * runtime when it was downloaded whole, and the partial blobs of a download that was + * interrupted (the runtime knows no model for those, so they are deleted as files). * - Install is checked before anything is downloaded: admin only in multi-user mode; the * model, the context and the coding agent must be a row the current recommendation offers; * the coding agent must be a harness row of the catalogue; `acknowledgedBytes` must be the @@ -80,13 +83,29 @@ * - A download's progress is kept in memory, written to the state file every few seconds and * when the job pauses or fails. The view shows the live figure and its speed. * - What blocks a session on an installed model (another program on the port, a runtime that - * keeps stopping, a model whose digest changed) is found live and shown as `error` on the - * view. It is never stored, and the state stays `installed`. - * - `launchRefusalMessage()` is what both session launch paths ask before they start a - * session on the managed endpoint: the digest is read again for every launch. + * keeps stopping or did not start, a model whose digest changed) is found live and shown as + * `error` on the view. It is never stored, and the state stays `installed`. + * - `launchRefusalMessage()` is what every path that STARTS a coding agent on the managed + * endpoint asks first: a new session with the model, a model applied to a session, the + * attach that starts an agent in a session that has none, and the rebuild after a reboot. + * The digest is read again for every one of them. Two things are not launches and are not + * asked: a pane that survived a server restart (its agent is already running), and a + * respawn cycle (it types into the agent that runs). * - With a model installed, `boot()` puts the saved endpoint record back when it is gone (a * config-backup restore can take it away), starts the runtime unless it was stopped, and * reads the digest again. + * - A runtime an earlier server left behind (a crash, a forced quit) is never left running + * under a deleted folder or beside a second one: its pid is stored the moment it is + * spawned, a start adopts or replaces it, and Stop, Remove and the boot of anything that + * is not a running installation end it (`RuntimeSupervisor.stop({ previousPid })`). + * - An installed model keeps the runtime VERSION it was installed with: when a later build + * pins another version, the stored one is started for as long as its folder is unpacked. + * - A job whose end could not be written to the state file (a full disk) is not lost: the + * outcome is kept in memory, the view shows it, Resume and a new install start from it, and + * the next verb writes it. The same holds for any `installing` in the file while this + * process runs no job, which is what a restart leaves too. + * - Stop and "Use another port" are refused (409 `sessions-open`) while live sessions use + * the model: their coding agents were given the address when they started. */ import { appendAdminAudit } from '../web/admin-audit.js'; @@ -124,6 +143,7 @@ import { type LocalModelInstallState, type LocalModelInstallStep, type LocalModelInstallView, + type LocalModelNextStep, type LocalModelProgress, type LocalModelRemoveRequest, type LocalModelResumeRequest, @@ -139,12 +159,14 @@ import { connectLocalModel, disconnectLocalModel, ensureLocalEndpoint, + inspectLocalEndpoint, isLocalModelEndpointId, LOCAL_MODEL_ENDPOINT_ID, moveLocalEndpoint, type ConnectLocalModelInput, type ConnectResult, type LocalEndpointInput, + type LocalEndpointInspection, } from './connect.js'; import { checkDeveloperTools } from './developer-tools.js'; import { formatGb } from './format.js'; @@ -164,8 +186,8 @@ import { type PullResult, type VerifyResult, } from './ollama-client.js'; -import { localModelPaths, type LocalModelPaths } from './paths.js'; -import { modelFitsBudget, recommend } from './recommend.js'; +import { localModelPaths, runtimeBinaryForVersion, type LocalModelPaths } from './paths.js'; +import { modelFitsBudget, needBytes, recommend } from './recommend.js'; import { chooseRuntimePort, createRuntimeSupervisor, @@ -173,6 +195,7 @@ import { isRuntimeInstalled, OLLAMA_RUNTIME_PIN, removeLocalModelFiles, + removePartialModelBlobs, RUNTIME_PORT_FIRST, type ChooseRuntimePortOptions, type RemoveLocalModelFilesOptions, @@ -304,7 +327,15 @@ export const INSTALL_STEP_TABLE: Readonly; detail?: string; nextModel?: { modelId: string; modelLabel: string } } = {} + extra: { + vars?: Record; + detail?: string; + nextModel?: { modelId: string; modelLabel: string }; + /** The next step in the state the error is shown in, when it is not the code's default. */ + next?: LocalModelNextStep; + } = {} ): LocalModelInstallError { - const next = LOCAL_MODEL_ERROR_NEXT[code]; + const next = extra.next ?? LOCAL_MODEL_ERROR_NEXT[code]; return { code, message: { @@ -542,8 +581,8 @@ export function buildInstallError( /** * What blocks a session in state `installed`, found live and never stored: another program - * on the port, a runtime that keeps stopping, a model that changed. `undefined` when nothing - * does, which includes a runtime the person stopped. Pure. + * on the port, a runtime that keeps stopping, a runtime that did not start, a model that + * changed. `undefined` when nothing does, which includes a runtime the person stopped. Pure. */ export function installedBlocker( state: LocalModelStateFile, @@ -554,6 +593,13 @@ export function installedBlocker( return buildInstallError('port-in-use', { vars: { port: String(state.runtime?.port ?? '') } }); } if (!live.running && live.gaveUp) return buildInstallError('runtime-stopped'); + if (!live.running && live.startFailed) { + // Start is the verb an installed model has: it tries the start again. + return buildInstallError('runtime-start-failed', { + next: 'start', + ...(live.startFailed.detail ? { detail: live.startFailed.detail } : {}), + }); + } if (live.digest === 'changed') return buildInstallError('model-changed'); return undefined; } @@ -646,6 +692,8 @@ export function buildInstallView(state: LocalModelStateFile, live: InstallLiveFa vars: state.error.vars, detail: state.error.detail, nextModel: live.nextModel, + // A job has no Start verb: what helps there is to run the step again. + ...(LOCAL_MODEL_ERROR_NEXT[state.error.code] === 'start' ? { next: 'retry' as const } : {}), }); } return view; @@ -667,10 +715,11 @@ export interface LocalModelActor { ip?: string; } -/** How a verb ended: the view after it, a 400 with its message, or a 409 with its conflict. */ +/** How a verb ended: the view after it, a refusal with its sentence, or a 409 with its conflict. */ export type LocalModelVerbResult = | { ok: true; view: LocalModelInstallView } - | { ok: false; status: 400; message: string } + /** 400: the request is not something the server offers. 507: nothing could be started because the disk is full. */ + | { ok: false; status: 400 | 507; message: LocalModelInstallMessage } | { ok: false; status: 409; conflict: LocalModelConflict }; /** @@ -710,6 +759,8 @@ export interface LocalModelServiceDeps { listRunning: (target: OllamaTarget, signal?: AbortSignal) => Promise; getRuntimeVersion: (target: OllamaTarget, signal?: AbortSignal) => Promise; deleteModel: (target: OllamaTarget, modelId: string, signal?: AbortSignal) => Promise; + /** `removePartialModelBlobs()` of src/local-model/runtime.ts: what an interrupted download left in the model store. Resolves with the bytes freed. */ + removePartialModelBlobs: (paths?: LocalModelPaths) => Promise; // src/local-model/developer-tools.ts checkDeveloperTools: () => Promise; // src/cli-install-runner.ts @@ -721,6 +772,8 @@ export interface LocalModelServiceDeps { /** Writes one admin audit line (`appendAdminAudit()`). */ audit: (entry: { admin: string; action: string; target: string; ip?: string; detail?: unknown }) => Promise; // src/local-model/connect.ts + /** What is saved under the reserved id and what the settings flag is, read before the connection is claimed. Writes nothing. */ + inspectLocalEndpoint: (input: LocalEndpointInput) => Promise; connectLocalModel: (input: ConnectLocalModelInput) => Promise; moveLocalEndpoint: (port: number) => Promise; /** Puts our endpoint record back when it is gone (after a config-backup restore). True when it was added. */ @@ -749,9 +802,10 @@ export interface LocalModelService { * yes, which is also the answer for every endpoint that is not this feature's. Otherwise the * error that refuses the launch. While the runtime child Codeman started is not alive * (whatever answers on the port then is not ours), the reason is the one that is known: - * `port-in-use`, `runtime-stopped`, else `runtime-not-running`. While it runs, the digest is - * read again: `model-changed` when it is not the stored one or the model is gone. The session - * launch path asks this before it starts a session with the managed endpoint. Never throws. + * `port-in-use`, `runtime-stopped`, `runtime-start-failed`, else `runtime-not-running`. + * While it runs, the digest is read again: `model-changed` when it is not the stored one or + * the model is gone. Every path that starts a coding agent on the managed endpoint asks + * this first (see the file header for the list). Never throws. */ launchRefusal(endpointId: string): Promise; /** @@ -767,10 +821,21 @@ export interface LocalModelService { install(request: LocalModelInstallRequest, actor: LocalModelActor): Promise; /** `POST /api/local-model/cancel`. Answers once the step in flight was interrupted. */ cancel(): Promise; - /** `POST /api/local-model/resume`. */ - resume(request: LocalModelResumeRequest, actor: LocalModelActor): Promise; - /** `POST /api/local-model/runtime`. */ - runtime(request: LocalModelRuntimeRequest): Promise; + /** + * `POST /api/local-model/resume`. `liveSessionNames` as for `remove()`: with `newPort` and a + * connection of ours, not empty answers 409 `sessions-open`. + */ + resume( + request: LocalModelResumeRequest, + actor: LocalModelActor, + liveSessionNames?: readonly string[] + ): Promise; + /** + * `POST /api/local-model/runtime`. `liveSessionNames` as for `remove()`: Stop and a start on + * another port answer 409 `sessions-open` while one of them uses the model, because their + * coding agents keep calling the address they were started with. + */ + runtime(request: LocalModelRuntimeRequest, liveSessionNames?: readonly string[]): Promise; /** * `DELETE /api/local-model`. `liveSessionNames` is what `sessionNamesUsingEndpoint()` found: * not empty answers 409 `sessions-open` while a connection of ours exists. Else: stop the @@ -788,13 +853,14 @@ export interface LocalModelService { * Called once when the server starts. An interrupted job becomes `paused`. With a model * installed: the saved endpoint record is put back when it is gone, a runtime that was not * stopped is started (an own orphan on the port is adopted or replaced, another listener is - * `port-in-use`) and the digest is read again. Never throws, and never keeps the server - * from starting. + * `port-in-use`) and the digest is read again. In every other case a runtime an earlier + * server left behind is ended: nothing would use it, and it holds the model's memory. + * Never throws, and never keeps the server from starting. */ boot(): Promise; /** Called when the server stops, beside `stopDeepSeekWeb()`: aborts a job in flight (it reads `paused` at the next boot) and stops the runtime. */ shutdown(): Promise; - /** What the runtime supervisor reports through `onChange`: a new pid after a restart, or the budget line. Stored, never thrown. */ + /** What the runtime supervisor reports through `onChange`: the pid of a child the moment it is spawned, a new pid after a restart, the budget line. Stored, never thrown. */ onRuntimeChange(status: RuntimeStatus): void; /** Resolves once no job is in flight. For `shutdown()` and for tests. */ idle(): Promise; @@ -842,13 +908,23 @@ function lazySupervisor(build: () => RuntimeSupervisor): RuntimeSupervisor { let instance: RuntimeSupervisor | undefined; return { start: (options) => (instance ??= build()).start(options), - stop: async () => { - if (instance) await instance.stop(); + stop: async (options) => { + // Built for a stored pid too: a runtime an earlier server left behind has no instance here. + if (instance || options?.previousPid !== undefined) await (instance ??= build()).stop(options); }, status: () => (instance ? instance.status() : { running: false, restarts: 0, gaveUp: false }), }; } +/** Whether a write failed for lack of space. */ +function isDiskFullError(err: unknown): boolean { + const code = typeof err === 'object' && err !== null ? (err as NodeJS.ErrnoException).code : undefined; + return code === 'ENOSPC' || code === 'EDQUOT' || /no space left on device/i.test(getErrorMessage(err)); +} + +/** How a job ended, as the `fail` event carries it. */ +type JobFailure = Omit, 'type'>; + /** Forgets the cached lookups of a CLI's binaries the way the CLI registry routes do, then probes it again. */ async function recheckCliInstalledNow(cli: string): Promise { const entry = getCli(cli); @@ -866,12 +942,22 @@ class LocalModelServiceImpl implements LocalModelService { private verbChain: Promise = Promise.resolve(); private recovery: Promise | undefined; private stopping = false; + /** A verb wrote `installing` and is about to start the job: the view must not take that for a job that is gone. */ + private jobStarting = false; + /** + * How the last job ended when the state file could not be told (a full disk): the failure, + * or nothing for a pause, and the download as it stood. Shown from memory and written by + * the next verb. + */ + private unrecorded: { failure?: JobFailure; progress?: { completedBytes: number; totalBytes: number } } | undefined; /** The download in flight, as last reported. The state file gets it every few seconds. */ private liveProgress: { completedBytes: number; totalBytes: number; bytesPerSecond?: number } | undefined; private lastProgressFlush = 0; private speedSamples: Array<{ at: number; bytes: number }> = []; /** The last start found another program on the stored port. */ private portInUse = false; + /** The last start failed for another reason, with what the runtime said. Cleared by a start that works and by Stop. */ + private startFailed: { detail?: string } | undefined; private digestStatus: 'same' | 'changed' | 'unknown' | undefined; private digestCheckedAt = 0; @@ -897,12 +983,14 @@ class LocalModelServiceImpl implements LocalModelService { listRunning, getRuntimeVersion, deleteModel, + removePartialModelBlobs, checkDeveloperTools: () => checkDeveloperTools(), stockInstallCommand, tryBeginCliInstall, runInstallCommand, recheckCliInstalled: recheckCliInstalledNow, audit: (entry) => appendAdminAudit({ ...entry, detail: entry.detail as Record | undefined }), + inspectLocalEndpoint, connectLocalModel, moveLocalEndpoint, ensureLocalEndpoint, @@ -930,7 +1018,7 @@ class LocalModelServiceImpl implements LocalModelService { return run; } - /** Applies one event to the stored state. False when the reducer refused it. */ + /** Applies one event of the running job to the stored state. False when the reducer refused it. */ private async dispatch(event: InstallJobEvent): Promise { const { after } = await this.deps.mutateState((current) => { const next = reduceInstallJob(current, event); @@ -939,6 +1027,76 @@ class LocalModelServiceImpl implements LocalModelService { return after !== undefined; } + /** + * `state` as it stands once a job that is over is counted as over: `installing` in the file + * while this process runs no job is a job that ended without the file knowing. It is + * `failed` with the outcome kept in memory, or `paused` when there is none (a restart, a + * Cancel whose write was refused). A paused or failed job with an outcome in memory is a + * Resume that could not be written. Pure apart from reading `this.job` and `this.unrecorded`. + */ + private settled(state: LocalModelStateFile): LocalModelStateFile { + if (this.job) return state; + const failure = this.unrecorded?.failure; + let next: LocalModelStateFile; + if (state.state === 'installing') { + next = failure + ? reduceInstallJob(state, { type: 'fail', ...failure }) + : reduceInstallJob(state, { type: 'restart' }); + } else if (failure && (state.state === 'paused' || state.state === 'failed')) { + // A Resume that could not even be written (the disk is full): the job did not start, + // and it says why instead of looking as if the click did nothing. + next = reduceInstallJob({ ...state, state: 'installing' }, { type: 'fail', ...failure }); + } else { + return state; + } + const progress = this.unrecorded?.progress; + return progress ? { ...next, progress } : next; + } + + /** The stored state as a view or a verb must see it: settled, unless a verb is just starting a job. */ + private async currentState(): Promise { + const state = await this.deps.readState(); + return this.jobStarting ? state : this.settled(state); + } + + /** + * Applies one event of a VERB to the stored state, after settling it (`settled()`), in one + * write. False when the reducer refused the event; the settlement is written all the same. + */ + private async verbDispatch(event: InstallJobEvent): Promise { + let applied = false; + let settledNow = false; + await this.deps.mutateState((current) => { + const base = this.settled(current); + settledNow = base !== current; + const next = reduceInstallJob(base, event); + applied = next !== base; + if (applied) return next; + return settledNow ? base : undefined; + }); + if (settledNow || applied) this.unrecorded = undefined; + return applied; + } + + /** + * Writes the settlement alone, so the file says what the view already shows. Best effort: a + * write that is refused (the disk is still full) changes nothing, the outcome stays in + * memory and the next verb tries again. + */ + private async settle(): Promise { + try { + let settledNow = false; + await this.deps.mutateState((current) => { + const base = this.settled(current); + settledNow = base !== current; + return settledNow ? base : undefined; + }); + if (settledNow) this.unrecorded = undefined; + } catch (err) { + console.warn(`[local-model] Could not write the state of the install: ${getErrorMessage(err)}`); + } + } + /** Writes facts a step learned. Never moves the state, and writes nothing once the job's state is gone. */ private async record(change: (current: LocalModelStateFile) => LocalModelStateFile | undefined): Promise { await this.deps.mutateState((current) => (current.state === 'idle' ? undefined : change(current))); @@ -946,16 +1104,12 @@ class LocalModelServiceImpl implements LocalModelService { /** * A job the state file still calls `installing` when this process has none is one a restart - * interrupted: it becomes `paused`. Done once, before the first view or verb, whether or not - * `boot()` ran yet. + * interrupted: it becomes `paused`. Written once per process, before the first view or verb, + * whether or not `boot()` ran yet. This is the one write a GET makes; a job that ends later + * without the file knowing is shown from memory (`settled()`) and written by the next verb. */ private recovered(): Promise { - this.recovery ??= this.dispatch({ type: 'restart' }).then( - () => undefined, - (err: unknown) => { - console.warn(`[local-model] Could not read the state at start: ${getErrorMessage(err)}`); - } - ); + this.recovery ??= this.settle(); return this.recovery; } @@ -967,6 +1121,28 @@ class LocalModelServiceImpl implements LocalModelService { } } + /** + * The runtime version to start: the pinned one when it is unpacked, else the version the + * state file records when THAT is still unpacked (a model installed by an earlier build + * with another pin keeps running), else the pinned one, whose start then says what is + * missing. + */ + private async runtimeVersionToRun(state: LocalModelStateFile): Promise { + const pinned = this.deps.pin.version; + const stored = state.runtime?.version; + if (!stored || stored === pinned || (await this.isRuntimeOnDisk())) return pinned; + try { + const paths = this.paths(); + const binary = runtimeBinaryForVersion(paths, stored); + if (await this.deps.isRuntimeInstalled({ ...this.deps.pin, version: stored }, { ...paths, binary })) { + return stored; + } + } catch { + /* not a version this build can name: the pinned one is tried */ + } + return pinned; + } + private errorVars(code: LocalModelErrorCode, state: LocalModelStateFile): Record | undefined { const model = state.model?.label ?? ''; switch (code) { @@ -987,15 +1163,38 @@ class LocalModelServiceImpl implements LocalModelService { } } - /** The best model the current recommendation offers that is not `failedId`. */ - private async nextModelId(failedId: string, options?: RecommendOptions): Promise { + /** + * The model to offer after `failed` did not work: the best one the current recommendation + * offers that is another model. With `smallerThan` (the memory the failed model needs, for + * `does-not-fit`) only a model that needs LESS is offered: one that needs more would fail + * the same way. `undefined` when there is none. + */ + private async nextModelId( + failed: { id: string; context: number }, + options?: RecommendOptions, + rule: { needsLessMemory?: boolean } = {} + ): Promise { try { const hw = await this.deps.readHardware(); const catalog = this.deps.getCatalog(); const facts = await this.deps.buildHarnessFacts(catalog); const rec = this.deps.recommend(hw, catalog, facts, options ?? (await this.recommendOptions())); - const rows = [rec.pick, ...rec.alternatives].filter((row): row is Candidate => row !== undefined); - return rows.find((row) => row.modelId !== failedId)?.modelId; + const rows = [rec.pick, ...rec.alternatives].filter( + (row): row is Candidate => row !== undefined && row.modelId !== failed.id + ); + if (!rule.needsLessMemory) return rows[0]?.modelId; + // What the failed model needs: its row when the recommendation still has it, else the + // catalogue's figures for it at the context that was chosen. + const failedRow = [rec.pick, ...rec.alternatives].find((row) => row?.modelId === failed.id); + const entry = catalog.models.find((model) => model.id === failed.id); + const limit = + failedRow && failedRow.context === failed.context + ? failedRow.needBytes + : entry + ? needBytes(entry, failed.context) + : undefined; + if (limit === undefined || !Number.isFinite(limit)) return undefined; + return rows.find((row) => row.needBytes < limit)?.modelId; } catch { return undefined; } @@ -1038,6 +1237,12 @@ class LocalModelServiceImpl implements LocalModelService { }); } + /** The download as it stands, for a job that ends: what a paused or failed job keeps. */ + private progressNow(): { completedBytes: number; totalBytes: number } | undefined { + const progress = this.liveProgress; + return progress ? { completedBytes: progress.completedBytes, totalBytes: progress.totalBytes } : undefined; + } + private clearProgress(): void { this.liveProgress = undefined; this.speedSamples = []; @@ -1052,17 +1257,24 @@ class LocalModelServiceImpl implements LocalModelService { return { ok: false, code: 'runtime-start-failed', detail: 'No port was chosen for the runtime yet.' }; } const { port } = state.runtime; + const version = await this.runtimeVersionToRun(state); let result: RuntimeStartResult; try { result = await this.deps.supervisor.start({ port, contextLength: state.model.context, ...(state.runtime.pid !== undefined ? { previousPid: state.runtime.pid } : {}), + ...(version !== this.deps.pin.version ? { version } : {}), }); } catch (err) { result = { ok: false, code: 'runtime-start-failed', detail: getErrorMessage(err) }; } this.portInUse = !result.ok && result.code === 'port-in-use'; + // Kept, so an installed model whose runtime does not start says so instead of only "stopped". + this.startFailed = + !result.ok && result.code === 'runtime-start-failed' + ? { ...(result.detail ? { detail: result.detail } : {}) } + : undefined; if (result.ok) { const { pid, measuredTotalBytes } = result; await this.record((current) => @@ -1071,7 +1283,7 @@ class LocalModelServiceImpl implements LocalModelService { ...current, runtime: { ...current.runtime, - version: this.deps.pin.version, + version, pid, ...(measuredTotalBytes !== undefined ? { measuredTotalBytes } : {}), }, @@ -1111,7 +1323,9 @@ class LocalModelServiceImpl implements LocalModelService { } onRuntimeChange(status: RuntimeStatus): void { - if (!status.running || status.pid === undefined) return; + // The pid is stored from the moment the child is spawned, not only once it answers: a + // server that is killed in between must still find its own runtime at the next start. + if (status.pid === undefined) return; const { pid, measuredTotalBytes } = status; void this.record((current) => { if (!current.runtime) return undefined; @@ -1149,6 +1363,7 @@ class LocalModelServiceImpl implements LocalModelService { running: status.running, gaveUp: status.gaveUp, portInUse: this.portInUse, + ...(this.startFailed ? { startFailed: this.startFailed } : {}), ...(state.state === 'installed' && this.digestStatus ? { digest: this.digestStatus } : {}), developerTools, runtimeInstalled: await this.isRuntimeOnDisk(), @@ -1162,7 +1377,7 @@ class LocalModelServiceImpl implements LocalModelService { async view(): Promise { try { await this.recovered(); - let state = await this.deps.readState(); + let state = await this.currentState(); if (state.state === 'installing' && this.liveProgress) { state = { ...state, @@ -1214,13 +1429,14 @@ class LocalModelServiceImpl implements LocalModelService { if (!isLocalModelEndpointId(endpointId)) return undefined; try { await this.recovered(); - const state = await this.deps.readState(); + const state = await this.currentState(); // A record with the reserved id that this feature did not write is not ours to judge. if (state.connection?.endpointId !== endpointId) return undefined; const status = this.deps.supervisor.status(); if (state.state !== 'installed' || !state.runtime || !state.model || !status.running) { if (this.portInUse) return buildInstallError('port-in-use', { vars: this.errorVars('port-in-use', state) }); if (status.gaveUp) return buildInstallError('runtime-stopped'); + if (this.startFailed) return buildInstallError('runtime-start-failed', { next: 'start' }); return buildInstallError('runtime-not-running'); } const digest = await this.refreshDigest(state); @@ -1253,7 +1469,10 @@ class LocalModelServiceImpl implements LocalModelService { try { const state = await this.deps.readState(); if (state.connection?.endpointId === endpointId && state.model && state.model.id !== modelId) { - return `Only ${state.model.label} is installed on this Mac. Remove it and install another model first.`; + return formatInstallMessage({ + text: LOCAL_MODEL_INSTALL_LABELS.refusedOtherModel, + vars: { model: state.model.label }, + }); } } catch { /* an unreadable state is "nothing of ours": nothing to refuse */ @@ -1269,8 +1488,8 @@ class LocalModelServiceImpl implements LocalModelService { install(request: LocalModelInstallRequest, actor: LocalModelActor): Promise { return this.exclusive(async () => { - await this.recovered(); - const state = await this.deps.readState(); + await this.settle(); + const state = await this.currentState(); if (state.state === 'installing') return conflict('job-running'); if (state.state === 'installed') return conflict('already-installed'); @@ -1284,7 +1503,7 @@ class LocalModelServiceImpl implements LocalModelService { const rows = [rec.pick, ...rec.alternatives].filter((row): row is Candidate => row !== undefined); const candidate = rows.find((row) => row.modelId === request.modelId); if (!candidate || candidate.context !== request.context) { - return { ok: false, status: 400, message: 'This model at this context is not offered for this Mac right now.' }; + return { ok: false, status: 400, message: { text: LOCAL_MODEL_INSTALL_LABELS.refusedModelNotOffered } }; } // A row without the list (an older shape) still names its one coding agent in the flat fields. const harnesses: CandidateHarness[] = candidate.harnesses ?? [ @@ -1292,13 +1511,13 @@ class LocalModelServiceImpl implements LocalModelService { ]; const harness = harnesses.find((row) => row.cli === request.cli); if (!harness || !catalog.harnesses.some((row) => row.cli === request.cli)) { - return { ok: false, status: 400, message: 'This coding agent is not offered for this model.' }; + return { ok: false, status: 400, message: { text: LOCAL_MODEL_INSTALL_LABELS.refusedAgentNotOffered } }; } if (!harness.installed && !this.deps.stockInstallCommand(harness.cli)) { return { ok: false, status: 400, - message: `${harness.cliLabel} is not installed and cannot be installed from here.`, + message: { text: LOCAL_MODEL_INSTALL_LABELS.refusedAgentNotInstallable, vars: { cli: harness.cliLabel } }, }; } const expectedBytes = candidate.installBytes ?? candidate.downloadBytes + (options.runtimeDownloadBytes ?? 0); @@ -1311,19 +1530,17 @@ class LocalModelServiceImpl implements LocalModelService { return { ok: false, status: 400, - message: formatInstallMessage( - buildInstallError('endpoint-id-taken', { vars: { id: LOCAL_MODEL_ENDPOINT_ID } }).message - ), + message: buildInstallError('endpoint-id-taken', { vars: { id: LOCAL_MODEL_ENDPOINT_ID } }).message, }; } - const started = await this.dispatch({ + const refused = await this.beginJob(actor, { type: 'install', model: { id: candidate.modelId, label: candidate.modelLabel, context: candidate.context }, cli: { id: harness.cli, label: harness.cliLabel }, steps: jobSteps({ agentInstalled: harness.installed }), }); - if (!started) return conflict('wrong-state'); + if (refused) return refused; void this.deps .audit({ admin: actor.username, @@ -1333,71 +1550,149 @@ class LocalModelServiceImpl implements LocalModelService { detail: { cli: harness.cli, context: candidate.context, bytes: expectedBytes }, }) .catch(() => {}); - this.startJob(actor); return this.answer(); }); } + /** + * Writes the event that starts a job (`install` or `resume`) and starts the job. `undefined`: + * started. A state file that cannot be written for lack of space is not a 500: the job did + * not start, and the view says `disk-full` with its "Try again" (a first install, which has + * no job to show it on, is answered with that sentence). + */ + private async beginJob( + actor: LocalModelActor, + event: Extract + ): Promise { + this.jobStarting = true; + try { + const started = await this.verbDispatch(event); + if (!started) return conflict('wrong-state'); + this.startJob(actor); + return undefined; + } catch (err) { + if (!isDiskFullError(err)) throw err; + // A first install has no job to show the failure on: the sentence goes back as it is. + if (event.type === 'install' && (await this.currentState().catch(() => undefined))?.state === 'idle') { + return { ok: false, status: 507, message: { text: LOCAL_MODEL_INSTALL_LABELS.errDiskFull } }; + } + const kept = this.unrecorded?.progress; + this.unrecorded = { + failure: { code: 'disk-full', detail: getErrorMessage(err) }, + ...(kept ? { progress: kept } : {}), + }; + return this.answer(); + } finally { + this.jobStarting = false; + } + } + cancel(): Promise { return this.exclusive(async () => { - await this.recovered(); - const state = await this.deps.readState(); - if (state.state !== 'installing') return conflict('wrong-state'); + // Taken before anything is awaited: a Cancel that arrives while a job runs is answered + // with the view after it, also when the job finishes in the same moment. const job = this.job; - if (job) { - job.controller.abort(); - await job.promise.catch(() => {}); + if (!job) { + // No job in this process: a job a restart interrupted is paused, never running. + await this.settle(); + return conflict('wrong-state'); } + job.controller.abort(); + await job.promise.catch(() => {}); // The step may have finished the job while it was being interrupted (the connect step // completes its two writes). Then there is nothing left to pause, and the view says so. - await this.flushProgress().catch(() => {}); - await this.dispatch({ type: 'cancel' }); + // Otherwise the job is paused with its download as it stood, in ONE write. + const progress = this.progressNow(); + try { + await this.deps.mutateState((current) => { + if (current.state !== 'installing') return undefined; + const kept = progress ? reduceInstallJob(current, { type: 'progress', ...progress }) : current; + return reduceInstallJob(kept, { type: 'cancel' }); + }); + } catch (err) { + // The job IS stopped. The file could not be told: the view shows the pause from memory. + console.warn(`[local-model] Could not write the paused install: ${getErrorMessage(err)}`); + this.unrecorded = { ...(progress ? { progress } : {}) }; + } this.clearProgress(); return this.answer(); }); } - resume(request: LocalModelResumeRequest, actor: LocalModelActor): Promise { + resume( + request: LocalModelResumeRequest, + actor: LocalModelActor, + liveSessionNames: readonly string[] = [] + ): Promise { return this.exclusive(async () => { - await this.recovered(); - const state = await this.deps.readState(); + await this.settle(); + const state = await this.currentState(); if (state.state === 'installing') return conflict('job-running'); if (state.state !== 'paused' && state.state !== 'failed') return conflict('wrong-state'); - if (request.newPort === true && state.runtime) await this.changePort(state); - const resumed = await this.dispatch({ type: 'resume' }); - if (!resumed) return conflict('wrong-state'); - this.startJob(actor); - return this.answer(); + if (request.newPort === true && state.runtime) { + const open = this.sessionsOpen(state, liveSessionNames); + if (open) return open; + await this.changePort(state); + } + return (await this.beginJob(actor, { type: 'resume' })) ?? this.answer(); }); } - /** Chooses another port than the stored one and moves the saved endpoint with it. False when no other port is free. */ + /** 409 `sessions-open` while live sessions use a connection of ours, else `undefined`. */ + private sessionsOpen( + state: LocalModelStateFile, + liveSessionNames: readonly string[] + ): LocalModelVerbResult | undefined { + if (!state.connection || liveSessionNames.length === 0) return undefined; + const sessions = [...liveSessionNames]; + return conflict('sessions-open', { sessions }, { sessions: sessions.join(', ') }); + } + + /** + * Chooses another port than the stored one and moves the saved endpoint with it. False when + * no other port is free. The state file first, then the saved endpoint: a state write that + * fails changes nothing, and a refused endpoint write puts the old port back, so the two + * never disagree. The stored pid stays: when it is still our own runtime on the OLD port, + * the next start ends it instead of leaving a second runtime behind. + */ private async changePort(state: LocalModelStateFile): Promise { if (!state.runtime) return false; - const port = await this.deps.chooseRuntimePort({ avoid: [state.runtime.port] }); + const previous = state.runtime.port; + const port = await this.deps.chooseRuntimePort({ avoid: [previous] }); if (port === undefined) return false; - // The saved endpoint first: when that write is refused, nothing has changed yet. - if (state.connection) await this.deps.moveLocalEndpoint(port); - await this.record((current) => { - if (!current.runtime) return undefined; - // The stored pid belonged to the old port: it must not make a listener on the new one "ours". - const { pid: _pid, ...runtime } = current.runtime; - return { ...current, runtime: { ...runtime, port } }; - }); + const setPort = (value: number): Promise => + this.record((current) => + current.runtime ? { ...current, runtime: { ...current.runtime, port: value } } : undefined + ); + await setPort(port); + if (state.connection) { + try { + await this.deps.moveLocalEndpoint(port); + } catch (err) { + await setPort(previous).catch(() => {}); + throw err; + } + } this.portInUse = false; return true; } - runtime(request: LocalModelRuntimeRequest): Promise { + runtime(request: LocalModelRuntimeRequest, liveSessionNames: readonly string[] = []): Promise { return this.exclusive(async () => { - await this.recovered(); - let state = await this.deps.readState(); + await this.settle(); + let state = await this.currentState(); if (state.state === 'installing') return conflict('job-running'); if (state.state !== 'installed' || !state.runtime) return conflict('wrong-state'); if (request.action === 'stop') { - await this.deps.supervisor.stop(); + // Their coding agents keep calling the address they were started with. + const open = this.sessionsOpen(state, liveSessionNames); + if (open) return open; + await this.deps.supervisor.stop({ + ...(state.runtime.pid !== undefined ? { previousPid: state.runtime.pid } : {}), + }); this.portInUse = false; + this.startFailed = undefined; await this.record((current) => current.runtime ? { ...current, runtime: { ...current.runtime, enabled: false } } : undefined ); @@ -1405,8 +1700,12 @@ class LocalModelServiceImpl implements LocalModelService { } if (request.newPort === true) { + const open = this.sessionsOpen(state, liveSessionNames); + if (open) return open; // The old child, when one is still alive, holds the old port: it must go first. - await this.deps.supervisor.stop(); + await this.deps.supervisor.stop({ + ...(state.runtime.pid !== undefined ? { previousPid: state.runtime.pid } : {}), + }); await this.changePort(state); } await this.record((current) => @@ -1425,15 +1724,17 @@ class LocalModelServiceImpl implements LocalModelService { actor?: LocalModelActor ): Promise { return this.exclusive(async () => { - await this.recovered(); - const state = await this.deps.readState(); + await this.settle(); + const state = await this.currentState(); if (state.state === 'installing') return conflict('job-running'); - if (state.connection && liveSessionNames.length > 0) { - const sessions = [...liveSessionNames]; - return conflict('sessions-open', { sessions }, { sessions: sessions.join(', ') }); - } + const open = this.sessionsOpen(state, liveSessionNames); + if (open) return open; - await this.deps.supervisor.stop(); + // Our child, and a runtime an earlier server left behind (known by the stored pid): no + // file is deleted under a process that still runs from it. + await this.deps.supervisor.stop({ + ...(state.runtime?.pid !== undefined ? { previousPid: state.runtime.pid } : {}), + }); // The two writes outside our folder are undone first: when one is refused, nothing was // deleted yet and the person can try again. if (state.connection) { @@ -1441,9 +1742,11 @@ class LocalModelServiceImpl implements LocalModelService { } const removeModel = request.removeModel === true; await this.deps.removeLocalModelFiles({ models: removeModel, paths: this.paths() }); - await this.dispatch({ type: 'remove' }); + await this.verbDispatch({ type: 'remove' }); this.clearProgress(); + this.unrecorded = undefined; this.portInUse = false; + this.startFailed = undefined; this.digestStatus = undefined; if (actor) { void this.deps @@ -1463,11 +1766,11 @@ class LocalModelServiceImpl implements LocalModelService { async boot(): Promise { try { await this.recovered(); - const state = await this.deps.readState(); - if (state.state !== 'installed' || !state.runtime || !state.model) return; + const state = await this.currentState(); + const installed = state.state === 'installed' && state.runtime !== undefined && state.model !== undefined; // The record of the connection can be gone while the model is still installed (a // config-backup restore of an older endpoint store): it is put back, never replaced. - if (state.connection) { + if (installed && state.connection && state.runtime && state.model) { const restored = await this.deps.ensureLocalEndpoint({ port: state.runtime.port, modelId: state.model.id, @@ -1476,7 +1779,15 @@ class LocalModelServiceImpl implements LocalModelService { if (restored) console.warn('[local-model] The saved endpoint of the model on this Mac was missing and was put back.'); } - if (!state.runtime.enabled) return; + if (!installed || !state.runtime?.enabled) { + // Nothing starts the runtime here (a job waits for a click, the person stopped the + // model). One an earlier server left behind is ended: it holds the model's memory, + // and a later Remove must not delete its folder under it. + if (state.runtime?.pid !== undefined) { + await this.deps.supervisor.stop({ previousPid: state.runtime.pid }); + } + return; + } const started = await this.startRuntime(state); if (!started.ok) { console.warn( @@ -1519,6 +1830,7 @@ class LocalModelServiceImpl implements LocalModelService { private startJob(actor: LocalModelActor): void { const controller = new AbortController(); this.clearProgress(); + this.unrecorded = undefined; const job = { controller, promise: this.runJob(actor, controller.signal).finally(() => { @@ -1538,7 +1850,12 @@ class LocalModelServiceImpl implements LocalModelService { try { outcome = await this.runStep(state.step, state, actor, signal); } catch (err) { - outcome = { kind: 'fail', code: 'internal', detail: getErrorMessage(err) }; + // A write of the step itself that found no space is a full disk, not a mystery. + outcome = { + kind: 'fail', + code: isDiskFullError(err) ? 'disk-full' : 'internal', + detail: getErrorMessage(err), + }; } if (outcome.kind === 'done') { // A step that finished stays finished, also when Cancel arrived while it ran. Its @@ -1550,19 +1867,38 @@ class LocalModelServiceImpl implements LocalModelService { // An interrupted step is not a failed one, whatever its last call answered. Cancel // writes `paused` itself; a server that is stopping writes nothing. if (outcome.kind === 'cancelled' || signal.aborted || this.stopping) return; - await this.flushProgress().catch(() => {}); - await this.dispatch({ - type: 'fail', + const failure: JobFailure = { code: outcome.code, ...(outcome.vars ? { vars: outcome.vars } : {}), ...(outcome.detail ? { detail: outcome.detail } : {}), ...(outcome.nextModelId ? { nextModelId: outcome.nextModelId } : {}), - }); + }; + const progress = this.progressNow(); + try { + // The failure and the download as it stood, in ONE write. + await this.deps.mutateState((current) => { + const kept = progress ? reduceInstallJob(current, { type: 'progress', ...progress }) : current; + const next = reduceInstallJob(kept, { type: 'fail', ...failure }); + return next === current ? undefined : next; + }); + } catch (err) { + // The disk that failed the step also refuses the state file. The outcome is not + // lost: the view shows it from memory and the next verb writes it. + console.warn(`[local-model] Could not write why the install stopped: ${getErrorMessage(err)}`); + this.unrecorded = { failure, ...(progress ? { progress } : {}) }; + } + this.clearProgress(); return; } } catch (err) { + // A write of the job itself was refused (the step that finished could not be recorded). console.warn(`[local-model] The install job stopped: ${getErrorMessage(err)}`); - await this.dispatch({ type: 'fail', code: 'internal', detail: getErrorMessage(err) }).catch(() => {}); + const progress = this.progressNow(); + this.unrecorded = { + failure: { code: isDiskFullError(err) ? 'disk-full' : 'internal', detail: getErrorMessage(err) }, + ...(progress ? { progress } : {}), + }; + this.clearProgress(); } } @@ -1629,7 +1965,7 @@ class LocalModelServiceImpl implements LocalModelService { const catalog = this.deps.getCatalog(); const options: RecommendOptions = { measuredTotalBytes, runtimeDownloadBytes: 0 }; if (!this.deps.modelFitsBudget(hw, catalog, state.model.id, state.model.context, options)) { - const nextModelId = await this.nextModelId(state.model.id, options); + const nextModelId = await this.nextModelId(state.model, options, { needsLessMemory: true }); return { kind: 'fail', code: 'does-not-fit', @@ -1657,6 +1993,15 @@ class LocalModelServiceImpl implements LocalModelService { /* a leftover is not a reason to stop: Remove with the models deletes it */ } if (signal.aborted) return CANCELLED; + // A download that was interrupted left partial blobs and no model: the runtime answers + // "not found" for it and deletes nothing. No download is in flight here, so every + // partial file in the store is a dead one. + try { + const freed = await this.deps.removePartialModelBlobs(this.paths()); + if (freed > 0) console.log(`[local-model] Deleted ${formatGb(freed)} GB a replaced download had left.`); + } catch { + /* as above */ + } await this.record((current) => { const { replacedModelId: _replaced, ...rest } = current; return rest; @@ -1669,7 +2014,16 @@ class LocalModelServiceImpl implements LocalModelService { }); if (!pulled.ok) { if (pulled.code === 'cancelled') return CANCELLED; - const nextModelId = pulled.code === 'model-not-found' ? await this.nextModelId(state.model.id) : undefined; + // A runtime that is gone for good looks like a broken network to the pull. It is not. + const status = this.deps.supervisor.status(); + if (pulled.code === 'pull-failed' && (status.gaveUp || !status.running)) { + return { + kind: 'fail', + code: 'runtime-stopped', + ...(pulled.detail ? { detail: pulled.detail } : {}), + }; + } + const nextModelId = pulled.code === 'model-not-found' ? await this.nextModelId(state.model) : undefined; return { kind: 'fail', code: pulled.code, @@ -1711,8 +2065,12 @@ class LocalModelServiceImpl implements LocalModelService { } vars = { model: state.model.label, actual, wanted: String(context) }; } - const offersNext = verified.code === 'does-not-fit' || verified.code === 'no-tool-call'; - const nextModelId = offersNext ? await this.nextModelId(id) : undefined; + const nextModelId = + verified.code === 'does-not-fit' + ? await this.nextModelId(state.model, undefined, { needsLessMemory: true }) + : verified.code === 'no-tool-call' + ? await this.nextModelId(state.model) + : undefined; return { kind: 'fail', code: verified.code, @@ -1815,13 +2173,41 @@ class LocalModelServiceImpl implements LocalModelService { return { kind: 'fail', code: 'connect-failed', detail: `The runtime does not list ${id}.` }; } + // The connection is CLAIMED in the state file before either write, so no record of ours + // can exist that the state file does not know: a state write that fails, or a server that + // is killed between the two, leaves at worst a claim with nothing behind it. + let claim = running.connection; + if (!claim) { + let found: LocalEndpointInspection; + try { + found = await this.deps.inspectLocalEndpoint({ port, modelId: id, context }); + } catch (err) { + return { kind: 'fail', code: 'connect-failed', detail: getErrorMessage(err) }; + } + if (signal.aborted) return CANCELLED; + // A record with our id that says something else is somebody else's: never overwritten. + if (found.record === 'other') { + return { kind: 'fail', code: 'endpoint-id-taken', vars: this.errorVars('endpoint-id-taken', state) }; + } + // A record that already says what ours would say can only be ours (the routes refuse + // the id): the flag was then switched on by us, and its earlier value is not known, so + // it is never switched off again. + claim = { + endpointId: LOCAL_MODEL_ENDPOINT_ID, + flagWasEnabled: found.record === 'same' ? true : found.flagEnabled, + }; + const claimed = claim; + await this.record((current) => ({ ...current, connection: claimed })); + } + if (signal.aborted) return CANCELLED; + // From here on the two writes run to their end: Cancel no longer interrupts. const connected = await this.deps.connectLocalModel({ port, modelId: id, context, - ours: running.connection !== undefined, - ...(running.connection ? { flagWasEnabled: running.connection.flagWasEnabled } : {}), + ours: true, + flagWasEnabled: claim.flagWasEnabled, }); if (!connected.ok) { return { @@ -1831,11 +2217,6 @@ class LocalModelServiceImpl implements LocalModelService { ...(connected.detail ? { detail: connected.detail } : {}), }; } - const flagWasEnabled = connected.flagWasEnabled; - await this.record((current) => ({ - ...current, - connection: { endpointId: LOCAL_MODEL_ENDPOINT_ID, flagWasEnabled }, - })); return DONE; } } diff --git a/src/local-model/ollama-client.ts b/src/local-model/ollama-client.ts index a597c138..90fe3bb7 100644 --- a/src/local-model/ollama-client.ts +++ b/src/local-model/ollama-client.ts @@ -31,6 +31,7 @@ */ import type { LocalModelErrorCode } from '../types.js'; +import { formatGb } from './format.js'; /** How often a pull that ended on an error line or a cut stream is started again. */ export const PULL_RETRIES = 5; @@ -683,7 +684,8 @@ export function judgeLoaded( return { ok: false, code: 'does-not-fit', - detail: `${model.sizeVram} of ${model.size} bytes are on the GPU`, + // A size a person reads: decimal GB, like every other size of this feature. + detail: `${formatGb(model.sizeVram)} of ${formatGb(model.size)} GB of the model are on the GPU`, }; } if (model.contextLength !== context) { diff --git a/src/types/local-model-install.ts b/src/types/local-model-install.ts index 3bd29022..2fdd8f2b 100644 --- a/src/types/local-model-install.ts +++ b/src/types/local-model-install.ts @@ -27,14 +27,19 @@ * | | | `already-installed`, `size-changed` | * | `POST /api/local-model/cancel` | none | 409 `wrong-state` (no job is running) | * | `POST /api/local-model/resume` | `LocalModelResumeRequest` | 409 `job-running`, `wrong-state` (not paused or | - * | | | failed) | - * | `POST /api/local-model/runtime` | `LocalModelRuntimeRequest` | 409 `wrong-state` (not installed), `job-running` | + * | | | failed), `sessions-open` (another port while live | + * | | | sessions use the model) | + * | `POST /api/local-model/runtime` | `LocalModelRuntimeRequest` | 409 `wrong-state` (not installed), `job-running`, | + * | | | `sessions-open` (stop, or another port, while live | + * | | | sessions use the model) | * | `DELETE /api/local-model` | `LocalModelRemoveRequest` | 409 `job-running`, `sessions-open` | * * A 409 is the error envelope plus `data: LocalModelConflict`: * `{ success: false, error, errorCode: 'CONFLICT', data: { reason, message, ... } }`. A 409 * WITHOUT `data` is a write of `settings.json` or of the endpoint store that was refused - * because the file does not parse; `error` names the fix. + * because the file does not parse; `error` names the fix. A 400 the service names carries its + * sentence the same way: `{ success: false, error, errorCode: 'INVALID_INPUT', data: { + * message } }` (`LocalModelRefusal`), so the window can print it in its own language. * * The rules of src/types/local-model.ts hold here too: * 1. A size a person reads travels twice: as bytes and as decimal GB text from `formatGb()` @@ -118,8 +123,10 @@ export const LOCAL_MODEL_INSTALL_STEPS: readonly LocalModelInstallStep[] = [ * | `runtime-download-failed` | the archive download still fails after its retries | * | `runtime-damaged` | the archive's sha256 is not the pinned one (the file was deleted) | * | `runtime-unpack-failed` | `/usr/bin/tar` failed | - * | `runtime-start-failed` | the runtime was started and never answered `/api/version` | - * | `runtime-stopped` | installed: the runtime crashed and was restarted 3 times, then given up | + * | `runtime-start-failed` | the runtime was started and never answered `/api/version`. Installed: | + * | | the last start failed (the detail says why), and Start tries again | + * | `runtime-stopped` | the runtime crashed and was restarted 3 times, then given up: on an | + * | | installed model, or during a job's download | * | `runtime-not-running` | installed: the runtime is not running (the person stopped it). Only a | * | | refused session start says it; the view shows no error for a stopped runtime | * | `port-in-use` | another program listens on the stored port, or no port of the span is free | @@ -170,6 +177,11 @@ export type LocalModelErrorCode = * Without a `nextModel` the window offers the chooser and Remove. * - `start`: "Start". `POST /api/local-model/runtime` with `action: 'start'`. * - `remove`: "Remove". `DELETE /api/local-model`. + * + * `LOCAL_MODEL_ERROR_NEXT` is the default of a code. The view adapts it to the state the + * error is shown in, so the button is always one the server accepts there: a code whose + * default is `start` says `retry` on a failed job (there is no Start verb for a job), and + * `runtime-start-failed` says `start` on an installed model (there is nothing to resume). */ export type LocalModelNextStep = 'retry' | 'other-port' | 'next-model' | 'start' | 'remove'; @@ -212,11 +224,18 @@ export type LocalModelDeveloperTools = * | `errContextNotApplied` | `model`; `actual`: the context the runtime reports; `wanted`: the chosen one | * | `errHarnessFailed` | `cli`: the coding agent's label | * | `errEndpointIdTaken` | `id`: the endpoint id this feature uses | + * | `refusedAgentNotInstallable` | `cli`: the coding agent's label | + * | `refusedOtherModel` | `model`: the label of the model that is installed | * | `conflictSizeChanged` | `size`: the server's own total as decimal GB | * | `conflictSessionsOpen` | `sessions`: the session names, joined with ', ' | * * `developerToolsMissing` is not sent: the view carries `developerTools` and the window prints * this sentence when it is `missing`. It is here so that its wording has one source. + * + * `endpointManaged` and `endpointIdReserved` are what the endpoint routes + * (src/web/routes/custom-model-routes.ts) answer for the record this feature manages and for + * its reserved id. They have no slot, so the App Settings toast that prints them translates + * them as it translates every other whole sentence. */ export const LOCAL_MODEL_INSTALL_LABELS = { errRuntimeDownloadFailed: @@ -246,6 +265,13 @@ export const LOCAL_MODEL_INSTALL_LABELS = { conflictSizeChanged: 'The download is {size} GB now. Check the sizes and confirm again.', conflictSessionsOpen: 'Sessions are still using this model: {sessions}. Close them first.', conflictWrongState: 'That is not possible in the current state.', + refusedModelNotOffered: 'This model at this context is not offered for this Mac right now.', + refusedAgentNotOffered: 'This coding agent is not offered for this model.', + refusedAgentNotInstallable: '{cli} is not installed and cannot be installed from here.', + refusedOtherModel: 'Only {model} is installed on this Mac. Remove it and install another model first.', + endpointManaged: + 'This endpoint belongs to the model Codeman runs on this Mac. Change or remove it in the local model window.', + endpointIdReserved: 'This endpoint id is reserved for the model Codeman runs on this Mac. Pick another id.', developerToolsMissing: 'Coding agents use git and Python. This Mac has no developer tools yet, so macOS will offer to install them the first time an agent needs one.', } as const; @@ -336,11 +362,12 @@ export interface LocalModelInstallError { code: LocalModelErrorCode; /** `LOCAL_MODEL_ERROR_LABELS[code]` with its slots filled. */ message: LocalModelInstallMessage; - /** `LOCAL_MODEL_ERROR_NEXT[code]`. */ + /** `LOCAL_MODEL_ERROR_NEXT[code]`, adapted to the state the error is shown in (see `LocalModelNextStep`). */ next: LocalModelNextStep; /** - * For `next-model`: the best model that still fits and is not the one that failed. Absent - * when there is none. + * For `next-model`: the model to try instead. After `does-not-fit` it is one that needs LESS + * memory than the one that failed; after the other codes the best other model that fits. + * Absent when there is none. */ nextModel?: { modelId: string; modelLabel: string }; /** @@ -484,6 +511,11 @@ export interface LocalModelRemoveRequest { removeModel?: boolean; } +/** The `data` of a 400 the service names: its sentence, for the window to print in its own language. */ +export interface LocalModelRefusal { + message: LocalModelInstallMessage; +} + /** What every verb answers on success: the view after the verb. */ export type LocalModelVerbAnswer = LocalModelInstallView; @@ -493,7 +525,9 @@ export type LocalModelVerbAnswer = LocalModelInstallView; * - `already-installed`: install while a model is installed. Remove it first. * - `size-changed`: `acknowledgedBytes` is not the server's total; `expectedBytes` and * `expectedGb` carry that total, so the window can show it and ask again. - * - `sessions-open`: remove while live sessions use the endpoint; `sessions` names them. + * - `sessions-open`: remove, stop or a change of port while live sessions use the endpoint; + * `sessions` names them. Their coding agents were given the runtime's address when they + * started, so they would go on calling a port nothing of ours holds. * - `wrong-state`: the verb makes no sense in the current state (see the route table above). */ export type LocalModelConflictReason = @@ -567,11 +601,17 @@ export interface LocalModelStateFile { cli?: { id: string; label: string }; /** The runtime, once its port was chosen. */ runtime?: { - /** The pinned version that was unpacked. */ + /** + * The runtime version this installation runs. A later build with another pin keeps + * starting this version for as long as its folder is unpacked. + */ version: string; /** The loopback port, chosen once by binding. */ port: number; - /** The pid of the child Codeman last started. Used at boot to tell our own orphan from another program. */ + /** + * The pid of the child Codeman last started, stored the moment it is spawned. Used to + * tell our own runtime from another program: at boot, and before the folder is deleted. + */ pid?: number; /** False after Stop: the runtime is not started at boot. */ enabled: boolean; @@ -582,7 +622,11 @@ export interface LocalModelStateFile { progress?: { completedBytes: number; totalBytes: number }; /** What the runtime reported when the job checked the loaded model. */ loaded?: { context: number; sizeBytes: number; sizeVramBytes: number }; - /** Present once the `connect` step wrote: what uninstall must undo. */ + /** + * Present once the `connect` step CLAIMED the connection, which it does before its two + * writes: what uninstall must undo. A claim with nothing behind it (the step was interrupted + * right after it) is harmless: a resumed job writes both, and uninstall removes nothing. + */ connection?: { /** The id of the endpoint record this feature wrote. */ endpointId: string; diff --git a/src/web/routes/custom-model-routes.ts b/src/web/routes/custom-model-routes.ts index 4516b516..d718f598 100644 --- a/src/web/routes/custom-model-routes.ts +++ b/src/web/routes/custom-model-routes.ts @@ -30,7 +30,7 @@ */ import type { FastifyInstance, FastifyRequest } from 'fastify'; -import { ApiErrorCode, createErrorResponse, type ApiResponse } from '../../types.js'; +import { ApiErrorCode, createErrorResponse, LOCAL_MODEL_INSTALL_LABELS, type ApiResponse } from '../../types.js'; import { isAdmin, parseBody, readJsonConfig, SETTINGS_PATH } from '../route-helpers.js'; import { isMultiUserMode } from '../../config/multiuser.js'; import { getDataDir } from '../../config/instance.js'; @@ -933,10 +933,7 @@ function refusedHostsWrite(err: unknown, reply: { code: (n: number) => unknown } /** The answer to an edit or a delete of the record the local model install manages. */ function managedEndpointRefusal(): ApiResponse { - return createErrorResponse( - ApiErrorCode.INVALID_INPUT, - 'This endpoint belongs to the model Codeman runs on this Mac. Change or remove it in the local model window.' - ); + return createErrorResponse(ApiErrorCode.INVALID_INPUT, LOCAL_MODEL_INSTALL_LABELS.endpointManaged); } export function registerCustomModelRoutes(app: FastifyInstance): void { @@ -956,10 +953,7 @@ export function registerCustomModelRoutes(app: FastifyInstance): void { const badDefault = invalidDefaultModel(host); if (badDefault) return badDefault; if (isLocalModelEndpointId(host.id)) { - return createErrorResponse( - ApiErrorCode.INVALID_INPUT, - `The endpoint id "${host.id}" is reserved for the model Codeman runs on this Mac. Pick another id.` - ); + return createErrorResponse(ApiErrorCode.INVALID_INPUT, LOCAL_MODEL_INSTALL_LABELS.endpointIdReserved); } try { return await mutateCustomModelHosts>((hosts) => { diff --git a/src/web/routes/local-model-routes.ts b/src/web/routes/local-model-routes.ts index 74c858ef..2cd8bc6a 100644 --- a/src/web/routes/local-model-routes.ts +++ b/src/web/routes/local-model-routes.ts @@ -40,11 +40,14 @@ * - A verb answers the view AFTER the verb. A refusal the service names is a 400 (the request * is not a row the server offers) or a 409 whose `data` is the `LocalModelConflict` * (`reason`, the sentence, and for `size-changed` the server's own total, for - * `sessions-open` the session names). A write of the settings or of the endpoint store - * that was refused (a file that does not parse) is a 409 without `data`, with the fix. + * `sessions-open` the session names). A 400 the service names carries its sentence as + * `data.message` too, so the window prints it in its own language. A write of the settings + * or of the endpoint store that was refused (a file that does not parse) is a 409 without + * `data`, with the fix. * - The verbs do the work in the install service; this file only gates, parses and maps. - * `DELETE` is the one that looks at the live sessions: it hands their names to the service, - * which refuses while one of them still uses the model. + * `DELETE`, `runtime` and `resume` look at the live sessions: they hand their names to the + * service, which refuses a removal, a stop and a change of port while one of them still + * uses the model. * - Every dependency is injected (`LocalModelRouteDeps`), so a route test spawns nothing and * does not need a Mac: it passes its own hardware, harness facts and service. */ @@ -61,6 +64,7 @@ import { type LocalModelCatalog, type LocalModelConflict, type LocalModelInstallView, + type LocalModelRefusal, type LocalModelStatus, type Recommendation, type RecommendOptions, @@ -152,8 +156,20 @@ export interface LocalModelConflictResponse { data: LocalModelConflict; } -/** What a verb answers: the view after it, an error envelope, or the 409 that carries the conflict. */ -export type LocalModelVerbResponse = ApiResponse | LocalModelConflictResponse; +/** The 400 (or 507) of a verb the service refused with a sentence: the error envelope plus that sentence as `data`. */ +export interface LocalModelRefusalResponse { + success: false; + /** The sentence in English, its slots filled. */ + error: string; + errorCode: ApiErrorCode; + data: LocalModelRefusal; +} + +/** What a verb answers: the view after it, an error envelope, or a refusal that carries its sentence. */ +export type LocalModelVerbResponse = + | ApiResponse + | LocalModelConflictResponse + | LocalModelRefusalResponse; /** 403 for a non-admin in multi-user mode, before anything else runs. `null`: go on. */ function adminOnly(req: FastifyRequest, reply: FastifyReply): ApiResponse | null { @@ -170,7 +186,14 @@ function actorOf(req: FastifyRequest): LocalModelActor { function respond(result: LocalModelVerbResult, reply: FastifyReply): LocalModelVerbResponse { if (result.ok) return { success: true, data: result.view }; reply.code(result.status); - if (result.status === 400) return createErrorResponse(ApiErrorCode.INVALID_INPUT, result.message); + if (result.status !== 409) { + return { + success: false, + error: formatInstallMessage(result.message), + errorCode: result.status === 400 ? ApiErrorCode.INVALID_INPUT : ApiErrorCode.OPERATION_FAILED, + data: { message: result.message }, + }; + } return { success: false, error: formatInstallMessage(result.conflict.message), @@ -258,14 +281,18 @@ export function registerLocalModelRoutes( const denied = adminOnly(req, reply); if (denied) return denied; const body = parseBody(LocalModelResumeSchema, req.body ?? {}); - return runVerb(reply, () => resolved.service().resume(body, actorOf(req))); + const sessionNames = sessionNamesUsingEndpoint(ctx.sessions.values()); + return runVerb(reply, () => resolved.service().resume(body, actorOf(req), sessionNames)); }); app.post('/api/local-model/runtime', async (req, reply): Promise => { const denied = adminOnly(req, reply); if (denied) return denied; const body = parseBody(LocalModelRuntimeSchema, req.body); - return runVerb(reply, () => resolved.service().runtime(body)); + // Asked at the moment of the request, like the DELETE: a stop or a change of port leaves + // a live session calling an address nothing of ours holds. + const sessionNames = sessionNamesUsingEndpoint(ctx.sessions.values()); + return runVerb(reply, () => resolved.service().runtime(body, sessionNames)); }); app.delete('/api/local-model', async (req, reply): Promise => { diff --git a/test/local-model-connect.test.ts b/test/local-model-connect.test.ts index c1dd2150..3630baf6 100644 --- a/test/local-model-connect.test.ts +++ b/test/local-model-connect.test.ts @@ -27,8 +27,10 @@ import { connectLocalModel, disconnectLocalModel, ensureLocalEndpoint, + inspectLocalEndpoint, isLocalModelEndpointId, isManagedLocalEndpoint, + isSameLocalEndpoint, LOCAL_MODEL_ENDPOINT_ID, LOCAL_MODEL_ENDPOINT_LABEL, moveLocalEndpoint, @@ -401,6 +403,76 @@ describe('compareModelDigest', () => { }); }); +describe('a record that says what ours would say is ours', () => { + it('isSameLocalEndpoint: the address, the one model, its context and no key', () => { + expect(isSameLocalEndpoint(RECORD, INPUT)).toBe(true); + // What discovery may have added does not make it somebody else's. + expect( + isSameLocalEndpoint( + { ...RECORD, modelContextLengths: { 'qwen3.6:35b-a3b': 131072 }, lastDiscoveredAt: '2026-11-06T10:00:00Z' }, + INPUT + ) + ).toBe(true); + const differs: Partial[] = [ + { baseUrl: 'http://127.0.0.1:11434' }, + { label: 'My own' }, + { models: ['qwen3.6:35b-a3b', 'gpt-oss:20b'] }, + { models: ['gpt-oss:20b'], defaultModelId: 'gpt-oss:20b' }, + { defaultContextLength: 65536 }, + { apiKey: 'sk-not-ours' }, + { authStyle: 'api-key' }, + ]; + for (const change of differs) expect(isSameLocalEndpoint({ ...RECORD, ...change }, INPUT)).toBe(false); + }); + + it('connectLocalModel takes such a record as ours although the caller did not say so', async () => { + writeSettings({}); + writeHosts([OTHER, RECORD]); + expect(await connectLocalModel({ ...INPUT, ours: false })).toEqual({ ok: true, flagWasEnabled: false }); + expect(await readCustomModelHosts(getDataDir())).toEqual([OTHER, RECORD]); + }); + + it('a record with our id that says something else is still somebody else: taken, and untouched', async () => { + const foreign = { ...RECORD, baseUrl: 'http://127.0.0.1:11434' }; + writeHosts([foreign]); + expect(await connectLocalModel({ ...INPUT, ours: false })).toEqual({ ok: false, code: 'endpoint-id-taken' }); + expect(await readCustomModelHosts(getDataDir())).toEqual([foreign]); + }); +}); + +describe('inspectLocalEndpoint: what the connect step finds before it claims the connection', () => { + it('nothing saved and no settings file: absent, and the flag is off', async () => { + expect(await inspectLocalEndpoint(INPUT)).toEqual({ record: 'absent', flagEnabled: false }); + }); + + it('reads the flag as it is now, and writes nothing', async () => { + writeSettings({ customModelEndpointsEnabled: true, tmuxHistoryLimit: 5000 }); + writeHosts([OTHER]); + const settingsBefore = readFileSync(SETTINGS_PATH, 'utf8'); + const hostsBefore = readFileSync(HOSTS(), 'utf8'); + expect(await inspectLocalEndpoint(INPUT)).toEqual({ record: 'absent', flagEnabled: true }); + expect(readFileSync(SETTINGS_PATH, 'utf8')).toBe(settingsBefore); + expect(readFileSync(HOSTS(), 'utf8')).toBe(hostsBefore); + }); + + it("tells our record from somebody else's under the same id", async () => { + writeHosts([RECORD]); + expect((await inspectLocalEndpoint(INPUT)).record).toBe('same'); + expect((await inspectLocalEndpoint({ ...INPUT, port: 11436 })).record).toBe('other'); + }); +}); + +describe('moveLocalEndpoint', () => { + it('a record that is already on the port is not written again', async () => { + writeHosts([RECORD]); + const before = readFileSync(HOSTS(), 'utf8'); + await moveLocalEndpoint(11435); + expect(readFileSync(HOSTS(), 'utf8')).toBe(before); + await moveLocalEndpoint(11440); + expect((await readCustomModelHosts(getDataDir()))[0].baseUrl).toBe('http://127.0.0.1:11440'); + }); +}); + describe('the runtime keeps a private key that is never served', () => { it('everything in the private home is on the sensitive list, and so is a runtime key under a real home', () => { const home = join(getDataDir(), 'local-model', 'home'); diff --git a/test/local-model-install-e2e.test.ts b/test/local-model-install-e2e.test.ts index 886c26ce..f87a5261 100644 --- a/test/local-model-install-e2e.test.ts +++ b/test/local-model-install-e2e.test.ts @@ -17,7 +17,8 @@ * * The walks: the whole install and its removal; a damaged runtime download; a model download * that fails twice and then succeeds; one that keeps failing; a model that calls no tool; - * Cancel during the model download; a server restart during the model download. + * Cancel during the model download; a server restart during the model download; a server + * that is KILLED during the model download and leaves its runtime running. * * "A server" here is one install service with its own runtime supervisor (`startServer()`). * A restart is `shutdown()` of the one and a new one over the same data dir. @@ -45,7 +46,11 @@ import { createLocalModelService, type LocalModelService } from '../src/local-mo import { memClassGb, formatGb } from '../src/local-model/format.js'; import { pullModel, PULL_RETRIES } from '../src/local-model/ollama-client.js'; import { localModelPaths, type LocalModelPaths } from '../src/local-model/paths.js'; -import { createRuntimeSupervisor, ensureRuntimeInstalled } from '../src/local-model/runtime.js'; +import { + createRuntimeSupervisor, + ensureRuntimeInstalled, + type RuntimeSupervisorDeps, +} from '../src/local-model/runtime.js'; import { readLocalModelState } from '../src/local-model/state-store.js'; import { registerLocalModelRoutes } from '../src/web/routes/local-model-routes.js'; import { SETTINGS_PATH } from '../src/web/route-helpers.js'; @@ -123,7 +128,7 @@ const archiveFetch: typeof fetch = async (input, init) => { * supervisor, over the data dir, the pin and the port of the test. It becomes `world.service`, * which is the one the routes ask. */ -function startServer(): LocalModelService { +function startServer(processes: Partial> = {}): LocalModelService { const pin = world.archive.pin(); const { paths, port } = world; const supervisor = createRuntimeSupervisor({ @@ -136,6 +141,7 @@ function startServer(): LocalModelService { env: { PATH: process.env.PATH, CODEMAN_PASSWORD: 'must-not-reach-the-runtime', OLLAMA_ORIGINS: '*' }, commandOfPid: async () => undefined, kill: () => {}, + ...processes, sleep: async () => {}, pin, paths, @@ -675,3 +681,83 @@ describe('the install from the first GET to Remove, real modules over a fake run expect(await readCustomModelHosts(getDataDir())).toMatchObject([{ id: 'local-ollama', models: [pick.modelId] }]); }, 60_000); }); + +describe('a server that is killed leaves its runtime running: the next server never deletes under it', () => { + /** + * Installs up to the model download, then "kills" the server: its job stops and it signals + * nothing, as a process that got SIGKILL does. Its runtime child goes on listening. Answers + * what the next server needs to find that process: `ps` and `kill` over the fake child. + */ + async function killedDuringTheDownload() { + const api = await routes(); + const pick = (await api.get()).recommendation.pick!; + const runtime = prepareRuntime(pick); + slowPull(runtime); + await api.install(pick); + await api.pulling(); + + const child = runtime.state.children[0]; + const signal = child.kill.bind(child); + // A killed server sends its child nothing. + child.kill = () => false; + await world.service.shutdown(); + expect(runtime.listening).toBe(true); + // The pid was stored the moment the child was spawned. + expect((await readLocalModelState()).runtime?.pid).toBe(child.pid); + + const signals: Array<{ pid: number; signal: string; folderThere: boolean }> = []; + const processes: Partial> = { + // What `ps -o command= -p ` prints for the runtime while it lives. + commandOfPid: async (pid) => (pid === child.pid && runtime.listening ? `${world.paths.binary} serve` : undefined), + kill: (pid, sig) => { + signals.push({ pid, signal: sig, folderThere: existsSync(world.paths.runtimeDir) }); + if (pid === child.pid) signal(sig); + }, + }; + return { ...api, pick, runtime, child, signals, processes }; + } + + it('Remove of the job it left ends that runtime BEFORE the folder is deleted', async () => { + const { app, runtime, child, signals, processes } = await killedDuringTheDownload(); + startServer(processes); + const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: true } }); + expect(removed.statusCode).toBe(200); + expect(removed.json().data.state).toBe('idle'); + // The process was signalled while its binary was still on disk, and it is gone. + expect(signals).toEqual([{ pid: child.pid, signal: 'SIGTERM', folderThere: true }]); + await vi.waitFor(() => expect(runtime.listening).toBe(false)); + expect(existsSync(world.paths.root)).toBe(false); + expect(await readLocalModelState()).toEqual(expect.objectContaining({ schema: 1, state: 'idle' })); + }, 60_000); + + it('the next start of the server ends it too: nothing would use it until a click', async () => { + const { get, settled, app, runtime, child, signals, processes } = await killedDuringTheDownload(); + const next = startServer(processes); + await next.boot(); + expect(signals.map((entry) => [entry.pid, entry.signal])).toEqual([[child.pid, 'SIGTERM']]); + await vi.waitFor(() => expect(runtime.listening).toBe(false)); + expect((await get()).install).toMatchObject({ state: 'paused', step: 'model', running: false }); + + // Continue starts a runtime of its own on the same port and finishes the install. + quickPull(runtime); + const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} }); + expect(resumed.statusCode).toBe(200); + const install = await settled(); + expect(install).toMatchObject({ state: 'installed', running: true }); + expect(runtime.state.children).toHaveLength(2); + }, 60_000); + + it('a pid that now runs something else is never signalled', async () => { + const { app, child, signals } = await killedDuringTheDownload(); + // The pid was reused by another program since. + startServer({ + commandOfPid: async () => '/Applications/Something.app/Contents/MacOS/Something', + kill: (pid, sig) => signals.push({ pid, signal: sig, folderThere: true }), + }); + const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: {} }); + expect(removed.statusCode).toBe(200); + expect(signals).toEqual([]); + // Left to the test's own cleanup. + child.crash(0); + }, 60_000); +}); diff --git a/test/local-model-install-job.test.ts b/test/local-model-install-job.test.ts index 740c4a81..dec84ee8 100644 --- a/test/local-model-install-job.test.ts +++ b/test/local-model-install-job.test.ts @@ -488,6 +488,30 @@ describe('installedBlocker', () => { }); }); + it('a runtime that did not start: what it said, and "Start" to try again (an installed model has no job to resume)', () => { + expect(installedBlocker(SAMPLE.installed, { ...LIVE, startFailed: { detail: 'spawn ENOENT' } })).toEqual({ + code: 'runtime-start-failed', + message: { text: 'The runtime did not start on this Mac.' }, + next: 'start', + detail: 'spawn ENOENT', + }); + expect(installedBlocker(SAMPLE.installed, { ...LIVE, startFailed: {} })).toEqual({ + code: 'runtime-start-failed', + message: { text: 'The runtime did not start on this Mac.' }, + next: 'start', + }); + // A runtime that runs has started: an older failure says nothing any more. + expect( + installedBlocker(SAMPLE.installed, { ...LIVE, running: true, digest: 'same', startFailed: { detail: 'x' } }) + ).toBeUndefined(); + }); + + it('the port and a runtime that keeps stopping are said before a failed start', () => { + const failed = { startFailed: { detail: 'x' } }; + expect(installedBlocker(SAMPLE.installed, { ...LIVE, ...failed, portInUse: true })?.code).toBe('port-in-use'); + expect(installedBlocker(SAMPLE.installed, { ...LIVE, ...failed, gaveUp: true })?.code).toBe('runtime-stopped'); + }); + it('a model that changed: "Remove"', () => { expect(installedBlocker(SAMPLE.installed, { ...LIVE, running: true, digest: 'changed' })).toEqual({ code: 'model-changed', @@ -617,6 +641,21 @@ describe('buildInstallView', () => { }); }); + it('a failed job never says "Start": a code whose own next step is that says "Try again" there', () => { + const state: LocalModelStateFile = { ...SAMPLE.failed, error: { code: 'runtime-stopped', detail: 'ECONNREFUSED' } }; + expect(buildInstallView(state, LIVE).error).toEqual({ + code: 'runtime-stopped', + message: { text: 'The runtime keeps stopping, so Codeman no longer restarts it.' }, + next: 'retry', + detail: 'ECONNREFUSED', + }); + // Every next step a failed job is shown with is one its actions can carry out. + for (const code of Object.keys(LOCAL_MODEL_ERROR_NEXT) as LocalModelErrorCode[]) { + const next = buildInstallView({ ...SAMPLE.failed, error: { code } }, LIVE).error?.next; + expect(next, code).not.toBe('start'); + } + }); + it('a next model is named only for an error whose next step is another model', () => { const view = buildInstallView(SAMPLE.failed, { ...LIVE, diff --git a/test/local-model-install-service.test.ts b/test/local-model-install-service.test.ts index 67da16a7..bae8ffbe 100644 --- a/test/local-model-install-service.test.ts +++ b/test/local-model-install-service.test.ts @@ -22,7 +22,7 @@ import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { dirname } from 'node:path'; import { getDataDir } from '../src/config/instance.js'; import { customModelHostsPath, readCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js'; -import { connectLocalModel, disconnectLocalModel } from '../src/local-model/connect.js'; +import { connectLocalModel, disconnectLocalModel, inspectLocalEndpoint } from '../src/local-model/connect.js'; import { createLocalModelService, type LocalModelActor, @@ -99,7 +99,14 @@ const FACTS: HarnessFact[] = [ }, ]; -function candidate(modelId: string, modelLabel: string, context: number, bytes: number, installed: boolean): Candidate { +function candidate( + modelId: string, + modelLabel: string, + context: number, + bytes: number, + installed: boolean, + needBytes = 36_000_000_000 +): Candidate { return { modelId, modelLabel, @@ -109,8 +116,8 @@ function candidate(modelId: string, modelLabel: string, context: number, bytes: cli: 'opencode', cliLabel: 'OpenCode', cliInstalled: installed, - needBytes: 30_000_000_000, - needGb: '30.0', + needBytes, + needGb: '0.0', downloadBytes: bytes, downloadGb: '0.0', budgetShare: 0.8, @@ -137,7 +144,8 @@ function recommendation(agentInstalled = false): Recommendation { headline: { text: 'Best fit for this Mac' }, notes: [], pick: candidate('qwen3.6:35b-a3b', 'Qwen3.6 35B-A3B', 131072, PULL_BYTES, agentInstalled), - alternatives: [candidate('gpt-oss:20b', 'GPT-OSS 20B', 131072, 13_793_441_427, agentInstalled)], + // Needs less memory than the pick, so it is what "does not fit" offers next. + alternatives: [candidate('gpt-oss:20b', 'GPT-OSS 20B', 131072, 13_793_441_427, agentInstalled, 26_600_000_000)], budgetBytes: 37_580_963_840, budgetGb: '37.6', budgetSource: 'estimated', @@ -187,20 +195,34 @@ function jobAt(step: LocalModelInstallStep, state: 'installing' | 'paused' = 'in // ---------- The harness ---------- -/** The state file in memory. Every write goes through the real parser, like a write to disk and a read back. */ +/** What a write to a full disk rejects with. */ +function noSpace(): Error { + return Object.assign(new Error('ENOSPC: no space left on device, write'), { code: 'ENOSPC' }); +} + +/** + * The state file in memory. Every write goes through the real parser, like a write to disk and + * a read back. `refuse` makes the disk full: while it answers true for a write, that write + * rejects with ENOSPC and the file keeps what it had, like the real temp file plus rename. + */ function memoryStore(initial: LocalModelStateFile = IDLE_LOCAL_MODEL_STATE) { let state = initial; let chain: Promise = Promise.resolve(); const history: LocalModelStateFile[] = []; + const disk = { + refuse: undefined as ((next: LocalModelStateFile, before: LocalModelStateFile) => boolean) | undefined, + }; return { get: () => state, history, + disk, readState: async () => state, mutateState: (fn: (current: LocalModelStateFile) => LocalModelStateFile | undefined) => { const run = chain.then(async () => { const before = state; const next = fn(before); if (!next) return { before, after: undefined }; + if (disk.refuse?.(next, before)) throw noSpace(); state = parseLocalModelState(JSON.stringify(next)); history.push(state); return { before, after: state }; @@ -256,7 +278,7 @@ function setup( ...(world.measured !== undefined ? { measuredTotalBytes: world.measured } : {}), }; }), - stop: vi.fn(async () => { + stop: vi.fn(async (_options?: { previousPid?: number }) => { calls.push('supervisor.stop'); world.running = false; }), @@ -307,6 +329,10 @@ function setup( calls.push('deleteModel'); return true; }), + removePartialModelBlobs: vi.fn(async () => { + calls.push('removePartialModelBlobs'); + return 0; + }), checkDeveloperTools: vi.fn(async () => 'missing' as const), stockInstallCommand: vi.fn((id: string) => (id === FACTS[0].cli ? INSTALL_COMMAND : undefined)), tryBeginCliInstall: vi.fn((): (() => void) | null => () => { @@ -322,6 +348,8 @@ function setup( return world.agentInstalled; }), audit: vi.fn(async () => {}), + // Nothing saved under the reserved id, and the endpoints setting is off: a first install. + inspectLocalEndpoint: vi.fn(async () => ({ record: 'absent' as 'absent' | 'same' | 'other', flagEnabled: false })), connectLocalModel: vi.fn(async () => { calls.push('connectLocalModel'); return { ok: true as const, flagWasEnabled: false }; @@ -401,12 +429,14 @@ describe('a job from the click to installed', () => { for (const [start] of deps.supervisor.start.mock.calls) { expect(start).toMatchObject({ port: 11435, contextLength: 131072 }); } - // The connection names the model as the runtime lists it, at the chosen context. + // The connection names the model as the runtime lists it, at the chosen context. It was + // claimed in the state file first, so the record is written as ours. expect(deps.connectLocalModel).toHaveBeenCalledWith({ port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072, - ours: false, + ours: true, + flagWasEnabled: false, }); }); @@ -602,7 +632,7 @@ describe('install is checked before anything is downloaded', () => { expect(result).toEqual({ ok: false, status: 400, - message: 'OpenCode is not installed and cannot be installed from here.', + message: { text: '{cli} is not installed and cannot be installed from here.', vars: { cli: 'OpenCode' } }, }); nothingStarted(h); }); @@ -614,8 +644,10 @@ describe('install is checked before anything is downloaded', () => { expect(result).toEqual({ ok: false, status: 400, - message: - 'A saved model connection named local-ollama already exists and was not created here. Rename or delete it in App Settings, then try again.', + message: { + text: 'A saved model connection named {id} already exists and was not created here. Rename or delete it in App Settings, then try again.', + vars: { id: 'local-ollama' }, + }, }); nothingStarted(h); }); @@ -823,6 +855,8 @@ describe('Cancel in every step', () => { }), }, }); + // What the claim keeps: the setting was on before this feature touched it. + h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'absent', flagEnabled: true }); await h.service.install(REQUEST, ADMIN); await vi.waitFor(() => expect(h.deps.connectLocalModel).toHaveBeenCalled()); const cancelling = h.service.cancel(); @@ -929,8 +963,9 @@ describe('Resume', () => { await h.service.idle(); expect(h.deps.chooseRuntimePort).toHaveBeenCalledWith({ avoid: [11435] }); expect(h.store.get()).toMatchObject({ state: 'installed', runtime: { port: 11436 } }); - // The pid that was stored belonged to the old port: it is not passed on as "ours". - expect(h.supervisor.start.mock.calls[0][0]).toEqual({ port: 11436, contextLength: 131072 }); + // The stored pid goes along: when it is still our own runtime on the OLD port, the start + // ends it instead of leaving a second runtime behind. + expect(h.supervisor.start.mock.calls[0][0]).toEqual({ port: 11436, contextLength: 131072, previousPid: 5000 }); expect(h.deps.connectLocalModel).toHaveBeenCalledWith(expect.objectContaining({ port: 11436 })); }); }); @@ -951,6 +986,16 @@ describe('a job a server restart interrupted', () => { const h = setup({ state: jobAt('verify') }); await h.service.boot(); expect(h.store.get()).toMatchObject({ state: 'paused', step: 'verify' }); + expect(h.supervisor.start).not.toHaveBeenCalled(); + // The runtime the killed server left behind is ended: nothing would use it until a click. + expect(h.calls).toEqual(['supervisor.stop']); + expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 5000 }); + }); + + it('boot() ends nothing for a job that never started a runtime', async () => { + const h = setup({ state: jobAt('runtime') }); + await h.service.boot(); + expect(h.store.get()).toMatchObject({ state: 'paused', step: 'runtime' }); expect(h.calls).toEqual([]); }); @@ -1280,9 +1325,9 @@ describe('every failure is a state with a code and a next step', () => { expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); }); - it('a saved endpoint with our id that is not ours: endpoint-id-taken with the id', async () => { + it('a saved endpoint with our id that is not ours: endpoint-id-taken with the id, and nothing is claimed or written', async () => { const h = setup(); - h.deps.connectLocalModel.mockResolvedValue({ ok: false, code: 'endpoint-id-taken' } as never); + h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'other', flagEnabled: true }); const view = await failed(h); expect(view.error).toMatchObject({ code: 'endpoint-id-taken', @@ -1290,6 +1335,7 @@ describe('every failure is a state with a code and a next step', () => { message: { vars: { id: 'local-ollama' } }, }); expect(h.store.get()).not.toHaveProperty('connection'); + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); }); it('a refused write of the connection: connect-failed with the reason', async () => { @@ -1629,7 +1675,8 @@ describe('the runtime verbs', () => { const result = await h.service.runtime({ action: 'start', newPort: true }); expect(h.deps.chooseRuntimePort).toHaveBeenCalledWith({ avoid: [11435] }); expect(h.deps.moveLocalEndpoint).toHaveBeenCalledWith(11437); - expect(h.supervisor.start).toHaveBeenLastCalledWith({ port: 11437, contextLength: 131072 }); + // The stored pid goes along, so our own runtime on the old port could not be left behind. + expect(h.supervisor.start).toHaveBeenLastCalledWith({ port: 11437, contextLength: 131072, previousPid: 4711 }); expect(result.ok && result.view).toMatchObject({ port: 11437, running: true }); expect(result.ok && result.view.error).toBeUndefined(); expect(h.store.get().runtime).toMatchObject({ port: 11437, enabled: true }); @@ -1781,7 +1828,10 @@ describe('Remove with the real connection: the settings flag', () => { /** Installs with the real connect step, then removes with the real disconnect. */ async function installThenRemove() { - const h = setup({ agentInstalled: true, overrides: { connectLocalModel, disconnectLocalModel } }); + const h = setup({ + agentInstalled: true, + overrides: { inspectLocalEndpoint, connectLocalModel, disconnectLocalModel }, + }); await installAndWait(h.service); expect(h.store.get().state).toBe('installed'); expect(flag()).toBe(true); @@ -1815,7 +1865,10 @@ describe('Remove with the real connection: the settings flag', () => { it('a resumed connect step does not forget that the flag was off before', async () => { writeSettings({ customModelEndpointsEnabled: false }); - const h = setup({ agentInstalled: true, overrides: { connectLocalModel, disconnectLocalModel } }); + const h = setup({ + agentInstalled: true, + overrides: { inspectLocalEndpoint, connectLocalModel, disconnectLocalModel }, + }); await installAndWait(h.service); // The connection is written a second time, as a resumed job would: the flag is on by now. const again = await connectLocalModel({ @@ -1897,3 +1950,641 @@ describe('view()', () => { expect(view.error).not.toHaveProperty('nextModel'); }); }); + +// ---------- The connection is claimed before it is written ---------- + +describe('the connect step claims the connection before it writes anything outside the folder', () => { + it('the state file knows the connection is ours before the endpoint record is written', async () => { + let claimAtWrite: LocalModelStateFile['connection']; + const h = setup(); + h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'absent', flagEnabled: true }); + h.deps.connectLocalModel.mockImplementation(async () => { + claimAtWrite = h.store.get().connection; + return { ok: true as const, flagWasEnabled: true }; + }); + await installAndWait(h.service); + // The claim carries the value the setting had BEFORE this feature switched it on. + expect(claimAtWrite).toEqual({ endpointId: 'local-ollama', flagWasEnabled: true }); + expect(h.deps.inspectLocalEndpoint).toHaveBeenCalledWith({ + port: 11435, + modelId: 'qwen3.6:35b-a3b', + context: 131072, + }); + }); + + it('a claim that cannot be written (a full disk) writes nothing outside, and "Try again" finishes the install', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('connect', 'paused') }); + // The claim is refused once; every other write lands. + let refused = false; + h.store.disk.refuse = (next, before) => { + if (refused || !next.connection || before.connection) return false; + refused = true; + return true; + }; + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ state: 'failed', step: 'connect', error: { code: 'disk-full' } }); + // Nothing was written outside our folder: the record and the setting are untouched. + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + expect(h.store.get()).not.toHaveProperty('connection'); + + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get()).toMatchObject({ + state: 'installed', + connection: { endpointId: 'local-ollama', flagWasEnabled: false }, + }); + expect(h.deps.connectLocalModel).toHaveBeenCalledTimes(1); + }); + + it('a record that already says what ours would say is ours, and the setting is then never switched off', async () => { + const h = setup({ state: jobAt('connect', 'paused') }); + h.deps.inspectLocalEndpoint.mockResolvedValue({ record: 'same', flagEnabled: true }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get()).toMatchObject({ + state: 'installed', + connection: { endpointId: 'local-ollama', flagWasEnabled: true }, + }); + }); + + it('an endpoint store that cannot be read: connect-failed, before anything is claimed', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('connect', 'paused') }); + h.deps.inspectLocalEndpoint.mockRejectedValue(new Error('custom-model-hosts.json is not a valid JSON list')); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ + state: 'failed', + error: { code: 'connect-failed', detail: 'custom-model-hosts.json is not a valid JSON list' }, + }); + expect(h.store.get()).not.toHaveProperty('connection'); + expect(h.deps.connectLocalModel).not.toHaveBeenCalled(); + }); + + it('a resumed job does not look again: the claim it has keeps the value from before', async () => { + const h = setup({ + state: { ...jobAt('connect', 'paused'), connection: { endpointId: 'local-ollama', flagWasEnabled: false } }, + }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.deps.inspectLocalEndpoint).not.toHaveBeenCalled(); + expect(h.deps.connectLocalModel).toHaveBeenCalledWith( + expect.objectContaining({ ours: true, flagWasEnabled: false }) + ); + expect(h.store.get().state).toBe('installed'); + }); +}); + +describe('the connect step with the real endpoint store and settings: a write that does not land', () => { + const HOSTS = () => customModelHostsPath(getDataDir()); + const real = { inspectLocalEndpoint, connectLocalModel, disconnectLocalModel }; + const hostIds = async () => (await readCustomModelHosts(getDataDir())).map((host) => host.id); + const flag = (): unknown => { + try { + return JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).customModelEndpointsEnabled; + } catch { + return undefined; + } + }; + + afterEach(() => { + rmSync(HOSTS(), { force: true }); + rmSync(SETTINGS_PATH, { force: true }); + }); + + it('the claim fails once: nothing is left behind, "Try again" installs, and Remove undoes both writes', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('connect', 'paused'), overrides: real }); + let refused = false; + h.store.disk.refuse = (next, before) => { + if (refused || !next.connection || before.connection) return false; + refused = true; + return true; + }; + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect((await h.service.view()).error?.code).toBe('disk-full'); + expect(await hostIds()).toEqual([]); + expect(flag()).toBeUndefined(); + + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + expect(await hostIds()).toEqual(['local-ollama']); + expect(flag()).toBe(true); + + const removed = await h.service.remove({ removeModel: true }, [], ADMIN); + expect(removed.ok && removed.view.state).toBe('idle'); + expect(await hostIds()).toEqual([]); + expect(flag()).toBe(false); + }); + + it('the server is killed after the claim and before the record: the next server finishes it on Continue', async () => { + // What the killed server left: the claim in the state file, nothing in the endpoint store. + const h = setup({ + state: { ...jobAt('connect'), connection: { endpointId: 'local-ollama', flagWasEnabled: false } }, + overrides: real, + }); + await h.service.boot(); + expect(await h.service.view()).toMatchObject({ state: 'paused', step: 'connect' }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + expect(await hostIds()).toEqual(['local-ollama']); + await h.service.remove({}, [], ADMIN); + expect(await hostIds()).toEqual([]); + expect(flag()).toBe(false); + }); + + it('the server is killed after both writes and before the last step was recorded: Continue ends installed', async () => { + const h = setup({ + state: { ...jobAt('connect'), connection: { endpointId: 'local-ollama', flagWasEnabled: false } }, + overrides: real, + }); + await connectLocalModel({ port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072, ours: true }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ state: 'installed' }); + expect(await hostIds()).toEqual(['local-ollama']); + }); + + it('a record of ours the state file does not know (written by an earlier order of the two) is taken as ours', async () => { + // The record and the flag exist, the state file has no claim. + await connectLocalModel({ port: 11435, modelId: 'qwen3.6:35b-a3b', context: 131072, ours: false }); + const h = setup({ state: jobAt('connect', 'paused'), overrides: real }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ state: 'installed' }); + // Remove takes the record out. The setting stays: what it was before is not known. + await h.service.remove({}, [], ADMIN); + expect(await hostIds()).toEqual([]); + expect(flag()).toBe(true); + }); + + it('a record with our id that says something else is never overwritten', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + mkdirSync(dirname(HOSTS()), { recursive: true }); + const foreign = { id: 'local-ollama', label: 'My own', baseUrl: 'http://127.0.0.1:11434' }; + writeFileSync(HOSTS(), JSON.stringify([foreign])); + const h = setup({ state: jobAt('connect', 'paused'), overrides: real }); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect((await h.service.view()).error?.code).toBe('endpoint-id-taken'); + expect(await readCustomModelHosts(getDataDir())).toEqual([foreign]); + expect(h.store.get()).not.toHaveProperty('connection'); + }); +}); + +// ---------- A job whose end could not be written ---------- + +describe('a state file that cannot be written when a step fails (a full disk)', () => { + /** A pull that fills the disk: from its failure on, every write of the state file is refused. */ + function diskFillsDuringPull(h: ReturnType) { + h.deps.pullModel.mockImplementation( + async (_target: unknown, _id: string, pull?: { onProgress?: (p: unknown) => void }) => { + pull?.onProgress?.({ completedBytes: 14_200_000_000, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 }); + h.store.disk.refuse = () => true; + return { ok: false as const, code: 'disk-full' as const, detail: 'write blob: no space left on device' }; + } + ); + } + + it('the job is not lost: the view says disk-full with the download as it stood, and no speed', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('model', 'paused') }); + diskFillsDuringPull(h); + await h.service.resume({}, ADMIN); + await h.service.idle(); + + // The file could not be told and still says the job runs. + expect(h.store.get().state).toBe('installing'); + const view = await h.service.view(); + expect(view).toMatchObject({ + state: 'failed', + step: 'model', + error: { code: 'disk-full', next: 'retry', detail: 'write blob: no space left on device' }, + actions: ['resume', 'install', 'remove'], + }); + expect(view.progress).toMatchObject({ completedBytes: 14_200_000_000, totalBytes: PULL_BYTES }); + expect(view.progress).not.toHaveProperty('bytesPerSecond'); + // A GET writes nothing: the file is as it was. + expect(h.store.get().state).toBe('installing'); + }); + + it('Cancel has nothing to cancel (409, not a 500), and "Try again" on a disk that is still full says so again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('model', 'paused') }); + diskFillsDuringPull(h); + await h.service.resume({}, ADMIN); + await h.service.idle(); + + expect(await h.service.cancel()).toMatchObject({ ok: false, status: 409, conflict: { reason: 'wrong-state' } }); + const again = await h.service.resume({}, ADMIN); + expect(again.ok && again.view).toMatchObject({ state: 'failed', error: { code: 'disk-full' } }); + expect(h.deps.pullModel).toHaveBeenCalledTimes(1); + }); + + it('once there is space again "Try again" writes the outcome, runs the step and installs', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('model', 'paused') }); + diskFillsDuringPull(h); + await h.service.resume({}, ADMIN); + await h.service.idle(); + + h.store.disk.refuse = undefined; + h.deps.pullModel.mockImplementation(async () => ({ ok: true as const, totalBytes: PULL_BYTES })); + const resumed = await h.service.resume({}, ADMIN); + expect(resumed.ok && resumed.view.state).toBe('installing'); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + expect((await h.service.view()).error).toBeUndefined(); + }); + + it('another verb writes what the view already showed: a new install replaces the failed job', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('model', 'paused'), runtimeOnDisk: true }); + diskFillsDuringPull(h); + await h.service.resume({}, ADMIN); + await h.service.idle(); + h.store.disk.refuse = undefined; + h.deps.pullModel.mockImplementation(async () => ({ ok: true as const, totalBytes: 13_793_441_427 })); + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + for (const row of [rec.pick!, ...rec.alternatives]) row.installBytes = row.downloadBytes; + return rec; + }); + const result = await installAndWait(h.service, { + modelId: 'gpt-oss:20b', + cli: 'opencode', + context: 131072, + acknowledgedBytes: 13_793_441_427, + }); + expect(result.ok).toBe(true); + expect(h.store.get()).toMatchObject({ state: 'installed', model: { id: 'gpt-oss:20b' } }); + }); + + it('Cancel while the disk is full stops the job and shows it paused, without throwing', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ + overrides: { + pullModel: vi.fn(async (_t: unknown, _id: string, pull?: PullOptions) => { + pull?.onProgress?.({ completedBytes: 6_200_000_000, totalBytes: PULL_BYTES, bytesPerSecond: 58_000_000 }); + await untilAborted(pull?.signal); + return { ok: false as const, code: 'cancelled' as const }; + }), + }, + }); + await h.service.install(REQUEST, ADMIN); + await vi.waitFor(() => expect(h.deps.pullModel).toHaveBeenCalled()); + h.store.disk.refuse = () => true; + const result = await h.service.cancel(); + expect(result.ok && result.view).toMatchObject({ + state: 'paused', + step: 'model', + progress: { completedBytes: 6_200_000_000 }, + actions: ['resume', 'install', 'remove'], + }); + // Space again: Continue writes the pause it could not write, and goes on. + h.store.disk.refuse = undefined; + h.deps.pullModel.mockImplementation(async () => ({ ok: true as const, totalBytes: PULL_BYTES })); + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + }); + + it('a first install on a full disk starts nothing and is answered with the sentence', async () => { + const h = setup(); + h.store.disk.refuse = () => true; + const result = await h.service.install(REQUEST, ADMIN); + expect(result).toEqual({ + ok: false, + status: 507, + message: { text: 'The disk is full. Free up space, then try again.' }, + }); + expect(h.calls).toEqual([]); + expect(h.store.get().state).toBe('idle'); + }); + + it('a step that finished but could not be recorded (a full disk) is disk-full, and the step runs again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: jobAt('verify', 'paused') }); + // The write that would move the job past `verify` is refused, and every write after it. + h.store.disk.refuse = (next, before) => before.step === 'verify' && next.step !== 'verify'; + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(await h.service.view()).toMatchObject({ state: 'failed', step: 'verify', error: { code: 'disk-full' } }); + h.store.disk.refuse = undefined; + await h.service.resume({}, ADMIN); + await h.service.idle(); + expect(h.store.get().state).toBe('installed'); + }); +}); + +// ---------- A runtime that does not start, and one an earlier server left behind ---------- + +describe('an installed model whose runtime does not start', () => { + it('says so on the view with what the runtime said, and Start tries again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: INSTALLED, runtimeOnDisk: true }); + h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'spawn ENOENT' }; + await h.service.boot(); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'installed', running: false, actions: ['start', 'remove'] }); + expect(view.error).toEqual({ + code: 'runtime-start-failed', + message: { text: 'The runtime did not start on this Mac.' }, + // Start is the verb an installed model has: there is no job to resume. + next: 'start', + detail: 'spawn ENOENT', + }); + // A session start is refused with the same reason, not with "start it first". + expect(await h.service.launchRefusalMessage('local-ollama', 'qwen3.6:35b-a3b')).toBe( + 'The runtime did not start on this Mac.' + ); + + // Start again, and it fails again: the view still says why. + const again = await h.service.runtime({ action: 'start' }); + expect(again.ok && again.view.error?.code).toBe('runtime-start-failed'); + + // Start again, and it works: the error is gone. + h.world.startResult = undefined; + const started = await h.service.runtime({ action: 'start' }); + expect(started.ok && started.view).toMatchObject({ running: true }); + expect(started.ok && started.view.error).toBeUndefined(); + }); + + it('Stop clears it: a model the person stopped shows no error', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: INSTALLED, runtimeOnDisk: true }); + h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'spawn ENOENT' }; + await h.service.boot(); + const stopped = await h.service.runtime({ action: 'stop' }); + expect(stopped.ok && stopped.view.error).toBeUndefined(); + }); +}); + +describe('a later build with another runtime pin', () => { + const OLD = '0.39.2'; + const OLD_STATE: LocalModelStateFile = { ...INSTALLED, runtime: { ...INSTALLED.runtime!, version: OLD } }; + /** Only the old version is unpacked: what an update of Codeman leaves on disk. */ + const onlyOldOnDisk = async (_pin?: unknown, paths?: LocalModelPaths) => + paths?.binary === `/data/local-model/runtime/ollama-${OLD}/ollama`; + + it('an installed model keeps running the version it was installed with', async () => { + const h = setup({ state: OLD_STATE }); + h.deps.isRuntimeInstalled.mockImplementation(onlyOldOnDisk as never); + await h.service.boot(); + expect(h.supervisor.start).toHaveBeenCalledWith({ + port: 11435, + contextLength: 131072, + previousPid: 4711, + version: OLD, + }); + // The stored version stays what runs, so the next start finds it again. + expect(h.store.get().runtime?.version).toBe(OLD); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'installed', running: true, actions: ['start-session', 'stop', 'remove'] }); + expect(view.error).toBeUndefined(); + // A NEW install would still download the pinned build. + expect(view.runtime).toMatchObject({ version: OLLAMA_RUNTIME_PIN.version, installed: false }); + }); + + it('with the pinned build unpacked that one runs, and the state says so', async () => { + const h = setup({ state: OLD_STATE, runtimeOnDisk: true }); + await h.service.boot(); + expect(h.supervisor.start).toHaveBeenCalledWith({ port: 11435, contextLength: 131072, previousPid: 4711 }); + expect(h.store.get().runtime?.version).toBe(OLLAMA_RUNTIME_PIN.version); + }); + + it('with neither on disk the start fails in the open: the view says the runtime did not start', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup({ state: OLD_STATE }); + h.world.startResult = { ok: false, code: 'runtime-start-failed', detail: 'spawn ENOENT' }; + await h.service.boot(); + expect((await h.service.view()).error).toMatchObject({ code: 'runtime-start-failed', next: 'start' }); + }); + + it('a version the state file could never name is not turned into a path', async () => { + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, version: '../../evil' } } }); + await h.service.boot(); + expect(h.supervisor.start).toHaveBeenCalledWith({ port: 11435, contextLength: 131072, previousPid: 4711 }); + }); +}); + +describe('a runtime an earlier server left behind', () => { + it.each(['paused', 'failed'] as const)('Remove of a %s job ends it before any file is deleted', async (state) => { + const base = jobAt('model', 'paused'); + const h = setup({ + state: state === 'paused' ? base : { ...base, state: 'failed', error: { code: 'pull-failed' } }, + }); + const removed = await h.service.remove({ removeModel: true }, [], ADMIN); + expect(removed.ok && removed.view.state).toBe('idle'); + expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 5000 }); + expect(h.calls.indexOf('supervisor.stop')).toBeLessThan(h.calls.indexOf('removeLocalModelFiles')); + }); + + it('Stop of an installed model ends it too, also when this server never started it', async () => { + const h = setup({ state: INSTALLED }); + await h.service.runtime({ action: 'stop' }); + expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 4711 }); + }); + + it('boot ends it for a model the person had stopped', async () => { + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } }); + await h.service.boot(); + expect(h.supervisor.start).not.toHaveBeenCalled(); + expect(h.supervisor.stop).toHaveBeenCalledWith({ previousPid: 4711 }); + }); + + it('the pid is stored the moment the child is spawned, before it answers', async () => { + const h = setup({ + state: { ...jobAt('model', 'paused'), runtime: { version: '0.40.15', port: 11435, enabled: true } }, + }); + h.service.onRuntimeChange({ running: false, pid: 7777, restarts: 0, gaveUp: false }); + await vi.waitFor(() => expect(h.store.get().runtime?.pid).toBe(7777)); + }); +}); + +// ---------- What a failed download is called, and what is offered next ---------- + +describe('a runtime that is gone for good during the download', () => { + it('is said as the runtime, not as the network, and "Try again" runs the step again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + h.deps.pullModel.mockImplementation(async () => { + h.world.running = false; + h.world.gaveUp = true; + return { ok: false as const, code: 'pull-failed' as const, detail: 'connect ECONNREFUSED 127.0.0.1:11435' }; + }); + await installAndWait(h.service); + const view = await h.service.view(); + expect(view).toMatchObject({ state: 'failed', step: 'model', actions: ['resume', 'install', 'remove'] }); + expect(view.error).toEqual({ + code: 'runtime-stopped', + message: { text: 'The runtime keeps stopping, so Codeman no longer restarts it.' }, + // A job has no Start verb: the step is run again. + next: 'retry', + detail: 'connect ECONNREFUSED 127.0.0.1:11435', + }); + }); + + it('a pull that fails while the runtime runs is still the download', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + h.deps.pullModel.mockResolvedValue({ ok: false, code: 'pull-failed', detail: 'unexpected EOF' } as never); + await installAndWait(h.service); + expect((await h.service.view()).error).toMatchObject({ code: 'pull-failed', next: 'retry' }); + }); +}); + +describe('the model offered after "does not fit"', () => { + it('needs less memory than the one that failed: a model that needs more is not offered', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + // The only other row needs MORE than the model that failed. + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + rec.alternatives = [candidate('qwen3.8:27b', 'Qwen3.8 27B', 131072, 17_700_000_000, false, 39_000_000_000)]; + return rec; + }); + h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'does-not-fit' } as never); + await installAndWait(h.service); + const view = await h.service.view(); + expect(view.error).toMatchObject({ code: 'does-not-fit', next: 'next-model' }); + expect(view.error).not.toHaveProperty('nextModel'); + }); + + it('skips a larger row for the first one that needs less', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + h.deps.getCatalog.mockReturnValue({ + ...CATALOG, + models: [...CATALOG.models, { id: 'qwen3.8:27b', label: 'Qwen3.8 27B' }], + } as unknown as LocalModelCatalog); + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + rec.alternatives = [ + candidate('qwen3.8:27b', 'Qwen3.8 27B', 131072, 17_700_000_000, false, 39_000_000_000), + candidate('gpt-oss:20b', 'GPT-OSS 20B', 131072, 13_793_441_427, false, 26_600_000_000), + ]; + return rec; + }); + h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'does-not-fit' } as never); + await installAndWait(h.service); + expect((await h.service.view()).error?.nextModel).toEqual({ modelId: 'gpt-oss:20b', modelLabel: 'GPT-OSS 20B' }); + }); + + it('after "no tool call" memory is not the question: the best other model is offered', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const h = setup(); + h.deps.getCatalog.mockReturnValue({ + ...CATALOG, + models: [...CATALOG.models, { id: 'qwen3.8:27b', label: 'Qwen3.8 27B' }], + } as unknown as LocalModelCatalog); + h.deps.recommend.mockImplementation(() => { + const rec = recommendation(); + rec.alternatives = [candidate('qwen3.8:27b', 'Qwen3.8 27B', 131072, 17_700_000_000, false, 39_000_000_000)]; + return rec; + }); + h.deps.verifyModel.mockResolvedValue({ ok: false, code: 'no-tool-call' } as never); + await installAndWait(h.service); + expect((await h.service.view()).error?.nextModel).toEqual({ modelId: 'qwen3.8:27b', modelLabel: 'Qwen3.8 27B' }); + }); +}); + +describe('what a replaced download left behind', () => { + it('the partial blobs are deleted as files, after the model was asked for and before the new pull', async () => { + const h = setup({ + state: { ...jobAt('model', 'paused'), replacedModelId: 'gpt-oss:20b' }, + }); + h.deps.removePartialModelBlobs.mockImplementation(async () => { + h.calls.push('removePartialModelBlobs'); + return 12_708_189_425; + }); + vi.spyOn(console, 'log').mockImplementation(() => {}); + await h.service.resume({}, ADMIN); + await h.service.idle(); + const order = h.calls.filter((call) => ['deleteModel', 'removePartialModelBlobs', 'pullModel'].includes(call)); + expect(order).toEqual(['deleteModel', 'removePartialModelBlobs', 'pullModel']); + expect(h.deps.removePartialModelBlobs).toHaveBeenCalledWith(PATHS); + expect(h.store.get()).not.toHaveProperty('replacedModelId'); + }); + + it('a job that replaced nothing deletes nothing', async () => { + const h = setup(); + await installAndWait(h.service); + expect(h.deps.deleteModel).not.toHaveBeenCalled(); + expect(h.deps.removePartialModelBlobs).not.toHaveBeenCalled(); + }); +}); + +// ---------- Live sessions keep the address they were started with ---------- + +describe('Stop and "Use another port" while live sessions use the model', () => { + it('Stop is 409 sessions-open with their names, and the runtime keeps running', async () => { + const h = setup({ state: INSTALLED }); + await h.service.boot(); + h.supervisor.stop.mockClear(); + const result = await h.service.runtime({ action: 'stop' }, ['w1-app', 'w4-lib']); + expect(result).toMatchObject({ + ok: false, + status: 409, + conflict: { + reason: 'sessions-open', + sessions: ['w1-app', 'w4-lib'], + message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen, vars: { sessions: 'w1-app, w4-lib' } }, + }, + }); + expect(h.supervisor.stop).not.toHaveBeenCalled(); + expect(h.store.get().runtime?.enabled).toBe(true); + }); + + it('a start on another port is refused the same way, and nothing moves', async () => { + const h = setup({ state: INSTALLED }); + const result = await h.service.runtime({ action: 'start', newPort: true }, ['w1-app']); + expect(result).toMatchObject({ ok: false, status: 409, conflict: { reason: 'sessions-open' } }); + expect(h.deps.chooseRuntimePort).not.toHaveBeenCalled(); + expect(h.deps.moveLocalEndpoint).not.toHaveBeenCalled(); + expect(h.store.get().runtime?.port).toBe(11435); + }); + + it('a plain Start is not refused: it changes no address', async () => { + const h = setup({ state: { ...INSTALLED, runtime: { ...INSTALLED.runtime!, enabled: false } } }); + const result = await h.service.runtime({ action: 'start' }, ['w1-app']); + expect(result.ok && result.view.running).toBe(true); + }); + + it('"Use another port" of a job is refused only when a connection of ours exists', async () => { + const failedOnPort: LocalModelStateFile = { + ...jobAt('connect', 'paused'), + state: 'failed', + error: { code: 'port-in-use', vars: { port: '11435' } }, + }; + const connected = setup({ + state: { ...failedOnPort, connection: { endpointId: 'local-ollama', flagWasEnabled: false } }, + }); + expect(await connected.service.resume({ newPort: true }, ADMIN, ['w1-app'])).toMatchObject({ + ok: false, + conflict: { reason: 'sessions-open', sessions: ['w1-app'] }, + }); + expect(connected.deps.chooseRuntimePort).not.toHaveBeenCalled(); + + // Without a connection of ours no session can be on the model: sessions elsewhere do not matter. + const fresh = setup({ state: failedOnPort }); + fresh.deps.chooseRuntimePort.mockResolvedValue(11436); + const result = await fresh.service.resume({ newPort: true }, ADMIN, ['w1-app']); + expect(result.ok).toBe(true); + await fresh.service.idle(); + }); + + it('a state write that fails before the saved endpoint moved changes nothing', async () => { + const h = setup({ state: INSTALLED }); + h.deps.chooseRuntimePort.mockResolvedValue(11437); + h.store.disk.refuse = (next) => next.runtime?.port === 11437; + await expect(h.service.runtime({ action: 'start', newPort: true })).rejects.toThrow('ENOSPC'); + expect(h.deps.moveLocalEndpoint).not.toHaveBeenCalled(); + expect(h.store.get().runtime?.port).toBe(11435); + }); +}); diff --git a/test/local-model-ollama-client.test.ts b/test/local-model-ollama-client.test.ts index 35fd13b0..94fcb670 100644 --- a/test/local-model-ollama-client.test.ts +++ b/test/local-model-ollama-client.test.ts @@ -708,7 +708,7 @@ describe('judgeLoaded', () => { expect(judged).toEqual({ ok: false, code: 'does-not-fit', - detail: '20000000000 of 23205798956 bytes are on the GPU', + detail: '20.0 of 23.2 GB of the model are on the GPU', }); // One byte is enough. expect(judgeLoaded([loaded({ sizeVram: 23_205_798_955 })], MODEL, CONTEXT)).toMatchObject({ code: 'does-not-fit' }); @@ -885,7 +885,7 @@ describe('verifyModel', () => { expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({ ok: false, code: 'does-not-fit', - detail: '18000000000 of 23205798956 bytes are on the GPU', + detail: '18.0 of 23.2 GB of the model are on the GPU', }); expect(server.requests.filter((r) => r.path === '/v1/chat/completions')).toHaveLength(1); }); diff --git a/test/routes/custom-model-routes.test.ts b/test/routes/custom-model-routes.test.ts index 7253c9aa..fc31495a 100644 --- a/test/routes/custom-model-routes.test.ts +++ b/test/routes/custom-model-routes.test.ts @@ -777,7 +777,7 @@ describe('the endpoint the local model install manages', () => { }); expect(res.json()).toEqual({ success: false, - error: 'The endpoint id "local-ollama" is reserved for the model Codeman runs on this Mac. Pick another id.', + error: 'This endpoint id is reserved for the model Codeman runs on this Mac. Pick another id.', errorCode: 'INVALID_INPUT', }); expect(await readCustomModelHosts(getDataDir())).toEqual([]); diff --git a/test/routes/local-model-routes.test.ts b/test/routes/local-model-routes.test.ts index 0ac50443..11ecb354 100644 --- a/test/routes/local-model-routes.test.ts +++ b/test/routes/local-model-routes.test.ts @@ -657,20 +657,43 @@ describe('the install verbs', () => { expect(f.service.install).toHaveBeenCalledWith(INSTALL_BODY, { username: 'alice', ip: '127.0.0.1' }); }); - it('a row the server does not offer is a 400 with the service own sentence', async () => { + it('a row the server does not offer is a 400 with the service own sentence, as text and as data', async () => { const f = fakes(); f.service.install.mockResolvedValue({ ok: false, status: 400, - message: 'This model at this context is not offered for this Mac right now.', + message: { text: LOCAL_MODEL_INSTALL_LABELS.refusedAgentNotInstallable, vars: { cli: 'OpenCode' } }, }); const { app } = await harness(f.deps); const res = await app.inject({ method: 'POST', url: '/api/local-model/install', payload: INSTALL_BODY }); expect(res.statusCode).toBe(400); + // `error` is the English sentence with its slots filled; `data.message` is the sentence + // with its slots apart, which the window prints in its own language. expect(res.json()).toEqual({ success: false, - error: 'This model at this context is not offered for this Mac right now.', + error: 'OpenCode is not installed and cannot be installed from here.', errorCode: 'INVALID_INPUT', + data: { + message: { text: '{cli} is not installed and cannot be installed from here.', vars: { cli: 'OpenCode' } }, + }, + }); + }); + + it('a first install on a full disk is a 507 with the sentence, not a 500', async () => { + const f = fakes(); + f.service.install.mockResolvedValue({ + ok: false, + status: 507, + message: { text: LOCAL_MODEL_INSTALL_LABELS.errDiskFull }, + }); + const { app } = await harness(f.deps); + const res = await app.inject({ method: 'POST', url: '/api/local-model/install', payload: INSTALL_BODY }); + expect(res.statusCode).toBe(507); + expect(res.json()).toEqual({ + success: false, + error: 'The disk is full. Free up space, then try again.', + errorCode: 'OPERATION_FAILED', + data: { message: { text: 'The disk is full. Free up space, then try again.' } }, }); }); @@ -743,6 +766,46 @@ describe('the install verbs', () => { expect(f.service.resume.mock.calls[0][1]).toEqual({ username: 'admin', ip: '127.0.0.1' }); }); + it('resume and runtime hand the service the live sessions on the model, read at the moment of the request', async () => { + const f = fakes(); + const sessions = new Map([ + ['a', { name: 'w1-app', customModel: { endpointId: 'local-ollama' } }], + ['b', { name: 'w2-app', customModel: { endpointId: 'office-gpu' } }], + ]); + const { app } = await createRouteTestHarness((instance) => + registerLocalModelRoutes(instance, { sessions }, f.deps) + ); + await app.inject({ method: 'POST', url: '/api/local-model/runtime', payload: { action: 'stop' } }); + await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: { newPort: true } }); + sessions.delete('a'); + await app.inject({ method: 'POST', url: '/api/local-model/runtime', payload: { action: 'stop' } }); + expect(f.service.runtime.mock.calls.map((args) => args[1])).toEqual([['w1-app'], []]); + expect(f.service.resume.mock.calls[0][2]).toEqual(['w1-app']); + }); + + it('a stop while a live session uses the model: 409 with the session names', async () => { + const f = fakes(); + f.service.runtime.mockImplementation(async (_request, names = []) => ({ + ok: false, + status: 409, + conflict: { + reason: 'sessions-open', + message: { text: LOCAL_MODEL_INSTALL_LABELS.conflictSessionsOpen, vars: { sessions: names.join(', ') } }, + sessions: [...names], + }, + })); + const { app } = await harness(f.deps, undefined, [ + { name: 'w1-app', customModel: { endpointId: 'local-ollama' } }, + ]); + const res = await app.inject({ method: 'POST', url: '/api/local-model/runtime', payload: { action: 'stop' } }); + expect(res.statusCode).toBe(409); + expect(res.json()).toMatchObject({ + errorCode: 'CONFLICT', + error: 'Sessions are still using this model: w1-app. Close them first.', + data: { reason: 'sessions-open', sessions: ['w1-app'] }, + }); + }); + it('resume refuses a body it does not know', async () => { const f = fakes(); const { app } = await harness(f.deps);