diff --git a/test/local-model-install-e2e.test.ts b/test/local-model-install-e2e.test.ts new file mode 100644 index 00000000..886c26ce --- /dev/null +++ b/test/local-model-install-e2e.test.ts @@ -0,0 +1,677 @@ +/** + * @fileoverview The local model install from the first GET to Remove, with the REAL modules + * joined together: the routes, the install service, the catalogue and `recommend()`, the + * runtime download and unpack, the supervisor, the pull, the checks, the connection and the + * state file. Only what would touch the world is a stand-in: + * + * - the runtime is the fake HTTP server of test/fixtures/fake-ollama.ts (one instance serves + * the archive, a second one is what the supervisor "starts" through the fake's `spawn`); + * - the pin points at the fake's small archive and carries that archive's own sha256, so the + * real download, the real hash comparison and the real `/usr/bin/tar` run; + * - the hardware is the owner's MacBook as its `sysctl` reported it (Apple M5 Pro, 48 GB); + * - the coding agent's installer is a function that says it ran; + * - the waits between retries are skipped (the retries themselves are the real ones). + * + * Nothing is downloaded from the network, no process is started and no fixed port is used + * (the runtime's port is one the system handed out). Files live in the test's temp data dir. + * + * The walks: the whole install and its removal; a damaged runtime download; a model download + * that fails twice and then succeeds; one that keeps failing; a model that calls no tool; + * Cancel during the model download; a server restart during the model download. + * + * "A server" here is one install service with its own runtime supervisor (`startServer()`). + * A restart is `shutdown()` of the one and a new one over the same data dir. + * + * The other tests give each module fakes of its neighbours. This one is what fails when two + * modules that each pass their own tests do not fit together. + * + * Port: the fake runtime binds port 0 twice; the routes use app.inject(). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { rm } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { createRouteTestHarness } from './routes/_route-test-utils.js'; +import { + createFakeOllama, + freeLoopbackPort, + MACBOOK_MEASURED_TOTAL_BYTES, + startFakeOllama, + type FakeOllama, +} from './fixtures/fake-ollama.js'; +import { dataPath, getDataDir } from '../src/config/instance.js'; +import { customModelHostsPath, readCustomModelHosts } from '../src/custom-model-hosts.js'; +import { createLocalModelService, type LocalModelService } from '../src/local-model/install-job.js'; +import { memClassGb, formatGb } from '../src/local-model/format.js'; +import { pullModel, PULL_RETRIES } from '../src/local-model/ollama-client.js'; +import { localModelPaths, type LocalModelPaths } from '../src/local-model/paths.js'; +import { createRuntimeSupervisor, ensureRuntimeInstalled } from '../src/local-model/runtime.js'; +import { readLocalModelState } from '../src/local-model/state-store.js'; +import { registerLocalModelRoutes } from '../src/web/routes/local-model-routes.js'; +import { SETTINGS_PATH } from '../src/web/route-helpers.js'; +import type { Candidate, HardwareFacts, HarnessFact, LocalModelInstallView, LocalModelStatus } from '../src/types.js'; + +const INSTALL_COMMAND = 'curl -fsSL https://opencode.ai/install | bash'; + +/** The owner's MacBook, from its sysctl capture of 2026-11-06. */ +const MACBOOK: HardwareFacts = { + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 51539607552, + memGb: formatGb(51539607552), + memClassGb: memClassGb(51539607552), + chip: 'Apple M5 Pro', + model: 'Mac17,13', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: 912308310016, + freeDiskGb: formatGb(912308310016), + source: 'sysctl', +}; + +interface World { + archive: FakeOllama; + runtime: FakeOllama | undefined; + port: number; + paths: LocalModelPaths; + agentInstalled: boolean; + installerRuns: number; + /** How many of the next archive downloads arrive with one byte changed (the size stays the pinned one). */ + damagedDownloads: number; + /** The server that is up. A restart replaces it. */ + service: LocalModelService; + /** Every server of the test, for the clean-up. */ + services: LocalModelService[]; + sessions: Map; +} + +let world: World; + +const HOSTS = () => customModelHostsPath(getDataDir()); +const STATE = () => dataPath('local-model.json'); + +const harnessFacts = async (): Promise => [ + { + cli: 'opencode', + label: 'OpenCode', + installed: world.agentInstalled, + installable: !world.agentInstalled, + ...(world.agentInstalled ? {} : { installCommand: INSTALL_COMMAND }), + acceptsModelId: () => true, + }, +]; + +/** + * The download of the runtime archive: the real `fetch` against the fake's archive. While + * `world.damagedDownloads` is above 0 the body arrives with its last byte changed, as a + * download damaged on the way does: the size is still the pinned one, only the hash can tell. + */ +const archiveFetch: typeof fetch = async (input, init) => { + const res = await fetch(input, init); + if (world.damagedDownloads <= 0 || !res.ok) return res; + world.damagedDownloads -= 1; + const body = Buffer.from(await res.arrayBuffer()); + body[body.length - 1] ^= 0xff; + return new Response(body, { status: res.status, headers: { 'content-length': String(body.length) } }); +}; + +/** + * One Codeman server as far as this feature goes: an install service with its own runtime + * supervisor, over the data dir, the pin and the port of the test. It becomes `world.service`, + * which is the one the routes ask. + */ +function startServer(): LocalModelService { + const pin = world.archive.pin(); + const { paths, port } = world; + const supervisor = createRuntimeSupervisor({ + // "Starting the runtime" makes the second fake listen on the port it was given. + spawn: (command, args, options) => { + if (!world.runtime) throw new Error('The test did not prepare a runtime'); + return world.runtime.spawn(command, args, options); + }, + fetch, + env: { PATH: process.env.PATH, CODEMAN_PASSWORD: 'must-not-reach-the-runtime', OLLAMA_ORIGINS: '*' }, + commandOfPid: async () => undefined, + kill: () => {}, + sleep: async () => {}, + pin, + paths, + onChange: (status) => service.onRuntimeChange(status), + }); + const service: LocalModelService = createLocalModelService({ + readHardware: async () => MACBOOK, + buildHarnessFacts: harnessFacts, + pin, + paths, + // The real download and unpack, told where the network is: the fake's archive. + ensureRuntimeInstalled: (options) => + ensureRuntimeInstalled({ ...options, deps: { fetch: archiveFetch, sleep: async () => {} } }), + // The real pull with its real retries, without the seconds between them. + pullModel: (target, modelId, options) => pullModel(target, modelId, { ...options, sleep: async () => {} }), + chooseRuntimePort: async () => port, + supervisor, + recheckCliInstalled: async () => world.agentInstalled, + tryBeginCliInstall: () => () => {}, + runInstallCommand: async () => { + world.installerRuns++; + world.agentInstalled = true; + return { code: 0, output: 'opencode installed\n', timedOut: false, aborted: false }; + }, + audit: async () => {}, + checkDeveloperTools: async () => 'missing', + }); + world.service = service; + world.services.push(service); + return service; +} + +beforeEach(async () => { + const archive = await startFakeOllama(); + world = { + archive, + runtime: undefined, + port: await freeLoopbackPort(), + paths: localModelPaths('0.40.15'), + agentInstalled: false, + installerRuns: 0, + damagedDownloads: 0, + service: undefined as unknown as LocalModelService, + services: [], + sessions: new Map(), + }; + startServer(); +}); + +afterEach(async () => { + for (const service of world.services) await service.shutdown(); + await world.archive.close(); + await world.runtime?.close(); + await rm(world.paths.root, { recursive: true, force: true }); + rmSync(STATE(), { force: true }); + rmSync(HOSTS(), { force: true }); + rmSync(SETTINGS_PATH, { force: true }); + vi.restoreAllMocks(); +}); + +async function routes() { + const { app } = await createRouteTestHarness((instance) => + registerLocalModelRoutes( + instance, + { sessions: world.sessions }, + { readHardware: async () => MACBOOK, buildHarnessFacts: harnessFacts, service: () => world.service } + ) + ); + const get = async (): Promise => + (await app.inject({ method: 'GET', url: '/api/local-model' })).json().data; + /** Polls the GET the way the window does, until the job is no longer running. */ + const settled = async (): Promise => { + let install = (await get()).install; + await vi.waitFor( + async () => { + install = (await get()).install; + expect(install.state).not.toBe('installing'); + }, + { timeout: 20_000, interval: 25 } + ); + return install; + }; + /** The click on "Download and install" for the row the server recommends, with the size it showed. */ + const install = (pick: Candidate) => + app.inject({ + method: 'POST', + url: '/api/local-model/install', + payload: { modelId: pick.modelId, cli: 'opencode', context: pick.context, acknowledgedBytes: pick.installBytes }, + }); + /** Polls until the model download is under way and has moved. */ + const pulling = async (): Promise => { + await vi.waitFor( + async () => { + const status = await get(); + expect(status.install.step).toBe('model'); + expect(status.install.progress?.completedBytes ?? 0).toBeGreaterThan(0); + }, + { timeout: 20_000, interval: 20 } + ); + }; + return { app, get, settled, install, pulling }; +} + +/** The runtime the supervisor will "start": it has exactly the model the server recommended. */ +function prepareRuntime(pick: Candidate): FakeOllama { + world.runtime = createFakeOllama({ + model: { id: pick.modelId, bytes: pick.downloadBytes }, + contextLength: pick.context, + }); + return world.runtime; +} + +/** A model download that takes a while: 400 lines, 5 ms apart. */ +function slowPull(runtime: FakeOllama): void { + runtime.switches.pull.progressLines = 400; + runtime.switches.pull.lineDelayMs = 5; +} + +function quickPull(runtime: FakeOllama): void { + runtime.switches.pull.progressLines = 20; + runtime.switches.pull.lineDelayMs = 0; +} + +describe('the install from the first GET to Remove, real modules over a fake runtime', () => { + it('recommends for the MacBook, installs what was confirmed, connects it, refuses a changed model, and removes all of it', async () => { + mkdirSync(dirname(SETTINGS_PATH), { recursive: true }); + writeFileSync(SETTINGS_PATH, JSON.stringify({ tmuxHistoryLimit: 5000 })); + const { app, get, settled, install: clickInstall } = await routes(); + + // ---- The chooser: what the window shows before anything is installed ---- + const before = await get(); + expect(before.install).toMatchObject({ + state: 'idle', + running: false, + developerTools: 'missing', + actions: ['install'], + folder: world.paths.root, + runtime: { version: '0.40.15', installed: false, downloadBytes: world.archive.archive.size }, + }); + expect(before.recommendation.verdict).toBe('ok'); + expect(before.recommendation.budgetSource).toBe('estimated'); + const pick = before.recommendation.pick!; + expect(pick).toBeDefined(); + // The MacBook's row: the one model that was run on a Mac like it, with its measured speed. + expect(pick).toMatchObject({ + modelId: 'qwen3.6:35b-a3b', + context: 131072, + fit: 'tight', + needSource: 'measured', + tested: true, + tag: { kind: 'tested' }, + }); + // The row carries the coding agent and the literal command the consent screen prints. + expect(pick.harnesses[0]).toMatchObject({ + cli: 'opencode', + cliLabel: 'OpenCode', + installed: false, + installCommand: INSTALL_COMMAND, + }); + // The server's own total: the model plus the runtime archive that is still to download. + expect(pick.installBytes).toBe(pick.downloadBytes + world.archive.archive.size); + expect(pick.installGb).toBe(formatGb(pick.installBytes)); + const runtime = prepareRuntime(pick); + + // ---- Consent is bound to the size shown ---- + const stale = await app.inject({ + method: 'POST', + url: '/api/local-model/install', + payload: { modelId: pick.modelId, cli: 'opencode', context: pick.context, acknowledgedBytes: pick.downloadBytes }, + }); + expect(stale.statusCode).toBe(409); + expect(stale.json().data).toMatchObject({ reason: 'size-changed', expectedBytes: pick.installBytes }); + expect(world.archive.state.archiveRequests).toBe(0); + + // ---- Install ---- + const started = await clickInstall(pick); + expect(started.statusCode).toBe(200); + expect(started.json().data).toMatchObject({ + state: 'installing', + steps: ['runtime', 'model', 'verify', 'agent', 'connect'], + model: { id: pick.modelId, label: pick.modelLabel }, + cli: 'opencode', + context: pick.context, + actions: ['cancel'], + }); + // One job at a time. + const second = await clickInstall(pick); + expect(second.statusCode).toBe(409); + expect(second.json().data).toMatchObject({ reason: 'job-running' }); + + const install = await settled(); + expect(install.error).toBeUndefined(); + expect(install).toMatchObject({ + state: 'installed', + running: true, + port: world.port, + model: { id: pick.modelId, label: pick.modelLabel }, + cli: 'opencode', + cliLabel: 'OpenCode', + context: pick.context, + endpointId: 'local-ollama', + folder: world.paths.root, + loaded: { context: pick.context, onGpu: true }, + actions: ['start-session', 'stop', 'remove'], + runtime: { installed: true, downloadBytes: 0, downloadGb: formatGb(0) }, + }); + + // ---- What really happened ---- + // The runtime was downloaded once, unpacked, and "started" with the allowlisted environment. + expect(world.archive.state.archiveRequests).toBe(1); + expect(existsSync(world.paths.binary)).toBe(true); + expect(existsSync(world.paths.archivePartial)).toBe(false); + expect(runtime.state.children).toHaveLength(1); + const env = runtime.state.children[0].env; + expect(env.OLLAMA_HOST).toBe(`127.0.0.1:${world.port}`); + expect(env.OLLAMA_CONTEXT_LENGTH).toBe(String(pick.context)); + expect(env.HOME).toBe(world.paths.homeDir); + expect(env.OLLAMA_MODELS).toBe(world.paths.modelsDir); + expect(Object.keys(env).filter((key) => key.startsWith('CODEMAN_'))).toEqual([]); + expect(env).not.toHaveProperty('OLLAMA_ORIGINS'); + // One pull, one coding agent install. + expect(runtime.state.pulls).toBe(1); + expect(world.installerRuns).toBe(1); + + // The state file: the measured budget, the pull's own total, the digest. + const state = await readLocalModelState(); + expect(state).toMatchObject({ + state: 'installed', + model: { id: pick.modelId, context: pick.context, pullBytes: runtime.model.bytes, digest: runtime.model.digest }, + runtime: { + version: '0.40.15', + port: world.port, + enabled: true, + measuredTotalBytes: MACBOOK_MEASURED_TOTAL_BYTES, + }, + connection: { endpointId: 'local-ollama', flagWasEnabled: false }, + }); + expect(JSON.stringify(state)).not.toContain(getDataDir()); + + // The two writes outside the folder, and nothing else changed there. + expect(await readCustomModelHosts(getDataDir())).toEqual([ + { + id: 'local-ollama', + label: 'This Mac', + baseUrl: `http://127.0.0.1:${world.port}`, + authStyle: 'bearer', + models: [pick.modelId], + defaultModelId: pick.modelId, + defaultContextLength: pick.context, + }, + ]); + expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8'))).toEqual({ + tmuxHistoryLimit: 5000, + customModelEndpointsEnabled: true, + }); + + // From now on the budget of the GET is the measured one. + const after = await get(); + expect(after.recommendation.budgetSource).toBe('measured'); + expect(after.recommendation.budgetBytes).toBeLessThanOrEqual(MACBOOK_MEASURED_TOTAL_BYTES); + + // ---- A session may start, until the model changes under its name ---- + expect(await world.service.launchRefusalMessage('local-ollama', pick.modelId)).toBeUndefined(); + runtime.model.digest = 'f'.repeat(64); + expect(await world.service.launchRefusalMessage('local-ollama', pick.modelId)).toBe( + 'The model on this Mac changed since it was installed. Remove it and install it again.' + ); + expect((await get()).install).toMatchObject({ + state: 'installed', + error: { code: 'model-changed', next: 'remove' }, + }); + + // ---- Remove ---- + world.sessions.set('s1', { name: 'w1-app', customModel: { endpointId: 'local-ollama' } }); + const refused = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: true } }); + expect(refused.statusCode).toBe(409); + expect(refused.json().data).toMatchObject({ reason: 'sessions-open', sessions: ['w1-app'] }); + expect(existsSync(world.paths.binary)).toBe(true); + + world.sessions.clear(); + const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: true } }); + expect(removed.statusCode).toBe(200); + expect(removed.json().data).toMatchObject({ state: 'idle', running: false, actions: ['install'] }); + // The child was stopped and the whole folder is gone: the runtime, the private home, the models. + expect(runtime.state.children[0].signals).toContain('SIGTERM'); + expect(runtime.listening).toBe(false); + expect(existsSync(world.paths.root)).toBe(false); + // Our record is gone, and the flag this feature switched on is off again. + expect(await readCustomModelHosts(getDataDir())).toEqual([]); + expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8'))).toEqual({ + tmuxHistoryLimit: 5000, + customModelEndpointsEnabled: false, + }); + expect((await readLocalModelState()).state).toBe('idle'); + // The chooser is back, with the runtime download in the total again. + const again = await get(); + expect(again.install.runtime).toMatchObject({ installed: false, downloadBytes: world.archive.archive.size }); + expect(again.recommendation.pick?.installBytes).toBe(pick.installBytes); + }, 60_000); + + it('a runtime download that arrives damaged is deleted before anything is unpacked, and Try again gets it', async () => { + const { get, settled, install: clickInstall, app } = await routes(); + const pick = (await get()).recommendation.pick!; + const runtime = prepareRuntime(pick); + world.damagedDownloads = 1; + + await clickInstall(pick); + const failed = await settled(); + expect(failed).toMatchObject({ + state: 'failed', + step: 'runtime', + error: { code: 'runtime-damaged', next: 'retry' }, + actions: ['resume', 'install', 'remove'], + runtime: { installed: false }, + }); + expect(failed.error?.detail).toContain(world.archive.archive.sha256); + // The hash was compared first: the file is gone, nothing was unpacked, nothing was started. + expect(world.archive.state.archiveRequests).toBe(1); + expect(existsSync(world.paths.archivePartial)).toBe(false); + expect(existsSync(world.paths.unpackDir)).toBe(false); + expect(existsSync(world.paths.runtimeDir)).toBe(false); + expect(runtime.state.children).toHaveLength(0); + expect(runtime.state.pulls).toBe(0); + + // "Try again" downloads the archive again, and this time it is the pinned one. + const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} }); + expect(resumed.statusCode).toBe(200); + expect((await settled()).state).toBe('installed'); + expect(world.archive.state.archiveRequests).toBe(2); + expect(existsSync(world.paths.binary)).toBe(true); + expect(runtime.state.children).toHaveLength(1); + }, 60_000); + + it('a model download that fails twice is asked again and finishes; Remove leaves another saved endpoint and the models', async () => { + writeFileSync(HOSTS(), JSON.stringify([{ id: 'office', label: 'Office', baseUrl: 'http://192.168.1.50:8080' }])); + const { app, get, settled, install: clickInstall } = await routes(); + const pick = (await get()).recommendation.pick!; + const runtime = prepareRuntime(pick); + // The real error line of a pull whose download kept failing, on the first two pulls. + runtime.switches.pull.fail = 'error-line'; + runtime.switches.pull.failTimes = 2; + + await clickInstall(pick); + const install = await settled(); + expect(install.error).toBeUndefined(); + expect(install.state).toBe('installed'); + expect(runtime.state.pulls).toBe(3); + // The size of the model is the pull's own total. + expect((await readLocalModelState()).model?.pullBytes).toBe(runtime.model.bytes); + expect(await readCustomModelHosts(getDataDir())).toMatchObject([{ id: 'office' }, { id: 'local-ollama' }]); + + // Remove without the box: the model store stays for a later install. + const removed = await app.inject({ method: 'DELETE', url: '/api/local-model', payload: { removeModel: false } }); + expect(removed.json().data.state).toBe('idle'); + expect(existsSync(world.paths.runtimeRoot)).toBe(false); + expect(existsSync(world.paths.homeDir)).toBe(false); + expect(existsSync(world.paths.modelsDir)).toBe(true); + // The other endpoint stays, and so does the flag it needs, although this feature switched it on. + expect(await readCustomModelHosts(getDataDir())).toEqual([ + { id: 'office', label: 'Office', baseUrl: 'http://192.168.1.50:8080' }, + ]); + expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).customModelEndpointsEnabled).toBe(true); + }, 60_000); + + it('a model download that keeps failing ends failed after its retries, and Try again continues it', async () => { + const { app, get, settled, install: clickInstall } = await routes(); + const pick = (await get()).recommendation.pick!; + const runtime = prepareRuntime(pick); + runtime.switches.pull.fail = 'error-line'; + + await clickInstall(pick); + const failed = await settled(); + expect(failed).toMatchObject({ + state: 'failed', + step: 'model', + error: { code: 'pull-failed', next: 'retry', detail: 'max retries exceeded: local error: tls: bad record MAC' }, + actions: ['resume', 'install', 'remove'], + }); + // The first pull and every retry, and no more. + expect(runtime.state.pulls).toBe(1 + PULL_RETRIES); + // The download as it stood, in the pull's own figures. + expect(failed.progress?.totalBytes).toBe(runtime.model.bytes); + expect(failed.progress?.completedBytes).toBeGreaterThan(0); + expect(failed.progress?.completedBytes).toBeLessThan(runtime.model.bytes); + // Nothing was checked, installed or connected. + expect(runtime.state.installed).toBe(false); + expect(world.installerRuns).toBe(0); + expect(existsSync(HOSTS())).toBe(false); + expect(existsSync(SETTINGS_PATH)).toBe(false); + + // The network is fine again: "Try again" enters the same step. + runtime.switches.pull.fail = 'none'; + const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} }); + expect(resumed.json().data).toMatchObject({ state: 'installing', step: 'model' }); + expect((await settled()).state).toBe('installed'); + expect(runtime.state.pulls).toBe(2 + PULL_RETRIES); + // The runtime was neither downloaded nor started a second time. + expect(world.archive.state.archiveRequests).toBe(1); + expect(runtime.state.children).toHaveLength(1); + }, 60_000); + + it('a model that does not call a tool ends failed with the next model to try, and Remove cleans up', async () => { + const { app, get, settled, install: clickInstall } = await routes(); + const pick = (await get()).recommendation.pick!; + const runtime = prepareRuntime(pick); + runtime.switches.tool = 'none'; + + await clickInstall(pick); + const install = await settled(); + expect(install).toMatchObject({ + state: 'failed', + step: 'verify', + error: { code: 'no-tool-call', next: 'next-model', message: { vars: { model: pick.modelLabel } } }, + actions: ['resume', 'install', 'remove'], + }); + // The model to try next is a row of the list the window has. + const offered = (await get()).recommendation; + expect(install.error?.nextModel?.modelId).not.toBe(pick.modelId); + expect(offered.alternatives.map((row) => row.modelId)).toContain(install.error?.nextModel?.modelId); + // The coding agent was not installed and nothing was connected. + expect(world.installerRuns).toBe(0); + expect(existsSync(HOSTS())).toBe(false); + expect(existsSync(SETTINGS_PATH)).toBe(false); + + // "Try again" with a model that now calls the tool finishes the same job. + runtime.switches.tool = 'call'; + const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} }); + expect(resumed.statusCode).toBe(200); + expect((await settled()).state).toBe('installed'); + // The runtime was not downloaded again and the model was not pulled again. + expect(world.archive.state.archiveRequests).toBe(1); + expect(runtime.state.pulls).toBe(1); + + const removed = await app.inject({ method: 'DELETE', url: '/api/local-model' }); + expect(removed.json().data.state).toBe('idle'); + // This feature switched the flag on and no other endpoint is saved: it is off again. + expect(JSON.parse(readFileSync(SETTINGS_PATH, 'utf8')).customModelEndpointsEnabled).toBe(false); + // The model was not asked to go: the store stays for a later install. + expect(existsSync(world.paths.runtimeDir)).toBe(false); + expect(existsSync(world.paths.modelsDir)).toBe(true); + }, 60_000); + + it('Cancel during the model download pauses the job, nothing goes on by itself, and Continue finishes it', async () => { + const { app, get, settled, install: clickInstall, pulling } = await routes(); + const pick = (await get()).recommendation.pick!; + const runtime = prepareRuntime(pick); + slowPull(runtime); + + await clickInstall(pick); + // The window sees the model download move. + await pulling(); + expect((await get()).install).toMatchObject({ state: 'installing', step: 'model' }); + + const cancelled = await app.inject({ method: 'POST', url: '/api/local-model/cancel' }); + expect(cancelled.statusCode).toBe(200); + const paused = cancelled.json().data as LocalModelInstallView; + expect(paused).toMatchObject({ state: 'paused', step: 'model', actions: ['resume', 'install', 'remove'] }); + // The bytes as they stood, and the pull's own total. + expect(paused.progress?.totalBytes).toBe(runtime.model.bytes); + expect(paused.progress?.completedBytes).toBeLessThan(runtime.model.bytes); + expect(paused.progress).not.toHaveProperty('bytesPerSecond'); + expect(runtime.state.installed).toBe(false); + + // Nothing goes on by itself. + const pullsAfterCancel = runtime.state.pulls; + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(runtime.state.pulls).toBe(pullsAfterCancel); + expect((await get()).install.state).toBe('paused'); + + quickPull(runtime); + const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume' }); + expect(resumed.json().data.state).toBe('installing'); + expect((await settled()).state).toBe('installed'); + expect(runtime.state.pulls).toBe(pullsAfterCancel + 1); + }, 60_000); + + it('a server restart during the model download comes back paused, waits for a click, and Continue finishes it', async () => { + const { app, get, settled, install: clickInstall, pulling } = await routes(); + const pick = (await get()).recommendation.pick!; + const runtime = prepareRuntime(pick); + slowPull(runtime); + + await clickInstall(pick); + await pulling(); + + // ---- The server stops ---- + const first = world.service; + await first.shutdown(); + // The runtime child was signalled and no longer listens; the download stopped with it. + expect(runtime.state.children).toHaveLength(1); + expect(runtime.state.children[0].signals).toContain('SIGTERM'); + expect(runtime.listening).toBe(false); + expect(runtime.state.installed).toBe(false); + // Stopping writes nothing: the file still says what the job was doing. + expect(await readLocalModelState()).toMatchObject({ state: 'installing', step: 'model' }); + const pullsBefore = runtime.state.pulls; + + // ---- A new server starts over the same data dir ---- + const second = startServer(); + expect(second).not.toBe(first); + await second.boot(); + const paused = (await get()).install; + expect(paused).toMatchObject({ + state: 'paused', + step: 'model', + steps: ['runtime', 'model', 'verify', 'agent', 'connect'], + model: { id: pick.modelId, label: pick.modelLabel }, + cli: 'opencode', + context: pick.context, + port: world.port, + running: false, + runtime: { installed: true, downloadBytes: 0 }, + actions: ['resume', 'install', 'remove'], + }); + expect(paused.error).toBeUndefined(); + expect(await readLocalModelState()).toMatchObject({ state: 'paused', step: 'model' }); + + // It never continues without a click: no runtime is started, nothing is pulled. + await new Promise((resolve) => setTimeout(resolve, 150)); + expect(runtime.state.children).toHaveLength(1); + expect(runtime.state.pulls).toBe(pullsBefore); + expect((await get()).install.state).toBe('paused'); + + // ---- Continue ---- + quickPull(runtime); + const resumed = await app.inject({ method: 'POST', url: '/api/local-model/resume', payload: {} }); + expect(resumed.statusCode).toBe(200); + expect(resumed.json().data.state).toBe('installing'); + const install = await settled(); + expect(install.error).toBeUndefined(); + expect(install).toMatchObject({ state: 'installed', running: true, endpointId: 'local-ollama' }); + // The new server started the runtime itself and pulled once more; the archive was not fetched again. + expect(runtime.state.children).toHaveLength(2); + expect(runtime.state.pulls).toBe(pullsBefore + 1); + expect(world.archive.state.archiveRequests).toBe(1); + expect(world.installerRuns).toBe(1); + expect(await readCustomModelHosts(getDataDir())).toMatchObject([{ id: 'local-ollama', models: [pick.modelId] }]); + }, 60_000); +});