diff --git a/src/local-model/paths.ts b/src/local-model/paths.ts index 9933d7ab..749ece9a 100644 --- a/src/local-model/paths.ts +++ b/src/local-model/paths.ts @@ -75,6 +75,19 @@ export function localModelPaths(version: string): LocalModelPaths { }; } +/** + * The binary of another unpacked runtime version beside the pinned one: `/ + * ollama-/ollama`. For an installed model whose runtime was unpacked by an earlier + * build with another pin: it keeps running the version it was installed with. Throws on a + * version that is not digits and dots, like `localModelPaths()`. Pure. + */ +export function runtimeBinaryForVersion(paths: LocalModelPaths, version: string): string { + if (!VERSION_PATTERN.test(version)) { + throw new Error(`Not a runtime version: ${JSON.stringify(version)}`); + } + return join(paths.runtimeRoot, `ollama-${version}`, 'ollama'); +} + /** * Whether `path` is the local model folder itself or lies inside it, judged on the resolved * text (no filesystem access, so a symlink is not followed: a caller that deletes checks the diff --git a/src/local-model/runtime.ts b/src/local-model/runtime.ts index dd6b70f5..0ff00614 100644 --- a/src/local-model/runtime.ts +++ b/src/local-model/runtime.ts @@ -29,12 +29,20 @@ * loses nothing and gives back what an orphan cannot: its log, so the measured budget, and * its exit. A child that exits by itself is restarted, at most `RUNTIME_MAX_RESTARTS` times * since the last `start()`; `stop()` sends SIGTERM. The supervisor does not write the state - * file: it reports through `onChange`, and its caller stores the pid and the port. + * file: it reports through `onChange`, and its caller stores the pid and the port. The pid + * is reported the moment the child is SPAWNED, before it answers, so a server that is + * killed during a start still leaves a pid that names the runtime as ours. `stop()` takes + * that stored pid too (`previousPid`): a runtime an earlier server left behind is ended the + * same way, when its command line is a binary under our runtime folder. `start()` runs the + * pinned version, or with `version` another one that is unpacked beside it (the version an + * installed model was installed with). * - BUDGET (`parseInferenceComputeTotal()`, pure). Right after it starts, the runtime prints * the device total in its log line "inference compute" (`total="37.4 GiB"`). The supervisor * reads the child's output for it and reports it as `measuredTotalBytes`. * - REMOVAL (`removeLocalModelFiles()`). Deletes the runtime, the private home and, when - * asked, the models. Only inside `LocalModelPaths.root`. + * asked, the models. Only inside `LocalModelPaths.root`, and never through a `root` that + * is itself a symbolic link. `removePartialModelBlobs()` deletes what an interrupted + * download left in the model store. * * Under vitest nothing spawns and nothing is fetched unless a test injects it: the default * `spawn` and `fetch` of the dependency objects refuse there, the default `commandOfPid` @@ -44,12 +52,12 @@ import { execFile, spawn as nodeSpawn, type ChildProcess, type SpawnOptions } from 'node:child_process'; import { createHash, type Hash } from 'node:crypto'; import { createReadStream } from 'node:fs'; -import { lstat, mkdir, open, realpath, rename, rm, rmdir, stat, unlink } from 'node:fs/promises'; +import { lstat, mkdir, open, readdir, realpath, rename, rm, rmdir, stat, unlink } from 'node:fs/promises'; import { createServer } from 'node:net'; import { join, resolve as resolvePath } from 'node:path'; import type { LocalModelErrorCode, LocalModelRuntimePin } from '../types.js'; import { getRuntimeVersion, listRunning } from './ollama-client.js'; -import { isInsideLocalModelRoot, localModelPaths, type LocalModelPaths } from './paths.js'; +import { isInsideLocalModelRoot, localModelPaths, runtimeBinaryForVersion, type LocalModelPaths } from './paths.js'; // ---------- The pin and the constants ---------- @@ -673,15 +681,37 @@ async function planRemoval(target: string, root: string): Promise { return { target, how: 'delete' }; } +/** + * Whether `root` may be deleted in: it is absent (nothing to do) or a real folder. A `root` + * that is itself a symbolic link is REFUSED: every check below compares against the real + * path of the root, so a link in its place would make whatever it points at pass as ours. + */ +async function rootIsDeletable(root: string): Promise<'absent' | 'folder'> { + let info; + try { + info = await lstat(root); + } catch (err) { + if (errnoOf(err) === 'ENOENT') return 'absent'; + throw err; + } + if (info.isSymbolicLink()) { + throw new Error(`Refusing to delete in ${root}: it is a symbolic link, not the folder Codeman made`); + } + if (!info.isDirectory()) throw new Error(`Refusing to delete in ${root}: it is not a folder`); + return 'folder'; +} + /** * Deletes what this feature put on disk: every runtime folder and archive, the private home * and, when asked, the models. Refuses any path that is not inside `LocalModelPaths.root`, - * follows no symlink out of it, and removes `root` itself only when it is empty afterwards. - * Every folder is judged BEFORE the first one is deleted, so a refusal deletes nothing. - * The state file and the coding agent are not its business. + * refuses a `root` that is a symbolic link, follows no symlink out of it, and removes `root` + * itself only when it is empty afterwards. Every folder is judged BEFORE the first one is + * deleted, so a refusal deletes nothing. The state file and the coding agent are not its + * business. */ export async function removeLocalModelFiles(options: RemoveLocalModelFilesOptions): Promise { const paths = options.paths ?? localModelPaths(OLLAMA_RUNTIME_PIN.version); + if ((await rootIsDeletable(paths.root)) === 'absent') return; const targets = [paths.runtimeRoot, paths.homeDir, ...(options.models ? [paths.modelsDir] : [])]; const plan: RemovalStep[] = []; for (const target of targets) plan.push(await planRemoval(target, paths.root)); @@ -694,6 +724,48 @@ export async function removeLocalModelFiles(options: RemoveLocalModelFilesOption await rmdir(paths.root).catch(() => undefined); } +/** The marker the runtime puts into the name of a blob it has not finished downloading. */ +const PARTIAL_BLOB_MARK = '-partial'; + +/** + * Deletes what an interrupted download left in the model store: the files directly in + * `/blobs` whose name carries the runtime's partial marker (`sha256--partial` + * and its numbered parts). A model the runtime has no manifest for cannot be deleted through + * the runtime (it answers "not found" and touches no blob), so its partial blobs are removed + * here. Call it only while NO download is in flight: every partial file is then a dead one. + * + * Only regular files, only that one folder, only inside `LocalModelPaths.root`; a `root`, a + * model store or a blob folder that is a symbolic link is left alone. Resolves with the bytes + * that were freed. Never throws: a file that cannot be deleted stays for Remove. + */ +export async function removePartialModelBlobs(paths?: LocalModelPaths): Promise { + let freed = 0; + try { + const resolved = paths ?? localModelPaths(OLLAMA_RUNTIME_PIN.version); + const blobs = join(resolved.modelsDir, 'blobs'); + if (!isInsideLocalModelRoot(blobs, resolved.root)) return 0; + for (const folder of [resolved.root, resolved.modelsDir, blobs]) { + const info = await lstat(folder); + if (info.isSymbolicLink() || !info.isDirectory()) return 0; + } + for (const name of await readdir(blobs)) { + if (!name.includes(PARTIAL_BLOB_MARK)) continue; + const file = join(blobs, name); + try { + const info = await lstat(file); + if (!info.isFile()) continue; + await unlink(file); + freed += info.size; + } catch { + // Gone already, or not ours to delete: Remove with the models takes the rest. + } + } + } catch { + // No model store yet: nothing was left behind. + } + return freed; +} + // ---------- Port ---------- /** Whether nothing holds `port` on 127.0.0.1, decided by binding it and letting go again. */ @@ -743,6 +815,11 @@ export async function chooseRuntimePort(options: ChooseRuntimePortOptions = {}): export interface RuntimeStatus { /** The child Codeman started (or adopted) is alive and answered `/api/version`. */ running: boolean; + /** + * The pid of the child. Present from the moment it is SPAWNED, so also while `running` is + * still false: the caller stores it at once, and a server killed during a start leaves a + * pid that names the runtime as ours. + */ pid?: number; port?: number; /** The context length the running child was started with. */ @@ -762,6 +839,22 @@ export interface RuntimeStartOptions { contextLength: number; /** The pid the state file holds from an earlier server: decides whether a listener on `port` is our own orphan. */ previousPid?: number; + /** + * Another unpacked runtime version than the pinned one: the version an installed model was + * installed with. Its binary is `runtimeBinaryForVersion()`, and it must answer this + * version. Absent: the pinned version. + */ + version?: string; +} + +/** What `stop()` may be told. */ +export interface RuntimeStopOptions { + /** + * The pid the state file holds. When no child of this process runs under it and its command + * line is a binary under our runtime folder, it is a runtime an earlier server left behind, + * and it is ended like our own child (SIGTERM, then SIGKILL). Any other pid is left alone. + */ + previousPid?: number; } /** How a `start()` ended. */ @@ -819,8 +912,12 @@ export interface RuntimeSupervisor { * and the budget line was read (or a few seconds passed without it). Never throws. */ start(options: RuntimeStartOptions): Promise; - /** SIGTERM to the child, SIGKILL after a grace period. Resolves once it is gone. Safe to call when nothing runs. */ - stop(): Promise; + /** + * SIGTERM to the child, SIGKILL after a grace period. Resolves once it is gone. Safe to call + * when nothing runs. With `previousPid` it also ends a runtime an earlier server left behind + * (see `RuntimeStopOptions`), so no caller deletes the runtime folder under a live process. + */ + stop(options?: RuntimeStopOptions): Promise; status(): RuntimeStatus; } @@ -884,7 +981,9 @@ export function createRuntimeSupervisor(overrides: Partial { + async function launch(port: number, contextLength: number, version: string, gen: number): Promise { const folders = paths(); let spawned: ChildProcess; try { + // The pinned build, or the one an installed model was installed with. + const binary = version === pin.version ? folders.binary : runtimeBinaryForVersion(folders, version); await mkdir(folders.homeDir, { recursive: true, mode: 0o700 }); await mkdir(folders.modelsDir, { recursive: true, mode: 0o700 }); // Asked to stop in the meantime: nothing is started only to be stopped again. if (gen !== generation) return { ok: false, detail: 'The start was replaced by a newer request' }; - spawned = deps.spawn(folders.binary, ['serve'], { + spawned = deps.spawn(binary, ['serve'], { env: runtimeEnv({ source: deps.env, home: folders.homeDir, @@ -1022,6 +1123,9 @@ export function createRuntimeSupervisor(overrides: Partial => { if (starting === spawned) starting = undefined; await stopChild(spawned); + // The pid that was announced at the spawn is no runtime any more. + if (state.pid === spawned.pid && child !== spawned) setState({ pid: undefined }); const log = tail.join('\n'); return { ok: false, detail: clip(log === '' ? reason : `${reason}\n${log}`) }; }; @@ -1104,10 +1210,10 @@ export function createRuntimeSupervisor(overrides: Partial= deadline) { @@ -1123,6 +1229,7 @@ export function createRuntimeSupervisor(overrides: Partial { @@ -1181,6 +1289,7 @@ export function createRuntimeSupervisor(overrides: Partial { const { port, contextLength, previousPid } = options; + const version = options.version ?? pin.version; if (!Number.isInteger(port) || port < 1 || port > 65535) { return { ok: false, code: 'runtime-start-failed', detail: `Not a port: ${String(port)}` }; } @@ -1189,7 +1298,13 @@ export function createRuntimeSupervisor(overrides: Partial undefined)) ? previousPid : undefined; - const version = await getRuntimeVersion(target); + const answering = await getRuntimeVersion(target); let orphan: 'none' | 'adopted' | 'replaced' = 'none'; if (ownOrphan !== undefined) { - if (version === pin.version && (await worthAdopting(target, contextLength))) { - adopt(ownOrphan, port, contextLength); + if (answering === version && (await worthAdopting(target, contextLength))) { + adopt(ownOrphan, port, contextLength, version); return { ok: true, pid: ownOrphan, port, orphan: 'adopted' }; } await stopOrphan(ownOrphan); orphan = 'replaced'; - } else if (version !== undefined) { + } else if (answering !== undefined) { return { ok: false, code: 'port-in-use', - detail: `A model runtime that Codeman did not start answers on port ${port} (version ${version})`, + detail: `A model runtime that Codeman did not start answers on port ${port} (version ${answering})`, }; } if (!(await portFreeWithin(port, orphan === 'replaced' ? STOP_GRACE_MS : 0))) { return { ok: false, code: 'port-in-use', detail: `Another program holds port ${port}` }; } - const launched = await launch(port, contextLength, gen); + const launched = await launch(port, contextLength, version, gen); if (!launched.ok) return { ok: false, code: 'runtime-start-failed', detail: launched.detail }; return { ok: true, @@ -1253,11 +1369,18 @@ export function createRuntimeSupervisor(overrides: Partial { + async function doStop(options: RuntimeStopOptions = {}): Promise { generation += 1; wanted = undefined; await restarting; await stopCurrent(); + runningVersion = undefined; + // A runtime an earlier server left behind: not a child of this process, known only by the + // pid the state file kept. Ended when (and only when) its command line is our own binary. + const { previousPid } = options; + if (previousPid !== undefined && Number.isInteger(previousPid) && previousPid > 1) { + if (isOwn(await deps.commandOfPid(previousPid).catch(() => undefined))) await stopOrphan(previousPid); + } setState({ running: false, pid: undefined, @@ -1274,10 +1397,10 @@ export function createRuntimeSupervisor(overrides: Partial doStart(options)).catch( (err): RuntimeStartResult => ({ ok: false, code: 'runtime-start-failed', detail: errorText(err) }) ), - stop: () => { + stop: (options) => { // At once, not when its turn comes: a start that is still waiting for the runtime gives up. generation += 1; - return serialize(doStop).catch(() => undefined); + return serialize(() => doStop(options)).catch(() => undefined); }, status: () => ({ ...state }), }; diff --git a/test/local-model-runtime.test.ts b/test/local-model-runtime.test.ts index 26886e5b..e6ecbc76 100644 --- a/test/local-model-runtime.test.ts +++ b/test/local-model-runtime.test.ts @@ -37,6 +37,7 @@ import { OLLAMA_RUNTIME_PIN, parseInferenceComputeTotal, removeLocalModelFiles, + removePartialModelBlobs, RUNTIME_DOWNLOAD_RETRIES, RUNTIME_KEEP_ALIVE, RUNTIME_MAX_RESTARTS, @@ -935,6 +936,42 @@ describe('removeLocalModelFiles', () => { expect(await readFile(join(outside, 'precious.gguf'), 'utf8')).toBe('the only copy'); }); + it('refuses a local model folder that is itself a symlink, and deletes nothing behind it', async () => { + // /local-model is a link to a folder somewhere else that happens to hold the + // three names this feature deletes. + const elsewhere = await outsideDir(); + await mkdir(join(elsewhere, 'home'), { recursive: true }); + await writeFile(join(elsewhere, 'home', 'thesis.txt'), 'the only copy'); + await mkdir(join(elsewhere, 'runtime'), { recursive: true }); + await mkdir(join(elsewhere, 'models', 'blobs'), { recursive: true }); + await writeFile(join(elsewhere, 'models', 'blobs', 'sha256-abc'), 'weights'); + await mkdir(join(paths.root, '..'), { recursive: true }); + await symlink(elsewhere, paths.root); + try { + await expect(removeLocalModelFiles({ models: true, paths })).rejects.toThrow(/symbolic link/); + expect(await readFile(join(elsewhere, 'home', 'thesis.txt'), 'utf8')).toBe('the only copy'); + expect(await readFile(join(elsewhere, 'models', 'blobs', 'sha256-abc'), 'utf8')).toBe('weights'); + expect(existsSync(join(elsewhere, 'runtime'))).toBe(true); + // The partial blobs are not deleted through the link either. + await writeFile(join(elsewhere, 'models', 'blobs', 'sha256-def-partial'), 'half'); + expect(await removePartialModelBlobs(paths)).toBe(0); + expect(existsSync(join(elsewhere, 'models', 'blobs', 'sha256-def-partial'))).toBe(true); + } finally { + // The link itself, never what it points at. + await rm(paths.root, { force: true }); + } + }); + + it('refuses a local model folder that is a file', async () => { + await mkdir(join(paths.root, '..'), { recursive: true }); + await writeFile(paths.root, 'not a folder'); + try { + await expect(removeLocalModelFiles({ models: true, paths })).rejects.toThrow(/not a folder/); + } finally { + await rm(paths.root, { force: true }); + } + }); + it('removes a symlink it meets inside a folder, not the tree behind it', async () => { const outside = await outsideDir(); await writeFile(join(outside, 'keep.txt'), 'keep'); @@ -980,3 +1017,54 @@ describe('removeLocalModelFiles', () => { expect(await readFile(join(outside, 'runtime', 'keep.txt'), 'utf8')).toBe('keep'); }); }); + +describe('removePartialModelBlobs', () => { + const blob = (name: string): string => join(paths.modelsDir, 'blobs', name); + + it('deletes what an interrupted download left, and keeps every finished blob and the manifests', async () => { + await mkdir(join(paths.modelsDir, 'blobs'), { recursive: true }); + await mkdir(join(paths.modelsDir, 'manifests', 'registry.ollama.ai'), { recursive: true }); + await writeFile(blob('sha256-aaa'), 'finished weights'); + await writeFile(blob('sha256-bbb-partial'), '12345'); + await writeFile(blob('sha256-bbb-partial-0'), '678'); + await writeFile(blob('sha256-bbb-partial-1'), '9'); + await writeFile(join(paths.modelsDir, 'manifests', 'registry.ollama.ai', 'latest'), '{}'); + + // The bytes that were freed: 5 + 3 + 1. + expect(await removePartialModelBlobs(paths)).toBe(9); + expect(existsSync(blob('sha256-bbb-partial'))).toBe(false); + expect(existsSync(blob('sha256-bbb-partial-0'))).toBe(false); + expect(existsSync(blob('sha256-bbb-partial-1'))).toBe(false); + expect(await readFile(blob('sha256-aaa'), 'utf8')).toBe('finished weights'); + expect(existsSync(join(paths.modelsDir, 'manifests', 'registry.ollama.ai', 'latest'))).toBe(true); + }); + + it('is fine without a model store, and deletes nothing a second time', async () => { + expect(await removePartialModelBlobs(paths)).toBe(0); + await mkdir(join(paths.modelsDir, 'blobs'), { recursive: true }); + await writeFile(blob('sha256-ccc-partial'), 'xx'); + expect(await removePartialModelBlobs(paths)).toBe(2); + expect(await removePartialModelBlobs(paths)).toBe(0); + }); + + it('touches only regular files directly in the blob folder', async () => { + const outside = await outsideDir(); + await writeFile(join(outside, 'keep-partial'), 'keep'); + await mkdir(join(paths.modelsDir, 'blobs', 'a-partial-folder'), { recursive: true }); + await writeFile(join(paths.modelsDir, 'blobs', 'a-partial-folder', 'inside-partial'), 'keep'); + // A link whose name carries the marker: the link stays, and so does what it points at. + await symlink(join(outside, 'keep-partial'), blob('sha256-ddd-partial')); + expect(await removePartialModelBlobs(paths)).toBe(0); + expect(await readFile(join(outside, 'keep-partial'), 'utf8')).toBe('keep'); + expect(existsSync(join(paths.modelsDir, 'blobs', 'a-partial-folder', 'inside-partial'))).toBe(true); + }); + + it('leaves a blob folder alone that is a symlink to somewhere else', async () => { + const outside = await outsideDir(); + await writeFile(join(outside, 'sha256-eee-partial'), "half of somebody else's model"); + await mkdir(paths.modelsDir, { recursive: true }); + await symlink(outside, join(paths.modelsDir, 'blobs')); + expect(await removePartialModelBlobs(paths)).toBe(0); + expect(existsSync(join(outside, 'sha256-eee-partial'))).toBe(true); + }); +}); diff --git a/test/local-model-supervisor.test.ts b/test/local-model-supervisor.test.ts index e51e4049..b3e883ad 100644 --- a/test/local-model-supervisor.test.ts +++ b/test/local-model-supervisor.test.ts @@ -25,7 +25,7 @@ import { rm } from 'node:fs/promises'; import { createServer, type Server } from 'node:net'; import type { ChildProcess, SpawnOptions } from 'node:child_process'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { localModelPaths, type LocalModelPaths } from '../src/local-model/paths.js'; +import { localModelPaths, runtimeBinaryForVersion, type LocalModelPaths } from '../src/local-model/paths.js'; import { createRuntimeSupervisor, RUNTIME_MAX_RESTARTS, @@ -794,3 +794,112 @@ describe('stop', () => { expect(process.listenerCount('exit')).toBe(before); }); }); + +// ---------- A runtime an earlier server left behind, and the pid that names it ---------- + +describe('the pid is told the moment the child is spawned', () => { + it('reports it before the child answers, so a server killed during the start leaves a pid', async () => { + // The child never answers: the start waits, and is told to stop. + const fake = fakeRuntime({ spawn: { neverListens: true } }); + const { supervisor, changes } = supervise(fake, { + startTimeoutMs: 60_000, + sleep: () => new Promise((resolve) => setTimeout(resolve, 5)), + }); + const starting = supervisor.start({ port, contextLength: CONTEXT }); + await until(() => children(fake).length === 1, 'the child to be spawned'); + await until(() => changes.some((status) => status.pid === children(fake)[0].pid), 'the pid to be reported'); + const early = changes.find((status) => status.pid === children(fake)[0].pid); + // Not running yet: it has not answered. + expect(early).toMatchObject({ running: false, pid: children(fake)[0].pid }); + await supervisor.stop(); + await starting; + // A start that failed leaves no pid behind. + expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false }); + }); +}); + +describe('stop with the pid the state file kept', () => { + it('ends a runtime of ours that this process never started', async () => { + let gone = false; + const fake = fakeRuntime(); + const { supervisor, kills } = supervise(fake, { + commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined), + kill: (pid, signal) => { + kills.push([pid, signal]); + gone = true; + }, + }); + await supervisor.stop({ previousPid: ORPHAN_PID }); + expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]); + // Nothing was spawned to do it. + expect(children(fake)).toHaveLength(0); + }); + + it('ends one that runs another version of ours: the folder decides, not the version', async () => { + let gone = false; + const fake = fakeRuntime(); + const { supervisor, kills } = supervise(fake, { + commandOfPid: async (pid) => + pid === ORPHAN_PID && !gone ? `${runtimeBinaryForVersion(paths, '0.39.2')} serve` : undefined, + kill: (pid, signal) => { + kills.push([pid, signal]); + gone = true; + }, + }); + await supervisor.stop({ previousPid: ORPHAN_PID }); + expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]); + }); + + it('never signals a pid that runs something else now, nor one that is gone', async () => { + const fake = fakeRuntime(); + const { supervisor, kills } = supervise(fake, { + commandOfPid: async (pid) => (pid === ORPHAN_PID ? '/usr/local/bin/ollama serve' : undefined), + }); + await supervisor.stop({ previousPid: ORPHAN_PID }); + await supervisor.stop({ previousPid: ORPHAN_PID + 1 }); + await supervisor.stop({ previousPid: 1 }); + await supervisor.stop({ previousPid: Number.NaN }); + expect(kills).toEqual([]); + }); + + it('stops its own child first, and the stored pid only when it is somebody else', async () => { + const fake = fakeRuntime(); + const { supervisor, kills } = supervise(fake); + const started = await supervisor.start({ port, contextLength: CONTEXT }); + expect(started.ok).toBe(true); + await supervisor.stop({ previousPid: children(fake)[0].pid }); + expect(children(fake)[0].signals).toEqual(['SIGTERM']); + // The child is gone by then: its pid names nothing of ours any more. + expect(kills).toEqual([]); + }); +}); + +describe('start with the version an installed model was installed with', () => { + it('runs that version from its own folder and expects it to answer as that version', async () => { + const fake = fakeRuntime({ version: '0.39.2' }); + const { supervisor } = supervise(fake, { pin: { ...fake.pin(), version: '0.40.15' } }); + const result = await supervisor.start({ port, contextLength: CONTEXT, version: '0.39.2' }); + expect(result).toMatchObject({ ok: true }); + expect(children(fake)[0].command).toBe(runtimeBinaryForVersion(paths, '0.39.2')); + // Asked again the same way: the child is kept. Asked for the pinned version: replaced. + await supervisor.start({ port, contextLength: CONTEXT, version: '0.39.2' }); + expect(children(fake)).toHaveLength(1); + }); + + it('a restart after a crash starts the same version again', async () => { + const fake = fakeRuntime({ version: '0.39.2' }); + const { supervisor } = supervise(fake, { pin: { ...fake.pin(), version: '0.40.15' } }); + await supervisor.start({ port, contextLength: CONTEXT, version: '0.39.2' }); + children(fake)[0].crash(); + await until(() => children(fake).length === 2 && supervisor.status().running, 'the restart'); + expect(children(fake)[1].command).toBe(runtimeBinaryForVersion(paths, '0.39.2')); + }); + + it('refuses a version that is no version before anything is spawned', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + const result = await supervisor.start({ port, contextLength: CONTEXT, version: '../../bin' }); + expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' }); + expect(children(fake)).toHaveLength(0); + }); +});