diff --git a/src/local-model/developer-tools.ts b/src/local-model/developer-tools.ts index 65e50c63..c7f7446b 100644 --- a/src/local-model/developer-tools.ts +++ b/src/local-model/developer-tools.ts @@ -1,9 +1,6 @@ /** * @fileoverview Whether this Mac has the developer tools a coding agent calls (git, Python). * - * CONTRACT STAGE: the function below has its final signature and a body that throws; the body - * is written next. - * * Why it exists: on a Mac without Xcode or the Command Line Tools, `git` and `python3` are * stubs that open macOS's "install the command line developer tools" window. A coding agent * runs both, so the install window says one honest sentence when the tools are missing @@ -15,19 +12,24 @@ * - macOS only. Anywhere else, and when the command does not answer in time, the answer is * `unknown`, and nothing is said. * - Never throws. The answer is cached for a short while, so a window that polls once a - * second does not spawn once a second. + * second does not spawn once a second. Only `present` and `missing` are cached: an + * `unknown` is asked again, because it says the check itself did not work. + * - Under vitest the default `run` spawns nothing and answers "did not start", so a test + * that wants an answer injects its own. */ +import { execFile } from 'node:child_process'; import type { LocalModelDeveloperTools } from '../types.js'; -/** Marks a body the contract stage left unwritten. */ -function notImplemented(what: string, ..._args: unknown[]): never { - throw new Error(`${what} is not implemented yet`); -} - /** How long `/usr/bin/xcode-select -p` may take. */ export const DEVELOPER_TOOLS_TIMEOUT_MS = 2_000; +/** How long an answer is reused. Installing the tools takes minutes, so a minute of staleness is not seen. */ +const DEVELOPER_TOOLS_CACHE_MS = 60_000; + +/** The one command this module runs. It prints the active developer directory and exits 0, or exits with an error when there is none. */ +const XCODE_SELECT = '/usr/bin/xcode-select'; + /** What the check calls. Each member defaults to the real thing; a test injects its own. */ export interface DeveloperToolsDeps { /** Default `process.platform`. */ @@ -36,7 +38,80 @@ export interface DeveloperToolsDeps { run: (file: string, args: readonly string[], timeoutMs: number) => Promise; } -/** `present` when `/usr/bin/xcode-select -p` exits 0, `missing` when it exits with anything else, `unknown` otherwise. */ +/** The default `run`: one bounded `execFile`, no shell. Spawns nothing under vitest. */ +function runBounded(file: string, args: readonly string[], timeoutMs: number): Promise { + return new Promise((resolve) => { + if (process.env.VITEST) return resolve(null); + try { + execFile(file, [...args], { timeout: timeoutMs }, (err) => { + if (!err) return resolve(0); + // A number is the exit code of a command that ran. Anything else (ENOENT, a timeout + // that killed it) is "it did not answer". + const code = (err as { code?: unknown }).code; + resolve(typeof code === 'number' && !(err as { killed?: boolean }).killed ? code : null); + }); + } catch { + resolve(null); + } + }); +} + +async function check(deps: DeveloperToolsDeps): Promise { + if (deps.platform !== 'darwin') return 'unknown'; + let code: number | null; + try { + code = await deps.run(XCODE_SELECT, ['-p'], DEVELOPER_TOOLS_TIMEOUT_MS); + } catch { + code = null; + } + if (code === null || !Number.isInteger(code)) return 'unknown'; + return code === 0 ? 'present' : 'missing'; +} + +/** + * A check that remembers its answer for a minute and shares one run among callers that ask at + * the same time. `now` is the clock of the cache; a test injects its own. + */ +export function createDeveloperToolsCheck( + deps: Partial & { now?: () => number } = {} +): () => Promise { + const resolved: DeveloperToolsDeps = { + platform: deps.platform ?? process.platform, + run: deps.run ?? runBounded, + }; + const now = deps.now ?? Date.now; + let cached: { answer: LocalModelDeveloperTools; at: number } | undefined; + let inFlight: Promise | undefined; + return () => { + if (cached && now() - cached.at < DEVELOPER_TOOLS_CACHE_MS) return Promise.resolve(cached.answer); + inFlight ??= check(resolved) + .then((answer) => { + if (answer !== 'unknown') cached = { answer, at: now() }; + return answer; + }) + .finally(() => { + inFlight = undefined; + }); + return inFlight; + }; +} + +/** The check of this server's own Mac. */ +let shared = createDeveloperToolsCheck(); + +/** Forgets the cached answer of this server's own Mac. For tests. */ +export function resetDeveloperToolsCache(): void { + shared = createDeveloperToolsCheck(); +} + +/** + * `present` when `/usr/bin/xcode-select -p` exits 0, `missing` when it exits with anything + * else, `unknown` otherwise. Without `deps` it is the cached check of this server's Mac; with + * a platform or a runner of the caller's it asks once and remembers nothing. + */ export async function checkDeveloperTools(deps: Partial = {}): Promise { - return notImplemented('checkDeveloperTools', deps); + if (deps.platform !== undefined || deps.run !== undefined) { + return check({ platform: deps.platform ?? process.platform, run: deps.run ?? runBounded }); + } + return shared(); } diff --git a/src/local-model/ollama-client.ts b/src/local-model/ollama-client.ts index bf7bb17b..a597c138 100644 --- a/src/local-model/ollama-client.ts +++ b/src/local-model/ollama-client.ts @@ -2,9 +2,6 @@ * @fileoverview The HTTP client of the model runtime: what Codeman asks its own runtime on * the loopback port, and the pure functions that read the answers. * - * CONTRACT STAGE: every function below has its final signature and a body that throws; the - * bodies are written next. - * * What it asks: * - `GET /api/version`, `GET /api/tags`, `GET /api/ps`, `GET /v1/models`, `DELETE /api/delete`. * - `POST /api/pull` with `stream: true`. The answer is one JSON object per line, and a @@ -23,24 +20,46 @@ * Rules: * - Nothing here throws for something a runtime can answer: every outcome is a result with * the job's error code. An abort signal is the result `cancelled`. + * - Every request is bounded: a listing by seconds, the two probes by minutes (the first + * chat loads the model). Only the pull has no deadline of its own, because its length is + * the download; it ends with the stream, and the caller's signal cancels it. * - The runtime does not enforce the context length (a prompt of 167,012 tokens was accepted * at 131072), so nothing here relies on it to. * - Every function takes its `fetch` from the target, so a test passes the fake runtime's - * address and spawns nothing. + * address and spawns nothing. The target is always a loopback address the caller built from + * the stored port: no function here takes a URL from a file or a request. */ import type { LocalModelErrorCode } from '../types.js'; -/** Marks a body the contract stage left unwritten. */ -function notImplemented(what: string, ..._args: unknown[]): never { - throw new Error(`${what} is not implemented yet`); -} - /** How often a pull that ended on an error line or a cut stream is started again. */ export const PULL_RETRIES = 5; /** `max_tokens` of the tool probe. */ export const TOOL_PROBE_MAX_TOKENS = 1024; +/** How long `/api/version` may take: it is asked while a runtime starts, many times. */ +const VERSION_TIMEOUT_MS = 2_000; +/** How long a listing or a delete may take. */ +const LIST_TIMEOUT_MS = 10_000; +/** How long the chat that loads the model may take. A 23 GB model loaded in 5.9 s on the test Mac; a cold disk is slower. */ +const CHAT_PROBE_TIMEOUT_MS = 180_000; +/** How long the tool probe may take. The model is loaded by then. */ +const TOOL_PROBE_TIMEOUT_MS = 120_000; +/** `max_tokens` of the chat that loads the model: its answer is not read. */ +const CHAT_PROBE_MAX_TOKENS = 16; +/** The least time between two progress reports of a pull. */ +const PULL_PROGRESS_INTERVAL_MS = 250; +/** The window the download speed is averaged over. */ +const PULL_SPEED_WINDOW_MS = 5_000; +/** The first wait before a pull is started again; it doubles with every retry. */ +const PULL_RETRY_DELAY_MS = 1_000; +/** A pull line longer than this is not a pull line. It is dropped, so a stream without line breaks cannot grow the buffer. */ +const MAX_PULL_LINE_CHARS = 1024 * 1024; +/** More layers than any model has. Lines for further ones are counted and not added up, so the layer table cannot grow without bound. */ +const MAX_PULL_LAYERS = 1024; +/** How much of a runtime's error text is kept as `detail`. */ +const MAX_DETAIL_CHARS = 400; + /** Which runtime to ask. */ export interface OllamaTarget { /** `http://127.0.0.1:`, without a trailing slash. */ @@ -49,6 +68,100 @@ export interface OllamaTarget { fetch?: typeof fetch; } +// ---------- Plumbing ---------- + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function asString(value: unknown): string | undefined { + return typeof value === 'string' && value !== '' ? value : undefined; +} + +/** A byte count or a token count as the runtime reports it: a finite number that is not negative. */ +function asCount(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined; +} + +function clip(text: string): string { + const trimmed = text.trim(); + return trimmed.length > MAX_DETAIL_CHARS ? `${trimmed.slice(0, MAX_DETAIL_CHARS)}...` : trimmed; +} + +function errorText(err: unknown): string { + if (err instanceof Error) { + // undici wraps the reason ("fetch failed") and keeps the useful part in `cause`. + const cause = (err as { cause?: unknown }).cause; + return clip(cause instanceof Error && cause.message ? `${err.message}: ${cause.message}` : err.message); + } + return clip(String(err)); +} + +/** The error text of a runtime answer: `{"error":"..."}` (its own routes) or `{"error":{"message":"..."}}` (the OpenAI routes). */ +function errorOfBody(body: unknown): string | undefined { + if (!isRecord(body)) return undefined; + const { error } = body; + if (typeof error === 'string') return asString(error); + if (isRecord(error)) return asString(error.message); + return undefined; +} + +/** The caller's signal joined with a deadline. */ +function bounded(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { + const deadline = AbortSignal.timeout(timeoutMs); + return signal ? AbortSignal.any([signal, deadline]) : deadline; +} + +/** An answer of the runtime: its status and its body as JSON (`undefined` when it is not JSON). */ +interface Answer { + status: number; + body: unknown; + text: string; +} + +/** + * One bounded request. `undefined` when nothing answered (refused, timed out, aborted): the + * caller looks at its own signal to tell an abort from the rest. + */ +async function ask( + target: OllamaTarget, + path: string, + init: { method?: string; json?: unknown }, + timeoutMs: number, + signal?: AbortSignal +): Promise { + const fetchFn = target.fetch ?? fetch; + try { + const res = await fetchFn(`${target.baseUrl}${path}`, { + method: init.method ?? 'GET', + ...(init.json !== undefined + ? { headers: { 'content-type': 'application/json' }, body: JSON.stringify(init.json) } + : {}), + signal: bounded(signal, timeoutMs), + }); + const text = await res.text(); + let body: unknown; + try { + body = text === '' ? undefined : JSON.parse(text); + } catch { + body = undefined; + } + return { status: res.status, body, text }; + } catch { + return undefined; + } +} + +/** A digest as it is stored and compared: lowercase hex without the `sha256:` prefix. */ +function normalizeDigest(value: unknown): string { + return typeof value === 'string' + ? value + .trim() + .toLowerCase() + .replace(/^sha256:/, '') + : ''; +} + // ---------- Listings ---------- /** One model of `GET /api/tags`: what is on disk. */ @@ -79,24 +192,69 @@ export interface OllamaPsModel { digest?: string; } +/** The `models` array of a listing, or an empty one. */ +function modelsOf(answer: Answer | undefined): Record[] { + if (!answer || answer.status !== 200 || !isRecord(answer.body)) return []; + const { models } = answer.body; + return Array.isArray(models) ? models.filter(isRecord) : []; +} + /** The runtime's version ('0.40.15'), or `undefined` when nothing answers there or the answer is not a runtime's. */ export async function getRuntimeVersion(target: OllamaTarget, signal?: AbortSignal): Promise { - return notImplemented('getRuntimeVersion', target, signal); + const answer = await ask(target, '/api/version', {}, VERSION_TIMEOUT_MS, signal); + if (!answer || answer.status !== 200 || !isRecord(answer.body)) return undefined; + return asString(answer.body.version); } /** The models on disk. An empty list when the runtime has none or does not answer. */ export async function listTags(target: OllamaTarget, signal?: AbortSignal): Promise { - return notImplemented('listTags', target, signal); + const listed: OllamaTagModel[] = []; + for (const row of modelsOf(await ask(target, '/api/tags', {}, LIST_TIMEOUT_MS, signal))) { + const name = asString(row.name) ?? asString(row.model); + if (!name) continue; + const details = isRecord(row.details) ? row.details : {}; + const format = asString(details.format); + const runner = asString(details.runner) ?? asString(row.runner); + const capabilities = Array.isArray(row.capabilities) + ? row.capabilities.filter((entry): entry is string => typeof entry === 'string') + : undefined; + listed.push({ + name, + size: asCount(row.size) ?? 0, + digest: normalizeDigest(row.digest), + ...(format ? { format } : {}), + ...(runner ? { runner } : {}), + ...(capabilities ? { capabilities } : {}), + }); + } + return listed; } /** The models loaded right now. An empty list when none is or the runtime does not answer. */ export async function listRunning(target: OllamaTarget, signal?: AbortSignal): Promise { - return notImplemented('listRunning', target, signal); + const listed: OllamaPsModel[] = []; + for (const row of modelsOf(await ask(target, '/api/ps', {}, LIST_TIMEOUT_MS, signal))) { + const name = asString(row.name) ?? asString(row.model); + if (!name) continue; + const digest = normalizeDigest(row.digest); + listed.push({ + name, + size: asCount(row.size) ?? 0, + sizeVram: asCount(row.size_vram) ?? 0, + contextLength: asCount(row.context_length) ?? 0, + ...(digest ? { digest } : {}), + }); + } + return listed; } /** The model ids `GET /v1/models` lists, exactly as listed (the runtime answers `data: null` for none). */ export async function listOpenAiModels(target: OllamaTarget, signal?: AbortSignal): Promise { - return notImplemented('listOpenAiModels', target, signal); + const answer = await ask(target, '/v1/models', {}, LIST_TIMEOUT_MS, signal); + if (!answer || answer.status !== 200 || !isRecord(answer.body)) return []; + const { data } = answer.body; + if (!Array.isArray(data)) return []; + return data.flatMap((row) => (isRecord(row) && typeof row.id === 'string' && row.id !== '' ? [row.id] : [])); } /** The digest `GET /api/tags` reports for `modelId`, or `undefined` when the model is not on disk. */ @@ -105,12 +263,21 @@ export async function readModelDigest( modelId: string, signal?: AbortSignal ): Promise { - return notImplemented('readModelDigest', target, modelId, signal); + const found = (await listTags(target, signal)).find((model) => model.name === modelId); + return found && found.digest !== '' ? found.digest : undefined; } /** Deletes a model from the runtime's store. True when it is gone afterwards (also when it never was there). */ export async function deleteModel(target: OllamaTarget, modelId: string, signal?: AbortSignal): Promise { - return notImplemented('deleteModel', target, modelId, signal); + const answer = await ask( + target, + '/api/delete', + { method: 'DELETE', json: { model: modelId } }, + LIST_TIMEOUT_MS, + signal + ); + // 404 is the runtime's answer for a model it does not have. + return answer !== undefined && (answer.status === 200 || answer.status === 404); } // ---------- Pull ---------- @@ -154,10 +321,104 @@ export interface PullParser { * `{"status":"writing manifest"}`, `{"status":"success"}` * - `{"error":"max retries exceeded: local error: tls: bad record MAC"}` * - * A line that is not JSON is skipped and still counted. + * A line that is not JSON is skipped and still counted. A blank line is not a line. */ export function createPullParser(): PullParser { - return notImplemented('createPullParser'); + const decoder = new TextDecoder(); + // One entry per layer: what the layer's last line said. A model has a handful of layers. + const layers = new Map(); + let pending = ''; + // True while a line longer than the limit is being skipped up to its line break. + let skipping = false; + let status: string | undefined; + let succeeded = false; + let error: string | undefined; + let lines = 0; + + function line(text: string): void { + const trimmed = text.trim(); + if (trimmed === '') return; + lines += 1; + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return; + } + if (!isRecord(parsed)) return; + const failure = errorOfBody(parsed); + if (failure !== undefined) { + error ??= clip(failure); + return; + } + const lineStatus = asString(parsed.status); + if (lineStatus !== undefined) { + status = lineStatus; + if (lineStatus === 'success') succeeded = true; + } + const total = asCount(parsed.total); + if (total === undefined) return; + // A layer is named by its digest; a line without one by its status text. + const key = asString(parsed.digest) ?? lineStatus; + if (key === undefined) return; + const known = layers.get(key); + if (!known && layers.size >= MAX_PULL_LAYERS) return; + // `completed` is absent until the layer's first bytes arrived: what was seen before stays. + const completed = asCount(parsed.completed) ?? known?.completed ?? 0; + layers.set(key, { total, completed: Math.min(completed, total) }); + } + + function feed(text: string): void { + let rest = text; + for (;;) { + const breakAt = rest.indexOf('\n'); + if (breakAt < 0) break; + const head = rest.slice(0, breakAt); + rest = rest.slice(breakAt + 1); + if (skipping) { + // The end of the line that was too long: it was counted when it was dropped. + skipping = false; + } else { + line(pending + head); + } + pending = ''; + } + if (skipping) return; + pending += rest; + if (pending.length > MAX_PULL_LINE_CHARS) { + pending = ''; + skipping = true; + lines += 1; + } + } + + return { + push(chunk) { + feed(typeof chunk === 'string' ? chunk : decoder.decode(chunk, { stream: true })); + }, + end() { + feed(decoder.decode()); + if (!skipping && pending !== '') line(pending); + pending = ''; + skipping = false; + }, + snapshot() { + let completedBytes = 0; + let totalBytes = 0; + for (const layer of layers.values()) { + completedBytes += layer.completed; + totalBytes += layer.total; + } + return { + completedBytes, + totalBytes, + ...(status !== undefined ? { status } : {}), + succeeded, + ...(error !== undefined ? { error } : {}), + lines, + }; + }, + }; } /** How far a pull is, for the caller's progress display. */ @@ -196,9 +457,145 @@ export type PullResult = } | { ok: false; code: 'cancelled' }; +/** How one request of a pull ended. */ +type PullAttempt = + | { kind: 'success'; totalBytes: number } + | { kind: 'cancelled' } + | { kind: 'fatal'; code: 'model-not-found' | 'disk-full'; detail: string } + | { kind: 'retry'; detail: string }; + +/** + * What an error text of the runtime means for the pull. A model the registry does not have and + * a full disk do not get better by asking again; everything else (a damaged download, a + * dropped connection) may. + */ +function classifyPullError(text: string): PullAttempt { + const detail = clip(text); + if (/file does not exist|manifest unknown|manifest\b.*\bnot found|model\b.*\bnot found/i.test(text)) { + return { kind: 'fatal', code: 'model-not-found', detail }; + } + if (/no space left on device|disk (is )?full|not enough (free )?(disk )?space/i.test(text)) { + return { kind: 'fatal', code: 'disk-full', detail }; + } + return { kind: 'retry', detail }; +} + +/** A wait that ends early when the signal aborts. */ +function waitFor(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) return resolve(); + const done = (): void => { + clearTimeout(timer); + signal?.removeEventListener('abort', done); + resolve(); + }; + const timer = setTimeout(done, ms); + signal?.addEventListener('abort', done, { once: true }); + }); +} + +/** One `POST /api/pull`, read to its end. */ +async function pullOnce(target: OllamaTarget, modelId: string, options: PullOptions): Promise { + const { signal, onProgress } = options; + const now = options.now ?? Date.now; + const fetchFn = target.fetch ?? fetch; + let res: Response; + try { + res = await fetchFn(`${target.baseUrl}/api/pull`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: modelId, stream: true }), + ...(signal ? { signal } : {}), + }); + } catch (err) { + return signal?.aborted ? { kind: 'cancelled' } : { kind: 'retry', detail: errorText(err) }; + } + + const parser = createPullParser(); + // The speed is the bytes of the last few seconds: (when, how far) pairs, the oldest first. + const samples: { at: number; completed: number }[] = []; + let lastReport = Number.NEGATIVE_INFINITY; + const report = (force: boolean): void => { + if (!onProgress) return; + const at = now(); + if (!force && at - lastReport < PULL_PROGRESS_INTERVAL_MS) return; + const snap = parser.snapshot(); + // Before the first layer line there is no size to show. + if (snap.totalBytes <= 0) return; + lastReport = at; + samples.push({ at, completed: snap.completedBytes }); + while (samples.length > 2 && at - samples[1].at >= PULL_SPEED_WINDOW_MS) samples.shift(); + const first = samples[0]; + const seconds = (at - first.at) / 1000; + const moved = snap.completedBytes - first.completed; + try { + onProgress({ + completedBytes: snap.completedBytes, + totalBytes: snap.totalBytes, + ...(seconds > 0 && moved >= 0 ? { bytesPerSecond: Math.round(moved / seconds) } : {}), + }); + } catch { + // A display that fails must not fail the download. + } + }; + + let streamError: string | undefined; + try { + if (res.body) { + for await (const chunk of res.body as unknown as AsyncIterable) { + parser.push(chunk); + report(false); + } + } + } catch (err) { + streamError = errorText(err); + } + // An abort ends the stream with an error too: it is a cancel, not a cut. + if (signal?.aborted) return { kind: 'cancelled' }; + parser.end(); + const snap = parser.snapshot(); + + if (snap.error !== undefined) return classifyPullError(snap.error); + if (res.status !== 200) return classifyPullError(`The runtime answered ${res.status} to the pull`); + if (!snap.succeeded) { + return { kind: 'retry', detail: streamError ?? 'The download ended before the runtime reported success' }; + } + report(true); + if (snap.totalBytes > 0) return { kind: 'success', totalBytes: snap.totalBytes }; + // A pull of a model that was already complete may name no layer. The size is then what the + // runtime lists for the model, which is the same figure a full pull reports as its total. + const listed = (await listTags(target, signal)).find((model) => model.name === modelId); + return { kind: 'success', totalBytes: listed?.size ?? 0 }; +} + /** Pulls `modelId` and waits for the end. Resuming is calling it again. Never throws. */ export async function pullModel(target: OllamaTarget, modelId: string, options: PullOptions = {}): Promise { - return notImplemented('pullModel', target, modelId, options); + const { signal } = options; + const retries = + options.retries !== undefined && Number.isInteger(options.retries) && options.retries >= 0 + ? options.retries + : PULL_RETRIES; + const sleep = options.sleep ?? ((ms: number) => waitFor(ms, signal)); + let lastDetail: string | undefined; + for (let attempt = 0; attempt <= retries; attempt++) { + if (signal?.aborted) return { ok: false, code: 'cancelled' }; + if (attempt > 0) { + await sleep(PULL_RETRY_DELAY_MS * 2 ** (attempt - 1)); + if (signal?.aborted) return { ok: false, code: 'cancelled' }; + } + let outcome: PullAttempt; + try { + outcome = await pullOnce(target, modelId, options); + } catch (err) { + // Only an injected `fetch` or clock can throw here. It must not end the pull as a crash. + outcome = { kind: 'retry', detail: errorText(err) }; + } + if (outcome.kind === 'success') return { ok: true, totalBytes: outcome.totalBytes }; + if (outcome.kind === 'cancelled') return { ok: false, code: 'cancelled' }; + if (outcome.kind === 'fatal') return { ok: false, code: outcome.code, detail: outcome.detail }; + lastDetail = outcome.detail; + } + return { ok: false, code: 'pull-failed', ...(lastDetail !== undefined ? { detail: lastDetail } : {}) }; } // ---------- The check after the pull ---------- @@ -218,12 +615,53 @@ export interface LoadedModel { /** The body of the tiny chat that loads the model: one short user message, `reasoning_effort: "none"`, a small `max_tokens`. Pure. */ export function buildChatProbeRequest(modelId: string): Record { - return notImplemented('buildChatProbeRequest', modelId); + return { + model: modelId, + messages: [{ role: 'user', content: 'Reply with the single word: ready' }], + reasoning_effort: 'none', + max_tokens: CHAT_PROBE_MAX_TOKENS, + temperature: 0, + stream: false, + }; +} + +/** A function tool of the tool probe, in the shape `/v1/chat/completions` takes. */ +function probeTool(name: string, description: string, properties: Record): Record { + return { + type: 'function', + function: { + name, + description, + parameters: { + type: 'object', + properties: Object.fromEntries( + Object.entries(properties).map(([key, about]) => [key, { type: 'string', description: about }]) + ), + required: Object.keys(properties), + }, + }, + }; } /** The body of the tool probe: a `tools` array, `reasoning_effort: "none"`, `max_tokens: TOOL_PROBE_MAX_TOKENS`. Pure. */ export function buildToolProbeRequest(modelId: string): Record { - return notImplemented('buildToolProbeRequest', modelId); + return { + model: modelId, + messages: [ + { role: 'system', content: 'You are a coding agent. Use the tools you are given to do what the user asks.' }, + { role: 'user', content: 'Read the file README.md and tell me what it says.' }, + ], + // The three tools a coding agent always has. The task needs the first. + tools: [ + probeTool('read_file', 'Read a file and return its content.', { path: 'The path of the file.' }), + probeTool('write_file', 'Write a file.', { path: 'The path of the file.', content: 'The new content.' }), + probeTool('run_command', 'Run a shell command and return its output.', { command: 'The command line.' }), + ], + reasoning_effort: 'none', + max_tokens: TOOL_PROBE_MAX_TOKENS, + temperature: 0, + stream: false, + }; } /** @@ -237,13 +675,43 @@ export function judgeLoaded( modelId: string, context: number ): { ok: true; loaded: LoadedModel } | { ok: false; code: VerifyFailureCode; detail?: string } { - return notImplemented('judgeLoaded', models, modelId, context); + const model = models.find((entry) => entry.name === modelId); + if (!model) return { ok: false, code: 'load-failed', detail: 'The runtime does not list the model as loaded' }; + if (model.size <= 0) + return { ok: false, code: 'load-failed', detail: 'The runtime reports no size for the loaded model' }; + if (model.sizeVram !== model.size) { + return { + ok: false, + code: 'does-not-fit', + detail: `${model.sizeVram} of ${model.size} bytes are on the GPU`, + }; + } + if (model.contextLength !== context) { + return { + ok: false, + code: 'context-not-applied', + detail: `The runtime reports a context of ${model.contextLength} tokens, not ${context}`, + }; + } + return { ok: true, loaded: { context: model.contextLength, sizeBytes: model.size, sizeVramBytes: model.sizeVram } }; +} + +/** Whether the arguments of a tool call are a JSON object: as text that parses to one, or already as one. */ +function argumentsAreObject(value: unknown): boolean { + if (isRecord(value)) return true; + if (typeof value !== 'string') return false; + try { + return isRecord(JSON.parse(value)); + } catch { + return false; + } } /** * Judges the answer of the tool probe. Pure. `status` is the HTTP status, `body` the parsed * JSON (or `undefined` when it did not parse). - * - a tool call whose `arguments` parse as a JSON object: passed; + * - a tool call whose `arguments` parse as a JSON object: passed (when the answer holds + * several calls, every one must have a name and such arguments); * - `finish_reason` is `length`: `probe-unfinished` (the token limit ended it, so it proves nothing); * - any other answer of status 200: `no-tool-call`; * - another status: `load-failed`. @@ -252,7 +720,33 @@ export function judgeToolProbe( status: number, body: unknown ): { ok: true } | { ok: false; code: VerifyFailureCode; detail?: string } { - return notImplemented('judgeToolProbe', status, body); + if (status !== 200) { + const said = errorOfBody(body); + return { ok: false, code: 'load-failed', detail: clip(said ?? `The runtime answered ${status}`) }; + } + const choice = isRecord(body) && Array.isArray(body.choices) ? body.choices[0] : undefined; + const message = isRecord(choice) && isRecord(choice.message) ? choice.message : undefined; + const calls = message && Array.isArray(message.tool_calls) ? message.tool_calls : []; + if (calls.length > 0) { + const wellFormed = calls.every( + (call) => + isRecord(call) && + isRecord(call.function) && + asString(call.function.name) !== undefined && + argumentsAreObject(call.function.arguments) + ); + if (wellFormed) return { ok: true }; + return { ok: false, code: 'no-tool-call', detail: 'The model called a tool with arguments that are not JSON' }; + } + if (isRecord(choice) && choice.finish_reason === 'length') { + return { ok: false, code: 'probe-unfinished', detail: `The answer ended at ${TOOL_PROBE_MAX_TOKENS} tokens` }; + } + const said = message ? asString(message.content) : undefined; + return { + ok: false, + code: 'no-tool-call', + detail: said ? clip(`The model answered in text: ${said}`) : 'The answer holds no tool call', + }; } /** How the check ended. */ @@ -271,5 +765,44 @@ export async function verifyModel( context: number, signal?: AbortSignal ): Promise { - return notImplemented('verifyModel', target, modelId, context, signal); + const cancelled: VerifyResult = { ok: false, code: 'cancelled' }; + if (signal?.aborted) return cancelled; + + // 1. The chat that loads the model. Its text is not read: only that it was answered. + const chat = await ask( + target, + '/v1/chat/completions', + { method: 'POST', json: buildChatProbeRequest(modelId) }, + CHAT_PROBE_TIMEOUT_MS, + signal + ); + if (signal?.aborted) return cancelled; + if (!chat) return { ok: false, code: 'load-failed', detail: 'The runtime did not answer the first request' }; + if (chat.status !== 200) { + return { + ok: false, + code: 'load-failed', + detail: clip(errorOfBody(chat.body) ?? `The runtime answered ${chat.status}`), + }; + } + + // 2. What the runtime says it loaded. + const judged = judgeLoaded(await listRunning(target, signal), modelId, context); + if (signal?.aborted) return cancelled; + if (!judged.ok) return judged; + + // 3. One tool call. + const probe = await ask( + target, + '/v1/chat/completions', + { method: 'POST', json: buildToolProbeRequest(modelId) }, + TOOL_PROBE_TIMEOUT_MS, + signal + ); + if (signal?.aborted) return cancelled; + if (!probe) + return { ok: false, code: 'probe-unfinished', detail: 'The runtime did not answer the tool request in time' }; + const tool = judgeToolProbe(probe.status, probe.body); + if (!tool.ok) return tool; + return { ok: true, loaded: judged.loaded }; } diff --git a/src/local-model/runtime.ts b/src/local-model/runtime.ts index 1cc334f5..dd6b70f5 100644 --- a/src/local-model/runtime.ts +++ b/src/local-model/runtime.ts @@ -2,9 +2,6 @@ * @fileoverview The model runtime: which build Codeman runs, how it gets onto the disk, and * how it is kept running. * - * CONTRACT STAGE: `OLLAMA_RUNTIME_PIN` and the constants are final. Every function below has - * its final signature and a body that throws; the bodies are written next. - * * What this module does: * - THE PIN. `OLLAMA_RUNTIME_PIN` is a compiled constant. No file, setting, environment * variable or request can change the URL, the size or the hash. @@ -26,10 +23,13 @@ * - SUPERVISOR (`createRuntimeSupervisor()`). The runtime is a NON-detached child of the * server. `start()` first looks at the port: a listener that answers `/api/version` and * whose pid (`previousPid`) runs a binary under our runtime folder is our own orphan and is - * adopted or replaced; any other listener is the answer `port-in-use`. A child that exits by - * itself is restarted, at most `RUNTIME_MAX_RESTARTS` times in a row; `stop()` sends SIGTERM. - * The supervisor does not write the state file: it reports through `onChange`, and its - * caller stores the pid and the port. + * adopted or replaced; any other listener is the answer `port-in-use`. An orphan is ADOPTED + * only when keeping it keeps something: it answers the pinned version and has a model loaded + * at the wanted context. Otherwise it is REPLACED (SIGTERM, then a child of our own), which + * loses nothing and gives back what an orphan cannot: its log, so the measured budget, and + * its exit. A child that exits by itself is restarted, at most `RUNTIME_MAX_RESTARTS` times + * since the last `start()`; `stop()` sends SIGTERM. The supervisor does not write the state + * file: it reports through `onChange`, and its caller stores the pid and the port. * - BUDGET (`parseInferenceComputeTotal()`, pure). Right after it starts, the runtime prints * the device total in its log line "inference compute" (`total="37.4 GiB"`). The supervisor * reads the child's output for it and reports it as `measuredTotalBytes`. @@ -37,17 +37,19 @@ * asked, the models. Only inside `LocalModelPaths.root`. * * Under vitest nothing spawns and nothing is fetched unless a test injects it: the default - * `spawn` and `fetch` of the dependency objects refuse there. + * `spawn` and `fetch` of the dependency objects refuse there, the default `commandOfPid` + * knows no pid and the default `kill` signals nothing. */ -import type { ChildProcess, SpawnOptions } from 'node:child_process'; +import { execFile, spawn as nodeSpawn, type ChildProcess, type SpawnOptions } from 'node:child_process'; +import { createHash, type Hash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { lstat, mkdir, open, realpath, rename, rm, rmdir, stat, unlink } from 'node:fs/promises'; +import { createServer } from 'node:net'; +import { join, resolve as resolvePath } from 'node:path'; import type { LocalModelErrorCode, LocalModelRuntimePin } from '../types.js'; -import type { LocalModelPaths } from './paths.js'; - -/** Marks a body the contract stage left unwritten. */ -function notImplemented(what: string, ..._args: unknown[]): never { - throw new Error(`${what} is not implemented yet`); -} +import { getRuntimeVersion, listRunning } from './ollama-client.js'; +import { isInsideLocalModelRoot, localModelPaths, type LocalModelPaths } from './paths.js'; // ---------- The pin and the constants ---------- @@ -74,8 +76,114 @@ export const RUNTIME_DOWNLOAD_RETRIES = 3; /** `OLLAMA_KEEP_ALIVE`: how long the model stays loaded after the last request. */ export const RUNTIME_KEEP_ALIVE = '10m'; +/** The name of the runtime's binary inside its folder and inside the archive. */ +const RUNTIME_BINARY_NAME = 'ollama'; +/** A download that delivers no byte for this long is treated as cut. */ +const DOWNLOAD_IDLE_TIMEOUT_MS = 60_000; +/** The first wait before a cut download is continued; it doubles with every retry. */ +const DOWNLOAD_RETRY_DELAY_MS = 500; +/** The least time between two progress reports of the download. */ +const DOWNLOAD_PROGRESS_INTERVAL_MS = 200; +/** How long `/usr/bin/tar` may take for the 579 MB of the runtime. */ +const UNTAR_TIMEOUT_MS = 180_000; +/** How long a started runtime has to answer `/api/version`. The first start on the test Mac took a few seconds. */ +const START_TIMEOUT_MS = 30_000; +/** The wait between two `/api/version` requests while the runtime starts. */ +const START_POLL_MS = 150; +/** How long `start()` waits for the budget line once the runtime answers. It is printed within a second. */ +const BUDGET_WAIT_MS = 3_000; +/** How long a runtime gets to exit after SIGTERM before it is sent SIGKILL. */ +const STOP_GRACE_MS = 5_000; +/** How long the exit is awaited after SIGKILL. */ +const STOP_KILL_WAIT_MS = 2_000; +/** The wait before a runtime that exited by itself is started again. */ +const RESTART_DELAY_MS = 1_000; +/** How often an adopted orphan is looked at: it is not our child, so its exit is not an event. */ +const ORPHAN_POLL_MS = 5_000; +/** How long `ps` may take. */ +const PS_TIMEOUT_MS = 2_000; +/** How many of the runtime's last log lines are kept for the detail of a failed start. */ +const LOG_TAIL_LINES = 12; +/** A log line longer than this is cut: the runtime logs request bodies in some modes. */ +const LOG_LINE_MAX_CHARS = 2_000; +/** How much of a failure text is kept as `detail`. */ +const MAX_DETAIL_CHARS = 600; + +const UNDER_VITEST = Boolean(process.env.VITEST); + +/** A default dependency that refuses under vitest: a test must inject its own. */ +class NotInjectedError extends Error { + constructor(what: string) { + super(`${what} is not available under vitest: the test must inject its own`); + this.name = 'NotInjectedError'; + } +} + +// ---------- Small helpers ---------- + +function clip(text: string): string { + const trimmed = text.trim(); + return trimmed.length > MAX_DETAIL_CHARS ? `${trimmed.slice(0, MAX_DETAIL_CHARS)}...` : trimmed; +} + +function errorText(err: unknown): string { + if (err instanceof Error) { + // undici wraps the reason ("fetch failed") and keeps the useful part in `cause`. + const cause = (err as { cause?: unknown }).cause; + return clip(cause instanceof Error && cause.message ? `${err.message}: ${cause.message}` : err.message); + } + return clip(String(err)); +} + +function errnoOf(err: unknown): string | undefined { + const code = (err as { code?: unknown } | null | undefined)?.code; + return typeof code === 'string' ? code : undefined; +} + +function isDiskFull(err: unknown): boolean { + const code = errnoOf(err); + return code === 'ENOSPC' || code === 'EDQUOT' || /no space left on device/i.test(errorText(err)); +} + +/** A wait on the real clock that does not keep the process alive. For grace periods, which an injected `sleep` must not shorten. */ +function pause(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms).unref(); + }); +} + +function realSleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** The size of a file, 0 when it is not there. */ +async function fileSize(path: string): Promise { + try { + const info = await stat(path); + return info.isFile() ? info.size : 0; + } catch { + return 0; + } +} + +async function isFile(path: string): Promise { + try { + return (await stat(path)).isFile(); + } catch { + return false; + } +} + // ---------- Pure ---------- +const BINARY_UNIT_BYTES: Record = { + B: 1, + KiB: 1024, + MiB: 1024 ** 2, + GiB: 1024 ** 3, + TiB: 1024 ** 4, +}; + /** * The device total of the runtime's log line "inference compute", in bytes, or `undefined` * for any other line. The real line (a MacBook, 2026-11-06): @@ -83,11 +191,16 @@ export const RUNTIME_KEEP_ALIVE = '10m'; * `time=... level=INFO source=types.go:32 msg="inference compute" id=0 ... type=iGPU total="37.4 GiB" available="37.4 GiB"` * * reads 40,157,944,217 (37.4 x 1024^3, rounded down). The value is the quoted text after - * ` total=`: a number and a binary unit (`KiB`, `MiB`, `GiB`, `TiB`). `total_vram=` of the + * ` total=`: a number and a binary unit (`B`, `KiB`, `MiB`, `GiB`, `TiB`). `total_vram=` of the * line "vram-based default context" is not read. */ export function parseInferenceComputeTotal(line: string): number | undefined { - return notImplemented('parseInferenceComputeTotal', line); + if (!line.includes('msg="inference compute"')) return undefined; + // The key must stand alone: `total_vram=` and `subtotal=` are other keys. + const match = /(?:^|\s)total="(\d+(?:\.\d+)?) ?(B|KiB|MiB|GiB|TiB)"/.exec(line); + if (!match) return undefined; + const bytes = Math.floor(Number(match[1]) * BINARY_UNIT_BYTES[match[2]]); + return Number.isFinite(bytes) && bytes > 0 ? bytes : undefined; } /** What `runtimeEnv()` is built from. */ @@ -104,18 +217,58 @@ export interface RuntimeEnvInput { contextLength: number; } +/** The names copied from the server's environment when they are set. Everything else stays behind. */ +const RUNTIME_ENV_COPIED = [ + 'PATH', + 'TMPDIR', + 'LANG', + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'NO_PROXY', + 'http_proxy', + 'https_proxy', + 'no_proxy', +] as const; + /** The environment of the runtime child: the allowlist of the file header, nothing more. Pure. */ export function runtimeEnv(input: RuntimeEnvInput): NodeJS.ProcessEnv { - return notImplemented('runtimeEnv', input); + const env: NodeJS.ProcessEnv = {}; + for (const name of RUNTIME_ENV_COPIED) { + const value = input.source[name]; + if (typeof value === 'string' && value !== '') env[name] = value; + } + env.HOME = input.home; + env.OLLAMA_HOST = `127.0.0.1:${input.port}`; + env.OLLAMA_MODELS = input.modelsDir; + env.OLLAMA_CONTEXT_LENGTH = String(input.contextLength); + env.OLLAMA_NO_CLOUD = '1'; + env.OLLAMA_KEEP_ALIVE = RUNTIME_KEEP_ALIVE; + env.OLLAMA_NUM_PARALLEL = '1'; + env.OLLAMA_MAX_LOADED_MODELS = '1'; + return env; +} + +/** A path without the `/private` macOS puts in front of `/var`, `/tmp` and `/etc`. */ +function withoutPrivatePrefix(path: string): string { + return path.startsWith('/private/') ? path.slice('/private'.length) : path; } /** * Whether `command` (one line of `ps -o command= -p `) is a process whose binary lies * under `runtimeRoot` (`LocalModelPaths.runtimeRoot`): our own runtime, so an orphan of an * earlier server. Pure. `/private` in front of a path is the same path on macOS. + * + * It tells our orphan from a runtime the user started, so that only ours is ever signalled. + * It is not a defence against another program of the same account, which can name itself + * anything. */ export function isOwnRuntimeCommand(command: string, runtimeRoot: string): boolean { - return notImplemented('isOwnRuntimeCommand', command, runtimeRoot); + const root = withoutPrivatePrefix(runtimeRoot.trim()).replace(/\/+$/, ''); + if (!root.startsWith('/') || root === '') return false; + const text = withoutPrivatePrefix(command.trimStart()); + if (!text.startsWith(`${root}/`)) return false; + // A `..` segment could lead out of the folder again. + return !/(?:^|\/)\.\.(?:\/|\s|$)/.test(text); } // ---------- Download and unpack ---------- @@ -147,6 +300,8 @@ export interface RuntimeInstallOptions { /** Called while the archive downloads, at most a few times a second. */ onProgress?: (progress: RuntimeDownloadProgress) => void; deps?: Partial; + /** How long the download may deliver nothing before it counts as cut. Default one minute. */ + idleTimeoutMs?: number; } /** @@ -171,9 +326,278 @@ export type RuntimeInstallResult = } | { ok: false; code: 'cancelled' }; +/** A failure of `ensureRuntimeInstalled()`, without the success arm. */ +type RuntimeInstallFailure = Exclude; + +const CANCELLED: RuntimeInstallFailure = { ok: false, code: 'cancelled' }; + +/** The default `fetch` of the download: the global one, refused under vitest. */ +const defaultFetch: typeof fetch = (input, init) => { + if (UNDER_VITEST) return Promise.reject(new NotInjectedError('fetch')); + return fetch(input, init); +}; + +/** The default `untar`: `/usr/bin/tar -xzf -C `, bounded. */ +function untarWithSystemTar(archive: string, intoDir: string, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + execFile( + '/usr/bin/tar', + ['-xzf', archive, '-C', intoDir], + { timeout: UNTAR_TIMEOUT_MS, ...(signal ? { signal } : {}) }, + (err, _stdout, stderr) => { + if (!err) return resolve(); + const said = typeof stderr === 'string' && stderr.trim() !== '' ? stderr : err.message; + reject(new Error(clip(said))); + } + ); + }); +} + /** Whether the pinned runtime is unpacked on disk (its binary exists). Never throws. */ export async function isRuntimeInstalled(pin?: LocalModelRuntimePin, paths?: LocalModelPaths): Promise { - return notImplemented('isRuntimeInstalled', pin, paths); + try { + const resolved = paths ?? localModelPaths((pin ?? OLLAMA_RUNTIME_PIN).version); + return await isFile(resolved.binary); + } catch { + return false; + } +} + +/** Feeds a file on disk into `hash`: the part of the archive an earlier run left behind. */ +async function hashFile(path: string, hash: Hash, signal?: AbortSignal): Promise { + for await (const chunk of createReadStream(path, signal ? { signal } : {})) { + hash.update(chunk as Buffer); + } +} + +/** How one request for the archive ended. `detail` says why it must be asked again. */ +type DownloadAttempt = + | { kind: 'ended'; detail?: string } + | { kind: 'cancelled' } + | { kind: 'failed'; result: RuntimeInstallFailure }; + +/** + * Downloads the pinned archive to `paths.archivePartial` and proves its hash. On success the + * file is complete and verified; nothing was unpacked. Never throws. + */ +async function downloadArchive( + pin: LocalModelRuntimePin, + paths: LocalModelPaths, + deps: RuntimeInstallDeps, + options: RuntimeInstallOptions +): Promise<{ ok: true } | RuntimeInstallFailure> { + const { signal, onProgress } = options; + const idleTimeoutMs = options.idleTimeoutMs ?? DOWNLOAD_IDLE_TIMEOUT_MS; + const partial = paths.archivePartial; + + // What an earlier run left: continue after it. More than the whole archive cannot be it. + let offset = await fileSize(partial); + if (offset > pin.bytes) { + await rm(partial, { force: true }); + offset = 0; + } + let hash = createHash('sha256'); + if (offset > 0) await hashFile(partial, hash, signal); + + let lastReport = Number.NEGATIVE_INFINITY; + const report = (force: boolean): void => { + if (!onProgress) return; + const at = Date.now(); + if (!force && at - lastReport < DOWNLOAD_PROGRESS_INTERVAL_MS) return; + lastReport = at; + try { + onProgress({ completedBytes: offset, totalBytes: pin.bytes }); + } catch { + // A display that fails must not fail the download. + } + }; + + /** One request, written to the file as far as it gets. */ + const attempt = async (): Promise => { + const controller = new AbortController(); + const onAbort = (): void => controller.abort(); + signal?.addEventListener('abort', onAbort, { once: true }); + let idle: NodeJS.Timeout | undefined; + let wentIdle = false; + const armIdle = (): void => { + clearTimeout(idle); + idle = setTimeout(() => { + wentIdle = true; + controller.abort(); + }, idleTimeoutMs); + }; + // Opened BEFORE the request, in append mode: once the answer is there, nothing is awaited + // until its body is being read. A body that is cut while nobody reads it loses what had + // arrived, and the continuation would start further back than it has to. + const file = await open(partial, 'a', 0o600); + // The file is deleted after it is closed: a download that cannot be the pinned archive. + let unlinkAfter = false; + const startOver = async (): Promise => { + await file.truncate(0); + offset = 0; + hash = createHash('sha256'); + }; + const damaged = (detail: string): DownloadAttempt => { + unlinkAfter = true; + return { kind: 'failed', result: { ok: false, code: 'runtime-damaged', detail } }; + }; + try { + armIdle(); + let res: Response; + try { + res = await deps.fetch(pin.url, { + ...(offset > 0 ? { headers: { range: `bytes=${offset}-` } } : {}), + signal: controller.signal, + redirect: 'follow', + }); + } catch (err) { + if (signal?.aborted) return { kind: 'cancelled' }; + if (err instanceof NotInjectedError) { + return { kind: 'failed', result: { ok: false, code: 'runtime-download-failed', detail: err.message } }; + } + return { kind: 'ended', detail: wentIdle ? 'The download server did not answer' : errorText(err) }; + } + + const discard = async (): Promise => { + await res.body?.cancel().catch(() => undefined); + }; + // The size the server says the whole archive has, when it says one. + let declaredTotal: number | undefined; + // The whole archive from its first byte although a range was asked for: the file starts over. + let restart = false; + if (res.status === 206) { + const range = /^bytes (\d+)-\d+\/(\d+)$/.exec(res.headers.get('content-range') ?? ''); + // A 206 that does not start where the file ends cannot be appended to it. + if (!range || Number(range[1]) !== offset) { + await discard(); + await startOver(); + return { kind: 'ended', detail: 'The server answered another range than the one asked for' }; + } + declaredTotal = Number(range[2]); + } else if (res.status === 200) { + restart = offset > 0; + const length = res.headers.get('content-length'); + if (length !== null && !res.headers.get('content-encoding') && /^\d+$/.test(length)) { + declaredTotal = Number(length); + } + } else if (res.status === 416) { + // The file on disk is not a prefix of what the server has. + await discard(); + await startOver(); + return { kind: 'ended', detail: 'The server refused the range (HTTP 416)' }; + } else { + await discard(); + return { kind: 'ended', detail: `The download server answered HTTP ${res.status}` }; + } + if (declaredTotal !== undefined && declaredTotal !== pin.bytes) { + await discard(); + return damaged(`The download is ${declaredTotal} bytes, the pinned archive has ${pin.bytes}`); + } + if (!res.body) return { kind: 'ended', detail: 'The download server sent no body' }; + + try { + if (restart) await startOver(); + for await (const chunk of res.body as unknown as AsyncIterable) { + armIdle(); + // Refused before a byte of it is written: a longer body can never have the pinned hash. + if (offset + chunk.length > pin.bytes) { + return damaged(`The download is longer than the ${pin.bytes} bytes of the pinned archive`); + } + await file.appendFile(chunk); + hash.update(chunk); + offset += chunk.length; + report(false); + } + } catch (err) { + // Whatever was written past the last hashed byte is cut off, so file and hash agree. + await file.truncate(offset).catch(() => undefined); + if (signal?.aborted) return { kind: 'cancelled' }; + if (isDiskFull(err)) { + return { kind: 'failed', result: { ok: false, code: 'disk-full', detail: errorText(err) } }; + } + return { + kind: 'ended', + detail: wentIdle + ? `The download delivered nothing for ${Math.max(1, Math.round(idleTimeoutMs / 1000))} s` + : errorText(err), + }; + } + return { kind: 'ended' }; + } finally { + clearTimeout(idle); + signal?.removeEventListener('abort', onAbort); + await file.close().catch(() => undefined); + // An empty file is no start of a download either. + if (unlinkAfter || offset === 0) await rm(partial, { force: true }).catch(() => undefined); + } + }; + + // A download that continues says at once how far it already is. + if (offset > 0) report(true); + let failures = 0; + while (offset < pin.bytes) { + if (signal?.aborted) return CANCELLED; + const outcome = await attempt(); + report(true); + if (outcome.kind === 'cancelled') return CANCELLED; + if (outcome.kind === 'failed') return outcome.result; + if (offset >= pin.bytes) break; + // Cut, or ended early: continue from where the file ends, a bounded number of times. + failures += 1; + if (failures > RUNTIME_DOWNLOAD_RETRIES) { + return { + ok: false, + code: 'runtime-download-failed', + detail: outcome.detail ?? `The download ended at ${offset} of ${pin.bytes} bytes`, + }; + } + await deps.sleep(DOWNLOAD_RETRY_DELAY_MS * 2 ** (failures - 1)); + } + if (signal?.aborted) return CANCELLED; + + // The proof, BEFORE anything is unpacked. + const digest = hash.digest('hex'); + if (digest !== pin.sha256.toLowerCase()) { + await rm(partial, { force: true }); + return { + ok: false, + code: 'runtime-damaged', + detail: `The sha256 of the download is ${digest}, the pinned archive has ${pin.sha256}`, + }; + } + return { ok: true }; +} + +/** + * Unpacks the verified archive into the temporary folder and renames it to the runtime folder. + * Never throws. + */ +async function unpackArchive( + paths: LocalModelPaths, + deps: RuntimeInstallDeps, + signal?: AbortSignal +): Promise<{ ok: true } | RuntimeInstallFailure> { + const fail = async (err: unknown): Promise => { + await rm(paths.unpackDir, { recursive: true, force: true }).catch(() => undefined); + if (signal?.aborted) return CANCELLED; + return { ok: false, code: isDiskFull(err) ? 'disk-full' : 'runtime-unpack-failed', detail: errorText(err) }; + }; + try { + await rm(paths.unpackDir, { recursive: true, force: true }); + await mkdir(paths.unpackDir, { recursive: true, mode: 0o700 }); + await deps.untar(paths.archivePartial, paths.unpackDir, signal); + if (signal?.aborted) return await fail(new Error('aborted')); + if (!(await isFile(join(paths.unpackDir, RUNTIME_BINARY_NAME)))) { + return await fail(new Error(`The archive holds no ${RUNTIME_BINARY_NAME} binary`)); + } + // A folder without its binary is a leftover of an unpack that was interrupted. + await rm(paths.runtimeDir, { recursive: true, force: true }); + await rename(paths.unpackDir, paths.runtimeDir); + await rm(paths.archivePartial, { force: true }); + return { ok: true }; + } catch (err) { + return fail(err); + } } /** @@ -182,7 +606,35 @@ export async function isRuntimeInstalled(pin?: LocalModelRuntimePin, paths?: Loc * again. Never throws: every outcome is a `RuntimeInstallResult`. */ export async function ensureRuntimeInstalled(options: RuntimeInstallOptions = {}): Promise { - return notImplemented('ensureRuntimeInstalled', options); + let phase: 'download' | 'unpack' = 'download'; + try { + const pin = options.pin ?? OLLAMA_RUNTIME_PIN; + const paths = options.paths ?? localModelPaths(pin.version); + const deps: RuntimeInstallDeps = { + fetch: options.deps?.fetch ?? defaultFetch, + untar: options.deps?.untar ?? untarWithSystemTar, + sleep: options.deps?.sleep ?? realSleep, + }; + if (await isFile(paths.binary)) return { ok: true, binaryPath: paths.binary, alreadyInstalled: true }; + if (options.signal?.aborted) return CANCELLED; + + await mkdir(paths.runtimeRoot, { recursive: true, mode: 0o700 }); + const downloaded = await downloadArchive(pin, paths, deps, options); + if (!downloaded.ok) return downloaded; + + phase = 'unpack'; + const unpacked = await unpackArchive(paths, deps, options.signal); + if (!unpacked.ok) return unpacked; + return { ok: true, binaryPath: paths.binary, alreadyInstalled: false }; + } catch (err) { + if (options.signal?.aborted) return CANCELLED; + if (isDiskFull(err)) return { ok: false, code: 'disk-full', detail: errorText(err) }; + return { + ok: false, + code: phase === 'download' ? 'runtime-download-failed' : 'runtime-unpack-failed', + detail: errorText(err), + }; + } } /** What `removeLocalModelFiles()` deletes. */ @@ -193,18 +645,70 @@ export interface RemoveLocalModelFilesOptions { paths?: LocalModelPaths; } +/** What is to be done with one folder of the local model: nothing, unlink the link in its place, or delete its tree. */ +type RemovalStep = { target: string; how: 'absent' | 'unlink' | 'delete' }; + +/** + * Decides what removing `target` means, and refuses what must not be removed. A symlink in the + * place of a folder is unlinked and never followed; a real folder is deleted only when its + * real path is still inside the real root. Nothing is deleted here. + */ +async function planRemoval(target: string, root: string): Promise { + if (!isInsideLocalModelRoot(target, root) || resolvePath(target) === resolvePath(root)) { + throw new Error(`Refusing to delete ${target}: it is not a folder inside ${root}`); + } + let info; + try { + info = await lstat(target); + } catch (err) { + if (errnoOf(err) === 'ENOENT') return { target, how: 'absent' }; + throw err; + } + // The link goes, what it points at stays: that is somebody else's folder. + if (info.isSymbolicLink()) return { target, how: 'unlink' }; + const [realTarget, realRoot] = await Promise.all([realpath(target), realpath(root)]); + if (!isInsideLocalModelRoot(realTarget, realRoot) || realTarget === realRoot) { + throw new Error(`Refusing to delete ${target}: it resolves to ${realTarget}, outside ${realRoot}`); + } + return { target, how: 'delete' }; +} + /** * Deletes what this feature put on disk: every runtime folder and archive, the private home * and, when asked, the models. Refuses any path that is not inside `LocalModelPaths.root`, * follows no symlink out of it, and removes `root` itself only when it is empty afterwards. + * Every folder is judged BEFORE the first one is deleted, so a refusal deletes nothing. * The state file and the coding agent are not its business. */ export async function removeLocalModelFiles(options: RemoveLocalModelFilesOptions): Promise { - return notImplemented('removeLocalModelFiles', options); + const paths = options.paths ?? localModelPaths(OLLAMA_RUNTIME_PIN.version); + const targets = [paths.runtimeRoot, paths.homeDir, ...(options.models ? [paths.modelsDir] : [])]; + const plan: RemovalStep[] = []; + for (const target of targets) plan.push(await planRemoval(target, paths.root)); + for (const step of plan) { + if (step.how === 'unlink') await unlink(step.target); + // `rm` removes a symlink it meets inside the tree, never the tree behind it. + if (step.how === 'delete') await rm(step.target, { recursive: true, force: true }); + } + // Empty now (the models went too, or there never were any): the folder itself goes. + await rmdir(paths.root).catch(() => undefined); } // ---------- Port ---------- +/** Whether nothing holds `port` on 127.0.0.1, decided by binding it and letting go again. */ +export function isLoopbackPortFree(port: number): Promise { + return new Promise((resolve) => { + if (!Number.isInteger(port) || port < 1 || port > 65535) return resolve(false); + const probe = createServer(); + probe.unref(); + probe.once('error', () => resolve(false)); + probe.listen(port, '127.0.0.1', () => { + probe.close(() => resolve(true)); + }); + }); +} + export interface ChooseRuntimePortOptions { /** Default `RUNTIME_PORT_FIRST`. */ first?: number; @@ -218,7 +722,19 @@ export interface ChooseRuntimePortOptions { /** The first free loopback port of the span, or `undefined` when none is free. Chosen once, then stored by the caller. */ export async function chooseRuntimePort(options: ChooseRuntimePortOptions = {}): Promise { - return notImplemented('chooseRuntimePort', options); + const first = options.first ?? RUNTIME_PORT_FIRST; + const span = options.span ?? RUNTIME_PORT_SPAN; + const isFree = options.isFree ?? isLoopbackPortFree; + const avoid = new Set(options.avoid ?? []); + for (let port = first; port < first + span; port++) { + if (avoid.has(port)) continue; + try { + if (await isFree(port)) return port; + } catch { + // A probe that fails says nothing about the port: it is not taken as free. + } + } + return undefined; } // ---------- Supervisor ---------- @@ -287,6 +803,12 @@ export interface RuntimeSupervisorDeps { paths: LocalModelPaths; /** Called after every change of `status()`: a start, an exit, a restart, the budget line. The caller stores pid and port. */ onChange: (status: RuntimeStatus) => void; + /** How long a started runtime has to answer `/api/version`. Default 30 seconds. */ + startTimeoutMs?: number; + /** How often an adopted orphan is looked at. Default 5 seconds. */ + orphanPollMs?: number; + /** How long `start()` waits for the budget line once the runtime answers. Default 3 seconds. */ + budgetWaitMs?: number; } /** The one supervisor of the runtime child. */ @@ -302,7 +824,461 @@ export interface RuntimeSupervisor { status(): RuntimeStatus; } +const defaultSpawn: RuntimeSupervisorDeps['spawn'] = (command, args, options) => { + if (UNDER_VITEST) throw new NotInjectedError('spawn'); + return nodeSpawn(command, [...args], options); +}; + +/** `ps -o command= -p `: the command line of a pid, `undefined` when it is gone. Spawns nothing under vitest. */ +function defaultCommandOfPid(pid: number): Promise { + return new Promise((resolve) => { + if (UNDER_VITEST || !Number.isInteger(pid) || pid <= 1) return resolve(undefined); + execFile('/bin/ps', ['-o', 'command=', '-p', String(pid)], { timeout: PS_TIMEOUT_MS }, (err, stdout) => { + const line = typeof stdout === 'string' ? stdout.trim() : ''; + resolve(err || line === '' ? undefined : line); + }); + }); +} + +const defaultKill: RuntimeSupervisorDeps['kill'] = (pid, signal) => { + if (UNDER_VITEST || !Number.isInteger(pid) || pid <= 1) return; + try { + process.kill(pid, signal); + } catch { + // Gone already. + } +}; + +function hasExited(child: ChildProcess): boolean { + return child.exitCode !== null || child.signalCode !== null; +} + +/** How one launch of the child ended. */ +type LaunchResult = { ok: true; pid: number; measuredTotalBytes?: number } | { ok: false; detail: string }; + /** A supervisor. The server keeps ONE (see `getLocalModelService()` in install-job.ts). */ -export function createRuntimeSupervisor(deps: Partial = {}): RuntimeSupervisor { - return notImplemented('createRuntimeSupervisor', deps); +export function createRuntimeSupervisor(overrides: Partial = {}): RuntimeSupervisor { + const pin = overrides.pin ?? OLLAMA_RUNTIME_PIN; + const deps = { + spawn: overrides.spawn ?? defaultSpawn, + fetch: overrides.fetch ?? defaultFetch, + env: overrides.env ?? process.env, + commandOfPid: overrides.commandOfPid ?? defaultCommandOfPid, + kill: overrides.kill ?? defaultKill, + isPortFree: overrides.isPortFree ?? isLoopbackPortFree, + sleep: overrides.sleep ?? realSleep, + onChange: overrides.onChange ?? ((): void => undefined), + startTimeoutMs: overrides.startTimeoutMs ?? START_TIMEOUT_MS, + orphanPollMs: overrides.orphanPollMs ?? ORPHAN_POLL_MS, + budgetWaitMs: overrides.budgetWaitMs ?? BUDGET_WAIT_MS, + }; + // Resolved on first use: `localModelPaths()` creates the data dir, which making a supervisor must not. + let resolvedPaths = overrides.paths; + const paths = (): LocalModelPaths => (resolvedPaths ??= localModelPaths(pin.version)); + + /** Our child, once it answered. */ + let child: ChildProcess | undefined; + /** A child that was spawned and has not answered yet. */ + let starting: ChildProcess | undefined; + /** An orphan of an earlier server that was adopted: not our child, so it is signalled by pid. */ + let adoptedPid: number | undefined; + let orphanTimer: NodeJS.Timeout | undefined; + /** What the last `start()` asked for: what a restart starts again. */ + let wanted: { port: number; contextLength: number } | undefined; + /** Bumped by every `start()` and `stop()`: work of an older generation stops when it notices. */ + let generation = 0; + /** The restart loop that is running, if one is. */ + let restarting: Promise | undefined; + /** `start()` and `stop()` run one after the other. */ + let queue: Promise = Promise.resolve(); + let state: RuntimeStatus = { running: false, restarts: 0, gaveUp: false }; + + function setState(patch: Partial): void { + const next: RuntimeStatus = { ...state, ...patch }; + for (const key of Object.keys(next) as (keyof RuntimeStatus)[]) { + if (next[key] === undefined) delete next[key]; + } + const changed = + Object.keys(next).length !== Object.keys(state).length || + (Object.keys(next) as (keyof RuntimeStatus)[]).some((key) => next[key] !== state[key]); + state = next; + if (!changed) return; + try { + deps.onChange({ ...state }); + } catch { + // A caller that throws must not stop the supervisor. + } + } + + function serialize(work: () => Promise): Promise { + const run = queue.then(work, work); + queue = run.catch(() => undefined); + return run; + } + + const isOwn = (command: string | undefined): boolean => + command !== undefined && isOwnRuntimeCommand(command, paths().runtimeRoot); + + /** SIGTERM, then SIGKILL after the grace period. Resolves once the child is gone. */ + async function stopChild(proc: ChildProcess): Promise { + // A child that never got a pid was never started: there is no exit to wait for. + if (hasExited(proc) || proc.pid === undefined) return; + const gone = new Promise((resolve) => proc.once('exit', () => resolve())); + try { + if (!proc.kill('SIGTERM')) return; + } catch { + return; + } + const force = setTimeout(() => { + try { + proc.kill('SIGKILL'); + } catch { + // Gone already. + } + }, STOP_GRACE_MS); + force.unref(); + await Promise.race([gone, pause(STOP_GRACE_MS + STOP_KILL_WAIT_MS)]); + clearTimeout(force); + } + + /** The same for a pid that is not our child: signalled through `deps.kill`, watched through `ps`. */ + async function stopOrphan(pid: number): Promise { + const goneWithin = async (ms: number): Promise => { + const until = Date.now() + ms; + for (;;) { + if (!isOwn(await deps.commandOfPid(pid))) return true; + if (Date.now() >= until) return false; + await pause(100); + } + }; + deps.kill(pid, 'SIGTERM'); + if (await goneWithin(STOP_GRACE_MS)) return; + deps.kill(pid, 'SIGKILL'); + await goneWithin(STOP_KILL_WAIT_MS); + } + + /** Stops whatever runs now: our child, one that is starting, an adopted orphan. */ + async function stopCurrent(): Promise { + clearInterval(orphanTimer); + orphanTimer = undefined; + const procs = [child, starting].filter((proc): proc is ChildProcess => proc !== undefined); + // Forgotten first, so that its exit is not taken for a crash. + child = undefined; + starting = undefined; + const orphan = adoptedPid; + adoptedPid = undefined; + await Promise.all(procs.map((proc) => stopChild(proc))); + if (orphan !== undefined) await stopOrphan(orphan); + } + + /** The runtime exited by itself: start it again, a bounded number of times. */ + function restart(): void { + if (restarting) return; + const gen = generation; + const loop = async (): Promise => { + while (gen === generation && wanted) { + if (state.restarts >= RUNTIME_MAX_RESTARTS) { + setState({ gaveUp: true }); + return; + } + setState({ restarts: state.restarts + 1 }); + await deps.sleep(RESTART_DELAY_MS); + if (gen !== generation || !wanted) return; + const launched = await launch(wanted.port, wanted.contextLength, gen); + // Running again: a later exit comes back here through the child's exit handler. + if (launched.ok) return; + } + }; + restarting = loop() + .catch(() => undefined) + .finally(() => { + restarting = undefined; + }); + } + + /** Spawns the runtime and waits until it answers. The child becomes `child` only then. */ + async function launch(port: number, contextLength: number, gen: number): Promise { + const folders = paths(); + let spawned: ChildProcess; + try { + await mkdir(folders.homeDir, { recursive: true, mode: 0o700 }); + await mkdir(folders.modelsDir, { recursive: true, mode: 0o700 }); + // Asked to stop in the meantime: nothing is started only to be stopped again. + if (gen !== generation) return { ok: false, detail: 'The start was replaced by a newer request' }; + spawned = deps.spawn(folders.binary, ['serve'], { + env: runtimeEnv({ + source: deps.env, + home: folders.homeDir, + modelsDir: folders.modelsDir, + port, + contextLength, + }), + cwd: folders.homeDir, + stdio: ['ignore', 'pipe', 'pipe'], + // A child of the server, in its process group: it is not left behind by a normal exit. + detached: false, + }); + } catch (err) { + return { ok: false, detail: errorText(err) }; + } + starting = spawned; + + // The child's output: the budget line, and the last lines for the detail of a failure. + // It is read for as long as the child lives, so a full pipe never blocks it. + const tail: string[] = []; + let measured: number | undefined; + let budgetSeen: () => void = () => undefined; + const budget = new Promise((resolve) => { + budgetSeen = resolve; + }); + const onLine = (text: string): void => { + const line = text.length > LOG_LINE_MAX_CHARS ? text.slice(0, LOG_LINE_MAX_CHARS) : text; + if (line.trim() === '') return; + tail.push(line); + if (tail.length > LOG_TAIL_LINES) tail.shift(); + const total = parseInferenceComputeTotal(line); + if (total === undefined) return; + measured = total; + if (child === spawned) setState({ measuredTotalBytes: total }); + budgetSeen(); + }; + for (const stream of [spawned.stdout, spawned.stderr]) { + if (!stream) continue; + let pending = ''; + stream.setEncoding('utf8'); + stream.on('data', (chunk: string) => { + pending += chunk; + let breakAt = pending.indexOf('\n'); + while (breakAt >= 0) { + onLine(pending.slice(0, breakAt)); + pending = pending.slice(breakAt + 1); + breakAt = pending.indexOf('\n'); + } + // A line without an end is cut, so the buffer cannot grow without bound. + if (pending.length > LOG_LINE_MAX_CHARS) pending = pending.slice(0, LOG_LINE_MAX_CHARS); + }); + stream.on('error', () => undefined); + } + + let ended: string | undefined; + let goneNow: () => void = () => undefined; + const gone = new Promise((resolve) => { + goneNow = resolve; + }); + // The last resort when the server exits without its shutdown: the child must not outlive it. + const onProcessExit = (): void => { + try { + spawned.kill('SIGTERM'); + } catch { + // Gone already. + } + }; + process.once('exit', onProcessExit); + spawned.once('error', (err) => { + // A spawn that failed emits no exit. + if (spawned.pid === undefined) process.off('exit', onProcessExit); + ended ??= `The runtime could not be started: ${errorText(err)}`; + goneNow(); + }); + spawned.once('exit', (code, signal) => { + process.off('exit', onProcessExit); + ended ??= signal ? `The runtime was ended by ${signal}` : `The runtime exited with code ${String(code)}`; + goneNow(); + if (starting === spawned) starting = undefined; + if (child !== spawned) return; + // Our running child exited by itself. + child = undefined; + setState({ running: false, pid: undefined }); + restart(); + }); + + const fail = async (reason: string): Promise => { + if (starting === spawned) starting = undefined; + await stopChild(spawned); + const log = tail.join('\n'); + return { ok: false, detail: clip(log === '' ? reason : `${reason}\n${log}`) }; + }; + + const target = { baseUrl: `http://127.0.0.1:${port}`, fetch: deps.fetch }; + const deadline = Date.now() + deps.startTimeoutMs; + for (;;) { + if (ended !== undefined) return fail(ended); + if (gen !== generation) return fail('The start was replaced by a newer request'); + const version = await getRuntimeVersion(target); + if (ended !== undefined) return fail(ended); + if (version !== undefined) { + if (version !== pin.version) return fail(`The runtime answers as version ${version}, not ${pin.version}`); + break; + } + if (Date.now() >= deadline) { + return fail(`The runtime did not answer within ${Math.round(deps.startTimeoutMs / 1000)} seconds`); + } + await deps.sleep(START_POLL_MS); + } + + // The budget line follows the first answer within a moment. An older runtime may not print it. + if (measured === undefined) await Promise.race([budget, gone, pause(deps.budgetWaitMs)]); + if (ended !== undefined) return fail(ended); + if (gen !== generation) return fail('The start was replaced by a newer request'); + + starting = undefined; + child = spawned; + const pid = spawned.pid ?? 0; + setState({ + running: true, + pid, + port, + contextLength, + measuredTotalBytes: measured, + gaveUp: false, + }); + return { ok: true, pid, ...(measured !== undefined ? { measuredTotalBytes: measured } : {}) }; + } + + /** Keeps an orphan of an earlier server: it is watched by pid, because its exit is no event of ours. */ + function adopt(pid: number, port: number, contextLength: number): void { + adoptedPid = pid; + setState({ running: true, pid, port, contextLength, measuredTotalBytes: undefined, gaveUp: false }); + let looking = false; + orphanTimer = setInterval(() => { + if (looking || adoptedPid !== pid) return; + looking = true; + void deps + .commandOfPid(pid) + .catch(() => undefined) + .then((command) => { + looking = false; + if (adoptedPid !== pid || isOwn(command)) return; + // The orphan is gone: from here on the runtime is a child of our own. + clearInterval(orphanTimer); + orphanTimer = undefined; + adoptedPid = undefined; + setState({ running: false, pid: undefined }); + restart(); + }); + }, deps.orphanPollMs); + orphanTimer.unref(); + } + + /** Whether keeping this orphan keeps something: a model is loaded, and at the wanted context. */ + async function worthAdopting( + target: { baseUrl: string; fetch: typeof fetch }, + contextLength: number + ): Promise { + const loaded = await listRunning(target); + return loaded.length > 0 && loaded.every((model) => model.contextLength === contextLength); + } + + /** Waits for the port to come free after an orphan was told to go. */ + async function portFreeWithin(port: number, ms: number): Promise { + const until = Date.now() + ms; + for (;;) { + if (await deps.isPortFree(port).catch(() => false)) return true; + if (Date.now() >= until) return false; + await pause(100); + } + } + + async function doStart(options: RuntimeStartOptions): Promise { + const { port, contextLength, previousPid } = options; + if (!Number.isInteger(port) || port < 1 || port > 65535) { + return { ok: false, code: 'runtime-start-failed', detail: `Not a port: ${String(port)}` }; + } + if (!Number.isInteger(contextLength) || contextLength < 1) { + return { ok: false, code: 'runtime-start-failed', detail: `Not a context length: ${String(contextLength)}` }; + } + // Already running as asked: nothing to do. + const alive = (child !== undefined && !hasExited(child)) || adoptedPid !== undefined; + if (alive && state.running && state.port === port && state.contextLength === contextLength) { + return { + ok: true, + pid: state.pid ?? 0, + port, + orphan: 'none', + ...(state.measuredTotalBytes !== undefined ? { measuredTotalBytes: state.measuredTotalBytes } : {}), + }; + } + + generation += 1; + const gen = generation; + await restarting; + await stopCurrent(); + wanted = { port, contextLength }; + setState({ + running: false, + pid: undefined, + port: undefined, + contextLength: undefined, + measuredTotalBytes: undefined, + restarts: 0, + gaveUp: false, + }); + + // Who holds the port, if anybody does. + const target = { baseUrl: `http://127.0.0.1:${port}`, fetch: deps.fetch }; + const ownOrphan = + previousPid !== undefined && + Number.isInteger(previousPid) && + isOwn(await deps.commandOfPid(previousPid).catch(() => undefined)) + ? previousPid + : undefined; + const version = await getRuntimeVersion(target); + let orphan: 'none' | 'adopted' | 'replaced' = 'none'; + if (ownOrphan !== undefined) { + if (version === pin.version && (await worthAdopting(target, contextLength))) { + adopt(ownOrphan, port, contextLength); + return { ok: true, pid: ownOrphan, port, orphan: 'adopted' }; + } + await stopOrphan(ownOrphan); + orphan = 'replaced'; + } else if (version !== undefined) { + return { + ok: false, + code: 'port-in-use', + detail: `A model runtime that Codeman did not start answers on port ${port} (version ${version})`, + }; + } + if (!(await portFreeWithin(port, orphan === 'replaced' ? STOP_GRACE_MS : 0))) { + return { ok: false, code: 'port-in-use', detail: `Another program holds port ${port}` }; + } + + const launched = await launch(port, contextLength, gen); + if (!launched.ok) return { ok: false, code: 'runtime-start-failed', detail: launched.detail }; + return { + ok: true, + pid: launched.pid, + port, + orphan, + ...(launched.measuredTotalBytes !== undefined ? { measuredTotalBytes: launched.measuredTotalBytes } : {}), + }; + } + + async function doStop(): Promise { + generation += 1; + wanted = undefined; + await restarting; + await stopCurrent(); + setState({ + running: false, + pid: undefined, + port: undefined, + contextLength: undefined, + measuredTotalBytes: undefined, + restarts: 0, + gaveUp: false, + }); + } + + return { + start: (options) => + serialize(() => doStart(options)).catch( + (err): RuntimeStartResult => ({ ok: false, code: 'runtime-start-failed', detail: errorText(err) }) + ), + stop: () => { + // At once, not when its turn comes: a start that is still waiting for the runtime gives up. + generation += 1; + return serialize(doStop).catch(() => undefined); + }, + status: () => ({ ...state }), + }; } diff --git a/test/local-model-developer-tools.test.ts b/test/local-model-developer-tools.test.ts new file mode 100644 index 00000000..8f443ca3 --- /dev/null +++ b/test/local-model-developer-tools.test.ts @@ -0,0 +1,153 @@ +/** + * @fileoverview Tests for src/local-model/developer-tools.ts: whether this Mac has the + * developer tools a coding agent calls. + * + * Pinned: the ONLY command it ever runs is `/usr/bin/xcode-select -p` (never `git` or + * `python3`, which open a system window on a Mac without the tools), with a timeout; exit 0 + * is `present`, any other exit is `missing`, no answer is `unknown`; off macOS nothing is run + * at all; nothing it is given can make it throw; and an answer is reused for a minute, so a + * window that polls once a second does not spawn once a second. + * + * Port: N/A (the runner is injected; nothing is spawned). + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + checkDeveloperTools, + createDeveloperToolsCheck, + DEVELOPER_TOOLS_TIMEOUT_MS, + resetDeveloperToolsCache, +} from '../src/local-model/developer-tools.js'; + +type Run = (file: string, args: readonly string[], timeoutMs: number) => Promise; + +/** A runner that answers `code` and records how it was called. */ +function runner(code: number | null): { run: Run; calls: [string, readonly string[], number][] } { + const calls: [string, readonly string[], number][] = []; + return { + calls, + run: async (file, args, timeoutMs) => { + calls.push([file, args, timeoutMs]); + return code; + }, + }; +} + +afterEach(() => { + resetDeveloperToolsCache(); + vi.restoreAllMocks(); +}); + +describe('checkDeveloperTools', () => { + it('runs exactly one command: /usr/bin/xcode-select -p, bounded', async () => { + const { run, calls } = runner(0); + expect(await checkDeveloperTools({ platform: 'darwin', run })).toBe('present'); + expect(calls).toEqual([['/usr/bin/xcode-select', ['-p'], DEVELOPER_TOOLS_TIMEOUT_MS]]); + expect(DEVELOPER_TOOLS_TIMEOUT_MS).toBe(2_000); + }); + + it('never names git, python or xcrun, whatever the answer is', async () => { + for (const code of [0, 1, 2, null]) { + const { run, calls } = runner(code); + await checkDeveloperTools({ platform: 'darwin', run }); + expect(calls).toHaveLength(1); + expect(calls[0][0]).toBe('/usr/bin/xcode-select'); + expect(JSON.stringify(calls)).not.toMatch(/git|python|xcrun/); + } + }); + + it('reads exit 0 as present and any other exit as missing', async () => { + expect(await checkDeveloperTools({ platform: 'darwin', run: runner(0).run })).toBe('present'); + // A Mac without the tools: "unable to get active developer directory", exit 2. + expect(await checkDeveloperTools({ platform: 'darwin', run: runner(2).run })).toBe('missing'); + expect(await checkDeveloperTools({ platform: 'darwin', run: runner(1).run })).toBe('missing'); + expect(await checkDeveloperTools({ platform: 'darwin', run: runner(127).run })).toBe('missing'); + }); + + it('reads no answer as unknown: a timeout, a command that could not be started, a runner that throws', async () => { + expect(await checkDeveloperTools({ platform: 'darwin', run: runner(null).run })).toBe('unknown'); + const throwing: Run = async () => { + throw new Error('spawn EACCES'); + }; + expect(await checkDeveloperTools({ platform: 'darwin', run: throwing })).toBe('unknown'); + // An exit code that is no whole number is no answer either. + expect(await checkDeveloperTools({ platform: 'darwin', run: runner(Number.NaN).run })).toBe('unknown'); + }); + + it('runs nothing off macOS and answers unknown', async () => { + for (const platform of ['linux', 'win32', 'freebsd']) { + const { run, calls } = runner(0); + expect(await checkDeveloperTools({ platform, run }), platform).toBe('unknown'); + expect(calls).toEqual([]); + } + }); + + it('spawns nothing with the default runner under vitest', async () => { + // The real `execFile` is never reached here: on a Mac this would otherwise run the command. + expect(await checkDeveloperTools({ platform: 'darwin' })).toBe('unknown'); + // And with no argument at all it answers for this machine without throwing. + expect(['present', 'missing', 'unknown']).toContain(await checkDeveloperTools()); + }); +}); + +describe('the cached check', () => { + it('reuses an answer for a minute, then asks again', async () => { + const { run, calls } = runner(2); + let clock = 1_000_000; + const check = createDeveloperToolsCheck({ platform: 'darwin', run, now: () => clock }); + expect(await check()).toBe('missing'); + // A window that polls once a second for a minute. + for (let second = 1; second < 60; second++) { + clock += 1_000; + expect(await check()).toBe('missing'); + } + expect(calls).toHaveLength(1); + clock += 1_000; + expect(await check()).toBe('missing'); + expect(calls).toHaveLength(2); + }); + + it('shares one run among callers that ask at the same time', async () => { + let release: (code: number) => void = () => undefined; + let started = 0; + const run: Run = () => { + started += 1; + return new Promise((resolve) => { + release = resolve; + }); + }; + const check = createDeveloperToolsCheck({ platform: 'darwin', run }); + const asked = [check(), check(), check()]; + await Promise.resolve(); + expect(started).toBe(1); + release(0); + expect(await Promise.all(asked)).toEqual(['present', 'present', 'present']); + }); + + it('does not remember "unknown": a check that did not work is tried again', async () => { + let calls = 0; + // No answer the first time, exit 0 from then on. + const run: Run = async () => (calls++ === 0 ? null : 0); + const check = createDeveloperToolsCheck({ platform: 'darwin', run, now: () => 5 }); + expect(await check()).toBe('unknown'); + expect(await check()).toBe('present'); + expect(await check()).toBe('present'); + expect(calls).toBe(2); + }); + + it('sees the tools once they were installed, after the minute', async () => { + const answers = [2, 0]; + let calls = 0; + let clock = 0; + const check = createDeveloperToolsCheck({ + platform: 'darwin', + run: async () => answers[calls++] ?? 0, + now: () => clock, + }); + expect(await check()).toBe('missing'); + clock = 59_999; + expect(await check()).toBe('missing'); + clock = 60_000; + expect(await check()).toBe('present'); + }); +}); diff --git a/test/local-model-ollama-client.test.ts b/test/local-model-ollama-client.test.ts new file mode 100644 index 00000000..35fd13b0 --- /dev/null +++ b/test/local-model-ollama-client.test.ts @@ -0,0 +1,950 @@ +/** + * @fileoverview Tests for src/local-model/ollama-client.ts: what Codeman asks its own model + * runtime, and the pure functions that read the answers. + * + * Laid out like the module: + * + * 1. The listings (`/api/version`, `/api/tags`, `/api/ps`, `/v1/models`, the delete), against + * the fake runtime, and what each answers when nothing listens. + * 2. The pull parser, pure: the two real line shapes fed in small pieces across line breaks, an + * error line, a resumed pull, and that it keeps no line. + * 3. `pullModel()` against the fake: the total it reports, progress, an error line and a cut + * stream that are retried, the cap of 5 retries, a model the registry does not have (no + * retry), a resumed pull, and Cancel. + * 4. The check after the pull: the two request bodies, the two judges (pure), and + * `verifyModel()` against the fake for every way it can end, the answer without a tool call + * among them. + * + * Port: none fixed (the fake runtime listens on 0). + */ + +import { afterEach, describe, expect, it } from 'vitest'; +import { + buildChatProbeRequest, + buildToolProbeRequest, + createPullParser, + deleteModel, + getRuntimeVersion, + judgeLoaded, + judgeToolProbe, + listOpenAiModels, + listRunning, + listTags, + PULL_RETRIES, + pullModel, + readModelDigest, + TOOL_PROBE_MAX_TOKENS, + verifyModel, + type OllamaPsModel, + type OllamaTarget, + type PullProgress, +} from '../src/local-model/ollama-client.js'; +import { + DEFAULT_FAKE_MODEL, + freeLoopbackPort, + REAL_PULL_ERROR_LINE, + startFakeOllama, + type FakeOllama, + type FakeOllamaOptions, +} from './fixtures/fake-ollama.js'; + +const MODEL = DEFAULT_FAKE_MODEL.id; +const MODEL_BYTES = DEFAULT_FAKE_MODEL.bytes; +const CONTEXT = 131072; + +let fake: FakeOllama | undefined; + +afterEach(async () => { + await fake?.close(); + fake = undefined; +}); + +/** Starts the fake and returns the target that points at it. */ +async function runtime(options: FakeOllamaOptions = {}): Promise<{ fake: FakeOllama; target: OllamaTarget }> { + fake = await startFakeOllama(options); + return { fake, target: { baseUrl: fake.baseUrl } }; +} + +/** A target nothing listens on. */ +async function deadTarget(): Promise { + return { baseUrl: `http://127.0.0.1:${await freeLoopbackPort()}` }; +} + +/** No waiting between retries. */ +const noSleep = async (): Promise => undefined; + +// ---------- 1. Listings ---------- + +describe('the listings', () => { + it('reads the version of a runtime', async () => { + const { target } = await runtime(); + expect(await getRuntimeVersion(target)).toBe('0.40.15'); + }); + + it('answers undefined for a port nothing listens on, and for a listener that is no runtime', async () => { + expect(await getRuntimeVersion(await deadTarget())).toBeUndefined(); + // Something answers, but not with a version. + const notARuntime: OllamaTarget = { + baseUrl: 'http://127.0.0.1:1', + fetch: async () => new Response('hello', { status: 200 }), + }; + expect(await getRuntimeVersion(notARuntime)).toBeUndefined(); + const wrongShape: OllamaTarget = { + baseUrl: 'http://127.0.0.1:1', + fetch: async () => new Response(JSON.stringify({ version: 42 }), { status: 200 }), + }; + expect(await getRuntimeVersion(wrongShape)).toBeUndefined(); + const refused: OllamaTarget = { + baseUrl: 'http://127.0.0.1:1', + fetch: async () => new Response(JSON.stringify({ version: '0.40.15' }), { status: 503 }), + }; + expect(await getRuntimeVersion(refused)).toBeUndefined(); + }); + + it('lists the model on disk with what a Mac reports: size, digest, format, runner', async () => { + const { target } = await runtime({ installed: true }); + expect(await listTags(target)).toEqual([ + { + name: 'qwen3.6:35b-a3b', + size: 23_105_798_956, + digest: '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba', + format: 'safetensors', + runner: 'mlx', + capabilities: ['completion', 'vision', 'tools', 'thinking'], + }, + ]); + expect(await readModelDigest(target, MODEL)).toBe(DEFAULT_FAKE_MODEL.digest); + expect(await readModelDigest(target, 'qwen3.5:9b')).toBeUndefined(); + }); + + it('stores a digest in one form: lowercase hex without the prefix', async () => { + const target: OllamaTarget = { + baseUrl: 'http://127.0.0.1:1', + fetch: async () => + new Response(JSON.stringify({ models: [{ name: MODEL, size: 1, digest: 'sha256:0CE5FF81' }] }), { + status: 200, + }), + }; + expect(await readModelDigest(target, MODEL)).toBe('0ce5ff81'); + }); + + it('lists nothing on an empty store, and reads the "data: null" of the OpenAI route as no model', async () => { + const { target } = await runtime(); + expect(await listTags(target)).toEqual([]); + expect(await listRunning(target)).toEqual([]); + expect(await listOpenAiModels(target)).toEqual([]); + expect(await readModelDigest(target, MODEL)).toBeUndefined(); + }); + + it('lists the model ids of the OpenAI route exactly as they are listed', async () => { + const { target } = await runtime({ installed: true }); + expect(await listOpenAiModels(target)).toEqual(['qwen3.6:35b-a3b']); + }); + + it('lists a loaded model with its memory, the part on the GPU and its context', async () => { + const { fake: server, target } = await runtime({ installed: true }); + expect(await listRunning(target)).toEqual([]); + server.state.loaded = true; + expect(await listRunning(target)).toEqual([ + { + name: 'qwen3.6:35b-a3b', + size: 23_205_798_956, + sizeVram: 23_205_798_956, + contextLength: 131072, + digest: DEFAULT_FAKE_MODEL.digest, + }, + ]); + }); + + it('answers an empty list, never an exception, when nothing listens', async () => { + const target = await deadTarget(); + expect(await listTags(target)).toEqual([]); + expect(await listRunning(target)).toEqual([]); + expect(await listOpenAiModels(target)).toEqual([]); + expect(await readModelDigest(target, MODEL)).toBeUndefined(); + }); + + it('skips rows that are not models instead of failing on them', async () => { + const target: OllamaTarget = { + baseUrl: 'http://127.0.0.1:1', + fetch: async () => + new Response(JSON.stringify({ models: [null, 'x', {}, { name: 'a:b' }, { model: 'c:d', size: -1 }] }), { + status: 200, + }), + }; + expect(await listTags(target)).toEqual([ + { name: 'a:b', size: 0, digest: '' }, + { name: 'c:d', size: 0, digest: '' }, + ]); + // A row without a digest has none to compare. + expect(await readModelDigest(target, 'a:b')).toBeUndefined(); + }); + + it('deletes a model, and calls a model that never was there gone too', async () => { + const { fake: server, target } = await runtime({ installed: true }); + expect(await deleteModel(target, MODEL)).toBe(true); + expect(server.requests.at(-1)).toMatchObject({ method: 'DELETE', path: '/api/delete', body: { model: MODEL } }); + expect(await listTags(target)).toEqual([]); + // The runtime answers 404 for it now. + expect(await deleteModel(target, MODEL)).toBe(true); + // No runtime: nothing was deleted, and it says so. + expect(await deleteModel(await deadTarget(), MODEL)).toBe(false); + }); + + it('gives up on a listing when the caller aborts', async () => { + const { target } = await runtime({ installed: true }); + const controller = new AbortController(); + controller.abort(); + expect(await listTags(target, controller.signal)).toEqual([]); + expect(await getRuntimeVersion(target, controller.signal)).toBeUndefined(); + }); +}); + +// ---------- 2. The pull parser ---------- + +describe('createPullParser', () => { + const mlxLine = (completed: number): string => + JSON.stringify({ status: 'pulling model', digest: 'sha256:model', total: MODEL_BYTES, completed }); + + it('adds up the line shape of an MLX build', () => { + const parser = createPullParser(); + parser.push('{"status":"pulling manifest"}\n'); + expect(parser.snapshot()).toEqual({ + completedBytes: 0, + totalBytes: 0, + status: 'pulling manifest', + succeeded: false, + lines: 1, + }); + parser.push(`${mlxLine(0)}\n${mlxLine(6_200_000_000)}\n`); + expect(parser.snapshot()).toMatchObject({ completedBytes: 6_200_000_000, totalBytes: 23_105_798_956, lines: 3 }); + parser.push(`${mlxLine(MODEL_BYTES)}\n{"status":"writing manifest"}\n{"status":"success"}\n`); + parser.end(); + expect(parser.snapshot()).toEqual({ + completedBytes: 23_105_798_956, + totalBytes: 23_105_798_956, + status: 'success', + succeeded: true, + lines: 6, + }); + }); + + it('reads the same stream whatever the size of the pieces, also across line breaks', () => { + const stream = + ['{"status":"pulling manifest"}', mlxLine(0), mlxLine(1_000), mlxLine(999_999_999), mlxLine(MODEL_BYTES)].join( + '\n' + ) + '\n{"status":"writing manifest"}\n{"status":"success"}\n'; + const whole = createPullParser(); + whole.push(stream); + whole.end(); + const expected = whole.snapshot(); + expect(expected).toMatchObject({ succeeded: true, completedBytes: MODEL_BYTES, lines: 7 }); + + for (const size of [1, 2, 3, 5, 7, 13, 64, 1000]) { + const parser = createPullParser(); + for (let at = 0; at < stream.length; at += size) parser.push(stream.slice(at, at + size)); + parser.end(); + expect(parser.snapshot(), `pieces of ${size}`).toEqual(expected); + } + // And as bytes, one at a time: what a socket may really deliver. + const bytes = new TextEncoder().encode(stream); + const parser = createPullParser(); + for (const byte of bytes) parser.push(Uint8Array.of(byte)); + parser.end(); + expect(parser.snapshot()).toEqual(expected); + }); + + it('survives a character of several bytes that is split between two pieces', () => { + const bytes = new TextEncoder().encode('{"status":"lädt Manifest ✓"}\n{"status":"success"}\n'); + for (let cut = 1; cut < bytes.length; cut++) { + const parser = createPullParser(); + parser.push(bytes.subarray(0, cut)); + parser.push(bytes.subarray(cut)); + parser.end(); + expect(parser.snapshot(), `cut at ${cut}`).toMatchObject({ succeeded: true, lines: 2 }); + } + const parser = createPullParser(); + parser.push(bytes.subarray(0, 14)); + parser.push(bytes.subarray(14)); + expect(parser.snapshot().lines).toBe(2); + }); + + it('adds up the layers of a GGUF build, each announced without "completed" first', () => { + const parser = createPullParser(); + const big = { status: 'pulling c5396e06af29', digest: 'sha256:c5396e06af29aaaa', total: 397_807_936 }; + const small = { status: 'pulling 66b9ea09bd5b', digest: 'sha256:66b9ea09bd5bbbbb', total: 68 }; + parser.push(`${JSON.stringify(big)}\n`); + // Known, nothing downloaded yet. + expect(parser.snapshot()).toMatchObject({ totalBytes: 397_807_936, completedBytes: 0 }); + parser.push(`${JSON.stringify({ ...big, completed: 100_000_000 })}\n`); + parser.push(`${JSON.stringify(small)}\n`); + // The total grows when a new layer appears; a line without "completed" keeps what was seen. + parser.push(`${JSON.stringify(big)}\n`); + expect(parser.snapshot()).toMatchObject({ totalBytes: 397_808_004, completedBytes: 100_000_000 }); + parser.push(`${JSON.stringify({ ...big, completed: 397_807_936 })}\n`); + parser.push(`${JSON.stringify({ ...small, completed: 68 })}\n`); + parser.push('{"status":"verifying sha256 digest"}\n{"status":"writing manifest"}\n{"status":"success"}\n'); + expect(parser.snapshot()).toEqual({ + completedBytes: 397_808_004, + totalBytes: 397_808_004, + status: 'success', + succeeded: true, + lines: 9, + }); + }); + + it('reads a resumed pull from where it starts, not from zero', () => { + const parser = createPullParser(); + parser.push(`{"status":"pulling manifest"}\n${mlxLine(11_000_000_000)}\n`); + expect(parser.snapshot()).toMatchObject({ completedBytes: 11_000_000_000, totalBytes: MODEL_BYTES }); + }); + + it('keeps the text of an error line, the real one included', () => { + const parser = createPullParser(); + parser.push(`{"status":"pulling manifest"}\n${mlxLine(5)}\n${REAL_PULL_ERROR_LINE}\n`); + parser.end(); + expect(parser.snapshot()).toEqual({ + completedBytes: 5, + totalBytes: MODEL_BYTES, + status: 'pulling model', + succeeded: false, + error: 'max retries exceeded: local error: tls: bad record MAC', + lines: 3, + }); + // The first error stays. + parser.push('{"error":"another"}\n'); + expect(parser.snapshot().error).toBe('max retries exceeded: local error: tls: bad record MAC'); + }); + + it('reads a last line that has no line break only when the body ends', () => { + const parser = createPullParser(); + parser.push('{"status":"success"}'); + expect(parser.snapshot()).toMatchObject({ succeeded: false, lines: 0 }); + parser.end(); + expect(parser.snapshot()).toMatchObject({ succeeded: true, lines: 1 }); + }); + + it('skips a line that is not JSON and still counts it; a blank line is no line', () => { + const parser = createPullParser(); + parser.push('not json\n\n \n[1,2]\n"text"\nnull\n{"status":"success"}\n'); + expect(parser.snapshot()).toEqual({ + completedBytes: 0, + totalBytes: 0, + status: 'success', + succeeded: true, + lines: 5, + }); + }); + + it('never lets "completed" pass "total", and ignores sizes that are no byte counts', () => { + const parser = createPullParser(); + parser.push('{"status":"pulling a","digest":"sha256:a","total":100,"completed":250}\n'); + parser.push('{"status":"pulling b","digest":"sha256:b","total":-5,"completed":1}\n'); + parser.push('{"status":"pulling c","digest":"sha256:c","total":"100","completed":1}\n'); + expect(parser.snapshot()).toMatchObject({ completedBytes: 100, totalBytes: 100 }); + }); + + it('keeps no line: a million progress lines leave it as small as ten', () => { + // What it holds is one entry per layer and the unfinished line. Proven by its answers: the + // totals are those of the LAST line of each layer, and the count is every line. + const parser = createPullParser(); + const lines = 200_000; + let block = ''; + for (let i = 1; i <= lines; i++) { + block += `${mlxLine(i * 1000)}\n`; + if (i % 5000 === 0) { + parser.push(block); + block = ''; + } + } + parser.end(); + expect(parser.snapshot()).toEqual({ + completedBytes: lines * 1000, + totalBytes: MODEL_BYTES, + status: 'pulling model', + succeeded: false, + lines, + }); + }); + + it('drops a "line" that never ends instead of buffering it', () => { + const parser = createPullParser(); + // Three megabytes without a line break, then the stream goes on normally. + for (let i = 0; i < 3; i++) parser.push('x'.repeat(1024 * 1024)); + parser.push('tail of the endless line\n{"status":"success"}\n'); + expect(parser.snapshot()).toMatchObject({ succeeded: true, lines: 2 }); + }); +}); + +// ---------- 3. pullModel ---------- + +describe('pullModel', () => { + it('pulls the model and reports the total of the pull itself', async () => { + const { fake: server, target } = await runtime(); + const seen: PullProgress[] = []; + const result = await pullModel(target, MODEL, { onProgress: (progress) => seen.push(progress) }); + expect(result).toEqual({ ok: true, totalBytes: 23_105_798_956 }); + expect(server.state.installed).toBe(true); + expect(server.requests.filter((r) => r.path === '/api/pull')).toEqual([ + expect.objectContaining({ method: 'POST', body: { model: MODEL, stream: true } }), + ]); + // The last report is the finished pull, and no report runs backwards. + expect(seen.at(-1)).toMatchObject({ completedBytes: MODEL_BYTES, totalBytes: MODEL_BYTES }); + for (let i = 1; i < seen.length; i++) { + expect(seen[i].completedBytes).toBeGreaterThanOrEqual(seen[i - 1].completedBytes); + } + for (const progress of seen) expect(progress.totalBytes).toBe(MODEL_BYTES); + }); + + it('adds up the layers of a GGUF pull to the same total', async () => { + const { target } = await runtime({ pull: { shape: 'gguf', progressLines: 6 } }); + expect(await pullModel(target, MODEL)).toEqual({ ok: true, totalBytes: MODEL_BYTES }); + }); + + it('reports a speed from the clock it is given', async () => { + const { target } = await runtime({ pull: { progressLines: 8, lineDelayMs: 5 } }); + const seen: PullProgress[] = []; + let clock = 0; + const result = await pullModel(target, MODEL, { + onProgress: (progress) => seen.push(progress), + // One second passes between two looks at the clock. + now: () => (clock += 1000), + }); + expect(result.ok).toBe(true); + const speeds = seen.flatMap((progress) => (progress.bytesPerSecond === undefined ? [] : [progress.bytesPerSecond])); + expect(speeds.length).toBeGreaterThan(0); + for (const speed of speeds) { + expect(Number.isInteger(speed)).toBe(true); + expect(speed).toBeGreaterThanOrEqual(0); + } + // The first report has nothing to compare with. + expect(seen[0].bytesPerSecond).toBeUndefined(); + expect(Math.max(...speeds)).toBeGreaterThan(0); + }); + + it('starts again after an error line, and the new pull continues where the last one stopped', async () => { + const { fake: server, target } = await runtime({ + pull: { fail: 'error-line', failTimes: 2, failAfterLines: 10, progressLines: 20 }, + }); + const waits: number[] = []; + const firstOfEachPull: number[] = []; + let last = -1; + const result = await pullModel(target, MODEL, { + sleep: async (ms) => { + waits.push(ms); + last = -1; + }, + onProgress: (progress) => { + if (last === -1) firstOfEachPull.push(progress.completedBytes); + last = progress.completedBytes; + }, + }); + expect(result).toEqual({ ok: true, totalBytes: MODEL_BYTES }); + // Two failures, then the one that succeeded. + expect(server.state.pulls).toBe(3); + // It waits before each retry, longer every time. + expect(waits).toEqual([1000, 2000]); + // The second and third pull do not start at zero. + expect(firstOfEachPull).toHaveLength(3); + expect(firstOfEachPull[1]).toBeGreaterThan(0); + expect(firstOfEachPull[2]).toBeGreaterThan(firstOfEachPull[1]); + }); + + it('starts again after a stream that was cut without a word', async () => { + const { fake: server, target } = await runtime({ pull: { fail: 'cut', failTimes: 1 } }); + expect(await pullModel(target, MODEL, { sleep: noSleep })).toEqual({ ok: true, totalBytes: MODEL_BYTES }); + expect(server.state.pulls).toBe(2); + }); + + it('gives up after 5 retries, with the error the runtime gave', async () => { + expect(PULL_RETRIES).toBe(5); + const { fake: server, target } = await runtime({ pull: { fail: 'error-line' } }); + const waits: number[] = []; + const result = await pullModel(target, MODEL, { sleep: async (ms) => void waits.push(ms) }); + expect(result).toEqual({ + ok: false, + code: 'pull-failed', + detail: 'max retries exceeded: local error: tls: bad record MAC', + }); + // The first pull and five more. + expect(server.state.pulls).toBe(6); + expect(waits).toEqual([1000, 2000, 4000, 8000, 16000]); + expect(server.state.installed).toBe(false); + }); + + it('gives up on a stream that is cut every time, and says that it ended early', async () => { + const { fake: server, target } = await runtime({ pull: { fail: 'cut' } }); + const result = await pullModel(target, MODEL, { sleep: noSleep, retries: 2 }); + expect(result).toMatchObject({ ok: false, code: 'pull-failed' }); + expect((result as { detail?: string }).detail).toBeTruthy(); + expect(server.state.pulls).toBe(3); + }); + + it('does not retry when 0 retries are asked for, and falls back to 5 for a nonsense count', async () => { + const { fake: server, target } = await runtime({ pull: { fail: 'error-line' } }); + expect(await pullModel(target, MODEL, { sleep: noSleep, retries: 0 })).toMatchObject({ code: 'pull-failed' }); + expect(server.state.pulls).toBe(1); + await pullModel(target, MODEL, { sleep: noSleep, retries: -3 }); + expect(server.state.pulls).toBe(1 + 6); + }); + + it('does not ask again for a model the registry does not have', async () => { + const { fake: server, target } = await runtime({ pull: { fail: 'not-found' } }); + const result = await pullModel(target, MODEL, { sleep: noSleep }); + expect(result).toEqual({ + ok: false, + code: 'model-not-found', + detail: 'pull model manifest: file does not exist', + }); + expect(server.state.pulls).toBe(1); + }); + + it('does not ask again when the disk is full', async () => { + const target: OllamaTarget = { + baseUrl: 'http://127.0.0.1:1', + fetch: async () => + new Response( + '{"status":"pulling manifest"}\n{"error":"write /models/blobs/sha256-abc-partial: no space left on device"}\n', + { status: 200 } + ), + }; + let asked = 0; + const counting: OllamaTarget = { + ...target, + fetch: (...args) => { + asked += 1; + return (target.fetch as typeof fetch)(...args); + }, + }; + expect(await pullModel(counting, MODEL, { sleep: noSleep })).toMatchObject({ ok: false, code: 'disk-full' }); + expect(asked).toBe(1); + }); + + it('retries, then fails, when no runtime answers at all', async () => { + const waits: number[] = []; + const result = await pullModel(await deadTarget(), MODEL, { sleep: async (ms) => void waits.push(ms), retries: 2 }); + expect(result).toMatchObject({ ok: false, code: 'pull-failed' }); + expect(waits).toHaveLength(2); + }); + + it('continues a pull that an earlier run left unfinished', async () => { + const { target } = await runtime({ pull: { resumeFrom: 11_000_000_000 } }); + const seen: number[] = []; + const result = await pullModel(target, MODEL, { onProgress: (progress) => seen.push(progress.completedBytes) }); + expect(result).toEqual({ ok: true, totalBytes: MODEL_BYTES }); + // The first figure the window shows is where the earlier run stopped. + expect(seen[0]).toBeGreaterThanOrEqual(11_000_000_000); + }); + + it('reports the size of a model that was complete already, when the pull names no layer', async () => { + // A runtime that answers "success" at once: the size is what it lists for the model. + const { fake: server } = await runtime({ installed: true }); + const target: OllamaTarget = { + baseUrl: server.baseUrl, + fetch: async (input, init) => + String(input).endsWith('/api/pull') + ? new Response('{"status":"pulling manifest"}\n{"status":"success"}\n', { status: 200 }) + : fetch(input, init), + }; + expect(await pullModel(target, MODEL)).toEqual({ ok: true, totalBytes: MODEL_BYTES }); + }); + + it('stops the download when the caller cancels, and does not start again', async () => { + const { fake: server, target } = await runtime({ pull: { progressLines: 200, lineDelayMs: 10 } }); + const controller = new AbortController(); + const seen: PullProgress[] = []; + const result = await pullModel(target, MODEL, { + signal: controller.signal, + sleep: noSleep, + onProgress: (progress) => { + seen.push(progress); + controller.abort(); + }, + }); + expect(result).toEqual({ ok: false, code: 'cancelled' }); + expect(seen).toHaveLength(1); + expect(server.state.pulls).toBe(1); + expect(server.state.installed).toBe(false); + // Pulling again is the resume: it starts where the cancelled one stopped. + server.switches.pull.lineDelayMs = 0; + const again: number[] = []; + expect(await pullModel(target, MODEL, { onProgress: (p) => again.push(p.completedBytes) })).toEqual({ + ok: true, + totalBytes: MODEL_BYTES, + }); + expect(again[0]).toBeGreaterThan(0); + }); + + it('answers cancelled at once for a signal that is already aborted', async () => { + const { fake: server, target } = await runtime(); + const controller = new AbortController(); + controller.abort(); + expect(await pullModel(target, MODEL, { signal: controller.signal })).toEqual({ ok: false, code: 'cancelled' }); + expect(server.state.pulls).toBe(0); + }); + + it('calls a cancel a cancel also on its last try, never a failed download', async () => { + // With no retry left, the cut stream of an abort must not read as "the download keeps stopping". + const { fake: server, target } = await runtime({ pull: { progressLines: 200, lineDelayMs: 10 } }); + const controller = new AbortController(); + const result = await pullModel(target, MODEL, { + signal: controller.signal, + retries: 0, + onProgress: () => controller.abort(), + }); + expect(result).toEqual({ ok: false, code: 'cancelled' }); + expect(server.state.pulls).toBe(1); + }); + + it('does not let a progress callback that throws fail the download', async () => { + const { fake: server, target } = await runtime(); + let called = 0; + const result = await pullModel(target, MODEL, { + sleep: noSleep, + onProgress: () => { + called += 1; + throw new Error('the window is gone'); + }, + }); + expect(called).toBeGreaterThan(0); + expect(result).toEqual({ ok: true, totalBytes: MODEL_BYTES }); + expect(server.state.pulls).toBe(1); + }); + + it('answers with a result, not an exception, when its own clock throws', async () => { + const { target } = await runtime(); + const result = await pullModel(target, MODEL, { + sleep: noSleep, + retries: 1, + onProgress: () => undefined, + now: () => { + throw new Error('no clock'); + }, + }); + expect(result).toEqual({ ok: false, code: 'pull-failed', detail: 'no clock' }); + }); +}); + +// ---------- 4. The check after the pull ---------- + +describe('the probe requests', () => { + it('builds a tiny chat with reasoning off', () => { + expect(buildChatProbeRequest(MODEL)).toEqual({ + model: MODEL, + messages: [{ role: 'user', content: 'Reply with the single word: ready' }], + reasoning_effort: 'none', + max_tokens: 16, + temperature: 0, + stream: false, + }); + expect(buildChatProbeRequest(MODEL)).not.toHaveProperty('tools'); + }); + + it('builds a tool probe with a tools array, reasoning off and 1024 tokens', () => { + expect(TOOL_PROBE_MAX_TOKENS).toBe(1024); + const request = buildToolProbeRequest(MODEL) as { + model: string; + messages: { role: string; content: string }[]; + tools: { type: string; function: { name: string; parameters: { type: string; required: string[] } } }[]; + reasoning_effort: string; + max_tokens: number; + stream: boolean; + }; + expect(request.model).toBe(MODEL); + expect(request.reasoning_effort).toBe('none'); + expect(request.max_tokens).toBe(1024); + expect(request.stream).toBe(false); + expect(request.tools.map((tool) => [tool.type, tool.function.name])).toEqual([ + ['function', 'read_file'], + ['function', 'write_file'], + ['function', 'run_command'], + ]); + for (const tool of request.tools) { + expect(tool.function.parameters.type).toBe('object'); + expect(tool.function.parameters.required.length).toBeGreaterThan(0); + } + // The task can only be done with a tool. + expect(request.messages.at(-1)).toEqual({ + role: 'user', + content: 'Read the file README.md and tell me what it says.', + }); + // Nothing forces the call: the model has to decide to make it. + expect(request).not.toHaveProperty('tool_choice'); + }); +}); + +describe('judgeLoaded', () => { + const loaded = (overrides: Partial = {}): OllamaPsModel => ({ + name: MODEL, + size: 23_205_798_956, + sizeVram: 23_205_798_956, + contextLength: CONTEXT, + ...overrides, + }); + + it('passes a model that is all on the GPU at the chosen context', () => { + expect(judgeLoaded([loaded()], MODEL, CONTEXT)).toEqual({ + ok: true, + loaded: { context: 131072, sizeBytes: 23_205_798_956, sizeVramBytes: 23_205_798_956 }, + }); + // Another loaded model beside it does not matter. + expect(judgeLoaded([loaded({ name: 'other:1', contextLength: 4096 }), loaded()], MODEL, CONTEXT).ok).toBe(true); + }); + + it('answers load-failed when the model is not listed', () => { + expect(judgeLoaded([], MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'load-failed' }); + expect(judgeLoaded([loaded({ name: 'other:1' })], MODEL, CONTEXT)).toMatchObject({ + ok: false, + code: 'load-failed', + }); + expect(judgeLoaded([loaded({ size: 0, sizeVram: 0 })], MODEL, CONTEXT)).toMatchObject({ + ok: false, + code: 'load-failed', + }); + }); + + it('answers does-not-fit when part of the model is not on the GPU', () => { + const judged = judgeLoaded([loaded({ sizeVram: 20_000_000_000 })], MODEL, CONTEXT); + expect(judged).toEqual({ + ok: false, + code: 'does-not-fit', + detail: '20000000000 of 23205798956 bytes are on the GPU', + }); + // One byte is enough. + expect(judgeLoaded([loaded({ sizeVram: 23_205_798_955 })], MODEL, CONTEXT)).toMatchObject({ code: 'does-not-fit' }); + expect(judgeLoaded([loaded({ sizeVram: 0 })], MODEL, CONTEXT)).toMatchObject({ code: 'does-not-fit' }); + }); + + it('answers context-not-applied when the runtime loaded another context', () => { + expect(judgeLoaded([loaded({ contextLength: 32768 })], MODEL, CONTEXT)).toEqual({ + ok: false, + code: 'context-not-applied', + detail: 'The runtime reports a context of 32768 tokens, not 131072', + }); + // A larger one is not the chosen one either. + expect(judgeLoaded([loaded({ contextLength: 262144 })], MODEL, CONTEXT)).toMatchObject({ + code: 'context-not-applied', + }); + }); + + it('looks at the GPU before the context', () => { + expect(judgeLoaded([loaded({ sizeVram: 1, contextLength: 4096 })], MODEL, CONTEXT)).toMatchObject({ + code: 'does-not-fit', + }); + }); +}); + +describe('judgeToolProbe', () => { + /** An answer of `/v1/chat/completions` with this message and finish reason. */ + const answer = (message: Record, finishReason: string): unknown => ({ + id: 'chatcmpl-1', + object: 'chat.completion', + choices: [{ index: 0, message: { role: 'assistant', ...message }, finish_reason: finishReason }], + }); + const call = (argumentsValue: unknown, name: unknown = 'read_file'): Record => ({ + id: 'call_1', + index: 0, + type: 'function', + function: { name, arguments: argumentsValue }, + }); + + it('passes on a tool call whose arguments parse as a JSON object', () => { + expect( + judgeToolProbe(200, answer({ content: '', tool_calls: [call('{"path":"README.md"}')] }, 'tool_calls')) + ).toEqual({ ok: true }); + // Arguments that are an object already, and an empty object, are arguments too. + expect(judgeToolProbe(200, answer({ tool_calls: [call({ path: 'README.md' })] }, 'tool_calls'))).toEqual({ + ok: true, + }); + expect(judgeToolProbe(200, answer({ tool_calls: [call('{}')] }, 'tool_calls'))).toEqual({ ok: true }); + // The finish reason is not what decides: the call is. + expect(judgeToolProbe(200, answer({ tool_calls: [call('{"path":"a"}')] }, 'stop'))).toEqual({ ok: true }); + }); + + it('answers no-tool-call for an answer in plain text', () => { + expect(judgeToolProbe(200, answer({ content: 'I would read the file, but I cannot.' }, 'stop'))).toEqual({ + ok: false, + code: 'no-tool-call', + detail: 'The model answered in text: I would read the file, but I cannot.', + }); + expect(judgeToolProbe(200, answer({ content: '', tool_calls: [] }, 'stop'))).toMatchObject({ + ok: false, + code: 'no-tool-call', + }); + }); + + it('answers no-tool-call for a tool call whose arguments are not a JSON object', () => { + for (const bad of ['{"path": ', 'README.md', '"README.md"', '[1]', 'null', '', 42, null, undefined]) { + expect(judgeToolProbe(200, answer({ tool_calls: [call(bad)] }, 'tool_calls')), JSON.stringify(bad)).toMatchObject( + { + ok: false, + code: 'no-tool-call', + } + ); + } + // A call without a name, and one good call beside a broken one. + expect(judgeToolProbe(200, answer({ tool_calls: [call('{}', '')] }, 'tool_calls'))).toMatchObject({ + code: 'no-tool-call', + }); + expect( + judgeToolProbe(200, answer({ tool_calls: [call('{"path":"a"}'), call('{"path": ')] }, 'tool_calls')) + ).toMatchObject({ code: 'no-tool-call' }); + expect(judgeToolProbe(200, answer({ tool_calls: ['read_file'] }, 'tool_calls'))).toMatchObject({ + code: 'no-tool-call', + }); + }); + + it('answers probe-unfinished when the token limit ended the answer: that proves nothing', () => { + expect( + judgeToolProbe(200, answer({ content: '', reasoning: 'The user wants me to read a file. I should' }, 'length')) + ).toEqual({ ok: false, code: 'probe-unfinished', detail: 'The answer ended at 1024 tokens' }); + }); + + it('answers load-failed for any other status, with what the runtime said', () => { + expect(judgeToolProbe(500, { error: { message: 'llm server error: out of memory' } })).toEqual({ + ok: false, + code: 'load-failed', + detail: 'llm server error: out of memory', + }); + expect(judgeToolProbe(404, { error: "model 'x' not found" })).toMatchObject({ + code: 'load-failed', + detail: "model 'x' not found", + }); + expect(judgeToolProbe(502, undefined)).toEqual({ + ok: false, + code: 'load-failed', + detail: 'The runtime answered 502', + }); + }); + + it('answers no-tool-call for a 200 it cannot read at all', () => { + for (const body of [undefined, null, 'text', [], {}, { choices: [] }, { choices: [null] }, { choices: [{}] }]) { + expect(judgeToolProbe(200, body), JSON.stringify(body)).toEqual({ + ok: false, + code: 'no-tool-call', + detail: 'The answer holds no tool call', + }); + } + }); +}); + +describe('verifyModel', () => { + it('passes a model that loads, sits on the GPU at the chosen context and calls a tool', async () => { + const { fake: server, target } = await runtime({ installed: true }); + expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({ + ok: true, + loaded: { context: 131072, sizeBytes: 23_205_798_956, sizeVramBytes: 23_205_798_956 }, + }); + // In this order: the chat that loads it, the process list, the tool probe. + expect(server.requests.map((r) => `${r.method} ${r.path}`)).toEqual([ + 'POST /v1/chat/completions', + 'GET /api/ps', + 'POST /v1/chat/completions', + ]); + const [chat, , probe] = server.requests.map((r) => r.body as Record); + expect(chat).toEqual(buildChatProbeRequest(MODEL)); + expect(chat).toMatchObject({ reasoning_effort: 'none' }); + expect(chat).not.toHaveProperty('tools'); + expect(probe).toEqual(buildToolProbeRequest(MODEL)); + expect(probe).toMatchObject({ reasoning_effort: 'none', max_tokens: 1024 }); + expect(Array.isArray(probe.tools)).toBe(true); + }); + + it('answers load-failed when the chat that loads the model fails, and asks nothing more', async () => { + const { fake: server, target } = await runtime({ installed: true, chat: 'error' }); + expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({ + ok: false, + code: 'load-failed', + detail: 'llm server error: out of memory', + }); + expect(server.requests).toHaveLength(1); + }); + + it('answers load-failed for a model that is not on disk', async () => { + const { target } = await runtime(); + expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'load-failed' }); + }); + + it('answers load-failed when no runtime answers', async () => { + expect(await verifyModel(await deadTarget(), MODEL, CONTEXT)).toEqual({ + ok: false, + code: 'load-failed', + detail: 'The runtime did not answer the first request', + }); + }); + + it('answers load-failed when the model is not in the process list after the chat', async () => { + const { fake: server, target } = await runtime({ installed: true, ps: { loaded: false } }); + expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'load-failed' }); + // The tool probe was not sent. + expect(server.requests.filter((r) => r.path === '/v1/chat/completions')).toHaveLength(1); + }); + + it('answers does-not-fit when part of the model is not on the GPU, before any tool probe', async () => { + const { fake: server, target } = await runtime({ installed: true, ps: { sizeVram: 18_000_000_000 } }); + expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({ + ok: false, + code: 'does-not-fit', + detail: '18000000000 of 23205798956 bytes are on the GPU', + }); + expect(server.requests.filter((r) => r.path === '/v1/chat/completions')).toHaveLength(1); + }); + + it('answers context-not-applied when the runtime loaded another context than the chosen one', async () => { + const { target } = await runtime({ installed: true, contextLength: 32768 }); + expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'context-not-applied' }); + // The same runtime passes when 32768 is what was chosen. + expect(await verifyModel(target, MODEL, 32768)).toMatchObject({ ok: true, loaded: { context: 32768 } }); + }); + + it('answers no-tool-call when the model answers the tool probe in text', async () => { + const { target } = await runtime({ installed: true, tool: 'none' }); + expect(await verifyModel(target, MODEL, CONTEXT)).toEqual({ + ok: false, + code: 'no-tool-call', + detail: 'The model answered in text: I would read the file, but I cannot.', + }); + }); + + it('answers no-tool-call when the arguments of the tool call do not parse', async () => { + const { target } = await runtime({ installed: true, tool: 'bad-arguments' }); + expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'no-tool-call' }); + }); + + it('answers probe-unfinished, not no-tool-call, when the token limit ended the answer', async () => { + const { target } = await runtime({ installed: true, tool: 'length' }); + expect(await verifyModel(target, MODEL, CONTEXT)).toMatchObject({ ok: false, code: 'probe-unfinished' }); + }); + + it('answers cancelled for an aborted signal, at whatever point it is noticed', async () => { + const { fake: server, target } = await runtime({ installed: true }); + const before = new AbortController(); + before.abort(); + expect(await verifyModel(target, MODEL, CONTEXT, before.signal)).toEqual({ ok: false, code: 'cancelled' }); + expect(server.requests).toHaveLength(0); + + // Aborted while the first request is on its way. + const during = new AbortController(); + const slow: OllamaTarget = { + baseUrl: server.baseUrl, + fetch: async (input, init) => { + during.abort(); + return fetch(input, init); + }, + }; + expect(await verifyModel(slow, MODEL, CONTEXT, during.signal)).toEqual({ ok: false, code: 'cancelled' }); + }); + + it('never throws, whatever the runtime sends back', async () => { + const answers = ['', 'not json', '[]', 'null', '{"choices":"x"}']; + for (const text of answers) { + const target: OllamaTarget = { + baseUrl: 'http://127.0.0.1:1', + fetch: async () => new Response(text, { status: 200 }), + }; + const result = await verifyModel(target, MODEL, CONTEXT); + // The chat was "answered", the process list lists nothing. + expect(result, text).toMatchObject({ ok: false, code: 'load-failed' }); + } + }); +}); diff --git a/test/local-model-runtime.test.ts b/test/local-model-runtime.test.ts new file mode 100644 index 00000000..26886e5b --- /dev/null +++ b/test/local-model-runtime.test.ts @@ -0,0 +1,982 @@ +/** + * @fileoverview Tests for src/local-model/runtime.ts, everything but the supervisor (that is + * test/local-model-supervisor.test.ts): the pin, the budget line, the child's environment, + * the download with its hash, the port, and the removal. + * + * Pinned: + * - The pin is the reviewed constant, and a test never reaches it: the default `fetch` + * refuses under vitest. + * - The budget parser reads the REAL log line of both test Macs and no other line. + * - The environment is an allowlist: no `CODEMAN_PASSWORD`, none of the user's own + * `OLLAMA_*`, nothing that is not named. + * - The download: a matching hash unpacks, a wrong hash deletes the file and unpacks + * NOTHING, a cut body is continued with `Range`, the retries are capped, a body longer + * than the pin is refused, Cancel keeps the partial file for the next run. + * - The port is chosen by binding. + * - The removal deletes only inside the one folder and follows no symlink out of it. + * + * Port: none fixed (the fake runtime listens on 0; the port tests bind 0 and read the port). + */ + +import { execFile } from 'node:child_process'; +import { createHash, randomBytes } from 'node:crypto'; +import { existsSync } from 'node:fs'; +import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { createServer, type AddressInfo, type Server } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { localModelPaths, type LocalModelPaths } from '../src/local-model/paths.js'; +import { + chooseRuntimePort, + ensureRuntimeInstalled, + isLoopbackPortFree, + isOwnRuntimeCommand, + isRuntimeInstalled, + OLLAMA_RUNTIME_PIN, + parseInferenceComputeTotal, + removeLocalModelFiles, + RUNTIME_DOWNLOAD_RETRIES, + RUNTIME_KEEP_ALIVE, + RUNTIME_MAX_RESTARTS, + RUNTIME_PORT_FIRST, + RUNTIME_PORT_SPAN, + runtimeEnv, + type RuntimeDownloadProgress, +} from '../src/local-model/runtime.js'; +import type { LocalModelRuntimePin } from '../src/types.js'; +import { + buildFakeArchive, + fakeRuntimeLog, + INFERENCE_COMPUTE_LINE_MACBOOK, + MACBOOK_MEASURED_TOTAL_BYTES, + REAL_INFERENCE_COMPUTE_LINE_MAC_MINI, + REAL_VRAM_DEFAULT_CONTEXT_LINE_MAC_MINI, + startFakeOllama, + type FakeOllama, +} from './fixtures/fake-ollama.js'; + +const execFileAsync = promisify(execFile); + +let fake: FakeOllama | undefined; +let paths: LocalModelPaths; +const cleanup: string[] = []; + +beforeEach(() => { + // Under the temp HOME test/setup.ts gives this file. + paths = localModelPaths('0.40.15'); +}); + +afterEach(async () => { + await fake?.close(); + fake = undefined; + await rm(paths.root, { recursive: true, force: true }); + for (const dir of cleanup.splice(0)) await rm(dir, { recursive: true, force: true }); +}); + +/** A temp folder outside the local model folder, removed after the test. */ +async function outsideDir(): Promise { + const dir = await mkdtemp(join(tmpdir(), 'local-model-outside-')); + cleanup.push(dir); + return dir; +} + +/** What `ensureRuntimeInstalled()` needs to reach the fake: the global `fetch`, and no waiting. */ +function deps(waits: number[] = []): { fetch: typeof fetch; sleep: (ms: number) => Promise } { + return { + fetch, + sleep: async (ms) => { + waits.push(ms); + }, + }; +} + +/** The archive requests the fake saw, as the `Range` header of each (`undefined` without one). */ +function archiveRanges(server: FakeOllama): (string | undefined)[] { + return server.requests + .filter((request) => request.path === '/download/ollama-darwin.tgz') + .map((request) => request.headers.range as string | undefined); +} + +/** A fake whose archive is about 40 KB: a runtime script plus bytes that do not compress. */ +async function fakeWithLargeArchive(archive: FakeOllama['switches']['archive'] | object = {}): Promise { + fake = await startFakeOllama({ + archive, + archiveFiles: { ollama: '#!/bin/sh\necho "fake runtime"\n', 'lib/blob.bin': randomBytes(30_000).toString('hex') }, + }); + return fake; +} + +// ---------- The pin and the constants ---------- + +describe('the pin', () => { + it('is the reviewed build: version, address, size and hash', () => { + expect(OLLAMA_RUNTIME_PIN).toEqual({ + id: 'ollama', + version: '0.40.15', + url: 'https://github.com/ollama/ollama/releases/download/v0.40.15/ollama-darwin.tgz', + bytes: 185_971_671, + sha256: '607c79d5d6315ef9d896db83adde736a4c0057196329843544e79b95afb260a8', + unpackedBytes: 579_000_000, + minMacosMajor: 14, + }); + }); + + it('cannot be changed by code that holds it', () => { + expect(Object.isFrozen(OLLAMA_RUNTIME_PIN)).toBe(true); + expect(() => { + (OLLAMA_RUNTIME_PIN as { url: string }).url = 'https://example.com/other.tgz'; + }).toThrow(); + expect(OLLAMA_RUNTIME_PIN.url).toMatch(/^https:\/\/github\.com\/ollama\/ollama\/releases\/download\/v0\.40\.15\//); + }); + + it('states the limits of the supervisor and the download', () => { + expect(RUNTIME_PORT_FIRST).toBe(11435); + expect(RUNTIME_PORT_SPAN).toBe(20); + expect(RUNTIME_MAX_RESTARTS).toBe(3); + expect(RUNTIME_DOWNLOAD_RETRIES).toBe(3); + expect(RUNTIME_KEEP_ALIVE).toBe('10m'); + }); + + it('is never downloaded by a test: the default fetch refuses under vitest', async () => { + const waits: number[] = []; + const result = await ensureRuntimeInstalled({ deps: { sleep: async (ms) => void waits.push(ms) } }); + expect(result).toEqual({ + ok: false, + code: 'runtime-download-failed', + detail: 'fetch is not available under vitest: the test must inject its own', + }); + // Refused at once, not after the retries. + expect(waits).toEqual([]); + expect(await isRuntimeInstalled()).toBe(false); + }); +}); + +// ---------- The budget line ---------- + +describe('parseInferenceComputeTotal', () => { + it('reads the device total of the real line of the Mac mini', () => { + // total="11.8 GiB": 11.8 x 1024^3, rounded down. + expect(parseInferenceComputeTotal(REAL_INFERENCE_COMPUTE_LINE_MAC_MINI)).toBe(12_670_153_523); + expect(12_670_153_523).toBe(Math.floor(11.8 * 1024 ** 3)); + }); + + it('reads 37.4 GiB, what the MacBook reported, as 40,157,944,217 bytes', () => { + expect(parseInferenceComputeTotal(INFERENCE_COMPUTE_LINE_MACBOOK)).toBe(40_157_944_217); + expect(parseInferenceComputeTotal(INFERENCE_COMPUTE_LINE_MACBOOK)).toBe(MACBOOK_MEASURED_TOTAL_BYTES); + // The two fields quoted in the measurement notes are enough. + expect(parseInferenceComputeTotal('msg="inference compute" total="37.4 GiB" available="37.4 GiB"')).toBe( + 40_157_944_217 + ); + }); + + it('does not read the line that follows it, which carries the total under another key', () => { + expect(parseInferenceComputeTotal(REAL_VRAM_DEFAULT_CONTEXT_LINE_MAC_MINI)).toBeUndefined(); + // Not even when that key sits on an "inference compute" line. + expect( + parseInferenceComputeTotal('msg="inference compute" total_vram="11.8 GiB" subtotal="1 GiB"') + ).toBeUndefined(); + }); + + it('finds exactly one line of a start-up log', () => { + const totals = fakeRuntimeLog('37.4 GiB').map(parseInferenceComputeTotal); + expect(totals).toEqual([undefined, undefined, 40_157_944_217, undefined]); + }); + + it('reads the binary units the runtime prints', () => { + const line = (total: string): string => `level=INFO msg="inference compute" id=0 total="${total}" available="1 B"`; + expect(parseInferenceComputeTotal(line('512 B'))).toBe(512); + expect(parseInferenceComputeTotal(line('8 KiB'))).toBe(8192); + expect(parseInferenceComputeTotal(line('12713.1 MiB'))).toBe(Math.floor(12713.1 * 1024 ** 2)); + expect(parseInferenceComputeTotal(line('96 GiB'))).toBe(96 * 1024 ** 3); + expect(parseInferenceComputeTotal(line('1.5 TiB'))).toBe(1.5 * 1024 ** 4); + // Without the space between the number and the unit. + expect(parseInferenceComputeTotal(line('16GiB'))).toBe(16 * 1024 ** 3); + }); + + it('answers undefined for anything it does not understand, and never throws', () => { + const line = (total: string): string => `msg="inference compute" total="${total}"`; + for (const text of [ + '', + 'total="37.4 GiB"', + 'msg="inference compute"', + line(''), + line('GiB'), + line('37.4'), + line('37.4 GB'), + line('-1 GiB'), + line('0 GiB'), + line('1e9 GiB'), + line('NaN GiB'), + 'msg="inference compute" total=37.4GiB', + 'msg="discovering available GPUs..."', + ]) { + expect(parseInferenceComputeTotal(text), JSON.stringify(text)).toBeUndefined(); + } + }); +}); + +// ---------- The environment ---------- + +describe('runtimeEnv', () => { + const input = (source: NodeJS.ProcessEnv): Parameters[0] => ({ + source, + home: '/data/local-model/home', + modelsDir: '/data/local-model/models', + port: 11435, + contextLength: 131072, + }); + + /** A server environment with everything a real one may hold. */ + const SERVER_ENV: NodeJS.ProcessEnv = { + PATH: '/usr/local/bin:/usr/bin:/bin', + HOME: '/Users/arkon', + TMPDIR: '/var/folders/xy/T/', + LANG: 'de_CH.UTF-8', + HTTP_PROXY: 'http://proxy:3128', + HTTPS_PROXY: 'http://proxy:3128', + NO_PROXY: 'localhost,127.0.0.1', + http_proxy: 'http://proxy-lower:3128', + https_proxy: 'http://proxy-lower:3128', + no_proxy: 'localhost', + // Codeman's own secrets and switches. + CODEMAN_PASSWORD: 'hunter2', + CODEMAN_USERNAME: 'admin', + CODEMAN_MUX: '1', + CODEMAN_API_URL: 'http://127.0.0.1:3000', + // The user's settings for their OWN runtime. + OLLAMA_HOST: '0.0.0.0:11434', + OLLAMA_ORIGINS: '*', + OLLAMA_MODELS: '/Users/arkon/.ollama/models', + OLLAMA_DEBUG: '1', + OLLAMA_DEBUG_LOG_REQUESTS: '1', + OLLAMA_KEEP_ALIVE: '-1', + OLLAMA_NUM_PARALLEL: '8', + OLLAMA_MAX_LOADED_MODELS: '0', + OLLAMA_CONTEXT_LENGTH: '4096', + OLLAMA_NO_CLOUD: '0', + OLLAMA_REMOTES: 'evil.example', + OLLAMA_AUTH: '1', + LLAMA_ARG_CACHE_RAM: '0', + // Other secrets a server environment carries. + ANTHROPIC_API_KEY: 'sk-ant-secret', + OPENAI_API_KEY: 'sk-secret', + GITHUB_TOKEN: 'ghp_secret', + AWS_SECRET_ACCESS_KEY: 'aws-secret', + SSH_AUTH_SOCK: '/tmp/ssh.sock', + USER: 'arkon', + SHELL: '/bin/zsh', + LC_ALL: 'C', + NODE_OPTIONS: '--inspect', + DYLD_INSERT_LIBRARIES: '/tmp/evil.dylib', + }; + + it('holds exactly the allowlist: what is copied, the private HOME, and seven settings', () => { + expect(runtimeEnv(input(SERVER_ENV))).toEqual({ + PATH: '/usr/local/bin:/usr/bin:/bin', + TMPDIR: '/var/folders/xy/T/', + LANG: 'de_CH.UTF-8', + HTTP_PROXY: 'http://proxy:3128', + HTTPS_PROXY: 'http://proxy:3128', + NO_PROXY: 'localhost,127.0.0.1', + http_proxy: 'http://proxy-lower:3128', + https_proxy: 'http://proxy-lower:3128', + no_proxy: 'localhost', + HOME: '/data/local-model/home', + OLLAMA_HOST: '127.0.0.1:11435', + OLLAMA_MODELS: '/data/local-model/models', + OLLAMA_CONTEXT_LENGTH: '131072', + OLLAMA_NO_CLOUD: '1', + OLLAMA_KEEP_ALIVE: '10m', + OLLAMA_NUM_PARALLEL: '1', + OLLAMA_MAX_LOADED_MODELS: '1', + }); + }); + + it('lets no CODEMAN_ variable through, the password least of all', () => { + const env = runtimeEnv(input(SERVER_ENV)); + expect(Object.keys(env).filter((name) => name.startsWith('CODEMAN_'))).toEqual([]); + expect(JSON.stringify(env)).not.toContain('hunter2'); + }); + + it("lets none of the user's own OLLAMA_ settings through: every one is ours or absent", () => { + const env = runtimeEnv(input(SERVER_ENV)); + expect( + Object.keys(env) + .filter((name) => name.startsWith('OLLAMA_')) + .sort() + ).toEqual([ + 'OLLAMA_CONTEXT_LENGTH', + 'OLLAMA_HOST', + 'OLLAMA_KEEP_ALIVE', + 'OLLAMA_MAX_LOADED_MODELS', + 'OLLAMA_MODELS', + 'OLLAMA_NO_CLOUD', + 'OLLAMA_NUM_PARALLEL', + ]); + // The inherited values lost against ours, and the ones we do not set are gone. + expect(env.OLLAMA_HOST).toBe('127.0.0.1:11435'); + expect(env.OLLAMA_KEEP_ALIVE).toBe('10m'); + expect(env.OLLAMA_MAX_LOADED_MODELS).toBe('1'); + for (const name of [ + 'OLLAMA_ORIGINS', + 'OLLAMA_DEBUG', + 'OLLAMA_DEBUG_LOG_REQUESTS', + 'OLLAMA_REMOTES', + 'OLLAMA_AUTH', + ]) { + expect(env, name).not.toHaveProperty(name); + } + expect(env).not.toHaveProperty('LLAMA_ARG_CACHE_RAM'); + }); + + it('carries no secret and no value of the server that is not on the list', () => { + const env = runtimeEnv(input(SERVER_ENV)); + const text = JSON.stringify(env); + for (const secret of ['sk-ant-secret', 'sk-secret', 'ghp_secret', 'aws-secret', 'evil', '/Users/arkon']) { + expect(text, secret).not.toContain(secret); + } + // Whatever is in the result and came from the server is one of the nine copied names. + const copied = [ + 'PATH', + 'TMPDIR', + 'LANG', + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'NO_PROXY', + 'http_proxy', + 'https_proxy', + 'no_proxy', + ]; + for (const [name, value] of Object.entries(env)) { + if (SERVER_ENV[name] === value) expect(copied, name).toContain(name); + } + }); + + it('binds the loopback address only, on the port it is given', () => { + for (const port of [11435, 11454, 40123]) { + expect(runtimeEnv({ ...input({}), port }).OLLAMA_HOST).toBe(`127.0.0.1:${port}`); + } + expect(runtimeEnv({ ...input({}), contextLength: 65536 }).OLLAMA_CONTEXT_LENGTH).toBe('65536'); + }); + + it('leaves out what the server does not have, and sets the rest all the same', () => { + expect(Object.keys(runtimeEnv(input({}))).sort()).toEqual([ + 'HOME', + 'OLLAMA_CONTEXT_LENGTH', + 'OLLAMA_HOST', + 'OLLAMA_KEEP_ALIVE', + 'OLLAMA_MAX_LOADED_MODELS', + 'OLLAMA_MODELS', + 'OLLAMA_NO_CLOUD', + 'OLLAMA_NUM_PARALLEL', + ]); + // An empty value is not copied either. + expect(runtimeEnv(input({ PATH: '', LANG: undefined }))).not.toHaveProperty('PATH'); + }); + + it('is pure: the server environment is not changed', () => { + const source = { ...SERVER_ENV }; + runtimeEnv(input(source)); + expect(source).toEqual(SERVER_ENV); + }); +}); + +// ---------- Whose process is it ---------- + +describe('isOwnRuntimeCommand', () => { + const root = '/Users/arkon/.codeman/local-model/runtime'; + + it('knows a runtime under our folder by its command line', () => { + expect(isOwnRuntimeCommand(`${root}/ollama-0.40.15/ollama serve`, root)).toBe(true); + expect(isOwnRuntimeCommand(` ${root}/ollama-0.40.15/ollama serve\n`, root)).toBe(true); + // Another version of ours, and a folder name with a space in it. + expect(isOwnRuntimeCommand(`${root}/ollama-0.39.0/ollama serve`, root)).toBe(true); + const spaced = '/Users/Jane Doe/.codeman/local-model/runtime'; + expect(isOwnRuntimeCommand(`${spaced}/ollama-0.40.15/ollama serve`, spaced)).toBe(true); + expect(isOwnRuntimeCommand(`${root}/ollama-0.40.15/ollama serve`, `${root}/`)).toBe(true); + }); + + it('reads /private in front of a path as the same path', () => { + const temp = '/var/folders/xy/T/codeman/local-model/runtime'; + expect(isOwnRuntimeCommand(`/private${temp}/ollama-0.40.15/ollama serve`, temp)).toBe(true); + expect(isOwnRuntimeCommand(`${temp}/ollama-0.40.15/ollama serve`, `/private${temp}`)).toBe(true); + }); + + it('does not take any other process for ours', () => { + for (const command of [ + // A runtime the user started. + '/usr/local/bin/ollama serve', + '/Applications/Ollama.app/Contents/Resources/ollama serve', + 'ollama serve', + // Our folder is only an argument, or a prefix of another folder. + `/bin/sh ${root}/ollama-0.40.15/ollama serve`, + `/usr/bin/tail -f ${root}/log`, + `${root}-other/ollama-0.40.15/ollama serve`, + `${root}`, + // A way back out of the folder. + `${root}/../../../../usr/local/bin/ollama serve`, + `${root}/ollama-0.40.15/../../../evil serve`, + '', + ]) { + expect(isOwnRuntimeCommand(command, root), command).toBe(false); + } + expect(isOwnRuntimeCommand('/anything/ollama serve', '')).toBe(false); + expect(isOwnRuntimeCommand('relative/ollama serve', 'relative')).toBe(false); + }); +}); + +// ---------- The download ---------- + +describe('ensureRuntimeInstalled', () => { + it('downloads, proves the hash, unpacks, and leaves a runtime that runs', async () => { + fake = await startFakeOllama(); + const seen: RuntimeDownloadProgress[] = []; + const result = await ensureRuntimeInstalled({ + pin: fake.pin(), + paths, + deps: deps(), + onProgress: (progress) => seen.push(progress), + }); + expect(result).toEqual({ ok: true, binaryPath: paths.binary, alreadyInstalled: false }); + expect(await isRuntimeInstalled(fake.pin(), paths)).toBe(true); + // Unpacked by the real tar: the file is executable and is the one of the archive. + expect((await stat(paths.binary)).mode & 0o111).not.toBe(0); + expect((await execFileAsync(paths.binary)).stdout).toBe('fake runtime\n'); + // The archive and the temporary folder are gone; one request was enough. + expect(existsSync(paths.archivePartial)).toBe(false); + expect(existsSync(paths.unpackDir)).toBe(false); + expect(archiveRanges(fake)).toEqual([undefined]); + // Progress ends at the whole archive, measured against the pin. + expect(seen.at(-1)).toEqual({ completedBytes: fake.archive.size, totalBytes: fake.archive.size }); + for (const progress of seen) expect(progress.totalBytes).toBe(fake.pin().bytes); + }); + + it('does nothing the second time: the runtime is already there', async () => { + fake = await startFakeOllama(); + await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); + const again = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); + expect(again).toEqual({ ok: true, binaryPath: paths.binary, alreadyInstalled: true }); + expect(fake.state.archiveRequests).toBe(1); + }); + + it('unpacks NOTHING when the hash is wrong, and deletes the download', async () => { + fake = await startFakeOllama(); + const wrongPin: LocalModelRuntimePin = { + ...fake.pin(), + sha256: createHash('sha256').update('other').digest('hex'), + }; + let unpacked = 0; + const result = await ensureRuntimeInstalled({ + pin: wrongPin, + paths, + deps: { + ...deps(), + untar: async () => { + unpacked += 1; + }, + }, + }); + expect(result).toMatchObject({ ok: false, code: 'runtime-damaged' }); + expect((result as { detail?: string }).detail).toContain(fake.archive.sha256); + expect(unpacked).toBe(0); + expect(existsSync(paths.archivePartial)).toBe(false); + expect(existsSync(paths.unpackDir)).toBe(false); + expect(existsSync(paths.runtimeDir)).toBe(false); + expect(await isRuntimeInstalled(wrongPin, paths)).toBe(false); + // The whole archive was read before the verdict: it is the hash that failed, not the size. + expect(fake.state.archiveRequests).toBe(1); + }); + + it('compares the hash whatever the case it is written in', async () => { + fake = await startFakeOllama(); + const upper: LocalModelRuntimePin = { ...fake.pin(), sha256: fake.archive.sha256.toUpperCase() }; + expect(await ensureRuntimeInstalled({ pin: upper, paths, deps: deps() })).toMatchObject({ ok: true }); + }); + + it('continues a download that was cut, with a Range request from where the file ends', async () => { + fake = await fakeWithLargeArchive({ cutAfterBytes: 1000 }); + const waits: number[] = []; + const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps(waits) }); + expect(result).toMatchObject({ ok: true, alreadyInstalled: false }); + // The first request from the start, the second for the rest. + expect(archiveRanges(fake)).toEqual([undefined, 'bytes=1000-']); + expect(waits).toEqual([500]); + expect((await execFileAsync(paths.binary)).stdout).toBe('fake runtime\n'); + }); + + it('gives up after 3 continuations and keeps the partial file for the next run', async () => { + expect(RUNTIME_DOWNLOAD_RETRIES).toBe(3); + fake = await fakeWithLargeArchive({ cutAfterBytes: 1000, cutTimes: Infinity }); + const waits: number[] = []; + const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps(waits) }); + expect(result).toMatchObject({ ok: false, code: 'runtime-download-failed' }); + expect((result as { detail?: string }).detail).toBeTruthy(); + // The first request and three more, each from where the last one stopped. + expect(archiveRanges(fake)).toEqual([undefined, 'bytes=1000-', 'bytes=2000-', 'bytes=3000-']); + expect(waits).toEqual([500, 1000, 2000]); + expect((await stat(paths.archivePartial)).size).toBe(4000); + expect(existsSync(paths.runtimeDir)).toBe(false); + + // "Try again": the next run continues the same file, it does not start over. + fake.switches.archive.cutAfterBytes = undefined; + expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); + expect(archiveRanges(fake).at(-1)).toBe('bytes=4000-'); + expect((await execFileAsync(paths.binary)).stdout).toBe('fake runtime\n'); + }); + + it('fails after the same number of tries when the server keeps answering an error', async () => { + fake = await startFakeOllama({ archive: { status: 503 } }); + const waits: number[] = []; + const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps(waits) }); + expect(result).toEqual({ + ok: false, + code: 'runtime-download-failed', + detail: 'The download server answered HTTP 503', + }); + expect(fake.state.archiveRequests).toBe(4); + expect(waits).toHaveLength(3); + }); + + it('starts over when the server ignores the Range request and sends the whole archive', async () => { + fake = await fakeWithLargeArchive({ ignoreRange: true }); + // A partial file of an earlier run: the true first 1500 bytes. + await mkdir(paths.runtimeRoot, { recursive: true }); + await writeFile(paths.archivePartial, fake.archive.body.subarray(0, 1500)); + const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); + expect(result).toMatchObject({ ok: true }); + expect(archiveRanges(fake)).toEqual(['bytes=1500-']); + // Appending the whole body to the 1500 bytes would not have had the pinned hash. + expect((await execFileAsync(paths.binary)).stdout).toBe('fake runtime\n'); + }); + + it('proves the WHOLE file: a partial file that is not the start of the archive ends as damaged', async () => { + fake = await fakeWithLargeArchive(); + await mkdir(paths.runtimeRoot, { recursive: true }); + await writeFile(paths.archivePartial, randomBytes(1500)); + const first = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); + expect(first).toMatchObject({ ok: false, code: 'runtime-damaged' }); + expect(existsSync(paths.archivePartial)).toBe(false); + expect(existsSync(paths.runtimeDir)).toBe(false); + // The damaged file is gone, so the next run downloads from the first byte. + expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); + expect(archiveRanges(fake)).toEqual(['bytes=1500-', undefined]); + }); + + it('throws away a partial file that is larger than the archive', async () => { + fake = await startFakeOllama(); + await mkdir(paths.runtimeRoot, { recursive: true }); + await writeFile(paths.archivePartial, randomBytes(fake.archive.size + 100)); + expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); + expect(archiveRanges(fake)).toEqual([undefined]); + }); + + it('verifies a file that is already complete without asking the server', async () => { + fake = await startFakeOllama(); + await mkdir(paths.runtimeRoot, { recursive: true }); + await writeFile(paths.archivePartial, fake.archive.body); + expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); + expect(fake.state.archiveRequests).toBe(0); + }); + + it('refuses a download whose size is not the pinned one, before it reads it', async () => { + fake = await startFakeOllama(); + const shortPin: LocalModelRuntimePin = { ...fake.pin(), bytes: fake.archive.size - 10 }; + const result = await ensureRuntimeInstalled({ pin: shortPin, paths, deps: deps() }); + expect(result).toEqual({ + ok: false, + code: 'runtime-damaged', + detail: `The download is ${fake.archive.size} bytes, the pinned archive has ${fake.archive.size - 10}`, + }); + expect(existsSync(paths.archivePartial)).toBe(false); + expect(fake.state.archiveRequests).toBe(1); + }); + + it('refuses a body that grows past the pinned size, when the server names no size', async () => { + fake = await startFakeOllama(); + const pin = fake.pin(); + const body = fake.archive.body; + let unpacked = 0; + // The right archive followed by more bytes, as a stream without a content-length. + const endless: typeof fetch = async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(body)); + controller.enqueue(new Uint8Array(1000)); + controller.close(); + }, + }), + { status: 200 } + ); + const result = await ensureRuntimeInstalled({ + pin, + paths, + deps: { + fetch: endless, + sleep: async () => undefined, + untar: async () => { + unpacked += 1; + }, + }, + }); + expect(result).toEqual({ + ok: false, + code: 'runtime-damaged', + detail: `The download is longer than the ${pin.bytes} bytes of the pinned archive`, + }); + expect(unpacked).toBe(0); + expect(existsSync(paths.archivePartial)).toBe(false); + }); + + /** A server that sends the first `bytes` of the archive and then nothing more, until it is aborted. */ + function stallingFetch(server: FakeOllama, bytes: number, counter: { requests: number }): typeof fetch { + return async (_input, init) => { + counter.requests += 1; + const signal = init?.signal ?? undefined; + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(server.archive.body.subarray(0, bytes))); + signal?.addEventListener('abort', () => controller.error(new DOMException('aborted', 'AbortError'))); + }, + }), + { status: 200 } + ); + }; + } + + it('stops when the caller cancels, and the next run continues the same file', async () => { + fake = await fakeWithLargeArchive(); + const counter = { requests: 0 }; + const controller = new AbortController(); + setTimeout(() => controller.abort(), 40); + const cancelled = await ensureRuntimeInstalled({ + pin: fake.pin(), + paths, + signal: controller.signal, + deps: { fetch: stallingFetch(fake, 2000, counter), sleep: async () => undefined }, + }); + expect(cancelled).toEqual({ ok: false, code: 'cancelled' }); + // One request, no retry after a cancel, and what arrived is kept. + expect(counter.requests).toBe(1); + expect((await stat(paths.archivePartial)).size).toBe(2000); + expect(existsSync(paths.runtimeDir)).toBe(false); + + expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); + expect(archiveRanges(fake)).toEqual(['bytes=2000-']); + }); + + it('answers cancelled at once for a signal that is already aborted', async () => { + fake = await startFakeOllama(); + const controller = new AbortController(); + controller.abort(); + const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, signal: controller.signal, deps: deps() }); + expect(result).toEqual({ ok: false, code: 'cancelled' }); + expect(fake.state.archiveRequests).toBe(0); + }); + + it('treats a download that delivers nothing as cut, a bounded number of times', async () => { + fake = await fakeWithLargeArchive(); + const counter = { requests: 0 }; + const result = await ensureRuntimeInstalled({ + pin: fake.pin(), + paths, + idleTimeoutMs: 30, + deps: { fetch: stallingFetch(fake, 500, counter), sleep: async () => undefined }, + }); + expect(result).toEqual({ + ok: false, + code: 'runtime-download-failed', + detail: 'The download delivered nothing for 1 s', + }); + expect(counter.requests).toBe(4); + }); + + it('answers runtime-unpack-failed when tar fails, keeps the proven archive, and unpacks it next time', async () => { + fake = await startFakeOllama(); + const failed = await ensureRuntimeInstalled({ + pin: fake.pin(), + paths, + deps: { + ...deps(), + untar: async () => { + throw new Error('tar: Error opening archive: Unrecognized archive format'); + }, + }, + }); + expect(failed).toEqual({ + ok: false, + code: 'runtime-unpack-failed', + detail: 'tar: Error opening archive: Unrecognized archive format', + }); + expect(existsSync(paths.unpackDir)).toBe(false); + expect(existsSync(paths.runtimeDir)).toBe(false); + expect((await stat(paths.archivePartial)).size).toBe(fake.archive.size); + + expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); + // Not downloaded a second time. + expect(fake.state.archiveRequests).toBe(1); + }); + + it('answers disk-full when unpacking runs out of space', async () => { + fake = await startFakeOllama(); + const result = await ensureRuntimeInstalled({ + pin: fake.pin(), + paths, + deps: { + ...deps(), + untar: async () => { + throw new Error("tar: ollama: Can't write: No space left on device"); + }, + }, + }); + expect(result).toMatchObject({ ok: false, code: 'disk-full' }); + }); + + it('refuses an archive that holds no runtime binary', async () => { + fake = await startFakeOllama({ archiveFiles: { 'README.txt': 'no binary here\n' } }); + const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); + expect(result).toEqual({ + ok: false, + code: 'runtime-unpack-failed', + detail: 'The archive holds no ollama binary', + }); + expect(existsSync(paths.runtimeDir)).toBe(false); + expect(existsSync(paths.unpackDir)).toBe(false); + }); + + it('replaces a runtime folder that an interrupted unpack left without its binary', async () => { + fake = await startFakeOllama(); + await mkdir(paths.runtimeDir, { recursive: true }); + await writeFile(join(paths.runtimeDir, 'half-written'), 'x'); + await mkdir(paths.unpackDir, { recursive: true }); + await writeFile(join(paths.unpackDir, 'stale'), 'x'); + expect(await isRuntimeInstalled(fake.pin(), paths)).toBe(false); + expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); + expect(existsSync(join(paths.runtimeDir, 'half-written'))).toBe(false); + expect(existsSync(join(paths.runtimeDir, 'stale'))).toBe(false); + expect(await readFile(paths.binary, 'utf8')).toContain('fake runtime'); + }); + + it('writes the download where only the owner can read it, inside the one folder', async () => { + fake = await fakeWithLargeArchive({ cutAfterBytes: 1000, cutTimes: Infinity }); + await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); + expect((await stat(paths.archivePartial)).mode & 0o077).toBe(0); + expect(paths.archivePartial.startsWith(paths.root)).toBe(true); + }); + + it('never throws: a version that is no version is an answer', async () => { + const badPin = { ...OLLAMA_RUNTIME_PIN, version: '../../etc' }; + expect(await isRuntimeInstalled(badPin)).toBe(false); + expect(await ensureRuntimeInstalled({ pin: badPin, deps: deps() })).toMatchObject({ + ok: false, + code: 'runtime-download-failed', + }); + }); +}); + +// ---------- The port ---------- + +describe('chooseRuntimePort', () => { + let holder: Server | undefined; + + afterEach(async () => { + await new Promise((resolve) => (holder ? holder.close(() => resolve()) : resolve())); + holder = undefined; + }); + + /** Binds a loopback port the system picks and keeps it. */ + async function holdPort(): Promise { + holder = createServer(); + await new Promise((resolve) => holder?.listen(0, '127.0.0.1', () => resolve())); + return (holder.address() as AddressInfo).port; + } + + it('takes the first port of the span that is free, in order', async () => { + const asked: number[] = []; + const port = await chooseRuntimePort({ + isFree: async (candidate) => { + asked.push(candidate); + return candidate >= 11438; + }, + }); + expect(port).toBe(11438); + expect(asked).toEqual([11435, 11436, 11437, 11438]); + }); + + it('starts at 11435 and tries 20 ports, never 11434', async () => { + const asked: number[] = []; + const port = await chooseRuntimePort({ + isFree: async (candidate) => { + asked.push(candidate); + return false; + }, + }); + expect(port).toBeUndefined(); + expect(asked).toHaveLength(20); + expect(asked[0]).toBe(11435); + expect(asked.at(-1)).toBe(11454); + expect(asked).not.toContain(11434); + }); + + it('skips the ports it is told to avoid: "Use another port"', async () => { + const asked: number[] = []; + const port = await chooseRuntimePort({ + avoid: [11435, 11436], + isFree: async (candidate) => { + asked.push(candidate); + return true; + }, + }); + expect(port).toBe(11437); + expect(asked).toEqual([11437]); + }); + + it('does not take a port whose probe failed for free', async () => { + const port = await chooseRuntimePort({ + first: 100, + span: 3, + isFree: async (candidate) => { + if (candidate === 100) throw new Error('EACCES'); + return candidate === 102; + }, + }); + expect(port).toBe(102); + }); + + it('decides by BINDING: a held port is not free, and is free again once it is let go', async () => { + const port = await holdPort(); + expect(await isLoopbackPortFree(port)).toBe(false); + expect(await chooseRuntimePort({ first: port, span: 1 })).toBeUndefined(); + await new Promise((resolve) => holder?.close(() => resolve())); + holder = undefined; + expect(await isLoopbackPortFree(port)).toBe(true); + expect(await chooseRuntimePort({ first: port, span: 1 })).toBe(port); + // The probe let go of it again: it can be bound right away. + expect(await isLoopbackPortFree(port)).toBe(true); + }); + + it('calls a number that is no port not free', async () => { + for (const port of [0, -1, 65536, 1.5, Number.NaN]) + expect(await isLoopbackPortFree(port), String(port)).toBe(false); + }); +}); + +// ---------- The removal ---------- + +describe('removeLocalModelFiles', () => { + /** Everything an install leaves on disk, plus the state file beside the folder. */ + async function installTree(): Promise { + await mkdir(join(paths.runtimeDir, 'lib'), { recursive: true }); + await writeFile(paths.binary, '#!/bin/sh\n'); + await writeFile(join(paths.runtimeDir, 'lib', 'libmlx.dylib'), 'x'); + await mkdir(join(paths.runtimeRoot, 'ollama-0.39.0'), { recursive: true }); + await writeFile(join(paths.runtimeRoot, 'ollama-0.39.0', 'ollama'), 'old'); + await writeFile(paths.archivePartial, 'partial'); + await mkdir(join(paths.homeDir, '.ollama'), { recursive: true }); + await writeFile(join(paths.homeDir, '.ollama', 'id_ed25519'), 'private key'); + await mkdir(join(paths.modelsDir, 'blobs'), { recursive: true }); + await writeFile(join(paths.modelsDir, 'blobs', 'sha256-abc'), 'weights'); + await writeFile(paths.stateFile, '{"schema":1,"state":"installed"}'); + } + + it('removes the runtime, every version of it, the archive and the private home, and keeps the models', async () => { + await installTree(); + await removeLocalModelFiles({ models: false, paths }); + expect(existsSync(paths.runtimeRoot)).toBe(false); + expect(existsSync(paths.homeDir)).toBe(false); + expect(await readFile(join(paths.modelsDir, 'blobs', 'sha256-abc'), 'utf8')).toBe('weights'); + // The folder still holds the models, so it stays. + expect(existsSync(paths.root)).toBe(true); + }); + + it('removes the models too when asked, and then the folder itself', async () => { + await installTree(); + await removeLocalModelFiles({ models: true, paths }); + expect(existsSync(paths.modelsDir)).toBe(false); + expect(existsSync(paths.root)).toBe(false); + }); + + it('leaves the state file alone: that is the business of its one writer', async () => { + await installTree(); + await removeLocalModelFiles({ models: true, paths }); + expect(await readFile(paths.stateFile, 'utf8')).toBe('{"schema":1,"state":"installed"}'); + await rm(paths.stateFile, { force: true }); + }); + + it('is fine with nothing to remove, and can be run twice', async () => { + await removeLocalModelFiles({ models: true, paths }); + await installTree(); + await removeLocalModelFiles({ models: true, paths }); + await removeLocalModelFiles({ models: true, paths }); + expect(existsSync(paths.root)).toBe(false); + await rm(paths.stateFile, { force: true }); + }); + + it('keeps a folder of the root that is not ours', async () => { + await installTree(); + await mkdir(join(paths.root, 'notes'), { recursive: true }); + await writeFile(join(paths.root, 'notes', 'mine.txt'), 'keep'); + await removeLocalModelFiles({ models: true, paths }); + expect(await readFile(join(paths.root, 'notes', 'mine.txt'), 'utf8')).toBe('keep'); + await rm(paths.stateFile, { force: true }); + }); + + it('unlinks a symlink in the place of a folder and never deletes what it points at', async () => { + const outside = await outsideDir(); + await writeFile(join(outside, 'precious.gguf'), 'the only copy'); + await mkdir(paths.root, { recursive: true }); + // The model store was moved to another disk and linked back. + await symlink(outside, paths.modelsDir); + await removeLocalModelFiles({ models: true, paths }); + expect(existsSync(paths.modelsDir)).toBe(false); + expect(await readFile(join(outside, 'precious.gguf'), 'utf8')).toBe('the only copy'); + }); + + it('removes a symlink it meets inside a folder, not the tree behind it', async () => { + const outside = await outsideDir(); + await writeFile(join(outside, 'keep.txt'), 'keep'); + await mkdir(paths.homeDir, { recursive: true }); + await symlink(outside, join(paths.homeDir, 'link-out')); + await removeLocalModelFiles({ models: false, paths }); + expect(existsSync(paths.homeDir)).toBe(false); + expect(await readFile(join(outside, 'keep.txt'), 'utf8')).toBe('keep'); + }); + + it('refuses a path that is not inside the one folder, and deletes nothing', async () => { + const outside = await outsideDir(); + await writeFile(join(outside, 'keep.txt'), 'keep'); + await installTree(); + const cases: Partial[] = [ + { runtimeRoot: outside }, + { homeDir: outside }, + { modelsDir: outside }, + // The folder itself, and a way back out of it. + { runtimeRoot: paths.root }, + { homeDir: join(paths.root, '..', 'elsewhere') }, + ]; + for (const change of cases) { + await expect(removeLocalModelFiles({ models: true, paths: { ...paths, ...change } })).rejects.toThrow(/Refusing/); + } + expect(await readFile(join(outside, 'keep.txt'), 'utf8')).toBe('keep'); + // A refusal deletes NOTHING: every folder is judged before the first one is removed. + expect(await readFile(paths.binary, 'utf8')).toBe('#!/bin/sh\n'); + expect(await readFile(join(paths.homeDir, '.ollama', 'id_ed25519'), 'utf8')).toBe('private key'); + expect(await readFile(join(paths.modelsDir, 'blobs', 'sha256-abc'), 'utf8')).toBe('weights'); + await rm(paths.stateFile, { force: true }); + }); + + it('refuses a folder whose real place is outside the real root', async () => { + // The root is real, but `runtime` inside it is reached through a linked parent. + const outside = await outsideDir(); + await mkdir(join(outside, 'runtime'), { recursive: true }); + await writeFile(join(outside, 'runtime', 'keep.txt'), 'keep'); + await mkdir(paths.root, { recursive: true }); + await symlink(outside, join(paths.root, 'nested')); + const twisted: LocalModelPaths = { ...paths, runtimeRoot: join(paths.root, 'nested', 'runtime') }; + await expect(removeLocalModelFiles({ models: false, paths: twisted })).rejects.toThrow(/Refusing/); + expect(await readFile(join(outside, 'runtime', 'keep.txt'), 'utf8')).toBe('keep'); + }); +}); diff --git a/test/local-model-supervisor.test.ts b/test/local-model-supervisor.test.ts new file mode 100644 index 00000000..e51e4049 --- /dev/null +++ b/test/local-model-supervisor.test.ts @@ -0,0 +1,796 @@ +/** + * @fileoverview Tests for the supervisor of src/local-model/runtime.ts: how Codeman starts, + * watches, restarts and stops its model runtime. + * + * Nothing real runs here. The child process is the fake of test/fixtures/fake-ollama.ts + * (`fake.spawn`): "starting" it makes the fake listen on the port of `OLLAMA_HOST` and write + * the runtime's start-up log, and killing it closes that listener. + * + * Pinned: + * - Start: one non-detached child, `/ollama serve`, with the allowlisted + * environment; it is ready when `/api/version` answers the pinned version; the device total + * of its log is reported as the measured budget. + * - The port: a listener that is not ours is `port-in-use` and is never signalled; our own + * orphan (the stored pid runs a binary under our runtime folder) is adopted when it holds a + * loaded model at the wanted context, and replaced otherwise. + * - A child that exits by itself is restarted at most 3 times, then given up. + * - Stop: SIGTERM, and nothing is restarted afterwards. + * - Under vitest nothing spawns unless the test injects it. + * + * Port: none fixed (each test reserves a free loopback port). + */ + +import { existsSync } from 'node:fs'; +import { rm } from 'node:fs/promises'; +import { createServer, type Server } from 'node:net'; +import type { ChildProcess, SpawnOptions } from 'node:child_process'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { localModelPaths, type LocalModelPaths } from '../src/local-model/paths.js'; +import { + createRuntimeSupervisor, + RUNTIME_MAX_RESTARTS, + type RuntimeStatus, + type RuntimeSupervisor, + type RuntimeSupervisorDeps, +} from '../src/local-model/runtime.js'; +import { + createFakeOllama, + FakeRuntimeChild, + freeLoopbackPort, + startFakeOllama, + type FakeOllama, + type FakeOllamaOptions, +} from './fixtures/fake-ollama.js'; + +const CONTEXT = 131072; +/** A pid of an "earlier server's" runtime. Above every real pid, like the fake's own. */ +const ORPHAN_PID = 9_800_000; + +let paths: LocalModelPaths; +let port: number; +const fakes: FakeOllama[] = []; +const supervisors: RuntimeSupervisor[] = []; +const servers: Server[] = []; + +beforeEach(async () => { + paths = localModelPaths('0.40.15'); + port = await freeLoopbackPort(); +}); + +afterEach(async () => { + for (const supervisor of supervisors.splice(0)) await supervisor.stop(); + for (const fake of fakes.splice(0)) await fake.close(); + for (const server of servers.splice(0)) await new Promise((resolve) => server.close(() => resolve())); + await rm(paths.root, { recursive: true, force: true }); +}); + +/** A fake runtime that does not listen until it is "spawned". */ +function fakeRuntime(options: FakeOllamaOptions = {}): FakeOllama { + const fake = createFakeOllama(options); + fakes.push(fake); + return fake; +} + +/** The children the fake has spawned so far. */ +function children(fake: FakeOllama): FakeRuntimeChild[] { + return fake.state.children; +} + +interface Harness { + supervisor: RuntimeSupervisor; + /** Every status `onChange` was called with. */ + changes: RuntimeStatus[]; + /** Every wait the supervisor asked for. */ + waits: number[]; + /** Every signal sent to a pid that is not our child. */ + kills: [number, NodeJS.Signals][]; +} + +/** A supervisor whose child is `fake`. Waits are instant unless a test passes its own `sleep`. */ +function supervise(fake: FakeOllama, overrides: Partial = {}): Harness { + const changes: RuntimeStatus[] = []; + const waits: number[] = []; + const kills: [number, NodeJS.Signals][] = []; + const supervisor = createRuntimeSupervisor({ + spawn: fake.spawn, + fetch, + env: { PATH: '/usr/bin:/bin' }, + pin: fake.pin(), + paths, + sleep: async (ms) => { + waits.push(ms); + }, + commandOfPid: async () => undefined, + kill: (pid, signal) => { + kills.push([pid, signal]); + }, + onChange: (status) => changes.push(status), + startTimeoutMs: 3_000, + budgetWaitMs: 500, + ...overrides, + }); + supervisors.push(supervisor); + return { supervisor, changes, waits, kills }; +} + +/** Waits until `check()` is true. Fails the test when it never is. */ +async function until(check: () => boolean, what: string, timeoutMs = 3_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!check()) { + if (Date.now() > deadline) throw new Error(`Timed out waiting for: ${what}`); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +/** The command line `ps` would print for a runtime of ours. */ +function ownCommand(): string { + return `${paths.binary} serve`; +} + +// ---------- Start ---------- + +describe('start', () => { + it('starts one child, the runtime binary with "serve", and answers once it is up', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + const result = await supervisor.start({ port, contextLength: CONTEXT }); + + expect(children(fake)).toHaveLength(1); + const child = children(fake)[0]; + expect(child.command).toBe(paths.binary); + expect(child.args).toEqual(['serve']); + expect(result).toEqual({ + ok: true, + pid: child.pid, + port, + orphan: 'none', + // 37.4 GiB, read from the child's log line "inference compute". + measuredTotalBytes: 40_157_944_217, + }); + expect(supervisor.status()).toEqual({ + running: true, + pid: child.pid, + port, + contextLength: CONTEXT, + measuredTotalBytes: 40_157_944_217, + restarts: 0, + gaveUp: false, + }); + expect(fake.port).toBe(port); + }); + + it('starts the child non-detached, with pipes, in the private home', async () => { + const fake = fakeRuntime(); + const seen: SpawnOptions[] = []; + const { supervisor } = supervise(fake, { + spawn: (command, args, options) => { + seen.push(options); + return fake.spawn(command, args, options); + }, + }); + await supervisor.start({ port, contextLength: CONTEXT }); + expect(seen).toHaveLength(1); + expect(seen[0].detached).toBe(false); + expect(seen[0].stdio).toEqual(['ignore', 'pipe', 'pipe']); + expect(seen[0].cwd).toBe(paths.homeDir); + expect(seen[0]).not.toHaveProperty('shell'); + // The folders it needs exist before it starts. + expect(existsSync(paths.homeDir)).toBe(true); + expect(existsSync(paths.modelsDir)).toBe(true); + }); + + it('gives the child the allowlisted environment and nothing of the server beyond it', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake, { + env: { + PATH: '/usr/bin:/bin', + HOME: '/Users/arkon', + LANG: 'en_US.UTF-8', + CODEMAN_PASSWORD: 'hunter2', + CODEMAN_API_URL: 'http://127.0.0.1:3000', + OLLAMA_ORIGINS: '*', + OLLAMA_HOST: '0.0.0.0:11434', + OLLAMA_KEEP_ALIVE: '-1', + ANTHROPIC_API_KEY: 'sk-ant-secret', + }, + }); + await supervisor.start({ port, contextLength: 65536 }); + expect(children(fake)[0].env).toEqual({ + PATH: '/usr/bin:/bin', + LANG: 'en_US.UTF-8', + HOME: paths.homeDir, + OLLAMA_HOST: `127.0.0.1:${port}`, + OLLAMA_MODELS: paths.modelsDir, + OLLAMA_CONTEXT_LENGTH: '65536', + OLLAMA_NO_CLOUD: '1', + OLLAMA_KEEP_ALIVE: '10m', + OLLAMA_NUM_PARALLEL: '1', + OLLAMA_MAX_LOADED_MODELS: '1', + }); + }); + + it('reads the budget the child logs, whatever the Mac', async () => { + const fake = fakeRuntime({ spawn: { total: '11.8 GiB' } }); + const { supervisor } = supervise(fake); + const result = await supervisor.start({ port, contextLength: CONTEXT }); + expect(result).toMatchObject({ ok: true, measuredTotalBytes: 12_670_153_523 }); + expect(supervisor.status().measuredTotalBytes).toBe(12_670_153_523); + }); + + it('still starts when the child prints no budget line, and reports none', async () => { + const fake = fakeRuntime({ spawn: { logsBudget: false } }); + const { supervisor } = supervise(fake, { budgetWaitMs: 60 }); + const result = await supervisor.start({ port, contextLength: CONTEXT }); + expect(result).toMatchObject({ ok: true, port }); + expect(result).not.toHaveProperty('measuredTotalBytes'); + expect(supervisor.status()).not.toHaveProperty('measuredTotalBytes'); + expect(supervisor.status().running).toBe(true); + }); + + it('is idempotent: a child that already runs as asked is kept', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + const first = await supervisor.start({ port, contextLength: CONTEXT }); + const second = await supervisor.start({ port, contextLength: CONTEXT }); + expect(second).toEqual(first); + expect(children(fake)).toHaveLength(1); + expect(children(fake)[0].signals).toEqual([]); + }); + + it('replaces a child that runs with another context length', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + await supervisor.start({ port, contextLength: CONTEXT }); + const result = await supervisor.start({ port, contextLength: 65536 }); + expect(result).toMatchObject({ ok: true, orphan: 'none' }); + expect(children(fake)).toHaveLength(2); + // The first got SIGTERM; the second runs with the new context. + expect(children(fake)[0].signals).toEqual(['SIGTERM']); + expect(children(fake)[1].env.OLLAMA_CONTEXT_LENGTH).toBe('65536'); + expect(supervisor.status()).toMatchObject({ running: true, pid: children(fake)[1].pid, contextLength: 65536 }); + // Replacing on purpose is not a crash. + expect(supervisor.status().restarts).toBe(0); + }); + + it('runs two starts one after the other, never two children at once', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + const [a, b] = await Promise.all([ + supervisor.start({ port, contextLength: CONTEXT }), + supervisor.start({ port, contextLength: CONTEXT }), + ]); + expect(a).toMatchObject({ ok: true }); + expect(b).toEqual(a); + expect(children(fake)).toHaveLength(1); + }); + + it('refuses a port or a context length that is none, without starting anything', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + for (const bad of [0, -1, 70_000, 1.5, Number.NaN]) { + expect(await supervisor.start({ port: bad, contextLength: CONTEXT }), String(bad)).toMatchObject({ + ok: false, + code: 'runtime-start-failed', + }); + } + for (const bad of [0, -5, 1.5, Number.NaN]) { + expect(await supervisor.start({ port, contextLength: bad }), String(bad)).toMatchObject({ + ok: false, + code: 'runtime-start-failed', + }); + } + expect(children(fake)).toHaveLength(0); + }); + + it('tells its caller every change, and survives a caller that throws', async () => { + const fake = fakeRuntime(); + const seen: RuntimeStatus[] = []; + const { supervisor } = supervise(fake, { + onChange: (status) => { + seen.push(status); + throw new Error('the state file is read-only'); + }, + }); + const result = await supervisor.start({ port, contextLength: CONTEXT }); + expect(result).toMatchObject({ ok: true }); + // The caller stores pid and port from this. + expect(seen.at(-1)).toMatchObject({ running: true, pid: children(fake)[0].pid, port, contextLength: CONTEXT }); + await supervisor.stop(); + expect(seen.at(-1)).toEqual({ running: false, restarts: 0, gaveUp: false }); + }); + + it('hands out a copy of its status', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + await supervisor.start({ port, contextLength: CONTEXT }); + const status = supervisor.status(); + status.running = false; + status.pid = 1; + expect(supervisor.status()).toMatchObject({ running: true, pid: children(fake)[0].pid }); + }); + + it("writes no state file: storing pid and port is the caller's business", async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + await supervisor.start({ port, contextLength: CONTEXT }); + await supervisor.stop(); + expect(existsSync(paths.stateFile)).toBe(false); + }); +}); + +// ---------- A start that fails ---------- + +describe('a start that fails', () => { + it('answers runtime-start-failed when the child never answers, and does not leave it running', async () => { + const fake = fakeRuntime({ spawn: { neverListens: true } }); + const { supervisor } = supervise(fake, { startTimeoutMs: 150 }); + const result = await supervisor.start({ port, contextLength: CONTEXT }); + expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' }); + const detail = (result as { detail?: string }).detail ?? ''; + expect(detail).toContain('did not answer'); + // The last lines of its log are the detail. + expect(detail).toContain('Listening on'); + expect(children(fake)).toHaveLength(1); + expect(children(fake)[0].signals).toEqual(['SIGTERM']); + expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false }); + }); + + it('answers runtime-start-failed when the child exits at once, with what it printed', async () => { + const fake = fakeRuntime({ spawn: { exitsAtOnce: 2 } }); + const { supervisor } = supervise(fake); + const result = await supervisor.start({ port, contextLength: CONTEXT }); + expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' }); + const detail = (result as { detail?: string }).detail ?? ''; + expect(detail).toContain('exited with code 2'); + expect(detail).toContain('address already in use'); + // A start that failed is not a crash to recover from: nothing is started again. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(children(fake)).toHaveLength(1); + expect(supervisor.status()).toMatchObject({ running: false, restarts: 0, gaveUp: false }); + }); + + it('answers runtime-start-failed when a runtime of another version answers', async () => { + const fake = fakeRuntime({ version: '0.39.2' }); + const { supervisor } = supervise(fake, { pin: { ...fake.pin(), version: '0.40.15' } }); + const result = await supervisor.start({ port, contextLength: CONTEXT }); + expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' }); + expect((result as { detail?: string }).detail).toContain('0.39.2'); + expect(children(fake)[0].signals).toEqual(['SIGTERM']); + }); + + it('answers runtime-start-failed when the binary cannot be started at all', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake, { + spawn: () => { + throw new Error('spawn EACCES'); + }, + }); + expect(await supervisor.start({ port, contextLength: CONTEXT })).toEqual({ + ok: false, + code: 'runtime-start-failed', + detail: 'spawn EACCES', + }); + }); + + it('spawns nothing under vitest unless the test injects a spawn', async () => { + // No `spawn` and no `fetch` given: the defaults refuse here. + const supervisor = createRuntimeSupervisor({ paths, startTimeoutMs: 200 }); + supervisors.push(supervisor); + const result = await supervisor.start({ port, contextLength: CONTEXT }); + // The refusal itself, not a failed attempt to run a binary that is not there. + expect(result).toEqual({ + ok: false, + code: 'runtime-start-failed', + detail: 'spawn is not available under vitest: the test must inject its own', + }); + expect(supervisor.status().running).toBe(false); + }); + + it('gives up a start at once when stop is called while it waits', async () => { + const fake = fakeRuntime({ spawn: { neverListens: true } }); + const { supervisor } = supervise(fake, { + startTimeoutMs: 20_000, + sleep: () => new Promise((resolve) => setTimeout(resolve, 10)), + }); + const starting = supervisor.start({ port, contextLength: CONTEXT }); + await until(() => children(fake).length === 1, 'the child to be spawned'); + const began = Date.now(); + await supervisor.stop(); + const result = await starting; + expect(result).toMatchObject({ ok: false, code: 'runtime-start-failed' }); + // Not after the 20 seconds a start may take. + expect(Date.now() - began).toBeLessThan(5_000); + expect(children(fake)[0].signals).toEqual(['SIGTERM']); + expect(supervisor.status().running).toBe(false); + }); +}); + +// ---------- The port ---------- + +describe('the port', () => { + /** A runtime that listens on `port` already, as an earlier server's would. */ + async function listeningRuntime(options: FakeOllamaOptions = {}): Promise { + const other = createFakeOllama(options); + fakes.push(other); + await other.listen(port); + return other; + } + + it('answers port-in-use for a runtime that is not ours, and does not touch it', async () => { + // The user's own runtime on our port. No pid of ours is stored. + const theirs = await listeningRuntime(); + const fake = fakeRuntime(); + const { supervisor, kills } = supervise(fake); + const result = await supervisor.start({ port, contextLength: CONTEXT }); + // Told apart from any other program on the port: it answers as a runtime, and is not ours. + expect(result).toEqual({ + ok: false, + code: 'port-in-use', + detail: `A model runtime that Codeman did not start answers on port ${port} (version 0.40.15)`, + }); + expect(children(fake)).toHaveLength(0); + expect(kills).toEqual([]); + expect(theirs.listening).toBe(true); + expect(supervisor.status().running).toBe(false); + }); + + it('answers port-in-use when the stored pid now runs something else', async () => { + const theirs = await listeningRuntime(); + const fake = fakeRuntime(); + const { supervisor, kills } = supervise(fake, { + // The pid was reused, or it is a runtime outside our folder. + commandOfPid: async () => '/usr/local/bin/ollama serve', + }); + const result = await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID }); + expect(result).toMatchObject({ ok: false, code: 'port-in-use' }); + expect(kills).toEqual([]); + expect(theirs.listening).toBe(true); + expect(children(fake)).toHaveLength(0); + }); + + it('answers port-in-use for a listener that is no runtime at all', async () => { + // Some other program: it takes the connection and hangs up. + const holder = createServer((socket) => socket.destroy()); + servers.push(holder); + await new Promise((resolve) => holder.listen(port, '127.0.0.1', () => resolve())); + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + expect(await supervisor.start({ port, contextLength: CONTEXT })).toEqual({ + ok: false, + code: 'port-in-use', + detail: `Another program holds port ${port}`, + }); + expect(children(fake)).toHaveLength(0); + }); + + it('ADOPTS its own orphan when that holds a loaded model at the wanted context', async () => { + const orphan = await listeningRuntime({ installed: true }); + orphan.state.loaded = true; + let gone = false; + const fake = fakeRuntime(); + const { supervisor, kills, changes } = supervise(fake, { + commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined), + }); + const result = await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID }); + // Kept as it is: nothing spawned, nothing signalled, the loaded model stays loaded. + expect(result).toEqual({ ok: true, pid: ORPHAN_PID, port, orphan: 'adopted' }); + expect(children(fake)).toHaveLength(0); + expect(kills).toEqual([]); + expect(orphan.listening).toBe(true); + // Its log is not ours to read, so there is no measured budget. + expect(supervisor.status()).toEqual({ + running: true, + pid: ORPHAN_PID, + port, + contextLength: CONTEXT, + restarts: 0, + gaveUp: false, + }); + expect(changes.at(-1)).toMatchObject({ running: true, pid: ORPHAN_PID }); + // So that the stop after this test finds it gone and does not wait out its grace period. + gone = true; + }); + + it('stops an adopted orphan by its pid', async () => { + const orphan = await listeningRuntime({ installed: true }); + orphan.state.loaded = true; + let gone = false; + const fake = fakeRuntime(); + const { supervisor, kills } = supervise(fake, { + commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined), + kill: (pid, signal) => { + kills.push([pid, signal]); + gone = true; + }, + }); + await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID }); + await supervisor.stop(); + expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]); + expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false }); + }); + + /** Starts with an orphan on the port that is ours and goes away when it is signalled. */ + async function startOverOrphan(orphanOptions: FakeOllamaOptions, loaded: boolean) { + const orphan = await listeningRuntime(orphanOptions); + orphan.state.loaded = loaded; + let gone = false; + const fake = fakeRuntime(); + const harness = supervise(fake, { + commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined), + kill: (pid, signal) => { + harness.kills.push([pid, signal]); + gone = true; + void orphan.close(); + }, + }); + const result = await harness.supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID }); + return { orphan, fake, result, ...harness }; + } + + it('REPLACES its own orphan when that has no model loaded: nothing is lost, and the budget is read again', async () => { + const { orphan, fake, result, kills, supervisor } = await startOverOrphan({ installed: true }, false); + expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]); + expect(orphan.listening).toBe(false); + expect(children(fake)).toHaveLength(1); + expect(result).toEqual({ + ok: true, + pid: children(fake)[0].pid, + port, + orphan: 'replaced', + measuredTotalBytes: 40_157_944_217, + }); + expect(supervisor.status()).toMatchObject({ running: true, pid: children(fake)[0].pid }); + }); + + it('replaces its own orphan when the model is loaded at another context', async () => { + const { fake, result, kills } = await startOverOrphan({ installed: true, contextLength: 32768 }, true); + expect(result).toMatchObject({ ok: true, orphan: 'replaced' }); + expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]); + expect(children(fake)[0].env.OLLAMA_CONTEXT_LENGTH).toBe(String(CONTEXT)); + }); + + it('replaces its own orphan when it is another version than the pinned one', async () => { + const { fake, result, kills } = await startOverOrphan({ installed: true, version: '0.39.2' }, true); + expect(result).toMatchObject({ ok: true, orphan: 'replaced' }); + expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]); + expect(children(fake)).toHaveLength(1); + }); + + it('replaces its own orphan that no longer answers, and starts on the free port', async () => { + // The stored pid is a runtime of ours, but nothing listens on the port. + let gone = false; + const fake = fakeRuntime(); + const { supervisor, kills } = supervise(fake, { + commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined), + kill: (pid, signal) => { + kills.push([pid, signal]); + gone = true; + }, + }); + const result = await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID }); + expect(result).toMatchObject({ ok: true, orphan: 'replaced' }); + expect(kills).toEqual([[ORPHAN_PID, 'SIGTERM']]); + }); + + it('never signals a pid whose command is not under our runtime folder', async () => { + for (const command of [ + '/usr/local/bin/ollama serve', + `${paths.runtimeRoot}-other/ollama serve`, + `/bin/sh -c ${paths.binary} serve`, + undefined, + ]) { + const fake = fakeRuntime(); + const { supervisor, kills } = supervise(fake, { commandOfPid: async () => command }); + const result = await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID }); + expect(result, String(command)).toMatchObject({ ok: true, orphan: 'none' }); + expect(kills, String(command)).toEqual([]); + await supervisor.stop(); + } + }); + + it('takes over from an adopted orphan that dies: the next runtime is a child of its own', async () => { + const orphan = await listeningRuntime({ installed: true }); + orphan.state.loaded = true; + let gone = false; + const fake = fakeRuntime(); + const { supervisor } = supervise(fake, { + commandOfPid: async (pid) => (pid === ORPHAN_PID && !gone ? ownCommand() : undefined), + orphanPollMs: 15, + }); + expect(await supervisor.start({ port, contextLength: CONTEXT, previousPid: ORPHAN_PID })).toMatchObject({ + orphan: 'adopted', + }); + // The orphan dies by itself. + await orphan.close(); + gone = true; + await until(() => children(fake).length === 1 && supervisor.status().running, 'a child of our own to run'); + expect(supervisor.status()).toMatchObject({ + running: true, + pid: children(fake)[0].pid, + port, + restarts: 1, + measuredTotalBytes: 40_157_944_217, + }); + }); +}); + +// ---------- Restart ---------- + +describe('a child that exits by itself', () => { + it('is started again, with the same port and context', async () => { + const fake = fakeRuntime(); + const { supervisor, waits, changes } = supervise(fake); + await supervisor.start({ port, contextLength: CONTEXT }); + const first = children(fake)[0]; + + first.crash(1); + await until(() => children(fake).length === 2 && supervisor.status().running, 'the runtime to run again'); + + const second = children(fake)[1]; + expect(second.env.OLLAMA_HOST).toBe(`127.0.0.1:${port}`); + expect(second.env.OLLAMA_CONTEXT_LENGTH).toBe(String(CONTEXT)); + expect(supervisor.status()).toMatchObject({ running: true, pid: second.pid, restarts: 1, gaveUp: false }); + // It waited a second before starting it again. + expect(waits).toContain(1000); + // The caller was told that it stopped, and that it runs again. + const running = changes.map((status) => status.running); + expect(running.indexOf(false, running.indexOf(true))).toBeGreaterThan(0); + expect(running.at(-1)).toBe(true); + // Nobody signalled the one that crashed. + expect(first.signals).toEqual([]); + }); + + it('is started again at most 3 times, then given up', async () => { + expect(RUNTIME_MAX_RESTARTS).toBe(3); + const fake = fakeRuntime(); + const { supervisor, changes } = supervise(fake); + await supervisor.start({ port, contextLength: CONTEXT }); + + for (let crash = 1; crash <= 3; crash++) { + children(fake).at(-1)?.crash(1); + await until(() => children(fake).length === crash + 1 && supervisor.status().running, `restart number ${crash}`); + expect(supervisor.status()).toMatchObject({ running: true, restarts: crash, gaveUp: false }); + } + + // The fourth exit is one too many. + children(fake).at(-1)?.crash(1); + await until(() => supervisor.status().gaveUp, 'the supervisor to give up'); + expect(supervisor.status()).toEqual({ + running: false, + port, + contextLength: CONTEXT, + measuredTotalBytes: 40_157_944_217, + restarts: 3, + gaveUp: true, + }); + // The first child and three restarts, and no fifth. + await new Promise((resolve) => setTimeout(resolve, 60)); + expect(children(fake)).toHaveLength(4); + expect(changes.at(-1)).toMatchObject({ running: false, gaveUp: true, restarts: 3 }); + }); + + it('counts a restart that does not come up as one of the three', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake, { startTimeoutMs: 100 }); + await supervisor.start({ port, contextLength: CONTEXT }); + // From now on the runtime exits as soon as it is started. + fake.switches.spawn.exitsAtOnce = 1; + children(fake)[0].crash(1); + await until(() => supervisor.status().gaveUp, 'the supervisor to give up'); + expect(children(fake)).toHaveLength(4); + expect(supervisor.status()).toMatchObject({ running: false, restarts: 3, gaveUp: true }); + }); + + it('starts counting again after a start that was asked for', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake, { startTimeoutMs: 100 }); + await supervisor.start({ port, contextLength: CONTEXT }); + fake.switches.spawn.exitsAtOnce = 1; + children(fake)[0].crash(1); + await until(() => supervisor.status().gaveUp, 'the supervisor to give up'); + + // The person presses Start: it runs again, and the count is back at 0. + fake.switches.spawn.exitsAtOnce = undefined; + const result = await supervisor.start({ port, contextLength: CONTEXT }); + expect(result).toMatchObject({ ok: true, orphan: 'none' }); + expect(supervisor.status()).toMatchObject({ running: true, restarts: 0, gaveUp: false }); + }); + + it('is not started again once stop was called, also while it waits to restart', async () => { + const fake = fakeRuntime(); + let release: () => void = () => undefined; + const { supervisor } = supervise(fake, { + // The wait before a restart does not end by itself; the polls of a start still do. + sleep: (ms) => + ms === 1000 + ? new Promise((resolve) => { + release = resolve; + }) + : Promise.resolve(), + }); + await supervisor.start({ port, contextLength: CONTEXT }); + children(fake)[0].crash(1); + await until(() => supervisor.status().restarts === 1, 'the restart to be scheduled'); + + const stopping = supervisor.stop(); + release(); + await stopping; + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(children(fake)).toHaveLength(1); + expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false }); + }); +}); + +// ---------- Stop ---------- + +describe('stop', () => { + it('sends the child SIGTERM and resolves once it is gone', async () => { + const fake = fakeRuntime(); + const { supervisor, changes } = supervise(fake); + await supervisor.start({ port, contextLength: CONTEXT }); + const child = children(fake)[0]; + + await supervisor.stop(); + + expect(child.signals).toEqual(['SIGTERM']); + expect(child.signalCode).toBe('SIGTERM'); + // Gone, and the port is free again. + expect(fake.listening).toBe(false); + await expect(fetch(`http://127.0.0.1:${port}/api/version`)).rejects.toThrow(); + expect(supervisor.status()).toEqual({ running: false, restarts: 0, gaveUp: false }); + expect(changes.at(-1)).toEqual({ running: false, restarts: 0, gaveUp: false }); + }); + + it('does not take the stop for a crash: nothing is started again', async () => { + const fake = fakeRuntime(); + const { supervisor, waits } = supervise(fake); + await supervisor.start({ port, contextLength: CONTEXT }); + await supervisor.stop(); + await new Promise((resolve) => setTimeout(resolve, 60)); + expect(children(fake)).toHaveLength(1); + expect(waits).not.toContain(1000); + }); + + it('is safe when nothing runs, and twice in a row', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + await supervisor.stop(); + await supervisor.start({ port, contextLength: CONTEXT }); + await Promise.all([supervisor.stop(), supervisor.stop()]); + await supervisor.stop(); + expect(children(fake)[0].signals).toEqual(['SIGTERM']); + expect(supervisor.status().running).toBe(false); + }); + + it('can be started again after a stop', async () => { + const fake = fakeRuntime(); + const { supervisor } = supervise(fake); + await supervisor.start({ port, contextLength: CONTEXT }); + await supervisor.stop(); + const again = await supervisor.start({ port, contextLength: CONTEXT }); + expect(again).toMatchObject({ ok: true, pid: children(fake)[1].pid, orphan: 'none' }); + expect(children(fake)).toHaveLength(2); + }); + + it('kills the child when the server process exits without its shutdown', async () => { + // The last resort for a forced exit: a listener on the process, for as long as the child lives. + const fake = fakeRuntime(); + const before = process.listenerCount('exit'); + const { supervisor } = supervise(fake); + await supervisor.start({ port, contextLength: CONTEXT }); + expect(process.listenerCount('exit')).toBe(before + 1); + const child = children(fake)[0] as unknown as ChildProcess & FakeRuntimeChild; + + // What Node does on `process.exit()`: it calls the listeners. + const added = process.listeners('exit').at(-1) as () => void; + const exited = new Promise((resolve) => child.once('exit', () => resolve())); + added(); + await exited; + expect(child.signals).toEqual(['SIGTERM']); + // The listener goes with its child. (This process did not really exit, so the supervisor + // took the exit for a crash and started another child, which has a listener of its own.) + expect(process.listeners('exit')).not.toContain(added); + await supervisor.stop(); + expect(process.listenerCount('exit')).toBe(before); + }); +});