/** * @fileoverview Tests for test/fixtures/fake-ollama.ts, the fake model runtime every local * model test runs against. A fake that answers in the wrong shape makes every test built on * it prove nothing, so this file pins each answer and each switch once. * * It also shows how the fake is used: `startFakeOllama()` for a client test, and * `createFakeOllama()` plus `fake.spawn` and `freeLoopbackPort()` for a supervisor test. * * Port: none fixed (the fake listens on 0; the spawn test reserves a free port first). */ import { execFile } from 'node:child_process'; import { createHash } from 'node:crypto'; import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; import { afterEach, describe, expect, it } from 'vitest'; import { buildFakeArchive, createFakeOllama, DEFAULT_FAKE_MODEL, fakeRuntimeLog, FakeRuntimeChild, freeLoopbackPort, INFERENCE_COMPUTE_LINE_MACBOOK, MACBOOK_MEASURED_TOTAL_BYTES, REAL_INFERENCE_COMPUTE_LINE_MAC_MINI, REAL_PULL_ERROR_LINE, startFakeOllama, type FakeOllama, } from './fixtures/fake-ollama.js'; const execFileAsync = promisify(execFile); let fake: FakeOllama | undefined; afterEach(async () => { await fake?.close(); fake = undefined; }); /** Every line of a pull, parsed. `cut` is true when the body ended in an error instead of a clean end. */ async function pull(target: FakeOllama, modelId = DEFAULT_FAKE_MODEL.id): Promise<{ lines: any[]; cut: boolean }> { const res = await fetch(`${target.baseUrl}/api/pull`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ model: modelId, stream: true }), }); let text = ''; let cut = false; const decoder = new TextDecoder(); try { for await (const chunk of res.body as unknown as AsyncIterable) { text += decoder.decode(chunk, { stream: true }); } } catch { cut = true; } const lines = text .split('\n') .filter((line) => line.trim() !== '') .map((line) => JSON.parse(line)); return { lines, cut }; } async function getJson(target: FakeOllama, path: string): Promise { return (await fetch(`${target.baseUrl}${path}`)).json(); } async function chat(target: FakeOllama, body: Record): Promise<{ status: number; json: any }> { const res = await fetch(`${target.baseUrl}/v1/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ model: DEFAULT_FAKE_MODEL.id, messages: [{ role: 'user', content: 'ping' }], ...body }), }); return { status: res.status, json: await res.json() }; } const TOOLS = [ { type: 'function', function: { name: 'read_file', description: 'Read a file', parameters: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'] }, }, }, ]; describe('the fake runtime: listings', () => { it('answers the version and an empty store in the real shapes', async () => { fake = await startFakeOllama(); expect(fake.port).toBeGreaterThan(0); expect(await (await fetch(`${fake.baseUrl}/`)).text()).toBe('Ollama is running'); expect(await getJson(fake, '/api/version')).toEqual({ version: '0.40.15' }); expect(await getJson(fake, '/api/tags')).toEqual({ models: [] }); expect(await getJson(fake, '/api/ps')).toEqual({ models: [] }); // The real runtime answers `data: null`, not an empty list, when it has no model. expect(await getJson(fake, '/v1/models')).toEqual({ object: 'list', data: null }); }); it('lists an installed model with the digest, format and runner a Mac reports', async () => { fake = await startFakeOllama({ installed: true }); const tags = await getJson(fake, '/api/tags'); expect(tags.models).toHaveLength(1); expect(tags.models[0]).toMatchObject({ name: 'qwen3.6:35b-a3b', size: 23_105_798_956, digest: '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba', details: { format: 'safetensors', quantization_level: 'nvfp4', runner: 'mlx' }, capabilities: ['completion', 'vision', 'tools', 'thinking'], }); expect((await getJson(fake, '/v1/models')).data.map((m: { id: string }) => m.id)).toEqual(['qwen3.6:35b-a3b']); }); it('records every request with its parsed body', async () => { fake = await startFakeOllama({ installed: true }); await chat(fake, { reasoning_effort: 'none', max_tokens: 16 }); expect(fake.requests.at(-1)).toMatchObject({ method: 'POST', path: '/v1/chat/completions', body: { model: 'qwen3.6:35b-a3b', reasoning_effort: 'none', max_tokens: 16 }, }); }); }); describe('the fake runtime: pull', () => { it('streams the MLX line shape of a Mac and installs the model', async () => { fake = await startFakeOllama({ pull: { progressLines: 4 } }); const { lines, cut } = await pull(fake); expect(cut).toBe(false); expect(lines[0]).toEqual({ status: 'pulling manifest' }); expect(lines[1]).toEqual({ status: 'pulling model', digest: 'sha256:model', total: 23_105_798_956, completed: 0 }); expect(lines.slice(-3)).toEqual([ { status: 'pulling model', digest: 'sha256:model', total: 23_105_798_956, completed: 23_105_798_956 }, { status: 'writing manifest' }, { status: 'success' }, ]); const completed = lines.filter((l) => l.status === 'pulling model').map((l) => l.completed); expect(completed).toEqual([...completed].sort((a, b) => a - b)); expect(fake.state.installed).toBe(true); expect((await getJson(fake, '/api/tags')).models).toHaveLength(1); }); it('streams as many progress lines as asked for', async () => { fake = await startFakeOllama({ pull: { progressLines: 5000 } }); const { lines } = await pull(fake); expect(lines.filter((l) => l.status === 'pulling model')).toHaveLength(5001); }); it('streams the GGUF shape: layers that announce themselves, then the digest check', async () => { fake = await startFakeOllama({ pull: { shape: 'gguf', progressLines: 2 } }); const { lines } = await pull(fake); const large = `sha256:${DEFAULT_FAKE_MODEL.digest}`; // The first line of a layer has no `completed`. expect(lines[1]).toEqual({ status: 'pulling 0ce5ff81f9ff', digest: large, total: 23_105_798_956 - 13_383 }); expect(lines.map((l) => l.status).slice(-3)).toEqual(['verifying sha256 digest', 'writing manifest', 'success']); // The totals of all layers add up to the model. const totals = new Map(); for (const l of lines) if (l.digest) totals.set(l.digest, l.total); expect([...totals.values()].reduce((a, b) => a + b, 0)).toBe(23_105_798_956); }); it('ends on the real error line, then resumes where it stopped and succeeds', async () => { fake = await startFakeOllama({ pull: { fail: 'error-line', failAfterLines: 3, failTimes: 1, progressLines: 6 } }); const first = await pull(fake); expect(first.cut).toBe(false); expect(JSON.stringify(first.lines.at(-1))).toBe(REAL_PULL_ERROR_LINE); expect(first.lines.some((l) => l.status === 'success')).toBe(false); expect(fake.state.installed).toBe(false); const reached = fake.state.completedBytes; expect(reached).toBeGreaterThan(0); const second = await pull(fake); // A resumed pull's first progress line starts at what is already on disk. expect(second.lines[1].completed).toBe(reached); expect(second.lines.at(-1)).toEqual({ status: 'success' }); expect(fake.state.pulls).toBe(2); }); it('cuts the stream without a last line', async () => { fake = await startFakeOllama({ pull: { fail: 'cut', failAfterLines: 2 } }); const { lines, cut } = await pull(fake); expect(cut).toBe(true); expect(lines.some((l) => l.status === 'success' || l.error)).toBe(false); expect(fake.state.installed).toBe(false); }); it('starts a first pull at resumeFrom', async () => { fake = await startFakeOllama({ pull: { resumeFrom: 197_411_434, progressLines: 2 } }); const { lines } = await pull(fake); expect(lines[1].completed).toBe(197_411_434); }); it('says so when the registry has no such model', async () => { fake = await startFakeOllama(); const unknown = await pull(fake, 'no-such:1b'); expect(unknown.lines.at(-1)).toEqual({ error: 'pull model manifest: file does not exist' }); fake.switches.pull.fail = 'not-found'; expect((await pull(fake)).lines.at(-1)).toEqual({ error: 'pull model manifest: file does not exist' }); }); it('stops sending when the client aborts, and remembers how far it got', async () => { fake = await startFakeOllama({ pull: { progressLines: 200, lineDelayMs: 5 } }); const controller = new AbortController(); const res = await fetch(`${fake.baseUrl}/api/pull`, { method: 'POST', body: JSON.stringify({ model: DEFAULT_FAKE_MODEL.id, stream: true }), signal: controller.signal, }); const reader = res.body!.getReader(); await reader.read(); controller.abort(); await reader.read().catch(() => undefined); await new Promise((resolve) => setTimeout(resolve, 50)); const reached = fake.state.completedBytes; expect(fake.state.installed).toBe(false); expect(reached).toBeLessThan(23_105_798_956); await new Promise((resolve) => setTimeout(resolve, 50)); expect(fake.state.completedBytes).toBe(reached); }); it('deletes the model', async () => { fake = await startFakeOllama({ installed: true }); const del = (): Promise => fetch(`${fake!.baseUrl}/api/delete`, { method: 'DELETE', body: JSON.stringify({ model: DEFAULT_FAKE_MODEL.id }), }); expect((await del()).status).toBe(200); expect(fake.state.installed).toBe(false); expect((await del()).status).toBe(404); }); }); describe('the fake runtime: chat, the process list and the tool probe', () => { it('answers a plain chat and lists the model as loaded, all on the GPU', async () => { fake = await startFakeOllama({ installed: true }); expect((await getJson(fake, '/api/ps')).models).toEqual([]); const { status, json } = await chat(fake, { reasoning_effort: 'none' }); expect(status).toBe(200); expect(json.choices[0]).toEqual({ index: 0, message: { role: 'assistant', content: 'pong' }, finish_reason: 'stop', }); const [loaded] = (await getJson(fake, '/api/ps')).models; expect(loaded).toMatchObject({ name: 'qwen3.6:35b-a3b', context_length: 131072, runner: 'mlx' }); expect(loaded.size_vram).toBe(loaded.size); }); it('can report a model that is partly off the GPU, or loaded at another context', async () => { fake = await startFakeOllama({ installed: true, ps: { size: 30e9, sizeVram: 20e9 } }); await chat(fake, {}); expect((await getJson(fake, '/api/ps')).models[0]).toMatchObject({ size: 30e9, size_vram: 20e9 }); fake.switches.ps = { contextLength: 32768 }; expect((await getJson(fake, '/api/ps')).models[0].context_length).toBe(32768); fake.switches.ps = { loaded: false }; expect((await getJson(fake, '/api/ps')).models).toEqual([]); }); it('answers 404 in the real error shape for a model it does not have', async () => { fake = await startFakeOllama(); const { status, json } = await chat(fake, {}); expect(status).toBe(404); expect(json).toEqual({ error: { message: "model 'qwen3.6:35b-a3b' not found", type: 'not_found_error', param: null, code: null }, }); }); it('can fail the chat that loads the model', async () => { fake = await startFakeOllama({ installed: true, chat: 'error' }); expect((await chat(fake, {})).status).toBe(500); expect((await getJson(fake, '/api/ps')).models).toEqual([]); }); it('calls the tool it was given, with arguments that parse', async () => { fake = await startFakeOllama({ installed: true }); const { json } = await chat(fake, { tools: TOOLS, reasoning_effort: 'none', max_tokens: 1024 }); const choice = json.choices[0]; expect(choice.finish_reason).toBe('tool_calls'); expect(choice.message.tool_calls).toHaveLength(1); expect(choice.message.tool_calls[0]).toMatchObject({ type: 'function', function: { name: 'read_file' } }); expect(JSON.parse(choice.message.tool_calls[0].function.arguments)).toEqual({ path: 'README.md' }); }); it('can answer without a tool call, run into the token limit, or send arguments that do not parse', async () => { fake = await startFakeOllama({ installed: true, tool: 'none' }); let choice = (await chat(fake, { tools: TOOLS })).json.choices[0]; expect(choice.finish_reason).toBe('stop'); expect(choice.message.tool_calls).toBeUndefined(); fake.switches.tool = 'length'; choice = (await chat(fake, { tools: TOOLS })).json.choices[0]; expect(choice.finish_reason).toBe('length'); expect(choice.message.content).toBe(''); expect(choice.message.tool_calls).toBeUndefined(); fake.switches.tool = 'bad-arguments'; choice = (await chat(fake, { tools: TOOLS })).json.choices[0]; expect(() => JSON.parse(choice.message.tool_calls[0].function.arguments)).toThrow(); }); }); describe('the fake runtime: the archive', () => { it('serves an archive whose size and hash are the pin it hands out', async () => { fake = await startFakeOllama(); const pin = fake.pin(); expect(pin.url).toBe(`${fake.baseUrl}/download/ollama-darwin.tgz`); const res = await fetch(pin.url); const body = Buffer.from(await res.arrayBuffer()); expect(res.status).toBe(200); expect(body.length).toBe(pin.bytes); expect(createHash('sha256').update(body).digest('hex')).toBe(pin.sha256); }); it('continues from a Range request', async () => { fake = await startFakeOllama(); const res = await fetch(fake.pin().url, { headers: { range: 'bytes=10-' } }); expect(res.status).toBe(206); expect(res.headers.get('content-range')).toBe(`bytes 10-${fake.archive.size - 1}/${fake.archive.size}`); expect(Buffer.from(await res.arrayBuffer()).equals(fake.archive.body.subarray(10))).toBe(true); expect((await fetch(fake.pin().url, { headers: { range: `bytes=${fake.archive.size}-` } })).status).toBe(416); }); it('cuts the body the first time only, so a Range request can finish it', async () => { fake = await startFakeOllama({ archive: { cutAfterBytes: 20 } }); const first = await fetch(fake.pin().url); const got: Buffer[] = []; await (async () => { for await (const chunk of first.body as unknown as AsyncIterable) got.push(Buffer.from(chunk)); })().catch(() => undefined); const head = Buffer.concat(got); expect(head.length).toBe(20); const rest = await fetch(fake.pin().url, { headers: { range: 'bytes=20-' } }); const whole = Buffer.concat([head, Buffer.from(await rest.arrayBuffer())]); expect(createHash('sha256').update(whole).digest('hex')).toBe(fake.archive.sha256); expect(fake.state.archiveRequests).toBe(2); }); it('can ignore Range, or answer another status', async () => { fake = await startFakeOllama({ archive: { ignoreRange: true } }); expect((await fetch(fake.pin().url, { headers: { range: 'bytes=10-' } })).status).toBe(200); fake.switches.archive.status = 503; expect((await fetch(fake.pin().url)).status).toBe(503); }); it('is a real archive that tar unpacks to an executable', async () => { const dir = await mkdtemp(join(tmpdir(), 'fake-ollama-archive-')); try { const archive = join(dir, 'runtime.tgz'); await writeFile(archive, buildFakeArchive({ ollama: '#!/bin/sh\necho hello\n', LICENSE: 'text\n' })); await execFileAsync('tar', ['-xzf', archive, '-C', dir]); expect(await readFile(join(dir, 'LICENSE'), 'utf8')).toBe('text\n'); expect((await stat(join(dir, 'ollama'))).mode & 0o111).not.toBe(0); expect((await execFileAsync(join(dir, 'ollama'))).stdout).toBe('hello\n'); } finally { await rm(dir, { recursive: true, force: true }); } }); }); describe('the fake runtime: the child process', () => { it('holds the real log lines with the device total', () => { expect(REAL_INFERENCE_COMPUTE_LINE_MAC_MINI).toContain('total="11.8 GiB" available="11.8 GiB"'); expect(INFERENCE_COMPUTE_LINE_MACBOOK).toContain('total="37.4 GiB" available="37.4 GiB"'); expect(MACBOOK_MEASURED_TOTAL_BYTES).toBe(Math.floor(37.4 * 1024 ** 3)); expect(fakeRuntimeLog('37.4 GiB')[2]).toContain('msg="inference compute"'); }); it('listens on the port of OLLAMA_HOST once spawned, logs the budget, and is gone when killed', async () => { const port = await freeLoopbackPort(); fake = createFakeOllama({ installed: true }); expect(fake.listening).toBe(false); const child = fake.spawn('/somewhere/ollama', ['serve'], { env: { OLLAMA_HOST: `127.0.0.1:${port}`, HOME: '/private/home' }, }) as unknown as FakeRuntimeChild; let log = ''; child.stderr.on('data', (chunk: Buffer) => (log += chunk.toString('utf8'))); await new Promise((resolve) => setTimeout(resolve, 50)); expect(child.pid).toBeGreaterThan(4_194_304); expect(child.env.HOME).toBe('/private/home'); expect(fake.port).toBe(port); expect(await getJson(fake, '/api/version')).toEqual({ version: '0.40.15' }); expect(log).toContain('msg="inference compute"'); expect(log).toContain('total="37.4 GiB"'); const exited = new Promise<[number | null, string | null]>((resolve) => child.once('exit', (code, signal) => resolve([code, signal])) ); expect(child.kill('SIGTERM')).toBe(true); expect(await exited).toEqual([null, 'SIGTERM']); expect(fake.listening).toBe(false); await expect(fetch(`http://127.0.0.1:${port}/api/version`)).rejects.toThrow(); expect(fake.state.children).toEqual([child]); }); it('can start without ever listening, exit at once, leave the budget out, or crash later', async () => { const port = await freeLoopbackPort(); const env = { OLLAMA_HOST: `127.0.0.1:${port}` }; fake = createFakeOllama({ spawn: { neverListens: true, logsBudget: false } }); const silent = fake.spawn('ollama', ['serve'], { env }) as unknown as FakeRuntimeChild; let log = ''; silent.stderr.on('data', (chunk: Buffer) => (log += chunk.toString('utf8'))); await new Promise((resolve) => setTimeout(resolve, 30)); expect(fake.listening).toBe(false); expect(log).toContain('Listening on'); expect(log).not.toContain('inference compute'); fake.switches.spawn = { neverListens: false, exitsAtOnce: 2, total: '37.4 GiB', logsBudget: true }; const dead = fake.spawn('ollama', ['serve'], { env }) as unknown as FakeRuntimeChild; const code = await new Promise((resolve) => dead.once('exit', resolve)); expect(code).toBe(2); fake.switches.spawn.exitsAtOnce = undefined; const alive = fake.spawn('ollama', ['serve'], { env }) as unknown as FakeRuntimeChild; await new Promise((resolve) => setTimeout(resolve, 30)); expect(fake.listening).toBe(true); const crashed = new Promise((resolve) => alive.once('exit', resolve)); alive.crash(137); expect(await crashed).toBe(137); expect(fake.listening).toBe(false); }); });