/** * @fileoverview Tests for src/local-model/runtime.ts, everything but the supervisor (that is * test/local-model-supervisor.test.ts): the pin, the budget line, the child's environment, * the download with its hash, the port, and the removal. * * Pinned: * - The pin is the reviewed constant, and a test never reaches it: the default `fetch` * refuses under vitest. * - The budget parser reads the REAL log line of both test Macs and no other line. * - The environment is an allowlist: no `CODEMAN_PASSWORD`, none of the user's own * `OLLAMA_*`, nothing that is not named. * - The download: a matching hash unpacks, a wrong hash deletes the file and unpacks * NOTHING, a cut body is continued with `Range`, the retries are capped, a body longer * than the pin is refused, Cancel keeps the partial file for the next run. * - The port is chosen by binding. * - The removal deletes only inside the one folder and follows no symlink out of it. * * Port: none fixed (the fake runtime listens on 0; the port tests bind 0 and read the port). */ import { execFile } from 'node:child_process'; import { createHash, randomBytes } from 'node:crypto'; import { existsSync } from 'node:fs'; import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { createServer, type AddressInfo, type Server } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { localModelPaths, type LocalModelPaths } from '../src/local-model/paths.js'; import { chooseRuntimePort, ensureRuntimeInstalled, isLoopbackPortFree, isOwnRuntimeCommand, isRuntimeInstalled, OLLAMA_RUNTIME_PIN, parseInferenceComputeTotal, removeLocalModelFiles, removePartialModelBlobs, RUNTIME_CTX_CHECKPOINTS, RUNTIME_DOWNLOAD_RETRIES, RUNTIME_HOST_PROMPT_CACHE_MIB, RUNTIME_KEEP_ALIVE, RUNTIME_KV_CACHE_TYPE, RUNTIME_MAX_RESTARTS, RUNTIME_PORT_FIRST, RUNTIME_PORT_SPAN, runtimeEnv, type RuntimeDownloadProgress, } from '../src/local-model/runtime.js'; import type { LocalModelRuntimePin } from '../src/types.js'; import { buildFakeArchive, fakeRuntimeLog, INFERENCE_COMPUTE_LINE_MACBOOK, MACBOOK_MEASURED_TOTAL_BYTES, REAL_INFERENCE_COMPUTE_LINE_MAC_MINI, REAL_VRAM_DEFAULT_CONTEXT_LINE_MAC_MINI, startFakeOllama, type FakeOllama, } from './fixtures/fake-ollama.js'; const execFileAsync = promisify(execFile); let fake: FakeOllama | undefined; let paths: LocalModelPaths; const cleanup: string[] = []; beforeEach(() => { // Under the temp HOME test/setup.ts gives this file. paths = localModelPaths('0.40.15'); }); afterEach(async () => { await fake?.close(); fake = undefined; await rm(paths.root, { recursive: true, force: true }); for (const dir of cleanup.splice(0)) await rm(dir, { recursive: true, force: true }); }); /** A temp folder outside the local model folder, removed after the test. */ async function outsideDir(): Promise { const dir = await mkdtemp(join(tmpdir(), 'local-model-outside-')); cleanup.push(dir); return dir; } /** What `ensureRuntimeInstalled()` needs to reach the fake: the global `fetch`, and no waiting. */ function deps(waits: number[] = []): { fetch: typeof fetch; sleep: (ms: number) => Promise } { return { fetch, sleep: async (ms) => { waits.push(ms); }, }; } /** The archive requests the fake saw, as the `Range` header of each (`undefined` without one). */ function archiveRanges(server: FakeOllama): (string | undefined)[] { return server.requests .filter((request) => request.path === '/download/ollama-darwin.tgz') .map((request) => request.headers.range as string | undefined); } /** A fake whose archive is about 40 KB: a runtime script plus bytes that do not compress. */ async function fakeWithLargeArchive(archive: FakeOllama['switches']['archive'] | object = {}): Promise { fake = await startFakeOllama({ archive, archiveFiles: { ollama: '#!/bin/sh\necho "fake runtime"\n', 'lib/blob.bin': randomBytes(30_000).toString('hex') }, }); return fake; } // ---------- The pin and the constants ---------- describe('the pin', () => { it('is the reviewed build: version, address, size and hash', () => { expect(OLLAMA_RUNTIME_PIN).toEqual({ id: 'ollama', version: '0.40.15', url: 'https://github.com/ollama/ollama/releases/download/v0.40.15/ollama-darwin.tgz', bytes: 185_971_671, sha256: '607c79d5d6315ef9d896db83adde736a4c0057196329843544e79b95afb260a8', unpackedBytes: 579_000_000, minMacosMajor: 14, }); }); it('cannot be changed by code that holds it', () => { expect(Object.isFrozen(OLLAMA_RUNTIME_PIN)).toBe(true); expect(() => { (OLLAMA_RUNTIME_PIN as { url: string }).url = 'https://example.com/other.tgz'; }).toThrow(); expect(OLLAMA_RUNTIME_PIN.url).toMatch(/^https:\/\/github\.com\/ollama\/ollama\/releases\/download\/v0\.40\.15\//); }); it('states the limits of the supervisor and the download', () => { expect(RUNTIME_PORT_FIRST).toBe(11435); expect(RUNTIME_PORT_SPAN).toBe(20); expect(RUNTIME_MAX_RESTARTS).toBe(3); expect(RUNTIME_DOWNLOAD_RETRIES).toBe(3); expect(RUNTIME_KEEP_ALIVE).toBe('10m'); }); it('is never downloaded by a test: the default fetch refuses under vitest', async () => { const waits: number[] = []; const result = await ensureRuntimeInstalled({ deps: { sleep: async (ms) => void waits.push(ms) } }); expect(result).toEqual({ ok: false, code: 'runtime-download-failed', detail: 'fetch is not available under vitest: the test must inject its own', }); // Refused at once, not after the retries. expect(waits).toEqual([]); expect(await isRuntimeInstalled()).toBe(false); }); }); // ---------- The budget line ---------- describe('parseInferenceComputeTotal', () => { it('reads the device total of the real line of the Mac mini', () => { // total="11.8 GiB": 11.8 x 1024^3, rounded down. expect(parseInferenceComputeTotal(REAL_INFERENCE_COMPUTE_LINE_MAC_MINI)).toBe(12_670_153_523); expect(12_670_153_523).toBe(Math.floor(11.8 * 1024 ** 3)); }); it('reads 37.4 GiB, what the MacBook reported, as 40,157,944,217 bytes', () => { expect(parseInferenceComputeTotal(INFERENCE_COMPUTE_LINE_MACBOOK)).toBe(40_157_944_217); expect(parseInferenceComputeTotal(INFERENCE_COMPUTE_LINE_MACBOOK)).toBe(MACBOOK_MEASURED_TOTAL_BYTES); // The two fields quoted in the measurement notes are enough. expect(parseInferenceComputeTotal('msg="inference compute" total="37.4 GiB" available="37.4 GiB"')).toBe( 40_157_944_217 ); }); it('does not read the line that follows it, which carries the total under another key', () => { expect(parseInferenceComputeTotal(REAL_VRAM_DEFAULT_CONTEXT_LINE_MAC_MINI)).toBeUndefined(); // Not even when that key sits on an "inference compute" line. expect( parseInferenceComputeTotal('msg="inference compute" total_vram="11.8 GiB" subtotal="1 GiB"') ).toBeUndefined(); }); it('finds exactly one line of a start-up log', () => { const totals = fakeRuntimeLog('37.4 GiB').map(parseInferenceComputeTotal); expect(totals).toEqual([undefined, undefined, 40_157_944_217, undefined]); }); it('reads the binary units the runtime prints', () => { const line = (total: string): string => `level=INFO msg="inference compute" id=0 total="${total}" available="1 B"`; expect(parseInferenceComputeTotal(line('512 B'))).toBe(512); expect(parseInferenceComputeTotal(line('8 KiB'))).toBe(8192); expect(parseInferenceComputeTotal(line('12713.1 MiB'))).toBe(Math.floor(12713.1 * 1024 ** 2)); expect(parseInferenceComputeTotal(line('96 GiB'))).toBe(96 * 1024 ** 3); expect(parseInferenceComputeTotal(line('1.5 TiB'))).toBe(1.5 * 1024 ** 4); // Without the space between the number and the unit. expect(parseInferenceComputeTotal(line('16GiB'))).toBe(16 * 1024 ** 3); }); it('answers undefined for anything it does not understand, and never throws', () => { const line = (total: string): string => `msg="inference compute" total="${total}"`; for (const text of [ '', 'total="37.4 GiB"', 'msg="inference compute"', line(''), line('GiB'), line('37.4'), line('37.4 GB'), line('-1 GiB'), line('0 GiB'), line('1e9 GiB'), line('NaN GiB'), 'msg="inference compute" total=37.4GiB', 'msg="discovering available GPUs..."', ]) { expect(parseInferenceComputeTotal(text), JSON.stringify(text)).toBeUndefined(); } }); }); // ---------- The environment ---------- describe('runtimeEnv', () => { const input = (source: NodeJS.ProcessEnv): Parameters[0] => ({ source, home: '/data/local-model/home', modelsDir: '/data/local-model/models', port: 11435, contextLength: 131072, }); /** A server environment with everything a real one may hold. */ const SERVER_ENV: NodeJS.ProcessEnv = { PATH: '/usr/local/bin:/usr/bin:/bin', HOME: '/Users/arkon', TMPDIR: '/var/folders/xy/T/', LANG: 'de_CH.UTF-8', HTTP_PROXY: 'http://proxy:3128', HTTPS_PROXY: 'http://proxy:3128', NO_PROXY: 'localhost,127.0.0.1', http_proxy: 'http://proxy-lower:3128', https_proxy: 'http://proxy-lower:3128', no_proxy: 'localhost', // Codeman's own secrets and switches. CODEMAN_PASSWORD: 'hunter2', CODEMAN_USERNAME: 'admin', CODEMAN_MUX: '1', CODEMAN_API_URL: 'http://127.0.0.1:3000', // The user's settings for their OWN runtime. OLLAMA_HOST: '0.0.0.0:11434', OLLAMA_ORIGINS: '*', OLLAMA_MODELS: '/Users/arkon/.ollama/models', OLLAMA_DEBUG: '1', OLLAMA_DEBUG_LOG_REQUESTS: '1', OLLAMA_KEEP_ALIVE: '-1', OLLAMA_NUM_PARALLEL: '8', OLLAMA_MAX_LOADED_MODELS: '0', OLLAMA_CONTEXT_LENGTH: '4096', OLLAMA_NO_CLOUD: '0', OLLAMA_REMOTES: 'evil.example', OLLAMA_AUTH: '1', LLAMA_ARG_CACHE_RAM: '0', LLAMA_ARG_CTX_CHECKPOINTS: '64', // Other secrets a server environment carries. ANTHROPIC_API_KEY: 'sk-ant-secret', OPENAI_API_KEY: 'sk-secret', GITHUB_TOKEN: 'ghp_secret', AWS_SECRET_ACCESS_KEY: 'aws-secret', SSH_AUTH_SOCK: '/tmp/ssh.sock', USER: 'arkon', SHELL: '/bin/zsh', LC_ALL: 'C', NODE_OPTIONS: '--inspect', DYLD_INSERT_LIBRARIES: '/tmp/evil.dylib', }; it('holds exactly the allowlist: what is copied, the private HOME, and eleven settings', () => { expect(runtimeEnv(input(SERVER_ENV))).toEqual({ PATH: '/usr/local/bin:/usr/bin:/bin', TMPDIR: '/var/folders/xy/T/', LANG: 'de_CH.UTF-8', HTTP_PROXY: 'http://proxy:3128', HTTPS_PROXY: 'http://proxy:3128', NO_PROXY: 'localhost,127.0.0.1', http_proxy: 'http://proxy-lower:3128', https_proxy: 'http://proxy-lower:3128', no_proxy: 'localhost', HOME: '/data/local-model/home', OLLAMA_HOST: '127.0.0.1:11435', OLLAMA_MODELS: '/data/local-model/models', OLLAMA_CONTEXT_LENGTH: '131072', OLLAMA_NO_CLOUD: '1', OLLAMA_KEEP_ALIVE: '10m', OLLAMA_NUM_PARALLEL: '1', OLLAMA_MAX_LOADED_MODELS: '1', // The memory settings the catalogue's llamacpp rows are computed for. OLLAMA_KV_CACHE_TYPE: 'q8_0', OLLAMA_FLASH_ATTENTION: '1', LLAMA_ARG_CACHE_RAM: '1024', LLAMA_ARG_CTX_CHECKPOINTS: '8', }); }); it('sets the llama.cpp memory settings from the exported constants, never from the server', () => { expect(RUNTIME_KV_CACHE_TYPE).toBe('q8_0'); expect(RUNTIME_HOST_PROMPT_CACHE_MIB).toBe(1024); expect(RUNTIME_CTX_CHECKPOINTS).toBe(8); const env = runtimeEnv(input(SERVER_ENV)); expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE); expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB)); expect(env.LLAMA_ARG_CTX_CHECKPOINTS).toBe(String(RUNTIME_CTX_CHECKPOINTS)); // The server's own `LLAMA_ARG_CACHE_RAM=0` and `LLAMA_ARG_CTX_CHECKPOINTS=64`, and a user's // q4_0 cache, lose against ours. const env2 = runtimeEnv(input({ ...SERVER_ENV, OLLAMA_KV_CACHE_TYPE: 'q4_0', OLLAMA_FLASH_ATTENTION: '0' })); expect([ env2.OLLAMA_KV_CACHE_TYPE, env2.OLLAMA_FLASH_ATTENTION, env2.LLAMA_ARG_CACHE_RAM, env2.LLAMA_ARG_CTX_CHECKPOINTS, ]).toEqual(['q8_0', '1', '1024', '8']); }); it('lets no CODEMAN_ variable through, the password least of all', () => { const env = runtimeEnv(input(SERVER_ENV)); expect(Object.keys(env).filter((name) => name.startsWith('CODEMAN_'))).toEqual([]); expect(JSON.stringify(env)).not.toContain('hunter2'); }); it("lets none of the user's own OLLAMA_ settings through: every one is ours or absent", () => { const env = runtimeEnv(input(SERVER_ENV)); expect( Object.keys(env) .filter((name) => name.startsWith('OLLAMA_')) .sort() ).toEqual([ 'OLLAMA_CONTEXT_LENGTH', 'OLLAMA_FLASH_ATTENTION', 'OLLAMA_HOST', 'OLLAMA_KEEP_ALIVE', 'OLLAMA_KV_CACHE_TYPE', 'OLLAMA_MAX_LOADED_MODELS', 'OLLAMA_MODELS', 'OLLAMA_NO_CLOUD', 'OLLAMA_NUM_PARALLEL', ]); // The inherited values lost against ours, and the ones we do not set are gone. expect(env.OLLAMA_HOST).toBe('127.0.0.1:11435'); expect(env.OLLAMA_KEEP_ALIVE).toBe('10m'); expect(env.OLLAMA_MAX_LOADED_MODELS).toBe('1'); for (const name of [ 'OLLAMA_ORIGINS', 'OLLAMA_DEBUG', 'OLLAMA_DEBUG_LOG_REQUESTS', 'OLLAMA_REMOTES', 'OLLAMA_AUTH', ]) { expect(env, name).not.toHaveProperty(name); } // The server's `LLAMA_ARG_CACHE_RAM=0` is not copied: the value is ours. expect(env.LLAMA_ARG_CACHE_RAM).toBe('1024'); }); it('carries no secret and no value of the server that is not on the list', () => { const env = runtimeEnv(input(SERVER_ENV)); const text = JSON.stringify(env); for (const secret of ['sk-ant-secret', 'sk-secret', 'ghp_secret', 'aws-secret', 'evil', '/Users/arkon']) { expect(text, secret).not.toContain(secret); } // Whatever is in the result and came from the server is one of the nine copied names. const copied = [ 'PATH', 'TMPDIR', 'LANG', 'HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy', ]; for (const [name, value] of Object.entries(env)) { if (SERVER_ENV[name] === value) expect(copied, name).toContain(name); } }); it('binds the loopback address only, on the port it is given', () => { for (const port of [11435, 11454, 40123]) { expect(runtimeEnv({ ...input({}), port }).OLLAMA_HOST).toBe(`127.0.0.1:${port}`); } expect(runtimeEnv({ ...input({}), contextLength: 65536 }).OLLAMA_CONTEXT_LENGTH).toBe('65536'); }); it('leaves out what the server does not have, and sets the rest all the same', () => { expect(Object.keys(runtimeEnv(input({}))).sort()).toEqual([ 'HOME', 'LLAMA_ARG_CACHE_RAM', 'LLAMA_ARG_CTX_CHECKPOINTS', 'OLLAMA_CONTEXT_LENGTH', 'OLLAMA_FLASH_ATTENTION', 'OLLAMA_HOST', 'OLLAMA_KEEP_ALIVE', 'OLLAMA_KV_CACHE_TYPE', 'OLLAMA_MAX_LOADED_MODELS', 'OLLAMA_MODELS', 'OLLAMA_NO_CLOUD', 'OLLAMA_NUM_PARALLEL', ]); // An empty value is not copied either. expect(runtimeEnv(input({ PATH: '', LANG: undefined }))).not.toHaveProperty('PATH'); }); it('is pure: the server environment is not changed', () => { const source = { ...SERVER_ENV }; runtimeEnv(input(source)); expect(source).toEqual(SERVER_ENV); }); }); // ---------- Whose process is it ---------- describe('isOwnRuntimeCommand', () => { const root = '/Users/arkon/.codeman/local-model/runtime'; it('knows a runtime under our folder by its command line', () => { expect(isOwnRuntimeCommand(`${root}/ollama-0.40.15/ollama serve`, root)).toBe(true); expect(isOwnRuntimeCommand(` ${root}/ollama-0.40.15/ollama serve\n`, root)).toBe(true); // Another version of ours, and a folder name with a space in it. expect(isOwnRuntimeCommand(`${root}/ollama-0.39.0/ollama serve`, root)).toBe(true); const spaced = '/Users/Jane Doe/.codeman/local-model/runtime'; expect(isOwnRuntimeCommand(`${spaced}/ollama-0.40.15/ollama serve`, spaced)).toBe(true); expect(isOwnRuntimeCommand(`${root}/ollama-0.40.15/ollama serve`, `${root}/`)).toBe(true); }); it('reads /private in front of a path as the same path', () => { const temp = '/var/folders/xy/T/codeman/local-model/runtime'; expect(isOwnRuntimeCommand(`/private${temp}/ollama-0.40.15/ollama serve`, temp)).toBe(true); expect(isOwnRuntimeCommand(`${temp}/ollama-0.40.15/ollama serve`, `/private${temp}`)).toBe(true); }); it('does not take any other process for ours', () => { for (const command of [ // A runtime the user started. '/usr/local/bin/ollama serve', '/Applications/Ollama.app/Contents/Resources/ollama serve', 'ollama serve', // Our folder is only an argument, or a prefix of another folder. `/bin/sh ${root}/ollama-0.40.15/ollama serve`, `/usr/bin/tail -f ${root}/log`, `${root}-other/ollama-0.40.15/ollama serve`, `${root}`, // A way back out of the folder. `${root}/../../../../usr/local/bin/ollama serve`, `${root}/ollama-0.40.15/../../../evil serve`, '', ]) { expect(isOwnRuntimeCommand(command, root), command).toBe(false); } expect(isOwnRuntimeCommand('/anything/ollama serve', '')).toBe(false); expect(isOwnRuntimeCommand('relative/ollama serve', 'relative')).toBe(false); }); }); // ---------- The download ---------- describe('ensureRuntimeInstalled', () => { it('downloads, proves the hash, unpacks, and leaves a runtime that runs', async () => { fake = await startFakeOllama(); const seen: RuntimeDownloadProgress[] = []; const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps(), onProgress: (progress) => seen.push(progress), }); expect(result).toEqual({ ok: true, binaryPath: paths.binary, alreadyInstalled: false }); expect(await isRuntimeInstalled(fake.pin(), paths)).toBe(true); // Unpacked by the real tar: the file is executable and is the one of the archive. expect((await stat(paths.binary)).mode & 0o111).not.toBe(0); expect((await execFileAsync(paths.binary)).stdout).toBe('fake runtime\n'); // The archive and the temporary folder are gone; one request was enough. expect(existsSync(paths.archivePartial)).toBe(false); expect(existsSync(paths.unpackDir)).toBe(false); expect(archiveRanges(fake)).toEqual([undefined]); // Progress ends at the whole archive, measured against the pin. expect(seen.at(-1)).toEqual({ completedBytes: fake.archive.size, totalBytes: fake.archive.size }); for (const progress of seen) expect(progress.totalBytes).toBe(fake.pin().bytes); }); it('does nothing the second time: the runtime is already there', async () => { fake = await startFakeOllama(); await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); const again = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); expect(again).toEqual({ ok: true, binaryPath: paths.binary, alreadyInstalled: true }); expect(fake.state.archiveRequests).toBe(1); }); it('unpacks NOTHING when the hash is wrong, and deletes the download', async () => { fake = await startFakeOllama(); const wrongPin: LocalModelRuntimePin = { ...fake.pin(), sha256: createHash('sha256').update('other').digest('hex'), }; let unpacked = 0; const result = await ensureRuntimeInstalled({ pin: wrongPin, paths, deps: { ...deps(), untar: async () => { unpacked += 1; }, }, }); expect(result).toMatchObject({ ok: false, code: 'runtime-damaged' }); expect((result as { detail?: string }).detail).toContain(fake.archive.sha256); expect(unpacked).toBe(0); expect(existsSync(paths.archivePartial)).toBe(false); expect(existsSync(paths.unpackDir)).toBe(false); expect(existsSync(paths.runtimeDir)).toBe(false); expect(await isRuntimeInstalled(wrongPin, paths)).toBe(false); // The whole archive was read before the verdict: it is the hash that failed, not the size. expect(fake.state.archiveRequests).toBe(1); }); it('compares the hash whatever the case it is written in', async () => { fake = await startFakeOllama(); const upper: LocalModelRuntimePin = { ...fake.pin(), sha256: fake.archive.sha256.toUpperCase() }; expect(await ensureRuntimeInstalled({ pin: upper, paths, deps: deps() })).toMatchObject({ ok: true }); }); it('continues a download that was cut, with a Range request from where the file ends', async () => { fake = await fakeWithLargeArchive({ cutAfterBytes: 1000 }); const waits: number[] = []; const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps(waits) }); expect(result).toMatchObject({ ok: true, alreadyInstalled: false }); // The first request from the start, the second for the rest. expect(archiveRanges(fake)).toEqual([undefined, 'bytes=1000-']); expect(waits).toEqual([500]); expect((await execFileAsync(paths.binary)).stdout).toBe('fake runtime\n'); }); it('gives up after 3 continuations and keeps the partial file for the next run', async () => { expect(RUNTIME_DOWNLOAD_RETRIES).toBe(3); fake = await fakeWithLargeArchive({ cutAfterBytes: 1000, cutTimes: Infinity }); const waits: number[] = []; const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps(waits) }); expect(result).toMatchObject({ ok: false, code: 'runtime-download-failed' }); expect((result as { detail?: string }).detail).toBeTruthy(); // The first request and three more, each from where the last one stopped. expect(archiveRanges(fake)).toEqual([undefined, 'bytes=1000-', 'bytes=2000-', 'bytes=3000-']); expect(waits).toEqual([500, 1000, 2000]); expect((await stat(paths.archivePartial)).size).toBe(4000); expect(existsSync(paths.runtimeDir)).toBe(false); // "Try again": the next run continues the same file, it does not start over. fake.switches.archive.cutAfterBytes = undefined; expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); expect(archiveRanges(fake).at(-1)).toBe('bytes=4000-'); expect((await execFileAsync(paths.binary)).stdout).toBe('fake runtime\n'); }); it('fails after the same number of tries when the server keeps answering an error', async () => { fake = await startFakeOllama({ archive: { status: 503 } }); const waits: number[] = []; const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps(waits) }); expect(result).toEqual({ ok: false, code: 'runtime-download-failed', detail: 'The download server answered HTTP 503', }); expect(fake.state.archiveRequests).toBe(4); expect(waits).toHaveLength(3); }); it('starts over when the server ignores the Range request and sends the whole archive', async () => { fake = await fakeWithLargeArchive({ ignoreRange: true }); // A partial file of an earlier run: the true first 1500 bytes. await mkdir(paths.runtimeRoot, { recursive: true }); await writeFile(paths.archivePartial, fake.archive.body.subarray(0, 1500)); const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); expect(result).toMatchObject({ ok: true }); expect(archiveRanges(fake)).toEqual(['bytes=1500-']); // Appending the whole body to the 1500 bytes would not have had the pinned hash. expect((await execFileAsync(paths.binary)).stdout).toBe('fake runtime\n'); }); it('proves the WHOLE file: a partial file that is not the start of the archive ends as damaged', async () => { fake = await fakeWithLargeArchive(); await mkdir(paths.runtimeRoot, { recursive: true }); await writeFile(paths.archivePartial, randomBytes(1500)); const first = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); expect(first).toMatchObject({ ok: false, code: 'runtime-damaged' }); expect(existsSync(paths.archivePartial)).toBe(false); expect(existsSync(paths.runtimeDir)).toBe(false); // The damaged file is gone, so the next run downloads from the first byte. expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); expect(archiveRanges(fake)).toEqual(['bytes=1500-', undefined]); }); it('throws away a partial file that is larger than the archive', async () => { fake = await startFakeOllama(); await mkdir(paths.runtimeRoot, { recursive: true }); await writeFile(paths.archivePartial, randomBytes(fake.archive.size + 100)); expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); expect(archiveRanges(fake)).toEqual([undefined]); }); it('verifies a file that is already complete without asking the server', async () => { fake = await startFakeOllama(); await mkdir(paths.runtimeRoot, { recursive: true }); await writeFile(paths.archivePartial, fake.archive.body); expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); expect(fake.state.archiveRequests).toBe(0); }); it('refuses a download whose size is not the pinned one, before it reads it', async () => { fake = await startFakeOllama(); const shortPin: LocalModelRuntimePin = { ...fake.pin(), bytes: fake.archive.size - 10 }; const result = await ensureRuntimeInstalled({ pin: shortPin, paths, deps: deps() }); expect(result).toEqual({ ok: false, code: 'runtime-damaged', detail: `The download is ${fake.archive.size} bytes, the pinned archive has ${fake.archive.size - 10}`, }); expect(existsSync(paths.archivePartial)).toBe(false); expect(fake.state.archiveRequests).toBe(1); }); it('refuses a body that grows past the pinned size, when the server names no size', async () => { fake = await startFakeOllama(); const pin = fake.pin(); const body = fake.archive.body; let unpacked = 0; // The right archive followed by more bytes, as a stream without a content-length. const endless: typeof fetch = async () => new Response( new ReadableStream({ start(controller) { controller.enqueue(new Uint8Array(body)); controller.enqueue(new Uint8Array(1000)); controller.close(); }, }), { status: 200 } ); const result = await ensureRuntimeInstalled({ pin, paths, deps: { fetch: endless, sleep: async () => undefined, untar: async () => { unpacked += 1; }, }, }); expect(result).toEqual({ ok: false, code: 'runtime-damaged', detail: `The download is longer than the ${pin.bytes} bytes of the pinned archive`, }); expect(unpacked).toBe(0); expect(existsSync(paths.archivePartial)).toBe(false); }); /** A server that sends the first `bytes` of the archive and then nothing more, until it is aborted. */ function stallingFetch(server: FakeOllama, bytes: number, counter: { requests: number }): typeof fetch { return async (_input, init) => { counter.requests += 1; const signal = init?.signal ?? undefined; return new Response( new ReadableStream({ start(controller) { controller.enqueue(new Uint8Array(server.archive.body.subarray(0, bytes))); signal?.addEventListener('abort', () => controller.error(new DOMException('aborted', 'AbortError'))); }, }), { status: 200 } ); }; } it('stops when the caller cancels, and the next run continues the same file', async () => { fake = await fakeWithLargeArchive(); const counter = { requests: 0 }; const controller = new AbortController(); setTimeout(() => controller.abort(), 40); const cancelled = await ensureRuntimeInstalled({ pin: fake.pin(), paths, signal: controller.signal, deps: { fetch: stallingFetch(fake, 2000, counter), sleep: async () => undefined }, }); expect(cancelled).toEqual({ ok: false, code: 'cancelled' }); // One request, no retry after a cancel, and what arrived is kept. expect(counter.requests).toBe(1); expect((await stat(paths.archivePartial)).size).toBe(2000); expect(existsSync(paths.runtimeDir)).toBe(false); expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); expect(archiveRanges(fake)).toEqual(['bytes=2000-']); }); it('answers cancelled at once for a signal that is already aborted', async () => { fake = await startFakeOllama(); const controller = new AbortController(); controller.abort(); const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, signal: controller.signal, deps: deps() }); expect(result).toEqual({ ok: false, code: 'cancelled' }); expect(fake.state.archiveRequests).toBe(0); }); it('treats a download that delivers nothing as cut, a bounded number of times', async () => { fake = await fakeWithLargeArchive(); const counter = { requests: 0 }; const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, idleTimeoutMs: 30, deps: { fetch: stallingFetch(fake, 500, counter), sleep: async () => undefined }, }); expect(result).toEqual({ ok: false, code: 'runtime-download-failed', detail: 'The download delivered nothing for 1 s', }); expect(counter.requests).toBe(4); }); it('answers runtime-unpack-failed when tar fails, keeps the proven archive, and unpacks it next time', async () => { fake = await startFakeOllama(); const failed = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: { ...deps(), untar: async () => { throw new Error('tar: Error opening archive: Unrecognized archive format'); }, }, }); expect(failed).toEqual({ ok: false, code: 'runtime-unpack-failed', detail: 'tar: Error opening archive: Unrecognized archive format', }); expect(existsSync(paths.unpackDir)).toBe(false); expect(existsSync(paths.runtimeDir)).toBe(false); expect((await stat(paths.archivePartial)).size).toBe(fake.archive.size); expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); // Not downloaded a second time. expect(fake.state.archiveRequests).toBe(1); }); it('answers disk-full when unpacking runs out of space', async () => { fake = await startFakeOllama(); const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: { ...deps(), untar: async () => { throw new Error("tar: ollama: Can't write: No space left on device"); }, }, }); expect(result).toMatchObject({ ok: false, code: 'disk-full' }); }); it('refuses an archive that holds no runtime binary', async () => { fake = await startFakeOllama({ archiveFiles: { 'README.txt': 'no binary here\n' } }); const result = await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); expect(result).toEqual({ ok: false, code: 'runtime-unpack-failed', detail: 'The archive holds no ollama binary', }); expect(existsSync(paths.runtimeDir)).toBe(false); expect(existsSync(paths.unpackDir)).toBe(false); }); it('replaces a runtime folder that an interrupted unpack left without its binary', async () => { fake = await startFakeOllama(); await mkdir(paths.runtimeDir, { recursive: true }); await writeFile(join(paths.runtimeDir, 'half-written'), 'x'); await mkdir(paths.unpackDir, { recursive: true }); await writeFile(join(paths.unpackDir, 'stale'), 'x'); expect(await isRuntimeInstalled(fake.pin(), paths)).toBe(false); expect(await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() })).toMatchObject({ ok: true }); expect(existsSync(join(paths.runtimeDir, 'half-written'))).toBe(false); expect(existsSync(join(paths.runtimeDir, 'stale'))).toBe(false); expect(await readFile(paths.binary, 'utf8')).toContain('fake runtime'); }); it('writes the download where only the owner can read it, inside the one folder', async () => { fake = await fakeWithLargeArchive({ cutAfterBytes: 1000, cutTimes: Infinity }); await ensureRuntimeInstalled({ pin: fake.pin(), paths, deps: deps() }); expect((await stat(paths.archivePartial)).mode & 0o077).toBe(0); expect(paths.archivePartial.startsWith(paths.root)).toBe(true); }); it('never throws: a version that is no version is an answer', async () => { const badPin = { ...OLLAMA_RUNTIME_PIN, version: '../../etc' }; expect(await isRuntimeInstalled(badPin)).toBe(false); expect(await ensureRuntimeInstalled({ pin: badPin, deps: deps() })).toMatchObject({ ok: false, code: 'runtime-download-failed', }); }); }); // ---------- The port ---------- describe('chooseRuntimePort', () => { let holder: Server | undefined; afterEach(async () => { await new Promise((resolve) => (holder ? holder.close(() => resolve()) : resolve())); holder = undefined; }); /** Binds a loopback port the system picks and keeps it. */ async function holdPort(): Promise { holder = createServer(); await new Promise((resolve) => holder?.listen(0, '127.0.0.1', () => resolve())); return (holder.address() as AddressInfo).port; } it('takes the first port of the span that is free, in order', async () => { const asked: number[] = []; const port = await chooseRuntimePort({ isFree: async (candidate) => { asked.push(candidate); return candidate >= 11438; }, }); expect(port).toBe(11438); expect(asked).toEqual([11435, 11436, 11437, 11438]); }); it('starts at 11435 and tries 20 ports, never 11434', async () => { const asked: number[] = []; const port = await chooseRuntimePort({ isFree: async (candidate) => { asked.push(candidate); return false; }, }); expect(port).toBeUndefined(); expect(asked).toHaveLength(20); expect(asked[0]).toBe(11435); expect(asked.at(-1)).toBe(11454); expect(asked).not.toContain(11434); }); it('skips the ports it is told to avoid: "Use another port"', async () => { const asked: number[] = []; const port = await chooseRuntimePort({ avoid: [11435, 11436], isFree: async (candidate) => { asked.push(candidate); return true; }, }); expect(port).toBe(11437); expect(asked).toEqual([11437]); }); it('does not take a port whose probe failed for free', async () => { const port = await chooseRuntimePort({ first: 100, span: 3, isFree: async (candidate) => { if (candidate === 100) throw new Error('EACCES'); return candidate === 102; }, }); expect(port).toBe(102); }); it('decides by BINDING: a held port is not free, and is free again once it is let go', async () => { const port = await holdPort(); expect(await isLoopbackPortFree(port)).toBe(false); expect(await chooseRuntimePort({ first: port, span: 1 })).toBeUndefined(); await new Promise((resolve) => holder?.close(() => resolve())); holder = undefined; expect(await isLoopbackPortFree(port)).toBe(true); expect(await chooseRuntimePort({ first: port, span: 1 })).toBe(port); // The probe let go of it again: it can be bound right away. expect(await isLoopbackPortFree(port)).toBe(true); }); it('calls a number that is no port not free', async () => { for (const port of [0, -1, 65536, 1.5, Number.NaN]) expect(await isLoopbackPortFree(port), String(port)).toBe(false); }); }); // ---------- The removal ---------- describe('removeLocalModelFiles', () => { /** Everything an install leaves on disk, plus the state file beside the folder. */ async function installTree(): Promise { await mkdir(join(paths.runtimeDir, 'lib'), { recursive: true }); await writeFile(paths.binary, '#!/bin/sh\n'); await writeFile(join(paths.runtimeDir, 'lib', 'libmlx.dylib'), 'x'); await mkdir(join(paths.runtimeRoot, 'ollama-0.39.0'), { recursive: true }); await writeFile(join(paths.runtimeRoot, 'ollama-0.39.0', 'ollama'), 'old'); await writeFile(paths.archivePartial, 'partial'); await mkdir(join(paths.homeDir, '.ollama'), { recursive: true }); await writeFile(join(paths.homeDir, '.ollama', 'id_ed25519'), 'private key'); await mkdir(join(paths.modelsDir, 'blobs'), { recursive: true }); await writeFile(join(paths.modelsDir, 'blobs', 'sha256-abc'), 'weights'); await writeFile(paths.stateFile, '{"schema":1,"state":"installed"}'); } it('removes the runtime, every version of it, the archive and the private home, and keeps the models', async () => { await installTree(); await removeLocalModelFiles({ models: false, paths }); expect(existsSync(paths.runtimeRoot)).toBe(false); expect(existsSync(paths.homeDir)).toBe(false); expect(await readFile(join(paths.modelsDir, 'blobs', 'sha256-abc'), 'utf8')).toBe('weights'); // The folder still holds the models, so it stays. expect(existsSync(paths.root)).toBe(true); }); it('removes the models too when asked, and then the folder itself', async () => { await installTree(); await removeLocalModelFiles({ models: true, paths }); expect(existsSync(paths.modelsDir)).toBe(false); expect(existsSync(paths.root)).toBe(false); }); it('leaves the state file alone: that is the business of its one writer', async () => { await installTree(); await removeLocalModelFiles({ models: true, paths }); expect(await readFile(paths.stateFile, 'utf8')).toBe('{"schema":1,"state":"installed"}'); await rm(paths.stateFile, { force: true }); }); it('is fine with nothing to remove, and can be run twice', async () => { await removeLocalModelFiles({ models: true, paths }); await installTree(); await removeLocalModelFiles({ models: true, paths }); await removeLocalModelFiles({ models: true, paths }); expect(existsSync(paths.root)).toBe(false); await rm(paths.stateFile, { force: true }); }); it('keeps a folder of the root that is not ours', async () => { await installTree(); await mkdir(join(paths.root, 'notes'), { recursive: true }); await writeFile(join(paths.root, 'notes', 'mine.txt'), 'keep'); await removeLocalModelFiles({ models: true, paths }); expect(await readFile(join(paths.root, 'notes', 'mine.txt'), 'utf8')).toBe('keep'); await rm(paths.stateFile, { force: true }); }); it('unlinks a symlink in the place of a folder and never deletes what it points at', async () => { const outside = await outsideDir(); await writeFile(join(outside, 'precious.gguf'), 'the only copy'); await mkdir(paths.root, { recursive: true }); // The model store was moved to another disk and linked back. await symlink(outside, paths.modelsDir); await removeLocalModelFiles({ models: true, paths }); expect(existsSync(paths.modelsDir)).toBe(false); expect(await readFile(join(outside, 'precious.gguf'), 'utf8')).toBe('the only copy'); }); it('refuses a local model folder that is itself a symlink, and deletes nothing behind it', async () => { // /local-model is a link to a folder somewhere else that happens to hold the // three names this feature deletes. const elsewhere = await outsideDir(); await mkdir(join(elsewhere, 'home'), { recursive: true }); await writeFile(join(elsewhere, 'home', 'thesis.txt'), 'the only copy'); await mkdir(join(elsewhere, 'runtime'), { recursive: true }); await mkdir(join(elsewhere, 'models', 'blobs'), { recursive: true }); await writeFile(join(elsewhere, 'models', 'blobs', 'sha256-abc'), 'weights'); await mkdir(join(paths.root, '..'), { recursive: true }); await symlink(elsewhere, paths.root); try { await expect(removeLocalModelFiles({ models: true, paths })).rejects.toThrow(/symbolic link/); expect(await readFile(join(elsewhere, 'home', 'thesis.txt'), 'utf8')).toBe('the only copy'); expect(await readFile(join(elsewhere, 'models', 'blobs', 'sha256-abc'), 'utf8')).toBe('weights'); expect(existsSync(join(elsewhere, 'runtime'))).toBe(true); // The partial blobs are not deleted through the link either. await writeFile(join(elsewhere, 'models', 'blobs', 'sha256-def-partial'), 'half'); expect(await removePartialModelBlobs(paths)).toBe(0); expect(existsSync(join(elsewhere, 'models', 'blobs', 'sha256-def-partial'))).toBe(true); } finally { // The link itself, never what it points at. await rm(paths.root, { force: true }); } }); it('refuses a local model folder that is a file', async () => { await mkdir(join(paths.root, '..'), { recursive: true }); await writeFile(paths.root, 'not a folder'); try { await expect(removeLocalModelFiles({ models: true, paths })).rejects.toThrow(/not a folder/); } finally { await rm(paths.root, { force: true }); } }); it('removes a symlink it meets inside a folder, not the tree behind it', async () => { const outside = await outsideDir(); await writeFile(join(outside, 'keep.txt'), 'keep'); await mkdir(paths.homeDir, { recursive: true }); await symlink(outside, join(paths.homeDir, 'link-out')); await removeLocalModelFiles({ models: false, paths }); expect(existsSync(paths.homeDir)).toBe(false); expect(await readFile(join(outside, 'keep.txt'), 'utf8')).toBe('keep'); }); it('refuses a path that is not inside the one folder, and deletes nothing', async () => { const outside = await outsideDir(); await writeFile(join(outside, 'keep.txt'), 'keep'); await installTree(); const cases: Partial[] = [ { runtimeRoot: outside }, { homeDir: outside }, { modelsDir: outside }, // The folder itself, and a way back out of it. { runtimeRoot: paths.root }, { homeDir: join(paths.root, '..', 'elsewhere') }, ]; for (const change of cases) { await expect(removeLocalModelFiles({ models: true, paths: { ...paths, ...change } })).rejects.toThrow(/Refusing/); } expect(await readFile(join(outside, 'keep.txt'), 'utf8')).toBe('keep'); // A refusal deletes NOTHING: every folder is judged before the first one is removed. expect(await readFile(paths.binary, 'utf8')).toBe('#!/bin/sh\n'); expect(await readFile(join(paths.homeDir, '.ollama', 'id_ed25519'), 'utf8')).toBe('private key'); expect(await readFile(join(paths.modelsDir, 'blobs', 'sha256-abc'), 'utf8')).toBe('weights'); await rm(paths.stateFile, { force: true }); }); it('refuses a folder whose real place is outside the real root', async () => { // The root is real, but `runtime` inside it is reached through a linked parent. const outside = await outsideDir(); await mkdir(join(outside, 'runtime'), { recursive: true }); await writeFile(join(outside, 'runtime', 'keep.txt'), 'keep'); await mkdir(paths.root, { recursive: true }); await symlink(outside, join(paths.root, 'nested')); const twisted: LocalModelPaths = { ...paths, runtimeRoot: join(paths.root, 'nested', 'runtime') }; await expect(removeLocalModelFiles({ models: false, paths: twisted })).rejects.toThrow(/Refusing/); expect(await readFile(join(outside, 'runtime', 'keep.txt'), 'utf8')).toBe('keep'); }); }); describe('removePartialModelBlobs', () => { const blob = (name: string): string => join(paths.modelsDir, 'blobs', name); it('deletes what an interrupted download left, and keeps every finished blob and the manifests', async () => { await mkdir(join(paths.modelsDir, 'blobs'), { recursive: true }); await mkdir(join(paths.modelsDir, 'manifests', 'registry.ollama.ai'), { recursive: true }); await writeFile(blob('sha256-aaa'), 'finished weights'); await writeFile(blob('sha256-bbb-partial'), '12345'); await writeFile(blob('sha256-bbb-partial-0'), '678'); await writeFile(blob('sha256-bbb-partial-1'), '9'); await writeFile(join(paths.modelsDir, 'manifests', 'registry.ollama.ai', 'latest'), '{}'); // The bytes that were freed: 5 + 3 + 1. expect(await removePartialModelBlobs(paths)).toBe(9); expect(existsSync(blob('sha256-bbb-partial'))).toBe(false); expect(existsSync(blob('sha256-bbb-partial-0'))).toBe(false); expect(existsSync(blob('sha256-bbb-partial-1'))).toBe(false); expect(await readFile(blob('sha256-aaa'), 'utf8')).toBe('finished weights'); expect(existsSync(join(paths.modelsDir, 'manifests', 'registry.ollama.ai', 'latest'))).toBe(true); }); it('is fine without a model store, and deletes nothing a second time', async () => { expect(await removePartialModelBlobs(paths)).toBe(0); await mkdir(join(paths.modelsDir, 'blobs'), { recursive: true }); await writeFile(blob('sha256-ccc-partial'), 'xx'); expect(await removePartialModelBlobs(paths)).toBe(2); expect(await removePartialModelBlobs(paths)).toBe(0); }); it('touches only regular files directly in the blob folder', async () => { const outside = await outsideDir(); await writeFile(join(outside, 'keep-partial'), 'keep'); await mkdir(join(paths.modelsDir, 'blobs', 'a-partial-folder'), { recursive: true }); await writeFile(join(paths.modelsDir, 'blobs', 'a-partial-folder', 'inside-partial'), 'keep'); // A link whose name carries the marker: the link stays, and so does what it points at. await symlink(join(outside, 'keep-partial'), blob('sha256-ddd-partial')); expect(await removePartialModelBlobs(paths)).toBe(0); expect(await readFile(join(outside, 'keep-partial'), 'utf8')).toBe('keep'); expect(existsSync(join(paths.modelsDir, 'blobs', 'a-partial-folder', 'inside-partial'))).toBe(true); }); it('leaves a blob folder alone that is a symlink to somewhere else', async () => { const outside = await outsideDir(); await writeFile(join(outside, 'sha256-eee-partial'), "half of somebody else's model"); await mkdir(paths.modelsDir, { recursive: true }); await symlink(outside, join(paths.modelsDir, 'blobs')); expect(await removePartialModelBlobs(paths)).toBe(0); expect(existsSync(join(outside, 'sha256-eee-partial'))).toBe(true); }); });