diff --git a/src/config/local-model-catalog.ts b/src/config/local-model-catalog.ts index 661ebd8b..a66a111e 100644 --- a/src/config/local-model-catalog.ts +++ b/src/config/local-model-catalog.ts @@ -2,16 +2,7 @@ * @fileoverview The local model catalogue: which models, harnesses and chips the recommendation * knows, as a compiled constant validated by a Zod schema. * - * SKELETON: the schema below is the shape of the contract (it mirrors `LocalModelCatalog` in - * src/types/local-model.ts) and the stock constant is EMPTY. Still to come here: - * - the model rows (ids and byte sizes from the registry manifests, `paramsB`, an empty - * `tested` array each; `flint:40b` stays out), the harness rows and the chips table; - * - the cross-row rules the schema cannot say field by field: unique model ids, unique harness - * `cli`s, `contexts` largest first, `chips` most specific first, `promptCacheTiers` ascending - * and starting at 0; - * - `getLocalModelCatalog()` parsing the constant once, on first use, never at import. - * - * Rules that hold from the start: + * Rules: * - There is NO override file and no runtime download data in this stage: nothing here reads * the data dir, and a data file can never decide what gets recommended. * - A model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`), so it can @@ -20,14 +11,71 @@ * registry id as data: no code may compare it with a literal. Whether that CLI exists and can * take a custom model is checked against the registry at call time, in * src/local-model/harness-facts.ts, never here. + * - The schema checks more than fields. Across rows it requires: unique model ids and ranks, + * unique harness `cli`s and ranks, `contexts` largest first, a `tested` entry naming a harness + * row, `chips` most specific first, and `promptCacheTiers` ascending from 0. + * - `getLocalModelCatalog()` parses the constant once, on first use and never at import, and + * throws on a bad row. test/local-model-catalog.test.ts pins that the stock constant parses, + * so a bad row fails the gate and never reaches the route. + * + * Where each figure of the stock rows comes from, and what was checked (2026-11-06): + * + * | Figure | Source | Checked | + * |----------------------|-----------------------------------------------------------|--------------------------------------| + * | `id`, `bytes` | registry manifest of the tag, sum of its layer sizes | yes, to the byte, by the facts review | + * | `license` | the model card (all Apache-2.0) | read by the research, not re-checked | + * | `paramsB` | 27.8 from the model card; the others are the count in the | no. Only the 12 billion line reads | + * | | model's name | it, and no row is near that line | + * | `kvBytesPerToken` | COMPUTED from each model's config: 2 x full-attention | no. Not measured in a running | + * | | layers x KV heads x head size x 2 bytes | runtime. The review called the | + * | | | gpt-oss figure plausible, no more | + * | `contexts`, `rank` | a judgement of the design | no | + * | `speedPer100Gbps` | one published run on an M1 Max (400 GB/s): generation | the run was read. It stands for less | + * | | only, short prompt. The two q8_0 rows are SCALED by file | than a session: no long prompt, no | + * | | size, not measured | thinking tokens | + * | `minMacosMajor` | the runtime's documentation (macOS 14) | yes, by the facts review | + * | `promptCacheTiers` | a decision of the design, not a measurement | no | + * | harness `minContext` | 65536: the runtime's documentation for both harnesses. | 65536 yes. 32768 no, which is why | + * | | 32768: recalled | that row is gated | + * | `chips` | the bandwidth column of | yes, read at the source for this | + * | | https://github.com/ggml-org/llama.cpp/discussions/4167 | file | + * + * `tested` is empty on every row: nobody has run any of these models with a harness on a Mac + * yet. Until a row has an entry, the recommendation never says "Recommended". */ import { z } from 'zod'; -import type { LocalModelCatalog } from '../types.js'; +import type { ChipEntry, LocalModelCatalog } from '../types.js'; /** A model id: lowercase name, one colon, a tag. No slash, so no registry host or namespace can ride along. */ export const LOCAL_MODEL_ID_PATTERN = /^[a-z0-9][a-z0-9._-]*:[a-z0-9][a-z0-9._-]*$/; +/** + * Whether `chip` (a `machdep.cpu.brand_string`, 'Apple M4 Pro') is the chip a `ChipEntry.match` + * ('M4 Pro') names: the string must END with the match, and the match must start at the + * beginning or after a space. Case is ignored. + * + * Stricter than "the match occurs in the string" on purpose. 'M4' is not the chip of + * 'Apple M4 Pro', 'M1' is not the chip of 'Apple M10', and a variant this table has never heard + * of ('Apple M6 Pro') matches no row instead of borrowing the bandwidth of its base chip. + */ +export function chipNameMatches(chip: string, match: string): boolean { + const name = chip.trim().toLowerCase(); + const wanted = match.trim().toLowerCase(); + if (wanted === '' || !name.endsWith(wanted)) return false; + const start = name.length - wanted.length; + return start === 0 || name[start - 1] === ' '; +} + +/** + * The memory bandwidth (GB/s) of `chip`, from the first row that names it. `undefined` when the + * chip string is absent or no row names it: the speed is then UNKNOWN, a bandwidth is never guessed. + */ +export function chipBandwidthGbps(chips: readonly ChipEntry[], chip: string | undefined): number | undefined { + if (!chip) return undefined; + return chips.find((row) => chipNameMatches(chip, row.match))?.gbps; +} + const isoDate = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'must be an ISO date (YYYY-MM-DD)'); const positiveInt = z.number().int().positive(); @@ -79,7 +127,20 @@ const PromptCacheTierSchema = z }) .strict(); -/** Validates a whole catalogue. Typed against `LocalModelCatalog`, so the two cannot drift apart. */ +/** The index of the first value that already occurred earlier in the list, or -1. */ +function firstRepeat(values: readonly T[]): number { + const seen = new Set(); + for (let i = 0; i < values.length; i++) { + if (seen.has(values[i])) return i; + seen.add(values[i]); + } + return -1; +} + +/** + * Validates a whole catalogue: every field, then the rules that hold across rows (see the file + * header). Typed against `LocalModelCatalog`, so the two cannot drift apart. + */ export const LocalModelCatalogSchema: z.ZodType = z .object({ schema: z.literal(1), @@ -90,33 +151,223 @@ export const LocalModelCatalogSchema: z.ZodType = z harnesses: z.array(LocalHarnessEntrySchema), chips: z.array(ChipEntrySchema), }) - .strict(); + .strict() + .superRefine((catalog, ctx) => { + const issue = (path: (string | number)[], message: string): void => { + ctx.addIssue({ code: 'custom', path, message }); + }; + + // A repeated id would make "the model" ambiguous, a repeated rank would make the order of + // two models depend on where they sit in the array. + const repeatedId = firstRepeat(catalog.models.map((m) => m.id)); + if (repeatedId >= 0) issue(['models', repeatedId, 'id'], 'model ids must be unique'); + const repeatedRank = firstRepeat(catalog.models.map((m) => m.rank)); + if (repeatedRank >= 0) issue(['models', repeatedRank, 'rank'], 'model ranks must be unique'); + + const harnessClis = catalog.harnesses.map((h) => h.cli); + const repeatedCli = firstRepeat(harnessClis); + if (repeatedCli >= 0) issue(['harnesses', repeatedCli, 'cli'], 'a harness may be listed only once'); + const repeatedHarnessRank = firstRepeat(catalog.harnesses.map((h) => h.rank)); + if (repeatedHarnessRank >= 0) issue(['harnesses', repeatedHarnessRank, 'rank'], 'harness ranks must be unique'); + + catalog.models.forEach((model, m) => { + for (let i = 1; i < model.contexts.length; i++) { + if (model.contexts[i] >= model.contexts[i - 1]) { + issue(['models', m, 'contexts', i], 'contexts must be listed largest first, without repeats'); + break; + } + } + // An entry for a harness the catalogue does not list could never match a candidate: a typo. + model.tested.forEach((entry, t) => { + if (!harnessClis.includes(entry.cli)) { + issue(['models', m, 'tested', t, 'cli'], 'a tested entry must name a harness of this catalogue'); + } + }); + }); + + // Most specific first ('M4 Max' before 'M4'): a row whose text sits inside a LATER row's text + // is the less specific of the two, so it is in the wrong place. A repeated match is the same + // mistake. + for (let later = 1; later < catalog.chips.length; later++) { + const laterMatch = catalog.chips[later].match.toLowerCase(); + const shadowing = catalog.chips.slice(0, later).find((row) => laterMatch.includes(row.match.toLowerCase())); + if (shadowing) { + issue( + ['chips', later, 'match'], + `chips must be listed most specific first: '${shadowing.match}' comes before '${catalog.chips[later].match}'` + ); + break; + } + } + + // The tiers are read as "the last one whose minimum is not above this Mac's memory", so they + // must start at 0 (every Mac has a tier) and ascend. + if (catalog.promptCacheTiers[0].minMemGb !== 0) { + issue(['promptCacheTiers', 0, 'minMemGb'], 'the first prompt cache tier must start at 0'); + } + for (let i = 1; i < catalog.promptCacheTiers.length; i++) { + if (catalog.promptCacheTiers[i].minMemGb <= catalog.promptCacheTiers[i - 1].minMemGb) { + issue(['promptCacheTiers', i, 'minMemGb'], 'prompt cache tiers must ascend by minMemGb'); + break; + } + } + }); + +const KIB = 1024; +const MIB = 1024 ** 2; +const GIB = 1024 ** 3; /** - * The catalogue that ships with Codeman. EMPTY for now (no model, harness or chip rows), but - * valid: with it `recommend()` can only answer verdict `none`. - * - * The two constants that are already decided are filled in: macOS 14 is the oldest version the - * runtime supports, and the prompt cache is 512 MiB on a Mac below 32 GB and 2 GiB from 32 GB. + * The catalogue that ships with Codeman. Never read directly outside this file and its test: + * callers use `getLocalModelCatalog()`, which validates it first. */ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { schema: 1, updated: '2026-11-06', + // The oldest macOS the model runtime supports. minMacosMajor: 14, + // Host memory the runtime's prompt cache may take beside the model: 512 MiB on a Mac below + // 32 GB, 2 GiB from 32 GB. Part of the need of every candidate. promptCacheTiers: [ - { minMemGb: 0, bytes: 512 * 1024 ** 2 }, - { minMemGb: 32, bytes: 2 * 1024 ** 3 }, + { minMemGb: 0, bytes: 512 * MIB }, + { minMemGb: 32, bytes: 2 * GIB }, + ], + // Ordered by rank. The higher rank wins among models that fit equally well and are equally + // fast. `flint:40b` is left out on purpose: its plain tag is the slow build on a Mac and it + // has an open bug with tool arguments. + models: [ + { + id: 'qwen3.5:9b', + label: 'Qwen3.5 9B', + license: 'Apache-2.0', + paramsB: 9, + bytes: 6_550_825_373, + kvBytesPerToken: 32 * KIB, + contexts: [65536, 32768], + rank: 1, + // 37.1 tokens per second measured on 400 GB/s. + speedPer100Gbps: 9.3, + tested: [], + }, + { + id: 'qwen3.5:9b-q8_0', + label: 'Qwen3.5 9B (Q8)', + license: 'Apache-2.0', + paramsB: 9, + bytes: 10_449_217_949, + kvBytesPerToken: 32 * KIB, + contexts: [65536, 32768], + rank: 2, + // Not measured: the 9.3 of the row above scaled by file size. + speedPer100Gbps: 5.8, + tested: [], + }, + { + id: 'gpt-oss:20b', + label: 'GPT-OSS 20B', + license: 'Apache-2.0', + paramsB: 20, + bytes: 13_793_441_427, + kvBytesPerToken: 24 * KIB, + contexts: [131072, 65536], + rank: 3, + // No speed figure: its speed is unknown on every chip. + tested: [], + }, + { + id: 'qwen3.6:35b-a3b', + label: 'Qwen3.6 35B-A3B', + license: 'Apache-2.0', + paramsB: 35, + bytes: 22_621_314_161, + kvBytesPerToken: 20 * KIB, + contexts: [131072, 65536], + rank: 4, + // 54.5 tokens per second measured on 400 GB/s (3 billion parameters are active per token). + speedPer100Gbps: 13.6, + tested: [], + }, + { + id: 'qwen3.8:27b', + label: 'Qwen3.8 27B', + license: 'Apache-2.0', + paramsB: 27.8, + bytes: 17_741_871_939, + kvBytesPerToken: 64 * KIB, + contexts: [131072, 65536, 32768], + rank: 5, + // 11.8 tokens per second measured on 400 GB/s. + speedPer100Gbps: 2.95, + tested: [], + }, + { + id: 'qwen3.8:27b-q8_0', + label: 'Qwen3.8 27B (Q8)', + license: 'Apache-2.0', + paramsB: 27.8, + bytes: 29_978_241_837, + kvBytesPerToken: 64 * KIB, + contexts: [262144, 131072], + rank: 6, + // Not measured: the 2.95 of the row above scaled by file size. + speedPer100Gbps: 1.7, + tested: [], + }, + { + id: 'qwen3.5:122b-a10b', + label: 'Qwen3.5 122B-A10B', + license: 'Apache-2.0', + paramsB: 122, + bytes: 81_370_035_877, + kvBytesPerToken: 24 * KIB, + contexts: [131072], + rank: 7, + // No speed figure: its speed is unknown on every chip. + tested: [], + }, + ], + // The higher rank wins when a model can run with more than one harness. + harnesses: [ + { cli: 'opencode', minContext: 65536, rank: 3 }, + // Sends a fixed prompt of about 36,000 tokens, so the first answer takes minutes. + { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true }, + // Gated: neither its 32768 minimum nor its handling of a `name:tag` model id is verified. + { cli: 'pi', minContext: 32768, rank: 1, gated: true }, + ], + // Memory bandwidth in GB/s, most specific first. Only chips that shipped and are in the source + // table are listed. Where one name covers two bandwidths (M3 Max 300 or 400, M4 Max 410 or 546, + // M5 Max 460 or 614) the LOWER one stands, so a speed computed from it is a floor. + chips: [ + { match: 'M5 Ultra', gbps: 1228 }, + { match: 'M3 Ultra', gbps: 800 }, + { match: 'M2 Ultra', gbps: 800 }, + { match: 'M1 Ultra', gbps: 800 }, + { match: 'M5 Max', gbps: 460 }, + { match: 'M4 Max', gbps: 410 }, + { match: 'M3 Max', gbps: 300 }, + { match: 'M2 Max', gbps: 400 }, + { match: 'M1 Max', gbps: 400 }, + { match: 'M5 Pro', gbps: 307 }, + { match: 'M4 Pro', gbps: 273 }, + { match: 'M3 Pro', gbps: 150 }, + { match: 'M2 Pro', gbps: 200 }, + { match: 'M1 Pro', gbps: 200 }, + { match: 'M5', gbps: 154 }, + { match: 'M4', gbps: 120 }, + { match: 'M3', gbps: 100 }, + { match: 'M2', gbps: 100 }, + { match: 'M1', gbps: 68 }, ], - models: [], - harnesses: [], - chips: [], }; +let validated: LocalModelCatalog | undefined; + /** - * The catalogue in force. There is no override in this stage, so this is the stock constant; - * callers go through this function anyway, so a later stage can change where it comes from - * without touching them. + * The catalogue in force: the stock constant, validated once on first use. Callers go through + * this function, so a later stage can change where the catalogue comes from without touching + * them. Throws when the constant breaks its own schema, which the catalogue test rules out. */ export function getLocalModelCatalog(): LocalModelCatalog { - return STOCK_LOCAL_MODEL_CATALOG; + validated ??= LocalModelCatalogSchema.parse(STOCK_LOCAL_MODEL_CATALOG); + return validated; } diff --git a/src/local-model/hardware.ts b/src/local-model/hardware.ts index 687d981a..5c49cfb7 100644 --- a/src/local-model/hardware.ts +++ b/src/local-model/hardware.ts @@ -1,8 +1,6 @@ /** * @fileoverview The hardware probe of the local model recommendation: what this Mac is. * - * SKELETON: the exported signatures are final, the bodies are not written yet. This module - * will hold: * - `parseSysctl()`: a pure parser of `key: value` lines. The command runs WITHOUT `-n`, so a * key that is missing on a machine (`sysctl.proc_translated` on Intel) cannot shift the * others, and a line it does not know is ignored. @@ -12,17 +10,30 @@ * * Rules: * - Never throws and never rejects: a failed probe is an answer (`source: 'os-fallback'`), and - * off macOS nothing is spawned (`source: 'not-probed'`). + * off macOS nothing is spawned and no disk is read (`source: 'not-probed'`). * - Never `system_profiler` (tens of milliseconds and a parse of free text), never a second * spawn, never a synchronous exec on the server's event loop. * - A non-zero exit is tolerated once `hw.memsize` parsed (a missing key makes sysctl exit 1). + * Without `hw.memsize` nothing else of that output is trusted: the answer is the fallback. + * - Only a line that ends in a newline counts. A cut-off tail is not a value: half of + * `hw.memsize` would read as a smaller, valid number. * - `iogpu.wired_limit_mb` of 0 means "not set": it is reported as 0 and only a value above 0 * is ever used as a limit. + * - The data dir can sit on a network mount that stopped answering, so the disk read is + * bounded too: the folder is probed with the bounded path probe first, the `statfs` is raced + * against a timeout, and a `statfs` that never came back is not started a second time. * - `memGb`, `memClassGb` and `freeDiskGb` are filled here through src/local-model/format.ts, * the one place a size is formatted. */ +import { execFile } from 'node:child_process'; +import { statfs } from 'node:fs/promises'; +import os from 'node:os'; +import { dirname } from 'node:path'; import type { HardwareFacts } from '../types.js'; +import { getDataDir } from '../config/instance.js'; +import { probePathKind, type PathProbeKind } from '../utils/bounded-path-probe.js'; +import { formatGb, memClassGb } from './format.js'; /** Absolute path, so the server's PATH (minimal under a service manager) cannot change what runs. */ export const SYSCTL_PATH = '/usr/sbin/sysctl'; @@ -44,6 +55,18 @@ export const SYSCTL_TIMEOUT_MS = 2000; /** How long one probe result is reused. */ export const HARDWARE_CACHE_TTL_MS = 60_000; +/** Upper bound of the wait for the free disk figure. */ +export const FREE_DISK_TIMEOUT_MS = 2000; + +/** Seven short lines are well under this. A larger output is not sysctl's. */ +const SYSCTL_MAX_OUTPUT_BYTES = 64 * 1024; + +/** Longest text value kept (a chip name, a model identifier, a version). */ +const MAX_TEXT_VALUE_LENGTH = 100; + +/** How many parent folders are tried when the data dir does not exist yet. */ +const FREE_DISK_MAX_PARENT_STEPS = 6; + /** What `parseSysctl()` read. Every field is absent when its key was missing or its value did not parse. */ export interface SysctlFacts { /** `hw.memsize`. */ @@ -82,25 +105,305 @@ export interface HardwareProbeDeps { now?: () => number; } +/** A whole number without sign, as sysctl prints one. */ +function parseCount(value: string): number | undefined { + if (!/^\d+$/.test(value)) return undefined; + const parsed = Number(value); + return Number.isSafeInteger(parsed) ? parsed : undefined; +} + +/** sysctl prints a flag as 0 or 1. Anything else is not a flag. */ +function parseFlag(value: string): boolean | undefined { + if (value === '1') return true; + if (value === '0') return false; + return undefined; +} + +function parseText(value: string): string | undefined { + return value !== '' && value.length <= MAX_TEXT_VALUE_LENGTH ? value : undefined; +} + /** * Parse the output of `sysctl ` (no `-n`): one `key: value` line per key it knows. - * Pure. Unknown lines, blank lines and error lines are skipped. + * Pure. Unknown lines, blank lines and error lines are skipped, and so is a last line that does + * not end in a newline. The first line of a key wins. */ export function parseSysctl(text: string): SysctlFacts { - void text; - throw new Error('not implemented: parseSysctl'); + const facts: SysctlFacts = {}; + const lines = text.split('\n'); + // What follows the last newline is either empty or a cut-off line. + lines.pop(); + for (const rawLine of lines) { + const line = rawLine.replace(/\r$/, ''); + const colon = line.indexOf(':'); + if (colon <= 0) continue; + const key = line.slice(0, colon).trim(); + const value = line.slice(colon + 1).trim(); + switch (key) { + case 'hw.memsize': { + const bytes = parseCount(value); + if (bytes !== undefined && bytes > 0) facts.memBytes ??= bytes; + break; + } + case 'machdep.cpu.brand_string': + facts.chip ??= parseText(value); + break; + case 'hw.optional.arm64': + facts.arm64 ??= parseFlag(value); + break; + case 'sysctl.proc_translated': + facts.translated ??= parseFlag(value); + break; + case 'iogpu.wired_limit_mb': + facts.wiredLimitMb ??= parseCount(value); + break; + case 'kern.osproductversion': { + const version = /^(\d+)(?:\.\d+)*$/.exec(value); + if (version && value.length <= MAX_TEXT_VALUE_LENGTH && facts.macosVersion === undefined) { + facts.macosVersion = value; + facts.macosMajor = Number(version[1]); + } + break; + } + case 'hw.model': + facts.model ??= parseText(value); + break; + default: + break; + } + } + return facts; } +/** + * The one spawn: `command` with `SYSCTL_KEYS` as its arguments and nothing else (no `-n`). + * Resolves stdout when the program ran, also after a non-zero exit (a key this machine lacks + * makes sysctl exit 1 while the other keys are printed). Resolves `undefined` when it could not + * be started, was killed (the timeout) or printed too much. Never rejects. + * + * The two parameters exist for the test, which runs a script of its own in place of sysctl. + */ +export function spawnSysctl( + command: string = SYSCTL_PATH, + timeoutMs: number = SYSCTL_TIMEOUT_MS +): Promise { + return new Promise((resolve) => { + try { + execFile( + command, + [...SYSCTL_KEYS], + { + timeout: timeoutMs, + maxBuffer: SYSCTL_MAX_OUTPUT_BYTES, + encoding: 'utf8', + windowsHide: true, + // sysctl needs nothing from the server's environment, which can hold a password. + env: { LC_ALL: 'C' }, + }, + (error, stdout) => { + if (!error) return resolve(stdout); + // A numeric code is an exit status: the program ran. A string code (ENOENT, EACCES, + // the buffer limit) or a signal means it did not run to its end. + const ranToEnd = typeof error.code === 'number' && !error.killed && !error.signal; + resolve(ranToEnd ? stdout : undefined); + } + ); + } catch { + resolve(undefined); + } + }); +} + +/** + * Free bytes on the filesystem that holds `path` (`bavail * bsize`: what an unprivileged + * process can use), or `undefined` when the filesystem does not say. Not bounded by itself: see + * `dataDirFreeBytes()`. + */ +export async function freeDiskBytesOf(path: string): Promise { + try { + const stat = await statfs(path); + const free = Number(stat.bavail) * Number(stat.bsize); + return Number.isFinite(free) && free >= 0 ? free : undefined; + } catch { + return undefined; + } +} + +/** The one `statfs` that may be pending. While it is, no second one is started. */ +let pendingStatfs: Promise | undefined; + +/** The IO of `dataDirFreeBytes()`. Each member defaults to the real thing; a test passes its own. */ +export interface FreeDiskDeps { + /** Default `getDataDir()`. */ + dataDir?: () => string; + /** Default the bounded path probe (`probePathKind`). */ + probeKind?: (path: string) => Promise; + /** Default `freeDiskBytesOf`. */ + freeBytesOf?: (path: string) => Promise; + /** Default `FREE_DISK_TIMEOUT_MS`. */ + timeoutMs?: number; +} + +/** + * Free bytes on the data dir's filesystem, bounded. The folder (or, when it does not exist yet, + * its nearest existing parent) is found with the bounded path probe, which never waits on a + * dead mount twice; the `statfs` itself is raced against a timeout. `undefined` when the + * filesystem does not answer in time, or is not known to be a folder. + */ +export async function dataDirFreeBytes(deps: FreeDiskDeps = {}): Promise { + // Concurrent probes share one promise in `readHardware()`, so a `statfs` that is still pending + // here was started by an EARLIER probe and has outlived its timeout: a dead mount. It holds + // one threadpool worker, and that is the most this module may ever hold. + if (pendingStatfs) return undefined; + + const probeKind = deps.probeKind ?? probePathKind; + let folder: string; + try { + folder = (deps.dataDir ?? getDataDir)(); + let kind = await probeKind(folder); + for (let step = 0; kind === 'absent' && step < FREE_DISK_MAX_PARENT_STEPS; step++) { + const parent = dirname(folder); + if (parent === folder) break; + folder = parent; + kind = await probeKind(folder); + } + // 'unknown' is a mount that does not answer (or an unreadable path): do not touch it. + if (kind !== 'directory') return undefined; + } catch { + return undefined; + } + + const started = (deps.freeBytesOf ?? freeDiskBytesOf)(folder).catch(() => undefined); + pendingStatfs = started; + const forget = (): void => { + if (pendingStatfs === started) pendingStatfs = undefined; + }; + started.then(forget, forget); + + let timer: ReturnType | undefined; + try { + return await Promise.race([ + started, + new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), deps.timeoutMs ?? FREE_DISK_TIMEOUT_MS); + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +function defaultCpuModel(): string | undefined { + return os.cpus()[0]?.model; +} + +/** A dependency that must never break the probe: its value, or `undefined` when it threw or rejected. */ +async function attempt(read: () => T | Promise): Promise { + try { + return await read(); + } catch { + return undefined; + } +} + +/** Memory and the three fields derived from it. */ +function memoryFields(memBytes: number | undefined): Pick { + const bytes = memBytes !== undefined && Number.isFinite(memBytes) && memBytes > 0 ? memBytes : 0; + return { memBytes: bytes, memGb: formatGb(bytes), memClassGb: memClassGb(bytes) }; +} + +async function probeHardware(deps: HardwareProbeDeps): Promise { + const platform = deps.platform ?? process.platform; + const arch = deps.arch ?? os.arch(); + const totalMemBytes = deps.totalMemBytes ?? os.totalmem; + const cpuModel = deps.cpuModel ?? defaultCpuModel; + + if (platform !== 'darwin') { + return { platform, arch, ...memoryFields(await attempt(totalMemBytes)), source: 'not-probed' }; + } + + const [stdout, freeDisk] = await Promise.all([ + attempt(deps.runSysctl ?? (() => spawnSysctl())), + attempt(deps.freeDiskBytes ?? (() => dataDirFreeBytes())), + ]); + const disk: Pick = + freeDisk !== undefined && Number.isFinite(freeDisk) && freeDisk >= 0 + ? { freeDiskBytes: freeDisk, freeDiskGb: formatGb(freeDisk) } + : {}; + + const facts = typeof stdout === 'string' ? parseSysctl(stdout) : {}; + if (facts.memBytes === undefined) { + const chip = await attempt(cpuModel); + return { + platform, + arch, + ...memoryFields(await attempt(totalMemBytes)), + ...(chip ? { chip } : {}), + ...disk, + source: 'os-fallback', + }; + } + + const chip = facts.chip ?? (await attempt(cpuModel)); + return { + platform, + arch, + ...(facts.arm64 !== undefined ? { arm64: facts.arm64 } : {}), + ...(facts.translated !== undefined ? { translated: facts.translated } : {}), + ...memoryFields(facts.memBytes), + ...(chip ? { chip } : {}), + ...(facts.model !== undefined ? { model: facts.model } : {}), + ...(facts.macosVersion !== undefined ? { macosVersion: facts.macosVersion } : {}), + ...(facts.macosMajor !== undefined ? { macosMajor: facts.macosMajor } : {}), + ...(facts.wiredLimitMb !== undefined ? { wiredLimitMb: facts.wiredLimitMb } : {}), + ...disk, + source: 'sysctl', + }; +} + +let cached: { at: number; facts: HardwareFacts } | undefined; +let inFlight: Promise | undefined; + /** * What this machine is. Cached for `HARDWARE_CACHE_TTL_MS`; concurrent callers share one probe. * Never rejects. + * + * The cache is one per process and does not look at `deps`: within the 60 s a second call + * returns the first call's answer whatever it injects. A test resets it first. */ -export async function readHardware(deps?: HardwareProbeDeps): Promise { - void deps; - throw new Error('not implemented: readHardware'); +export async function readHardware(deps: HardwareProbeDeps = {}): Promise { + const now = deps.now ?? Date.now; + if (cached) { + const age = now() - cached.at; + if (age >= 0 && age < HARDWARE_CACHE_TTL_MS) return cached.facts; + } + if (inFlight) return inFlight; + + const probe = probeHardware(deps) + .catch( + // Unreachable by design (every read above is guarded), and still an answer if it is reached. + (): HardwareFacts => ({ + platform: deps.platform ?? process.platform, + arch: deps.arch ?? os.arch(), + ...memoryFields(undefined), + source: (deps.platform ?? process.platform) === 'darwin' ? 'os-fallback' : 'not-probed', + }) + ) + .then((facts) => { + cached = { at: now(), facts }; + return facts; + }) + .finally(() => { + if (inFlight === probe) inFlight = undefined; + }); + inFlight = probe; + return probe; } -/** Drop the cached probe result (tests, and nothing else). */ +/** Drop the cached probe result and the memory of a pending disk read (tests, and nothing else). */ export function resetHardwareCache(): void { - throw new Error('not implemented: resetHardwareCache'); + cached = undefined; + inFlight = undefined; + pendingStatfs = undefined; } diff --git a/src/local-model/recommend.ts b/src/local-model/recommend.ts index 2536e63c..a0513093 100644 --- a/src/local-model/recommend.ts +++ b/src/local-model/recommend.ts @@ -2,54 +2,231 @@ * @fileoverview The recommendation itself: which model, at which context, with which harness * fits this Mac. Pure: no IO, no clock, no CLI registry, no data dir. * - * SKELETON: the exported signatures are final, the bodies are not written yet. This module - * will hold: + * The math, in the order it runs: * - `deviceBudget()`: L is the wired limit when it is above 0, else 2/3 of memory below 36 GB * and 3/4 from 36 GB. B = min(L - 1 GiB, memory - reserve), with a reserve of 6 GiB up to - * 18 GB, 8 GiB up to 48 GB and 12 GiB from 64 GB. + * 18 GB, 8 GiB up to 48 GB and 12 GiB above. * - `needBytes()`: F = weights + kvBytesPerToken * ctx + 1 GiB + the prompt cache tier of the * catalogue for this Mac's memory. - * - `recommend()`: the candidates (model, context, harness) with `ctx >= harness.minContext`, - * the harness installed or installable, accepting the model id and not gated; the fit (F/B up - * to 0.85 good, up to 0.98 tight, above out); the speed (ok, slow or unknown: a chip that is - * in no chips row is unknown, a bandwidth is never guessed); the ranking (good before tight, - * then ok before unknown before slow, then model rank, harness rank, larger context); and the - * verdict. + * - Fit: F/B up to 0.85 is good, up to 0.98 is tight, above is out. + * - Speed: tokens per second = the model's `speedPer100Gbps` x the chip's bandwidth / 100. + * Below 10 is slow. A chip that is in no chips row, or a model without a speed figure, is + * UNKNOWN: a bandwidth is never guessed. + * - Candidates: (model, context, harness) with `ctx >= harness.minContext`, the harness not + * gated, installed or installable, and accepting the model id. + * - Ranking, lexicographic: good before tight; ok before unknown before slow; the higher model + * rank; the higher harness rank; the larger context. The pick is the first candidate, the + * alternatives are the next four MODELS, each at its own best context and harness. + * + * Verdict `none`, in this order (the first that applies is the reason): + * 1. `not-macos`: the server does not run on macOS. + * 2. `intel`: a Mac without Apple silicon. + * 3. `macos`: macOS is older than the catalogue's minimum (an unknown version does not block). + * 4. `memory`: no model fits the budget at any of its contexts, whatever the harness. + * 5. `no-harness`: no harness row is usable (not gated, known to the caller, installed or + * installable). + * 6. `memory`: models fit, but none at a context a usable harness needs. + * 7. `no-harness`: models fit at such a context, but no usable harness accepts their id. + * 8. `disk`: for every remaining model, free disk minus the download is below 10 GiB (an + * unknown free disk does not block). * * Rules: - * - Verdict `limited` when the best candidate's model has fewer than 12 billion parameters. - * - The word "Recommended" (`LOCAL_MODEL_LABELS.headlineTested`) only for a pick whose model - * has a `tested` entry matching this Mac. Otherwise `headlineOk` plus `noteUntested`. - * - Reasons for `none`: not-macos, intel, memory, disk (free disk minus the download is below - * 10 GiB for every fitting model), macos, no-harness. + * - Verdict `limited` when the pick's model has fewer than 12 billion parameters. + * - The word "Recommended" (`LOCAL_MODEL_LABELS.headlineTested`) only for verdict `ok` with a + * pick whose model has a `tested` entry for exactly this chip, this memory size and this + * harness. Otherwise `headlineOk` (or `headlineLimited`) plus `noteUntested`. * - Every label in the result comes from `LOCAL_MODEL_LABELS`, every size text from * `formatGb()`. No number is spliced into a sentence here. * - No CLI id is ever compared with a literal: a harness is a catalogue row plus a * `HarnessFact`. */ -import type { - DeviceBudget, - HardwareFacts, - HarnessFact, - LocalModelCatalog, - LocalModelEntry, - Recommendation, +import { + LOCAL_MODEL_LABELS, + LOCAL_MODEL_NONE_LABELS, + type Candidate, + type DeviceBudget, + type FitGrade, + type HardwareFacts, + type HarnessFact, + type LocalHarnessEntry, + type LocalModelCatalog, + type LocalModelEntry, + type NoneReason, + type Recommendation, + type SpeedGrade, } from '../types.js'; +import { chipBandwidthGbps } from '../config/local-model-catalog.js'; +import { formatGb, GIB, MIB } from './format.js'; + +/** Need over budget up to this share is a `good` fit. */ +export const FIT_GOOD_MAX_SHARE = 0.85; +/** Need over budget up to this share is a `tight` fit; above it the model is out. */ +export const FIT_TIGHT_MAX_SHARE = 0.98; +/** An estimated speed below this many tokens per second is `slow`. */ +export const SLOW_BELOW_TOKENS_PER_SECOND = 10; +/** A pick whose model has fewer than this many billion parameters makes the verdict `limited`. */ +export const LIMITED_BELOW_PARAMS_B = 12; +/** What must stay free on the disk after the model download. */ +export const DISK_HEADROOM_BYTES = 10 * GIB; +/** What the runtime needs beside the weights, the KV cache and the prompt cache (compute buffers). */ +export const RUNTIME_OVERHEAD_BYTES = GIB; +/** How many other models are listed under the pick. */ +export const MAX_ALTERNATIVES = 4; + +/** A memory size of at least this many GB (as sold) gets 3/4 of its memory as device limit, below it 2/3. */ +const LARGE_LIMIT_FROM_GB = 36; + +/** The memory the system, Codeman, a browser and the agent keep for themselves, by the size the Mac is sold as. */ +function reserveBytes(memClassGb: number): number { + if (memClassGb <= 18) return 6 * GIB; + if (memClassGb <= 48) return 8 * GIB; + return 12 * GIB; +} /** The memory a model may use on this Mac: the device limit L, the budget B and where L came from. */ export function deviceBudget(hw: HardwareFacts): DeviceBudget { - void hw; - throw new Error('not implemented: deviceBudget'); + const memBytes = Number.isFinite(hw.memBytes) && hw.memBytes > 0 ? hw.memBytes : 0; + // 0 means "not set" (both test Macs report 0): only a value above 0 is a limit. + const wired = hw.wiredLimitMb !== undefined && Number.isFinite(hw.wiredLimitMb) && hw.wiredLimitMb > 0; + const limitBytes = wired + ? Math.floor((hw.wiredLimitMb as number) * MIB) + : Math.floor(memBytes * (hw.memClassGb < LARGE_LIMIT_FROM_GB ? 2 / 3 : 3 / 4)); + const budgetBytes = Math.max(0, Math.min(limitBytes - GIB, memBytes - reserveBytes(hw.memClassGb))); + return { limitBytes, budgetBytes, source: wired ? 'wired-limit' : 'estimated' }; +} + +/** The prompt cache of the last tier whose minimum is not above this Mac's memory size. */ +function promptCacheBytes(catalog: LocalModelCatalog, memClassGb: number): number { + let bytes = 0; + for (const tier of catalog.promptCacheTiers) { + if (tier.minMemGb <= memClassGb) bytes = tier.bytes; + } + return bytes; } /** F: the bytes `model` needs in memory at a context of `ctx` tokens on this Mac. */ export function needBytes(model: LocalModelEntry, ctx: number, catalog: LocalModelCatalog, hw: HardwareFacts): number { - void model; - void ctx; - void catalog; - void hw; - throw new Error('not implemented: needBytes'); + return model.bytes + model.kvBytesPerToken * ctx + RUNTIME_OVERHEAD_BYTES + promptCacheBytes(catalog, hw.memClassGb); +} + +function fitGrade(need: number, budgetBytes: number): FitGrade { + if (budgetBytes <= 0) return 'out'; + const share = need / budgetBytes; + if (share <= FIT_GOOD_MAX_SHARE) return 'good'; + if (share <= FIT_TIGHT_MAX_SHARE) return 'tight'; + return 'out'; +} + +/** + * Whether the MACHINE is Apple silicon. `hw.optional.arm64` decides when it was read. Without + * it: a process under Rosetta, an arm64 process or an Apple chip name each prove Apple silicon + * (`arch` alone cannot say no, because Node under Rosetta reports x64). + */ +function isAppleSilicon(hw: HardwareFacts): boolean { + if (hw.arm64 !== undefined) return hw.arm64; + if (hw.translated === true || hw.arch === 'arm64') return true; + return /^Apple\b/i.test(hw.chip ?? ''); +} + +function round(value: number, decimals: number): number { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +} + +/** A candidate plus the two ranks the ordering needs, which are not part of the payload. */ +interface Ranked { + candidate: Candidate; + modelRank: number; + harnessRank: number; +} + +/** A catalogue harness row with the caller's fact about it. */ +interface UsableHarness { + row: LocalHarnessEntry; + fact: HarnessFact; +} + +const FIT_ORDER: Record = { good: 0, tight: 1 }; +const SPEED_ORDER: Record = { ok: 0, unknown: 1, slow: 2 }; + +/** Best first. */ +function compareRanked(a: Ranked, b: Ranked): number { + return ( + FIT_ORDER[a.candidate.fit] - FIT_ORDER[b.candidate.fit] || + SPEED_ORDER[a.candidate.speed] - SPEED_ORDER[b.candidate.speed] || + b.modelRank - a.modelRank || + b.harnessRank - a.harnessRank || + b.candidate.context - a.candidate.context + ); +} + +/** A function supplied by the caller must not be able to break a pure answer. */ +function accepts(fact: HarnessFact, modelId: string): boolean { + try { + return fact.acceptsModelId(modelId) === true; + } catch { + return false; + } +} + +function buildCandidate( + model: LocalModelEntry, + ctx: number, + harness: UsableHarness, + need: number, + fit: Candidate['fit'], + hw: HardwareFacts, + catalog: LocalModelCatalog, + budgetBytes: number +): Candidate { + const gbps = chipBandwidthGbps(catalog.chips, hw.chip); + const tokensPerSecond = + gbps !== undefined && model.speedPer100Gbps !== undefined + ? round((model.speedPer100Gbps * gbps) / 100, 1) + : undefined; + const speed: SpeedGrade = + tokensPerSecond === undefined ? 'unknown' : tokensPerSecond < SLOW_BELOW_TOKENS_PER_SECOND ? 'slow' : 'ok'; + // "A Mac like this" is exact: the same chip string, the same memory size as sold, the same harness. + const tested = model.tested.some( + (entry) => entry.chip === hw.chip && entry.memGb === hw.memClassGb && entry.cli === harness.row.cli + ); + return { + modelId: model.id, + modelLabel: model.label, + license: model.license, + paramsB: model.paramsB, + context: ctx, + cli: harness.row.cli, + cliLabel: harness.fact.label, + cliInstalled: harness.fact.installed, + ...(harness.row.slowFirstTurn ? { slowFirstTurn: true } : {}), + needBytes: need, + needGb: formatGb(need), + downloadBytes: model.bytes, + downloadGb: formatGb(model.bytes), + budgetShare: round(need / budgetBytes, 3), + fit, + fitLabel: fit === 'good' ? LOCAL_MODEL_LABELS.fitGood : LOCAL_MODEL_LABELS.fitTight, + speed, + ...(speed === 'slow' ? { speedLabel: LOCAL_MODEL_LABELS.speedSlow } : {}), + ...(speed === 'unknown' ? { speedLabel: LOCAL_MODEL_LABELS.speedUnknown } : {}), + ...(tokensPerSecond !== undefined ? { tokensPerSecond } : {}), + tested, + }; +} + +function none(reason: NoneReason, budget: DeviceBudget, vars?: Record): Recommendation { + return { + verdict: 'none', + reason, + headline: { text: LOCAL_MODEL_NONE_LABELS[reason], ...(vars ? { vars } : {}) }, + notes: [], + alternatives: [], + budgetBytes: budget.budgetBytes, + budgetGb: formatGb(budget.budgetBytes), + budgetSource: budget.source, + }; } /** @@ -61,8 +238,99 @@ export function recommend( catalog: LocalModelCatalog, harnessFacts: readonly HarnessFact[] ): Recommendation { - void hw; - void catalog; - void harnessFacts; - throw new Error('not implemented: recommend'); + // Off macOS and on an Intel Mac there is no budget to compute: a model has no memory to run in. + const noBudget: DeviceBudget = { limitBytes: 0, budgetBytes: 0, source: 'estimated' }; + if (hw.platform !== 'darwin') return none('not-macos', noBudget); + if (!isAppleSilicon(hw)) return none('intel', noBudget); + + const budget = deviceBudget(hw); + if (hw.macosMajor !== undefined && hw.macosMajor < catalog.minMacosMajor) { + return none('macos', budget, { + version: hw.macosVersion ?? String(hw.macosMajor), + minimum: String(catalog.minMacosMajor), + }); + } + + // Every (model, context) that fits the budget, before any harness is looked at. + const fitting: { model: LocalModelEntry; ctx: number; need: number; fit: Candidate['fit'] }[] = []; + for (const model of catalog.models) { + for (const ctx of model.contexts) { + const need = needBytes(model, ctx, catalog, hw); + const fit = fitGrade(need, budget.budgetBytes); + if (fit !== 'out') fitting.push({ model, ctx, need, fit }); + } + } + const tooLittleMemory = (): Recommendation => none('memory', budget, { memory: hw.memGb }); + if (fitting.length === 0) return tooLittleMemory(); + + // A gated row is never offered, whatever the caller knows about its CLI. + const usable: UsableHarness[] = []; + for (const row of catalog.harnesses) { + if (row.gated) continue; + const fact = harnessFacts.find((candidate) => candidate.cli === row.cli); + if (fact && (fact.installed || fact.installable)) usable.push({ row, fact }); + } + if (usable.length === 0) return none('no-harness', budget); + + const atHarnessContext = fitting.flatMap((entry) => + usable.filter((harness) => entry.ctx >= harness.row.minContext).map((harness) => ({ ...entry, harness })) + ); + if (atHarnessContext.length === 0) return tooLittleMemory(); + + const accepted = atHarnessContext.filter((entry) => accepts(entry.harness.fact, entry.model.id)); + if (accepted.length === 0) return none('no-harness', budget); + + // The download must leave headroom on the disk. An unknown free disk does not block. + const free = hw.freeDiskBytes; + const downloadable = + free === undefined ? accepted : accepted.filter((entry) => free - entry.model.bytes >= DISK_HEADROOM_BYTES); + if (free !== undefined && downloadable.length === 0) { + const smallest = Math.min(...accepted.map((entry) => entry.model.bytes)); + return none('disk', budget, { + download: formatGb(smallest), + free: hw.freeDiskGb ?? formatGb(free), + missing: formatGb(smallest + DISK_HEADROOM_BYTES - free), + }); + } + + const ranked: Ranked[] = downloadable + .map((entry) => ({ + candidate: buildCandidate( + entry.model, + entry.ctx, + entry.harness, + entry.need, + entry.fit, + hw, + catalog, + budget.budgetBytes + ), + modelRank: entry.model.rank, + harnessRank: entry.harness.row.rank, + })) + .sort(compareRanked); + + // The first candidate of each model is that model at its best context and harness. + const bestPerModel: Candidate[] = []; + for (const { candidate } of ranked) { + if (!bestPerModel.some((kept) => kept.modelId === candidate.modelId)) bestPerModel.push(candidate); + } + const [pick, ...others] = bestPerModel; + + const limited = pick.paramsB < LIMITED_BELOW_PARAMS_B; + const headline = limited + ? LOCAL_MODEL_LABELS.headlineLimited + : pick.tested + ? LOCAL_MODEL_LABELS.headlineTested + : LOCAL_MODEL_LABELS.headlineOk; + return { + verdict: limited ? 'limited' : 'ok', + headline: { text: headline }, + notes: pick.tested ? [] : [{ text: LOCAL_MODEL_LABELS.noteUntested }], + pick, + alternatives: others.slice(0, MAX_ALTERNATIVES), + budgetBytes: budget.budgetBytes, + budgetGb: formatGb(budget.budgetBytes), + budgetSource: budget.source, + }; } diff --git a/src/types/local-model.ts b/src/types/local-model.ts index 5298c3bb..77352d64 100644 --- a/src/types/local-model.ts +++ b/src/types/local-model.ts @@ -129,11 +129,14 @@ export interface LocalHarnessEntry { /** * Memory bandwidth of one chip family. `LocalModelCatalog.chips` is ordered most specific first - * ('M4 Max' before 'M4') and the first row whose `match` occurs in `HardwareFacts.chip` wins. A - * chip that matches no row has an UNKNOWN speed: a bandwidth is never guessed. + * ('M4 Max' before 'M4') and the first row that names `HardwareFacts.chip` wins. A row names a + * chip when the chip string ENDS with its `match`, starting at the beginning or after a space, + * case ignored (`chipNameMatches()` in src/config/local-model-catalog.ts): 'M4' is not the chip + * of 'Apple M4 Pro', and a variant the table has never heard of ('Apple M6 Pro') matches no row. + * A chip that matches no row has an UNKNOWN speed: a bandwidth is never guessed. */ export interface ChipEntry { - /** Text that must occur in the chip string ('M4 Max'). */ + /** The chip name the chip string must end with ('M4 Max'). */ match: string; /** Memory bandwidth in GB/s. */ gbps: number; @@ -305,7 +308,7 @@ export interface Candidate { downloadBytes: number; /** `downloadBytes` as decimal GB text, no unit. */ downloadGb: string; - /** `needBytes / Recommendation.budgetBytes` (0.83). */ + /** `needBytes / Recommendation.budgetBytes`, rounded to three decimals (0.83). The fit grade is decided on the unrounded ratio. */ budgetShare: number; /** A candidate in a `Recommendation` is never `out`. */ fit: Exclude; @@ -339,7 +342,7 @@ export interface Recommendation { pick?: Candidate; /** The next models that fit (at most four, each at its best context and harness), best first. Empty when `verdict` is `none`. */ alternatives: Candidate[]; - /** `DeviceBudget.budgetBytes`. 0 when the budget could not be computed (not macOS). */ + /** `DeviceBudget.budgetBytes`. 0 when no budget is computed: not macOS, or a Mac without Apple silicon. */ budgetBytes: number; /** `budgetBytes` as decimal GB text, no unit. */ budgetGb: string; diff --git a/test/cli-registry-no-id-branching.test.ts b/test/cli-registry-no-id-branching.test.ts index c746bdfb..cf31be83 100644 --- a/test/cli-registry-no-id-branching.test.ts +++ b/test/cli-registry-no-id-branching.test.ts @@ -167,7 +167,6 @@ const ALLOWED_BRANCHES: Record = { ), "tui/tui-render.ts::mode !== 'claude'": allow(1, 'cosmetic: suppress the mode badge for the default mode'), - // Legacy Config plumbing, same category as the `===` entries above. "web/routes/session-routes.ts::mode !== 'omp'": allow( 2, @@ -212,8 +211,10 @@ const ID_ALT = IDS.join('|'); */ const BRANCH_PATTERN = new RegExp( [ - // mode === 'codex' / mode !== 'codex' - `\\b(?:mode|id|agentType)\\s*[!=]==\\s*'(?:${ID_ALT})'`, + // mode === 'codex' / mode !== 'codex'. `cli` is the name the local model catalogue gives + // the same thing (`LocalHarnessEntry.cli`, `Candidate.cli`), so `harness.cli === 'claude'` + // is the same banned branch under another field name. + `\\b(?:mode|id|agentType|cli)\\s*[!=]==\\s*'(?:${ID_ALT})'`, // case 'codex': `\\bcase\\s+'(?:${ID_ALT})'\\s*:`, // ['codex', 'gemini'].includes(mode) — the id list IS the branch, wherever `mode` sits @@ -267,7 +268,7 @@ function scan(): { findings: Finding[]; filesScanned: number } { lines.forEach((line, i) => { BRANCH_PATTERN.lastIndex = 0; // shared /g regex — see utils/regex-patterns.ts for (const match of line.matchAll(BRANCH_PATTERN)) { - const expression = match[0].replace(/\s+/g, ' ').replace(/^(?:id|agentType)/, 'mode'); + const expression = match[0].replace(/\s+/g, ' ').replace(/^(?:id|agentType|cli)/, 'mode'); const posix = rel.split(sep).join('/'); findings.push({ file: posix, expression, line: i + 1, key: `${posix}::${expression}` }); } @@ -337,6 +338,8 @@ describe('no CLI-id branching outside the stock catalog', () => { "if (mode !== 'shell' && mode !== 'deepseek') { doSomething(); }", "switch (mode) { case 'gemini': return 1; }", "if (['codex', 'gemini'].includes(mode)) { doSomething(); }", + "if (harness.cli === 'claude') { doSomething(); }", + "const others = candidates.filter((c) => c.cli !== 'opencode');", ]; for (const sample of samples) { BRANCH_PATTERN.lastIndex = 0; diff --git a/test/local-model-catalog.test.ts b/test/local-model-catalog.test.ts new file mode 100644 index 00000000..5f72b764 --- /dev/null +++ b/test/local-model-catalog.test.ts @@ -0,0 +1,383 @@ +/** + * @fileoverview Tests for src/config/local-model-catalog.ts: the compiled catalogue of models, + * harnesses and chips, its schema, and the chip lookup. + * + * Three things are pinned: + * + * 1. The stock constant PARSES. `getLocalModelCatalog()` validates it on first use and throws on + * a bad row, so this test is what keeps a bad row from ever reaching the route. + * 2. The schema refuses what must never get in: a model id that could name another registry + * (a slash, a host), an unknown key (a `runtime` block with a download URL), and rows that + * break a rule across rows (a repeated id, contexts in the wrong order, a chip row that + * hides a more specific one). + * 3. The chip lookup never guesses: a chip that no row names has no bandwidth, including one + * whose name merely contains a known name ('Apple M10', 'Apple M6 Pro'). + * + * Port: N/A (pure data and functions). + */ + +import { describe, expect, it } from 'vitest'; +import { + chipBandwidthGbps, + chipNameMatches, + getLocalModelCatalog, + LOCAL_MODEL_ID_PATTERN, + LocalModelCatalogSchema, + STOCK_LOCAL_MODEL_CATALOG, +} from '../src/config/local-model-catalog.js'; +import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js'; + +/** A deep copy of the stock catalogue that a test may break. */ +function copy(): LocalModelCatalog { + return structuredClone(STOCK_LOCAL_MODEL_CATALOG); +} + +/** The message of every issue the schema raises for `catalog`, or an empty list when it parses. */ +function issuesOf(catalog: unknown): string[] { + const result = LocalModelCatalogSchema.safeParse(catalog); + return result.success ? [] : result.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`); +} + +/** The stock catalogue with its first model changed. */ +/** `url` is no field of a model row: one test passes it to prove the schema refuses an unknown key. */ +function withFirstModel(change: Partial>): unknown { + const catalog = copy(); + return { ...catalog, models: [{ ...catalog.models[0], ...change }, ...catalog.models.slice(1)] }; +} + +describe('the stock catalogue', () => { + it('passes its own schema', () => { + expect(issuesOf(STOCK_LOCAL_MODEL_CATALOG)).toEqual([]); + }); + + it('is what the getter returns, validated once', () => { + const first = getLocalModelCatalog(); + expect(first).toEqual(STOCK_LOCAL_MODEL_CATALOG); + // Parsed on first use, then reused: the same object, not a new parse per request. + expect(getLocalModelCatalog()).toBe(first); + }); + + it('lists the seven models of the design, without flint:40b', () => { + expect(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => m.id)).toEqual([ + 'qwen3.5:9b', + 'qwen3.5:9b-q8_0', + 'gpt-oss:20b', + 'qwen3.6:35b-a3b', + 'qwen3.8:27b', + 'qwen3.8:27b-q8_0', + 'qwen3.5:122b-a10b', + ]); + expect(STOCK_LOCAL_MODEL_CATALOG.models.some((m) => m.id.startsWith('flint'))).toBe(false); + }); + + it('carries the byte sizes of the registry manifests', () => { + // The sum of the layer sizes of each tag's manifest, as fetched on 2026-11-06. + expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.bytes]))).toEqual({ + 'qwen3.5:9b': 6_550_825_373, + 'qwen3.5:9b-q8_0': 10_449_217_949, + 'gpt-oss:20b': 13_793_441_427, + 'qwen3.6:35b-a3b': 22_621_314_161, + 'qwen3.8:27b': 17_741_871_939, + 'qwen3.8:27b-q8_0': 29_978_241_837, + 'qwen3.5:122b-a10b': 81_370_035_877, + }); + }); + + it('carries the KV cost per token of each model', () => { + expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.kvBytesPerToken]))).toEqual({ + 'qwen3.5:9b': 32_768, + 'qwen3.5:9b-q8_0': 32_768, + 'gpt-oss:20b': 24_576, + 'qwen3.6:35b-a3b': 20_480, + 'qwen3.8:27b': 65_536, + 'qwen3.8:27b-q8_0': 65_536, + 'qwen3.5:122b-a10b': 24_576, + }); + }); + + it('gives every model a parameter count and an empty tested list', () => { + for (const model of STOCK_LOCAL_MODEL_CATALOG.models) { + expect(model.paramsB, model.id).toBeGreaterThan(0); + // Nobody has run any of these with a harness on a Mac yet. + expect(model.tested, model.id).toEqual([]); + expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN); + expect(model.license).toBe('Apache-2.0'); + } + // The two 9B rows are the ones under the 12 billion line of the `limited` verdict. + expect(STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual([ + 'qwen3.5:9b', + 'qwen3.5:9b-q8_0', + ]); + }); + + it('has a speed figure only where one was measured or scaled from a measurement', () => { + expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({ + 'qwen3.5:9b': 9.3, + 'qwen3.5:9b-q8_0': 5.8, + 'gpt-oss:20b': null, + 'qwen3.6:35b-a3b': 13.6, + 'qwen3.8:27b': 2.95, + 'qwen3.8:27b-q8_0': 1.7, + 'qwen3.5:122b-a10b': null, + }); + }); + + it('lists its harnesses as data rows, one of them gated', () => { + expect(STOCK_LOCAL_MODEL_CATALOG.harnesses).toEqual([ + { cli: 'opencode', minContext: 65536, rank: 3 }, + { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true }, + { cli: 'pi', minContext: 32768, rank: 1, gated: true }, + ]); + }); + + it('counts the prompt cache as 512 MiB below 32 GB and 2 GiB from 32 GB', () => { + expect(STOCK_LOCAL_MODEL_CATALOG.promptCacheTiers).toEqual([ + { minMemGb: 0, bytes: 512 * 1024 ** 2 }, + { minMemGb: 32, bytes: 2 * 1024 ** 3 }, + ]); + expect(STOCK_LOCAL_MODEL_CATALOG.minMacosMajor).toBe(14); + expect(STOCK_LOCAL_MODEL_CATALOG.schema).toBe(1); + }); + + it('holds no download data: no URL, no digest, no runtime block', () => { + const text = JSON.stringify(STOCK_LOCAL_MODEL_CATALOG); + expect(text).not.toMatch(/https?:/); + expect(text).not.toMatch(/sha256/i); + expect(Object.keys(STOCK_LOCAL_MODEL_CATALOG).sort()).toEqual( + ['chips', 'harnesses', 'minMacosMajor', 'models', 'promptCacheTiers', 'schema', 'updated'].sort() + ); + }); +}); + +describe('the model id rule', () => { + it('accepts name:tag in lowercase', () => { + for (const id of ['qwen3.5:9b', 'gpt-oss:20b', 'qwen3.8:27b-q8_0', 'a:b', 'x_y.z-1:2.3_4']) { + expect(LOCAL_MODEL_ID_PATTERN.test(id), id).toBe(true); + } + }); + + it('refuses anything that could name another registry, namespace or no tag at all', () => { + const bad = [ + 'evil.example/x/y:z', + 'library/qwen3.5:9b', + 'registry.ollama.ai/library/qwen3.5:9b', + 'Qwen3.5:9b', + 'qwen3.5:9B', + 'qwen3.5', + 'qwen3.5:', + ':9b', + 'qwen3.5:9b:extra', + 'qwen 3.5:9b', + '-qwen:9b', + 'qwen:-9b', + 'qwen3.5:9b\n', + '', + ]; + for (const id of bad) expect(LOCAL_MODEL_ID_PATTERN.test(id), JSON.stringify(id)).toBe(false); + }); + + it('makes the schema reject a catalogue with such an id', () => { + for (const id of ['evil.example/x/y:z', 'Qwen3.5:9b', 'qwen3.5', 'qwen3.5:9b ']) { + const issues = issuesOf(withFirstModel({ id })); + expect(issues, id).toHaveLength(1); + expect(issues[0]).toContain('models.0.id'); + } + }); +}); + +describe('the schema, field by field', () => { + it('refuses a key it does not know, at the top and inside a row', () => { + // The whole point of having no `runtime` block: a data row can never carry a download URL. + expect(issuesOf({ ...copy(), runtime: { url: 'https://example.com/x.tgz', sha256: 'abc' } })).not.toEqual([]); + expect(issuesOf(withFirstModel({ url: 'https://example.com/model' }))).not.toEqual([]); + const catalog = copy(); + expect(issuesOf({ ...catalog, harnesses: [{ ...catalog.harnesses[0], dialect: 'openai' }] })).not.toEqual([]); + expect(issuesOf({ ...catalog, chips: [{ match: 'M4', gbps: 120, cores: 10 }] })).not.toEqual([]); + }); + + it('requires paramsB and tested on every model row', () => { + for (const field of ['paramsB', 'tested'] as const) { + const row: Record = { ...copy().models[0] }; + delete row[field]; + const issues = issuesOf({ ...copy(), models: [row] }); + expect(issues, field).toHaveLength(1); + expect(issues[0]).toContain(`models.0.${field}`); + } + }); + + it('refuses sizes and counts that are not positive whole numbers', () => { + expect(issuesOf(withFirstModel({ bytes: 0 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ bytes: 1.5 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ kvBytesPerToken: -1 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ contexts: [] }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ paramsB: 0 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ speedPer100Gbps: 0 }))).not.toEqual([]); + }); + + it('accepts a tested entry and checks its shape', () => { + const good = { chip: 'Apple M4', memGb: 16, cli: 'opencode', date: '2026-11-06' }; + expect(issuesOf(withFirstModel({ tested: [good] }))).toEqual([]); + expect(issuesOf(withFirstModel({ tested: [{ ...good, date: '6 November 2026' }] }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ tested: [{ ...good, memGb: 0 }] }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ tested: [{ ...good, result: 'pass' }] }))).not.toEqual([]); + }); + + it('refuses another schema version and a date that is not ISO', () => { + expect(issuesOf({ ...copy(), schema: 2 })).not.toEqual([]); + expect(issuesOf({ ...copy(), updated: 'today' })).not.toEqual([]); + }); +}); + +describe('the schema, across rows', () => { + it('refuses a repeated model id', () => { + const catalog = copy(); + catalog.models[1] = { ...catalog.models[1], id: catalog.models[0].id }; + expect(issuesOf(catalog)).toEqual(['models.1.id: model ids must be unique']); + }); + + it('refuses a repeated model rank, which would leave the order of two models to chance', () => { + const catalog = copy(); + catalog.models[1] = { ...catalog.models[1], rank: catalog.models[0].rank }; + expect(issuesOf(catalog)).toEqual(['models.1.rank: model ranks must be unique']); + }); + + it('refuses a harness listed twice and a repeated harness rank', () => { + const twice = copy(); + twice.harnesses.push({ cli: twice.harnesses[0].cli, minContext: 32768, rank: 9 }); + expect(issuesOf(twice)).toEqual(['harnesses.3.cli: a harness may be listed only once']); + + const sameRank = copy(); + sameRank.harnesses[1] = { ...sameRank.harnesses[1], rank: sameRank.harnesses[0].rank }; + expect(issuesOf(sameRank)).toEqual(['harnesses.1.rank: harness ranks must be unique']); + }); + + it('refuses contexts that are not listed largest first, or that repeat', () => { + expect(issuesOf(withFirstModel({ contexts: [32768, 65536] }))).toEqual([ + 'models.0.contexts.1: contexts must be listed largest first, without repeats', + ]); + expect(issuesOf(withFirstModel({ contexts: [65536, 65536] }))).toEqual([ + 'models.0.contexts.1: contexts must be listed largest first, without repeats', + ]); + }); + + it('refuses a tested entry for a harness the catalogue does not list', () => { + const entry = { chip: 'Apple M4', memGb: 16, cli: 'not-a-harness', date: '2026-11-06' }; + expect(issuesOf(withFirstModel({ tested: [entry] }))).toEqual([ + 'models.0.tested.0.cli: a tested entry must name a harness of this catalogue', + ]); + }); + + it('refuses chips that are not listed most specific first', () => { + const catalog = copy(); + catalog.chips = [ + { match: 'M4', gbps: 120 }, + { match: 'M4 Max', gbps: 410 }, + ]; + expect(issuesOf(catalog)).toEqual([ + "chips.1.match: chips must be listed most specific first: 'M4' comes before 'M4 Max'", + ]); + + const repeated = copy(); + repeated.chips = [ + { match: 'M4', gbps: 120 }, + { match: 'm4', gbps: 130 }, + ]; + expect(issuesOf(repeated)).toHaveLength(1); + }); + + it('refuses prompt cache tiers that do not start at 0 or do not ascend', () => { + const late = copy(); + late.promptCacheTiers = [{ minMemGb: 16, bytes: 1 }]; + expect(issuesOf(late)).toEqual(['promptCacheTiers.0.minMemGb: the first prompt cache tier must start at 0']); + + const unordered = copy(); + unordered.promptCacheTiers = [ + { minMemGb: 0, bytes: 1 }, + { minMemGb: 32, bytes: 2 }, + { minMemGb: 24, bytes: 3 }, + ]; + expect(issuesOf(unordered)).toEqual(['promptCacheTiers.2.minMemGb: prompt cache tiers must ascend by minMemGb']); + }); +}); + +describe('the chips table', () => { + const chips = STOCK_LOCAL_MODEL_CATALOG.chips; + + it('holds every chip of the source table that shipped', () => { + // Bandwidth column of https://github.com/ggml-org/llama.cpp/discussions/4167, read on + // 2026-11-06. A name that covers two bandwidths has the lower one. + expect(Object.fromEntries(chips.map((row) => [row.match, row.gbps]))).toEqual({ + 'M5 Ultra': 1228, + 'M3 Ultra': 800, + 'M2 Ultra': 800, + 'M1 Ultra': 800, + 'M5 Max': 460, + 'M4 Max': 410, + 'M3 Max': 300, + 'M2 Max': 400, + 'M1 Max': 400, + 'M5 Pro': 307, + 'M4 Pro': 273, + 'M3 Pro': 150, + 'M2 Pro': 200, + 'M1 Pro': 200, + M5: 154, + M4: 120, + M3: 100, + M2: 100, + M1: 68, + }); + }); + + it('finds each row under the name macOS reports for that chip', () => { + // No row is hidden by an earlier one: 'Apple M4 Max' is 410, not the 120 of 'M4'. + for (const row of chips) { + expect(chipBandwidthGbps(chips, `Apple ${row.match}`), row.match).toBe(row.gbps); + } + }); + + it('knows the chips of the two test Macs', () => { + expect(chipBandwidthGbps(chips, 'Apple M4')).toBe(120); + expect(chipBandwidthGbps(chips, 'Apple M5 Pro')).toBe(307); + }); + + it('never guesses: a chip that no row names has no bandwidth', () => { + const unknown = [ + // A base chip that is not in the table. + 'Apple M6', + // A variant the table has never heard of must not borrow the figure of its base chip. + 'Apple M6 Pro', + 'Apple M4 Ultra', + 'Apple M5 Extreme', + // A name that merely contains a known name. + 'Apple M10', + 'Apple M12 Pro', + 'Apple M1 Max (Virtual)', + 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz', + 'VirtualApple @ 2.50GHz processor', + 'Max', + '', + ]; + for (const chip of unknown) expect(chipBandwidthGbps(chips, chip), chip).toBeUndefined(); + expect(chipBandwidthGbps(chips, undefined)).toBeUndefined(); + expect(chipBandwidthGbps([], 'Apple M4')).toBeUndefined(); + }); +}); + +describe('chipNameMatches', () => { + it('matches the whole chip name at the end of the string, ignoring case and padding', () => { + expect(chipNameMatches('Apple M4 Pro', 'M4 Pro')).toBe(true); + expect(chipNameMatches('Apple M4', 'M4')).toBe(true); + expect(chipNameMatches('M4', 'M4')).toBe(true); + expect(chipNameMatches(' apple m4 pro ', 'M4 PRO')).toBe(true); + }); + + it('does not match a part of a longer name', () => { + expect(chipNameMatches('Apple M4 Pro', 'M4')).toBe(false); + expect(chipNameMatches('Apple M14', 'M4')).toBe(false); + expect(chipNameMatches('Apple M10', 'M1')).toBe(false); + expect(chipNameMatches('Apple XM4', 'M4')).toBe(false); + expect(chipNameMatches('Apple M4', '')).toBe(false); + expect(chipNameMatches('', 'M4')).toBe(false); + }); +}); diff --git a/test/local-model-format.test.ts b/test/local-model-format.test.ts new file mode 100644 index 00000000..2f3bd3fb --- /dev/null +++ b/test/local-model-format.test.ts @@ -0,0 +1,86 @@ +/** + * @fileoverview Tests for src/local-model/format.ts: the ONE place a byte count becomes a size a + * person reads, and the memory class a Mac is sold as. + * + * What is pinned here is a contract with the browser: it prints the text it was sent and adds + * ' GB' itself, so the text must be decimal GB, carry no unit, and round the same way everywhere. + * + * Port: N/A (pure functions). + */ + +import { describe, expect, it } from 'vitest'; +import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js'; + +describe('byte units', () => { + it('keeps decimal GB (what a person reads) apart from the binary units of the budget math', () => { + expect(GB).toBe(1_000_000_000); + expect(GIB).toBe(1_073_741_824); + expect(MIB).toBe(1_048_576); + }); +}); + +describe('formatGb', () => { + it('prints decimal GB, so a Mac sold as 16 GB reads 17.2 and one sold as 48 GB reads 51.5', () => { + expect(formatGb(17_179_869_184)).toBe('17.2'); + expect(formatGb(51_539_607_552)).toBe('51.5'); + }); + + it('prints the registry sizes of the catalogue models as the registry shows them', () => { + expect(formatGb(6_550_825_373)).toBe('6.6'); + expect(formatGb(13_793_441_427)).toBe('13.8'); + expect(formatGb(17_741_871_939)).toBe('17.7'); + expect(formatGb(22_621_314_161)).toBe('22.6'); + expect(formatGb(81_370_035_877)).toBe('81.4'); + }); + + it('keeps one decimal below 100 and none from 100', () => { + expect(formatGb(1 * GB)).toBe('1.0'); + expect(formatGb(99.94 * GB)).toBe('99.9'); + // Rounds to 100.0, which is already "from 100". + expect(formatGb(99.96 * GB)).toBe('100'); + expect(formatGb(100 * GB)).toBe('100'); + // The free disk of the two test Macs, from their `df -k` lines. + expect(formatGb(890_926_084 * 1024)).toBe('912'); + expect(formatGb(41_752_848 * 1024)).toBe('42.8'); + }); + + it('never prints a unit, a thousands separator or an exponent', () => { + for (const bytes of [1, 5 * MIB, 3 * GIB, 250 * GB, 4000 * GB, 2.5e15]) { + expect(formatGb(bytes)).toMatch(/^\d+(\.\d)?$/); + } + }); + + it("reads '0.0' for anything that is not a positive size", () => { + for (const bytes of [0, -1, -5 * GB, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) { + expect(formatGb(bytes)).toBe('0.0'); + } + }); + + it('shows a size too small for one decimal as 0.0, never as an empty text', () => { + expect(formatGb(10 * MIB)).toBe('0.0'); + expect(formatGb(512 * MIB)).toBe('0.5'); + }); +}); + +describe('memClassGb', () => { + it('names the size a Mac is sold as: whole binary gigabytes', () => { + for (const sold of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192]) { + expect(memClassGb(sold * GIB)).toBe(sold); + } + // The exact `hw.memsize` of the two test Macs. + expect(memClassGb(17_179_869_184)).toBe(16); + expect(memClassGb(51_539_607_552)).toBe(48); + }); + + it('rounds a size that is not a whole number of gigabytes', () => { + // `hw.memsize_usable` of the 16 GB test Mac is a little below 16 GiB. + expect(memClassGb(16_631_808_000)).toBe(15); + expect(memClassGb(16 * GIB - 100 * MIB)).toBe(16); + }); + + it('reads 0 for anything that is not a positive size', () => { + for (const bytes of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(memClassGb(bytes)).toBe(0); + } + }); +}); diff --git a/test/local-model-hardware.test.ts b/test/local-model-hardware.test.ts new file mode 100644 index 00000000..0bcaa61a --- /dev/null +++ b/test/local-model-hardware.test.ts @@ -0,0 +1,749 @@ +/** + * @fileoverview Tests for src/local-model/hardware.ts: the probe that tells the recommendation + * what this Mac is. + * + * Two halves, mirroring the module: + * + * 1. `parseSysctl()`, pure. Fed the REAL output of the probe command on the two test Macs (a + * MacBook Pro with an M5 Pro and 48 GB, a Mac mini with an M4 and 16 GB), copied here + * character for character with the extra lines the capture holds, plus an Intel-shaped + * output (three of the seven keys do not exist there) and outputs cut off mid-line. + * 2. `readHardware()`, with every piece of IO injected: nothing is spawned and no real disk is + * read. What is pinned: a failed spawn is an ANSWER (the `os-fallback` shape), nothing is + * spawned off macOS, the result is cached for 60 s and concurrent callers share one probe. + * + * The IO itself is tested apart, against things this suite controls: + * + * 3. `spawnSysctl()` runs a small shell script of the test in place of sysctl. That proves what + * a fake cannot: the arguments are the seven keys and nothing else, the output of a program + * that exits 1 is still read (an Intel Mac lacks three keys, so sysctl exits 1 there), a + * program that hangs is given up on, and the server's environment is not handed over. + * 4. `dataDirFreeBytes()` with injected probes: a folder that does not exist yet, a mount that + * does not answer, and a `statfs` that never comes back, which must not be started twice. + * + * One test at the end runs the real defaults once (the real spawn and the real `statfs`), to + * prove they cannot reject on whatever machine runs the suite. On Linux `sysctl` knows none of + * the keys, so that run ends in the fallback; on a Mac it ends in a real answer. Both are fine. + * + * Port: N/A (no server). + */ + +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { isAbsolute, join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + dataDirFreeBytes, + FREE_DISK_TIMEOUT_MS, + freeDiskBytesOf, + HARDWARE_CACHE_TTL_MS, + parseSysctl, + readHardware, + resetHardwareCache, + spawnSysctl, + SYSCTL_KEYS, + SYSCTL_PATH, + SYSCTL_TIMEOUT_MS, + type HardwareProbeDeps, +} from '../src/local-model/hardware.js'; + +/** + * The capture of the MacBook Pro (Mac17,13), 2026-11-06. The last four lines are not part of + * the probe: two extra keys the capture asked for, the shell's exit status and a `df -k` line. + */ +const MACBOOK_SYSCTL = `hw.memsize: 51539607552 +machdep.cpu.brand_string: Apple M5 Pro +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac17,13 +hw.perflevel0.physicalcpu: 6 +hw.perflevel1.physicalcpu: 12 +exit=0 +/dev/disk3s5 971094180 54383564 890926084 6% 542146 8909260840 0% /System/Volumes/Data +`; + +/** The capture of the Mac mini (Mac16,10), 2026-11-06, with the same four extra lines. */ +const MACMINI_SYSCTL = `hw.memsize: 17179869184 +machdep.cpu.brand_string: Apple M4 +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac16,10 +hw.perflevel0.physicalcpu: 4 +hw.perflevel1.physicalcpu: 6 +exit=0 +/dev/disk3s5 239362496 172954124 41752848 81% 2353838 417528480 1% /System/Volumes/Data +`; + +/** Free disk of the two Macs: the "available" column of their `df -k` lines, in bytes. */ +const MACBOOK_FREE_DISK = 890_926_084 * 1024; +const MACMINI_FREE_DISK = 41_752_848 * 1024; + +/** + * What an Intel Mac prints: `hw.optional.arm64`, `sysctl.proc_translated` and + * `iogpu.wired_limit_mb` do not exist there, so sysctl leaves their lines out (and exits 1). + * NOT a capture: no Intel Mac was available, the shape is recalled. + */ +const INTEL_SYSCTL = `hw.memsize: 17179869184 +machdep.cpu.brand_string: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz +kern.osproductversion: 13.6.7 +hw.model: MacBookPro15,1 +`; + +/** Deps that spawn nothing and read no disk. A test overrides what it is about. */ +function deps(overrides: HardwareProbeDeps = {}): HardwareProbeDeps { + return { + platform: 'darwin', + arch: 'arm64', + runSysctl: async () => MACMINI_SYSCTL, + freeDiskBytes: async () => MACMINI_FREE_DISK, + totalMemBytes: () => 8 * 1024 ** 3, + cpuModel: () => 'Fallback CPU', + now: () => 1_000_000, + ...overrides, + }; +} + +beforeEach(() => { + resetHardwareCache(); +}); + +afterEach(() => { + resetHardwareCache(); +}); + +describe('the probe command', () => { + it('is an absolute path and asks for exactly the seven keys, never with -n', () => { + expect(isAbsolute(SYSCTL_PATH)).toBe(true); + expect(SYSCTL_PATH).toBe('/usr/sbin/sysctl'); + expect([...SYSCTL_KEYS]).toEqual([ + 'hw.memsize', + 'machdep.cpu.brand_string', + 'hw.optional.arm64', + 'sysctl.proc_translated', + 'iogpu.wired_limit_mb', + 'kern.osproductversion', + 'hw.model', + ]); + // Every argument is a key name. `-n` would print bare values, and a key missing on a + // machine would then shift every value after it onto the wrong key. + for (const key of SYSCTL_KEYS) expect(key).toMatch(/^[a-z][a-z0-9_.]*$/); + }); + + it('is bounded, and its result is reused for 60 seconds', () => { + expect(SYSCTL_TIMEOUT_MS).toBe(2000); + expect(HARDWARE_CACHE_TTL_MS).toBe(60_000); + }); +}); + +describe('parseSysctl on the two real Macs', () => { + it('reads the MacBook Pro: M5 Pro, 48 GB, macOS 27.2, no wired limit set', () => { + expect(parseSysctl(MACBOOK_SYSCTL)).toEqual({ + memBytes: 51_539_607_552, + chip: 'Apple M5 Pro', + arm64: true, + translated: false, + wiredLimitMb: 0, + macosVersion: '27.2', + macosMajor: 27, + model: 'Mac17,13', + }); + }); + + it('reads the Mac mini: M4, 16 GB, macOS 27.2, no wired limit set', () => { + expect(parseSysctl(MACMINI_SYSCTL)).toEqual({ + memBytes: 17_179_869_184, + chip: 'Apple M4', + arm64: true, + translated: false, + wiredLimitMb: 0, + macosVersion: '27.2', + macosMajor: 27, + model: 'Mac16,10', + }); + }); + + it('skips the lines it does not know: other keys, the exit status, the df line', () => { + const facts = parseSysctl(MACMINI_SYSCTL); + expect(Object.keys(facts).sort()).toEqual( + ['arm64', 'chip', 'macosMajor', 'macosVersion', 'memBytes', 'model', 'translated', 'wiredLimitMb'].sort() + ); + }); + + it('reports a wired limit of 0 as 0 (not set), and a set one as its number', () => { + expect(parseSysctl(MACMINI_SYSCTL).wiredLimitMb).toBe(0); + expect(parseSysctl('iogpu.wired_limit_mb: 14000\n').wiredLimitMb).toBe(14000); + }); +}); + +describe('parseSysctl on other shapes', () => { + it('reads an Intel Mac, whose output lacks three of the keys, without shifting a value', () => { + expect(parseSysctl(INTEL_SYSCTL)).toEqual({ + memBytes: 17_179_869_184, + chip: 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz', + macosVersion: '13.6.7', + macosMajor: 13, + model: 'MacBookPro15,1', + }); + }); + + it('reads an Intel Mac that does print the arm64 flag, as 0', () => { + const facts = parseSysctl(`hw.optional.arm64: 0\n${INTEL_SYSCTL}`); + expect(facts.arm64).toBe(false); + expect(facts.translated).toBeUndefined(); + }); + + it('reads a process under Rosetta: the machine is arm64 and the process is translated', () => { + const facts = parseSysctl(MACMINI_SYSCTL.replace('sysctl.proc_translated: 0', 'sysctl.proc_translated: 1')); + expect(facts.arm64).toBe(true); + expect(facts.translated).toBe(true); + }); + + it('drops a line that was cut off: a truncated output never yields half a value', () => { + // Cut in the middle of the chip name. The memory line before it is whole. + const cutInChip = 'hw.memsize: 17179869184\nmachdep.cpu.brand_string: Apple M'; + expect(parseSysctl(cutInChip)).toEqual({ memBytes: 17_179_869_184 }); + + // Cut in the middle of the memory figure: '1717' would be a valid, wrong number. + expect(parseSysctl('hw.memsize: 1717')).toEqual({}); + + // Cut right after a whole line: everything up to there counts. + const cutAfterLine = MACMINI_SYSCTL.slice(0, MACMINI_SYSCTL.indexOf('hw.optional.arm64')); + expect(parseSysctl(cutAfterLine)).toEqual({ memBytes: 17_179_869_184, chip: 'Apple M4' }); + }); + + it('reads nothing from an empty output or from error text', () => { + expect(parseSysctl('')).toEqual({}); + expect(parseSysctl('\n\n')).toEqual({}); + expect( + parseSysctl("sysctl: unknown oid 'hw.memsize'\nsysctl: cannot stat /proc/sys/hw/memsize: No such file\n") + ).toEqual({}); + }); + + it('leaves out a value that does not parse instead of guessing', () => { + const facts = parseSysctl( + [ + 'hw.memsize: lots', + 'machdep.cpu.brand_string: ', + 'hw.optional.arm64: yes', + 'sysctl.proc_translated: 2', + 'iogpu.wired_limit_mb: -1', + 'kern.osproductversion: Sequoia', + `hw.model: ${'x'.repeat(200)}`, + '', + ].join('\n') + ); + expect(facts).toEqual({}); + }); + + it('refuses a memory size of 0 or one too large to be a number of bytes', () => { + expect(parseSysctl('hw.memsize: 0\n').memBytes).toBeUndefined(); + expect(parseSysctl('hw.memsize: 99999999999999999999999\n').memBytes).toBeUndefined(); + }); + + it('accepts Windows line ends and padding, and the first line of a key wins', () => { + const facts = parseSysctl('hw.memsize: 17179869184 \r\nhw.model: Mac16,10\r\nhw.model: Other9,9\r\n'); + expect(facts).toEqual({ memBytes: 17_179_869_184, model: 'Mac16,10' }); + }); + + it('reads the macOS major version from one, two or three numbers', () => { + expect(parseSysctl('kern.osproductversion: 14\n')).toMatchObject({ macosVersion: '14', macosMajor: 14 }); + expect(parseSysctl('kern.osproductversion: 15.7\n')).toMatchObject({ macosVersion: '15.7', macosMajor: 15 }); + expect(parseSysctl('kern.osproductversion: 13.6.7\n')).toMatchObject({ macosVersion: '13.6.7', macosMajor: 13 }); + }); +}); + +describe('readHardware on the two real Macs', () => { + it('describes the MacBook Pro', async () => { + const hw = await readHardware( + deps({ runSysctl: async () => MACBOOK_SYSCTL, freeDiskBytes: async () => MACBOOK_FREE_DISK }) + ); + expect(hw).toEqual({ + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 51_539_607_552, + memGb: '51.5', + memClassGb: 48, + chip: 'Apple M5 Pro', + model: 'Mac17,13', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: MACBOOK_FREE_DISK, + freeDiskGb: '912', + source: 'sysctl', + }); + }); + + it('describes the Mac mini', async () => { + const hw = await readHardware(deps()); + expect(hw).toEqual({ + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 17_179_869_184, + memGb: '17.2', + memClassGb: 16, + chip: 'Apple M4', + model: 'Mac16,10', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: MACMINI_FREE_DISK, + freeDiskGb: '42.8', + source: 'sysctl', + }); + }); + + it('is plain JSON: it survives a round trip unchanged', async () => { + const hw = await readHardware(deps()); + expect(JSON.parse(JSON.stringify(hw))).toEqual(hw); + }); + + it('describes an Intel Mac from its shorter output: no arm64 flag, an x64 process', async () => { + const hw = await readHardware(deps({ arch: 'x64', runSysctl: async () => INTEL_SYSCTL })); + expect(hw.source).toBe('sysctl'); + expect(hw.arch).toBe('x64'); + expect(hw.chip).toBe('Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'); + expect(hw.macosMajor).toBe(13); + expect(hw).not.toHaveProperty('arm64'); + expect(hw).not.toHaveProperty('translated'); + expect(hw).not.toHaveProperty('wiredLimitMb'); + }); + + it('takes the chip name from node:os when sysctl printed everything but that', async () => { + const hw = await readHardware( + deps({ runSysctl: async () => MACMINI_SYSCTL.replace('machdep.cpu.brand_string: Apple M4\n', '') }) + ); + expect(hw.source).toBe('sysctl'); + expect(hw.chip).toBe('Fallback CPU'); + }); +}); + +describe('readHardware when the probe fails', () => { + it('answers from node:os when the spawn could not run or timed out', async () => { + const hw = await readHardware( + deps({ + runSysctl: async () => undefined, + totalMemBytes: () => 34_359_738_368, + cpuModel: () => 'Apple M2 Max', + }) + ); + expect(hw).toEqual({ + platform: 'darwin', + arch: 'arm64', + memBytes: 34_359_738_368, + memGb: '34.4', + memClassGb: 32, + chip: 'Apple M2 Max', + freeDiskBytes: MACMINI_FREE_DISK, + freeDiskGb: '42.8', + source: 'os-fallback', + }); + }); + + it('never rejects when a dependency throws or rejects', async () => { + const hw = await readHardware( + deps({ + runSysctl: () => Promise.reject(new Error('spawn EACCES')), + freeDiskBytes: () => { + throw new Error('statfs failed'); + }, + cpuModel: () => { + throw new Error('no cpus'); + }, + }) + ); + expect(hw.source).toBe('os-fallback'); + expect(hw.memBytes).toBe(8 * 1024 ** 3); + expect(hw).not.toHaveProperty('chip'); + expect(hw).not.toHaveProperty('freeDiskBytes'); + expect(hw).not.toHaveProperty('freeDiskGb'); + }); + + it('still answers when even the memory size cannot be read', async () => { + const hw = await readHardware( + deps({ + runSysctl: async () => undefined, + totalMemBytes: () => { + throw new Error('no memory figure'); + }, + }) + ); + expect(hw).toMatchObject({ source: 'os-fallback', memBytes: 0, memGb: '0.0', memClassGb: 0 }); + }); + + it('trusts nothing of an output that has no memory size', async () => { + // A truncated or foreign output: the chip and version lines are whole, hw.memsize is not there. + const withoutMemory = MACMINI_SYSCTL.replace('hw.memsize: 17179869184\n', ''); + const hw = await readHardware(deps({ runSysctl: async () => withoutMemory })); + expect(hw.source).toBe('os-fallback'); + expect(hw.memBytes).toBe(8 * 1024 ** 3); + expect(hw.chip).toBe('Fallback CPU'); + expect(hw).not.toHaveProperty('macosVersion'); + expect(hw).not.toHaveProperty('macosMajor'); + expect(hw).not.toHaveProperty('model'); + expect(hw).not.toHaveProperty('wiredLimitMb'); + expect(hw).not.toHaveProperty('arm64'); + }); + + it('falls back when the output was cut off inside the memory figure', async () => { + const hw = await readHardware(deps({ runSysctl: async () => 'hw.memsize: 1717' })); + expect(hw.source).toBe('os-fallback'); + expect(hw.memBytes).toBe(8 * 1024 ** 3); + }); + + it('leaves the free disk out when it is unknown, and keeps the rest', async () => { + const hw = await readHardware(deps({ freeDiskBytes: async () => undefined })); + expect(hw.source).toBe('sysctl'); + expect(hw).not.toHaveProperty('freeDiskBytes'); + expect(hw).not.toHaveProperty('freeDiskGb'); + }); + + it('reports a full disk as 0 bytes free, which is a figure and not "unknown"', async () => { + const hw = await readHardware(deps({ freeDiskBytes: async () => 0 })); + expect(hw.freeDiskBytes).toBe(0); + expect(hw.freeDiskGb).toBe('0.0'); + }); +}); + +describe('readHardware off macOS', () => { + it('spawns nothing and reads no disk', async () => { + let spawned = 0; + let diskRead = 0; + const hw = await readHardware( + deps({ + platform: 'linux', + arch: 'x64', + runSysctl: async () => { + spawned++; + return MACMINI_SYSCTL; + }, + freeDiskBytes: async () => { + diskRead++; + return 1; + }, + totalMemBytes: () => 67_430_023_168, + }) + ); + expect(spawned).toBe(0); + expect(diskRead).toBe(0); + expect(hw).toEqual({ + platform: 'linux', + arch: 'x64', + memBytes: 67_430_023_168, + memGb: '67.4', + memClassGb: 63, + source: 'not-probed', + }); + }); +}); + +describe('the cache of readHardware', () => { + it('reuses one result for 60 seconds, then probes again', async () => { + let clock = 5_000_000; + let spawned = 0; + const counting = deps({ + now: () => clock, + runSysctl: async () => { + spawned++; + return spawned === 1 ? MACMINI_SYSCTL : MACBOOK_SYSCTL; + }, + }); + + const first = await readHardware(counting); + clock += HARDWARE_CACHE_TTL_MS - 1; + const second = await readHardware(counting); + expect(spawned).toBe(1); + expect(second).toBe(first); + + clock += 1; + const third = await readHardware(counting); + expect(spawned).toBe(2); + expect(third.chip).toBe('Apple M5 Pro'); + }); + + it('caches a failed probe too: a broken sysctl is not spawned on every request', async () => { + let spawned = 0; + const failing = deps({ + runSysctl: async () => { + spawned++; + return undefined; + }, + }); + expect((await readHardware(failing)).source).toBe('os-fallback'); + expect((await readHardware(failing)).source).toBe('os-fallback'); + expect(spawned).toBe(1); + }); + + it('lets concurrent callers share one probe', async () => { + let spawned = 0; + let release: (text: string) => void = () => {}; + const slow = deps({ + runSysctl: () => { + spawned++; + return new Promise((resolve) => { + release = resolve; + }); + }, + }); + const callers = [readHardware(slow), readHardware(slow), readHardware(slow)]; + // Let the three calls reach the spawn before it answers. + await new Promise((resolve) => setImmediate(resolve)); + release(MACMINI_SYSCTL); + const results = await Promise.all(callers); + expect(spawned).toBe(1); + expect(results[1]).toBe(results[0]); + expect(results[2]).toBe(results[0]); + }); + + it('probes again after a reset, and does not trust a clock that ran backwards', async () => { + let clock = 9_000_000; + let spawned = 0; + const counting = deps({ + now: () => clock, + runSysctl: async () => { + spawned++; + return MACMINI_SYSCTL; + }, + }); + await readHardware(counting); + resetHardwareCache(); + await readHardware(counting); + expect(spawned).toBe(2); + + clock -= 10_000; + await readHardware(counting); + expect(spawned).toBe(3); + }); +}); + +// The scripts below are POSIX shell. Codeman's server does not run on Windows. +describe.skipIf(process.platform === 'win32')('spawnSysctl, against a script of the test', () => { + let folder: string; + + beforeEach(() => { + folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-spawn-')); + }); + + afterEach(() => { + rmSync(folder, { recursive: true, force: true }); + }); + + /** An executable shell script standing in for sysctl. */ + function script(body: string, mode = 0o755): string { + const path = join(folder, 'fake-sysctl'); + writeFileSync(path, `#!/bin/sh\n${body}\n`, { mode }); + return path; + } + + it('passes the seven keys as arguments and nothing else: no -n, no shell', async () => { + // printf, not echo: echo would swallow a `-n` argument, the very thing this test looks for. + const output = await spawnSysctl(script('printf \'%s\\n\' "$@"')); + expect(output?.trimEnd().split('\n')).toEqual([...SYSCTL_KEYS]); + }); + + it('reads the output of a program that exits 1, as sysctl does on an Intel Mac', async () => { + // Three keys are unknown there: their errors go to stderr, the other four lines to stdout. + const output = await spawnSysctl( + script( + [ + "echo 'hw.memsize: 17179869184'", + "echo 'machdep.cpu.brand_string: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'", + `echo "sysctl: unknown oid 'hw.optional.arm64'" >&2`, + `echo "sysctl: unknown oid 'sysctl.proc_translated'" >&2`, + `echo "sysctl: unknown oid 'iogpu.wired_limit_mb'" >&2`, + "echo 'kern.osproductversion: 13.6.7'", + "echo 'hw.model: MacBookPro15,1'", + 'exit 1', + ].join('\n') + ) + ); + expect(output).toBeDefined(); + expect(output).not.toContain('unknown oid'); + expect(parseSysctl(output as string)).toEqual({ + memBytes: 17_179_869_184, + chip: 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz', + macosVersion: '13.6.7', + macosMajor: 13, + model: 'MacBookPro15,1', + }); + }); + + it('answers undefined for a program that does not exist or may not be run', async () => { + expect(await spawnSysctl(join(folder, 'no-such-program'))).toBeUndefined(); + expect(await spawnSysctl(script('echo "hw.memsize: 1"', 0o644))).toBeUndefined(); + }); + + it('gives up on a program that hangs, and does not hand over half an output', async () => { + const started = Date.now(); + // `exec` so the signal reaches the sleeper itself and nothing is left behind. + const output = await spawnSysctl(script("echo 'hw.memsize: 17179869184'\nexec sleep 20"), 200); + expect(output).toBeUndefined(); + expect(Date.now() - started).toBeLessThan(5000); + }); + + it('answers undefined for an output far larger than seven lines', async () => { + expect(await spawnSysctl(script("head -c 200000 /dev/zero | tr '\\0' 'x'"))).toBeUndefined(); + }); + + it("does not pass the server's environment to the program", async () => { + const name = 'CODEMAN_LOCAL_MODEL_TEST_SECRET'; + process.env[name] = 'hunter2'; + try { + const output = await spawnSysctl(script(`echo "secret=\${${name}:-unset}"`)); + expect(output?.trim()).toBe('secret=unset'); + } finally { + delete process.env[name]; + } + }); +}); + +describe('dataDirFreeBytes', () => { + let folder: string; + + beforeEach(() => { + folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-disk-')); + }); + + afterEach(() => { + rmSync(folder, { recursive: true, force: true }); + }); + + it('reads the free bytes of the data dir with the real probe and the real statfs', async () => { + const free = await dataDirFreeBytes({ dataDir: () => folder }); + expect(typeof free).toBe('number'); + expect(free).toBeGreaterThanOrEqual(0); + }); + + it('uses the nearest existing parent when the data dir does not exist yet', async () => { + const asked: string[] = []; + const free = await dataDirFreeBytes({ + dataDir: () => join(folder, 'not', 'yet', 'there'), + freeBytesOf: async (path) => { + asked.push(path); + return 123; + }, + }); + expect(free).toBe(123); + expect(asked).toEqual([folder]); + }); + + it('does not touch a folder whose mount does not answer, or a path that is a file', async () => { + for (const kind of ['unknown', 'file'] as const) { + let touched = 0; + const free = await dataDirFreeBytes({ + dataDir: () => folder, + probeKind: async () => kind, + freeBytesOf: async () => { + touched++; + return 1; + }, + }); + expect(free, kind).toBeUndefined(); + expect(touched, kind).toBe(0); + } + }); + + it('stops walking up after a few parents', async () => { + let probes = 0; + const free = await dataDirFreeBytes({ + dataDir: () => '/a/b/c/d/e/f/g/h/i/j/k/l', + probeKind: async () => { + probes++; + return 'absent'; + }, + freeBytesOf: async () => 1, + }); + expect(free).toBeUndefined(); + expect(probes).toBe(7); + }); + + it('gives up on a statfs that does not come back, and never starts a second one meanwhile', async () => { + let started = 0; + let finish: (bytes: number) => void = () => {}; + const hanging = { + dataDir: () => folder, + freeBytesOf: () => { + started++; + return new Promise((resolve) => { + finish = resolve; + }); + }, + timeoutMs: 20, + }; + + // The first read waits its timeout and answers "unknown". + expect(await dataDirFreeBytes(hanging)).toBeUndefined(); + expect(started).toBe(1); + + // While that statfs is still out, later reads answer at once and start nothing. + const before = Date.now(); + expect(await dataDirFreeBytes(hanging)).toBeUndefined(); + expect(await dataDirFreeBytes({ ...hanging, timeoutMs: 60_000 })).toBeUndefined(); + expect(Date.now() - before).toBeLessThan(1000); + expect(started).toBe(1); + + // Once it finally answers, the next read works again. + finish(5); + await new Promise((resolve) => setImmediate(resolve)); + expect(await dataDirFreeBytes({ ...hanging, freeBytesOf: async () => 42 })).toBe(42); + }); + + it('never rejects when one of its reads does', async () => { + const failing = async (): Promise => { + throw new Error('EIO'); + }; + expect(await dataDirFreeBytes({ dataDir: () => folder, freeBytesOf: failing })).toBeUndefined(); + expect(await dataDirFreeBytes({ dataDir: () => folder, probeKind: failing })).toBeUndefined(); + expect( + await dataDirFreeBytes({ + dataDir: () => { + throw new Error('no home'); + }, + }) + ).toBeUndefined(); + }); + + it('is bounded by default', () => { + expect(FREE_DISK_TIMEOUT_MS).toBe(2000); + }); +}); + +describe('the real defaults', () => { + let folder: string; + + beforeEach(() => { + folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-hw-')); + }); + + afterEach(() => { + rmSync(folder, { recursive: true, force: true }); + }); + + it('reads the free bytes of a real folder, and undefined for one that does not exist', async () => { + const free = await freeDiskBytesOf(folder); + expect(typeof free).toBe('number'); + expect(free).toBeGreaterThanOrEqual(0); + expect(await freeDiskBytesOf(join(folder, 'missing', 'deeper'))).toBeUndefined(); + }); + + it('never rejects with the real spawn and the real disk read, whatever machine this is', async () => { + // Pretend to be a Mac so the spawn and the disk read really run. On Linux /usr/sbin/sysctl + // either does not exist or knows none of the keys: the answer is then the fallback. + const hw = await readHardware({ platform: 'darwin' }); + expect(['sysctl', 'os-fallback']).toContain(hw.source); + expect(hw.memBytes).toBeGreaterThan(0); + expect(hw.memGb).toMatch(/^\d+(\.\d)?$/); + if (hw.freeDiskBytes !== undefined) { + expect(hw.freeDiskBytes).toBeGreaterThanOrEqual(0); + expect(hw.freeDiskGb).toMatch(/^\d+(\.\d)?$/); + } + }); +}); diff --git a/test/local-model-recommend.test.ts b/test/local-model-recommend.test.ts new file mode 100644 index 00000000..86a2c327 --- /dev/null +++ b/test/local-model-recommend.test.ts @@ -0,0 +1,1298 @@ +/** + * @fileoverview Tests for src/local-model/recommend.ts: the pure function that turns "what this + * Mac is" into "which local model fits it". + * + * Laid out like the module: + * + * 1. The budget (`deviceBudget`) for every memory size Apple sells from 8 to 128 GB, and the + * need of a model at a context (`needBytes`). + * 2. The two REAL Macs of the project, from their captured `sysctl` output through the real + * parser: the MacBook Pro (M5 Pro, 48 GB) and the Mac mini (M4, 16 GB). Each test states the + * expected answer and why. + * 3. Every reason for verdict `none`, one fixture per reason, and their order. + * 4. The `limited` verdict, the three-state speed in the ranking, harness rows (a gated one is + * never chosen), the `tested` flag that alone may turn on the word "Recommended", and the + * shape of the answer (labels from the fixed list, sizes in one unit). + * + * The stock catalogue is used wherever the answer for a real Mac is at stake. Small synthetic + * catalogues (validated by the real schema) isolate one rule of the ranking at a time. + * + * Port: N/A (pure functions; the two hardware fixtures go through `readHardware` with every + * piece of IO injected). + */ + +import { beforeEach, describe, expect, it } from 'vitest'; +import { + getLocalModelCatalog, + LocalModelCatalogSchema, + STOCK_LOCAL_MODEL_CATALOG, +} from '../src/config/local-model-catalog.js'; +import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js'; +import { readHardware, resetHardwareCache } from '../src/local-model/hardware.js'; +import { + deviceBudget, + DISK_HEADROOM_BYTES, + FIT_GOOD_MAX_SHARE, + FIT_TIGHT_MAX_SHARE, + LIMITED_BELOW_PARAMS_B, + MAX_ALTERNATIVES, + needBytes, + recommend, + SLOW_BELOW_TOKENS_PER_SECOND, +} from '../src/local-model/recommend.js'; +import { + LOCAL_MODEL_LABELS, + type Candidate, + type HardwareFacts, + type HarnessFact, + type LocalHarnessEntry, + type LocalModelCatalog, + type LocalModelEntry, + type Recommendation, +} from '../src/types.js'; + +// ---------- Fixtures ---------- + +/** The real probe output of the MacBook Pro (Mac17,13), captured 2026-11-06, extra lines included. */ +const MACBOOK_SYSCTL = `hw.memsize: 51539607552 +machdep.cpu.brand_string: Apple M5 Pro +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac17,13 +hw.perflevel0.physicalcpu: 6 +hw.perflevel1.physicalcpu: 12 +exit=0 +/dev/disk3s5 971094180 54383564 890926084 6% 542146 8909260840 0% /System/Volumes/Data +`; + +/** The real probe output of the Mac mini (Mac16,10), captured 2026-11-06, extra lines included. */ +const MACMINI_SYSCTL = `hw.memsize: 17179869184 +machdep.cpu.brand_string: Apple M4 +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac16,10 +hw.perflevel0.physicalcpu: 4 +hw.perflevel1.physicalcpu: 6 +exit=0 +/dev/disk3s5 239362496 172954124 41752848 81% 2353838 417528480 1% /System/Volumes/Data +`; + +/** The hardware facts of a real Mac: its captured output through the real parser, no IO. */ +async function realMac(sysctlOutput: string, freeDiskKb: number): Promise { + resetHardwareCache(); + return readHardware({ + platform: 'darwin', + arch: 'arm64', + runSysctl: async () => sysctlOutput, + freeDiskBytes: async () => freeDiskKb * 1024, + totalMemBytes: () => 0, + cpuModel: () => undefined, + now: () => 0, + }); +} + +/** An Apple silicon Mac with `soldGb` of memory, healthy in every other respect. */ +function mac(soldGb: number, chip: string, overrides: Partial = {}): HardwareFacts { + const memBytes = soldGb * GIB; + return { + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes, + memGb: formatGb(memBytes), + memClassGb: memClassGb(memBytes), + chip, + model: 'Mac0,0', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: 500 * GB, + freeDiskGb: '500', + source: 'sysctl', + ...overrides, + }; +} + +/** A copy of `facts` without the given fields (an absent field, not an `undefined` one). */ +function without(facts: HardwareFacts, ...fields: (keyof HardwareFacts)[]): HardwareFacts { + const copy: Record = { ...facts }; + for (const field of fields) delete copy[field]; + return copy as unknown as HardwareFacts; +} + +/** Display labels of the stock harnesses, as the CLI registry would supply them. Data, not a branch. */ +const HARNESS_LABELS: Record = { opencode: 'OpenCode', claude: 'Claude Code', pi: 'Pi' }; + +/** What the server knows about one harness: installed and accepting every model id, unless overridden. */ +function fact(cli: string, overrides: Partial = {}): HarnessFact { + return { + cli, + label: HARNESS_LABELS[cli] ?? cli, + installed: true, + installable: false, + acceptsModelId: () => true, + ...overrides, + }; +} + +const STOCK = getLocalModelCatalog(); + +/** Every stock harness installed, the gated one included. */ +const ALL_INSTALLED: HarnessFact[] = STOCK.harnesses.map((row) => fact(row.cli)); + +/** The stock catalogue with a change applied, validated by the real schema. */ +function stockWith(change: (catalog: LocalModelCatalog) => void): LocalModelCatalog { + const catalog = structuredClone(STOCK_LOCAL_MODEL_CATALOG); + change(catalog); + return LocalModelCatalogSchema.parse(catalog); +} + +/** A model row for a synthetic catalogue: 20 billion parameters, 10 GiB, one small context. */ +function model(id: string, overrides: Partial = {}): LocalModelEntry { + return { + id, + label: id, + license: 'Apache-2.0', + paramsB: 20, + bytes: 10 * GIB, + kvBytesPerToken: 1, + contexts: [1024], + rank: 1, + tested: [], + ...overrides, + }; +} + +/** + * A synthetic catalogue: no prompt cache, one harness `alpha` that works from 1024 tokens, and + * one chip row ('M4 Max' at 400 GB/s, so tokens per second = `speedPer100Gbps` x 4). + */ +function catalogOf(models: LocalModelEntry[], harnesses?: LocalHarnessEntry[]): LocalModelCatalog { + return LocalModelCatalogSchema.parse({ + schema: 1, + updated: '2026-11-06', + minMacosMajor: 14, + promptCacheTiers: [{ minMemGb: 0, bytes: 0 }], + models, + harnesses: harnesses ?? [{ cli: 'alpha', minContext: 1024, rank: 1 }], + chips: [{ match: 'M4 Max', gbps: 400 }], + }); +} + +/** A Mac whose budget is exactly 100 GiB: a wired limit of 101 GiB on a 128 GB machine. */ +const HUNDRED_GIB_MAC = mac(128, 'Apple M4 Max', { wiredLimitMb: 101 * 1024 }); +const HUNDRED_GIB = 100 * GIB; + +/** What a synthetic model needs beside its weights at its 1024-token context: 1 GiB plus the KV cache. */ +const SYNTHETIC_OVERHEAD = GIB + 1024; + +/** A model whose need is exactly `need` bytes in a synthetic catalogue. */ +function modelNeeding(id: string, need: number, overrides: Partial = {}): LocalModelEntry { + return model(id, { bytes: need - SYNTHETIC_OVERHEAD, ...overrides }); +} + +/** `id@context` of the pick and of each alternative, in order. */ +function order(result: Recommendation): string[] { + return [result.pick, ...result.alternatives] + .filter((candidate): candidate is Candidate => candidate !== undefined) + .map((candidate) => `${candidate.modelId}@${candidate.context}`); +} + +/** Every string anywhere in a value. */ +function stringsIn(value: unknown): string[] { + if (typeof value === 'string') return [value]; + if (Array.isArray(value)) return value.flatMap(stringsIn); + if (value && typeof value === 'object') return Object.values(value).flatMap(stringsIn); + return []; +} + +function deepFreeze(value: T): T { + if (value && typeof value === 'object') { + for (const inner of Object.values(value)) deepFreeze(inner); + Object.freeze(value); + } + return value; +} + +/** The Macs the sweeps run over: every memory size, with a slow, a fast and an unknown chip. */ +const SWEEP: HardwareFacts[] = [8, 16, 18, 24, 32, 36, 48, 64, 96, 128].flatMap((soldGb) => + ['Apple M4', 'Apple M4 Max', 'Apple M5 Ultra', 'Apple M6'].map((chip) => mac(soldGb, chip)) +); + +beforeEach(() => { + resetHardwareCache(); +}); + +// ---------- 1. Budget and need ---------- + +describe('deviceBudget', () => { + // Sold size, device limit L and budget B in GiB. L is 2/3 of memory below 36 GB and 3/4 from + // 36 GB; B = min(L - 1, memory - reserve). Only at 8 GB is the reserve the smaller bound. + const TIERS: [soldGb: number, limitGib: number, budgetGib: number][] = [ + [8, 16 / 3, 2], + [16, 32 / 3, 29 / 3], + [18, 12, 11], + [24, 16, 15], + [32, 64 / 3, 61 / 3], + [36, 27, 26], + [48, 36, 35], + [64, 48, 47], + [96, 72, 71], + [128, 96, 95], + ]; + + it.each(TIERS)('gives a %i GB Mac a limit of %f GiB and a budget of %f GiB', (soldGb, limitGib, budgetGib) => { + const budget = deviceBudget(mac(soldGb, 'Apple M4')); + // Whole bytes: a third of a byte is rounded down. + expect(Math.abs(budget.limitBytes - limitGib * GIB)).toBeLessThan(1); + expect(Math.abs(budget.budgetBytes - budgetGib * GIB)).toBeLessThan(1); + expect(Number.isInteger(budget.limitBytes)).toBe(true); + expect(Number.isInteger(budget.budgetBytes)).toBe(true); + expect(budget.source).toBe('estimated'); + }); + + it('matches the rounded budgets of the design table', () => { + const rounded = TIERS.map(([soldGb]) => [ + soldGb, + (deviceBudget(mac(soldGb, 'Apple M4')).budgetBytes / GIB).toFixed(1), + ]); + expect(rounded).toEqual([ + [8, '2.0'], + [16, '9.7'], + [18, '11.0'], + [24, '15.0'], + [32, '20.3'], + [36, '26.0'], + [48, '35.0'], + [64, '47.0'], + [96, '71.0'], + [128, '95.0'], + ]); + }); + + it('keeps a reserve of 6 GiB up to 18 GB, 8 GiB up to 48 GB and 12 GiB from 64 GB', () => { + // A wired limit far above the memory leaves "memory - reserve" as the only bound. + const reserveOf = (soldGb: number): number => { + const budget = deviceBudget(mac(soldGb, 'Apple M4', { wiredLimitMb: 1024 * 1024 })); + return (soldGb * GIB - budget.budgetBytes) / GIB; + }; + expect([8, 16, 18].map(reserveOf)).toEqual([6, 6, 6]); + expect([24, 32, 36, 48].map(reserveOf)).toEqual([8, 8, 8, 8]); + expect([64, 96, 128].map(reserveOf)).toEqual([12, 12, 12]); + }); + + it('uses the wired limit when it is above 0', () => { + // 14000 MiB on the 16 GB Mac: L - 1 GiB is 12.67 GiB, memory - reserve is 10 GiB. + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 14000 }))).toEqual({ + limitBytes: 14000 * MIB, + budgetBytes: 10 * GIB, + source: 'wired-limit', + }); + // A limit BELOW the estimate is honoured too: 8192 MiB leaves 7 GiB. + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 8192 }))).toEqual({ + limitBytes: 8 * GIB, + budgetBytes: 7 * GIB, + source: 'wired-limit', + }); + }); + + it('reads a wired limit of 0, or none at all, as "not set"', () => { + const estimated = deviceBudget(mac(16, 'Apple M4')); + expect(estimated.source).toBe('estimated'); + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 0 }))).toEqual(estimated); + expect(deviceBudget(without(mac(16, 'Apple M4'), 'wiredLimitMb'))).toEqual(estimated); + }); + + it('never goes below 0', () => { + expect(deviceBudget(mac(4, 'Apple M1')).budgetBytes).toBe(0); + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 512 })).budgetBytes).toBe(0); + expect(deviceBudget({ ...mac(16, 'Apple M4'), memBytes: 0, memClassGb: 0 })).toEqual({ + limitBytes: 0, + budgetBytes: 0, + source: 'estimated', + }); + }); +}); + +describe('needBytes', () => { + const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; + + it('adds weights, the KV cache of the context, 1 GiB and the prompt cache', () => { + // 6,550,825,373 + 32768 x 65536 + 1 GiB + 512 MiB on a Mac below 32 GB. + expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + 2 * GIB + GIB + 512 * MIB); + expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(10_308_921_757); + // Half the context, half the KV cache. + expect(needBytes(nineB, 32768, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + GIB + GIB + 512 * MIB); + }); + + it('counts a prompt cache of 512 MiB below 32 GB of memory and 2 GiB from 32 GB', () => { + const base = 6_550_825_373 + 2 * GIB + GIB; + for (const soldGb of [8, 16, 18, 24]) { + expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(512 * MIB); + } + for (const soldGb of [32, 36, 48, 64, 96, 128]) { + expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(2 * GIB); + } + }); + + it('does not depend on the chip or the harness', () => { + expect(needBytes(nineB, 65536, STOCK, mac(48, 'Apple M1'))).toBe( + needBytes(nineB, 65536, STOCK, mac(48, 'Apple M5 Ultra')) + ); + }); +}); + +describe('the fit grade', () => { + const pickOf = (need: number): Candidate | undefined => + recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', need)]), [fact('alpha')]).pick; + + it('has the budget this group assumes', () => { + expect(deviceBudget(HUNDRED_GIB_MAC).budgetBytes).toBe(HUNDRED_GIB); + expect(FIT_GOOD_MAX_SHARE).toBe(0.85); + expect(FIT_TIGHT_MAX_SHARE).toBe(0.98); + }); + + it('is good up to 0.85 of the budget', () => { + expect(pickOf(0.5 * HUNDRED_GIB)).toMatchObject({ fit: 'good', fitLabel: 'Good', budgetShare: 0.5 }); + expect(pickOf(0.85 * HUNDRED_GIB)).toMatchObject({ fit: 'good', budgetShare: 0.85 }); + }); + + it('is tight above 0.85 and up to 0.98', () => { + expect(pickOf(0.85 * HUNDRED_GIB + 1)).toMatchObject({ fit: 'tight', fitLabel: 'Tight' }); + expect(pickOf(0.9 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.9 }); + expect(pickOf(0.98 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.98 }); + }); + + it('is out above 0.98: the model is not offered at all', () => { + const result = recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', 0.98 * HUNDRED_GIB + 1)]), [ + fact('alpha'), + ]); + expect(result.verdict).toBe('none'); + expect(result.reason).toBe('memory'); + expect(result.pick).toBeUndefined(); + }); +}); + +// ---------- 2. The two real Macs ---------- + +describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { + /** + * Expected: Qwen3.6 35B-A3B at 131,072 tokens with OpenCode, Good, about 42 tokens per second. + * + * Why. The budget is 35 GiB (3/4 of 48 GiB minus 1). Five models fit it as Good. The M5 Pro is + * in the chips table (307 GB/s), so each model with a speed figure gets an estimate: + * the dense 27B computes to 2.95 x 3.07 = 9.1 tokens per second, under the 10 that separate + * slow from ok, while the 35B-A3B (3 billion parameters active per token) computes to 41.8. + * Speed ranks before model rank, so the 35B-A3B wins although the 27B has the higher rank. The + * 27B stays in the list, last, marked Slow. Both harnesses can run it; OpenCode has the + * higher rank. + */ + it('gets Qwen3.6 35B-A3B at 131072 tokens with OpenCode', async () => { + const hw = await realMac(MACBOOK_SYSCTL, 890_926_084); + expect(hw.chip).toBe('Apple M5 Pro'); + expect(hw.memClassGb).toBe(48); + + const result = recommend(hw, STOCK, ALL_INSTALLED); + + expect(result.verdict).toBe('ok'); + expect(result.reason).toBeUndefined(); + expect(result.budgetBytes).toBe(35 * GIB); + expect(result.budgetGb).toBe('37.6'); + expect(result.budgetSource).toBe('estimated'); + expect(result.pick).toEqual({ + modelId: 'qwen3.6:35b-a3b', + modelLabel: 'Qwen3.6 35B-A3B', + license: 'Apache-2.0', + paramsB: 35, + context: 131072, + cli: 'opencode', + cliLabel: 'OpenCode', + cliInstalled: true, + // 22,621,314,161 + 20480 x 131072 + 1 GiB + 2 GiB. + needBytes: 28_526_894_193, + needGb: '28.5', + downloadBytes: 22_621_314_161, + downloadGb: '22.6', + budgetShare: 0.759, + fit: 'good', + fitLabel: 'Good', + speed: 'ok', + tokensPerSecond: 41.8, + tested: false, + }); + }); + + it('says "Best fit", not "Recommended", and that nobody tested it on such a Mac', async () => { + const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + }); + + it('lists the other four models that fit: the fast ones, the one of unknown speed, the slow one', async () => { + const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED); + expect(result.alternatives.map((c) => [c.modelId, c.context, c.fit, c.speed, c.tokensPerSecond ?? null])).toEqual([ + ['qwen3.5:9b-q8_0', 65536, 'good', 'ok', 17.8], + ['qwen3.5:9b', 65536, 'good', 'ok', 28.6], + ['gpt-oss:20b', 131072, 'good', 'unknown', null], + ['qwen3.8:27b', 131072, 'good', 'slow', 9.1], + ]); + expect(result.alternatives[2].speedLabel).toBe('Speed unknown'); + expect(result.alternatives[3].speedLabel).toBe('Slow'); + // The two that do not fit 35 GiB at any context are not listed. + expect(order(result)).not.toContain('qwen3.8:27b-q8_0@131072'); + expect(order(result).some((entry) => entry.startsWith('qwen3.5:122b'))).toBe(false); + }); + + it('would get the 27B instead if its chip were in no chips row', async () => { + // The same Mac with the M5 Pro row removed: every speed is unknown, so model rank decides. + const hw = await realMac(MACBOOK_SYSCTL, 890_926_084); + const withoutChip = stockWith((catalog) => { + catalog.chips = catalog.chips.filter((row) => row.match !== 'M5 Pro'); + }); + const result = recommend(hw, withoutChip, ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', context: 131072, speed: 'unknown' }); + expect(result.pick).not.toHaveProperty('tokensPerSecond'); + }); +}); + +describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { + /** + * Expected: no recommendation, reason `memory`. + * + * Why. The budget is 9.67 GiB (2/3 of 16 GiB minus 1). The only model near it is the 9B. Both + * harnesses that may be offered need 65,536 tokens of context, and there the 9B needs + * 6.10 GiB of weights + 2 GiB of KV cache + 1 GiB + 0.5 GiB of prompt cache = 9.60 GiB, which + * is 0.993 of the budget: above the 0.98 where "tight" ends. At 32,768 tokens it would fit + * (0.890, tight), but the one harness that works at that context is gated. + */ + it('gets no recommendation: nothing fits at the context a harness needs', async () => { + const hw = await realMac(MACMINI_SYSCTL, 41_752_848); + expect(hw.chip).toBe('Apple M4'); + expect(hw.memClassGb).toBe(16); + + const result = recommend(hw, STOCK, ALL_INSTALLED); + + expect(result).toEqual({ + verdict: 'none', + reason: 'memory', + headline: { + text: 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', + vars: { memory: '17.2' }, + }, + notes: [], + alternatives: [], + // floor(17,179,869,184 x 2/3) - 1 GiB. + budgetBytes: 10_379_504_298, + budgetGb: '10.4', + budgetSource: 'estimated', + }); + }); + + it('misses by a hair at 65536 tokens and would fit at 32768', async () => { + const hw = await realMac(MACMINI_SYSCTL, 41_752_848); + const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; + const budget = deviceBudget(hw).budgetBytes; + const shareAt64k = needBytes(nineB, 65536, STOCK, hw) / budget; + const shareAt32k = needBytes(nineB, 32768, STOCK, hw) / budget; + expect(shareAt64k).toBeCloseTo(0.9932, 4); + expect(shareAt64k).toBeGreaterThan(FIT_TIGHT_MAX_SHARE); + expect(shareAt32k).toBeCloseTo(0.8898, 4); + expect(shareAt32k).toBeLessThan(FIT_TIGHT_MAX_SHARE); + // Every other model needs more than the whole budget at its smallest context. + for (const other of STOCK.models.filter((m) => m.id !== 'qwen3.5:9b')) { + const smallest = other.contexts[other.contexts.length - 1]; + expect(needBytes(other, smallest, STOCK, hw), other.id).toBeGreaterThan(budget); + } + }); + + it('would get the 9B at 32768 tokens, as a limited answer, once the gated harness is opened', async () => { + const hw = await realMac(MACMINI_SYSCTL, 41_752_848); + const ungated = stockWith((catalog) => { + catalog.harnesses = catalog.harnesses.map((row) => ({ + cli: row.cli, + minContext: row.minContext, + rank: row.rank, + })); + }); + const result = recommend(hw, ungated, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.pick).toMatchObject({ + modelId: 'qwen3.5:9b', + context: 32768, + cli: 'pi', + fit: 'tight', + budgetShare: 0.89, + speed: 'ok', + // 9.3 x 1.2. + tokensPerSecond: 11.2, + }); + }); + + it('would get the 9B at 65536 tokens if a wired limit had been set on it', async () => { + // iogpu.wired_limit_mb = 14000 lifts the budget to 10 GiB (memory minus the 6 GiB reserve). + const hw = { ...(await realMac(MACMINI_SYSCTL, 41_752_848)), wiredLimitMb: 14000 }; + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.budgetSource).toBe('wired-limit'); + expect(result.budgetBytes).toBe(10 * GIB); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', context: 65536, cli: 'opencode', fit: 'tight' }); + }); +}); + +describe('every memory size, with the stock catalogue', () => { + // What the rules compute today. A change of a catalogue figure or of a threshold that moves + // one of these answers has to change this table on purpose. + const EXPECTED: [soldGb: number, chip: string, answer: string][] = [ + [8, 'Apple M2', 'none memory'], + [16, 'Apple M4', 'none memory'], + // The 9B at 65536 tokens is 0.87 of 11 GiB. + [18, 'Apple M3 Pro', 'limited qwen3.5:9b@65536 tight ok'], + // gpt-oss:20b at 65536 tokens needs 15.8 GiB of the 15 GiB budget: out. + [24, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], + // gpt-oss:20b fits only as Tight here (0.853 at 65536 tokens, 0.927 at 131072), and Good + // ranks before Tight. The Q8 9B is Good but slow on a base M4 (7 tokens per second). + [32, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], + [32, 'Apple M1 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + // The 27B (0.905) and the 35B-A3B (0.974) are Tight; gpt-oss:20b is Good but has no speed + // figure, and a known speed ranks before an unknown one. + [36, 'Apple M4 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + // On a Pro chip the dense 27B is slow (8.1 tokens per second), the 35B-A3B is not. + [48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'], + [48, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], + // The Q8 27B fits 47 GiB but computes to 7 tokens per second on this chip. + [64, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], + [96, 'Apple M3 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], + // The 122B fits 95 GiB only as Tight (0.861) and has no speed figure. + [128, 'Apple M5 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], + ]; + + it.each(EXPECTED)('%i GB, %s: %s', (soldGb, chip, answer) => { + const result = recommend(mac(soldGb, chip), STOCK, ALL_INSTALLED); + const actual = result.pick + ? `${result.verdict} ${result.pick.modelId}@${result.pick.context} ${result.pick.fit} ${result.pick.speed}` + : `${result.verdict} ${result.reason}`; + expect(actual).toBe(answer); + }); + + it('always pairs the pick with the top-ranked harness, OpenCode', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + if (result.pick) expect(result.pick.cli).toBe('opencode'); + } + }); +}); + +// ---------- 3. Verdict none ---------- + +describe('verdict none', () => { + const bigMac = mac(48, 'Apple M4 Max'); + + /** A `none` answer has no pick, no alternatives, no notes, and the reason's sentence as headline. */ + function expectNone(result: Recommendation, reason: Recommendation['reason'], headline: string): void { + expect(result.verdict).toBe('none'); + expect(result.reason).toBe(reason); + expect(result.headline.text).toBe(headline); + expect(result.pick).toBeUndefined(); + expect(result.alternatives).toEqual([]); + expect(result.notes).toEqual([]); + } + + describe('not-macos', () => { + it('answers so on Linux, with no budget at all', () => { + const linux: HardwareFacts = { + platform: 'linux', + arch: 'x64', + memBytes: 64 * GIB, + memGb: formatGb(64 * GIB), + memClassGb: 64, + source: 'not-probed', + }; + const result = recommend(linux, STOCK, ALL_INSTALLED); + expectNone(result, 'not-macos', LOCAL_MODEL_LABELS.noneNotMacos); + expect(result.headline.vars).toBeUndefined(); + expect(result.budgetBytes).toBe(0); + expect(result.budgetGb).toBe('0.0'); + expect(result.budgetSource).toBe('estimated'); + }); + + it('comes before every other reason', () => { + for (const platform of ['linux', 'win32', 'freebsd']) { + const hw = mac(8, 'Intel(R) Core(TM) i7', { platform, arch: 'x64', arm64: false, macosMajor: 12 }); + expect(recommend(hw, STOCK, []).reason, platform).toBe('not-macos'); + } + }); + }); + + describe('intel', () => { + const intelChip = 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'; + + it('answers so for an Intel Mac, whose sysctl has no arm64 flag', () => { + const hw = without(mac(32, intelChip, { arch: 'x64' }), 'arm64', 'translated', 'wiredLimitMb'); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expectNone(result, 'intel', 'This Mac has an Intel processor. Local coding models need Apple silicon.'); + expect(result.budgetBytes).toBe(0); + expect(result.budgetGb).toBe('0.0'); + }); + + it('answers so when the arm64 flag reads 0', () => { + const hw = without(mac(32, intelChip, { arch: 'x64', arm64: false }), 'translated'); + expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('intel'); + }); + + it('comes before the macOS version and the memory', () => { + const hw = without(mac(8, intelChip, { arch: 'x64', macosMajor: 12, macosVersion: '12.7' }), 'arm64'); + expect(recommend(hw, STOCK, []).reason).toBe('intel'); + }); + + it('does not mistake Node under Rosetta for an Intel Mac', () => { + // An x64 process on Apple silicon: the machine flag says arm64, the process is translated. + const rosetta = mac(48, 'Apple M4 Max', { arch: 'x64', translated: true }); + expect(recommend(rosetta, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + // The same when only the translation flag was read. + expect(recommend(without(rosetta, 'arm64'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + }); + + it('decides from node:os alone when the probe failed', () => { + const fallback = (arch: string, chip: string): HardwareFacts => ({ + platform: 'darwin', + arch, + memBytes: 48 * GIB, + memGb: formatGb(48 * GIB), + memClassGb: 48, + chip, + source: 'os-fallback', + }); + // An arm64 process can only run on Apple silicon. + expect(recommend(fallback('arm64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + // An x64 process with an Apple chip name is Rosetta. + expect(recommend(fallback('x64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + // An x64 process with any other chip name is taken for an Intel Mac. + expect(recommend(fallback('x64', intelChip), STOCK, ALL_INSTALLED).reason).toBe('intel'); + expect(recommend(without(fallback('x64', intelChip), 'chip'), STOCK, ALL_INSTALLED).reason).toBe('intel'); + }); + }); + + describe('macos', () => { + it('answers so below the catalogue minimum, naming both versions', () => { + const result = recommend( + mac(48, 'Apple M1 Max', { macosVersion: '13.6.7', macosMajor: 13 }), + STOCK, + ALL_INSTALLED + ); + expectNone(result, 'macos', 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.'); + expect(result.headline.vars).toEqual({ version: '13.6.7', minimum: '14' }); + // The budget is still reported: the Mac itself is able. + expect(result.budgetBytes).toBe(35 * GIB); + }); + + it('accepts the minimum itself and anything newer', () => { + for (const macosMajor of [14, 15, 26, 27]) { + const hw = mac(48, 'Apple M4 Max', { macosMajor, macosVersion: `${macosMajor}.0` }); + expect(recommend(hw, STOCK, ALL_INSTALLED).verdict, String(macosMajor)).toBe('ok'); + } + }); + + it('does not block on a version that could not be read', () => { + const hw = without(mac(48, 'Apple M4 Max'), 'macosMajor', 'macosVersion'); + expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + }); + + it('comes before the memory', () => { + const hw = mac(8, 'Apple M1', { macosVersion: '12.7', macosMajor: 12 }); + expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('macos'); + }); + }); + + describe('memory', () => { + it('answers so at 8 GB, where nothing fits at any context', () => { + const result = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); + expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemory); + // The size is the decimal figure of the hardware line, without the unit. + expect(result.headline.vars).toEqual({ memory: '8.6' }); + expect(result.budgetBytes).toBe(2 * GIB); + expect(result.budgetGb).toBe('2.1'); + }); + + it('answers so when models fit, but only below the context a harness needs', () => { + // The 16 GB case of the Mac mini, with harnesses that are all present. + const result = recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED); + expect(result.reason).toBe('memory'); + expect(result.headline.vars).toEqual({ memory: '17.2' }); + }); + + it('comes before no-harness when nothing fits at all', () => { + expect(recommend(mac(8, 'Apple M2'), STOCK, []).reason).toBe('memory'); + }); + + it('answers so for an empty catalogue', () => { + expect(recommend(bigMac, catalogOf([]), [fact('alpha')]).reason).toBe('memory'); + }); + + it('answers so when the memory size is unknown', () => { + const hw = { ...mac(48, 'Apple M4 Max'), memBytes: 0, memGb: '0.0', memClassGb: 0 }; + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.reason).toBe('memory'); + expect(result.budgetBytes).toBe(0); + }); + }); + + describe('no-harness', () => { + const headline = 'No coding agent that Codeman can connect to a local model is available on this Mac.'; + + it('answers so when the caller knows no catalogue harness', () => { + const result = recommend(bigMac, STOCK, []); + expectNone(result, 'no-harness', headline); + expect(result.headline.vars).toBeUndefined(); + expect(result.budgetBytes).toBe(35 * GIB); + }); + + it('answers so when no harness is installed or installable', () => { + const facts = STOCK.harnesses.map((row) => fact(row.cli, { installed: false, installable: false })); + expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); + }); + + it('answers so when only the gated harness is on the machine', () => { + const gatedRows = STOCK.harnesses.filter((row) => row.gated); + expect(gatedRows).toHaveLength(1); + expect( + recommend( + bigMac, + STOCK, + gatedRows.map((row) => fact(row.cli)) + ).reason + ).toBe('no-harness'); + }); + + it('answers so when a harness is there but accepts no model id', () => { + const facts = ALL_INSTALLED.map((entry) => ({ ...entry, acceptsModelId: () => false })); + expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); + }); + + it('ignores a fact for a CLI that is not a catalogue harness', () => { + expect(recommend(bigMac, STOCK, [fact('codex'), fact('some-other-cli')]).reason).toBe('no-harness'); + }); + }); + + describe('disk', () => { + it('answers so when every fitting model would leave under 10 GiB free', () => { + // 12 GiB free: even the smallest download (6.55 GB) leaves 5.9 GiB. + const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expectNone( + result, + 'disk', + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.' + ); + // download + 10 GiB - free = 6,550,825,373 + 10,737,418,240 - 12,884,901,888 bytes. + expect(result.headline.vars).toEqual({ download: '6.6', free: '12.9', missing: '4.4' }); + expect(result.budgetBytes).toBe(35 * GIB); + }); + + it('names the smallest model that fits the MEMORY, not the smallest of the catalogue', () => { + // On a 48 GB Mac whose harness refuses both 9B ids, the smallest candidate is gpt-oss:20b. + const facts = ALL_INSTALLED.map((entry) => ({ + ...entry, + acceptsModelId: (id: string) => !id.startsWith('qwen3.5:9b'), + })); + const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); + const result = recommend(hw, STOCK, facts); + expect(result.reason).toBe('disk'); + expect(result.headline.vars?.download).toBe('13.8'); + }); + + it('drops only the models that do not leave 10 GiB, and picks among the rest', () => { + // 20 GiB free: the 9B downloads (6.1 and 9.7 GiB) leave 10 GiB or more, the others do not. + const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 20 * GIB, freeDiskGb: formatGb(20 * GIB) }); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(order(result)).toEqual(['qwen3.5:9b-q8_0@65536', 'qwen3.5:9b@65536']); + }); + + it('allows a download that leaves exactly 10 GiB', () => { + const exact = STOCK.models[0].bytes + DISK_HEADROOM_BYTES; + expect(DISK_HEADROOM_BYTES).toBe(10 * GIB); + const fits = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact }), STOCK, ALL_INSTALLED); + expect(fits.pick?.modelId).toBe('qwen3.5:9b'); + const short = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact - 1 }), STOCK, ALL_INSTALLED); + expect(short.reason).toBe('disk'); + }); + + it('does not block when the free disk is unknown', () => { + const hw = without(mac(48, 'Apple M4 Max'), 'freeDiskBytes', 'freeDiskGb'); + expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + }); + + it('formats the free size itself when the hardware carries only the bytes', () => { + const hw = without(mac(48, 'Apple M4 Max', { freeDiskBytes: 3 * GB }), 'freeDiskGb'); + expect(recommend(hw, STOCK, ALL_INSTALLED).headline.vars?.free).toBe('3.0'); + }); + + it('comes after the memory and the harness', () => { + const fullDisk = { freeDiskBytes: 0, freeDiskGb: '0.0' }; + expect(recommend(mac(16, 'Apple M4', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('memory'); + expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, []).reason).toBe('no-harness'); + expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('disk'); + }); + }); +}); + +// ---------- 4. Verdicts, ranking, harnesses, tested ---------- + +describe('verdict limited', () => { + it('is the answer when the best model that fits has fewer than 12 billion parameters', () => { + expect(LIMITED_BELOW_PARAMS_B).toBe(12); + const result = recommend(mac(24, 'Apple M4'), STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.reason).toBeUndefined(); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', paramsB: 9, context: 65536, fit: 'good' }); + expect(result.headline).toEqual({ + text: 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', + }); + expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); + }); + + it('holds for a tight fit too (18 GB)', () => { + const result = recommend(mac(18, 'Apple M3 Pro'), STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', fit: 'tight', fitLabel: 'Tight', budgetShare: 0.873 }); + expect(result.alternatives).toEqual([]); + }); + + it('draws the line at 12 billion: 11.9 is limited, 12 is ok', () => { + const verdictFor = (paramsB: number): string => + recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { paramsB })]), [fact('alpha')]).verdict; + expect(verdictFor(11.9)).toBe('limited'); + expect(verdictFor(12)).toBe('ok'); + expect(verdictFor(9)).toBe('limited'); + expect(verdictFor(27.8)).toBe('ok'); + }); + + it('is about the PICK: a larger model further down the list does not lift it', () => { + // 32 GB: the 20B fits only as Tight, so a 9B is the pick and the answer is limited. + const result = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.alternatives.map((c) => [c.modelId, c.fit])).toContainEqual(['gpt-oss:20b', 'tight']); + }); + + it('never says "Recommended", for any Mac', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + if (result.verdict === 'limited') { + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + } + } + }); +}); + +describe('speed', () => { + it('is the model figure times the chip bandwidth, rounded to one decimal', () => { + const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + // 2.95 x 410 / 100. + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', speed: 'ok', tokensPerSecond: 12.1 }); + expect(result.pick).not.toHaveProperty('speedLabel'); + }); + + it('is slow below 10 tokens per second, and says so', () => { + expect(SLOW_BELOW_TOKENS_PER_SECOND).toBe(10); + const speedOf = (speedPer100Gbps: number): Candidate | undefined => + recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { speedPer100Gbps })]), [fact('alpha')]).pick; + // The synthetic chip has 400 GB/s: 2.5 is 10.0 tokens per second, 2.475 is 9.9. + expect(speedOf(2.5)).toMatchObject({ speed: 'ok', tokensPerSecond: 10 }); + expect(speedOf(2.475)).toMatchObject({ speed: 'slow', tokensPerSecond: 9.9, speedLabel: 'Slow' }); + expect(speedOf(0.5)).toMatchObject({ speed: 'slow', tokensPerSecond: 2, speedLabel: 'Slow' }); + }); + + it('is unknown for a model without a figure, on any chip', () => { + // 48 GB: five models fit, so all of them are listed and gpt-oss:20b is among them. + for (const chip of ['Apple M1', 'Apple M4 Max', 'Apple M5 Ultra']) { + const result = recommend(mac(48, chip), STOCK, ALL_INSTALLED); + const gptOss = [result.pick, ...result.alternatives].find((c) => c?.modelId === 'gpt-oss:20b'); + expect(gptOss, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); + expect(gptOss).not.toHaveProperty('tokensPerSecond'); + } + }); + + it('is unknown for every model on a chip that is in no chips row: no bandwidth is guessed', () => { + for (const chip of ['Apple M6', 'Apple M6 Pro', 'Apple M10', 'Some Future Chip']) { + const result = recommend(mac(64, chip), STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + expect(candidate, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); + expect(candidate).not.toHaveProperty('tokensPerSecond'); + } + } + // And when the chip name itself is missing. + const result = recommend(without(mac(64, 'x'), 'chip'), STOCK, ALL_INSTALLED); + expect(result.pick?.speed).toBe('unknown'); + }); +}); + +describe('the ranking', () => { + it('puts ok before unknown before slow, ahead of the model rank', () => { + // Three models that fit equally well. The ranks run AGAINST the speed. + const catalog = catalogOf([ + model('slow-top:1', { rank: 3, speedPer100Gbps: 1 }), + model('unknown-mid:1', { rank: 2 }), + model('ok-low:1', { rank: 1, speedPer100Gbps: 5 }), + ]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(order(result)).toEqual(['ok-low:1@1024', 'unknown-mid:1@1024', 'slow-top:1@1024']); + expect([result.pick, ...result.alternatives].map((c) => c?.speed)).toEqual(['ok', 'unknown', 'slow']); + }); + + it('ranks by model rank alone when every speed is unknown', () => { + // 'Apple M6' is in no chips row. + const result = recommend(mac(48, 'Apple M6'), STOCK, ALL_INSTALLED); + expect(order(result)).toEqual([ + 'qwen3.8:27b@131072', + 'qwen3.6:35b-a3b@131072', + 'gpt-oss:20b@131072', + 'qwen3.5:9b-q8_0@65536', + 'qwen3.5:9b@65536', + ]); + }); + + it('ranks an unknown speed below a known ok one and above a slow one, on a real tier', () => { + // 48 GB, M4 Pro: three ok models, then gpt-oss:20b (unknown), then the 27B (slow). + const result = recommend(mac(48, 'Apple M4 Pro'), STOCK, ALL_INSTALLED); + expect([result.pick, ...result.alternatives].map((c) => [c?.modelId, c?.speed])).toEqual([ + ['qwen3.6:35b-a3b', 'ok'], + ['qwen3.5:9b-q8_0', 'ok'], + ['qwen3.5:9b', 'ok'], + ['gpt-oss:20b', 'unknown'], + ['qwen3.8:27b', 'slow'], + ]); + }); + + it('puts a good fit before a tight one, ahead of speed and rank', () => { + const catalog = catalogOf([ + modelNeeding('tight-fast-top:1', 0.9 * HUNDRED_GIB, { rank: 9, speedPer100Gbps: 20 }), + modelNeeding('good-slow-low:1', 0.5 * HUNDRED_GIB, { rank: 1, speedPer100Gbps: 1 }), + ]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(order(result)).toEqual(['good-slow-low:1@1024', 'tight-fast-top:1@1024']); + }); + + it('takes the higher model rank among equals', () => { + const catalog = catalogOf([model('low:1', { rank: 1 }), model('high:1', { rank: 5 }), model('mid:1', { rank: 3 })]); + expect(order(recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]))).toEqual([ + 'high:1@1024', + 'mid:1@1024', + 'low:1@1024', + ]); + }); + + it('takes the larger context of one model when both fit equally well', () => { + // 1 KiB per token: 4 GiB of KV cache at 4,194,304 tokens, still a good fit. + const catalog = catalogOf([model('probe:1', { kvBytesPerToken: 1024, contexts: [4_194_304, 1024] })]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(result.pick?.context).toBe(4_194_304); + expect(result.alternatives).toEqual([]); + }); + + it('steps a model down to the context at which it fits well', () => { + // Good at the small context, tight at the large one: the good one is that model's entry. + const catalog = catalogOf([ + model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] }), + ]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(result.pick).toMatchObject({ context: 1024, fit: 'good' }); + }); + + it('lists each model once, never the pick again, and at most four alternatives', () => { + expect(MAX_ALTERNATIVES).toBe(4); + // Six models fit a 128 GB Mac; the pick plus four are shown. + const result = recommend(mac(128, 'Apple M5 Ultra'), STOCK, ALL_INSTALLED); + expect(result.alternatives).toHaveLength(4); + const ids = [result.pick, ...result.alternatives].map((c) => c?.modelId); + expect(new Set(ids).size).toBe(ids.length); + for (const hw of SWEEP) { + const swept = recommend(hw, STOCK, ALL_INSTALLED); + const sweptIds = [swept.pick, ...swept.alternatives].filter(Boolean).map((c) => c?.modelId); + expect(new Set(sweptIds).size).toBe(sweptIds.length); + expect(swept.alternatives.length).toBeLessThanOrEqual(MAX_ALTERNATIVES); + } + }); +}); + +describe('harnesses', () => { + const hw = mac(48, 'Apple M4 Max'); + + it('never chooses a gated harness, even when it is the only one that could run a model', () => { + const gated = STOCK.harnesses.filter((row) => row.gated).map((row) => row.cli); + expect(gated).toEqual(['pi']); + for (const swept of SWEEP) { + const result = recommend(swept, STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + if (candidate) expect(gated).not.toContain(candidate.cli); + } + } + // 16 GB: the gated harness alone works at 32768 tokens, where the 9B would fit. Still none. + expect(recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED).verdict).toBe('none'); + }); + + it('never offers a context below the minimum of the harness', () => { + for (const swept of SWEEP) { + const result = recommend(swept, STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + if (!candidate) continue; + const row = STOCK.harnesses.find((harness) => harness.cli === candidate.cli); + expect(candidate.context).toBeGreaterThanOrEqual(row?.minContext ?? Number.POSITIVE_INFINITY); + } + } + }); + + it('takes the higher-ranked harness and reports its label and that it is installed', () => { + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.pick).toMatchObject({ cli: 'opencode', cliLabel: 'OpenCode', cliInstalled: true }); + expect(result.pick).not.toHaveProperty('slowFirstTurn'); + }); + + it('falls back to the next harness when the first is not on the machine, and flags its slow first turn', () => { + const facts = [fact('opencode', { installed: false, installable: false }), fact('claude'), fact('pi')]; + const result = recommend(hw, STOCK, facts); + expect(result.pick).toMatchObject({ + modelId: 'qwen3.8:27b', + cli: 'claude', + cliLabel: 'Claude Code', + cliInstalled: true, + slowFirstTurn: true, + }); + // The same when the caller has no fact for it at all. + expect(recommend(hw, STOCK, [fact('claude')]).pick?.cli).toBe('claude'); + }); + + it('offers a harness that is not installed but can be, and says it is not installed', () => { + const facts = [fact('opencode', { installed: false, installable: true }), fact('claude')]; + const result = recommend(hw, STOCK, facts); + expect(result.pick).toMatchObject({ cli: 'opencode', cliInstalled: false }); + }); + + it('pairs a model only with a harness that accepts its id', () => { + const refusing27b = (id: string): boolean => id !== 'qwen3.8:27b'; + // OpenCode refuses the 27B: it goes with the next harness. + const one = recommend(hw, STOCK, [fact('opencode', { acceptsModelId: refusing27b }), fact('claude')]); + expect(one.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'claude' }); + expect(one.alternatives[0]).toMatchObject({ modelId: 'qwen3.6:35b-a3b', cli: 'opencode' }); + // Both refuse it: the model is not offered. + const both = recommend(hw, STOCK, [ + fact('opencode', { acceptsModelId: refusing27b }), + fact('claude', { acceptsModelId: refusing27b }), + ]); + expect(both.pick?.modelId).toBe('qwen3.6:35b-a3b'); + expect(order(both)).not.toContain('qwen3.8:27b@131072'); + }); + + it('treats a harness check that throws as a refusal, and still answers', () => { + const throwing = fact('opencode', { + acceptsModelId: () => { + throw new Error('registry not loaded'); + }, + }); + const result = recommend(hw, STOCK, [throwing, fact('claude')]); + expect(result.verdict).toBe('ok'); + expect(result.pick?.cli).toBe('claude'); + expect(recommend(hw, STOCK, [throwing]).reason).toBe('no-harness'); + }); + + it('puts the fit and the model before the harness rank', () => { + // `big` needs a large context and is the higher-ranked harness; `small` works from 1024. + const harnesses: LocalHarnessEntry[] = [ + { cli: 'big', minContext: 8_388_608, rank: 2 }, + { cli: 'small', minContext: 1024, rank: 1 }, + ]; + const tightAtLarge = catalogOf( + [model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })], + harnesses + ); + // Tight at the context `big` needs, good at the small one: the good fit with `small` wins. + expect(recommend(HUNDRED_GIB_MAC, tightAtLarge, [fact('big'), fact('small')]).pick).toMatchObject({ + cli: 'small', + context: 1024, + fit: 'good', + }); + // Good at both: the higher-ranked harness wins, at the context it needs. + const goodAtBoth = catalogOf( + [model('probe:1', { bytes: 10 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })], + harnesses + ); + expect(recommend(HUNDRED_GIB_MAC, goodAtBoth, [fact('big'), fact('small')]).pick).toMatchObject({ + cli: 'big', + context: 8_388_608, + }); + }); +}); + +describe('the tested flag and the word "Recommended"', () => { + const testedOn = (chip: string, memGb: number, cli: string) => ({ chip, memGb, cli, date: '2026-11-06' }); + const with27bTested = (chip: string, memGb: number, cli: string): LocalModelCatalog => + stockWith((catalog) => { + const row = catalog.models.find((m) => m.id === 'qwen3.8:27b') as LocalModelEntry; + row.tested = [testedOn(chip, memGb, cli)]; + }); + + it('is off for every model of the stock catalogue, so "Recommended" is never said today', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + if (candidate) expect(candidate.tested).toBe(false); + } + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + if (result.verdict !== 'none') { + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + } + } + }); + + it('turns on "Recommended for this Mac" when the pick was tested on exactly such a Mac', () => { + const result = recommend(mac(48, 'Apple M4 Max'), with27bTested('Apple M4 Max', 48, 'opencode'), ALL_INSTALLED); + expect(result.verdict).toBe('ok'); + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: true }); + expect(result.headline).toEqual({ text: 'Recommended for this Mac' }); + // The "not yet tested" note goes away with it. + expect(result.notes).toEqual([]); + }); + + it('stays off when the chip, the memory size or the harness differs', () => { + const hw = mac(48, 'Apple M4 Max'); + const cases: LocalModelCatalog[] = [ + with27bTested('Apple M4 Pro', 48, 'opencode'), + with27bTested('Apple M4', 48, 'opencode'), + with27bTested('Apple M4 Max', 64, 'opencode'), + with27bTested('Apple M4 Max', 48, 'claude'), + ]; + for (const catalog of cases) { + const result = recommend(hw, catalog, ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: false }); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); + } + }); + + it('follows the harness: tested with the second harness counts when that one is the pick', () => { + const catalog = with27bTested('Apple M4 Max', 48, 'claude'); + const result = recommend(mac(48, 'Apple M4 Max'), catalog, [fact('claude')]); + expect(result.pick).toMatchObject({ cli: 'claude', tested: true }); + expect(result.headline.text).toBe('Recommended for this Mac'); + }); + + it('does not change the ranking: a tested model further down stays an alternative', () => { + // On the M4 Pro the pick is the 35B-A3B. A tested 27B is flagged in the list, no more. + const result = recommend(mac(48, 'Apple M4 Pro'), with27bTested('Apple M4 Pro', 48, 'opencode'), ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', tested: false }); + expect(result.headline.text).toBe('Best fit for this Mac'); + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + expect(result.alternatives.find((c) => c.modelId === 'qwen3.8:27b')?.tested).toBe(true); + }); + + it('never says "Recommended" for a limited answer, tested or not', () => { + const catalog = stockWith((stock) => { + const row = stock.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; + row.tested = [testedOn('Apple M4', 24, 'opencode')]; + }); + const result = recommend(mac(24, 'Apple M4'), catalog, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.pick?.tested).toBe(true); + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + // It WAS tested, so the "not yet tested" note would be false. + expect(result.notes).toEqual([]); + }); +}); + +describe('the shape of the answer', () => { + const labels = new Set(Object.values(LOCAL_MODEL_LABELS)); + + it('takes every sentence and label from the fixed list', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(labels.has(result.headline.text)).toBe(true); + for (const note of result.notes) expect(labels.has(note.text)).toBe(true); + for (const candidate of [result.pick, ...result.alternatives]) { + if (!candidate) continue; + expect(labels.has(candidate.fitLabel)).toBe(true); + if (candidate.speedLabel !== undefined) expect(labels.has(candidate.speedLabel)).toBe(true); + expect(candidate.speedLabel === undefined).toBe(candidate.speed === 'ok'); + expect(candidate.tokensPerSecond === undefined).toBe(candidate.speed === 'unknown'); + } + } + }); + + it('fills every placeholder of a headline and nothing else', () => { + const answers = [ + recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { macosMajor: 13, macosVersion: '13.0' }), STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { freeDiskBytes: GB, freeDiskGb: '1.0' }), STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4'), STOCK, []), + recommend(mac(48, 'Apple M4'), STOCK, ALL_INSTALLED), + ]; + expect(answers.map((answer) => answer.reason ?? answer.verdict)).toEqual([ + 'memory', + 'macos', + 'disk', + 'no-harness', + 'ok', + ]); + for (const answer of answers) { + const slots = [...answer.headline.text.matchAll(/\{(\w+)\}/g)].map((match) => match[1]).sort(); + expect(Object.keys(answer.headline.vars ?? {}).sort()).toEqual(slots); + // `{name}` belongs to the browser's translator. + expect(slots).not.toContain('name'); + for (const value of Object.values(answer.headline.vars ?? {})) expect(typeof value).toBe('string'); + } + }); + + it('prints every size in one unit: decimal GB from the one formatter, never GiB', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.budgetGb).toBe(formatGb(result.budgetBytes)); + for (const candidate of [result.pick, ...result.alternatives]) { + if (!candidate) continue; + expect(candidate.needGb).toBe(formatGb(candidate.needBytes)); + expect(candidate.downloadGb).toBe(formatGb(candidate.downloadBytes)); + expect(candidate.budgetShare).toBeCloseTo(candidate.needBytes / result.budgetBytes, 3); + } + expect(JSON.stringify(result)).not.toMatch(/GiB|MiB/); + } + }); + + it('adds up for a person: download, plus what the context and the runtime take, is the need', () => { + const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const pick = result.pick as Candidate; + // 17.7 GB of weights + 8 GiB of KV cache + 1 GiB + 2 GiB of prompt cache, all in decimal GB. + expect(pick.downloadGb).toBe('17.7'); + expect(pick.needGb).toBe('29.6'); + expect(formatGb(pick.needBytes - pick.downloadBytes)).toBe('11.8'); + expect(result.budgetGb).toBe('37.6'); + expect(pick.needBytes).toBeLessThan(result.budgetBytes); + }); + + it('is plain JSON and has exactly the fields of its verdict', () => { + const ok = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + expect(JSON.parse(JSON.stringify(ok))).toEqual(ok); + expect(Object.keys(ok).sort()).toEqual( + ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'pick', 'verdict'].sort() + ); + const none = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); + expect(JSON.parse(JSON.stringify(none))).toEqual(none); + expect(Object.keys(none).sort()).toEqual( + ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'reason', 'verdict'].sort() + ); + }); + + it('is pure: frozen inputs are enough, and the same inputs give the same answer', () => { + const hw = deepFreeze(mac(48, 'Apple M4 Pro')); + const catalog = deepFreeze(structuredClone(STOCK_LOCAL_MODEL_CATALOG)); + const facts = deepFreeze(ALL_INSTALLED.map((entry) => ({ ...entry }))); + const first = recommend(hw, catalog, facts); + const second = recommend(hw, catalog, facts); + expect(second).toEqual(first); + expect(first.verdict).toBe('ok'); + }); +});