From 92c20fe572ff8bbc32cd2efa39817fa78518b09c Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 16:27:33 +0100 Subject: [PATCH] feat(local-model): hardware probe, model catalogue and the pure recommendation The core of the read-only recommendation: what this Mac is, what the catalogue knows, and which model fits. Nothing is downloaded, installed or written. - hardware.ts: one bounded async spawn of /usr/sbin/sysctl with seven keys, without -n, parsed as "key: value" lines by the pure parseSysctl(). Free disk of the data dir through statfs behind the bounded path probe. Fallbacks from node:os, a 60 s cache, never throws, never system_profiler. A wired limit of 0 means "not set". - local-model-catalog.ts: seven models, three harness rows (pi gated) and the chips table as a compiled constant, validated by a Zod schema on first use. No override file. Model ids are name:tag in lowercase without a slash; every model carries paramsB and an (empty) tested list. Chip names match on the end of the chip string, so 'M4' is not the chip of 'Apple M4 Pro'. - recommend.ts: device budget (the wired limit when above 0, else 2/3 of memory below 36 GB and 3/4 from 36 GB, less the reserve), need = weights + KV cache + 1 GiB + prompt cache, fit good up to 0.85 and tight up to 0.98, speed ok, unknown or slow (a chip in no chips row is unknown), and the ranking. Verdicts ok, limited (best model under 12 billion parameters) and none with its reason. "Recommended" only for a pick with a tested entry for this Mac, which no row has today. - types/local-model.ts: comments brought in line with the code (chip matching, budgetShare rounding, a budget of 0 on an Intel Mac). - cli-registry-no-id-branching.test.ts: the guard also catches a comparison of a `cli` field with a CLI id (plus one stray blank line Prettier refused). Tests: test/local-model-format.test.ts, test/local-model-hardware.test.ts (the captured output of two real Macs), test/local-model-catalog.test.ts, test/local-model-recommend.test.ts. Co-Authored-By: Claude Fable 5.1 --- src/config/local-model-catalog.ts | 305 ++++- src/local-model/hardware.ts | 325 +++++- src/local-model/recommend.ts | 332 +++++- src/types/local-model.ts | 13 +- test/cli-registry-no-id-branching.test.ts | 11 +- test/local-model-catalog.test.ts | 383 ++++++ test/local-model-format.test.ts | 86 ++ test/local-model-hardware.test.ts | 749 ++++++++++++ test/local-model-recommend.test.ts | 1298 +++++++++++++++++++++ 9 files changed, 3423 insertions(+), 79 deletions(-) create mode 100644 test/local-model-catalog.test.ts create mode 100644 test/local-model-format.test.ts create mode 100644 test/local-model-hardware.test.ts create mode 100644 test/local-model-recommend.test.ts diff --git a/src/config/local-model-catalog.ts b/src/config/local-model-catalog.ts index 661ebd8b..a66a111e 100644 --- a/src/config/local-model-catalog.ts +++ b/src/config/local-model-catalog.ts @@ -2,16 +2,7 @@ * @fileoverview The local model catalogue: which models, harnesses and chips the recommendation * knows, as a compiled constant validated by a Zod schema. * - * SKELETON: the schema below is the shape of the contract (it mirrors `LocalModelCatalog` in - * src/types/local-model.ts) and the stock constant is EMPTY. Still to come here: - * - the model rows (ids and byte sizes from the registry manifests, `paramsB`, an empty - * `tested` array each; `flint:40b` stays out), the harness rows and the chips table; - * - the cross-row rules the schema cannot say field by field: unique model ids, unique harness - * `cli`s, `contexts` largest first, `chips` most specific first, `promptCacheTiers` ascending - * and starting at 0; - * - `getLocalModelCatalog()` parsing the constant once, on first use, never at import. - * - * Rules that hold from the start: + * Rules: * - There is NO override file and no runtime download data in this stage: nothing here reads * the data dir, and a data file can never decide what gets recommended. * - A model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`), so it can @@ -20,14 +11,71 @@ * registry id as data: no code may compare it with a literal. Whether that CLI exists and can * take a custom model is checked against the registry at call time, in * src/local-model/harness-facts.ts, never here. + * - The schema checks more than fields. Across rows it requires: unique model ids and ranks, + * unique harness `cli`s and ranks, `contexts` largest first, a `tested` entry naming a harness + * row, `chips` most specific first, and `promptCacheTiers` ascending from 0. + * - `getLocalModelCatalog()` parses the constant once, on first use and never at import, and + * throws on a bad row. test/local-model-catalog.test.ts pins that the stock constant parses, + * so a bad row fails the gate and never reaches the route. + * + * Where each figure of the stock rows comes from, and what was checked (2026-11-06): + * + * | Figure | Source | Checked | + * |----------------------|-----------------------------------------------------------|--------------------------------------| + * | `id`, `bytes` | registry manifest of the tag, sum of its layer sizes | yes, to the byte, by the facts review | + * | `license` | the model card (all Apache-2.0) | read by the research, not re-checked | + * | `paramsB` | 27.8 from the model card; the others are the count in the | no. Only the 12 billion line reads | + * | | model's name | it, and no row is near that line | + * | `kvBytesPerToken` | COMPUTED from each model's config: 2 x full-attention | no. Not measured in a running | + * | | layers x KV heads x head size x 2 bytes | runtime. The review called the | + * | | | gpt-oss figure plausible, no more | + * | `contexts`, `rank` | a judgement of the design | no | + * | `speedPer100Gbps` | one published run on an M1 Max (400 GB/s): generation | the run was read. It stands for less | + * | | only, short prompt. The two q8_0 rows are SCALED by file | than a session: no long prompt, no | + * | | size, not measured | thinking tokens | + * | `minMacosMajor` | the runtime's documentation (macOS 14) | yes, by the facts review | + * | `promptCacheTiers` | a decision of the design, not a measurement | no | + * | harness `minContext` | 65536: the runtime's documentation for both harnesses. | 65536 yes. 32768 no, which is why | + * | | 32768: recalled | that row is gated | + * | `chips` | the bandwidth column of | yes, read at the source for this | + * | | https://github.com/ggml-org/llama.cpp/discussions/4167 | file | + * + * `tested` is empty on every row: nobody has run any of these models with a harness on a Mac + * yet. Until a row has an entry, the recommendation never says "Recommended". */ import { z } from 'zod'; -import type { LocalModelCatalog } from '../types.js'; +import type { ChipEntry, LocalModelCatalog } from '../types.js'; /** A model id: lowercase name, one colon, a tag. No slash, so no registry host or namespace can ride along. */ export const LOCAL_MODEL_ID_PATTERN = /^[a-z0-9][a-z0-9._-]*:[a-z0-9][a-z0-9._-]*$/; +/** + * Whether `chip` (a `machdep.cpu.brand_string`, 'Apple M4 Pro') is the chip a `ChipEntry.match` + * ('M4 Pro') names: the string must END with the match, and the match must start at the + * beginning or after a space. Case is ignored. + * + * Stricter than "the match occurs in the string" on purpose. 'M4' is not the chip of + * 'Apple M4 Pro', 'M1' is not the chip of 'Apple M10', and a variant this table has never heard + * of ('Apple M6 Pro') matches no row instead of borrowing the bandwidth of its base chip. + */ +export function chipNameMatches(chip: string, match: string): boolean { + const name = chip.trim().toLowerCase(); + const wanted = match.trim().toLowerCase(); + if (wanted === '' || !name.endsWith(wanted)) return false; + const start = name.length - wanted.length; + return start === 0 || name[start - 1] === ' '; +} + +/** + * The memory bandwidth (GB/s) of `chip`, from the first row that names it. `undefined` when the + * chip string is absent or no row names it: the speed is then UNKNOWN, a bandwidth is never guessed. + */ +export function chipBandwidthGbps(chips: readonly ChipEntry[], chip: string | undefined): number | undefined { + if (!chip) return undefined; + return chips.find((row) => chipNameMatches(chip, row.match))?.gbps; +} + const isoDate = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'must be an ISO date (YYYY-MM-DD)'); const positiveInt = z.number().int().positive(); @@ -79,7 +127,20 @@ const PromptCacheTierSchema = z }) .strict(); -/** Validates a whole catalogue. Typed against `LocalModelCatalog`, so the two cannot drift apart. */ +/** The index of the first value that already occurred earlier in the list, or -1. */ +function firstRepeat(values: readonly T[]): number { + const seen = new Set(); + for (let i = 0; i < values.length; i++) { + if (seen.has(values[i])) return i; + seen.add(values[i]); + } + return -1; +} + +/** + * Validates a whole catalogue: every field, then the rules that hold across rows (see the file + * header). Typed against `LocalModelCatalog`, so the two cannot drift apart. + */ export const LocalModelCatalogSchema: z.ZodType = z .object({ schema: z.literal(1), @@ -90,33 +151,223 @@ export const LocalModelCatalogSchema: z.ZodType = z harnesses: z.array(LocalHarnessEntrySchema), chips: z.array(ChipEntrySchema), }) - .strict(); + .strict() + .superRefine((catalog, ctx) => { + const issue = (path: (string | number)[], message: string): void => { + ctx.addIssue({ code: 'custom', path, message }); + }; + + // A repeated id would make "the model" ambiguous, a repeated rank would make the order of + // two models depend on where they sit in the array. + const repeatedId = firstRepeat(catalog.models.map((m) => m.id)); + if (repeatedId >= 0) issue(['models', repeatedId, 'id'], 'model ids must be unique'); + const repeatedRank = firstRepeat(catalog.models.map((m) => m.rank)); + if (repeatedRank >= 0) issue(['models', repeatedRank, 'rank'], 'model ranks must be unique'); + + const harnessClis = catalog.harnesses.map((h) => h.cli); + const repeatedCli = firstRepeat(harnessClis); + if (repeatedCli >= 0) issue(['harnesses', repeatedCli, 'cli'], 'a harness may be listed only once'); + const repeatedHarnessRank = firstRepeat(catalog.harnesses.map((h) => h.rank)); + if (repeatedHarnessRank >= 0) issue(['harnesses', repeatedHarnessRank, 'rank'], 'harness ranks must be unique'); + + catalog.models.forEach((model, m) => { + for (let i = 1; i < model.contexts.length; i++) { + if (model.contexts[i] >= model.contexts[i - 1]) { + issue(['models', m, 'contexts', i], 'contexts must be listed largest first, without repeats'); + break; + } + } + // An entry for a harness the catalogue does not list could never match a candidate: a typo. + model.tested.forEach((entry, t) => { + if (!harnessClis.includes(entry.cli)) { + issue(['models', m, 'tested', t, 'cli'], 'a tested entry must name a harness of this catalogue'); + } + }); + }); + + // Most specific first ('M4 Max' before 'M4'): a row whose text sits inside a LATER row's text + // is the less specific of the two, so it is in the wrong place. A repeated match is the same + // mistake. + for (let later = 1; later < catalog.chips.length; later++) { + const laterMatch = catalog.chips[later].match.toLowerCase(); + const shadowing = catalog.chips.slice(0, later).find((row) => laterMatch.includes(row.match.toLowerCase())); + if (shadowing) { + issue( + ['chips', later, 'match'], + `chips must be listed most specific first: '${shadowing.match}' comes before '${catalog.chips[later].match}'` + ); + break; + } + } + + // The tiers are read as "the last one whose minimum is not above this Mac's memory", so they + // must start at 0 (every Mac has a tier) and ascend. + if (catalog.promptCacheTiers[0].minMemGb !== 0) { + issue(['promptCacheTiers', 0, 'minMemGb'], 'the first prompt cache tier must start at 0'); + } + for (let i = 1; i < catalog.promptCacheTiers.length; i++) { + if (catalog.promptCacheTiers[i].minMemGb <= catalog.promptCacheTiers[i - 1].minMemGb) { + issue(['promptCacheTiers', i, 'minMemGb'], 'prompt cache tiers must ascend by minMemGb'); + break; + } + } + }); + +const KIB = 1024; +const MIB = 1024 ** 2; +const GIB = 1024 ** 3; /** - * The catalogue that ships with Codeman. EMPTY for now (no model, harness or chip rows), but - * valid: with it `recommend()` can only answer verdict `none`. - * - * The two constants that are already decided are filled in: macOS 14 is the oldest version the - * runtime supports, and the prompt cache is 512 MiB on a Mac below 32 GB and 2 GiB from 32 GB. + * The catalogue that ships with Codeman. Never read directly outside this file and its test: + * callers use `getLocalModelCatalog()`, which validates it first. */ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { schema: 1, updated: '2026-11-06', + // The oldest macOS the model runtime supports. minMacosMajor: 14, + // Host memory the runtime's prompt cache may take beside the model: 512 MiB on a Mac below + // 32 GB, 2 GiB from 32 GB. Part of the need of every candidate. promptCacheTiers: [ - { minMemGb: 0, bytes: 512 * 1024 ** 2 }, - { minMemGb: 32, bytes: 2 * 1024 ** 3 }, + { minMemGb: 0, bytes: 512 * MIB }, + { minMemGb: 32, bytes: 2 * GIB }, + ], + // Ordered by rank. The higher rank wins among models that fit equally well and are equally + // fast. `flint:40b` is left out on purpose: its plain tag is the slow build on a Mac and it + // has an open bug with tool arguments. + models: [ + { + id: 'qwen3.5:9b', + label: 'Qwen3.5 9B', + license: 'Apache-2.0', + paramsB: 9, + bytes: 6_550_825_373, + kvBytesPerToken: 32 * KIB, + contexts: [65536, 32768], + rank: 1, + // 37.1 tokens per second measured on 400 GB/s. + speedPer100Gbps: 9.3, + tested: [], + }, + { + id: 'qwen3.5:9b-q8_0', + label: 'Qwen3.5 9B (Q8)', + license: 'Apache-2.0', + paramsB: 9, + bytes: 10_449_217_949, + kvBytesPerToken: 32 * KIB, + contexts: [65536, 32768], + rank: 2, + // Not measured: the 9.3 of the row above scaled by file size. + speedPer100Gbps: 5.8, + tested: [], + }, + { + id: 'gpt-oss:20b', + label: 'GPT-OSS 20B', + license: 'Apache-2.0', + paramsB: 20, + bytes: 13_793_441_427, + kvBytesPerToken: 24 * KIB, + contexts: [131072, 65536], + rank: 3, + // No speed figure: its speed is unknown on every chip. + tested: [], + }, + { + id: 'qwen3.6:35b-a3b', + label: 'Qwen3.6 35B-A3B', + license: 'Apache-2.0', + paramsB: 35, + bytes: 22_621_314_161, + kvBytesPerToken: 20 * KIB, + contexts: [131072, 65536], + rank: 4, + // 54.5 tokens per second measured on 400 GB/s (3 billion parameters are active per token). + speedPer100Gbps: 13.6, + tested: [], + }, + { + id: 'qwen3.8:27b', + label: 'Qwen3.8 27B', + license: 'Apache-2.0', + paramsB: 27.8, + bytes: 17_741_871_939, + kvBytesPerToken: 64 * KIB, + contexts: [131072, 65536, 32768], + rank: 5, + // 11.8 tokens per second measured on 400 GB/s. + speedPer100Gbps: 2.95, + tested: [], + }, + { + id: 'qwen3.8:27b-q8_0', + label: 'Qwen3.8 27B (Q8)', + license: 'Apache-2.0', + paramsB: 27.8, + bytes: 29_978_241_837, + kvBytesPerToken: 64 * KIB, + contexts: [262144, 131072], + rank: 6, + // Not measured: the 2.95 of the row above scaled by file size. + speedPer100Gbps: 1.7, + tested: [], + }, + { + id: 'qwen3.5:122b-a10b', + label: 'Qwen3.5 122B-A10B', + license: 'Apache-2.0', + paramsB: 122, + bytes: 81_370_035_877, + kvBytesPerToken: 24 * KIB, + contexts: [131072], + rank: 7, + // No speed figure: its speed is unknown on every chip. + tested: [], + }, + ], + // The higher rank wins when a model can run with more than one harness. + harnesses: [ + { cli: 'opencode', minContext: 65536, rank: 3 }, + // Sends a fixed prompt of about 36,000 tokens, so the first answer takes minutes. + { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true }, + // Gated: neither its 32768 minimum nor its handling of a `name:tag` model id is verified. + { cli: 'pi', minContext: 32768, rank: 1, gated: true }, + ], + // Memory bandwidth in GB/s, most specific first. Only chips that shipped and are in the source + // table are listed. Where one name covers two bandwidths (M3 Max 300 or 400, M4 Max 410 or 546, + // M5 Max 460 or 614) the LOWER one stands, so a speed computed from it is a floor. + chips: [ + { match: 'M5 Ultra', gbps: 1228 }, + { match: 'M3 Ultra', gbps: 800 }, + { match: 'M2 Ultra', gbps: 800 }, + { match: 'M1 Ultra', gbps: 800 }, + { match: 'M5 Max', gbps: 460 }, + { match: 'M4 Max', gbps: 410 }, + { match: 'M3 Max', gbps: 300 }, + { match: 'M2 Max', gbps: 400 }, + { match: 'M1 Max', gbps: 400 }, + { match: 'M5 Pro', gbps: 307 }, + { match: 'M4 Pro', gbps: 273 }, + { match: 'M3 Pro', gbps: 150 }, + { match: 'M2 Pro', gbps: 200 }, + { match: 'M1 Pro', gbps: 200 }, + { match: 'M5', gbps: 154 }, + { match: 'M4', gbps: 120 }, + { match: 'M3', gbps: 100 }, + { match: 'M2', gbps: 100 }, + { match: 'M1', gbps: 68 }, ], - models: [], - harnesses: [], - chips: [], }; +let validated: LocalModelCatalog | undefined; + /** - * The catalogue in force. There is no override in this stage, so this is the stock constant; - * callers go through this function anyway, so a later stage can change where it comes from - * without touching them. + * The catalogue in force: the stock constant, validated once on first use. Callers go through + * this function, so a later stage can change where the catalogue comes from without touching + * them. Throws when the constant breaks its own schema, which the catalogue test rules out. */ export function getLocalModelCatalog(): LocalModelCatalog { - return STOCK_LOCAL_MODEL_CATALOG; + validated ??= LocalModelCatalogSchema.parse(STOCK_LOCAL_MODEL_CATALOG); + return validated; } diff --git a/src/local-model/hardware.ts b/src/local-model/hardware.ts index 687d981a..5c49cfb7 100644 --- a/src/local-model/hardware.ts +++ b/src/local-model/hardware.ts @@ -1,8 +1,6 @@ /** * @fileoverview The hardware probe of the local model recommendation: what this Mac is. * - * SKELETON: the exported signatures are final, the bodies are not written yet. This module - * will hold: * - `parseSysctl()`: a pure parser of `key: value` lines. The command runs WITHOUT `-n`, so a * key that is missing on a machine (`sysctl.proc_translated` on Intel) cannot shift the * others, and a line it does not know is ignored. @@ -12,17 +10,30 @@ * * Rules: * - Never throws and never rejects: a failed probe is an answer (`source: 'os-fallback'`), and - * off macOS nothing is spawned (`source: 'not-probed'`). + * off macOS nothing is spawned and no disk is read (`source: 'not-probed'`). * - Never `system_profiler` (tens of milliseconds and a parse of free text), never a second * spawn, never a synchronous exec on the server's event loop. * - A non-zero exit is tolerated once `hw.memsize` parsed (a missing key makes sysctl exit 1). + * Without `hw.memsize` nothing else of that output is trusted: the answer is the fallback. + * - Only a line that ends in a newline counts. A cut-off tail is not a value: half of + * `hw.memsize` would read as a smaller, valid number. * - `iogpu.wired_limit_mb` of 0 means "not set": it is reported as 0 and only a value above 0 * is ever used as a limit. + * - The data dir can sit on a network mount that stopped answering, so the disk read is + * bounded too: the folder is probed with the bounded path probe first, the `statfs` is raced + * against a timeout, and a `statfs` that never came back is not started a second time. * - `memGb`, `memClassGb` and `freeDiskGb` are filled here through src/local-model/format.ts, * the one place a size is formatted. */ +import { execFile } from 'node:child_process'; +import { statfs } from 'node:fs/promises'; +import os from 'node:os'; +import { dirname } from 'node:path'; import type { HardwareFacts } from '../types.js'; +import { getDataDir } from '../config/instance.js'; +import { probePathKind, type PathProbeKind } from '../utils/bounded-path-probe.js'; +import { formatGb, memClassGb } from './format.js'; /** Absolute path, so the server's PATH (minimal under a service manager) cannot change what runs. */ export const SYSCTL_PATH = '/usr/sbin/sysctl'; @@ -44,6 +55,18 @@ export const SYSCTL_TIMEOUT_MS = 2000; /** How long one probe result is reused. */ export const HARDWARE_CACHE_TTL_MS = 60_000; +/** Upper bound of the wait for the free disk figure. */ +export const FREE_DISK_TIMEOUT_MS = 2000; + +/** Seven short lines are well under this. A larger output is not sysctl's. */ +const SYSCTL_MAX_OUTPUT_BYTES = 64 * 1024; + +/** Longest text value kept (a chip name, a model identifier, a version). */ +const MAX_TEXT_VALUE_LENGTH = 100; + +/** How many parent folders are tried when the data dir does not exist yet. */ +const FREE_DISK_MAX_PARENT_STEPS = 6; + /** What `parseSysctl()` read. Every field is absent when its key was missing or its value did not parse. */ export interface SysctlFacts { /** `hw.memsize`. */ @@ -82,25 +105,305 @@ export interface HardwareProbeDeps { now?: () => number; } +/** A whole number without sign, as sysctl prints one. */ +function parseCount(value: string): number | undefined { + if (!/^\d+$/.test(value)) return undefined; + const parsed = Number(value); + return Number.isSafeInteger(parsed) ? parsed : undefined; +} + +/** sysctl prints a flag as 0 or 1. Anything else is not a flag. */ +function parseFlag(value: string): boolean | undefined { + if (value === '1') return true; + if (value === '0') return false; + return undefined; +} + +function parseText(value: string): string | undefined { + return value !== '' && value.length <= MAX_TEXT_VALUE_LENGTH ? value : undefined; +} + /** * Parse the output of `sysctl ` (no `-n`): one `key: value` line per key it knows. - * Pure. Unknown lines, blank lines and error lines are skipped. + * Pure. Unknown lines, blank lines and error lines are skipped, and so is a last line that does + * not end in a newline. The first line of a key wins. */ export function parseSysctl(text: string): SysctlFacts { - void text; - throw new Error('not implemented: parseSysctl'); + const facts: SysctlFacts = {}; + const lines = text.split('\n'); + // What follows the last newline is either empty or a cut-off line. + lines.pop(); + for (const rawLine of lines) { + const line = rawLine.replace(/\r$/, ''); + const colon = line.indexOf(':'); + if (colon <= 0) continue; + const key = line.slice(0, colon).trim(); + const value = line.slice(colon + 1).trim(); + switch (key) { + case 'hw.memsize': { + const bytes = parseCount(value); + if (bytes !== undefined && bytes > 0) facts.memBytes ??= bytes; + break; + } + case 'machdep.cpu.brand_string': + facts.chip ??= parseText(value); + break; + case 'hw.optional.arm64': + facts.arm64 ??= parseFlag(value); + break; + case 'sysctl.proc_translated': + facts.translated ??= parseFlag(value); + break; + case 'iogpu.wired_limit_mb': + facts.wiredLimitMb ??= parseCount(value); + break; + case 'kern.osproductversion': { + const version = /^(\d+)(?:\.\d+)*$/.exec(value); + if (version && value.length <= MAX_TEXT_VALUE_LENGTH && facts.macosVersion === undefined) { + facts.macosVersion = value; + facts.macosMajor = Number(version[1]); + } + break; + } + case 'hw.model': + facts.model ??= parseText(value); + break; + default: + break; + } + } + return facts; } +/** + * The one spawn: `command` with `SYSCTL_KEYS` as its arguments and nothing else (no `-n`). + * Resolves stdout when the program ran, also after a non-zero exit (a key this machine lacks + * makes sysctl exit 1 while the other keys are printed). Resolves `undefined` when it could not + * be started, was killed (the timeout) or printed too much. Never rejects. + * + * The two parameters exist for the test, which runs a script of its own in place of sysctl. + */ +export function spawnSysctl( + command: string = SYSCTL_PATH, + timeoutMs: number = SYSCTL_TIMEOUT_MS +): Promise { + return new Promise((resolve) => { + try { + execFile( + command, + [...SYSCTL_KEYS], + { + timeout: timeoutMs, + maxBuffer: SYSCTL_MAX_OUTPUT_BYTES, + encoding: 'utf8', + windowsHide: true, + // sysctl needs nothing from the server's environment, which can hold a password. + env: { LC_ALL: 'C' }, + }, + (error, stdout) => { + if (!error) return resolve(stdout); + // A numeric code is an exit status: the program ran. A string code (ENOENT, EACCES, + // the buffer limit) or a signal means it did not run to its end. + const ranToEnd = typeof error.code === 'number' && !error.killed && !error.signal; + resolve(ranToEnd ? stdout : undefined); + } + ); + } catch { + resolve(undefined); + } + }); +} + +/** + * Free bytes on the filesystem that holds `path` (`bavail * bsize`: what an unprivileged + * process can use), or `undefined` when the filesystem does not say. Not bounded by itself: see + * `dataDirFreeBytes()`. + */ +export async function freeDiskBytesOf(path: string): Promise { + try { + const stat = await statfs(path); + const free = Number(stat.bavail) * Number(stat.bsize); + return Number.isFinite(free) && free >= 0 ? free : undefined; + } catch { + return undefined; + } +} + +/** The one `statfs` that may be pending. While it is, no second one is started. */ +let pendingStatfs: Promise | undefined; + +/** The IO of `dataDirFreeBytes()`. Each member defaults to the real thing; a test passes its own. */ +export interface FreeDiskDeps { + /** Default `getDataDir()`. */ + dataDir?: () => string; + /** Default the bounded path probe (`probePathKind`). */ + probeKind?: (path: string) => Promise; + /** Default `freeDiskBytesOf`. */ + freeBytesOf?: (path: string) => Promise; + /** Default `FREE_DISK_TIMEOUT_MS`. */ + timeoutMs?: number; +} + +/** + * Free bytes on the data dir's filesystem, bounded. The folder (or, when it does not exist yet, + * its nearest existing parent) is found with the bounded path probe, which never waits on a + * dead mount twice; the `statfs` itself is raced against a timeout. `undefined` when the + * filesystem does not answer in time, or is not known to be a folder. + */ +export async function dataDirFreeBytes(deps: FreeDiskDeps = {}): Promise { + // Concurrent probes share one promise in `readHardware()`, so a `statfs` that is still pending + // here was started by an EARLIER probe and has outlived its timeout: a dead mount. It holds + // one threadpool worker, and that is the most this module may ever hold. + if (pendingStatfs) return undefined; + + const probeKind = deps.probeKind ?? probePathKind; + let folder: string; + try { + folder = (deps.dataDir ?? getDataDir)(); + let kind = await probeKind(folder); + for (let step = 0; kind === 'absent' && step < FREE_DISK_MAX_PARENT_STEPS; step++) { + const parent = dirname(folder); + if (parent === folder) break; + folder = parent; + kind = await probeKind(folder); + } + // 'unknown' is a mount that does not answer (or an unreadable path): do not touch it. + if (kind !== 'directory') return undefined; + } catch { + return undefined; + } + + const started = (deps.freeBytesOf ?? freeDiskBytesOf)(folder).catch(() => undefined); + pendingStatfs = started; + const forget = (): void => { + if (pendingStatfs === started) pendingStatfs = undefined; + }; + started.then(forget, forget); + + let timer: ReturnType | undefined; + try { + return await Promise.race([ + started, + new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), deps.timeoutMs ?? FREE_DISK_TIMEOUT_MS); + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +function defaultCpuModel(): string | undefined { + return os.cpus()[0]?.model; +} + +/** A dependency that must never break the probe: its value, or `undefined` when it threw or rejected. */ +async function attempt(read: () => T | Promise): Promise { + try { + return await read(); + } catch { + return undefined; + } +} + +/** Memory and the three fields derived from it. */ +function memoryFields(memBytes: number | undefined): Pick { + const bytes = memBytes !== undefined && Number.isFinite(memBytes) && memBytes > 0 ? memBytes : 0; + return { memBytes: bytes, memGb: formatGb(bytes), memClassGb: memClassGb(bytes) }; +} + +async function probeHardware(deps: HardwareProbeDeps): Promise { + const platform = deps.platform ?? process.platform; + const arch = deps.arch ?? os.arch(); + const totalMemBytes = deps.totalMemBytes ?? os.totalmem; + const cpuModel = deps.cpuModel ?? defaultCpuModel; + + if (platform !== 'darwin') { + return { platform, arch, ...memoryFields(await attempt(totalMemBytes)), source: 'not-probed' }; + } + + const [stdout, freeDisk] = await Promise.all([ + attempt(deps.runSysctl ?? (() => spawnSysctl())), + attempt(deps.freeDiskBytes ?? (() => dataDirFreeBytes())), + ]); + const disk: Pick = + freeDisk !== undefined && Number.isFinite(freeDisk) && freeDisk >= 0 + ? { freeDiskBytes: freeDisk, freeDiskGb: formatGb(freeDisk) } + : {}; + + const facts = typeof stdout === 'string' ? parseSysctl(stdout) : {}; + if (facts.memBytes === undefined) { + const chip = await attempt(cpuModel); + return { + platform, + arch, + ...memoryFields(await attempt(totalMemBytes)), + ...(chip ? { chip } : {}), + ...disk, + source: 'os-fallback', + }; + } + + const chip = facts.chip ?? (await attempt(cpuModel)); + return { + platform, + arch, + ...(facts.arm64 !== undefined ? { arm64: facts.arm64 } : {}), + ...(facts.translated !== undefined ? { translated: facts.translated } : {}), + ...memoryFields(facts.memBytes), + ...(chip ? { chip } : {}), + ...(facts.model !== undefined ? { model: facts.model } : {}), + ...(facts.macosVersion !== undefined ? { macosVersion: facts.macosVersion } : {}), + ...(facts.macosMajor !== undefined ? { macosMajor: facts.macosMajor } : {}), + ...(facts.wiredLimitMb !== undefined ? { wiredLimitMb: facts.wiredLimitMb } : {}), + ...disk, + source: 'sysctl', + }; +} + +let cached: { at: number; facts: HardwareFacts } | undefined; +let inFlight: Promise | undefined; + /** * What this machine is. Cached for `HARDWARE_CACHE_TTL_MS`; concurrent callers share one probe. * Never rejects. + * + * The cache is one per process and does not look at `deps`: within the 60 s a second call + * returns the first call's answer whatever it injects. A test resets it first. */ -export async function readHardware(deps?: HardwareProbeDeps): Promise { - void deps; - throw new Error('not implemented: readHardware'); +export async function readHardware(deps: HardwareProbeDeps = {}): Promise { + const now = deps.now ?? Date.now; + if (cached) { + const age = now() - cached.at; + if (age >= 0 && age < HARDWARE_CACHE_TTL_MS) return cached.facts; + } + if (inFlight) return inFlight; + + const probe = probeHardware(deps) + .catch( + // Unreachable by design (every read above is guarded), and still an answer if it is reached. + (): HardwareFacts => ({ + platform: deps.platform ?? process.platform, + arch: deps.arch ?? os.arch(), + ...memoryFields(undefined), + source: (deps.platform ?? process.platform) === 'darwin' ? 'os-fallback' : 'not-probed', + }) + ) + .then((facts) => { + cached = { at: now(), facts }; + return facts; + }) + .finally(() => { + if (inFlight === probe) inFlight = undefined; + }); + inFlight = probe; + return probe; } -/** Drop the cached probe result (tests, and nothing else). */ +/** Drop the cached probe result and the memory of a pending disk read (tests, and nothing else). */ export function resetHardwareCache(): void { - throw new Error('not implemented: resetHardwareCache'); + cached = undefined; + inFlight = undefined; + pendingStatfs = undefined; } diff --git a/src/local-model/recommend.ts b/src/local-model/recommend.ts index 2536e63c..a0513093 100644 --- a/src/local-model/recommend.ts +++ b/src/local-model/recommend.ts @@ -2,54 +2,231 @@ * @fileoverview The recommendation itself: which model, at which context, with which harness * fits this Mac. Pure: no IO, no clock, no CLI registry, no data dir. * - * SKELETON: the exported signatures are final, the bodies are not written yet. This module - * will hold: + * The math, in the order it runs: * - `deviceBudget()`: L is the wired limit when it is above 0, else 2/3 of memory below 36 GB * and 3/4 from 36 GB. B = min(L - 1 GiB, memory - reserve), with a reserve of 6 GiB up to - * 18 GB, 8 GiB up to 48 GB and 12 GiB from 64 GB. + * 18 GB, 8 GiB up to 48 GB and 12 GiB above. * - `needBytes()`: F = weights + kvBytesPerToken * ctx + 1 GiB + the prompt cache tier of the * catalogue for this Mac's memory. - * - `recommend()`: the candidates (model, context, harness) with `ctx >= harness.minContext`, - * the harness installed or installable, accepting the model id and not gated; the fit (F/B up - * to 0.85 good, up to 0.98 tight, above out); the speed (ok, slow or unknown: a chip that is - * in no chips row is unknown, a bandwidth is never guessed); the ranking (good before tight, - * then ok before unknown before slow, then model rank, harness rank, larger context); and the - * verdict. + * - Fit: F/B up to 0.85 is good, up to 0.98 is tight, above is out. + * - Speed: tokens per second = the model's `speedPer100Gbps` x the chip's bandwidth / 100. + * Below 10 is slow. A chip that is in no chips row, or a model without a speed figure, is + * UNKNOWN: a bandwidth is never guessed. + * - Candidates: (model, context, harness) with `ctx >= harness.minContext`, the harness not + * gated, installed or installable, and accepting the model id. + * - Ranking, lexicographic: good before tight; ok before unknown before slow; the higher model + * rank; the higher harness rank; the larger context. The pick is the first candidate, the + * alternatives are the next four MODELS, each at its own best context and harness. + * + * Verdict `none`, in this order (the first that applies is the reason): + * 1. `not-macos`: the server does not run on macOS. + * 2. `intel`: a Mac without Apple silicon. + * 3. `macos`: macOS is older than the catalogue's minimum (an unknown version does not block). + * 4. `memory`: no model fits the budget at any of its contexts, whatever the harness. + * 5. `no-harness`: no harness row is usable (not gated, known to the caller, installed or + * installable). + * 6. `memory`: models fit, but none at a context a usable harness needs. + * 7. `no-harness`: models fit at such a context, but no usable harness accepts their id. + * 8. `disk`: for every remaining model, free disk minus the download is below 10 GiB (an + * unknown free disk does not block). * * Rules: - * - Verdict `limited` when the best candidate's model has fewer than 12 billion parameters. - * - The word "Recommended" (`LOCAL_MODEL_LABELS.headlineTested`) only for a pick whose model - * has a `tested` entry matching this Mac. Otherwise `headlineOk` plus `noteUntested`. - * - Reasons for `none`: not-macos, intel, memory, disk (free disk minus the download is below - * 10 GiB for every fitting model), macos, no-harness. + * - Verdict `limited` when the pick's model has fewer than 12 billion parameters. + * - The word "Recommended" (`LOCAL_MODEL_LABELS.headlineTested`) only for verdict `ok` with a + * pick whose model has a `tested` entry for exactly this chip, this memory size and this + * harness. Otherwise `headlineOk` (or `headlineLimited`) plus `noteUntested`. * - Every label in the result comes from `LOCAL_MODEL_LABELS`, every size text from * `formatGb()`. No number is spliced into a sentence here. * - No CLI id is ever compared with a literal: a harness is a catalogue row plus a * `HarnessFact`. */ -import type { - DeviceBudget, - HardwareFacts, - HarnessFact, - LocalModelCatalog, - LocalModelEntry, - Recommendation, +import { + LOCAL_MODEL_LABELS, + LOCAL_MODEL_NONE_LABELS, + type Candidate, + type DeviceBudget, + type FitGrade, + type HardwareFacts, + type HarnessFact, + type LocalHarnessEntry, + type LocalModelCatalog, + type LocalModelEntry, + type NoneReason, + type Recommendation, + type SpeedGrade, } from '../types.js'; +import { chipBandwidthGbps } from '../config/local-model-catalog.js'; +import { formatGb, GIB, MIB } from './format.js'; + +/** Need over budget up to this share is a `good` fit. */ +export const FIT_GOOD_MAX_SHARE = 0.85; +/** Need over budget up to this share is a `tight` fit; above it the model is out. */ +export const FIT_TIGHT_MAX_SHARE = 0.98; +/** An estimated speed below this many tokens per second is `slow`. */ +export const SLOW_BELOW_TOKENS_PER_SECOND = 10; +/** A pick whose model has fewer than this many billion parameters makes the verdict `limited`. */ +export const LIMITED_BELOW_PARAMS_B = 12; +/** What must stay free on the disk after the model download. */ +export const DISK_HEADROOM_BYTES = 10 * GIB; +/** What the runtime needs beside the weights, the KV cache and the prompt cache (compute buffers). */ +export const RUNTIME_OVERHEAD_BYTES = GIB; +/** How many other models are listed under the pick. */ +export const MAX_ALTERNATIVES = 4; + +/** A memory size of at least this many GB (as sold) gets 3/4 of its memory as device limit, below it 2/3. */ +const LARGE_LIMIT_FROM_GB = 36; + +/** The memory the system, Codeman, a browser and the agent keep for themselves, by the size the Mac is sold as. */ +function reserveBytes(memClassGb: number): number { + if (memClassGb <= 18) return 6 * GIB; + if (memClassGb <= 48) return 8 * GIB; + return 12 * GIB; +} /** The memory a model may use on this Mac: the device limit L, the budget B and where L came from. */ export function deviceBudget(hw: HardwareFacts): DeviceBudget { - void hw; - throw new Error('not implemented: deviceBudget'); + const memBytes = Number.isFinite(hw.memBytes) && hw.memBytes > 0 ? hw.memBytes : 0; + // 0 means "not set" (both test Macs report 0): only a value above 0 is a limit. + const wired = hw.wiredLimitMb !== undefined && Number.isFinite(hw.wiredLimitMb) && hw.wiredLimitMb > 0; + const limitBytes = wired + ? Math.floor((hw.wiredLimitMb as number) * MIB) + : Math.floor(memBytes * (hw.memClassGb < LARGE_LIMIT_FROM_GB ? 2 / 3 : 3 / 4)); + const budgetBytes = Math.max(0, Math.min(limitBytes - GIB, memBytes - reserveBytes(hw.memClassGb))); + return { limitBytes, budgetBytes, source: wired ? 'wired-limit' : 'estimated' }; +} + +/** The prompt cache of the last tier whose minimum is not above this Mac's memory size. */ +function promptCacheBytes(catalog: LocalModelCatalog, memClassGb: number): number { + let bytes = 0; + for (const tier of catalog.promptCacheTiers) { + if (tier.minMemGb <= memClassGb) bytes = tier.bytes; + } + return bytes; } /** F: the bytes `model` needs in memory at a context of `ctx` tokens on this Mac. */ export function needBytes(model: LocalModelEntry, ctx: number, catalog: LocalModelCatalog, hw: HardwareFacts): number { - void model; - void ctx; - void catalog; - void hw; - throw new Error('not implemented: needBytes'); + return model.bytes + model.kvBytesPerToken * ctx + RUNTIME_OVERHEAD_BYTES + promptCacheBytes(catalog, hw.memClassGb); +} + +function fitGrade(need: number, budgetBytes: number): FitGrade { + if (budgetBytes <= 0) return 'out'; + const share = need / budgetBytes; + if (share <= FIT_GOOD_MAX_SHARE) return 'good'; + if (share <= FIT_TIGHT_MAX_SHARE) return 'tight'; + return 'out'; +} + +/** + * Whether the MACHINE is Apple silicon. `hw.optional.arm64` decides when it was read. Without + * it: a process under Rosetta, an arm64 process or an Apple chip name each prove Apple silicon + * (`arch` alone cannot say no, because Node under Rosetta reports x64). + */ +function isAppleSilicon(hw: HardwareFacts): boolean { + if (hw.arm64 !== undefined) return hw.arm64; + if (hw.translated === true || hw.arch === 'arm64') return true; + return /^Apple\b/i.test(hw.chip ?? ''); +} + +function round(value: number, decimals: number): number { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +} + +/** A candidate plus the two ranks the ordering needs, which are not part of the payload. */ +interface Ranked { + candidate: Candidate; + modelRank: number; + harnessRank: number; +} + +/** A catalogue harness row with the caller's fact about it. */ +interface UsableHarness { + row: LocalHarnessEntry; + fact: HarnessFact; +} + +const FIT_ORDER: Record = { good: 0, tight: 1 }; +const SPEED_ORDER: Record = { ok: 0, unknown: 1, slow: 2 }; + +/** Best first. */ +function compareRanked(a: Ranked, b: Ranked): number { + return ( + FIT_ORDER[a.candidate.fit] - FIT_ORDER[b.candidate.fit] || + SPEED_ORDER[a.candidate.speed] - SPEED_ORDER[b.candidate.speed] || + b.modelRank - a.modelRank || + b.harnessRank - a.harnessRank || + b.candidate.context - a.candidate.context + ); +} + +/** A function supplied by the caller must not be able to break a pure answer. */ +function accepts(fact: HarnessFact, modelId: string): boolean { + try { + return fact.acceptsModelId(modelId) === true; + } catch { + return false; + } +} + +function buildCandidate( + model: LocalModelEntry, + ctx: number, + harness: UsableHarness, + need: number, + fit: Candidate['fit'], + hw: HardwareFacts, + catalog: LocalModelCatalog, + budgetBytes: number +): Candidate { + const gbps = chipBandwidthGbps(catalog.chips, hw.chip); + const tokensPerSecond = + gbps !== undefined && model.speedPer100Gbps !== undefined + ? round((model.speedPer100Gbps * gbps) / 100, 1) + : undefined; + const speed: SpeedGrade = + tokensPerSecond === undefined ? 'unknown' : tokensPerSecond < SLOW_BELOW_TOKENS_PER_SECOND ? 'slow' : 'ok'; + // "A Mac like this" is exact: the same chip string, the same memory size as sold, the same harness. + const tested = model.tested.some( + (entry) => entry.chip === hw.chip && entry.memGb === hw.memClassGb && entry.cli === harness.row.cli + ); + return { + modelId: model.id, + modelLabel: model.label, + license: model.license, + paramsB: model.paramsB, + context: ctx, + cli: harness.row.cli, + cliLabel: harness.fact.label, + cliInstalled: harness.fact.installed, + ...(harness.row.slowFirstTurn ? { slowFirstTurn: true } : {}), + needBytes: need, + needGb: formatGb(need), + downloadBytes: model.bytes, + downloadGb: formatGb(model.bytes), + budgetShare: round(need / budgetBytes, 3), + fit, + fitLabel: fit === 'good' ? LOCAL_MODEL_LABELS.fitGood : LOCAL_MODEL_LABELS.fitTight, + speed, + ...(speed === 'slow' ? { speedLabel: LOCAL_MODEL_LABELS.speedSlow } : {}), + ...(speed === 'unknown' ? { speedLabel: LOCAL_MODEL_LABELS.speedUnknown } : {}), + ...(tokensPerSecond !== undefined ? { tokensPerSecond } : {}), + tested, + }; +} + +function none(reason: NoneReason, budget: DeviceBudget, vars?: Record): Recommendation { + return { + verdict: 'none', + reason, + headline: { text: LOCAL_MODEL_NONE_LABELS[reason], ...(vars ? { vars } : {}) }, + notes: [], + alternatives: [], + budgetBytes: budget.budgetBytes, + budgetGb: formatGb(budget.budgetBytes), + budgetSource: budget.source, + }; } /** @@ -61,8 +238,99 @@ export function recommend( catalog: LocalModelCatalog, harnessFacts: readonly HarnessFact[] ): Recommendation { - void hw; - void catalog; - void harnessFacts; - throw new Error('not implemented: recommend'); + // Off macOS and on an Intel Mac there is no budget to compute: a model has no memory to run in. + const noBudget: DeviceBudget = { limitBytes: 0, budgetBytes: 0, source: 'estimated' }; + if (hw.platform !== 'darwin') return none('not-macos', noBudget); + if (!isAppleSilicon(hw)) return none('intel', noBudget); + + const budget = deviceBudget(hw); + if (hw.macosMajor !== undefined && hw.macosMajor < catalog.minMacosMajor) { + return none('macos', budget, { + version: hw.macosVersion ?? String(hw.macosMajor), + minimum: String(catalog.minMacosMajor), + }); + } + + // Every (model, context) that fits the budget, before any harness is looked at. + const fitting: { model: LocalModelEntry; ctx: number; need: number; fit: Candidate['fit'] }[] = []; + for (const model of catalog.models) { + for (const ctx of model.contexts) { + const need = needBytes(model, ctx, catalog, hw); + const fit = fitGrade(need, budget.budgetBytes); + if (fit !== 'out') fitting.push({ model, ctx, need, fit }); + } + } + const tooLittleMemory = (): Recommendation => none('memory', budget, { memory: hw.memGb }); + if (fitting.length === 0) return tooLittleMemory(); + + // A gated row is never offered, whatever the caller knows about its CLI. + const usable: UsableHarness[] = []; + for (const row of catalog.harnesses) { + if (row.gated) continue; + const fact = harnessFacts.find((candidate) => candidate.cli === row.cli); + if (fact && (fact.installed || fact.installable)) usable.push({ row, fact }); + } + if (usable.length === 0) return none('no-harness', budget); + + const atHarnessContext = fitting.flatMap((entry) => + usable.filter((harness) => entry.ctx >= harness.row.minContext).map((harness) => ({ ...entry, harness })) + ); + if (atHarnessContext.length === 0) return tooLittleMemory(); + + const accepted = atHarnessContext.filter((entry) => accepts(entry.harness.fact, entry.model.id)); + if (accepted.length === 0) return none('no-harness', budget); + + // The download must leave headroom on the disk. An unknown free disk does not block. + const free = hw.freeDiskBytes; + const downloadable = + free === undefined ? accepted : accepted.filter((entry) => free - entry.model.bytes >= DISK_HEADROOM_BYTES); + if (free !== undefined && downloadable.length === 0) { + const smallest = Math.min(...accepted.map((entry) => entry.model.bytes)); + return none('disk', budget, { + download: formatGb(smallest), + free: hw.freeDiskGb ?? formatGb(free), + missing: formatGb(smallest + DISK_HEADROOM_BYTES - free), + }); + } + + const ranked: Ranked[] = downloadable + .map((entry) => ({ + candidate: buildCandidate( + entry.model, + entry.ctx, + entry.harness, + entry.need, + entry.fit, + hw, + catalog, + budget.budgetBytes + ), + modelRank: entry.model.rank, + harnessRank: entry.harness.row.rank, + })) + .sort(compareRanked); + + // The first candidate of each model is that model at its best context and harness. + const bestPerModel: Candidate[] = []; + for (const { candidate } of ranked) { + if (!bestPerModel.some((kept) => kept.modelId === candidate.modelId)) bestPerModel.push(candidate); + } + const [pick, ...others] = bestPerModel; + + const limited = pick.paramsB < LIMITED_BELOW_PARAMS_B; + const headline = limited + ? LOCAL_MODEL_LABELS.headlineLimited + : pick.tested + ? LOCAL_MODEL_LABELS.headlineTested + : LOCAL_MODEL_LABELS.headlineOk; + return { + verdict: limited ? 'limited' : 'ok', + headline: { text: headline }, + notes: pick.tested ? [] : [{ text: LOCAL_MODEL_LABELS.noteUntested }], + pick, + alternatives: others.slice(0, MAX_ALTERNATIVES), + budgetBytes: budget.budgetBytes, + budgetGb: formatGb(budget.budgetBytes), + budgetSource: budget.source, + }; } diff --git a/src/types/local-model.ts b/src/types/local-model.ts index 5298c3bb..77352d64 100644 --- a/src/types/local-model.ts +++ b/src/types/local-model.ts @@ -129,11 +129,14 @@ export interface LocalHarnessEntry { /** * Memory bandwidth of one chip family. `LocalModelCatalog.chips` is ordered most specific first - * ('M4 Max' before 'M4') and the first row whose `match` occurs in `HardwareFacts.chip` wins. A - * chip that matches no row has an UNKNOWN speed: a bandwidth is never guessed. + * ('M4 Max' before 'M4') and the first row that names `HardwareFacts.chip` wins. A row names a + * chip when the chip string ENDS with its `match`, starting at the beginning or after a space, + * case ignored (`chipNameMatches()` in src/config/local-model-catalog.ts): 'M4' is not the chip + * of 'Apple M4 Pro', and a variant the table has never heard of ('Apple M6 Pro') matches no row. + * A chip that matches no row has an UNKNOWN speed: a bandwidth is never guessed. */ export interface ChipEntry { - /** Text that must occur in the chip string ('M4 Max'). */ + /** The chip name the chip string must end with ('M4 Max'). */ match: string; /** Memory bandwidth in GB/s. */ gbps: number; @@ -305,7 +308,7 @@ export interface Candidate { downloadBytes: number; /** `downloadBytes` as decimal GB text, no unit. */ downloadGb: string; - /** `needBytes / Recommendation.budgetBytes` (0.83). */ + /** `needBytes / Recommendation.budgetBytes`, rounded to three decimals (0.83). The fit grade is decided on the unrounded ratio. */ budgetShare: number; /** A candidate in a `Recommendation` is never `out`. */ fit: Exclude; @@ -339,7 +342,7 @@ export interface Recommendation { pick?: Candidate; /** The next models that fit (at most four, each at its best context and harness), best first. Empty when `verdict` is `none`. */ alternatives: Candidate[]; - /** `DeviceBudget.budgetBytes`. 0 when the budget could not be computed (not macOS). */ + /** `DeviceBudget.budgetBytes`. 0 when no budget is computed: not macOS, or a Mac without Apple silicon. */ budgetBytes: number; /** `budgetBytes` as decimal GB text, no unit. */ budgetGb: string; diff --git a/test/cli-registry-no-id-branching.test.ts b/test/cli-registry-no-id-branching.test.ts index c746bdfb..cf31be83 100644 --- a/test/cli-registry-no-id-branching.test.ts +++ b/test/cli-registry-no-id-branching.test.ts @@ -167,7 +167,6 @@ const ALLOWED_BRANCHES: Record = { ), "tui/tui-render.ts::mode !== 'claude'": allow(1, 'cosmetic: suppress the mode badge for the default mode'), - // Legacy Config plumbing, same category as the `===` entries above. "web/routes/session-routes.ts::mode !== 'omp'": allow( 2, @@ -212,8 +211,10 @@ const ID_ALT = IDS.join('|'); */ const BRANCH_PATTERN = new RegExp( [ - // mode === 'codex' / mode !== 'codex' - `\\b(?:mode|id|agentType)\\s*[!=]==\\s*'(?:${ID_ALT})'`, + // mode === 'codex' / mode !== 'codex'. `cli` is the name the local model catalogue gives + // the same thing (`LocalHarnessEntry.cli`, `Candidate.cli`), so `harness.cli === 'claude'` + // is the same banned branch under another field name. + `\\b(?:mode|id|agentType|cli)\\s*[!=]==\\s*'(?:${ID_ALT})'`, // case 'codex': `\\bcase\\s+'(?:${ID_ALT})'\\s*:`, // ['codex', 'gemini'].includes(mode) — the id list IS the branch, wherever `mode` sits @@ -267,7 +268,7 @@ function scan(): { findings: Finding[]; filesScanned: number } { lines.forEach((line, i) => { BRANCH_PATTERN.lastIndex = 0; // shared /g regex — see utils/regex-patterns.ts for (const match of line.matchAll(BRANCH_PATTERN)) { - const expression = match[0].replace(/\s+/g, ' ').replace(/^(?:id|agentType)/, 'mode'); + const expression = match[0].replace(/\s+/g, ' ').replace(/^(?:id|agentType|cli)/, 'mode'); const posix = rel.split(sep).join('/'); findings.push({ file: posix, expression, line: i + 1, key: `${posix}::${expression}` }); } @@ -337,6 +338,8 @@ describe('no CLI-id branching outside the stock catalog', () => { "if (mode !== 'shell' && mode !== 'deepseek') { doSomething(); }", "switch (mode) { case 'gemini': return 1; }", "if (['codex', 'gemini'].includes(mode)) { doSomething(); }", + "if (harness.cli === 'claude') { doSomething(); }", + "const others = candidates.filter((c) => c.cli !== 'opencode');", ]; for (const sample of samples) { BRANCH_PATTERN.lastIndex = 0; diff --git a/test/local-model-catalog.test.ts b/test/local-model-catalog.test.ts new file mode 100644 index 00000000..5f72b764 --- /dev/null +++ b/test/local-model-catalog.test.ts @@ -0,0 +1,383 @@ +/** + * @fileoverview Tests for src/config/local-model-catalog.ts: the compiled catalogue of models, + * harnesses and chips, its schema, and the chip lookup. + * + * Three things are pinned: + * + * 1. The stock constant PARSES. `getLocalModelCatalog()` validates it on first use and throws on + * a bad row, so this test is what keeps a bad row from ever reaching the route. + * 2. The schema refuses what must never get in: a model id that could name another registry + * (a slash, a host), an unknown key (a `runtime` block with a download URL), and rows that + * break a rule across rows (a repeated id, contexts in the wrong order, a chip row that + * hides a more specific one). + * 3. The chip lookup never guesses: a chip that no row names has no bandwidth, including one + * whose name merely contains a known name ('Apple M10', 'Apple M6 Pro'). + * + * Port: N/A (pure data and functions). + */ + +import { describe, expect, it } from 'vitest'; +import { + chipBandwidthGbps, + chipNameMatches, + getLocalModelCatalog, + LOCAL_MODEL_ID_PATTERN, + LocalModelCatalogSchema, + STOCK_LOCAL_MODEL_CATALOG, +} from '../src/config/local-model-catalog.js'; +import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js'; + +/** A deep copy of the stock catalogue that a test may break. */ +function copy(): LocalModelCatalog { + return structuredClone(STOCK_LOCAL_MODEL_CATALOG); +} + +/** The message of every issue the schema raises for `catalog`, or an empty list when it parses. */ +function issuesOf(catalog: unknown): string[] { + const result = LocalModelCatalogSchema.safeParse(catalog); + return result.success ? [] : result.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`); +} + +/** The stock catalogue with its first model changed. */ +/** `url` is no field of a model row: one test passes it to prove the schema refuses an unknown key. */ +function withFirstModel(change: Partial>): unknown { + const catalog = copy(); + return { ...catalog, models: [{ ...catalog.models[0], ...change }, ...catalog.models.slice(1)] }; +} + +describe('the stock catalogue', () => { + it('passes its own schema', () => { + expect(issuesOf(STOCK_LOCAL_MODEL_CATALOG)).toEqual([]); + }); + + it('is what the getter returns, validated once', () => { + const first = getLocalModelCatalog(); + expect(first).toEqual(STOCK_LOCAL_MODEL_CATALOG); + // Parsed on first use, then reused: the same object, not a new parse per request. + expect(getLocalModelCatalog()).toBe(first); + }); + + it('lists the seven models of the design, without flint:40b', () => { + expect(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => m.id)).toEqual([ + 'qwen3.5:9b', + 'qwen3.5:9b-q8_0', + 'gpt-oss:20b', + 'qwen3.6:35b-a3b', + 'qwen3.8:27b', + 'qwen3.8:27b-q8_0', + 'qwen3.5:122b-a10b', + ]); + expect(STOCK_LOCAL_MODEL_CATALOG.models.some((m) => m.id.startsWith('flint'))).toBe(false); + }); + + it('carries the byte sizes of the registry manifests', () => { + // The sum of the layer sizes of each tag's manifest, as fetched on 2026-11-06. + expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.bytes]))).toEqual({ + 'qwen3.5:9b': 6_550_825_373, + 'qwen3.5:9b-q8_0': 10_449_217_949, + 'gpt-oss:20b': 13_793_441_427, + 'qwen3.6:35b-a3b': 22_621_314_161, + 'qwen3.8:27b': 17_741_871_939, + 'qwen3.8:27b-q8_0': 29_978_241_837, + 'qwen3.5:122b-a10b': 81_370_035_877, + }); + }); + + it('carries the KV cost per token of each model', () => { + expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.kvBytesPerToken]))).toEqual({ + 'qwen3.5:9b': 32_768, + 'qwen3.5:9b-q8_0': 32_768, + 'gpt-oss:20b': 24_576, + 'qwen3.6:35b-a3b': 20_480, + 'qwen3.8:27b': 65_536, + 'qwen3.8:27b-q8_0': 65_536, + 'qwen3.5:122b-a10b': 24_576, + }); + }); + + it('gives every model a parameter count and an empty tested list', () => { + for (const model of STOCK_LOCAL_MODEL_CATALOG.models) { + expect(model.paramsB, model.id).toBeGreaterThan(0); + // Nobody has run any of these with a harness on a Mac yet. + expect(model.tested, model.id).toEqual([]); + expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN); + expect(model.license).toBe('Apache-2.0'); + } + // The two 9B rows are the ones under the 12 billion line of the `limited` verdict. + expect(STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual([ + 'qwen3.5:9b', + 'qwen3.5:9b-q8_0', + ]); + }); + + it('has a speed figure only where one was measured or scaled from a measurement', () => { + expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({ + 'qwen3.5:9b': 9.3, + 'qwen3.5:9b-q8_0': 5.8, + 'gpt-oss:20b': null, + 'qwen3.6:35b-a3b': 13.6, + 'qwen3.8:27b': 2.95, + 'qwen3.8:27b-q8_0': 1.7, + 'qwen3.5:122b-a10b': null, + }); + }); + + it('lists its harnesses as data rows, one of them gated', () => { + expect(STOCK_LOCAL_MODEL_CATALOG.harnesses).toEqual([ + { cli: 'opencode', minContext: 65536, rank: 3 }, + { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true }, + { cli: 'pi', minContext: 32768, rank: 1, gated: true }, + ]); + }); + + it('counts the prompt cache as 512 MiB below 32 GB and 2 GiB from 32 GB', () => { + expect(STOCK_LOCAL_MODEL_CATALOG.promptCacheTiers).toEqual([ + { minMemGb: 0, bytes: 512 * 1024 ** 2 }, + { minMemGb: 32, bytes: 2 * 1024 ** 3 }, + ]); + expect(STOCK_LOCAL_MODEL_CATALOG.minMacosMajor).toBe(14); + expect(STOCK_LOCAL_MODEL_CATALOG.schema).toBe(1); + }); + + it('holds no download data: no URL, no digest, no runtime block', () => { + const text = JSON.stringify(STOCK_LOCAL_MODEL_CATALOG); + expect(text).not.toMatch(/https?:/); + expect(text).not.toMatch(/sha256/i); + expect(Object.keys(STOCK_LOCAL_MODEL_CATALOG).sort()).toEqual( + ['chips', 'harnesses', 'minMacosMajor', 'models', 'promptCacheTiers', 'schema', 'updated'].sort() + ); + }); +}); + +describe('the model id rule', () => { + it('accepts name:tag in lowercase', () => { + for (const id of ['qwen3.5:9b', 'gpt-oss:20b', 'qwen3.8:27b-q8_0', 'a:b', 'x_y.z-1:2.3_4']) { + expect(LOCAL_MODEL_ID_PATTERN.test(id), id).toBe(true); + } + }); + + it('refuses anything that could name another registry, namespace or no tag at all', () => { + const bad = [ + 'evil.example/x/y:z', + 'library/qwen3.5:9b', + 'registry.ollama.ai/library/qwen3.5:9b', + 'Qwen3.5:9b', + 'qwen3.5:9B', + 'qwen3.5', + 'qwen3.5:', + ':9b', + 'qwen3.5:9b:extra', + 'qwen 3.5:9b', + '-qwen:9b', + 'qwen:-9b', + 'qwen3.5:9b\n', + '', + ]; + for (const id of bad) expect(LOCAL_MODEL_ID_PATTERN.test(id), JSON.stringify(id)).toBe(false); + }); + + it('makes the schema reject a catalogue with such an id', () => { + for (const id of ['evil.example/x/y:z', 'Qwen3.5:9b', 'qwen3.5', 'qwen3.5:9b ']) { + const issues = issuesOf(withFirstModel({ id })); + expect(issues, id).toHaveLength(1); + expect(issues[0]).toContain('models.0.id'); + } + }); +}); + +describe('the schema, field by field', () => { + it('refuses a key it does not know, at the top and inside a row', () => { + // The whole point of having no `runtime` block: a data row can never carry a download URL. + expect(issuesOf({ ...copy(), runtime: { url: 'https://example.com/x.tgz', sha256: 'abc' } })).not.toEqual([]); + expect(issuesOf(withFirstModel({ url: 'https://example.com/model' }))).not.toEqual([]); + const catalog = copy(); + expect(issuesOf({ ...catalog, harnesses: [{ ...catalog.harnesses[0], dialect: 'openai' }] })).not.toEqual([]); + expect(issuesOf({ ...catalog, chips: [{ match: 'M4', gbps: 120, cores: 10 }] })).not.toEqual([]); + }); + + it('requires paramsB and tested on every model row', () => { + for (const field of ['paramsB', 'tested'] as const) { + const row: Record = { ...copy().models[0] }; + delete row[field]; + const issues = issuesOf({ ...copy(), models: [row] }); + expect(issues, field).toHaveLength(1); + expect(issues[0]).toContain(`models.0.${field}`); + } + }); + + it('refuses sizes and counts that are not positive whole numbers', () => { + expect(issuesOf(withFirstModel({ bytes: 0 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ bytes: 1.5 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ kvBytesPerToken: -1 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ contexts: [] }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ paramsB: 0 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ speedPer100Gbps: 0 }))).not.toEqual([]); + }); + + it('accepts a tested entry and checks its shape', () => { + const good = { chip: 'Apple M4', memGb: 16, cli: 'opencode', date: '2026-11-06' }; + expect(issuesOf(withFirstModel({ tested: [good] }))).toEqual([]); + expect(issuesOf(withFirstModel({ tested: [{ ...good, date: '6 November 2026' }] }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ tested: [{ ...good, memGb: 0 }] }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ tested: [{ ...good, result: 'pass' }] }))).not.toEqual([]); + }); + + it('refuses another schema version and a date that is not ISO', () => { + expect(issuesOf({ ...copy(), schema: 2 })).not.toEqual([]); + expect(issuesOf({ ...copy(), updated: 'today' })).not.toEqual([]); + }); +}); + +describe('the schema, across rows', () => { + it('refuses a repeated model id', () => { + const catalog = copy(); + catalog.models[1] = { ...catalog.models[1], id: catalog.models[0].id }; + expect(issuesOf(catalog)).toEqual(['models.1.id: model ids must be unique']); + }); + + it('refuses a repeated model rank, which would leave the order of two models to chance', () => { + const catalog = copy(); + catalog.models[1] = { ...catalog.models[1], rank: catalog.models[0].rank }; + expect(issuesOf(catalog)).toEqual(['models.1.rank: model ranks must be unique']); + }); + + it('refuses a harness listed twice and a repeated harness rank', () => { + const twice = copy(); + twice.harnesses.push({ cli: twice.harnesses[0].cli, minContext: 32768, rank: 9 }); + expect(issuesOf(twice)).toEqual(['harnesses.3.cli: a harness may be listed only once']); + + const sameRank = copy(); + sameRank.harnesses[1] = { ...sameRank.harnesses[1], rank: sameRank.harnesses[0].rank }; + expect(issuesOf(sameRank)).toEqual(['harnesses.1.rank: harness ranks must be unique']); + }); + + it('refuses contexts that are not listed largest first, or that repeat', () => { + expect(issuesOf(withFirstModel({ contexts: [32768, 65536] }))).toEqual([ + 'models.0.contexts.1: contexts must be listed largest first, without repeats', + ]); + expect(issuesOf(withFirstModel({ contexts: [65536, 65536] }))).toEqual([ + 'models.0.contexts.1: contexts must be listed largest first, without repeats', + ]); + }); + + it('refuses a tested entry for a harness the catalogue does not list', () => { + const entry = { chip: 'Apple M4', memGb: 16, cli: 'not-a-harness', date: '2026-11-06' }; + expect(issuesOf(withFirstModel({ tested: [entry] }))).toEqual([ + 'models.0.tested.0.cli: a tested entry must name a harness of this catalogue', + ]); + }); + + it('refuses chips that are not listed most specific first', () => { + const catalog = copy(); + catalog.chips = [ + { match: 'M4', gbps: 120 }, + { match: 'M4 Max', gbps: 410 }, + ]; + expect(issuesOf(catalog)).toEqual([ + "chips.1.match: chips must be listed most specific first: 'M4' comes before 'M4 Max'", + ]); + + const repeated = copy(); + repeated.chips = [ + { match: 'M4', gbps: 120 }, + { match: 'm4', gbps: 130 }, + ]; + expect(issuesOf(repeated)).toHaveLength(1); + }); + + it('refuses prompt cache tiers that do not start at 0 or do not ascend', () => { + const late = copy(); + late.promptCacheTiers = [{ minMemGb: 16, bytes: 1 }]; + expect(issuesOf(late)).toEqual(['promptCacheTiers.0.minMemGb: the first prompt cache tier must start at 0']); + + const unordered = copy(); + unordered.promptCacheTiers = [ + { minMemGb: 0, bytes: 1 }, + { minMemGb: 32, bytes: 2 }, + { minMemGb: 24, bytes: 3 }, + ]; + expect(issuesOf(unordered)).toEqual(['promptCacheTiers.2.minMemGb: prompt cache tiers must ascend by minMemGb']); + }); +}); + +describe('the chips table', () => { + const chips = STOCK_LOCAL_MODEL_CATALOG.chips; + + it('holds every chip of the source table that shipped', () => { + // Bandwidth column of https://github.com/ggml-org/llama.cpp/discussions/4167, read on + // 2026-11-06. A name that covers two bandwidths has the lower one. + expect(Object.fromEntries(chips.map((row) => [row.match, row.gbps]))).toEqual({ + 'M5 Ultra': 1228, + 'M3 Ultra': 800, + 'M2 Ultra': 800, + 'M1 Ultra': 800, + 'M5 Max': 460, + 'M4 Max': 410, + 'M3 Max': 300, + 'M2 Max': 400, + 'M1 Max': 400, + 'M5 Pro': 307, + 'M4 Pro': 273, + 'M3 Pro': 150, + 'M2 Pro': 200, + 'M1 Pro': 200, + M5: 154, + M4: 120, + M3: 100, + M2: 100, + M1: 68, + }); + }); + + it('finds each row under the name macOS reports for that chip', () => { + // No row is hidden by an earlier one: 'Apple M4 Max' is 410, not the 120 of 'M4'. + for (const row of chips) { + expect(chipBandwidthGbps(chips, `Apple ${row.match}`), row.match).toBe(row.gbps); + } + }); + + it('knows the chips of the two test Macs', () => { + expect(chipBandwidthGbps(chips, 'Apple M4')).toBe(120); + expect(chipBandwidthGbps(chips, 'Apple M5 Pro')).toBe(307); + }); + + it('never guesses: a chip that no row names has no bandwidth', () => { + const unknown = [ + // A base chip that is not in the table. + 'Apple M6', + // A variant the table has never heard of must not borrow the figure of its base chip. + 'Apple M6 Pro', + 'Apple M4 Ultra', + 'Apple M5 Extreme', + // A name that merely contains a known name. + 'Apple M10', + 'Apple M12 Pro', + 'Apple M1 Max (Virtual)', + 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz', + 'VirtualApple @ 2.50GHz processor', + 'Max', + '', + ]; + for (const chip of unknown) expect(chipBandwidthGbps(chips, chip), chip).toBeUndefined(); + expect(chipBandwidthGbps(chips, undefined)).toBeUndefined(); + expect(chipBandwidthGbps([], 'Apple M4')).toBeUndefined(); + }); +}); + +describe('chipNameMatches', () => { + it('matches the whole chip name at the end of the string, ignoring case and padding', () => { + expect(chipNameMatches('Apple M4 Pro', 'M4 Pro')).toBe(true); + expect(chipNameMatches('Apple M4', 'M4')).toBe(true); + expect(chipNameMatches('M4', 'M4')).toBe(true); + expect(chipNameMatches(' apple m4 pro ', 'M4 PRO')).toBe(true); + }); + + it('does not match a part of a longer name', () => { + expect(chipNameMatches('Apple M4 Pro', 'M4')).toBe(false); + expect(chipNameMatches('Apple M14', 'M4')).toBe(false); + expect(chipNameMatches('Apple M10', 'M1')).toBe(false); + expect(chipNameMatches('Apple XM4', 'M4')).toBe(false); + expect(chipNameMatches('Apple M4', '')).toBe(false); + expect(chipNameMatches('', 'M4')).toBe(false); + }); +}); diff --git a/test/local-model-format.test.ts b/test/local-model-format.test.ts new file mode 100644 index 00000000..2f3bd3fb --- /dev/null +++ b/test/local-model-format.test.ts @@ -0,0 +1,86 @@ +/** + * @fileoverview Tests for src/local-model/format.ts: the ONE place a byte count becomes a size a + * person reads, and the memory class a Mac is sold as. + * + * What is pinned here is a contract with the browser: it prints the text it was sent and adds + * ' GB' itself, so the text must be decimal GB, carry no unit, and round the same way everywhere. + * + * Port: N/A (pure functions). + */ + +import { describe, expect, it } from 'vitest'; +import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js'; + +describe('byte units', () => { + it('keeps decimal GB (what a person reads) apart from the binary units of the budget math', () => { + expect(GB).toBe(1_000_000_000); + expect(GIB).toBe(1_073_741_824); + expect(MIB).toBe(1_048_576); + }); +}); + +describe('formatGb', () => { + it('prints decimal GB, so a Mac sold as 16 GB reads 17.2 and one sold as 48 GB reads 51.5', () => { + expect(formatGb(17_179_869_184)).toBe('17.2'); + expect(formatGb(51_539_607_552)).toBe('51.5'); + }); + + it('prints the registry sizes of the catalogue models as the registry shows them', () => { + expect(formatGb(6_550_825_373)).toBe('6.6'); + expect(formatGb(13_793_441_427)).toBe('13.8'); + expect(formatGb(17_741_871_939)).toBe('17.7'); + expect(formatGb(22_621_314_161)).toBe('22.6'); + expect(formatGb(81_370_035_877)).toBe('81.4'); + }); + + it('keeps one decimal below 100 and none from 100', () => { + expect(formatGb(1 * GB)).toBe('1.0'); + expect(formatGb(99.94 * GB)).toBe('99.9'); + // Rounds to 100.0, which is already "from 100". + expect(formatGb(99.96 * GB)).toBe('100'); + expect(formatGb(100 * GB)).toBe('100'); + // The free disk of the two test Macs, from their `df -k` lines. + expect(formatGb(890_926_084 * 1024)).toBe('912'); + expect(formatGb(41_752_848 * 1024)).toBe('42.8'); + }); + + it('never prints a unit, a thousands separator or an exponent', () => { + for (const bytes of [1, 5 * MIB, 3 * GIB, 250 * GB, 4000 * GB, 2.5e15]) { + expect(formatGb(bytes)).toMatch(/^\d+(\.\d)?$/); + } + }); + + it("reads '0.0' for anything that is not a positive size", () => { + for (const bytes of [0, -1, -5 * GB, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) { + expect(formatGb(bytes)).toBe('0.0'); + } + }); + + it('shows a size too small for one decimal as 0.0, never as an empty text', () => { + expect(formatGb(10 * MIB)).toBe('0.0'); + expect(formatGb(512 * MIB)).toBe('0.5'); + }); +}); + +describe('memClassGb', () => { + it('names the size a Mac is sold as: whole binary gigabytes', () => { + for (const sold of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192]) { + expect(memClassGb(sold * GIB)).toBe(sold); + } + // The exact `hw.memsize` of the two test Macs. + expect(memClassGb(17_179_869_184)).toBe(16); + expect(memClassGb(51_539_607_552)).toBe(48); + }); + + it('rounds a size that is not a whole number of gigabytes', () => { + // `hw.memsize_usable` of the 16 GB test Mac is a little below 16 GiB. + expect(memClassGb(16_631_808_000)).toBe(15); + expect(memClassGb(16 * GIB - 100 * MIB)).toBe(16); + }); + + it('reads 0 for anything that is not a positive size', () => { + for (const bytes of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(memClassGb(bytes)).toBe(0); + } + }); +}); diff --git a/test/local-model-hardware.test.ts b/test/local-model-hardware.test.ts new file mode 100644 index 00000000..0bcaa61a --- /dev/null +++ b/test/local-model-hardware.test.ts @@ -0,0 +1,749 @@ +/** + * @fileoverview Tests for src/local-model/hardware.ts: the probe that tells the recommendation + * what this Mac is. + * + * Two halves, mirroring the module: + * + * 1. `parseSysctl()`, pure. Fed the REAL output of the probe command on the two test Macs (a + * MacBook Pro with an M5 Pro and 48 GB, a Mac mini with an M4 and 16 GB), copied here + * character for character with the extra lines the capture holds, plus an Intel-shaped + * output (three of the seven keys do not exist there) and outputs cut off mid-line. + * 2. `readHardware()`, with every piece of IO injected: nothing is spawned and no real disk is + * read. What is pinned: a failed spawn is an ANSWER (the `os-fallback` shape), nothing is + * spawned off macOS, the result is cached for 60 s and concurrent callers share one probe. + * + * The IO itself is tested apart, against things this suite controls: + * + * 3. `spawnSysctl()` runs a small shell script of the test in place of sysctl. That proves what + * a fake cannot: the arguments are the seven keys and nothing else, the output of a program + * that exits 1 is still read (an Intel Mac lacks three keys, so sysctl exits 1 there), a + * program that hangs is given up on, and the server's environment is not handed over. + * 4. `dataDirFreeBytes()` with injected probes: a folder that does not exist yet, a mount that + * does not answer, and a `statfs` that never comes back, which must not be started twice. + * + * One test at the end runs the real defaults once (the real spawn and the real `statfs`), to + * prove they cannot reject on whatever machine runs the suite. On Linux `sysctl` knows none of + * the keys, so that run ends in the fallback; on a Mac it ends in a real answer. Both are fine. + * + * Port: N/A (no server). + */ + +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { isAbsolute, join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + dataDirFreeBytes, + FREE_DISK_TIMEOUT_MS, + freeDiskBytesOf, + HARDWARE_CACHE_TTL_MS, + parseSysctl, + readHardware, + resetHardwareCache, + spawnSysctl, + SYSCTL_KEYS, + SYSCTL_PATH, + SYSCTL_TIMEOUT_MS, + type HardwareProbeDeps, +} from '../src/local-model/hardware.js'; + +/** + * The capture of the MacBook Pro (Mac17,13), 2026-11-06. The last four lines are not part of + * the probe: two extra keys the capture asked for, the shell's exit status and a `df -k` line. + */ +const MACBOOK_SYSCTL = `hw.memsize: 51539607552 +machdep.cpu.brand_string: Apple M5 Pro +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac17,13 +hw.perflevel0.physicalcpu: 6 +hw.perflevel1.physicalcpu: 12 +exit=0 +/dev/disk3s5 971094180 54383564 890926084 6% 542146 8909260840 0% /System/Volumes/Data +`; + +/** The capture of the Mac mini (Mac16,10), 2026-11-06, with the same four extra lines. */ +const MACMINI_SYSCTL = `hw.memsize: 17179869184 +machdep.cpu.brand_string: Apple M4 +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac16,10 +hw.perflevel0.physicalcpu: 4 +hw.perflevel1.physicalcpu: 6 +exit=0 +/dev/disk3s5 239362496 172954124 41752848 81% 2353838 417528480 1% /System/Volumes/Data +`; + +/** Free disk of the two Macs: the "available" column of their `df -k` lines, in bytes. */ +const MACBOOK_FREE_DISK = 890_926_084 * 1024; +const MACMINI_FREE_DISK = 41_752_848 * 1024; + +/** + * What an Intel Mac prints: `hw.optional.arm64`, `sysctl.proc_translated` and + * `iogpu.wired_limit_mb` do not exist there, so sysctl leaves their lines out (and exits 1). + * NOT a capture: no Intel Mac was available, the shape is recalled. + */ +const INTEL_SYSCTL = `hw.memsize: 17179869184 +machdep.cpu.brand_string: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz +kern.osproductversion: 13.6.7 +hw.model: MacBookPro15,1 +`; + +/** Deps that spawn nothing and read no disk. A test overrides what it is about. */ +function deps(overrides: HardwareProbeDeps = {}): HardwareProbeDeps { + return { + platform: 'darwin', + arch: 'arm64', + runSysctl: async () => MACMINI_SYSCTL, + freeDiskBytes: async () => MACMINI_FREE_DISK, + totalMemBytes: () => 8 * 1024 ** 3, + cpuModel: () => 'Fallback CPU', + now: () => 1_000_000, + ...overrides, + }; +} + +beforeEach(() => { + resetHardwareCache(); +}); + +afterEach(() => { + resetHardwareCache(); +}); + +describe('the probe command', () => { + it('is an absolute path and asks for exactly the seven keys, never with -n', () => { + expect(isAbsolute(SYSCTL_PATH)).toBe(true); + expect(SYSCTL_PATH).toBe('/usr/sbin/sysctl'); + expect([...SYSCTL_KEYS]).toEqual([ + 'hw.memsize', + 'machdep.cpu.brand_string', + 'hw.optional.arm64', + 'sysctl.proc_translated', + 'iogpu.wired_limit_mb', + 'kern.osproductversion', + 'hw.model', + ]); + // Every argument is a key name. `-n` would print bare values, and a key missing on a + // machine would then shift every value after it onto the wrong key. + for (const key of SYSCTL_KEYS) expect(key).toMatch(/^[a-z][a-z0-9_.]*$/); + }); + + it('is bounded, and its result is reused for 60 seconds', () => { + expect(SYSCTL_TIMEOUT_MS).toBe(2000); + expect(HARDWARE_CACHE_TTL_MS).toBe(60_000); + }); +}); + +describe('parseSysctl on the two real Macs', () => { + it('reads the MacBook Pro: M5 Pro, 48 GB, macOS 27.2, no wired limit set', () => { + expect(parseSysctl(MACBOOK_SYSCTL)).toEqual({ + memBytes: 51_539_607_552, + chip: 'Apple M5 Pro', + arm64: true, + translated: false, + wiredLimitMb: 0, + macosVersion: '27.2', + macosMajor: 27, + model: 'Mac17,13', + }); + }); + + it('reads the Mac mini: M4, 16 GB, macOS 27.2, no wired limit set', () => { + expect(parseSysctl(MACMINI_SYSCTL)).toEqual({ + memBytes: 17_179_869_184, + chip: 'Apple M4', + arm64: true, + translated: false, + wiredLimitMb: 0, + macosVersion: '27.2', + macosMajor: 27, + model: 'Mac16,10', + }); + }); + + it('skips the lines it does not know: other keys, the exit status, the df line', () => { + const facts = parseSysctl(MACMINI_SYSCTL); + expect(Object.keys(facts).sort()).toEqual( + ['arm64', 'chip', 'macosMajor', 'macosVersion', 'memBytes', 'model', 'translated', 'wiredLimitMb'].sort() + ); + }); + + it('reports a wired limit of 0 as 0 (not set), and a set one as its number', () => { + expect(parseSysctl(MACMINI_SYSCTL).wiredLimitMb).toBe(0); + expect(parseSysctl('iogpu.wired_limit_mb: 14000\n').wiredLimitMb).toBe(14000); + }); +}); + +describe('parseSysctl on other shapes', () => { + it('reads an Intel Mac, whose output lacks three of the keys, without shifting a value', () => { + expect(parseSysctl(INTEL_SYSCTL)).toEqual({ + memBytes: 17_179_869_184, + chip: 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz', + macosVersion: '13.6.7', + macosMajor: 13, + model: 'MacBookPro15,1', + }); + }); + + it('reads an Intel Mac that does print the arm64 flag, as 0', () => { + const facts = parseSysctl(`hw.optional.arm64: 0\n${INTEL_SYSCTL}`); + expect(facts.arm64).toBe(false); + expect(facts.translated).toBeUndefined(); + }); + + it('reads a process under Rosetta: the machine is arm64 and the process is translated', () => { + const facts = parseSysctl(MACMINI_SYSCTL.replace('sysctl.proc_translated: 0', 'sysctl.proc_translated: 1')); + expect(facts.arm64).toBe(true); + expect(facts.translated).toBe(true); + }); + + it('drops a line that was cut off: a truncated output never yields half a value', () => { + // Cut in the middle of the chip name. The memory line before it is whole. + const cutInChip = 'hw.memsize: 17179869184\nmachdep.cpu.brand_string: Apple M'; + expect(parseSysctl(cutInChip)).toEqual({ memBytes: 17_179_869_184 }); + + // Cut in the middle of the memory figure: '1717' would be a valid, wrong number. + expect(parseSysctl('hw.memsize: 1717')).toEqual({}); + + // Cut right after a whole line: everything up to there counts. + const cutAfterLine = MACMINI_SYSCTL.slice(0, MACMINI_SYSCTL.indexOf('hw.optional.arm64')); + expect(parseSysctl(cutAfterLine)).toEqual({ memBytes: 17_179_869_184, chip: 'Apple M4' }); + }); + + it('reads nothing from an empty output or from error text', () => { + expect(parseSysctl('')).toEqual({}); + expect(parseSysctl('\n\n')).toEqual({}); + expect( + parseSysctl("sysctl: unknown oid 'hw.memsize'\nsysctl: cannot stat /proc/sys/hw/memsize: No such file\n") + ).toEqual({}); + }); + + it('leaves out a value that does not parse instead of guessing', () => { + const facts = parseSysctl( + [ + 'hw.memsize: lots', + 'machdep.cpu.brand_string: ', + 'hw.optional.arm64: yes', + 'sysctl.proc_translated: 2', + 'iogpu.wired_limit_mb: -1', + 'kern.osproductversion: Sequoia', + `hw.model: ${'x'.repeat(200)}`, + '', + ].join('\n') + ); + expect(facts).toEqual({}); + }); + + it('refuses a memory size of 0 or one too large to be a number of bytes', () => { + expect(parseSysctl('hw.memsize: 0\n').memBytes).toBeUndefined(); + expect(parseSysctl('hw.memsize: 99999999999999999999999\n').memBytes).toBeUndefined(); + }); + + it('accepts Windows line ends and padding, and the first line of a key wins', () => { + const facts = parseSysctl('hw.memsize: 17179869184 \r\nhw.model: Mac16,10\r\nhw.model: Other9,9\r\n'); + expect(facts).toEqual({ memBytes: 17_179_869_184, model: 'Mac16,10' }); + }); + + it('reads the macOS major version from one, two or three numbers', () => { + expect(parseSysctl('kern.osproductversion: 14\n')).toMatchObject({ macosVersion: '14', macosMajor: 14 }); + expect(parseSysctl('kern.osproductversion: 15.7\n')).toMatchObject({ macosVersion: '15.7', macosMajor: 15 }); + expect(parseSysctl('kern.osproductversion: 13.6.7\n')).toMatchObject({ macosVersion: '13.6.7', macosMajor: 13 }); + }); +}); + +describe('readHardware on the two real Macs', () => { + it('describes the MacBook Pro', async () => { + const hw = await readHardware( + deps({ runSysctl: async () => MACBOOK_SYSCTL, freeDiskBytes: async () => MACBOOK_FREE_DISK }) + ); + expect(hw).toEqual({ + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 51_539_607_552, + memGb: '51.5', + memClassGb: 48, + chip: 'Apple M5 Pro', + model: 'Mac17,13', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: MACBOOK_FREE_DISK, + freeDiskGb: '912', + source: 'sysctl', + }); + }); + + it('describes the Mac mini', async () => { + const hw = await readHardware(deps()); + expect(hw).toEqual({ + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 17_179_869_184, + memGb: '17.2', + memClassGb: 16, + chip: 'Apple M4', + model: 'Mac16,10', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: MACMINI_FREE_DISK, + freeDiskGb: '42.8', + source: 'sysctl', + }); + }); + + it('is plain JSON: it survives a round trip unchanged', async () => { + const hw = await readHardware(deps()); + expect(JSON.parse(JSON.stringify(hw))).toEqual(hw); + }); + + it('describes an Intel Mac from its shorter output: no arm64 flag, an x64 process', async () => { + const hw = await readHardware(deps({ arch: 'x64', runSysctl: async () => INTEL_SYSCTL })); + expect(hw.source).toBe('sysctl'); + expect(hw.arch).toBe('x64'); + expect(hw.chip).toBe('Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'); + expect(hw.macosMajor).toBe(13); + expect(hw).not.toHaveProperty('arm64'); + expect(hw).not.toHaveProperty('translated'); + expect(hw).not.toHaveProperty('wiredLimitMb'); + }); + + it('takes the chip name from node:os when sysctl printed everything but that', async () => { + const hw = await readHardware( + deps({ runSysctl: async () => MACMINI_SYSCTL.replace('machdep.cpu.brand_string: Apple M4\n', '') }) + ); + expect(hw.source).toBe('sysctl'); + expect(hw.chip).toBe('Fallback CPU'); + }); +}); + +describe('readHardware when the probe fails', () => { + it('answers from node:os when the spawn could not run or timed out', async () => { + const hw = await readHardware( + deps({ + runSysctl: async () => undefined, + totalMemBytes: () => 34_359_738_368, + cpuModel: () => 'Apple M2 Max', + }) + ); + expect(hw).toEqual({ + platform: 'darwin', + arch: 'arm64', + memBytes: 34_359_738_368, + memGb: '34.4', + memClassGb: 32, + chip: 'Apple M2 Max', + freeDiskBytes: MACMINI_FREE_DISK, + freeDiskGb: '42.8', + source: 'os-fallback', + }); + }); + + it('never rejects when a dependency throws or rejects', async () => { + const hw = await readHardware( + deps({ + runSysctl: () => Promise.reject(new Error('spawn EACCES')), + freeDiskBytes: () => { + throw new Error('statfs failed'); + }, + cpuModel: () => { + throw new Error('no cpus'); + }, + }) + ); + expect(hw.source).toBe('os-fallback'); + expect(hw.memBytes).toBe(8 * 1024 ** 3); + expect(hw).not.toHaveProperty('chip'); + expect(hw).not.toHaveProperty('freeDiskBytes'); + expect(hw).not.toHaveProperty('freeDiskGb'); + }); + + it('still answers when even the memory size cannot be read', async () => { + const hw = await readHardware( + deps({ + runSysctl: async () => undefined, + totalMemBytes: () => { + throw new Error('no memory figure'); + }, + }) + ); + expect(hw).toMatchObject({ source: 'os-fallback', memBytes: 0, memGb: '0.0', memClassGb: 0 }); + }); + + it('trusts nothing of an output that has no memory size', async () => { + // A truncated or foreign output: the chip and version lines are whole, hw.memsize is not there. + const withoutMemory = MACMINI_SYSCTL.replace('hw.memsize: 17179869184\n', ''); + const hw = await readHardware(deps({ runSysctl: async () => withoutMemory })); + expect(hw.source).toBe('os-fallback'); + expect(hw.memBytes).toBe(8 * 1024 ** 3); + expect(hw.chip).toBe('Fallback CPU'); + expect(hw).not.toHaveProperty('macosVersion'); + expect(hw).not.toHaveProperty('macosMajor'); + expect(hw).not.toHaveProperty('model'); + expect(hw).not.toHaveProperty('wiredLimitMb'); + expect(hw).not.toHaveProperty('arm64'); + }); + + it('falls back when the output was cut off inside the memory figure', async () => { + const hw = await readHardware(deps({ runSysctl: async () => 'hw.memsize: 1717' })); + expect(hw.source).toBe('os-fallback'); + expect(hw.memBytes).toBe(8 * 1024 ** 3); + }); + + it('leaves the free disk out when it is unknown, and keeps the rest', async () => { + const hw = await readHardware(deps({ freeDiskBytes: async () => undefined })); + expect(hw.source).toBe('sysctl'); + expect(hw).not.toHaveProperty('freeDiskBytes'); + expect(hw).not.toHaveProperty('freeDiskGb'); + }); + + it('reports a full disk as 0 bytes free, which is a figure and not "unknown"', async () => { + const hw = await readHardware(deps({ freeDiskBytes: async () => 0 })); + expect(hw.freeDiskBytes).toBe(0); + expect(hw.freeDiskGb).toBe('0.0'); + }); +}); + +describe('readHardware off macOS', () => { + it('spawns nothing and reads no disk', async () => { + let spawned = 0; + let diskRead = 0; + const hw = await readHardware( + deps({ + platform: 'linux', + arch: 'x64', + runSysctl: async () => { + spawned++; + return MACMINI_SYSCTL; + }, + freeDiskBytes: async () => { + diskRead++; + return 1; + }, + totalMemBytes: () => 67_430_023_168, + }) + ); + expect(spawned).toBe(0); + expect(diskRead).toBe(0); + expect(hw).toEqual({ + platform: 'linux', + arch: 'x64', + memBytes: 67_430_023_168, + memGb: '67.4', + memClassGb: 63, + source: 'not-probed', + }); + }); +}); + +describe('the cache of readHardware', () => { + it('reuses one result for 60 seconds, then probes again', async () => { + let clock = 5_000_000; + let spawned = 0; + const counting = deps({ + now: () => clock, + runSysctl: async () => { + spawned++; + return spawned === 1 ? MACMINI_SYSCTL : MACBOOK_SYSCTL; + }, + }); + + const first = await readHardware(counting); + clock += HARDWARE_CACHE_TTL_MS - 1; + const second = await readHardware(counting); + expect(spawned).toBe(1); + expect(second).toBe(first); + + clock += 1; + const third = await readHardware(counting); + expect(spawned).toBe(2); + expect(third.chip).toBe('Apple M5 Pro'); + }); + + it('caches a failed probe too: a broken sysctl is not spawned on every request', async () => { + let spawned = 0; + const failing = deps({ + runSysctl: async () => { + spawned++; + return undefined; + }, + }); + expect((await readHardware(failing)).source).toBe('os-fallback'); + expect((await readHardware(failing)).source).toBe('os-fallback'); + expect(spawned).toBe(1); + }); + + it('lets concurrent callers share one probe', async () => { + let spawned = 0; + let release: (text: string) => void = () => {}; + const slow = deps({ + runSysctl: () => { + spawned++; + return new Promise((resolve) => { + release = resolve; + }); + }, + }); + const callers = [readHardware(slow), readHardware(slow), readHardware(slow)]; + // Let the three calls reach the spawn before it answers. + await new Promise((resolve) => setImmediate(resolve)); + release(MACMINI_SYSCTL); + const results = await Promise.all(callers); + expect(spawned).toBe(1); + expect(results[1]).toBe(results[0]); + expect(results[2]).toBe(results[0]); + }); + + it('probes again after a reset, and does not trust a clock that ran backwards', async () => { + let clock = 9_000_000; + let spawned = 0; + const counting = deps({ + now: () => clock, + runSysctl: async () => { + spawned++; + return MACMINI_SYSCTL; + }, + }); + await readHardware(counting); + resetHardwareCache(); + await readHardware(counting); + expect(spawned).toBe(2); + + clock -= 10_000; + await readHardware(counting); + expect(spawned).toBe(3); + }); +}); + +// The scripts below are POSIX shell. Codeman's server does not run on Windows. +describe.skipIf(process.platform === 'win32')('spawnSysctl, against a script of the test', () => { + let folder: string; + + beforeEach(() => { + folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-spawn-')); + }); + + afterEach(() => { + rmSync(folder, { recursive: true, force: true }); + }); + + /** An executable shell script standing in for sysctl. */ + function script(body: string, mode = 0o755): string { + const path = join(folder, 'fake-sysctl'); + writeFileSync(path, `#!/bin/sh\n${body}\n`, { mode }); + return path; + } + + it('passes the seven keys as arguments and nothing else: no -n, no shell', async () => { + // printf, not echo: echo would swallow a `-n` argument, the very thing this test looks for. + const output = await spawnSysctl(script('printf \'%s\\n\' "$@"')); + expect(output?.trimEnd().split('\n')).toEqual([...SYSCTL_KEYS]); + }); + + it('reads the output of a program that exits 1, as sysctl does on an Intel Mac', async () => { + // Three keys are unknown there: their errors go to stderr, the other four lines to stdout. + const output = await spawnSysctl( + script( + [ + "echo 'hw.memsize: 17179869184'", + "echo 'machdep.cpu.brand_string: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'", + `echo "sysctl: unknown oid 'hw.optional.arm64'" >&2`, + `echo "sysctl: unknown oid 'sysctl.proc_translated'" >&2`, + `echo "sysctl: unknown oid 'iogpu.wired_limit_mb'" >&2`, + "echo 'kern.osproductversion: 13.6.7'", + "echo 'hw.model: MacBookPro15,1'", + 'exit 1', + ].join('\n') + ) + ); + expect(output).toBeDefined(); + expect(output).not.toContain('unknown oid'); + expect(parseSysctl(output as string)).toEqual({ + memBytes: 17_179_869_184, + chip: 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz', + macosVersion: '13.6.7', + macosMajor: 13, + model: 'MacBookPro15,1', + }); + }); + + it('answers undefined for a program that does not exist or may not be run', async () => { + expect(await spawnSysctl(join(folder, 'no-such-program'))).toBeUndefined(); + expect(await spawnSysctl(script('echo "hw.memsize: 1"', 0o644))).toBeUndefined(); + }); + + it('gives up on a program that hangs, and does not hand over half an output', async () => { + const started = Date.now(); + // `exec` so the signal reaches the sleeper itself and nothing is left behind. + const output = await spawnSysctl(script("echo 'hw.memsize: 17179869184'\nexec sleep 20"), 200); + expect(output).toBeUndefined(); + expect(Date.now() - started).toBeLessThan(5000); + }); + + it('answers undefined for an output far larger than seven lines', async () => { + expect(await spawnSysctl(script("head -c 200000 /dev/zero | tr '\\0' 'x'"))).toBeUndefined(); + }); + + it("does not pass the server's environment to the program", async () => { + const name = 'CODEMAN_LOCAL_MODEL_TEST_SECRET'; + process.env[name] = 'hunter2'; + try { + const output = await spawnSysctl(script(`echo "secret=\${${name}:-unset}"`)); + expect(output?.trim()).toBe('secret=unset'); + } finally { + delete process.env[name]; + } + }); +}); + +describe('dataDirFreeBytes', () => { + let folder: string; + + beforeEach(() => { + folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-disk-')); + }); + + afterEach(() => { + rmSync(folder, { recursive: true, force: true }); + }); + + it('reads the free bytes of the data dir with the real probe and the real statfs', async () => { + const free = await dataDirFreeBytes({ dataDir: () => folder }); + expect(typeof free).toBe('number'); + expect(free).toBeGreaterThanOrEqual(0); + }); + + it('uses the nearest existing parent when the data dir does not exist yet', async () => { + const asked: string[] = []; + const free = await dataDirFreeBytes({ + dataDir: () => join(folder, 'not', 'yet', 'there'), + freeBytesOf: async (path) => { + asked.push(path); + return 123; + }, + }); + expect(free).toBe(123); + expect(asked).toEqual([folder]); + }); + + it('does not touch a folder whose mount does not answer, or a path that is a file', async () => { + for (const kind of ['unknown', 'file'] as const) { + let touched = 0; + const free = await dataDirFreeBytes({ + dataDir: () => folder, + probeKind: async () => kind, + freeBytesOf: async () => { + touched++; + return 1; + }, + }); + expect(free, kind).toBeUndefined(); + expect(touched, kind).toBe(0); + } + }); + + it('stops walking up after a few parents', async () => { + let probes = 0; + const free = await dataDirFreeBytes({ + dataDir: () => '/a/b/c/d/e/f/g/h/i/j/k/l', + probeKind: async () => { + probes++; + return 'absent'; + }, + freeBytesOf: async () => 1, + }); + expect(free).toBeUndefined(); + expect(probes).toBe(7); + }); + + it('gives up on a statfs that does not come back, and never starts a second one meanwhile', async () => { + let started = 0; + let finish: (bytes: number) => void = () => {}; + const hanging = { + dataDir: () => folder, + freeBytesOf: () => { + started++; + return new Promise((resolve) => { + finish = resolve; + }); + }, + timeoutMs: 20, + }; + + // The first read waits its timeout and answers "unknown". + expect(await dataDirFreeBytes(hanging)).toBeUndefined(); + expect(started).toBe(1); + + // While that statfs is still out, later reads answer at once and start nothing. + const before = Date.now(); + expect(await dataDirFreeBytes(hanging)).toBeUndefined(); + expect(await dataDirFreeBytes({ ...hanging, timeoutMs: 60_000 })).toBeUndefined(); + expect(Date.now() - before).toBeLessThan(1000); + expect(started).toBe(1); + + // Once it finally answers, the next read works again. + finish(5); + await new Promise((resolve) => setImmediate(resolve)); + expect(await dataDirFreeBytes({ ...hanging, freeBytesOf: async () => 42 })).toBe(42); + }); + + it('never rejects when one of its reads does', async () => { + const failing = async (): Promise => { + throw new Error('EIO'); + }; + expect(await dataDirFreeBytes({ dataDir: () => folder, freeBytesOf: failing })).toBeUndefined(); + expect(await dataDirFreeBytes({ dataDir: () => folder, probeKind: failing })).toBeUndefined(); + expect( + await dataDirFreeBytes({ + dataDir: () => { + throw new Error('no home'); + }, + }) + ).toBeUndefined(); + }); + + it('is bounded by default', () => { + expect(FREE_DISK_TIMEOUT_MS).toBe(2000); + }); +}); + +describe('the real defaults', () => { + let folder: string; + + beforeEach(() => { + folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-hw-')); + }); + + afterEach(() => { + rmSync(folder, { recursive: true, force: true }); + }); + + it('reads the free bytes of a real folder, and undefined for one that does not exist', async () => { + const free = await freeDiskBytesOf(folder); + expect(typeof free).toBe('number'); + expect(free).toBeGreaterThanOrEqual(0); + expect(await freeDiskBytesOf(join(folder, 'missing', 'deeper'))).toBeUndefined(); + }); + + it('never rejects with the real spawn and the real disk read, whatever machine this is', async () => { + // Pretend to be a Mac so the spawn and the disk read really run. On Linux /usr/sbin/sysctl + // either does not exist or knows none of the keys: the answer is then the fallback. + const hw = await readHardware({ platform: 'darwin' }); + expect(['sysctl', 'os-fallback']).toContain(hw.source); + expect(hw.memBytes).toBeGreaterThan(0); + expect(hw.memGb).toMatch(/^\d+(\.\d)?$/); + if (hw.freeDiskBytes !== undefined) { + expect(hw.freeDiskBytes).toBeGreaterThanOrEqual(0); + expect(hw.freeDiskGb).toMatch(/^\d+(\.\d)?$/); + } + }); +}); diff --git a/test/local-model-recommend.test.ts b/test/local-model-recommend.test.ts new file mode 100644 index 00000000..86a2c327 --- /dev/null +++ b/test/local-model-recommend.test.ts @@ -0,0 +1,1298 @@ +/** + * @fileoverview Tests for src/local-model/recommend.ts: the pure function that turns "what this + * Mac is" into "which local model fits it". + * + * Laid out like the module: + * + * 1. The budget (`deviceBudget`) for every memory size Apple sells from 8 to 128 GB, and the + * need of a model at a context (`needBytes`). + * 2. The two REAL Macs of the project, from their captured `sysctl` output through the real + * parser: the MacBook Pro (M5 Pro, 48 GB) and the Mac mini (M4, 16 GB). Each test states the + * expected answer and why. + * 3. Every reason for verdict `none`, one fixture per reason, and their order. + * 4. The `limited` verdict, the three-state speed in the ranking, harness rows (a gated one is + * never chosen), the `tested` flag that alone may turn on the word "Recommended", and the + * shape of the answer (labels from the fixed list, sizes in one unit). + * + * The stock catalogue is used wherever the answer for a real Mac is at stake. Small synthetic + * catalogues (validated by the real schema) isolate one rule of the ranking at a time. + * + * Port: N/A (pure functions; the two hardware fixtures go through `readHardware` with every + * piece of IO injected). + */ + +import { beforeEach, describe, expect, it } from 'vitest'; +import { + getLocalModelCatalog, + LocalModelCatalogSchema, + STOCK_LOCAL_MODEL_CATALOG, +} from '../src/config/local-model-catalog.js'; +import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js'; +import { readHardware, resetHardwareCache } from '../src/local-model/hardware.js'; +import { + deviceBudget, + DISK_HEADROOM_BYTES, + FIT_GOOD_MAX_SHARE, + FIT_TIGHT_MAX_SHARE, + LIMITED_BELOW_PARAMS_B, + MAX_ALTERNATIVES, + needBytes, + recommend, + SLOW_BELOW_TOKENS_PER_SECOND, +} from '../src/local-model/recommend.js'; +import { + LOCAL_MODEL_LABELS, + type Candidate, + type HardwareFacts, + type HarnessFact, + type LocalHarnessEntry, + type LocalModelCatalog, + type LocalModelEntry, + type Recommendation, +} from '../src/types.js'; + +// ---------- Fixtures ---------- + +/** The real probe output of the MacBook Pro (Mac17,13), captured 2026-11-06, extra lines included. */ +const MACBOOK_SYSCTL = `hw.memsize: 51539607552 +machdep.cpu.brand_string: Apple M5 Pro +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac17,13 +hw.perflevel0.physicalcpu: 6 +hw.perflevel1.physicalcpu: 12 +exit=0 +/dev/disk3s5 971094180 54383564 890926084 6% 542146 8909260840 0% /System/Volumes/Data +`; + +/** The real probe output of the Mac mini (Mac16,10), captured 2026-11-06, extra lines included. */ +const MACMINI_SYSCTL = `hw.memsize: 17179869184 +machdep.cpu.brand_string: Apple M4 +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac16,10 +hw.perflevel0.physicalcpu: 4 +hw.perflevel1.physicalcpu: 6 +exit=0 +/dev/disk3s5 239362496 172954124 41752848 81% 2353838 417528480 1% /System/Volumes/Data +`; + +/** The hardware facts of a real Mac: its captured output through the real parser, no IO. */ +async function realMac(sysctlOutput: string, freeDiskKb: number): Promise { + resetHardwareCache(); + return readHardware({ + platform: 'darwin', + arch: 'arm64', + runSysctl: async () => sysctlOutput, + freeDiskBytes: async () => freeDiskKb * 1024, + totalMemBytes: () => 0, + cpuModel: () => undefined, + now: () => 0, + }); +} + +/** An Apple silicon Mac with `soldGb` of memory, healthy in every other respect. */ +function mac(soldGb: number, chip: string, overrides: Partial = {}): HardwareFacts { + const memBytes = soldGb * GIB; + return { + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes, + memGb: formatGb(memBytes), + memClassGb: memClassGb(memBytes), + chip, + model: 'Mac0,0', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: 500 * GB, + freeDiskGb: '500', + source: 'sysctl', + ...overrides, + }; +} + +/** A copy of `facts` without the given fields (an absent field, not an `undefined` one). */ +function without(facts: HardwareFacts, ...fields: (keyof HardwareFacts)[]): HardwareFacts { + const copy: Record = { ...facts }; + for (const field of fields) delete copy[field]; + return copy as unknown as HardwareFacts; +} + +/** Display labels of the stock harnesses, as the CLI registry would supply them. Data, not a branch. */ +const HARNESS_LABELS: Record = { opencode: 'OpenCode', claude: 'Claude Code', pi: 'Pi' }; + +/** What the server knows about one harness: installed and accepting every model id, unless overridden. */ +function fact(cli: string, overrides: Partial = {}): HarnessFact { + return { + cli, + label: HARNESS_LABELS[cli] ?? cli, + installed: true, + installable: false, + acceptsModelId: () => true, + ...overrides, + }; +} + +const STOCK = getLocalModelCatalog(); + +/** Every stock harness installed, the gated one included. */ +const ALL_INSTALLED: HarnessFact[] = STOCK.harnesses.map((row) => fact(row.cli)); + +/** The stock catalogue with a change applied, validated by the real schema. */ +function stockWith(change: (catalog: LocalModelCatalog) => void): LocalModelCatalog { + const catalog = structuredClone(STOCK_LOCAL_MODEL_CATALOG); + change(catalog); + return LocalModelCatalogSchema.parse(catalog); +} + +/** A model row for a synthetic catalogue: 20 billion parameters, 10 GiB, one small context. */ +function model(id: string, overrides: Partial = {}): LocalModelEntry { + return { + id, + label: id, + license: 'Apache-2.0', + paramsB: 20, + bytes: 10 * GIB, + kvBytesPerToken: 1, + contexts: [1024], + rank: 1, + tested: [], + ...overrides, + }; +} + +/** + * A synthetic catalogue: no prompt cache, one harness `alpha` that works from 1024 tokens, and + * one chip row ('M4 Max' at 400 GB/s, so tokens per second = `speedPer100Gbps` x 4). + */ +function catalogOf(models: LocalModelEntry[], harnesses?: LocalHarnessEntry[]): LocalModelCatalog { + return LocalModelCatalogSchema.parse({ + schema: 1, + updated: '2026-11-06', + minMacosMajor: 14, + promptCacheTiers: [{ minMemGb: 0, bytes: 0 }], + models, + harnesses: harnesses ?? [{ cli: 'alpha', minContext: 1024, rank: 1 }], + chips: [{ match: 'M4 Max', gbps: 400 }], + }); +} + +/** A Mac whose budget is exactly 100 GiB: a wired limit of 101 GiB on a 128 GB machine. */ +const HUNDRED_GIB_MAC = mac(128, 'Apple M4 Max', { wiredLimitMb: 101 * 1024 }); +const HUNDRED_GIB = 100 * GIB; + +/** What a synthetic model needs beside its weights at its 1024-token context: 1 GiB plus the KV cache. */ +const SYNTHETIC_OVERHEAD = GIB + 1024; + +/** A model whose need is exactly `need` bytes in a synthetic catalogue. */ +function modelNeeding(id: string, need: number, overrides: Partial = {}): LocalModelEntry { + return model(id, { bytes: need - SYNTHETIC_OVERHEAD, ...overrides }); +} + +/** `id@context` of the pick and of each alternative, in order. */ +function order(result: Recommendation): string[] { + return [result.pick, ...result.alternatives] + .filter((candidate): candidate is Candidate => candidate !== undefined) + .map((candidate) => `${candidate.modelId}@${candidate.context}`); +} + +/** Every string anywhere in a value. */ +function stringsIn(value: unknown): string[] { + if (typeof value === 'string') return [value]; + if (Array.isArray(value)) return value.flatMap(stringsIn); + if (value && typeof value === 'object') return Object.values(value).flatMap(stringsIn); + return []; +} + +function deepFreeze(value: T): T { + if (value && typeof value === 'object') { + for (const inner of Object.values(value)) deepFreeze(inner); + Object.freeze(value); + } + return value; +} + +/** The Macs the sweeps run over: every memory size, with a slow, a fast and an unknown chip. */ +const SWEEP: HardwareFacts[] = [8, 16, 18, 24, 32, 36, 48, 64, 96, 128].flatMap((soldGb) => + ['Apple M4', 'Apple M4 Max', 'Apple M5 Ultra', 'Apple M6'].map((chip) => mac(soldGb, chip)) +); + +beforeEach(() => { + resetHardwareCache(); +}); + +// ---------- 1. Budget and need ---------- + +describe('deviceBudget', () => { + // Sold size, device limit L and budget B in GiB. L is 2/3 of memory below 36 GB and 3/4 from + // 36 GB; B = min(L - 1, memory - reserve). Only at 8 GB is the reserve the smaller bound. + const TIERS: [soldGb: number, limitGib: number, budgetGib: number][] = [ + [8, 16 / 3, 2], + [16, 32 / 3, 29 / 3], + [18, 12, 11], + [24, 16, 15], + [32, 64 / 3, 61 / 3], + [36, 27, 26], + [48, 36, 35], + [64, 48, 47], + [96, 72, 71], + [128, 96, 95], + ]; + + it.each(TIERS)('gives a %i GB Mac a limit of %f GiB and a budget of %f GiB', (soldGb, limitGib, budgetGib) => { + const budget = deviceBudget(mac(soldGb, 'Apple M4')); + // Whole bytes: a third of a byte is rounded down. + expect(Math.abs(budget.limitBytes - limitGib * GIB)).toBeLessThan(1); + expect(Math.abs(budget.budgetBytes - budgetGib * GIB)).toBeLessThan(1); + expect(Number.isInteger(budget.limitBytes)).toBe(true); + expect(Number.isInteger(budget.budgetBytes)).toBe(true); + expect(budget.source).toBe('estimated'); + }); + + it('matches the rounded budgets of the design table', () => { + const rounded = TIERS.map(([soldGb]) => [ + soldGb, + (deviceBudget(mac(soldGb, 'Apple M4')).budgetBytes / GIB).toFixed(1), + ]); + expect(rounded).toEqual([ + [8, '2.0'], + [16, '9.7'], + [18, '11.0'], + [24, '15.0'], + [32, '20.3'], + [36, '26.0'], + [48, '35.0'], + [64, '47.0'], + [96, '71.0'], + [128, '95.0'], + ]); + }); + + it('keeps a reserve of 6 GiB up to 18 GB, 8 GiB up to 48 GB and 12 GiB from 64 GB', () => { + // A wired limit far above the memory leaves "memory - reserve" as the only bound. + const reserveOf = (soldGb: number): number => { + const budget = deviceBudget(mac(soldGb, 'Apple M4', { wiredLimitMb: 1024 * 1024 })); + return (soldGb * GIB - budget.budgetBytes) / GIB; + }; + expect([8, 16, 18].map(reserveOf)).toEqual([6, 6, 6]); + expect([24, 32, 36, 48].map(reserveOf)).toEqual([8, 8, 8, 8]); + expect([64, 96, 128].map(reserveOf)).toEqual([12, 12, 12]); + }); + + it('uses the wired limit when it is above 0', () => { + // 14000 MiB on the 16 GB Mac: L - 1 GiB is 12.67 GiB, memory - reserve is 10 GiB. + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 14000 }))).toEqual({ + limitBytes: 14000 * MIB, + budgetBytes: 10 * GIB, + source: 'wired-limit', + }); + // A limit BELOW the estimate is honoured too: 8192 MiB leaves 7 GiB. + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 8192 }))).toEqual({ + limitBytes: 8 * GIB, + budgetBytes: 7 * GIB, + source: 'wired-limit', + }); + }); + + it('reads a wired limit of 0, or none at all, as "not set"', () => { + const estimated = deviceBudget(mac(16, 'Apple M4')); + expect(estimated.source).toBe('estimated'); + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 0 }))).toEqual(estimated); + expect(deviceBudget(without(mac(16, 'Apple M4'), 'wiredLimitMb'))).toEqual(estimated); + }); + + it('never goes below 0', () => { + expect(deviceBudget(mac(4, 'Apple M1')).budgetBytes).toBe(0); + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 512 })).budgetBytes).toBe(0); + expect(deviceBudget({ ...mac(16, 'Apple M4'), memBytes: 0, memClassGb: 0 })).toEqual({ + limitBytes: 0, + budgetBytes: 0, + source: 'estimated', + }); + }); +}); + +describe('needBytes', () => { + const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; + + it('adds weights, the KV cache of the context, 1 GiB and the prompt cache', () => { + // 6,550,825,373 + 32768 x 65536 + 1 GiB + 512 MiB on a Mac below 32 GB. + expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + 2 * GIB + GIB + 512 * MIB); + expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(10_308_921_757); + // Half the context, half the KV cache. + expect(needBytes(nineB, 32768, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + GIB + GIB + 512 * MIB); + }); + + it('counts a prompt cache of 512 MiB below 32 GB of memory and 2 GiB from 32 GB', () => { + const base = 6_550_825_373 + 2 * GIB + GIB; + for (const soldGb of [8, 16, 18, 24]) { + expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(512 * MIB); + } + for (const soldGb of [32, 36, 48, 64, 96, 128]) { + expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(2 * GIB); + } + }); + + it('does not depend on the chip or the harness', () => { + expect(needBytes(nineB, 65536, STOCK, mac(48, 'Apple M1'))).toBe( + needBytes(nineB, 65536, STOCK, mac(48, 'Apple M5 Ultra')) + ); + }); +}); + +describe('the fit grade', () => { + const pickOf = (need: number): Candidate | undefined => + recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', need)]), [fact('alpha')]).pick; + + it('has the budget this group assumes', () => { + expect(deviceBudget(HUNDRED_GIB_MAC).budgetBytes).toBe(HUNDRED_GIB); + expect(FIT_GOOD_MAX_SHARE).toBe(0.85); + expect(FIT_TIGHT_MAX_SHARE).toBe(0.98); + }); + + it('is good up to 0.85 of the budget', () => { + expect(pickOf(0.5 * HUNDRED_GIB)).toMatchObject({ fit: 'good', fitLabel: 'Good', budgetShare: 0.5 }); + expect(pickOf(0.85 * HUNDRED_GIB)).toMatchObject({ fit: 'good', budgetShare: 0.85 }); + }); + + it('is tight above 0.85 and up to 0.98', () => { + expect(pickOf(0.85 * HUNDRED_GIB + 1)).toMatchObject({ fit: 'tight', fitLabel: 'Tight' }); + expect(pickOf(0.9 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.9 }); + expect(pickOf(0.98 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.98 }); + }); + + it('is out above 0.98: the model is not offered at all', () => { + const result = recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', 0.98 * HUNDRED_GIB + 1)]), [ + fact('alpha'), + ]); + expect(result.verdict).toBe('none'); + expect(result.reason).toBe('memory'); + expect(result.pick).toBeUndefined(); + }); +}); + +// ---------- 2. The two real Macs ---------- + +describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { + /** + * Expected: Qwen3.6 35B-A3B at 131,072 tokens with OpenCode, Good, about 42 tokens per second. + * + * Why. The budget is 35 GiB (3/4 of 48 GiB minus 1). Five models fit it as Good. The M5 Pro is + * in the chips table (307 GB/s), so each model with a speed figure gets an estimate: + * the dense 27B computes to 2.95 x 3.07 = 9.1 tokens per second, under the 10 that separate + * slow from ok, while the 35B-A3B (3 billion parameters active per token) computes to 41.8. + * Speed ranks before model rank, so the 35B-A3B wins although the 27B has the higher rank. The + * 27B stays in the list, last, marked Slow. Both harnesses can run it; OpenCode has the + * higher rank. + */ + it('gets Qwen3.6 35B-A3B at 131072 tokens with OpenCode', async () => { + const hw = await realMac(MACBOOK_SYSCTL, 890_926_084); + expect(hw.chip).toBe('Apple M5 Pro'); + expect(hw.memClassGb).toBe(48); + + const result = recommend(hw, STOCK, ALL_INSTALLED); + + expect(result.verdict).toBe('ok'); + expect(result.reason).toBeUndefined(); + expect(result.budgetBytes).toBe(35 * GIB); + expect(result.budgetGb).toBe('37.6'); + expect(result.budgetSource).toBe('estimated'); + expect(result.pick).toEqual({ + modelId: 'qwen3.6:35b-a3b', + modelLabel: 'Qwen3.6 35B-A3B', + license: 'Apache-2.0', + paramsB: 35, + context: 131072, + cli: 'opencode', + cliLabel: 'OpenCode', + cliInstalled: true, + // 22,621,314,161 + 20480 x 131072 + 1 GiB + 2 GiB. + needBytes: 28_526_894_193, + needGb: '28.5', + downloadBytes: 22_621_314_161, + downloadGb: '22.6', + budgetShare: 0.759, + fit: 'good', + fitLabel: 'Good', + speed: 'ok', + tokensPerSecond: 41.8, + tested: false, + }); + }); + + it('says "Best fit", not "Recommended", and that nobody tested it on such a Mac', async () => { + const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + }); + + it('lists the other four models that fit: the fast ones, the one of unknown speed, the slow one', async () => { + const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED); + expect(result.alternatives.map((c) => [c.modelId, c.context, c.fit, c.speed, c.tokensPerSecond ?? null])).toEqual([ + ['qwen3.5:9b-q8_0', 65536, 'good', 'ok', 17.8], + ['qwen3.5:9b', 65536, 'good', 'ok', 28.6], + ['gpt-oss:20b', 131072, 'good', 'unknown', null], + ['qwen3.8:27b', 131072, 'good', 'slow', 9.1], + ]); + expect(result.alternatives[2].speedLabel).toBe('Speed unknown'); + expect(result.alternatives[3].speedLabel).toBe('Slow'); + // The two that do not fit 35 GiB at any context are not listed. + expect(order(result)).not.toContain('qwen3.8:27b-q8_0@131072'); + expect(order(result).some((entry) => entry.startsWith('qwen3.5:122b'))).toBe(false); + }); + + it('would get the 27B instead if its chip were in no chips row', async () => { + // The same Mac with the M5 Pro row removed: every speed is unknown, so model rank decides. + const hw = await realMac(MACBOOK_SYSCTL, 890_926_084); + const withoutChip = stockWith((catalog) => { + catalog.chips = catalog.chips.filter((row) => row.match !== 'M5 Pro'); + }); + const result = recommend(hw, withoutChip, ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', context: 131072, speed: 'unknown' }); + expect(result.pick).not.toHaveProperty('tokensPerSecond'); + }); +}); + +describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { + /** + * Expected: no recommendation, reason `memory`. + * + * Why. The budget is 9.67 GiB (2/3 of 16 GiB minus 1). The only model near it is the 9B. Both + * harnesses that may be offered need 65,536 tokens of context, and there the 9B needs + * 6.10 GiB of weights + 2 GiB of KV cache + 1 GiB + 0.5 GiB of prompt cache = 9.60 GiB, which + * is 0.993 of the budget: above the 0.98 where "tight" ends. At 32,768 tokens it would fit + * (0.890, tight), but the one harness that works at that context is gated. + */ + it('gets no recommendation: nothing fits at the context a harness needs', async () => { + const hw = await realMac(MACMINI_SYSCTL, 41_752_848); + expect(hw.chip).toBe('Apple M4'); + expect(hw.memClassGb).toBe(16); + + const result = recommend(hw, STOCK, ALL_INSTALLED); + + expect(result).toEqual({ + verdict: 'none', + reason: 'memory', + headline: { + text: 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', + vars: { memory: '17.2' }, + }, + notes: [], + alternatives: [], + // floor(17,179,869,184 x 2/3) - 1 GiB. + budgetBytes: 10_379_504_298, + budgetGb: '10.4', + budgetSource: 'estimated', + }); + }); + + it('misses by a hair at 65536 tokens and would fit at 32768', async () => { + const hw = await realMac(MACMINI_SYSCTL, 41_752_848); + const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; + const budget = deviceBudget(hw).budgetBytes; + const shareAt64k = needBytes(nineB, 65536, STOCK, hw) / budget; + const shareAt32k = needBytes(nineB, 32768, STOCK, hw) / budget; + expect(shareAt64k).toBeCloseTo(0.9932, 4); + expect(shareAt64k).toBeGreaterThan(FIT_TIGHT_MAX_SHARE); + expect(shareAt32k).toBeCloseTo(0.8898, 4); + expect(shareAt32k).toBeLessThan(FIT_TIGHT_MAX_SHARE); + // Every other model needs more than the whole budget at its smallest context. + for (const other of STOCK.models.filter((m) => m.id !== 'qwen3.5:9b')) { + const smallest = other.contexts[other.contexts.length - 1]; + expect(needBytes(other, smallest, STOCK, hw), other.id).toBeGreaterThan(budget); + } + }); + + it('would get the 9B at 32768 tokens, as a limited answer, once the gated harness is opened', async () => { + const hw = await realMac(MACMINI_SYSCTL, 41_752_848); + const ungated = stockWith((catalog) => { + catalog.harnesses = catalog.harnesses.map((row) => ({ + cli: row.cli, + minContext: row.minContext, + rank: row.rank, + })); + }); + const result = recommend(hw, ungated, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.pick).toMatchObject({ + modelId: 'qwen3.5:9b', + context: 32768, + cli: 'pi', + fit: 'tight', + budgetShare: 0.89, + speed: 'ok', + // 9.3 x 1.2. + tokensPerSecond: 11.2, + }); + }); + + it('would get the 9B at 65536 tokens if a wired limit had been set on it', async () => { + // iogpu.wired_limit_mb = 14000 lifts the budget to 10 GiB (memory minus the 6 GiB reserve). + const hw = { ...(await realMac(MACMINI_SYSCTL, 41_752_848)), wiredLimitMb: 14000 }; + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.budgetSource).toBe('wired-limit'); + expect(result.budgetBytes).toBe(10 * GIB); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', context: 65536, cli: 'opencode', fit: 'tight' }); + }); +}); + +describe('every memory size, with the stock catalogue', () => { + // What the rules compute today. A change of a catalogue figure or of a threshold that moves + // one of these answers has to change this table on purpose. + const EXPECTED: [soldGb: number, chip: string, answer: string][] = [ + [8, 'Apple M2', 'none memory'], + [16, 'Apple M4', 'none memory'], + // The 9B at 65536 tokens is 0.87 of 11 GiB. + [18, 'Apple M3 Pro', 'limited qwen3.5:9b@65536 tight ok'], + // gpt-oss:20b at 65536 tokens needs 15.8 GiB of the 15 GiB budget: out. + [24, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], + // gpt-oss:20b fits only as Tight here (0.853 at 65536 tokens, 0.927 at 131072), and Good + // ranks before Tight. The Q8 9B is Good but slow on a base M4 (7 tokens per second). + [32, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], + [32, 'Apple M1 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + // The 27B (0.905) and the 35B-A3B (0.974) are Tight; gpt-oss:20b is Good but has no speed + // figure, and a known speed ranks before an unknown one. + [36, 'Apple M4 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + // On a Pro chip the dense 27B is slow (8.1 tokens per second), the 35B-A3B is not. + [48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'], + [48, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], + // The Q8 27B fits 47 GiB but computes to 7 tokens per second on this chip. + [64, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], + [96, 'Apple M3 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], + // The 122B fits 95 GiB only as Tight (0.861) and has no speed figure. + [128, 'Apple M5 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], + ]; + + it.each(EXPECTED)('%i GB, %s: %s', (soldGb, chip, answer) => { + const result = recommend(mac(soldGb, chip), STOCK, ALL_INSTALLED); + const actual = result.pick + ? `${result.verdict} ${result.pick.modelId}@${result.pick.context} ${result.pick.fit} ${result.pick.speed}` + : `${result.verdict} ${result.reason}`; + expect(actual).toBe(answer); + }); + + it('always pairs the pick with the top-ranked harness, OpenCode', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + if (result.pick) expect(result.pick.cli).toBe('opencode'); + } + }); +}); + +// ---------- 3. Verdict none ---------- + +describe('verdict none', () => { + const bigMac = mac(48, 'Apple M4 Max'); + + /** A `none` answer has no pick, no alternatives, no notes, and the reason's sentence as headline. */ + function expectNone(result: Recommendation, reason: Recommendation['reason'], headline: string): void { + expect(result.verdict).toBe('none'); + expect(result.reason).toBe(reason); + expect(result.headline.text).toBe(headline); + expect(result.pick).toBeUndefined(); + expect(result.alternatives).toEqual([]); + expect(result.notes).toEqual([]); + } + + describe('not-macos', () => { + it('answers so on Linux, with no budget at all', () => { + const linux: HardwareFacts = { + platform: 'linux', + arch: 'x64', + memBytes: 64 * GIB, + memGb: formatGb(64 * GIB), + memClassGb: 64, + source: 'not-probed', + }; + const result = recommend(linux, STOCK, ALL_INSTALLED); + expectNone(result, 'not-macos', LOCAL_MODEL_LABELS.noneNotMacos); + expect(result.headline.vars).toBeUndefined(); + expect(result.budgetBytes).toBe(0); + expect(result.budgetGb).toBe('0.0'); + expect(result.budgetSource).toBe('estimated'); + }); + + it('comes before every other reason', () => { + for (const platform of ['linux', 'win32', 'freebsd']) { + const hw = mac(8, 'Intel(R) Core(TM) i7', { platform, arch: 'x64', arm64: false, macosMajor: 12 }); + expect(recommend(hw, STOCK, []).reason, platform).toBe('not-macos'); + } + }); + }); + + describe('intel', () => { + const intelChip = 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'; + + it('answers so for an Intel Mac, whose sysctl has no arm64 flag', () => { + const hw = without(mac(32, intelChip, { arch: 'x64' }), 'arm64', 'translated', 'wiredLimitMb'); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expectNone(result, 'intel', 'This Mac has an Intel processor. Local coding models need Apple silicon.'); + expect(result.budgetBytes).toBe(0); + expect(result.budgetGb).toBe('0.0'); + }); + + it('answers so when the arm64 flag reads 0', () => { + const hw = without(mac(32, intelChip, { arch: 'x64', arm64: false }), 'translated'); + expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('intel'); + }); + + it('comes before the macOS version and the memory', () => { + const hw = without(mac(8, intelChip, { arch: 'x64', macosMajor: 12, macosVersion: '12.7' }), 'arm64'); + expect(recommend(hw, STOCK, []).reason).toBe('intel'); + }); + + it('does not mistake Node under Rosetta for an Intel Mac', () => { + // An x64 process on Apple silicon: the machine flag says arm64, the process is translated. + const rosetta = mac(48, 'Apple M4 Max', { arch: 'x64', translated: true }); + expect(recommend(rosetta, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + // The same when only the translation flag was read. + expect(recommend(without(rosetta, 'arm64'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + }); + + it('decides from node:os alone when the probe failed', () => { + const fallback = (arch: string, chip: string): HardwareFacts => ({ + platform: 'darwin', + arch, + memBytes: 48 * GIB, + memGb: formatGb(48 * GIB), + memClassGb: 48, + chip, + source: 'os-fallback', + }); + // An arm64 process can only run on Apple silicon. + expect(recommend(fallback('arm64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + // An x64 process with an Apple chip name is Rosetta. + expect(recommend(fallback('x64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + // An x64 process with any other chip name is taken for an Intel Mac. + expect(recommend(fallback('x64', intelChip), STOCK, ALL_INSTALLED).reason).toBe('intel'); + expect(recommend(without(fallback('x64', intelChip), 'chip'), STOCK, ALL_INSTALLED).reason).toBe('intel'); + }); + }); + + describe('macos', () => { + it('answers so below the catalogue minimum, naming both versions', () => { + const result = recommend( + mac(48, 'Apple M1 Max', { macosVersion: '13.6.7', macosMajor: 13 }), + STOCK, + ALL_INSTALLED + ); + expectNone(result, 'macos', 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.'); + expect(result.headline.vars).toEqual({ version: '13.6.7', minimum: '14' }); + // The budget is still reported: the Mac itself is able. + expect(result.budgetBytes).toBe(35 * GIB); + }); + + it('accepts the minimum itself and anything newer', () => { + for (const macosMajor of [14, 15, 26, 27]) { + const hw = mac(48, 'Apple M4 Max', { macosMajor, macosVersion: `${macosMajor}.0` }); + expect(recommend(hw, STOCK, ALL_INSTALLED).verdict, String(macosMajor)).toBe('ok'); + } + }); + + it('does not block on a version that could not be read', () => { + const hw = without(mac(48, 'Apple M4 Max'), 'macosMajor', 'macosVersion'); + expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + }); + + it('comes before the memory', () => { + const hw = mac(8, 'Apple M1', { macosVersion: '12.7', macosMajor: 12 }); + expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('macos'); + }); + }); + + describe('memory', () => { + it('answers so at 8 GB, where nothing fits at any context', () => { + const result = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); + expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemory); + // The size is the decimal figure of the hardware line, without the unit. + expect(result.headline.vars).toEqual({ memory: '8.6' }); + expect(result.budgetBytes).toBe(2 * GIB); + expect(result.budgetGb).toBe('2.1'); + }); + + it('answers so when models fit, but only below the context a harness needs', () => { + // The 16 GB case of the Mac mini, with harnesses that are all present. + const result = recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED); + expect(result.reason).toBe('memory'); + expect(result.headline.vars).toEqual({ memory: '17.2' }); + }); + + it('comes before no-harness when nothing fits at all', () => { + expect(recommend(mac(8, 'Apple M2'), STOCK, []).reason).toBe('memory'); + }); + + it('answers so for an empty catalogue', () => { + expect(recommend(bigMac, catalogOf([]), [fact('alpha')]).reason).toBe('memory'); + }); + + it('answers so when the memory size is unknown', () => { + const hw = { ...mac(48, 'Apple M4 Max'), memBytes: 0, memGb: '0.0', memClassGb: 0 }; + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.reason).toBe('memory'); + expect(result.budgetBytes).toBe(0); + }); + }); + + describe('no-harness', () => { + const headline = 'No coding agent that Codeman can connect to a local model is available on this Mac.'; + + it('answers so when the caller knows no catalogue harness', () => { + const result = recommend(bigMac, STOCK, []); + expectNone(result, 'no-harness', headline); + expect(result.headline.vars).toBeUndefined(); + expect(result.budgetBytes).toBe(35 * GIB); + }); + + it('answers so when no harness is installed or installable', () => { + const facts = STOCK.harnesses.map((row) => fact(row.cli, { installed: false, installable: false })); + expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); + }); + + it('answers so when only the gated harness is on the machine', () => { + const gatedRows = STOCK.harnesses.filter((row) => row.gated); + expect(gatedRows).toHaveLength(1); + expect( + recommend( + bigMac, + STOCK, + gatedRows.map((row) => fact(row.cli)) + ).reason + ).toBe('no-harness'); + }); + + it('answers so when a harness is there but accepts no model id', () => { + const facts = ALL_INSTALLED.map((entry) => ({ ...entry, acceptsModelId: () => false })); + expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); + }); + + it('ignores a fact for a CLI that is not a catalogue harness', () => { + expect(recommend(bigMac, STOCK, [fact('codex'), fact('some-other-cli')]).reason).toBe('no-harness'); + }); + }); + + describe('disk', () => { + it('answers so when every fitting model would leave under 10 GiB free', () => { + // 12 GiB free: even the smallest download (6.55 GB) leaves 5.9 GiB. + const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expectNone( + result, + 'disk', + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.' + ); + // download + 10 GiB - free = 6,550,825,373 + 10,737,418,240 - 12,884,901,888 bytes. + expect(result.headline.vars).toEqual({ download: '6.6', free: '12.9', missing: '4.4' }); + expect(result.budgetBytes).toBe(35 * GIB); + }); + + it('names the smallest model that fits the MEMORY, not the smallest of the catalogue', () => { + // On a 48 GB Mac whose harness refuses both 9B ids, the smallest candidate is gpt-oss:20b. + const facts = ALL_INSTALLED.map((entry) => ({ + ...entry, + acceptsModelId: (id: string) => !id.startsWith('qwen3.5:9b'), + })); + const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); + const result = recommend(hw, STOCK, facts); + expect(result.reason).toBe('disk'); + expect(result.headline.vars?.download).toBe('13.8'); + }); + + it('drops only the models that do not leave 10 GiB, and picks among the rest', () => { + // 20 GiB free: the 9B downloads (6.1 and 9.7 GiB) leave 10 GiB or more, the others do not. + const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 20 * GIB, freeDiskGb: formatGb(20 * GIB) }); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(order(result)).toEqual(['qwen3.5:9b-q8_0@65536', 'qwen3.5:9b@65536']); + }); + + it('allows a download that leaves exactly 10 GiB', () => { + const exact = STOCK.models[0].bytes + DISK_HEADROOM_BYTES; + expect(DISK_HEADROOM_BYTES).toBe(10 * GIB); + const fits = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact }), STOCK, ALL_INSTALLED); + expect(fits.pick?.modelId).toBe('qwen3.5:9b'); + const short = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact - 1 }), STOCK, ALL_INSTALLED); + expect(short.reason).toBe('disk'); + }); + + it('does not block when the free disk is unknown', () => { + const hw = without(mac(48, 'Apple M4 Max'), 'freeDiskBytes', 'freeDiskGb'); + expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + }); + + it('formats the free size itself when the hardware carries only the bytes', () => { + const hw = without(mac(48, 'Apple M4 Max', { freeDiskBytes: 3 * GB }), 'freeDiskGb'); + expect(recommend(hw, STOCK, ALL_INSTALLED).headline.vars?.free).toBe('3.0'); + }); + + it('comes after the memory and the harness', () => { + const fullDisk = { freeDiskBytes: 0, freeDiskGb: '0.0' }; + expect(recommend(mac(16, 'Apple M4', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('memory'); + expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, []).reason).toBe('no-harness'); + expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('disk'); + }); + }); +}); + +// ---------- 4. Verdicts, ranking, harnesses, tested ---------- + +describe('verdict limited', () => { + it('is the answer when the best model that fits has fewer than 12 billion parameters', () => { + expect(LIMITED_BELOW_PARAMS_B).toBe(12); + const result = recommend(mac(24, 'Apple M4'), STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.reason).toBeUndefined(); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', paramsB: 9, context: 65536, fit: 'good' }); + expect(result.headline).toEqual({ + text: 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', + }); + expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); + }); + + it('holds for a tight fit too (18 GB)', () => { + const result = recommend(mac(18, 'Apple M3 Pro'), STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', fit: 'tight', fitLabel: 'Tight', budgetShare: 0.873 }); + expect(result.alternatives).toEqual([]); + }); + + it('draws the line at 12 billion: 11.9 is limited, 12 is ok', () => { + const verdictFor = (paramsB: number): string => + recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { paramsB })]), [fact('alpha')]).verdict; + expect(verdictFor(11.9)).toBe('limited'); + expect(verdictFor(12)).toBe('ok'); + expect(verdictFor(9)).toBe('limited'); + expect(verdictFor(27.8)).toBe('ok'); + }); + + it('is about the PICK: a larger model further down the list does not lift it', () => { + // 32 GB: the 20B fits only as Tight, so a 9B is the pick and the answer is limited. + const result = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.alternatives.map((c) => [c.modelId, c.fit])).toContainEqual(['gpt-oss:20b', 'tight']); + }); + + it('never says "Recommended", for any Mac', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + if (result.verdict === 'limited') { + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + } + } + }); +}); + +describe('speed', () => { + it('is the model figure times the chip bandwidth, rounded to one decimal', () => { + const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + // 2.95 x 410 / 100. + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', speed: 'ok', tokensPerSecond: 12.1 }); + expect(result.pick).not.toHaveProperty('speedLabel'); + }); + + it('is slow below 10 tokens per second, and says so', () => { + expect(SLOW_BELOW_TOKENS_PER_SECOND).toBe(10); + const speedOf = (speedPer100Gbps: number): Candidate | undefined => + recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { speedPer100Gbps })]), [fact('alpha')]).pick; + // The synthetic chip has 400 GB/s: 2.5 is 10.0 tokens per second, 2.475 is 9.9. + expect(speedOf(2.5)).toMatchObject({ speed: 'ok', tokensPerSecond: 10 }); + expect(speedOf(2.475)).toMatchObject({ speed: 'slow', tokensPerSecond: 9.9, speedLabel: 'Slow' }); + expect(speedOf(0.5)).toMatchObject({ speed: 'slow', tokensPerSecond: 2, speedLabel: 'Slow' }); + }); + + it('is unknown for a model without a figure, on any chip', () => { + // 48 GB: five models fit, so all of them are listed and gpt-oss:20b is among them. + for (const chip of ['Apple M1', 'Apple M4 Max', 'Apple M5 Ultra']) { + const result = recommend(mac(48, chip), STOCK, ALL_INSTALLED); + const gptOss = [result.pick, ...result.alternatives].find((c) => c?.modelId === 'gpt-oss:20b'); + expect(gptOss, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); + expect(gptOss).not.toHaveProperty('tokensPerSecond'); + } + }); + + it('is unknown for every model on a chip that is in no chips row: no bandwidth is guessed', () => { + for (const chip of ['Apple M6', 'Apple M6 Pro', 'Apple M10', 'Some Future Chip']) { + const result = recommend(mac(64, chip), STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + expect(candidate, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); + expect(candidate).not.toHaveProperty('tokensPerSecond'); + } + } + // And when the chip name itself is missing. + const result = recommend(without(mac(64, 'x'), 'chip'), STOCK, ALL_INSTALLED); + expect(result.pick?.speed).toBe('unknown'); + }); +}); + +describe('the ranking', () => { + it('puts ok before unknown before slow, ahead of the model rank', () => { + // Three models that fit equally well. The ranks run AGAINST the speed. + const catalog = catalogOf([ + model('slow-top:1', { rank: 3, speedPer100Gbps: 1 }), + model('unknown-mid:1', { rank: 2 }), + model('ok-low:1', { rank: 1, speedPer100Gbps: 5 }), + ]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(order(result)).toEqual(['ok-low:1@1024', 'unknown-mid:1@1024', 'slow-top:1@1024']); + expect([result.pick, ...result.alternatives].map((c) => c?.speed)).toEqual(['ok', 'unknown', 'slow']); + }); + + it('ranks by model rank alone when every speed is unknown', () => { + // 'Apple M6' is in no chips row. + const result = recommend(mac(48, 'Apple M6'), STOCK, ALL_INSTALLED); + expect(order(result)).toEqual([ + 'qwen3.8:27b@131072', + 'qwen3.6:35b-a3b@131072', + 'gpt-oss:20b@131072', + 'qwen3.5:9b-q8_0@65536', + 'qwen3.5:9b@65536', + ]); + }); + + it('ranks an unknown speed below a known ok one and above a slow one, on a real tier', () => { + // 48 GB, M4 Pro: three ok models, then gpt-oss:20b (unknown), then the 27B (slow). + const result = recommend(mac(48, 'Apple M4 Pro'), STOCK, ALL_INSTALLED); + expect([result.pick, ...result.alternatives].map((c) => [c?.modelId, c?.speed])).toEqual([ + ['qwen3.6:35b-a3b', 'ok'], + ['qwen3.5:9b-q8_0', 'ok'], + ['qwen3.5:9b', 'ok'], + ['gpt-oss:20b', 'unknown'], + ['qwen3.8:27b', 'slow'], + ]); + }); + + it('puts a good fit before a tight one, ahead of speed and rank', () => { + const catalog = catalogOf([ + modelNeeding('tight-fast-top:1', 0.9 * HUNDRED_GIB, { rank: 9, speedPer100Gbps: 20 }), + modelNeeding('good-slow-low:1', 0.5 * HUNDRED_GIB, { rank: 1, speedPer100Gbps: 1 }), + ]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(order(result)).toEqual(['good-slow-low:1@1024', 'tight-fast-top:1@1024']); + }); + + it('takes the higher model rank among equals', () => { + const catalog = catalogOf([model('low:1', { rank: 1 }), model('high:1', { rank: 5 }), model('mid:1', { rank: 3 })]); + expect(order(recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]))).toEqual([ + 'high:1@1024', + 'mid:1@1024', + 'low:1@1024', + ]); + }); + + it('takes the larger context of one model when both fit equally well', () => { + // 1 KiB per token: 4 GiB of KV cache at 4,194,304 tokens, still a good fit. + const catalog = catalogOf([model('probe:1', { kvBytesPerToken: 1024, contexts: [4_194_304, 1024] })]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(result.pick?.context).toBe(4_194_304); + expect(result.alternatives).toEqual([]); + }); + + it('steps a model down to the context at which it fits well', () => { + // Good at the small context, tight at the large one: the good one is that model's entry. + const catalog = catalogOf([ + model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] }), + ]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(result.pick).toMatchObject({ context: 1024, fit: 'good' }); + }); + + it('lists each model once, never the pick again, and at most four alternatives', () => { + expect(MAX_ALTERNATIVES).toBe(4); + // Six models fit a 128 GB Mac; the pick plus four are shown. + const result = recommend(mac(128, 'Apple M5 Ultra'), STOCK, ALL_INSTALLED); + expect(result.alternatives).toHaveLength(4); + const ids = [result.pick, ...result.alternatives].map((c) => c?.modelId); + expect(new Set(ids).size).toBe(ids.length); + for (const hw of SWEEP) { + const swept = recommend(hw, STOCK, ALL_INSTALLED); + const sweptIds = [swept.pick, ...swept.alternatives].filter(Boolean).map((c) => c?.modelId); + expect(new Set(sweptIds).size).toBe(sweptIds.length); + expect(swept.alternatives.length).toBeLessThanOrEqual(MAX_ALTERNATIVES); + } + }); +}); + +describe('harnesses', () => { + const hw = mac(48, 'Apple M4 Max'); + + it('never chooses a gated harness, even when it is the only one that could run a model', () => { + const gated = STOCK.harnesses.filter((row) => row.gated).map((row) => row.cli); + expect(gated).toEqual(['pi']); + for (const swept of SWEEP) { + const result = recommend(swept, STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + if (candidate) expect(gated).not.toContain(candidate.cli); + } + } + // 16 GB: the gated harness alone works at 32768 tokens, where the 9B would fit. Still none. + expect(recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED).verdict).toBe('none'); + }); + + it('never offers a context below the minimum of the harness', () => { + for (const swept of SWEEP) { + const result = recommend(swept, STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + if (!candidate) continue; + const row = STOCK.harnesses.find((harness) => harness.cli === candidate.cli); + expect(candidate.context).toBeGreaterThanOrEqual(row?.minContext ?? Number.POSITIVE_INFINITY); + } + } + }); + + it('takes the higher-ranked harness and reports its label and that it is installed', () => { + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.pick).toMatchObject({ cli: 'opencode', cliLabel: 'OpenCode', cliInstalled: true }); + expect(result.pick).not.toHaveProperty('slowFirstTurn'); + }); + + it('falls back to the next harness when the first is not on the machine, and flags its slow first turn', () => { + const facts = [fact('opencode', { installed: false, installable: false }), fact('claude'), fact('pi')]; + const result = recommend(hw, STOCK, facts); + expect(result.pick).toMatchObject({ + modelId: 'qwen3.8:27b', + cli: 'claude', + cliLabel: 'Claude Code', + cliInstalled: true, + slowFirstTurn: true, + }); + // The same when the caller has no fact for it at all. + expect(recommend(hw, STOCK, [fact('claude')]).pick?.cli).toBe('claude'); + }); + + it('offers a harness that is not installed but can be, and says it is not installed', () => { + const facts = [fact('opencode', { installed: false, installable: true }), fact('claude')]; + const result = recommend(hw, STOCK, facts); + expect(result.pick).toMatchObject({ cli: 'opencode', cliInstalled: false }); + }); + + it('pairs a model only with a harness that accepts its id', () => { + const refusing27b = (id: string): boolean => id !== 'qwen3.8:27b'; + // OpenCode refuses the 27B: it goes with the next harness. + const one = recommend(hw, STOCK, [fact('opencode', { acceptsModelId: refusing27b }), fact('claude')]); + expect(one.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'claude' }); + expect(one.alternatives[0]).toMatchObject({ modelId: 'qwen3.6:35b-a3b', cli: 'opencode' }); + // Both refuse it: the model is not offered. + const both = recommend(hw, STOCK, [ + fact('opencode', { acceptsModelId: refusing27b }), + fact('claude', { acceptsModelId: refusing27b }), + ]); + expect(both.pick?.modelId).toBe('qwen3.6:35b-a3b'); + expect(order(both)).not.toContain('qwen3.8:27b@131072'); + }); + + it('treats a harness check that throws as a refusal, and still answers', () => { + const throwing = fact('opencode', { + acceptsModelId: () => { + throw new Error('registry not loaded'); + }, + }); + const result = recommend(hw, STOCK, [throwing, fact('claude')]); + expect(result.verdict).toBe('ok'); + expect(result.pick?.cli).toBe('claude'); + expect(recommend(hw, STOCK, [throwing]).reason).toBe('no-harness'); + }); + + it('puts the fit and the model before the harness rank', () => { + // `big` needs a large context and is the higher-ranked harness; `small` works from 1024. + const harnesses: LocalHarnessEntry[] = [ + { cli: 'big', minContext: 8_388_608, rank: 2 }, + { cli: 'small', minContext: 1024, rank: 1 }, + ]; + const tightAtLarge = catalogOf( + [model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })], + harnesses + ); + // Tight at the context `big` needs, good at the small one: the good fit with `small` wins. + expect(recommend(HUNDRED_GIB_MAC, tightAtLarge, [fact('big'), fact('small')]).pick).toMatchObject({ + cli: 'small', + context: 1024, + fit: 'good', + }); + // Good at both: the higher-ranked harness wins, at the context it needs. + const goodAtBoth = catalogOf( + [model('probe:1', { bytes: 10 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })], + harnesses + ); + expect(recommend(HUNDRED_GIB_MAC, goodAtBoth, [fact('big'), fact('small')]).pick).toMatchObject({ + cli: 'big', + context: 8_388_608, + }); + }); +}); + +describe('the tested flag and the word "Recommended"', () => { + const testedOn = (chip: string, memGb: number, cli: string) => ({ chip, memGb, cli, date: '2026-11-06' }); + const with27bTested = (chip: string, memGb: number, cli: string): LocalModelCatalog => + stockWith((catalog) => { + const row = catalog.models.find((m) => m.id === 'qwen3.8:27b') as LocalModelEntry; + row.tested = [testedOn(chip, memGb, cli)]; + }); + + it('is off for every model of the stock catalogue, so "Recommended" is never said today', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + if (candidate) expect(candidate.tested).toBe(false); + } + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + if (result.verdict !== 'none') { + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + } + } + }); + + it('turns on "Recommended for this Mac" when the pick was tested on exactly such a Mac', () => { + const result = recommend(mac(48, 'Apple M4 Max'), with27bTested('Apple M4 Max', 48, 'opencode'), ALL_INSTALLED); + expect(result.verdict).toBe('ok'); + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: true }); + expect(result.headline).toEqual({ text: 'Recommended for this Mac' }); + // The "not yet tested" note goes away with it. + expect(result.notes).toEqual([]); + }); + + it('stays off when the chip, the memory size or the harness differs', () => { + const hw = mac(48, 'Apple M4 Max'); + const cases: LocalModelCatalog[] = [ + with27bTested('Apple M4 Pro', 48, 'opencode'), + with27bTested('Apple M4', 48, 'opencode'), + with27bTested('Apple M4 Max', 64, 'opencode'), + with27bTested('Apple M4 Max', 48, 'claude'), + ]; + for (const catalog of cases) { + const result = recommend(hw, catalog, ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: false }); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); + } + }); + + it('follows the harness: tested with the second harness counts when that one is the pick', () => { + const catalog = with27bTested('Apple M4 Max', 48, 'claude'); + const result = recommend(mac(48, 'Apple M4 Max'), catalog, [fact('claude')]); + expect(result.pick).toMatchObject({ cli: 'claude', tested: true }); + expect(result.headline.text).toBe('Recommended for this Mac'); + }); + + it('does not change the ranking: a tested model further down stays an alternative', () => { + // On the M4 Pro the pick is the 35B-A3B. A tested 27B is flagged in the list, no more. + const result = recommend(mac(48, 'Apple M4 Pro'), with27bTested('Apple M4 Pro', 48, 'opencode'), ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', tested: false }); + expect(result.headline.text).toBe('Best fit for this Mac'); + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + expect(result.alternatives.find((c) => c.modelId === 'qwen3.8:27b')?.tested).toBe(true); + }); + + it('never says "Recommended" for a limited answer, tested or not', () => { + const catalog = stockWith((stock) => { + const row = stock.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; + row.tested = [testedOn('Apple M4', 24, 'opencode')]; + }); + const result = recommend(mac(24, 'Apple M4'), catalog, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.pick?.tested).toBe(true); + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + // It WAS tested, so the "not yet tested" note would be false. + expect(result.notes).toEqual([]); + }); +}); + +describe('the shape of the answer', () => { + const labels = new Set(Object.values(LOCAL_MODEL_LABELS)); + + it('takes every sentence and label from the fixed list', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(labels.has(result.headline.text)).toBe(true); + for (const note of result.notes) expect(labels.has(note.text)).toBe(true); + for (const candidate of [result.pick, ...result.alternatives]) { + if (!candidate) continue; + expect(labels.has(candidate.fitLabel)).toBe(true); + if (candidate.speedLabel !== undefined) expect(labels.has(candidate.speedLabel)).toBe(true); + expect(candidate.speedLabel === undefined).toBe(candidate.speed === 'ok'); + expect(candidate.tokensPerSecond === undefined).toBe(candidate.speed === 'unknown'); + } + } + }); + + it('fills every placeholder of a headline and nothing else', () => { + const answers = [ + recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { macosMajor: 13, macosVersion: '13.0' }), STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { freeDiskBytes: GB, freeDiskGb: '1.0' }), STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4'), STOCK, []), + recommend(mac(48, 'Apple M4'), STOCK, ALL_INSTALLED), + ]; + expect(answers.map((answer) => answer.reason ?? answer.verdict)).toEqual([ + 'memory', + 'macos', + 'disk', + 'no-harness', + 'ok', + ]); + for (const answer of answers) { + const slots = [...answer.headline.text.matchAll(/\{(\w+)\}/g)].map((match) => match[1]).sort(); + expect(Object.keys(answer.headline.vars ?? {}).sort()).toEqual(slots); + // `{name}` belongs to the browser's translator. + expect(slots).not.toContain('name'); + for (const value of Object.values(answer.headline.vars ?? {})) expect(typeof value).toBe('string'); + } + }); + + it('prints every size in one unit: decimal GB from the one formatter, never GiB', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.budgetGb).toBe(formatGb(result.budgetBytes)); + for (const candidate of [result.pick, ...result.alternatives]) { + if (!candidate) continue; + expect(candidate.needGb).toBe(formatGb(candidate.needBytes)); + expect(candidate.downloadGb).toBe(formatGb(candidate.downloadBytes)); + expect(candidate.budgetShare).toBeCloseTo(candidate.needBytes / result.budgetBytes, 3); + } + expect(JSON.stringify(result)).not.toMatch(/GiB|MiB/); + } + }); + + it('adds up for a person: download, plus what the context and the runtime take, is the need', () => { + const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const pick = result.pick as Candidate; + // 17.7 GB of weights + 8 GiB of KV cache + 1 GiB + 2 GiB of prompt cache, all in decimal GB. + expect(pick.downloadGb).toBe('17.7'); + expect(pick.needGb).toBe('29.6'); + expect(formatGb(pick.needBytes - pick.downloadBytes)).toBe('11.8'); + expect(result.budgetGb).toBe('37.6'); + expect(pick.needBytes).toBeLessThan(result.budgetBytes); + }); + + it('is plain JSON and has exactly the fields of its verdict', () => { + const ok = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + expect(JSON.parse(JSON.stringify(ok))).toEqual(ok); + expect(Object.keys(ok).sort()).toEqual( + ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'pick', 'verdict'].sort() + ); + const none = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); + expect(JSON.parse(JSON.stringify(none))).toEqual(none); + expect(Object.keys(none).sort()).toEqual( + ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'reason', 'verdict'].sort() + ); + }); + + it('is pure: frozen inputs are enough, and the same inputs give the same answer', () => { + const hw = deepFreeze(mac(48, 'Apple M4 Pro')); + const catalog = deepFreeze(structuredClone(STOCK_LOCAL_MODEL_CATALOG)); + const facts = deepFreeze(ALL_INSTALLED.map((entry) => ({ ...entry }))); + const first = recommend(hw, catalog, facts); + const second = recommend(hw, catalog, facts); + expect(second).toEqual(first); + expect(first.verdict).toBe('ok'); + }); +});