/** * @fileoverview Tests for src/local-model/recommend.ts: the pure function that turns "what this * Mac is" into "which local model fits it". * * Laid out like the module: * * 1. The budget (`deviceBudget`) for every memory size Apple sells from 8 to 128 GB, and the * need of a model at a context (`needBytes`). * 2. The two REAL Macs of the project, from their captured `sysctl` output through the real * parser: the MacBook Pro (M5 Pro, 48 GB) and the Mac mini (M4, 16 GB). Each test states the * expected answer and why. * 3. Every reason for verdict `none`, one fixture per reason, and their order. * 4. The `limited` verdict, the three-state speed in the ranking, harness rows (a gated one is * never chosen), the `tested` flag that alone may turn on the word "Recommended", and the * shape of the answer (labels from the fixed list, sizes in one unit). * * The stock catalogue is used wherever the answer for a real Mac is at stake. Small synthetic * catalogues (validated by the real schema) isolate one rule of the ranking at a time. * * Port: N/A (pure functions; the two hardware fixtures go through `readHardware` with every * piece of IO injected). */ import { beforeEach, describe, expect, it } from 'vitest'; import { getLocalModelCatalog, LocalModelCatalogSchema, STOCK_LOCAL_MODEL_CATALOG, } from '../src/config/local-model-catalog.js'; import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js'; import { readHardware, resetHardwareCache } from '../src/local-model/hardware.js'; import { deviceBudget, DISK_HEADROOM_BYTES, FIT_GOOD_MAX_SHARE, FIT_TIGHT_MAX_SHARE, LIMITED_BELOW_PARAMS_B, MAX_ALTERNATIVES, needBytes, recommend, SLOW_BELOW_TOKENS_PER_SECOND, } from '../src/local-model/recommend.js'; import { LOCAL_MODEL_LABELS, type Candidate, type HardwareFacts, type HarnessFact, type LocalHarnessEntry, type LocalModelCatalog, type LocalModelEntry, type Recommendation, } from '../src/types.js'; // ---------- Fixtures ---------- /** The real probe output of the MacBook Pro (Mac17,13), captured 2026-11-06, extra lines included. */ const MACBOOK_SYSCTL = `hw.memsize: 51539607552 machdep.cpu.brand_string: Apple M5 Pro hw.optional.arm64: 1 sysctl.proc_translated: 0 iogpu.wired_limit_mb: 0 kern.osproductversion: 27.2 hw.model: Mac17,13 hw.perflevel0.physicalcpu: 6 hw.perflevel1.physicalcpu: 12 exit=0 /dev/disk3s5 971094180 54383564 890926084 6% 542146 8909260840 0% /System/Volumes/Data `; /** The real probe output of the Mac mini (Mac16,10), captured 2026-11-06, extra lines included. */ const MACMINI_SYSCTL = `hw.memsize: 17179869184 machdep.cpu.brand_string: Apple M4 hw.optional.arm64: 1 sysctl.proc_translated: 0 iogpu.wired_limit_mb: 0 kern.osproductversion: 27.2 hw.model: Mac16,10 hw.perflevel0.physicalcpu: 4 hw.perflevel1.physicalcpu: 6 exit=0 /dev/disk3s5 239362496 172954124 41752848 81% 2353838 417528480 1% /System/Volumes/Data `; /** The hardware facts of a real Mac: its captured output through the real parser, no IO. */ async function realMac(sysctlOutput: string, freeDiskKb: number): Promise { resetHardwareCache(); return readHardware({ platform: 'darwin', arch: 'arm64', runSysctl: async () => sysctlOutput, freeDiskBytes: async () => freeDiskKb * 1024, totalMemBytes: () => 0, cpuModel: () => undefined, now: () => 0, }); } /** An Apple silicon Mac with `soldGb` of memory, healthy in every other respect. */ function mac(soldGb: number, chip: string, overrides: Partial = {}): HardwareFacts { const memBytes = soldGb * GIB; return { platform: 'darwin', arch: 'arm64', arm64: true, translated: false, memBytes, memGb: formatGb(memBytes), memClassGb: memClassGb(memBytes), chip, model: 'Mac0,0', macosVersion: '27.2', macosMajor: 27, wiredLimitMb: 0, freeDiskBytes: 500 * GB, freeDiskGb: '500', source: 'sysctl', ...overrides, }; } /** A copy of `facts` without the given fields (an absent field, not an `undefined` one). */ function without(facts: HardwareFacts, ...fields: (keyof HardwareFacts)[]): HardwareFacts { const copy: Record = { ...facts }; for (const field of fields) delete copy[field]; return copy as unknown as HardwareFacts; } /** Display labels of the stock harnesses, as the CLI registry would supply them. Data, not a branch. */ const HARNESS_LABELS: Record = { opencode: 'OpenCode', claude: 'Claude Code', pi: 'Pi' }; /** What the server knows about one harness: installed and accepting every model id, unless overridden. */ function fact(cli: string, overrides: Partial = {}): HarnessFact { return { cli, label: HARNESS_LABELS[cli] ?? cli, installed: true, installable: false, acceptsModelId: () => true, ...overrides, }; } const STOCK = getLocalModelCatalog(); /** Every stock harness installed, the gated one included. */ const ALL_INSTALLED: HarnessFact[] = STOCK.harnesses.map((row) => fact(row.cli)); /** The stock catalogue with a change applied, validated by the real schema. */ function stockWith(change: (catalog: LocalModelCatalog) => void): LocalModelCatalog { const catalog = structuredClone(STOCK_LOCAL_MODEL_CATALOG); change(catalog); return LocalModelCatalogSchema.parse(catalog); } /** A model row for a synthetic catalogue: 20 billion parameters, 10 GiB, one small context. */ function model(id: string, overrides: Partial = {}): LocalModelEntry { return { id, label: id, license: 'Apache-2.0', paramsB: 20, bytes: 10 * GIB, kvBytesPerToken: 1, contexts: [1024], rank: 1, tested: [], ...overrides, }; } /** * A synthetic catalogue: no prompt cache, one harness `alpha` that works from 1024 tokens, and * one chip row ('M4 Max' at 400 GB/s, so tokens per second = `speedPer100Gbps` x 4). */ function catalogOf(models: LocalModelEntry[], harnesses?: LocalHarnessEntry[]): LocalModelCatalog { return LocalModelCatalogSchema.parse({ schema: 1, updated: '2026-11-06', minMacosMajor: 14, promptCacheTiers: [{ minMemGb: 0, bytes: 0 }], models, harnesses: harnesses ?? [{ cli: 'alpha', minContext: 1024, rank: 1 }], chips: [{ match: 'M4 Max', gbps: 400 }], }); } /** A Mac whose budget is exactly 100 GiB: a wired limit of 101 GiB on a 128 GB machine. */ const HUNDRED_GIB_MAC = mac(128, 'Apple M4 Max', { wiredLimitMb: 101 * 1024 }); const HUNDRED_GIB = 100 * GIB; /** What a synthetic model needs beside its weights at its 1024-token context: 1 GiB plus the KV cache. */ const SYNTHETIC_OVERHEAD = GIB + 1024; /** A model whose need is exactly `need` bytes in a synthetic catalogue. */ function modelNeeding(id: string, need: number, overrides: Partial = {}): LocalModelEntry { return model(id, { bytes: need - SYNTHETIC_OVERHEAD, ...overrides }); } /** `id@context` of the pick and of each alternative, in order. */ function order(result: Recommendation): string[] { return [result.pick, ...result.alternatives] .filter((candidate): candidate is Candidate => candidate !== undefined) .map((candidate) => `${candidate.modelId}@${candidate.context}`); } /** Every string anywhere in a value. */ function stringsIn(value: unknown): string[] { if (typeof value === 'string') return [value]; if (Array.isArray(value)) return value.flatMap(stringsIn); if (value && typeof value === 'object') return Object.values(value).flatMap(stringsIn); return []; } function deepFreeze(value: T): T { if (value && typeof value === 'object') { for (const inner of Object.values(value)) deepFreeze(inner); Object.freeze(value); } return value; } /** The Macs the sweeps run over: every memory size, with a slow, a fast and an unknown chip. */ const SWEEP: HardwareFacts[] = [8, 16, 18, 24, 32, 36, 48, 64, 96, 128].flatMap((soldGb) => ['Apple M4', 'Apple M4 Max', 'Apple M5 Ultra', 'Apple M6'].map((chip) => mac(soldGb, chip)) ); beforeEach(() => { resetHardwareCache(); }); // ---------- 1. Budget and need ---------- describe('deviceBudget', () => { // Sold size, device limit L and budget B in GiB. L is 2/3 of memory below 36 GB and 3/4 from // 36 GB; B = min(L - 1, memory - reserve). Only at 8 GB is the reserve the smaller bound. const TIERS: [soldGb: number, limitGib: number, budgetGib: number][] = [ [8, 16 / 3, 2], [16, 32 / 3, 29 / 3], [18, 12, 11], [24, 16, 15], [32, 64 / 3, 61 / 3], [36, 27, 26], [48, 36, 35], [64, 48, 47], [96, 72, 71], [128, 96, 95], ]; it.each(TIERS)('gives a %i GB Mac a limit of %f GiB and a budget of %f GiB', (soldGb, limitGib, budgetGib) => { const budget = deviceBudget(mac(soldGb, 'Apple M4')); // Whole bytes: a third of a byte is rounded down. expect(Math.abs(budget.limitBytes - limitGib * GIB)).toBeLessThan(1); expect(Math.abs(budget.budgetBytes - budgetGib * GIB)).toBeLessThan(1); expect(Number.isInteger(budget.limitBytes)).toBe(true); expect(Number.isInteger(budget.budgetBytes)).toBe(true); expect(budget.source).toBe('estimated'); }); it('matches the rounded budgets of the design table', () => { const rounded = TIERS.map(([soldGb]) => [ soldGb, (deviceBudget(mac(soldGb, 'Apple M4')).budgetBytes / GIB).toFixed(1), ]); expect(rounded).toEqual([ [8, '2.0'], [16, '9.7'], [18, '11.0'], [24, '15.0'], [32, '20.3'], [36, '26.0'], [48, '35.0'], [64, '47.0'], [96, '71.0'], [128, '95.0'], ]); }); it('keeps a reserve of 6 GiB up to 18 GB, 8 GiB up to 48 GB and 12 GiB from 64 GB', () => { // A wired limit far above the memory leaves "memory - reserve" as the only bound. const reserveOf = (soldGb: number): number => { const budget = deviceBudget(mac(soldGb, 'Apple M4', { wiredLimitMb: 1024 * 1024 })); return (soldGb * GIB - budget.budgetBytes) / GIB; }; expect([8, 16, 18].map(reserveOf)).toEqual([6, 6, 6]); expect([24, 32, 36, 48].map(reserveOf)).toEqual([8, 8, 8, 8]); expect([64, 96, 128].map(reserveOf)).toEqual([12, 12, 12]); }); it('uses the wired limit when it is above 0', () => { // 14000 MiB on the 16 GB Mac: L - 1 GiB is 12.67 GiB, memory - reserve is 10 GiB. expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 14000 }))).toEqual({ limitBytes: 14000 * MIB, budgetBytes: 10 * GIB, source: 'wired-limit', }); // A limit BELOW the estimate is honoured too: 8192 MiB leaves 7 GiB. expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 8192 }))).toEqual({ limitBytes: 8 * GIB, budgetBytes: 7 * GIB, source: 'wired-limit', }); }); it('reads a wired limit of 0, or none at all, as "not set"', () => { const estimated = deviceBudget(mac(16, 'Apple M4')); expect(estimated.source).toBe('estimated'); expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 0 }))).toEqual(estimated); expect(deviceBudget(without(mac(16, 'Apple M4'), 'wiredLimitMb'))).toEqual(estimated); }); it('never goes below 0', () => { expect(deviceBudget(mac(4, 'Apple M1')).budgetBytes).toBe(0); expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 512 })).budgetBytes).toBe(0); expect(deviceBudget({ ...mac(16, 'Apple M4'), memBytes: 0, memClassGb: 0 })).toEqual({ limitBytes: 0, budgetBytes: 0, source: 'estimated', }); }); }); describe('needBytes', () => { const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; it('adds weights, the KV cache of the context, 1 GiB and the prompt cache', () => { // 6,550,825,373 + 32768 x 65536 + 1 GiB + 512 MiB on a Mac below 32 GB. expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + 2 * GIB + GIB + 512 * MIB); expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(10_308_921_757); // Half the context, half the KV cache. expect(needBytes(nineB, 32768, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + GIB + GIB + 512 * MIB); }); it('counts a prompt cache of 512 MiB below 32 GB of memory and 2 GiB from 32 GB', () => { const base = 6_550_825_373 + 2 * GIB + GIB; for (const soldGb of [8, 16, 18, 24]) { expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(512 * MIB); } for (const soldGb of [32, 36, 48, 64, 96, 128]) { expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(2 * GIB); } }); it('does not depend on the chip or the harness', () => { expect(needBytes(nineB, 65536, STOCK, mac(48, 'Apple M1'))).toBe( needBytes(nineB, 65536, STOCK, mac(48, 'Apple M5 Ultra')) ); }); }); describe('the fit grade', () => { const pickOf = (need: number): Candidate | undefined => recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', need)]), [fact('alpha')]).pick; it('has the budget this group assumes', () => { expect(deviceBudget(HUNDRED_GIB_MAC).budgetBytes).toBe(HUNDRED_GIB); expect(FIT_GOOD_MAX_SHARE).toBe(0.85); expect(FIT_TIGHT_MAX_SHARE).toBe(0.98); }); it('is good up to 0.85 of the budget', () => { expect(pickOf(0.5 * HUNDRED_GIB)).toMatchObject({ fit: 'good', fitLabel: 'Good', budgetShare: 0.5 }); expect(pickOf(0.85 * HUNDRED_GIB)).toMatchObject({ fit: 'good', budgetShare: 0.85 }); }); it('is tight above 0.85 and up to 0.98', () => { expect(pickOf(0.85 * HUNDRED_GIB + 1)).toMatchObject({ fit: 'tight', fitLabel: 'Tight' }); expect(pickOf(0.9 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.9 }); expect(pickOf(0.98 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.98 }); }); it('is out above 0.98: the model is not offered at all', () => { const result = recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', 0.98 * HUNDRED_GIB + 1)]), [ fact('alpha'), ]); expect(result.verdict).toBe('none'); expect(result.reason).toBe('memory'); expect(result.pick).toBeUndefined(); }); }); // ---------- 2. The two real Macs ---------- describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { /** * Expected: Qwen3.6 35B-A3B at 131,072 tokens with OpenCode, Good, about 42 tokens per second. * * Why. The budget is 35 GiB (3/4 of 48 GiB minus 1). Five models fit it as Good. The M5 Pro is * in the chips table (307 GB/s), so each model with a speed figure gets an estimate: * the dense 27B computes to 2.95 x 3.07 = 9.1 tokens per second, under the 10 that separate * slow from ok, while the 35B-A3B (3 billion parameters active per token) computes to 41.8. * Speed ranks before model rank, so the 35B-A3B wins although the 27B has the higher rank. The * 27B stays in the list, last, marked Slow. Both harnesses can run it; OpenCode has the * higher rank. */ it('gets Qwen3.6 35B-A3B at 131072 tokens with OpenCode', async () => { const hw = await realMac(MACBOOK_SYSCTL, 890_926_084); expect(hw.chip).toBe('Apple M5 Pro'); expect(hw.memClassGb).toBe(48); const result = recommend(hw, STOCK, ALL_INSTALLED); expect(result.verdict).toBe('ok'); expect(result.reason).toBeUndefined(); expect(result.budgetBytes).toBe(35 * GIB); expect(result.budgetGb).toBe('37.6'); expect(result.budgetSource).toBe('estimated'); expect(result.pick).toEqual({ modelId: 'qwen3.6:35b-a3b', modelLabel: 'Qwen3.6 35B-A3B', license: 'Apache-2.0', paramsB: 35, context: 131072, cli: 'opencode', cliLabel: 'OpenCode', cliInstalled: true, // 22,621,314,161 + 20480 x 131072 + 1 GiB + 2 GiB. needBytes: 28_526_894_193, needGb: '28.5', downloadBytes: 22_621_314_161, downloadGb: '22.6', budgetShare: 0.759, fit: 'good', fitLabel: 'Good', speed: 'ok', tokensPerSecond: 41.8, tested: false, }); }); it('says "Best fit", not "Recommended", and that nobody tested it on such a Mac', async () => { const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED); expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); }); it('lists the other four models that fit: the fast ones, the one of unknown speed, the slow one', async () => { const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED); expect(result.alternatives.map((c) => [c.modelId, c.context, c.fit, c.speed, c.tokensPerSecond ?? null])).toEqual([ ['qwen3.5:9b-q8_0', 65536, 'good', 'ok', 17.8], ['qwen3.5:9b', 65536, 'good', 'ok', 28.6], ['gpt-oss:20b', 131072, 'good', 'unknown', null], ['qwen3.8:27b', 131072, 'good', 'slow', 9.1], ]); expect(result.alternatives[2].speedLabel).toBe('Speed unknown'); expect(result.alternatives[3].speedLabel).toBe('Slow'); // The two that do not fit 35 GiB at any context are not listed. expect(order(result)).not.toContain('qwen3.8:27b-q8_0@131072'); expect(order(result).some((entry) => entry.startsWith('qwen3.5:122b'))).toBe(false); }); it('would get the 27B instead if its chip were in no chips row', async () => { // The same Mac with the M5 Pro row removed: every speed is unknown, so model rank decides. const hw = await realMac(MACBOOK_SYSCTL, 890_926_084); const withoutChip = stockWith((catalog) => { catalog.chips = catalog.chips.filter((row) => row.match !== 'M5 Pro'); }); const result = recommend(hw, withoutChip, ALL_INSTALLED); expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', context: 131072, speed: 'unknown' }); expect(result.pick).not.toHaveProperty('tokensPerSecond'); }); }); describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { /** * Expected: no recommendation, reason `memory`. * * Why. The budget is 9.67 GiB (2/3 of 16 GiB minus 1). The only model near it is the 9B. Both * harnesses that may be offered need 65,536 tokens of context, and there the 9B needs * 6.10 GiB of weights + 2 GiB of KV cache + 1 GiB + 0.5 GiB of prompt cache = 9.60 GiB, which * is 0.993 of the budget: above the 0.98 where "tight" ends. At 32,768 tokens it would fit * (0.890, tight), but the one harness that works at that context is gated. */ it('gets no recommendation: nothing fits at the context a harness needs', async () => { const hw = await realMac(MACMINI_SYSCTL, 41_752_848); expect(hw.chip).toBe('Apple M4'); expect(hw.memClassGb).toBe(16); const result = recommend(hw, STOCK, ALL_INSTALLED); expect(result).toEqual({ verdict: 'none', reason: 'memory', headline: { text: 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', vars: { memory: '17.2' }, }, notes: [], alternatives: [], // floor(17,179,869,184 x 2/3) - 1 GiB. budgetBytes: 10_379_504_298, budgetGb: '10.4', budgetSource: 'estimated', }); }); it('misses by a hair at 65536 tokens and would fit at 32768', async () => { const hw = await realMac(MACMINI_SYSCTL, 41_752_848); const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; const budget = deviceBudget(hw).budgetBytes; const shareAt64k = needBytes(nineB, 65536, STOCK, hw) / budget; const shareAt32k = needBytes(nineB, 32768, STOCK, hw) / budget; expect(shareAt64k).toBeCloseTo(0.9932, 4); expect(shareAt64k).toBeGreaterThan(FIT_TIGHT_MAX_SHARE); expect(shareAt32k).toBeCloseTo(0.8898, 4); expect(shareAt32k).toBeLessThan(FIT_TIGHT_MAX_SHARE); // Every other model needs more than the whole budget at its smallest context. for (const other of STOCK.models.filter((m) => m.id !== 'qwen3.5:9b')) { const smallest = other.contexts[other.contexts.length - 1]; expect(needBytes(other, smallest, STOCK, hw), other.id).toBeGreaterThan(budget); } }); it('would get the 9B at 32768 tokens, as a limited answer, once the gated harness is opened', async () => { const hw = await realMac(MACMINI_SYSCTL, 41_752_848); const ungated = stockWith((catalog) => { catalog.harnesses = catalog.harnesses.map((row) => ({ cli: row.cli, minContext: row.minContext, rank: row.rank, })); }); const result = recommend(hw, ungated, ALL_INSTALLED); expect(result.verdict).toBe('limited'); expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', context: 32768, cli: 'pi', fit: 'tight', budgetShare: 0.89, speed: 'ok', // 9.3 x 1.2. tokensPerSecond: 11.2, }); }); it('would get the 9B at 65536 tokens if a wired limit had been set on it', async () => { // iogpu.wired_limit_mb = 14000 lifts the budget to 10 GiB (memory minus the 6 GiB reserve). const hw = { ...(await realMac(MACMINI_SYSCTL, 41_752_848)), wiredLimitMb: 14000 }; const result = recommend(hw, STOCK, ALL_INSTALLED); expect(result.verdict).toBe('limited'); expect(result.budgetSource).toBe('wired-limit'); expect(result.budgetBytes).toBe(10 * GIB); expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', context: 65536, cli: 'opencode', fit: 'tight' }); }); }); describe('every memory size, with the stock catalogue', () => { // What the rules compute today. A change of a catalogue figure or of a threshold that moves // one of these answers has to change this table on purpose. const EXPECTED: [soldGb: number, chip: string, answer: string][] = [ [8, 'Apple M2', 'none memory'], [16, 'Apple M4', 'none memory'], // The 9B at 65536 tokens is 0.87 of 11 GiB. [18, 'Apple M3 Pro', 'limited qwen3.5:9b@65536 tight ok'], // gpt-oss:20b at 65536 tokens needs 15.8 GiB of the 15 GiB budget: out. [24, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], // gpt-oss:20b fits only as Tight here (0.853 at 65536 tokens, 0.927 at 131072), and Good // ranks before Tight. The Q8 9B is Good but slow on a base M4 (7 tokens per second). [32, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], [32, 'Apple M1 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], // The 27B (0.905) and the 35B-A3B (0.974) are Tight; gpt-oss:20b is Good but has no speed // figure, and a known speed ranks before an unknown one. [36, 'Apple M4 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], // On a Pro chip the dense 27B is slow (8.1 tokens per second), the 35B-A3B is not. [48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'], [48, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], // The Q8 27B fits 47 GiB but computes to 7 tokens per second on this chip. [64, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], [96, 'Apple M3 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], // The 122B fits 95 GiB only as Tight (0.861) and has no speed figure. [128, 'Apple M5 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], ]; it.each(EXPECTED)('%i GB, %s: %s', (soldGb, chip, answer) => { const result = recommend(mac(soldGb, chip), STOCK, ALL_INSTALLED); const actual = result.pick ? `${result.verdict} ${result.pick.modelId}@${result.pick.context} ${result.pick.fit} ${result.pick.speed}` : `${result.verdict} ${result.reason}`; expect(actual).toBe(answer); }); it('always pairs the pick with the top-ranked harness, OpenCode', () => { for (const hw of SWEEP) { const result = recommend(hw, STOCK, ALL_INSTALLED); if (result.pick) expect(result.pick.cli).toBe('opencode'); } }); }); // ---------- 3. Verdict none ---------- describe('verdict none', () => { const bigMac = mac(48, 'Apple M4 Max'); /** A `none` answer has no pick, no alternatives, no notes, and the reason's sentence as headline. */ function expectNone(result: Recommendation, reason: Recommendation['reason'], headline: string): void { expect(result.verdict).toBe('none'); expect(result.reason).toBe(reason); expect(result.headline.text).toBe(headline); expect(result.pick).toBeUndefined(); expect(result.alternatives).toEqual([]); expect(result.notes).toEqual([]); } describe('not-macos', () => { it('answers so on Linux, with no budget at all', () => { const linux: HardwareFacts = { platform: 'linux', arch: 'x64', memBytes: 64 * GIB, memGb: formatGb(64 * GIB), memClassGb: 64, source: 'not-probed', }; const result = recommend(linux, STOCK, ALL_INSTALLED); expectNone(result, 'not-macos', LOCAL_MODEL_LABELS.noneNotMacos); expect(result.headline.vars).toBeUndefined(); expect(result.budgetBytes).toBe(0); expect(result.budgetGb).toBe('0.0'); expect(result.budgetSource).toBe('estimated'); }); it('comes before every other reason', () => { for (const platform of ['linux', 'win32', 'freebsd']) { const hw = mac(8, 'Intel(R) Core(TM) i7', { platform, arch: 'x64', arm64: false, macosMajor: 12 }); expect(recommend(hw, STOCK, []).reason, platform).toBe('not-macos'); } }); }); describe('intel', () => { const intelChip = 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'; it('answers so for an Intel Mac, whose sysctl has no arm64 flag', () => { const hw = without(mac(32, intelChip, { arch: 'x64' }), 'arm64', 'translated', 'wiredLimitMb'); const result = recommend(hw, STOCK, ALL_INSTALLED); expectNone(result, 'intel', 'This Mac has an Intel processor. Local coding models need Apple silicon.'); expect(result.budgetBytes).toBe(0); expect(result.budgetGb).toBe('0.0'); }); it('answers so when the arm64 flag reads 0', () => { const hw = without(mac(32, intelChip, { arch: 'x64', arm64: false }), 'translated'); expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('intel'); }); it('comes before the macOS version and the memory', () => { const hw = without(mac(8, intelChip, { arch: 'x64', macosMajor: 12, macosVersion: '12.7' }), 'arm64'); expect(recommend(hw, STOCK, []).reason).toBe('intel'); }); it('does not mistake Node under Rosetta for an Intel Mac', () => { // An x64 process on Apple silicon: the machine flag says arm64, the process is translated. const rosetta = mac(48, 'Apple M4 Max', { arch: 'x64', translated: true }); expect(recommend(rosetta, STOCK, ALL_INSTALLED).verdict).toBe('ok'); // The same when only the translation flag was read. expect(recommend(without(rosetta, 'arm64'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); }); it('decides from node:os alone when the probe failed', () => { const fallback = (arch: string, chip: string): HardwareFacts => ({ platform: 'darwin', arch, memBytes: 48 * GIB, memGb: formatGb(48 * GIB), memClassGb: 48, chip, source: 'os-fallback', }); // An arm64 process can only run on Apple silicon. expect(recommend(fallback('arm64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); // An x64 process with an Apple chip name is Rosetta. expect(recommend(fallback('x64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); // An x64 process with any other chip name is taken for an Intel Mac. expect(recommend(fallback('x64', intelChip), STOCK, ALL_INSTALLED).reason).toBe('intel'); expect(recommend(without(fallback('x64', intelChip), 'chip'), STOCK, ALL_INSTALLED).reason).toBe('intel'); }); }); describe('macos', () => { it('answers so below the catalogue minimum, naming both versions', () => { const result = recommend( mac(48, 'Apple M1 Max', { macosVersion: '13.6.7', macosMajor: 13 }), STOCK, ALL_INSTALLED ); expectNone(result, 'macos', 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.'); expect(result.headline.vars).toEqual({ version: '13.6.7', minimum: '14' }); // The budget is still reported: the Mac itself is able. expect(result.budgetBytes).toBe(35 * GIB); }); it('accepts the minimum itself and anything newer', () => { for (const macosMajor of [14, 15, 26, 27]) { const hw = mac(48, 'Apple M4 Max', { macosMajor, macosVersion: `${macosMajor}.0` }); expect(recommend(hw, STOCK, ALL_INSTALLED).verdict, String(macosMajor)).toBe('ok'); } }); it('does not block on a version that could not be read', () => { const hw = without(mac(48, 'Apple M4 Max'), 'macosMajor', 'macosVersion'); expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); }); it('comes before the memory', () => { const hw = mac(8, 'Apple M1', { macosVersion: '12.7', macosMajor: 12 }); expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('macos'); }); }); describe('memory', () => { it('answers so at 8 GB, where nothing fits at any context', () => { const result = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemory); // The size is the decimal figure of the hardware line, without the unit. expect(result.headline.vars).toEqual({ memory: '8.6' }); expect(result.budgetBytes).toBe(2 * GIB); expect(result.budgetGb).toBe('2.1'); }); it('answers so when models fit, but only below the context a harness needs', () => { // The 16 GB case of the Mac mini, with harnesses that are all present. const result = recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED); expect(result.reason).toBe('memory'); expect(result.headline.vars).toEqual({ memory: '17.2' }); }); it('comes before no-harness when nothing fits at all', () => { expect(recommend(mac(8, 'Apple M2'), STOCK, []).reason).toBe('memory'); }); it('answers so for an empty catalogue', () => { expect(recommend(bigMac, catalogOf([]), [fact('alpha')]).reason).toBe('memory'); }); it('answers so when the memory size is unknown', () => { const hw = { ...mac(48, 'Apple M4 Max'), memBytes: 0, memGb: '0.0', memClassGb: 0 }; const result = recommend(hw, STOCK, ALL_INSTALLED); expect(result.reason).toBe('memory'); expect(result.budgetBytes).toBe(0); }); }); describe('no-harness', () => { const headline = 'No coding agent that Codeman can connect to a local model is available on this Mac.'; it('answers so when the caller knows no catalogue harness', () => { const result = recommend(bigMac, STOCK, []); expectNone(result, 'no-harness', headline); expect(result.headline.vars).toBeUndefined(); expect(result.budgetBytes).toBe(35 * GIB); }); it('answers so when no harness is installed or installable', () => { const facts = STOCK.harnesses.map((row) => fact(row.cli, { installed: false, installable: false })); expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); }); it('answers so when only the gated harness is on the machine', () => { const gatedRows = STOCK.harnesses.filter((row) => row.gated); expect(gatedRows).toHaveLength(1); expect( recommend( bigMac, STOCK, gatedRows.map((row) => fact(row.cli)) ).reason ).toBe('no-harness'); }); it('answers so when a harness is there but accepts no model id', () => { const facts = ALL_INSTALLED.map((entry) => ({ ...entry, acceptsModelId: () => false })); expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); }); it('ignores a fact for a CLI that is not a catalogue harness', () => { expect(recommend(bigMac, STOCK, [fact('codex'), fact('some-other-cli')]).reason).toBe('no-harness'); }); }); describe('disk', () => { it('answers so when every fitting model would leave under 10 GiB free', () => { // 12 GiB free: even the smallest download (6.55 GB) leaves 5.9 GiB. const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); const result = recommend(hw, STOCK, ALL_INSTALLED); expectNone( result, 'disk', 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.' ); // download + 10 GiB - free = 6,550,825,373 + 10,737,418,240 - 12,884,901,888 bytes. expect(result.headline.vars).toEqual({ download: '6.6', free: '12.9', missing: '4.4' }); expect(result.budgetBytes).toBe(35 * GIB); }); it('names the smallest model that fits the MEMORY, not the smallest of the catalogue', () => { // On a 48 GB Mac whose harness refuses both 9B ids, the smallest candidate is gpt-oss:20b. const facts = ALL_INSTALLED.map((entry) => ({ ...entry, acceptsModelId: (id: string) => !id.startsWith('qwen3.5:9b'), })); const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); const result = recommend(hw, STOCK, facts); expect(result.reason).toBe('disk'); expect(result.headline.vars?.download).toBe('13.8'); }); it('drops only the models that do not leave 10 GiB, and picks among the rest', () => { // 20 GiB free: the 9B downloads (6.1 and 9.7 GiB) leave 10 GiB or more, the others do not. const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 20 * GIB, freeDiskGb: formatGb(20 * GIB) }); const result = recommend(hw, STOCK, ALL_INSTALLED); expect(result.verdict).toBe('limited'); expect(order(result)).toEqual(['qwen3.5:9b-q8_0@65536', 'qwen3.5:9b@65536']); }); it('allows a download that leaves exactly 10 GiB', () => { const exact = STOCK.models[0].bytes + DISK_HEADROOM_BYTES; expect(DISK_HEADROOM_BYTES).toBe(10 * GIB); const fits = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact }), STOCK, ALL_INSTALLED); expect(fits.pick?.modelId).toBe('qwen3.5:9b'); const short = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact - 1 }), STOCK, ALL_INSTALLED); expect(short.reason).toBe('disk'); }); it('does not block when the free disk is unknown', () => { const hw = without(mac(48, 'Apple M4 Max'), 'freeDiskBytes', 'freeDiskGb'); expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); }); it('formats the free size itself when the hardware carries only the bytes', () => { const hw = without(mac(48, 'Apple M4 Max', { freeDiskBytes: 3 * GB }), 'freeDiskGb'); expect(recommend(hw, STOCK, ALL_INSTALLED).headline.vars?.free).toBe('3.0'); }); it('comes after the memory and the harness', () => { const fullDisk = { freeDiskBytes: 0, freeDiskGb: '0.0' }; expect(recommend(mac(16, 'Apple M4', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('memory'); expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, []).reason).toBe('no-harness'); expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('disk'); }); }); }); // ---------- 4. Verdicts, ranking, harnesses, tested ---------- describe('verdict limited', () => { it('is the answer when the best model that fits has fewer than 12 billion parameters', () => { expect(LIMITED_BELOW_PARAMS_B).toBe(12); const result = recommend(mac(24, 'Apple M4'), STOCK, ALL_INSTALLED); expect(result.verdict).toBe('limited'); expect(result.reason).toBeUndefined(); expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', paramsB: 9, context: 65536, fit: 'good' }); expect(result.headline).toEqual({ text: 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', }); expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); }); it('holds for a tight fit too (18 GB)', () => { const result = recommend(mac(18, 'Apple M3 Pro'), STOCK, ALL_INSTALLED); expect(result.verdict).toBe('limited'); expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', fit: 'tight', fitLabel: 'Tight', budgetShare: 0.873 }); expect(result.alternatives).toEqual([]); }); it('draws the line at 12 billion: 11.9 is limited, 12 is ok', () => { const verdictFor = (paramsB: number): string => recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { paramsB })]), [fact('alpha')]).verdict; expect(verdictFor(11.9)).toBe('limited'); expect(verdictFor(12)).toBe('ok'); expect(verdictFor(9)).toBe('limited'); expect(verdictFor(27.8)).toBe('ok'); }); it('is about the PICK: a larger model further down the list does not lift it', () => { // 32 GB: the 20B fits only as Tight, so a 9B is the pick and the answer is limited. const result = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED); expect(result.verdict).toBe('limited'); expect(result.alternatives.map((c) => [c.modelId, c.fit])).toContainEqual(['gpt-oss:20b', 'tight']); }); it('never says "Recommended", for any Mac', () => { for (const hw of SWEEP) { const result = recommend(hw, STOCK, ALL_INSTALLED); if (result.verdict === 'limited') { expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); } } }); }); describe('speed', () => { it('is the model figure times the chip bandwidth, rounded to one decimal', () => { const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); // 2.95 x 410 / 100. expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', speed: 'ok', tokensPerSecond: 12.1 }); expect(result.pick).not.toHaveProperty('speedLabel'); }); it('is slow below 10 tokens per second, and says so', () => { expect(SLOW_BELOW_TOKENS_PER_SECOND).toBe(10); const speedOf = (speedPer100Gbps: number): Candidate | undefined => recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { speedPer100Gbps })]), [fact('alpha')]).pick; // The synthetic chip has 400 GB/s: 2.5 is 10.0 tokens per second, 2.475 is 9.9. expect(speedOf(2.5)).toMatchObject({ speed: 'ok', tokensPerSecond: 10 }); expect(speedOf(2.475)).toMatchObject({ speed: 'slow', tokensPerSecond: 9.9, speedLabel: 'Slow' }); expect(speedOf(0.5)).toMatchObject({ speed: 'slow', tokensPerSecond: 2, speedLabel: 'Slow' }); }); it('is unknown for a model without a figure, on any chip', () => { // 48 GB: five models fit, so all of them are listed and gpt-oss:20b is among them. for (const chip of ['Apple M1', 'Apple M4 Max', 'Apple M5 Ultra']) { const result = recommend(mac(48, chip), STOCK, ALL_INSTALLED); const gptOss = [result.pick, ...result.alternatives].find((c) => c?.modelId === 'gpt-oss:20b'); expect(gptOss, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); expect(gptOss).not.toHaveProperty('tokensPerSecond'); } }); it('is unknown for every model on a chip that is in no chips row: no bandwidth is guessed', () => { for (const chip of ['Apple M6', 'Apple M6 Pro', 'Apple M10', 'Some Future Chip']) { const result = recommend(mac(64, chip), STOCK, ALL_INSTALLED); for (const candidate of [result.pick, ...result.alternatives]) { expect(candidate, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); expect(candidate).not.toHaveProperty('tokensPerSecond'); } } // And when the chip name itself is missing. const result = recommend(without(mac(64, 'x'), 'chip'), STOCK, ALL_INSTALLED); expect(result.pick?.speed).toBe('unknown'); }); }); describe('the ranking', () => { it('puts ok before unknown before slow, ahead of the model rank', () => { // Three models that fit equally well. The ranks run AGAINST the speed. const catalog = catalogOf([ model('slow-top:1', { rank: 3, speedPer100Gbps: 1 }), model('unknown-mid:1', { rank: 2 }), model('ok-low:1', { rank: 1, speedPer100Gbps: 5 }), ]); const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); expect(order(result)).toEqual(['ok-low:1@1024', 'unknown-mid:1@1024', 'slow-top:1@1024']); expect([result.pick, ...result.alternatives].map((c) => c?.speed)).toEqual(['ok', 'unknown', 'slow']); }); it('ranks by model rank alone when every speed is unknown', () => { // 'Apple M6' is in no chips row. const result = recommend(mac(48, 'Apple M6'), STOCK, ALL_INSTALLED); expect(order(result)).toEqual([ 'qwen3.8:27b@131072', 'qwen3.6:35b-a3b@131072', 'gpt-oss:20b@131072', 'qwen3.5:9b-q8_0@65536', 'qwen3.5:9b@65536', ]); }); it('ranks an unknown speed below a known ok one and above a slow one, on a real tier', () => { // 48 GB, M4 Pro: three ok models, then gpt-oss:20b (unknown), then the 27B (slow). const result = recommend(mac(48, 'Apple M4 Pro'), STOCK, ALL_INSTALLED); expect([result.pick, ...result.alternatives].map((c) => [c?.modelId, c?.speed])).toEqual([ ['qwen3.6:35b-a3b', 'ok'], ['qwen3.5:9b-q8_0', 'ok'], ['qwen3.5:9b', 'ok'], ['gpt-oss:20b', 'unknown'], ['qwen3.8:27b', 'slow'], ]); }); it('puts a good fit before a tight one, ahead of speed and rank', () => { const catalog = catalogOf([ modelNeeding('tight-fast-top:1', 0.9 * HUNDRED_GIB, { rank: 9, speedPer100Gbps: 20 }), modelNeeding('good-slow-low:1', 0.5 * HUNDRED_GIB, { rank: 1, speedPer100Gbps: 1 }), ]); const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); expect(order(result)).toEqual(['good-slow-low:1@1024', 'tight-fast-top:1@1024']); }); it('takes the higher model rank among equals', () => { const catalog = catalogOf([model('low:1', { rank: 1 }), model('high:1', { rank: 5 }), model('mid:1', { rank: 3 })]); expect(order(recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]))).toEqual([ 'high:1@1024', 'mid:1@1024', 'low:1@1024', ]); }); it('takes the larger context of one model when both fit equally well', () => { // 1 KiB per token: 4 GiB of KV cache at 4,194,304 tokens, still a good fit. const catalog = catalogOf([model('probe:1', { kvBytesPerToken: 1024, contexts: [4_194_304, 1024] })]); const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); expect(result.pick?.context).toBe(4_194_304); expect(result.alternatives).toEqual([]); }); it('steps a model down to the context at which it fits well', () => { // Good at the small context, tight at the large one: the good one is that model's entry. const catalog = catalogOf([ model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] }), ]); const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); expect(result.pick).toMatchObject({ context: 1024, fit: 'good' }); }); it('lists each model once, never the pick again, and at most four alternatives', () => { expect(MAX_ALTERNATIVES).toBe(4); // Six models fit a 128 GB Mac; the pick plus four are shown. const result = recommend(mac(128, 'Apple M5 Ultra'), STOCK, ALL_INSTALLED); expect(result.alternatives).toHaveLength(4); const ids = [result.pick, ...result.alternatives].map((c) => c?.modelId); expect(new Set(ids).size).toBe(ids.length); for (const hw of SWEEP) { const swept = recommend(hw, STOCK, ALL_INSTALLED); const sweptIds = [swept.pick, ...swept.alternatives].filter(Boolean).map((c) => c?.modelId); expect(new Set(sweptIds).size).toBe(sweptIds.length); expect(swept.alternatives.length).toBeLessThanOrEqual(MAX_ALTERNATIVES); } }); }); describe('harnesses', () => { const hw = mac(48, 'Apple M4 Max'); it('never chooses a gated harness, even when it is the only one that could run a model', () => { const gated = STOCK.harnesses.filter((row) => row.gated).map((row) => row.cli); expect(gated).toEqual(['pi']); for (const swept of SWEEP) { const result = recommend(swept, STOCK, ALL_INSTALLED); for (const candidate of [result.pick, ...result.alternatives]) { if (candidate) expect(gated).not.toContain(candidate.cli); } } // 16 GB: the gated harness alone works at 32768 tokens, where the 9B would fit. Still none. expect(recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED).verdict).toBe('none'); }); it('never offers a context below the minimum of the harness', () => { for (const swept of SWEEP) { const result = recommend(swept, STOCK, ALL_INSTALLED); for (const candidate of [result.pick, ...result.alternatives]) { if (!candidate) continue; const row = STOCK.harnesses.find((harness) => harness.cli === candidate.cli); expect(candidate.context).toBeGreaterThanOrEqual(row?.minContext ?? Number.POSITIVE_INFINITY); } } }); it('takes the higher-ranked harness and reports its label and that it is installed', () => { const result = recommend(hw, STOCK, ALL_INSTALLED); expect(result.pick).toMatchObject({ cli: 'opencode', cliLabel: 'OpenCode', cliInstalled: true }); expect(result.pick).not.toHaveProperty('slowFirstTurn'); }); it('falls back to the next harness when the first is not on the machine, and flags its slow first turn', () => { const facts = [fact('opencode', { installed: false, installable: false }), fact('claude'), fact('pi')]; const result = recommend(hw, STOCK, facts); expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'claude', cliLabel: 'Claude Code', cliInstalled: true, slowFirstTurn: true, }); // The same when the caller has no fact for it at all. expect(recommend(hw, STOCK, [fact('claude')]).pick?.cli).toBe('claude'); }); it('offers a harness that is not installed but can be, and says it is not installed', () => { const facts = [fact('opencode', { installed: false, installable: true }), fact('claude')]; const result = recommend(hw, STOCK, facts); expect(result.pick).toMatchObject({ cli: 'opencode', cliInstalled: false }); }); it('pairs a model only with a harness that accepts its id', () => { const refusing27b = (id: string): boolean => id !== 'qwen3.8:27b'; // OpenCode refuses the 27B: it goes with the next harness. const one = recommend(hw, STOCK, [fact('opencode', { acceptsModelId: refusing27b }), fact('claude')]); expect(one.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'claude' }); expect(one.alternatives[0]).toMatchObject({ modelId: 'qwen3.6:35b-a3b', cli: 'opencode' }); // Both refuse it: the model is not offered. const both = recommend(hw, STOCK, [ fact('opencode', { acceptsModelId: refusing27b }), fact('claude', { acceptsModelId: refusing27b }), ]); expect(both.pick?.modelId).toBe('qwen3.6:35b-a3b'); expect(order(both)).not.toContain('qwen3.8:27b@131072'); }); it('treats a harness check that throws as a refusal, and still answers', () => { const throwing = fact('opencode', { acceptsModelId: () => { throw new Error('registry not loaded'); }, }); const result = recommend(hw, STOCK, [throwing, fact('claude')]); expect(result.verdict).toBe('ok'); expect(result.pick?.cli).toBe('claude'); expect(recommend(hw, STOCK, [throwing]).reason).toBe('no-harness'); }); it('puts the fit and the model before the harness rank', () => { // `big` needs a large context and is the higher-ranked harness; `small` works from 1024. const harnesses: LocalHarnessEntry[] = [ { cli: 'big', minContext: 8_388_608, rank: 2 }, { cli: 'small', minContext: 1024, rank: 1 }, ]; const tightAtLarge = catalogOf( [model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })], harnesses ); // Tight at the context `big` needs, good at the small one: the good fit with `small` wins. expect(recommend(HUNDRED_GIB_MAC, tightAtLarge, [fact('big'), fact('small')]).pick).toMatchObject({ cli: 'small', context: 1024, fit: 'good', }); // Good at both: the higher-ranked harness wins, at the context it needs. const goodAtBoth = catalogOf( [model('probe:1', { bytes: 10 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })], harnesses ); expect(recommend(HUNDRED_GIB_MAC, goodAtBoth, [fact('big'), fact('small')]).pick).toMatchObject({ cli: 'big', context: 8_388_608, }); }); }); describe('the tested flag and the word "Recommended"', () => { const testedOn = (chip: string, memGb: number, cli: string) => ({ chip, memGb, cli, date: '2026-11-06' }); const with27bTested = (chip: string, memGb: number, cli: string): LocalModelCatalog => stockWith((catalog) => { const row = catalog.models.find((m) => m.id === 'qwen3.8:27b') as LocalModelEntry; row.tested = [testedOn(chip, memGb, cli)]; }); it('is off for every model of the stock catalogue, so "Recommended" is never said today', () => { for (const hw of SWEEP) { const result = recommend(hw, STOCK, ALL_INSTALLED); for (const candidate of [result.pick, ...result.alternatives]) { if (candidate) expect(candidate.tested).toBe(false); } expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); if (result.verdict !== 'none') { expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); } } }); it('turns on "Recommended for this Mac" when the pick was tested on exactly such a Mac', () => { const result = recommend(mac(48, 'Apple M4 Max'), with27bTested('Apple M4 Max', 48, 'opencode'), ALL_INSTALLED); expect(result.verdict).toBe('ok'); expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: true }); expect(result.headline).toEqual({ text: 'Recommended for this Mac' }); // The "not yet tested" note goes away with it. expect(result.notes).toEqual([]); }); it('stays off when the chip, the memory size or the harness differs', () => { const hw = mac(48, 'Apple M4 Max'); const cases: LocalModelCatalog[] = [ with27bTested('Apple M4 Pro', 48, 'opencode'), with27bTested('Apple M4', 48, 'opencode'), with27bTested('Apple M4 Max', 64, 'opencode'), with27bTested('Apple M4 Max', 48, 'claude'), ]; for (const catalog of cases) { const result = recommend(hw, catalog, ALL_INSTALLED); expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: false }); expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); } }); it('follows the harness: tested with the second harness counts when that one is the pick', () => { const catalog = with27bTested('Apple M4 Max', 48, 'claude'); const result = recommend(mac(48, 'Apple M4 Max'), catalog, [fact('claude')]); expect(result.pick).toMatchObject({ cli: 'claude', tested: true }); expect(result.headline.text).toBe('Recommended for this Mac'); }); it('does not change the ranking: a tested model further down stays an alternative', () => { // On the M4 Pro the pick is the 35B-A3B. A tested 27B is flagged in the list, no more. const result = recommend(mac(48, 'Apple M4 Pro'), with27bTested('Apple M4 Pro', 48, 'opencode'), ALL_INSTALLED); expect(result.pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', tested: false }); expect(result.headline.text).toBe('Best fit for this Mac'); expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); expect(result.alternatives.find((c) => c.modelId === 'qwen3.8:27b')?.tested).toBe(true); }); it('never says "Recommended" for a limited answer, tested or not', () => { const catalog = stockWith((stock) => { const row = stock.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; row.tested = [testedOn('Apple M4', 24, 'opencode')]; }); const result = recommend(mac(24, 'Apple M4'), catalog, ALL_INSTALLED); expect(result.verdict).toBe('limited'); expect(result.pick?.tested).toBe(true); expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); // It WAS tested, so the "not yet tested" note would be false. expect(result.notes).toEqual([]); }); }); describe('the shape of the answer', () => { const labels = new Set(Object.values(LOCAL_MODEL_LABELS)); it('takes every sentence and label from the fixed list', () => { for (const hw of SWEEP) { const result = recommend(hw, STOCK, ALL_INSTALLED); expect(labels.has(result.headline.text)).toBe(true); for (const note of result.notes) expect(labels.has(note.text)).toBe(true); for (const candidate of [result.pick, ...result.alternatives]) { if (!candidate) continue; expect(labels.has(candidate.fitLabel)).toBe(true); if (candidate.speedLabel !== undefined) expect(labels.has(candidate.speedLabel)).toBe(true); expect(candidate.speedLabel === undefined).toBe(candidate.speed === 'ok'); expect(candidate.tokensPerSecond === undefined).toBe(candidate.speed === 'unknown'); } } }); it('fills every placeholder of a headline and nothing else', () => { const answers = [ recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED), recommend(mac(48, 'Apple M4', { macosMajor: 13, macosVersion: '13.0' }), STOCK, ALL_INSTALLED), recommend(mac(48, 'Apple M4', { freeDiskBytes: GB, freeDiskGb: '1.0' }), STOCK, ALL_INSTALLED), recommend(mac(48, 'Apple M4'), STOCK, []), recommend(mac(48, 'Apple M4'), STOCK, ALL_INSTALLED), ]; expect(answers.map((answer) => answer.reason ?? answer.verdict)).toEqual([ 'memory', 'macos', 'disk', 'no-harness', 'ok', ]); for (const answer of answers) { const slots = [...answer.headline.text.matchAll(/\{(\w+)\}/g)].map((match) => match[1]).sort(); expect(Object.keys(answer.headline.vars ?? {}).sort()).toEqual(slots); // `{name}` belongs to the browser's translator. expect(slots).not.toContain('name'); for (const value of Object.values(answer.headline.vars ?? {})) expect(typeof value).toBe('string'); } }); it('prints every size in one unit: decimal GB from the one formatter, never GiB', () => { for (const hw of SWEEP) { const result = recommend(hw, STOCK, ALL_INSTALLED); expect(result.budgetGb).toBe(formatGb(result.budgetBytes)); for (const candidate of [result.pick, ...result.alternatives]) { if (!candidate) continue; expect(candidate.needGb).toBe(formatGb(candidate.needBytes)); expect(candidate.downloadGb).toBe(formatGb(candidate.downloadBytes)); expect(candidate.budgetShare).toBeCloseTo(candidate.needBytes / result.budgetBytes, 3); } expect(JSON.stringify(result)).not.toMatch(/GiB|MiB/); } }); it('adds up for a person: download, plus what the context and the runtime take, is the need', () => { const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); const pick = result.pick as Candidate; // 17.7 GB of weights + 8 GiB of KV cache + 1 GiB + 2 GiB of prompt cache, all in decimal GB. expect(pick.downloadGb).toBe('17.7'); expect(pick.needGb).toBe('29.6'); expect(formatGb(pick.needBytes - pick.downloadBytes)).toBe('11.8'); expect(result.budgetGb).toBe('37.6'); expect(pick.needBytes).toBeLessThan(result.budgetBytes); }); it('is plain JSON and has exactly the fields of its verdict', () => { const ok = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); expect(JSON.parse(JSON.stringify(ok))).toEqual(ok); expect(Object.keys(ok).sort()).toEqual( ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'pick', 'verdict'].sort() ); const none = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); expect(JSON.parse(JSON.stringify(none))).toEqual(none); expect(Object.keys(none).sort()).toEqual( ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'reason', 'verdict'].sort() ); }); it('is pure: frozen inputs are enough, and the same inputs give the same answer', () => { const hw = deepFreeze(mac(48, 'Apple M4 Pro')); const catalog = deepFreeze(structuredClone(STOCK_LOCAL_MODEL_CATALOG)); const facts = deepFreeze(ALL_INSTALLED.map((entry) => ({ ...entry }))); const first = recommend(hw, catalog, facts); const second = recommend(hw, catalog, facts); expect(second).toEqual(first); expect(first.verdict).toBe('ok'); }); });