mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-17 19:59:33 +01:00
1298 lines
56 KiB
TypeScript
1298 lines
56 KiB
TypeScript
/**
|
|||
|
|
* @fileoverview Tests for src/local-model/recommend.ts: the pure function that turns "what this
|
||
|
|
* Mac is" into "which local model fits it".
|
||
|
|
*
|
||
|
|
* Laid out like the module:
|
||
|
|
*
|
||
|
|
* 1. The budget (`deviceBudget`) for every memory size Apple sells from 8 to 128 GB, and the
|
||
|
|
* need of a model at a context (`needBytes`).
|
||
|
|
* 2. The two REAL Macs of the project, from their captured `sysctl` output through the real
|
||
|
|
* parser: the MacBook Pro (M5 Pro, 48 GB) and the Mac mini (M4, 16 GB). Each test states the
|
||
|
|
* expected answer and why.
|
||
|
|
* 3. Every reason for verdict `none`, one fixture per reason, and their order.
|
||
|
|
* 4. The `limited` verdict, the three-state speed in the ranking, harness rows (a gated one is
|
||
|
|
* never chosen), the `tested` flag that alone may turn on the word "Recommended", and the
|
||
|
|
* shape of the answer (labels from the fixed list, sizes in one unit).
|
||
|
|
*
|
||
|
|
* The stock catalogue is used wherever the answer for a real Mac is at stake. Small synthetic
|
||
|
|
* catalogues (validated by the real schema) isolate one rule of the ranking at a time.
|
||
|
|
*
|
||
|
|
* Port: N/A (pure functions; the two hardware fixtures go through `readHardware` with every
|
||
|
|
* piece of IO injected).
|
||
|
|
*/
|
||
|
|
|
||
|
|
import { beforeEach, describe, expect, it } from 'vitest';
|
||
|
|
import {
|
||
|
|
getLocalModelCatalog,
|
||
|
|
LocalModelCatalogSchema,
|
||
|
|
STOCK_LOCAL_MODEL_CATALOG,
|
||
|
|
} from '../src/config/local-model-catalog.js';
|
||
|
|
import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js';
|
||
|
|
import { readHardware, resetHardwareCache } from '../src/local-model/hardware.js';
|
||
|
|
import {
|
||
|
|
deviceBudget,
|
||
|
|
DISK_HEADROOM_BYTES,
|
||
|
|
FIT_GOOD_MAX_SHARE,
|
||
|
|
FIT_TIGHT_MAX_SHARE,
|
||
|
|
LIMITED_BELOW_PARAMS_B,
|
||
|
|
MAX_ALTERNATIVES,
|
||
|
|
needBytes,
|
||
|
|
recommend,
|
||
|
|
SLOW_BELOW_TOKENS_PER_SECOND,
|
||
|
|
} from '../src/local-model/recommend.js';
|
||
|
|
import {
|
||
|
|
LOCAL_MODEL_LABELS,
|
||
|
|
type Candidate,
|
||
|
|
type HardwareFacts,
|
||
|
|
type HarnessFact,
|
||
|
|
type LocalHarnessEntry,
|
||
|
|
type LocalModelCatalog,
|
||
|
|
type LocalModelEntry,
|
||
|
|
type Recommendation,
|
||
|
|
} from '../src/types.js';
|
||
|
|
|
||
|
|
// ---------- Fixtures ----------
|
||
|
|
|
||
|
|
/** The real probe output of the MacBook Pro (Mac17,13), captured 2026-11-06, extra lines included. */
|
||
|
|
const MACBOOK_SYSCTL = `hw.memsize: 51539607552
|
||
|
|
machdep.cpu.brand_string: Apple M5 Pro
|
||
|
|
hw.optional.arm64: 1
|
||
|
|
sysctl.proc_translated: 0
|
||
|
|
iogpu.wired_limit_mb: 0
|
||
|
|
kern.osproductversion: 27.2
|
||
|
|
hw.model: Mac17,13
|
||
|
|
hw.perflevel0.physicalcpu: 6
|
||
|
|
hw.perflevel1.physicalcpu: 12
|
||
|
|
exit=0
|
||
|
|
/dev/disk3s5 971094180 54383564 890926084 6% 542146 8909260840 0% /System/Volumes/Data
|
||
|
|
`;
|
||
|
|
|
||
|
|
/** The real probe output of the Mac mini (Mac16,10), captured 2026-11-06, extra lines included. */
|
||
|
|
const MACMINI_SYSCTL = `hw.memsize: 17179869184
|
||
|
|
machdep.cpu.brand_string: Apple M4
|
||
|
|
hw.optional.arm64: 1
|
||
|
|
sysctl.proc_translated: 0
|
||
|
|
iogpu.wired_limit_mb: 0
|
||
|
|
kern.osproductversion: 27.2
|
||
|
|
hw.model: Mac16,10
|
||
|
|
hw.perflevel0.physicalcpu: 4
|
||
|
|
hw.perflevel1.physicalcpu: 6
|
||
|
|
exit=0
|
||
|
|
/dev/disk3s5 239362496 172954124 41752848 81% 2353838 417528480 1% /System/Volumes/Data
|
||
|
|
`;
|
||
|
|
|
||
|
|
/** The hardware facts of a real Mac: its captured output through the real parser, no IO. */
|
||
|
|
async function realMac(sysctlOutput: string, freeDiskKb: number): Promise<HardwareFacts> {
|
||
|
|
resetHardwareCache();
|
||
|
|
return readHardware({
|
||
|
|
platform: 'darwin',
|
||
|
|
arch: 'arm64',
|
||
|
|
runSysctl: async () => sysctlOutput,
|
||
|
|
freeDiskBytes: async () => freeDiskKb * 1024,
|
||
|
|
totalMemBytes: () => 0,
|
||
|
|
cpuModel: () => undefined,
|
||
|
|
now: () => 0,
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
/** An Apple silicon Mac with `soldGb` of memory, healthy in every other respect. */
|
||
|
|
function mac(soldGb: number, chip: string, overrides: Partial<HardwareFacts> = {}): HardwareFacts {
|
||
|
|
const memBytes = soldGb * GIB;
|
||
|
|
return {
|
||
|
|
platform: 'darwin',
|
||
|
|
arch: 'arm64',
|
||
|
|
arm64: true,
|
||
|
|
translated: false,
|
||
|
|
memBytes,
|
||
|
|
memGb: formatGb(memBytes),
|
||
|
|
memClassGb: memClassGb(memBytes),
|
||
|
|
chip,
|
||
|
|
model: 'Mac0,0',
|
||
|
|
macosVersion: '27.2',
|
||
|
|
macosMajor: 27,
|
||
|
|
wiredLimitMb: 0,
|
||
|
|
freeDiskBytes: 500 * GB,
|
||
|
|
freeDiskGb: '500',
|
||
|
|
source: 'sysctl',
|
||
|
|
...overrides,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/** A copy of `facts` without the given fields (an absent field, not an `undefined` one). */
|
||
|
|
function without(facts: HardwareFacts, ...fields: (keyof HardwareFacts)[]): HardwareFacts {
|
||
|
|
const copy: Record<string, unknown> = { ...facts };
|
||
|
|
for (const field of fields) delete copy[field];
|
||
|
|
return copy as unknown as HardwareFacts;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Display labels of the stock harnesses, as the CLI registry would supply them. Data, not a branch. */
|
||
|
|
const HARNESS_LABELS: Record<string, string> = { opencode: 'OpenCode', claude: 'Claude Code', pi: 'Pi' };
|
||
|
|
|
||
|
|
/** What the server knows about one harness: installed and accepting every model id, unless overridden. */
|
||
|
|
function fact(cli: string, overrides: Partial<HarnessFact> = {}): HarnessFact {
|
||
|
|
return {
|
||
|
|
cli,
|
||
|
|
label: HARNESS_LABELS[cli] ?? cli,
|
||
|
|
installed: true,
|
||
|
|
installable: false,
|
||
|
|
acceptsModelId: () => true,
|
||
|
|
...overrides,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
const STOCK = getLocalModelCatalog();
|
||
|
|
|
||
|
|
/** Every stock harness installed, the gated one included. */
|
||
|
|
const ALL_INSTALLED: HarnessFact[] = STOCK.harnesses.map((row) => fact(row.cli));
|
||
|
|
|
||
|
|
/** The stock catalogue with a change applied, validated by the real schema. */
|
||
|
|
function stockWith(change: (catalog: LocalModelCatalog) => void): LocalModelCatalog {
|
||
|
|
const catalog = structuredClone(STOCK_LOCAL_MODEL_CATALOG);
|
||
|
|
change(catalog);
|
||
|
|
return LocalModelCatalogSchema.parse(catalog);
|
||
|
|
}
|
||
|
|
|
||
|
|
/** A model row for a synthetic catalogue: 20 billion parameters, 10 GiB, one small context. */
|
||
|
|
function model(id: string, overrides: Partial<LocalModelEntry> = {}): LocalModelEntry {
|
||
|
|
return {
|
||
|
|
id,
|
||
|
|
label: id,
|
||
|
|
license: 'Apache-2.0',
|
||
|
|
paramsB: 20,
|
||
|
|
bytes: 10 * GIB,
|
||
|
|
kvBytesPerToken: 1,
|
||
|
|
contexts: [1024],
|
||
|
|
rank: 1,
|
||
|
|
tested: [],
|
||
|
|
...overrides,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* A synthetic catalogue: no prompt cache, one harness `alpha` that works from 1024 tokens, and
|
||
|
|
* one chip row ('M4 Max' at 400 GB/s, so tokens per second = `speedPer100Gbps` x 4).
|
||
|
|
*/
|
||
|
|
function catalogOf(models: LocalModelEntry[], harnesses?: LocalHarnessEntry[]): LocalModelCatalog {
|
||
|
|
return LocalModelCatalogSchema.parse({
|
||
|
|
schema: 1,
|
||
|
|
updated: '2026-11-06',
|
||
|
|
minMacosMajor: 14,
|
||
|
|
promptCacheTiers: [{ minMemGb: 0, bytes: 0 }],
|
||
|
|
models,
|
||
|
|
harnesses: harnesses ?? [{ cli: 'alpha', minContext: 1024, rank: 1 }],
|
||
|
|
chips: [{ match: 'M4 Max', gbps: 400 }],
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
/** A Mac whose budget is exactly 100 GiB: a wired limit of 101 GiB on a 128 GB machine. */
|
||
|
|
const HUNDRED_GIB_MAC = mac(128, 'Apple M4 Max', { wiredLimitMb: 101 * 1024 });
|
||
|
|
const HUNDRED_GIB = 100 * GIB;
|
||
|
|
|
||
|
|
/** What a synthetic model needs beside its weights at its 1024-token context: 1 GiB plus the KV cache. */
|
||
|
|
const SYNTHETIC_OVERHEAD = GIB + 1024;
|
||
|
|
|
||
|
|
/** A model whose need is exactly `need` bytes in a synthetic catalogue. */
|
||
|
|
function modelNeeding(id: string, need: number, overrides: Partial<LocalModelEntry> = {}): LocalModelEntry {
|
||
|
|
return model(id, { bytes: need - SYNTHETIC_OVERHEAD, ...overrides });
|
||
|
|
}
|
||
|
|
|
||
|
|
/** `id@context` of the pick and of each alternative, in order. */
|
||
|
|
function order(result: Recommendation): string[] {
|
||
|
|
return [result.pick, ...result.alternatives]
|
||
|
|
.filter((candidate): candidate is Candidate => candidate !== undefined)
|
||
|
|
.map((candidate) => `${candidate.modelId}@${candidate.context}`);
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Every string anywhere in a value. */
|
||
|
|
function stringsIn(value: unknown): string[] {
|
||
|
|
if (typeof value === 'string') return [value];
|
||
|
|
if (Array.isArray(value)) return value.flatMap(stringsIn);
|
||
|
|
if (value && typeof value === 'object') return Object.values(value).flatMap(stringsIn);
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
|
||
|
|
function deepFreeze<T>(value: T): T {
|
||
|
|
if (value && typeof value === 'object') {
|
||
|
|
for (const inner of Object.values(value)) deepFreeze(inner);
|
||
|
|
Object.freeze(value);
|
||
|
|
}
|
||
|
|
return value;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** The Macs the sweeps run over: every memory size, with a slow, a fast and an unknown chip. */
|
||
|
|
const SWEEP: HardwareFacts[] = [8, 16, 18, 24, 32, 36, 48, 64, 96, 128].flatMap((soldGb) =>
|
||
|
|
['Apple M4', 'Apple M4 Max', 'Apple M5 Ultra', 'Apple M6'].map((chip) => mac(soldGb, chip))
|
||
|
|
);
|
||
|
|
|
||
|
|
beforeEach(() => {
|
||
|
|
resetHardwareCache();
|
||
|
|
});
|
||
|
|
|
||
|
|
// ---------- 1. Budget and need ----------
|
||
|
|
|
||
|
|
describe('deviceBudget', () => {
|
||
|
|
// Sold size, device limit L and budget B in GiB. L is 2/3 of memory below 36 GB and 3/4 from
|
||
|
|
// 36 GB; B = min(L - 1, memory - reserve). Only at 8 GB is the reserve the smaller bound.
|
||
|
|
const TIERS: [soldGb: number, limitGib: number, budgetGib: number][] = [
|
||
|
|
[8, 16 / 3, 2],
|
||
|
|
[16, 32 / 3, 29 / 3],
|
||
|
|
[18, 12, 11],
|
||
|
|
[24, 16, 15],
|
||
|
|
[32, 64 / 3, 61 / 3],
|
||
|
|
[36, 27, 26],
|
||
|
|
[48, 36, 35],
|
||
|
|
[64, 48, 47],
|
||
|
|
[96, 72, 71],
|
||
|
|
[128, 96, 95],
|
||
|
|
];
|
||
|
|
|
||
|
|
it.each(TIERS)('gives a %i GB Mac a limit of %f GiB and a budget of %f GiB', (soldGb, limitGib, budgetGib) => {
|
||
|
|
const budget = deviceBudget(mac(soldGb, 'Apple M4'));
|
||
|
|
// Whole bytes: a third of a byte is rounded down.
|
||
|
|
expect(Math.abs(budget.limitBytes - limitGib * GIB)).toBeLessThan(1);
|
||
|
|
expect(Math.abs(budget.budgetBytes - budgetGib * GIB)).toBeLessThan(1);
|
||
|
|
expect(Number.isInteger(budget.limitBytes)).toBe(true);
|
||
|
|
expect(Number.isInteger(budget.budgetBytes)).toBe(true);
|
||
|
|
expect(budget.source).toBe('estimated');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('matches the rounded budgets of the design table', () => {
|
||
|
|
const rounded = TIERS.map(([soldGb]) => [
|
||
|
|
soldGb,
|
||
|
|
(deviceBudget(mac(soldGb, 'Apple M4')).budgetBytes / GIB).toFixed(1),
|
||
|
|
]);
|
||
|
|
expect(rounded).toEqual([
|
||
|
|
[8, '2.0'],
|
||
|
|
[16, '9.7'],
|
||
|
|
[18, '11.0'],
|
||
|
|
[24, '15.0'],
|
||
|
|
[32, '20.3'],
|
||
|
|
[36, '26.0'],
|
||
|
|
[48, '35.0'],
|
||
|
|
[64, '47.0'],
|
||
|
|
[96, '71.0'],
|
||
|
|
[128, '95.0'],
|
||
|
|
]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('keeps a reserve of 6 GiB up to 18 GB, 8 GiB up to 48 GB and 12 GiB from 64 GB', () => {
|
||
|
|
// A wired limit far above the memory leaves "memory - reserve" as the only bound.
|
||
|
|
const reserveOf = (soldGb: number): number => {
|
||
|
|
const budget = deviceBudget(mac(soldGb, 'Apple M4', { wiredLimitMb: 1024 * 1024 }));
|
||
|
|
return (soldGb * GIB - budget.budgetBytes) / GIB;
|
||
|
|
};
|
||
|
|
expect([8, 16, 18].map(reserveOf)).toEqual([6, 6, 6]);
|
||
|
|
expect([24, 32, 36, 48].map(reserveOf)).toEqual([8, 8, 8, 8]);
|
||
|
|
expect([64, 96, 128].map(reserveOf)).toEqual([12, 12, 12]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('uses the wired limit when it is above 0', () => {
|
||
|
|
// 14000 MiB on the 16 GB Mac: L - 1 GiB is 12.67 GiB, memory - reserve is 10 GiB.
|
||
|
|
expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 14000 }))).toEqual({
|
||
|
|
limitBytes: 14000 * MIB,
|
||
|
|
budgetBytes: 10 * GIB,
|
||
|
|
source: 'wired-limit',
|
||
|
|
});
|
||
|
|
// A limit BELOW the estimate is honoured too: 8192 MiB leaves 7 GiB.
|
||
|
|
expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 8192 }))).toEqual({
|
||
|
|
limitBytes: 8 * GIB,
|
||
|
|
budgetBytes: 7 * GIB,
|
||
|
|
source: 'wired-limit',
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
it('reads a wired limit of 0, or none at all, as "not set"', () => {
|
||
|
|
const estimated = deviceBudget(mac(16, 'Apple M4'));
|
||
|
|
expect(estimated.source).toBe('estimated');
|
||
|
|
expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 0 }))).toEqual(estimated);
|
||
|
|
expect(deviceBudget(without(mac(16, 'Apple M4'), 'wiredLimitMb'))).toEqual(estimated);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('never goes below 0', () => {
|
||
|
|
expect(deviceBudget(mac(4, 'Apple M1')).budgetBytes).toBe(0);
|
||
|
|
expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 512 })).budgetBytes).toBe(0);
|
||
|
|
expect(deviceBudget({ ...mac(16, 'Apple M4'), memBytes: 0, memClassGb: 0 })).toEqual({
|
||
|
|
limitBytes: 0,
|
||
|
|
budgetBytes: 0,
|
||
|
|
source: 'estimated',
|
||
|
|
});
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('needBytes', () => {
|
||
|
|
const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry;
|
||
|
|
|
||
|
|
it('adds weights, the KV cache of the context, 1 GiB and the prompt cache', () => {
|
||
|
|
// 6,550,825,373 + 32768 x 65536 + 1 GiB + 512 MiB on a Mac below 32 GB.
|
||
|
|
expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + 2 * GIB + GIB + 512 * MIB);
|
||
|
|
expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(10_308_921_757);
|
||
|
|
// Half the context, half the KV cache.
|
||
|
|
expect(needBytes(nineB, 32768, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + GIB + GIB + 512 * MIB);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('counts a prompt cache of 512 MiB below 32 GB of memory and 2 GiB from 32 GB', () => {
|
||
|
|
const base = 6_550_825_373 + 2 * GIB + GIB;
|
||
|
|
for (const soldGb of [8, 16, 18, 24]) {
|
||
|
|
expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(512 * MIB);
|
||
|
|
}
|
||
|
|
for (const soldGb of [32, 36, 48, 64, 96, 128]) {
|
||
|
|
expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(2 * GIB);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('does not depend on the chip or the harness', () => {
|
||
|
|
expect(needBytes(nineB, 65536, STOCK, mac(48, 'Apple M1'))).toBe(
|
||
|
|
needBytes(nineB, 65536, STOCK, mac(48, 'Apple M5 Ultra'))
|
||
|
|
);
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('the fit grade', () => {
|
||
|
|
const pickOf = (need: number): Candidate | undefined =>
|
||
|
|
recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', need)]), [fact('alpha')]).pick;
|
||
|
|
|
||
|
|
it('has the budget this group assumes', () => {
|
||
|
|
expect(deviceBudget(HUNDRED_GIB_MAC).budgetBytes).toBe(HUNDRED_GIB);
|
||
|
|
expect(FIT_GOOD_MAX_SHARE).toBe(0.85);
|
||
|
|
expect(FIT_TIGHT_MAX_SHARE).toBe(0.98);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is good up to 0.85 of the budget', () => {
|
||
|
|
expect(pickOf(0.5 * HUNDRED_GIB)).toMatchObject({ fit: 'good', fitLabel: 'Good', budgetShare: 0.5 });
|
||
|
|
expect(pickOf(0.85 * HUNDRED_GIB)).toMatchObject({ fit: 'good', budgetShare: 0.85 });
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is tight above 0.85 and up to 0.98', () => {
|
||
|
|
expect(pickOf(0.85 * HUNDRED_GIB + 1)).toMatchObject({ fit: 'tight', fitLabel: 'Tight' });
|
||
|
|
expect(pickOf(0.9 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.9 });
|
||
|
|
expect(pickOf(0.98 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.98 });
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is out above 0.98: the model is not offered at all', () => {
|
||
|
|
const result = recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', 0.98 * HUNDRED_GIB + 1)]), [
|
||
|
|
fact('alpha'),
|
||
|
|
]);
|
||
|
|
expect(result.verdict).toBe('none');
|
||
|
|
expect(result.reason).toBe('memory');
|
||
|
|
expect(result.pick).toBeUndefined();
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
// ---------- 2. The two real Macs ----------
|
||
|
|
|
||
|
|
describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => {
|
||
|
|
/**
|
||
|
|
* Expected: Qwen3.6 35B-A3B at 131,072 tokens with OpenCode, Good, about 42 tokens per second.
|
||
|
|
*
|
||
|
|
* Why. The budget is 35 GiB (3/4 of 48 GiB minus 1). Five models fit it as Good. The M5 Pro is
|
||
|
|
* in the chips table (307 GB/s), so each model with a speed figure gets an estimate:
|
||
|
|
* the dense 27B computes to 2.95 x 3.07 = 9.1 tokens per second, under the 10 that separate
|
||
|
|
* slow from ok, while the 35B-A3B (3 billion parameters active per token) computes to 41.8.
|
||
|
|
* Speed ranks before model rank, so the 35B-A3B wins although the 27B has the higher rank. The
|
||
|
|
* 27B stays in the list, last, marked Slow. Both harnesses can run it; OpenCode has the
|
||
|
|
* higher rank.
|
||
|
|
*/
|
||
|
|
it('gets Qwen3.6 35B-A3B at 131072 tokens with OpenCode', async () => {
|
||
|
|
const hw = await realMac(MACBOOK_SYSCTL, 890_926_084);
|
||
|
|
expect(hw.chip).toBe('Apple M5 Pro');
|
||
|
|
expect(hw.memClassGb).toBe(48);
|
||
|
|
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
|
||
|
|
expect(result.verdict).toBe('ok');
|
||
|
|
expect(result.reason).toBeUndefined();
|
||
|
|
expect(result.budgetBytes).toBe(35 * GIB);
|
||
|
|
expect(result.budgetGb).toBe('37.6');
|
||
|
|
expect(result.budgetSource).toBe('estimated');
|
||
|
|
expect(result.pick).toEqual({
|
||
|
|
modelId: 'qwen3.6:35b-a3b',
|
||
|
|
modelLabel: 'Qwen3.6 35B-A3B',
|
||
|
|
license: 'Apache-2.0',
|
||
|
|
paramsB: 35,
|
||
|
|
context: 131072,
|
||
|
|
cli: 'opencode',
|
||
|
|
cliLabel: 'OpenCode',
|
||
|
|
cliInstalled: true,
|
||
|
|
// 22,621,314,161 + 20480 x 131072 + 1 GiB + 2 GiB.
|
||
|
|
needBytes: 28_526_894_193,
|
||
|
|
needGb: '28.5',
|
||
|
|
downloadBytes: 22_621_314_161,
|
||
|
|
downloadGb: '22.6',
|
||
|
|
budgetShare: 0.759,
|
||
|
|
fit: 'good',
|
||
|
|
fitLabel: 'Good',
|
||
|
|
speed: 'ok',
|
||
|
|
tokensPerSecond: 41.8,
|
||
|
|
tested: false,
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
it('says "Best fit", not "Recommended", and that nobody tested it on such a Mac', async () => {
|
||
|
|
const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.headline).toEqual({ text: 'Best fit for this Mac' });
|
||
|
|
expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]);
|
||
|
|
expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('lists the other four models that fit: the fast ones, the one of unknown speed, the slow one', async () => {
|
||
|
|
const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.alternatives.map((c) => [c.modelId, c.context, c.fit, c.speed, c.tokensPerSecond ?? null])).toEqual([
|
||
|
|
['qwen3.5:9b-q8_0', 65536, 'good', 'ok', 17.8],
|
||
|
|
['qwen3.5:9b', 65536, 'good', 'ok', 28.6],
|
||
|
|
['gpt-oss:20b', 131072, 'good', 'unknown', null],
|
||
|
|
['qwen3.8:27b', 131072, 'good', 'slow', 9.1],
|
||
|
|
]);
|
||
|
|
expect(result.alternatives[2].speedLabel).toBe('Speed unknown');
|
||
|
|
expect(result.alternatives[3].speedLabel).toBe('Slow');
|
||
|
|
// The two that do not fit 35 GiB at any context are not listed.
|
||
|
|
expect(order(result)).not.toContain('qwen3.8:27b-q8_0@131072');
|
||
|
|
expect(order(result).some((entry) => entry.startsWith('qwen3.5:122b'))).toBe(false);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('would get the 27B instead if its chip were in no chips row', async () => {
|
||
|
|
// The same Mac with the M5 Pro row removed: every speed is unknown, so model rank decides.
|
||
|
|
const hw = await realMac(MACBOOK_SYSCTL, 890_926_084);
|
||
|
|
const withoutChip = stockWith((catalog) => {
|
||
|
|
catalog.chips = catalog.chips.filter((row) => row.match !== 'M5 Pro');
|
||
|
|
});
|
||
|
|
const result = recommend(hw, withoutChip, ALL_INSTALLED);
|
||
|
|
expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', context: 131072, speed: 'unknown' });
|
||
|
|
expect(result.pick).not.toHaveProperty('tokensPerSecond');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => {
|
||
|
|
/**
|
||
|
|
* Expected: no recommendation, reason `memory`.
|
||
|
|
*
|
||
|
|
* Why. The budget is 9.67 GiB (2/3 of 16 GiB minus 1). The only model near it is the 9B. Both
|
||
|
|
* harnesses that may be offered need 65,536 tokens of context, and there the 9B needs
|
||
|
|
* 6.10 GiB of weights + 2 GiB of KV cache + 1 GiB + 0.5 GiB of prompt cache = 9.60 GiB, which
|
||
|
|
* is 0.993 of the budget: above the 0.98 where "tight" ends. At 32,768 tokens it would fit
|
||
|
|
* (0.890, tight), but the one harness that works at that context is gated.
|
||
|
|
*/
|
||
|
|
it('gets no recommendation: nothing fits at the context a harness needs', async () => {
|
||
|
|
const hw = await realMac(MACMINI_SYSCTL, 41_752_848);
|
||
|
|
expect(hw.chip).toBe('Apple M4');
|
||
|
|
expect(hw.memClassGb).toBe(16);
|
||
|
|
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
|
||
|
|
expect(result).toEqual({
|
||
|
|
verdict: 'none',
|
||
|
|
reason: 'memory',
|
||
|
|
headline: {
|
||
|
|
text: 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.',
|
||
|
|
vars: { memory: '17.2' },
|
||
|
|
},
|
||
|
|
notes: [],
|
||
|
|
alternatives: [],
|
||
|
|
// floor(17,179,869,184 x 2/3) - 1 GiB.
|
||
|
|
budgetBytes: 10_379_504_298,
|
||
|
|
budgetGb: '10.4',
|
||
|
|
budgetSource: 'estimated',
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
it('misses by a hair at 65536 tokens and would fit at 32768', async () => {
|
||
|
|
const hw = await realMac(MACMINI_SYSCTL, 41_752_848);
|
||
|
|
const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry;
|
||
|
|
const budget = deviceBudget(hw).budgetBytes;
|
||
|
|
const shareAt64k = needBytes(nineB, 65536, STOCK, hw) / budget;
|
||
|
|
const shareAt32k = needBytes(nineB, 32768, STOCK, hw) / budget;
|
||
|
|
expect(shareAt64k).toBeCloseTo(0.9932, 4);
|
||
|
|
expect(shareAt64k).toBeGreaterThan(FIT_TIGHT_MAX_SHARE);
|
||
|
|
expect(shareAt32k).toBeCloseTo(0.8898, 4);
|
||
|
|
expect(shareAt32k).toBeLessThan(FIT_TIGHT_MAX_SHARE);
|
||
|
|
// Every other model needs more than the whole budget at its smallest context.
|
||
|
|
for (const other of STOCK.models.filter((m) => m.id !== 'qwen3.5:9b')) {
|
||
|
|
const smallest = other.contexts[other.contexts.length - 1];
|
||
|
|
expect(needBytes(other, smallest, STOCK, hw), other.id).toBeGreaterThan(budget);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('would get the 9B at 32768 tokens, as a limited answer, once the gated harness is opened', async () => {
|
||
|
|
const hw = await realMac(MACMINI_SYSCTL, 41_752_848);
|
||
|
|
const ungated = stockWith((catalog) => {
|
||
|
|
catalog.harnesses = catalog.harnesses.map((row) => ({
|
||
|
|
cli: row.cli,
|
||
|
|
minContext: row.minContext,
|
||
|
|
rank: row.rank,
|
||
|
|
}));
|
||
|
|
});
|
||
|
|
const result = recommend(hw, ungated, ALL_INSTALLED);
|
||
|
|
expect(result.verdict).toBe('limited');
|
||
|
|
expect(result.pick).toMatchObject({
|
||
|
|
modelId: 'qwen3.5:9b',
|
||
|
|
context: 32768,
|
||
|
|
cli: 'pi',
|
||
|
|
fit: 'tight',
|
||
|
|
budgetShare: 0.89,
|
||
|
|
speed: 'ok',
|
||
|
|
// 9.3 x 1.2.
|
||
|
|
tokensPerSecond: 11.2,
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
it('would get the 9B at 65536 tokens if a wired limit had been set on it', async () => {
|
||
|
|
// iogpu.wired_limit_mb = 14000 lifts the budget to 10 GiB (memory minus the 6 GiB reserve).
|
||
|
|
const hw = { ...(await realMac(MACMINI_SYSCTL, 41_752_848)), wiredLimitMb: 14000 };
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.verdict).toBe('limited');
|
||
|
|
expect(result.budgetSource).toBe('wired-limit');
|
||
|
|
expect(result.budgetBytes).toBe(10 * GIB);
|
||
|
|
expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', context: 65536, cli: 'opencode', fit: 'tight' });
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('every memory size, with the stock catalogue', () => {
|
||
|
|
// What the rules compute today. A change of a catalogue figure or of a threshold that moves
|
||
|
|
// one of these answers has to change this table on purpose.
|
||
|
|
const EXPECTED: [soldGb: number, chip: string, answer: string][] = [
|
||
|
|
[8, 'Apple M2', 'none memory'],
|
||
|
|
[16, 'Apple M4', 'none memory'],
|
||
|
|
// The 9B at 65536 tokens is 0.87 of 11 GiB.
|
||
|
|
[18, 'Apple M3 Pro', 'limited qwen3.5:9b@65536 tight ok'],
|
||
|
|
// gpt-oss:20b at 65536 tokens needs 15.8 GiB of the 15 GiB budget: out.
|
||
|
|
[24, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'],
|
||
|
|
// gpt-oss:20b fits only as Tight here (0.853 at 65536 tokens, 0.927 at 131072), and Good
|
||
|
|
// ranks before Tight. The Q8 9B is Good but slow on a base M4 (7 tokens per second).
|
||
|
|
[32, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'],
|
||
|
|
[32, 'Apple M1 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'],
|
||
|
|
// The 27B (0.905) and the 35B-A3B (0.974) are Tight; gpt-oss:20b is Good but has no speed
|
||
|
|
// figure, and a known speed ranks before an unknown one.
|
||
|
|
[36, 'Apple M4 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'],
|
||
|
|
// On a Pro chip the dense 27B is slow (8.1 tokens per second), the 35B-A3B is not.
|
||
|
|
[48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'],
|
||
|
|
[48, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'],
|
||
|
|
// The Q8 27B fits 47 GiB but computes to 7 tokens per second on this chip.
|
||
|
|
[64, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'],
|
||
|
|
[96, 'Apple M3 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'],
|
||
|
|
// The 122B fits 95 GiB only as Tight (0.861) and has no speed figure.
|
||
|
|
[128, 'Apple M5 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'],
|
||
|
|
];
|
||
|
|
|
||
|
|
it.each(EXPECTED)('%i GB, %s: %s', (soldGb, chip, answer) => {
|
||
|
|
const result = recommend(mac(soldGb, chip), STOCK, ALL_INSTALLED);
|
||
|
|
const actual = result.pick
|
||
|
|
? `${result.verdict} ${result.pick.modelId}@${result.pick.context} ${result.pick.fit} ${result.pick.speed}`
|
||
|
|
: `${result.verdict} ${result.reason}`;
|
||
|
|
expect(actual).toBe(answer);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('always pairs the pick with the top-ranked harness, OpenCode', () => {
|
||
|
|
for (const hw of SWEEP) {
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
if (result.pick) expect(result.pick.cli).toBe('opencode');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
// ---------- 3. Verdict none ----------
|
||
|
|
|
||
|
|
describe('verdict none', () => {
|
||
|
|
const bigMac = mac(48, 'Apple M4 Max');
|
||
|
|
|
||
|
|
/** A `none` answer has no pick, no alternatives, no notes, and the reason's sentence as headline. */
|
||
|
|
function expectNone(result: Recommendation, reason: Recommendation['reason'], headline: string): void {
|
||
|
|
expect(result.verdict).toBe('none');
|
||
|
|
expect(result.reason).toBe(reason);
|
||
|
|
expect(result.headline.text).toBe(headline);
|
||
|
|
expect(result.pick).toBeUndefined();
|
||
|
|
expect(result.alternatives).toEqual([]);
|
||
|
|
expect(result.notes).toEqual([]);
|
||
|
|
}
|
||
|
|
|
||
|
|
describe('not-macos', () => {
|
||
|
|
it('answers so on Linux, with no budget at all', () => {
|
||
|
|
const linux: HardwareFacts = {
|
||
|
|
platform: 'linux',
|
||
|
|
arch: 'x64',
|
||
|
|
memBytes: 64 * GIB,
|
||
|
|
memGb: formatGb(64 * GIB),
|
||
|
|
memClassGb: 64,
|
||
|
|
source: 'not-probed',
|
||
|
|
};
|
||
|
|
const result = recommend(linux, STOCK, ALL_INSTALLED);
|
||
|
|
expectNone(result, 'not-macos', LOCAL_MODEL_LABELS.noneNotMacos);
|
||
|
|
expect(result.headline.vars).toBeUndefined();
|
||
|
|
expect(result.budgetBytes).toBe(0);
|
||
|
|
expect(result.budgetGb).toBe('0.0');
|
||
|
|
expect(result.budgetSource).toBe('estimated');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('comes before every other reason', () => {
|
||
|
|
for (const platform of ['linux', 'win32', 'freebsd']) {
|
||
|
|
const hw = mac(8, 'Intel(R) Core(TM) i7', { platform, arch: 'x64', arm64: false, macosMajor: 12 });
|
||
|
|
expect(recommend(hw, STOCK, []).reason, platform).toBe('not-macos');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('intel', () => {
|
||
|
|
const intelChip = 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz';
|
||
|
|
|
||
|
|
it('answers so for an Intel Mac, whose sysctl has no arm64 flag', () => {
|
||
|
|
const hw = without(mac(32, intelChip, { arch: 'x64' }), 'arm64', 'translated', 'wiredLimitMb');
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
expectNone(result, 'intel', 'This Mac has an Intel processor. Local coding models need Apple silicon.');
|
||
|
|
expect(result.budgetBytes).toBe(0);
|
||
|
|
expect(result.budgetGb).toBe('0.0');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('answers so when the arm64 flag reads 0', () => {
|
||
|
|
const hw = without(mac(32, intelChip, { arch: 'x64', arm64: false }), 'translated');
|
||
|
|
expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('intel');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('comes before the macOS version and the memory', () => {
|
||
|
|
const hw = without(mac(8, intelChip, { arch: 'x64', macosMajor: 12, macosVersion: '12.7' }), 'arm64');
|
||
|
|
expect(recommend(hw, STOCK, []).reason).toBe('intel');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('does not mistake Node under Rosetta for an Intel Mac', () => {
|
||
|
|
// An x64 process on Apple silicon: the machine flag says arm64, the process is translated.
|
||
|
|
const rosetta = mac(48, 'Apple M4 Max', { arch: 'x64', translated: true });
|
||
|
|
expect(recommend(rosetta, STOCK, ALL_INSTALLED).verdict).toBe('ok');
|
||
|
|
// The same when only the translation flag was read.
|
||
|
|
expect(recommend(without(rosetta, 'arm64'), STOCK, ALL_INSTALLED).verdict).toBe('ok');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('decides from node:os alone when the probe failed', () => {
|
||
|
|
const fallback = (arch: string, chip: string): HardwareFacts => ({
|
||
|
|
platform: 'darwin',
|
||
|
|
arch,
|
||
|
|
memBytes: 48 * GIB,
|
||
|
|
memGb: formatGb(48 * GIB),
|
||
|
|
memClassGb: 48,
|
||
|
|
chip,
|
||
|
|
source: 'os-fallback',
|
||
|
|
});
|
||
|
|
// An arm64 process can only run on Apple silicon.
|
||
|
|
expect(recommend(fallback('arm64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok');
|
||
|
|
// An x64 process with an Apple chip name is Rosetta.
|
||
|
|
expect(recommend(fallback('x64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok');
|
||
|
|
// An x64 process with any other chip name is taken for an Intel Mac.
|
||
|
|
expect(recommend(fallback('x64', intelChip), STOCK, ALL_INSTALLED).reason).toBe('intel');
|
||
|
|
expect(recommend(without(fallback('x64', intelChip), 'chip'), STOCK, ALL_INSTALLED).reason).toBe('intel');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('macos', () => {
|
||
|
|
it('answers so below the catalogue minimum, naming both versions', () => {
|
||
|
|
const result = recommend(
|
||
|
|
mac(48, 'Apple M1 Max', { macosVersion: '13.6.7', macosMajor: 13 }),
|
||
|
|
STOCK,
|
||
|
|
ALL_INSTALLED
|
||
|
|
);
|
||
|
|
expectNone(result, 'macos', 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.');
|
||
|
|
expect(result.headline.vars).toEqual({ version: '13.6.7', minimum: '14' });
|
||
|
|
// The budget is still reported: the Mac itself is able.
|
||
|
|
expect(result.budgetBytes).toBe(35 * GIB);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('accepts the minimum itself and anything newer', () => {
|
||
|
|
for (const macosMajor of [14, 15, 26, 27]) {
|
||
|
|
const hw = mac(48, 'Apple M4 Max', { macosMajor, macosVersion: `${macosMajor}.0` });
|
||
|
|
expect(recommend(hw, STOCK, ALL_INSTALLED).verdict, String(macosMajor)).toBe('ok');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('does not block on a version that could not be read', () => {
|
||
|
|
const hw = without(mac(48, 'Apple M4 Max'), 'macosMajor', 'macosVersion');
|
||
|
|
expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('comes before the memory', () => {
|
||
|
|
const hw = mac(8, 'Apple M1', { macosVersion: '12.7', macosMajor: 12 });
|
||
|
|
expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('macos');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('memory', () => {
|
||
|
|
it('answers so at 8 GB, where nothing fits at any context', () => {
|
||
|
|
const result = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED);
|
||
|
|
expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemory);
|
||
|
|
// The size is the decimal figure of the hardware line, without the unit.
|
||
|
|
expect(result.headline.vars).toEqual({ memory: '8.6' });
|
||
|
|
expect(result.budgetBytes).toBe(2 * GIB);
|
||
|
|
expect(result.budgetGb).toBe('2.1');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('answers so when models fit, but only below the context a harness needs', () => {
|
||
|
|
// The 16 GB case of the Mac mini, with harnesses that are all present.
|
||
|
|
const result = recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.reason).toBe('memory');
|
||
|
|
expect(result.headline.vars).toEqual({ memory: '17.2' });
|
||
|
|
});
|
||
|
|
|
||
|
|
it('comes before no-harness when nothing fits at all', () => {
|
||
|
|
expect(recommend(mac(8, 'Apple M2'), STOCK, []).reason).toBe('memory');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('answers so for an empty catalogue', () => {
|
||
|
|
expect(recommend(bigMac, catalogOf([]), [fact('alpha')]).reason).toBe('memory');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('answers so when the memory size is unknown', () => {
|
||
|
|
const hw = { ...mac(48, 'Apple M4 Max'), memBytes: 0, memGb: '0.0', memClassGb: 0 };
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.reason).toBe('memory');
|
||
|
|
expect(result.budgetBytes).toBe(0);
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('no-harness', () => {
|
||
|
|
const headline = 'No coding agent that Codeman can connect to a local model is available on this Mac.';
|
||
|
|
|
||
|
|
it('answers so when the caller knows no catalogue harness', () => {
|
||
|
|
const result = recommend(bigMac, STOCK, []);
|
||
|
|
expectNone(result, 'no-harness', headline);
|
||
|
|
expect(result.headline.vars).toBeUndefined();
|
||
|
|
expect(result.budgetBytes).toBe(35 * GIB);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('answers so when no harness is installed or installable', () => {
|
||
|
|
const facts = STOCK.harnesses.map((row) => fact(row.cli, { installed: false, installable: false }));
|
||
|
|
expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('answers so when only the gated harness is on the machine', () => {
|
||
|
|
const gatedRows = STOCK.harnesses.filter((row) => row.gated);
|
||
|
|
expect(gatedRows).toHaveLength(1);
|
||
|
|
expect(
|
||
|
|
recommend(
|
||
|
|
bigMac,
|
||
|
|
STOCK,
|
||
|
|
gatedRows.map((row) => fact(row.cli))
|
||
|
|
).reason
|
||
|
|
).toBe('no-harness');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('answers so when a harness is there but accepts no model id', () => {
|
||
|
|
const facts = ALL_INSTALLED.map((entry) => ({ ...entry, acceptsModelId: () => false }));
|
||
|
|
expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('ignores a fact for a CLI that is not a catalogue harness', () => {
|
||
|
|
expect(recommend(bigMac, STOCK, [fact('codex'), fact('some-other-cli')]).reason).toBe('no-harness');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('disk', () => {
|
||
|
|
it('answers so when every fitting model would leave under 10 GiB free', () => {
|
||
|
|
// 12 GiB free: even the smallest download (6.55 GB) leaves 5.9 GiB.
|
||
|
|
const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) });
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
expectNone(
|
||
|
|
result,
|
||
|
|
'disk',
|
||
|
|
'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.'
|
||
|
|
);
|
||
|
|
// download + 10 GiB - free = 6,550,825,373 + 10,737,418,240 - 12,884,901,888 bytes.
|
||
|
|
expect(result.headline.vars).toEqual({ download: '6.6', free: '12.9', missing: '4.4' });
|
||
|
|
expect(result.budgetBytes).toBe(35 * GIB);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('names the smallest model that fits the MEMORY, not the smallest of the catalogue', () => {
|
||
|
|
// On a 48 GB Mac whose harness refuses both 9B ids, the smallest candidate is gpt-oss:20b.
|
||
|
|
const facts = ALL_INSTALLED.map((entry) => ({
|
||
|
|
...entry,
|
||
|
|
acceptsModelId: (id: string) => !id.startsWith('qwen3.5:9b'),
|
||
|
|
}));
|
||
|
|
const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) });
|
||
|
|
const result = recommend(hw, STOCK, facts);
|
||
|
|
expect(result.reason).toBe('disk');
|
||
|
|
expect(result.headline.vars?.download).toBe('13.8');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('drops only the models that do not leave 10 GiB, and picks among the rest', () => {
|
||
|
|
// 20 GiB free: the 9B downloads (6.1 and 9.7 GiB) leave 10 GiB or more, the others do not.
|
||
|
|
const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 20 * GIB, freeDiskGb: formatGb(20 * GIB) });
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.verdict).toBe('limited');
|
||
|
|
expect(order(result)).toEqual(['qwen3.5:9b-q8_0@65536', 'qwen3.5:9b@65536']);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('allows a download that leaves exactly 10 GiB', () => {
|
||
|
|
const exact = STOCK.models[0].bytes + DISK_HEADROOM_BYTES;
|
||
|
|
expect(DISK_HEADROOM_BYTES).toBe(10 * GIB);
|
||
|
|
const fits = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact }), STOCK, ALL_INSTALLED);
|
||
|
|
expect(fits.pick?.modelId).toBe('qwen3.5:9b');
|
||
|
|
const short = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact - 1 }), STOCK, ALL_INSTALLED);
|
||
|
|
expect(short.reason).toBe('disk');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('does not block when the free disk is unknown', () => {
|
||
|
|
const hw = without(mac(48, 'Apple M4 Max'), 'freeDiskBytes', 'freeDiskGb');
|
||
|
|
expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('formats the free size itself when the hardware carries only the bytes', () => {
|
||
|
|
const hw = without(mac(48, 'Apple M4 Max', { freeDiskBytes: 3 * GB }), 'freeDiskGb');
|
||
|
|
expect(recommend(hw, STOCK, ALL_INSTALLED).headline.vars?.free).toBe('3.0');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('comes after the memory and the harness', () => {
|
||
|
|
const fullDisk = { freeDiskBytes: 0, freeDiskGb: '0.0' };
|
||
|
|
expect(recommend(mac(16, 'Apple M4', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('memory');
|
||
|
|
expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, []).reason).toBe('no-harness');
|
||
|
|
expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('disk');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
// ---------- 4. Verdicts, ranking, harnesses, tested ----------
|
||
|
|
|
||
|
|
describe('verdict limited', () => {
|
||
|
|
it('is the answer when the best model that fits has fewer than 12 billion parameters', () => {
|
||
|
|
expect(LIMITED_BELOW_PARAMS_B).toBe(12);
|
||
|
|
const result = recommend(mac(24, 'Apple M4'), STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.verdict).toBe('limited');
|
||
|
|
expect(result.reason).toBeUndefined();
|
||
|
|
expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', paramsB: 9, context: 65536, fit: 'good' });
|
||
|
|
expect(result.headline).toEqual({
|
||
|
|
text: 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.',
|
||
|
|
});
|
||
|
|
expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('holds for a tight fit too (18 GB)', () => {
|
||
|
|
const result = recommend(mac(18, 'Apple M3 Pro'), STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.verdict).toBe('limited');
|
||
|
|
expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', fit: 'tight', fitLabel: 'Tight', budgetShare: 0.873 });
|
||
|
|
expect(result.alternatives).toEqual([]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('draws the line at 12 billion: 11.9 is limited, 12 is ok', () => {
|
||
|
|
const verdictFor = (paramsB: number): string =>
|
||
|
|
recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { paramsB })]), [fact('alpha')]).verdict;
|
||
|
|
expect(verdictFor(11.9)).toBe('limited');
|
||
|
|
expect(verdictFor(12)).toBe('ok');
|
||
|
|
expect(verdictFor(9)).toBe('limited');
|
||
|
|
expect(verdictFor(27.8)).toBe('ok');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is about the PICK: a larger model further down the list does not lift it', () => {
|
||
|
|
// 32 GB: the 20B fits only as Tight, so a 9B is the pick and the answer is limited.
|
||
|
|
const result = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.verdict).toBe('limited');
|
||
|
|
expect(result.alternatives.map((c) => [c.modelId, c.fit])).toContainEqual(['gpt-oss:20b', 'tight']);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('never says "Recommended", for any Mac', () => {
|
||
|
|
for (const hw of SWEEP) {
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
if (result.verdict === 'limited') {
|
||
|
|
expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited);
|
||
|
|
expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('speed', () => {
|
||
|
|
it('is the model figure times the chip bandwidth, rounded to one decimal', () => {
|
||
|
|
const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED);
|
||
|
|
// 2.95 x 410 / 100.
|
||
|
|
expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', speed: 'ok', tokensPerSecond: 12.1 });
|
||
|
|
expect(result.pick).not.toHaveProperty('speedLabel');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is slow below 10 tokens per second, and says so', () => {
|
||
|
|
expect(SLOW_BELOW_TOKENS_PER_SECOND).toBe(10);
|
||
|
|
const speedOf = (speedPer100Gbps: number): Candidate | undefined =>
|
||
|
|
recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { speedPer100Gbps })]), [fact('alpha')]).pick;
|
||
|
|
// The synthetic chip has 400 GB/s: 2.5 is 10.0 tokens per second, 2.475 is 9.9.
|
||
|
|
expect(speedOf(2.5)).toMatchObject({ speed: 'ok', tokensPerSecond: 10 });
|
||
|
|
expect(speedOf(2.475)).toMatchObject({ speed: 'slow', tokensPerSecond: 9.9, speedLabel: 'Slow' });
|
||
|
|
expect(speedOf(0.5)).toMatchObject({ speed: 'slow', tokensPerSecond: 2, speedLabel: 'Slow' });
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is unknown for a model without a figure, on any chip', () => {
|
||
|
|
// 48 GB: five models fit, so all of them are listed and gpt-oss:20b is among them.
|
||
|
|
for (const chip of ['Apple M1', 'Apple M4 Max', 'Apple M5 Ultra']) {
|
||
|
|
const result = recommend(mac(48, chip), STOCK, ALL_INSTALLED);
|
||
|
|
const gptOss = [result.pick, ...result.alternatives].find((c) => c?.modelId === 'gpt-oss:20b');
|
||
|
|
expect(gptOss, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' });
|
||
|
|
expect(gptOss).not.toHaveProperty('tokensPerSecond');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is unknown for every model on a chip that is in no chips row: no bandwidth is guessed', () => {
|
||
|
|
for (const chip of ['Apple M6', 'Apple M6 Pro', 'Apple M10', 'Some Future Chip']) {
|
||
|
|
const result = recommend(mac(64, chip), STOCK, ALL_INSTALLED);
|
||
|
|
for (const candidate of [result.pick, ...result.alternatives]) {
|
||
|
|
expect(candidate, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' });
|
||
|
|
expect(candidate).not.toHaveProperty('tokensPerSecond');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// And when the chip name itself is missing.
|
||
|
|
const result = recommend(without(mac(64, 'x'), 'chip'), STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.pick?.speed).toBe('unknown');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('the ranking', () => {
|
||
|
|
it('puts ok before unknown before slow, ahead of the model rank', () => {
|
||
|
|
// Three models that fit equally well. The ranks run AGAINST the speed.
|
||
|
|
const catalog = catalogOf([
|
||
|
|
model('slow-top:1', { rank: 3, speedPer100Gbps: 1 }),
|
||
|
|
model('unknown-mid:1', { rank: 2 }),
|
||
|
|
model('ok-low:1', { rank: 1, speedPer100Gbps: 5 }),
|
||
|
|
]);
|
||
|
|
const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]);
|
||
|
|
expect(order(result)).toEqual(['ok-low:1@1024', 'unknown-mid:1@1024', 'slow-top:1@1024']);
|
||
|
|
expect([result.pick, ...result.alternatives].map((c) => c?.speed)).toEqual(['ok', 'unknown', 'slow']);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('ranks by model rank alone when every speed is unknown', () => {
|
||
|
|
// 'Apple M6' is in no chips row.
|
||
|
|
const result = recommend(mac(48, 'Apple M6'), STOCK, ALL_INSTALLED);
|
||
|
|
expect(order(result)).toEqual([
|
||
|
|
'qwen3.8:27b@131072',
|
||
|
|
'qwen3.6:35b-a3b@131072',
|
||
|
|
'gpt-oss:20b@131072',
|
||
|
|
'qwen3.5:9b-q8_0@65536',
|
||
|
|
'qwen3.5:9b@65536',
|
||
|
|
]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('ranks an unknown speed below a known ok one and above a slow one, on a real tier', () => {
|
||
|
|
// 48 GB, M4 Pro: three ok models, then gpt-oss:20b (unknown), then the 27B (slow).
|
||
|
|
const result = recommend(mac(48, 'Apple M4 Pro'), STOCK, ALL_INSTALLED);
|
||
|
|
expect([result.pick, ...result.alternatives].map((c) => [c?.modelId, c?.speed])).toEqual([
|
||
|
|
['qwen3.6:35b-a3b', 'ok'],
|
||
|
|
['qwen3.5:9b-q8_0', 'ok'],
|
||
|
|
['qwen3.5:9b', 'ok'],
|
||
|
|
['gpt-oss:20b', 'unknown'],
|
||
|
|
['qwen3.8:27b', 'slow'],
|
||
|
|
]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('puts a good fit before a tight one, ahead of speed and rank', () => {
|
||
|
|
const catalog = catalogOf([
|
||
|
|
modelNeeding('tight-fast-top:1', 0.9 * HUNDRED_GIB, { rank: 9, speedPer100Gbps: 20 }),
|
||
|
|
modelNeeding('good-slow-low:1', 0.5 * HUNDRED_GIB, { rank: 1, speedPer100Gbps: 1 }),
|
||
|
|
]);
|
||
|
|
const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]);
|
||
|
|
expect(order(result)).toEqual(['good-slow-low:1@1024', 'tight-fast-top:1@1024']);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('takes the higher model rank among equals', () => {
|
||
|
|
const catalog = catalogOf([model('low:1', { rank: 1 }), model('high:1', { rank: 5 }), model('mid:1', { rank: 3 })]);
|
||
|
|
expect(order(recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]))).toEqual([
|
||
|
|
'high:1@1024',
|
||
|
|
'mid:1@1024',
|
||
|
|
'low:1@1024',
|
||
|
|
]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('takes the larger context of one model when both fit equally well', () => {
|
||
|
|
// 1 KiB per token: 4 GiB of KV cache at 4,194,304 tokens, still a good fit.
|
||
|
|
const catalog = catalogOf([model('probe:1', { kvBytesPerToken: 1024, contexts: [4_194_304, 1024] })]);
|
||
|
|
const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]);
|
||
|
|
expect(result.pick?.context).toBe(4_194_304);
|
||
|
|
expect(result.alternatives).toEqual([]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('steps a model down to the context at which it fits well', () => {
|
||
|
|
// Good at the small context, tight at the large one: the good one is that model's entry.
|
||
|
|
const catalog = catalogOf([
|
||
|
|
model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] }),
|
||
|
|
]);
|
||
|
|
const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]);
|
||
|
|
expect(result.pick).toMatchObject({ context: 1024, fit: 'good' });
|
||
|
|
});
|
||
|
|
|
||
|
|
it('lists each model once, never the pick again, and at most four alternatives', () => {
|
||
|
|
expect(MAX_ALTERNATIVES).toBe(4);
|
||
|
|
// Six models fit a 128 GB Mac; the pick plus four are shown.
|
||
|
|
const result = recommend(mac(128, 'Apple M5 Ultra'), STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.alternatives).toHaveLength(4);
|
||
|
|
const ids = [result.pick, ...result.alternatives].map((c) => c?.modelId);
|
||
|
|
expect(new Set(ids).size).toBe(ids.length);
|
||
|
|
for (const hw of SWEEP) {
|
||
|
|
const swept = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
const sweptIds = [swept.pick, ...swept.alternatives].filter(Boolean).map((c) => c?.modelId);
|
||
|
|
expect(new Set(sweptIds).size).toBe(sweptIds.length);
|
||
|
|
expect(swept.alternatives.length).toBeLessThanOrEqual(MAX_ALTERNATIVES);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('harnesses', () => {
|
||
|
|
const hw = mac(48, 'Apple M4 Max');
|
||
|
|
|
||
|
|
it('never chooses a gated harness, even when it is the only one that could run a model', () => {
|
||
|
|
const gated = STOCK.harnesses.filter((row) => row.gated).map((row) => row.cli);
|
||
|
|
expect(gated).toEqual(['pi']);
|
||
|
|
for (const swept of SWEEP) {
|
||
|
|
const result = recommend(swept, STOCK, ALL_INSTALLED);
|
||
|
|
for (const candidate of [result.pick, ...result.alternatives]) {
|
||
|
|
if (candidate) expect(gated).not.toContain(candidate.cli);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// 16 GB: the gated harness alone works at 32768 tokens, where the 9B would fit. Still none.
|
||
|
|
expect(recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED).verdict).toBe('none');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('never offers a context below the minimum of the harness', () => {
|
||
|
|
for (const swept of SWEEP) {
|
||
|
|
const result = recommend(swept, STOCK, ALL_INSTALLED);
|
||
|
|
for (const candidate of [result.pick, ...result.alternatives]) {
|
||
|
|
if (!candidate) continue;
|
||
|
|
const row = STOCK.harnesses.find((harness) => harness.cli === candidate.cli);
|
||
|
|
expect(candidate.context).toBeGreaterThanOrEqual(row?.minContext ?? Number.POSITIVE_INFINITY);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('takes the higher-ranked harness and reports its label and that it is installed', () => {
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.pick).toMatchObject({ cli: 'opencode', cliLabel: 'OpenCode', cliInstalled: true });
|
||
|
|
expect(result.pick).not.toHaveProperty('slowFirstTurn');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('falls back to the next harness when the first is not on the machine, and flags its slow first turn', () => {
|
||
|
|
const facts = [fact('opencode', { installed: false, installable: false }), fact('claude'), fact('pi')];
|
||
|
|
const result = recommend(hw, STOCK, facts);
|
||
|
|
expect(result.pick).toMatchObject({
|
||
|
|
modelId: 'qwen3.8:27b',
|
||
|
|
cli: 'claude',
|
||
|
|
cliLabel: 'Claude Code',
|
||
|
|
cliInstalled: true,
|
||
|
|
slowFirstTurn: true,
|
||
|
|
});
|
||
|
|
// The same when the caller has no fact for it at all.
|
||
|
|
expect(recommend(hw, STOCK, [fact('claude')]).pick?.cli).toBe('claude');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('offers a harness that is not installed but can be, and says it is not installed', () => {
|
||
|
|
const facts = [fact('opencode', { installed: false, installable: true }), fact('claude')];
|
||
|
|
const result = recommend(hw, STOCK, facts);
|
||
|
|
expect(result.pick).toMatchObject({ cli: 'opencode', cliInstalled: false });
|
||
|
|
});
|
||
|
|
|
||
|
|
it('pairs a model only with a harness that accepts its id', () => {
|
||
|
|
const refusing27b = (id: string): boolean => id !== 'qwen3.8:27b';
|
||
|
|
// OpenCode refuses the 27B: it goes with the next harness.
|
||
|
|
const one = recommend(hw, STOCK, [fact('opencode', { acceptsModelId: refusing27b }), fact('claude')]);
|
||
|
|
expect(one.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'claude' });
|
||
|
|
expect(one.alternatives[0]).toMatchObject({ modelId: 'qwen3.6:35b-a3b', cli: 'opencode' });
|
||
|
|
// Both refuse it: the model is not offered.
|
||
|
|
const both = recommend(hw, STOCK, [
|
||
|
|
fact('opencode', { acceptsModelId: refusing27b }),
|
||
|
|
fact('claude', { acceptsModelId: refusing27b }),
|
||
|
|
]);
|
||
|
|
expect(both.pick?.modelId).toBe('qwen3.6:35b-a3b');
|
||
|
|
expect(order(both)).not.toContain('qwen3.8:27b@131072');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('treats a harness check that throws as a refusal, and still answers', () => {
|
||
|
|
const throwing = fact('opencode', {
|
||
|
|
acceptsModelId: () => {
|
||
|
|
throw new Error('registry not loaded');
|
||
|
|
},
|
||
|
|
});
|
||
|
|
const result = recommend(hw, STOCK, [throwing, fact('claude')]);
|
||
|
|
expect(result.verdict).toBe('ok');
|
||
|
|
expect(result.pick?.cli).toBe('claude');
|
||
|
|
expect(recommend(hw, STOCK, [throwing]).reason).toBe('no-harness');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('puts the fit and the model before the harness rank', () => {
|
||
|
|
// `big` needs a large context and is the higher-ranked harness; `small` works from 1024.
|
||
|
|
const harnesses: LocalHarnessEntry[] = [
|
||
|
|
{ cli: 'big', minContext: 8_388_608, rank: 2 },
|
||
|
|
{ cli: 'small', minContext: 1024, rank: 1 },
|
||
|
|
];
|
||
|
|
const tightAtLarge = catalogOf(
|
||
|
|
[model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })],
|
||
|
|
harnesses
|
||
|
|
);
|
||
|
|
// Tight at the context `big` needs, good at the small one: the good fit with `small` wins.
|
||
|
|
expect(recommend(HUNDRED_GIB_MAC, tightAtLarge, [fact('big'), fact('small')]).pick).toMatchObject({
|
||
|
|
cli: 'small',
|
||
|
|
context: 1024,
|
||
|
|
fit: 'good',
|
||
|
|
});
|
||
|
|
// Good at both: the higher-ranked harness wins, at the context it needs.
|
||
|
|
const goodAtBoth = catalogOf(
|
||
|
|
[model('probe:1', { bytes: 10 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })],
|
||
|
|
harnesses
|
||
|
|
);
|
||
|
|
expect(recommend(HUNDRED_GIB_MAC, goodAtBoth, [fact('big'), fact('small')]).pick).toMatchObject({
|
||
|
|
cli: 'big',
|
||
|
|
context: 8_388_608,
|
||
|
|
});
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('the tested flag and the word "Recommended"', () => {
|
||
|
|
const testedOn = (chip: string, memGb: number, cli: string) => ({ chip, memGb, cli, date: '2026-11-06' });
|
||
|
|
const with27bTested = (chip: string, memGb: number, cli: string): LocalModelCatalog =>
|
||
|
|
stockWith((catalog) => {
|
||
|
|
const row = catalog.models.find((m) => m.id === 'qwen3.8:27b') as LocalModelEntry;
|
||
|
|
row.tested = [testedOn(chip, memGb, cli)];
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is off for every model of the stock catalogue, so "Recommended" is never said today', () => {
|
||
|
|
for (const hw of SWEEP) {
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
for (const candidate of [result.pick, ...result.alternatives]) {
|
||
|
|
if (candidate) expect(candidate.tested).toBe(false);
|
||
|
|
}
|
||
|
|
expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i);
|
||
|
|
if (result.verdict !== 'none') {
|
||
|
|
expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('turns on "Recommended for this Mac" when the pick was tested on exactly such a Mac', () => {
|
||
|
|
const result = recommend(mac(48, 'Apple M4 Max'), with27bTested('Apple M4 Max', 48, 'opencode'), ALL_INSTALLED);
|
||
|
|
expect(result.verdict).toBe('ok');
|
||
|
|
expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: true });
|
||
|
|
expect(result.headline).toEqual({ text: 'Recommended for this Mac' });
|
||
|
|
// The "not yet tested" note goes away with it.
|
||
|
|
expect(result.notes).toEqual([]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('stays off when the chip, the memory size or the harness differs', () => {
|
||
|
|
const hw = mac(48, 'Apple M4 Max');
|
||
|
|
const cases: LocalModelCatalog[] = [
|
||
|
|
with27bTested('Apple M4 Pro', 48, 'opencode'),
|
||
|
|
with27bTested('Apple M4', 48, 'opencode'),
|
||
|
|
with27bTested('Apple M4 Max', 64, 'opencode'),
|
||
|
|
with27bTested('Apple M4 Max', 48, 'claude'),
|
||
|
|
];
|
||
|
|
for (const catalog of cases) {
|
||
|
|
const result = recommend(hw, catalog, ALL_INSTALLED);
|
||
|
|
expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: false });
|
||
|
|
expect(result.headline).toEqual({ text: 'Best fit for this Mac' });
|
||
|
|
expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('follows the harness: tested with the second harness counts when that one is the pick', () => {
|
||
|
|
const catalog = with27bTested('Apple M4 Max', 48, 'claude');
|
||
|
|
const result = recommend(mac(48, 'Apple M4 Max'), catalog, [fact('claude')]);
|
||
|
|
expect(result.pick).toMatchObject({ cli: 'claude', tested: true });
|
||
|
|
expect(result.headline.text).toBe('Recommended for this Mac');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('does not change the ranking: a tested model further down stays an alternative', () => {
|
||
|
|
// On the M4 Pro the pick is the 35B-A3B. A tested 27B is flagged in the list, no more.
|
||
|
|
const result = recommend(mac(48, 'Apple M4 Pro'), with27bTested('Apple M4 Pro', 48, 'opencode'), ALL_INSTALLED);
|
||
|
|
expect(result.pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', tested: false });
|
||
|
|
expect(result.headline.text).toBe('Best fit for this Mac');
|
||
|
|
expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]);
|
||
|
|
expect(result.alternatives.find((c) => c.modelId === 'qwen3.8:27b')?.tested).toBe(true);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('never says "Recommended" for a limited answer, tested or not', () => {
|
||
|
|
const catalog = stockWith((stock) => {
|
||
|
|
const row = stock.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry;
|
||
|
|
row.tested = [testedOn('Apple M4', 24, 'opencode')];
|
||
|
|
});
|
||
|
|
const result = recommend(mac(24, 'Apple M4'), catalog, ALL_INSTALLED);
|
||
|
|
expect(result.verdict).toBe('limited');
|
||
|
|
expect(result.pick?.tested).toBe(true);
|
||
|
|
expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited);
|
||
|
|
expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i);
|
||
|
|
// It WAS tested, so the "not yet tested" note would be false.
|
||
|
|
expect(result.notes).toEqual([]);
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('the shape of the answer', () => {
|
||
|
|
const labels = new Set<string>(Object.values(LOCAL_MODEL_LABELS));
|
||
|
|
|
||
|
|
it('takes every sentence and label from the fixed list', () => {
|
||
|
|
for (const hw of SWEEP) {
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
expect(labels.has(result.headline.text)).toBe(true);
|
||
|
|
for (const note of result.notes) expect(labels.has(note.text)).toBe(true);
|
||
|
|
for (const candidate of [result.pick, ...result.alternatives]) {
|
||
|
|
if (!candidate) continue;
|
||
|
|
expect(labels.has(candidate.fitLabel)).toBe(true);
|
||
|
|
if (candidate.speedLabel !== undefined) expect(labels.has(candidate.speedLabel)).toBe(true);
|
||
|
|
expect(candidate.speedLabel === undefined).toBe(candidate.speed === 'ok');
|
||
|
|
expect(candidate.tokensPerSecond === undefined).toBe(candidate.speed === 'unknown');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('fills every placeholder of a headline and nothing else', () => {
|
||
|
|
const answers = [
|
||
|
|
recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED),
|
||
|
|
recommend(mac(48, 'Apple M4', { macosMajor: 13, macosVersion: '13.0' }), STOCK, ALL_INSTALLED),
|
||
|
|
recommend(mac(48, 'Apple M4', { freeDiskBytes: GB, freeDiskGb: '1.0' }), STOCK, ALL_INSTALLED),
|
||
|
|
recommend(mac(48, 'Apple M4'), STOCK, []),
|
||
|
|
recommend(mac(48, 'Apple M4'), STOCK, ALL_INSTALLED),
|
||
|
|
];
|
||
|
|
expect(answers.map((answer) => answer.reason ?? answer.verdict)).toEqual([
|
||
|
|
'memory',
|
||
|
|
'macos',
|
||
|
|
'disk',
|
||
|
|
'no-harness',
|
||
|
|
'ok',
|
||
|
|
]);
|
||
|
|
for (const answer of answers) {
|
||
|
|
const slots = [...answer.headline.text.matchAll(/\{(\w+)\}/g)].map((match) => match[1]).sort();
|
||
|
|
expect(Object.keys(answer.headline.vars ?? {}).sort()).toEqual(slots);
|
||
|
|
// `{name}` belongs to the browser's translator.
|
||
|
|
expect(slots).not.toContain('name');
|
||
|
|
for (const value of Object.values(answer.headline.vars ?? {})) expect(typeof value).toBe('string');
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('prints every size in one unit: decimal GB from the one formatter, never GiB', () => {
|
||
|
|
for (const hw of SWEEP) {
|
||
|
|
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||
|
|
expect(result.budgetGb).toBe(formatGb(result.budgetBytes));
|
||
|
|
for (const candidate of [result.pick, ...result.alternatives]) {
|
||
|
|
if (!candidate) continue;
|
||
|
|
expect(candidate.needGb).toBe(formatGb(candidate.needBytes));
|
||
|
|
expect(candidate.downloadGb).toBe(formatGb(candidate.downloadBytes));
|
||
|
|
expect(candidate.budgetShare).toBeCloseTo(candidate.needBytes / result.budgetBytes, 3);
|
||
|
|
}
|
||
|
|
expect(JSON.stringify(result)).not.toMatch(/GiB|MiB/);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
it('adds up for a person: download, plus what the context and the runtime take, is the need', () => {
|
||
|
|
const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED);
|
||
|
|
const pick = result.pick as Candidate;
|
||
|
|
// 17.7 GB of weights + 8 GiB of KV cache + 1 GiB + 2 GiB of prompt cache, all in decimal GB.
|
||
|
|
expect(pick.downloadGb).toBe('17.7');
|
||
|
|
expect(pick.needGb).toBe('29.6');
|
||
|
|
expect(formatGb(pick.needBytes - pick.downloadBytes)).toBe('11.8');
|
||
|
|
expect(result.budgetGb).toBe('37.6');
|
||
|
|
expect(pick.needBytes).toBeLessThan(result.budgetBytes);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is plain JSON and has exactly the fields of its verdict', () => {
|
||
|
|
const ok = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED);
|
||
|
|
expect(JSON.parse(JSON.stringify(ok))).toEqual(ok);
|
||
|
|
expect(Object.keys(ok).sort()).toEqual(
|
||
|
|
['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'pick', 'verdict'].sort()
|
||
|
|
);
|
||
|
|
const none = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED);
|
||
|
|
expect(JSON.parse(JSON.stringify(none))).toEqual(none);
|
||
|
|
expect(Object.keys(none).sort()).toEqual(
|
||
|
|
['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'reason', 'verdict'].sort()
|
||
|
|
);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('is pure: frozen inputs are enough, and the same inputs give the same answer', () => {
|
||
|
|
const hw = deepFreeze(mac(48, 'Apple M4 Pro'));
|
||
|
|
const catalog = deepFreeze(structuredClone(STOCK_LOCAL_MODEL_CATALOG));
|
||
|
|
const facts = deepFreeze(ALL_INSTALLED.map((entry) => ({ ...entry })));
|
||
|
|
const first = recommend(hw, catalog, facts);
|
||
|
|
const second = recommend(hw, catalog, facts);
|
||
|
|
expect(second).toEqual(first);
|
||
|
|
expect(first.verdict).toBe('ok');
|
||
|
|
});
|
||
|
|
});
|