diff --git a/src/config/local-model-catalog.ts b/src/config/local-model-catalog.ts index a1e578b9..63990a54 100644 --- a/src/config/local-model-catalog.ts +++ b/src/config/local-model-catalog.ts @@ -8,6 +8,11 @@ * is downloaded is a compiled constant of its own (src/local-model/runtime.ts). * - A model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`), so it can * only ever name a model of the default registry and namespace. + * - A model row names the `runner` a Mac uses for its tag, and its memory figures follow that + * runner's rule (MEMORY below). A row's figures are its OWN: measured on that model, or + * computed from that model's architecture. test/local-model-catalog.test.ts recomputes every + * computed figure from a table of architectures and refuses a row that is not in it, so a + * new row needs its model's config.json read, not a figure borrowed from a neighbour. * - A harness is a ROW (`{ cli, minContext, rank, gated?, slowFirstTurn? }`). The `cli` is a CLI * registry id as data: no code may compare it with a literal. Whether that CLI exists and can * take a custom model is checked against the registry at call time, in @@ -19,48 +24,112 @@ * throws on a bad row. test/local-model-catalog.test.ts pins that the stock constant parses, * so a bad row fails the gate and never reaches the route. * - * Where each figure of the stock rows comes from, and what was checked (2026-11-06): + * Where each figure of the stock rows comes from, and what was checked (2026-11-08). [V] read + * at the source that day, [M] measured on a Mac, [C] computed from [V] inputs: * * | Figure | Source | Checked | * |----------------------|-----------------------------------------------------------|--------------------------------------| - * | `id` | the registry's tag | yes, by the facts review | - * | `bytes` | `qwen3.6:35b-a3b`: the total of a real pull on a Mac | that row: measured. It is an MLX | - * | | (23,105,798,956). Every other row: the sum of the layer | build on a Mac, 2% above its | - * | | sizes of the tag's registry manifest | manifest. The others: not on a Mac | - * | `heldBytesPerToken`, | measured for `qwen3.6:35b-a3b` on an Apple M5 Pro with | that row: measured. The others carry | - * | `overheadBytes`, | 48 GB: 36,000 to 38,000 bytes held per token of context, | the SAME three figures, not their | - * | `peakExtraBytes` | 2.5 to 3 GB beside the weights, and about 5 GB more for a | own: nobody ran them. `figures` says | - * | | moment while a long prompt is processed | which row is which | - * | `license` | the model card (all Apache-2.0) | read by the research, not re-checked | - * | `paramsB` | 27.8 from the model card; the others are the count in the | no. Only the 12 billion line reads | - * | | model's name | it, and no row is near that line | - * | `contexts`, `rank` | a judgement of the design | no | - * | `speedPer100Gbps` | one published run on an M1 Max (400 GB/s): generation | the run was read. It stands for less | - * | | only, short prompt. The two q8_0 rows are SCALED by file | than a session: no long prompt, no | - * | | size, not measured | thinking tokens | + * | `id` | the registry's tag. The GGUF tags in lowercase: the | [V] every row: the registry answers | + * | | registry spells them `-q4_K_M`, and an id here is | the lowercase id with the same | + * | | lowercase (`LOCAL_MODEL_ID_PATTERN`) | manifest. NOT checked: that the | + * | | | runtime lists the pulled model under | + * | | | that name (Codeman compares exactly) | + * | `runner` | the build the runtime 0.40.15 pulls on Apple silicon: the | [V] every row | + * | | MLX child of a manifest list, else the one manifest | | + * | | (`runnerPreferences()`: mlx, then llamacpp) | | + * | `bytes` | every layer plus the config of that build: the `total` | [V] every row. [M] for | + * | | a pull reports | `qwen3.6:35b-a3b`, byte for byte | + * | `heldBytesPerToken`, | `qwen3.6:35b-a3b`: [M] on an Apple M5 Pro with 48 GB. | the others: [C], never run on a Mac. | + * | `overheadBytes`, | Every other row: [C] from its model's config.json for its | `figures` says which row is which. | + * | `peakExtraBytes` | runner, see MEMORY below. No row borrows another's | test/local-model-catalog.test.ts | + * | | figures: two rows share figures only when they share the | recomputes every one | + * | | architecture and the runner | | + * | `license`, `paramsB` | the model card (`paramsB` is the TOTAL, not the active | [V] | + * | | parameters) | | + * | `rank` | a judgement on the agentic coding results of the vendors' | no: vendor tables disagree with each | + * | | cards (SWE-bench Verified, SWE-bench Pro, Terminal-Bench) | other by several points | + * | `contexts` | up to the model's own window, and at least the 65536 a | no | + * | | harness needs | | + * | `speedPer100Gbps` | one published run per model family on an M1 Max | the run was read. It stands for less | + * | | (400 GB/s): generation only, short prompt. A row that is | than a session: no long prompt, no | + * | | not that run is SCALED from it, and says how | thinking tokens. An MTP build is | + * | | | faster than the run without it | * | `tested` | our own run of the model with a harness on a real Mac | one entry, with its measured speed | * | `minMacosMajor` | the runtime's documentation (macOS 14) | yes, by the facts review | - * | harness `minContext` | 65536: the runtime's documentation for both harnesses. | 65536 yes. 32768 no, which is why | - * | | 32768: recalled | that row is gated | + * | harness `minContext` | 65536: the runtime's documentation for OpenCode ("64k or | 65536 yes. 32768 no, which is why | + * | | higher"), and OpenCode v1.19.5's own fixed prompt (about | that row is gated | + * | | 8,000 to 10,000 tokens) and compaction rule: at 32768 it | | + * | | leaves 14,000 to 16,000 tokens, less than one default | | + * | | 2,000-line file read | | * | `chips` | the bandwidth column of | yes, read at the source for this | * | | https://github.com/ggml-org/llama.cpp/discussions/4167 | file | * - * The memory figures decide a lot and rest on ONE measured model, so for every other row they - * can be wrong in either direction: - * - For the 9B rows they may be too HIGH (the 5 GB for a moment was seen with a prompt of - * 89,000 tokens on the 35B model). That errs on the careful side: a small Mac is told - * "none" before it is told to download something that may not fit. - * - For the two dense `qwen3.8:27b` rows the per-token figure may be too LOW: computed from - * its config alone that model holds 64 KiB per token, more than the 37,000 bytes measured - * on the 35B (whose own computed figure was 20 KiB). Their need at a large context is - * therefore the least certain figure of the catalogue. - * A row gets its own figures when somebody measures it. Until then the check after the - * download (the runtime must report the whole model on the GPU) is what catches a row that - * was judged too kindly. + * MEMORY. The need is `bytes + heldBytesPerToken x context + overheadBytes + peakExtraBytes` + * (src/local-model/recommend.ts). The two runners spend memory differently, so a row's three + * figures are filled by the rule of its runner: + * + * - `llamacpp` (llama-server on Metal; runtime v0.40.15 source, llm/llama_server.go). It + * allocates the K/V cache for the WHOLE context when the model loads, in the cache type the + * runtime sets (`RUNTIME_KV_CACHE_TYPE` in src/local-model/runtime.ts, q8_0: 34 bytes per + * 32 values), and does not grow after that. + * - held = attention layers x K/V heads x head dim x 2 (K and V) x 34/32. The one + * multi-token-prediction layer of a Qwen3.5-family GGUF that carries it (the `-mtp` + * tags, and `qwen3.8:27b-q8_0`) is counted as one more attention layer: whether + * llama.cpp gives it a cache was not checked, so this errs on the careful side. + * - overhead = the fixed caches (the f32 recurrent state of the linear-attention layers, + * or the sliding-window cache of `n_swa + 512` cells), + 1.0 GB for the runner's own + * buffers (calibrated on a third-party table of a 27B model on llama.cpp: 8.4 GB at + * 4,096 tokens with 7.15 GB of weights), + the 1024 MiB host prompt cache that + * `LLAMA_ARG_CACHE_RAM` allows (`RUNTIME_HOST_PROMPT_CACHE_MIB`). + * - peak = 0: its compute buffers are allocated at load and are inside the 1.0 GB. + * The one outside check of this rule: a third party measured 65,600 bytes per token of + * growth at f16 for a model of the Qwen3.8-27B architecture, where its config gives 65,536. + * - `mlx` (runtime v0.40.15 source, mlxrunner/). It grows the K/V cache in bf16 with the + * tokens used, keeps the recurrent state in f32 and a snapshot of it every 8,192 tokens, + * and keeps up to 8 GiB of paged-out prefix snapshots (prefix_cache.go `maxPagedOutBytes`, + * not configurable). Third-party OpenCode sessions show that growth (runtime issues 17924 + * and 18131), and the runtime books an MLX model at its weights alone (issue 19417). + * - `qwen3.6:35b-a3b`: MEASURED. 36,000 to 38,000 bytes held per token after prompts of + * 89,000 and 167,000 tokens, 2.5 to 3 GB beside the weights, about 5 GB more for a + * moment while a long prompt was processed. NOT covered by that measurement: a session + * of many requests, in which the prefix snapshots may add up to 8 GiB. Its figures stay + * the measured ones (with 8 GiB on top it would not be offered on the Mac it ran on), + * and a 30-request session on it is the first thing to measure next. + * - every other `mlx` row: held = the K/V cache in bf16 + one recurrent snapshot per + * 8,192 tokens, computed from the config, x 37,000 / 28,340: the ratio of what the + * measured row really held to the same computation for it (same runner, same + * architecture family). Overhead = the measured row's 3 GB + the whole 8 GiB prefix + * budget. Peak = the measured row's 5 GB, NOT scaled: what makes it up was not + * measured. + * + * After the download the install checks that the runtime holds the whole model on the GPU. + * For a `llamacpp` row that is the runtime's own reckoning of the load, K/V cache included. + * For an `mlx` row it proves only the weights (issue 19417): the figures above are all that + * stands between such a row and swap. + * + * Models set aside on purpose (a weekly review should look at them again): + * - `bonsai-2:27b`, `bonsai-2:9b` (ternary builds of Qwen3.8-27B and Qwen3.5-9B; the 27B + * would make a 24 GB Mac `ok`): the vendor reports no SWE-bench and no agent loop, and + * whether 2-bit weights keep tool calls reliable was never measured. + * - `ornith-1.5:9b`, `ornith-1.5:35b`, `ornith-1.5:397b` (GGUF only): strong cards, but their + * manifests name no renderer and no tool parser, so whether tool calls work through the + * runtime is unknown (runtime pull request 17965 is open). `ornith-1.5:122b` has both. + * - `gpt-oss:20b`: weak in agentic coding (SWE-bench Verified 52.4, Terminal-Bench 2.1 15.2 + * in NVIDIA's table). + * - `flint:40b`: runtime issue 19191 breaks integer tool arguments after the first call. + * - `lfm2.5:24b-a2b`: its licence allows commercial use only below USD 10M of revenue. + * - `laguna-xs-2.1`, `laguna-s-2.1`: the runtime's own page names an open problem on macOS. + * - `qwen3.8-flash-next:125b`: the Qwen Community License, not an open licence. + * - The plain tags `qwen3.5:9b`, `mimo-v2.6:9b` and `qwen3.8:27b`: a Mac pulls their MLX + * builds, whose 8 GiB of prefix snapshots do not fit the Macs a 9B or a 27B is for. Their + * GGUF builds are listed instead. + * - `qwen3.5:122b-a10b`: replaced by `ornith-1.5:122b`, a fine-tune of it (SWE-bench + * Verified 84.5 against 72.0) whose MLX build carries a renderer and a tool parser. * * `tested` has one entry: `qwen3.6:35b-a3b` with OpenCode on an Apple M5 Pro with 48 GB, * 2026-11-06, 67.7 tokens per second. Only a Mac of exactly that chip and memory size is told - * "Recommended" and shown that speed in place of the estimate. + * "Recommended" and shown that speed in place of the estimate. Every other row was never run + * on a Mac by us, and its row says `estimated` and "not run on a Mac like this yet". */ import { z } from 'zod'; @@ -114,6 +183,7 @@ const LocalModelEntrySchema = z label: z.string().min(1).max(80), license: z.string().min(1).max(40), paramsB: z.number().positive(), + runner: z.enum(['mlx', 'llamacpp']), bytes: positiveInt, heldBytesPerToken: positiveInt, overheadBytes: z.number().int().nonnegative(), @@ -218,8 +288,8 @@ export const LocalModelCatalogSchema: z.ZodType = z /** * The memory figures measured for `qwen3.6:35b-a3b` on an Apple M5 Pro with 48 GB (2026-11-06, - * the MLX runner), in decimal bytes. That row carries them as `measured`; every other row - * carries the same three as `estimated`, see the file header. + * the MLX runner), in decimal bytes. Only that row carries them; see the file header (MEMORY) + * for what they do not cover. */ const MEASURED_MEMORY = { /** 36,000 to 38,000 bytes stayed held per token of context after a prompt of 89,000 and one of 167,000 tokens. */ @@ -230,117 +300,195 @@ const MEASURED_MEMORY = { peakExtraBytes: 5_000_000_000, } as const; +/** The llama.cpp runner's own buffers, compute buffers included (file header, MEMORY). */ +const LLAMACPP_RUNNER_BYTES = 1_000_000_000; +/** The host prompt cache llama-server may keep: `LLAMA_ARG_CACHE_RAM`, 1024 MiB (`RUNTIME_HOST_PROMPT_CACHE_MIB`). */ +const LLAMACPP_PROMPT_CACHE_BYTES = 1024 * 1024 * 1024; +/** The prefix snapshots the MLX runner may keep: `maxPagedOutBytes`, 8 GiB. */ +const MLX_PREFIX_CACHE_BYTES = 8 * 1024 * 1024 * 1024; + /** * The catalogue that ships with Codeman. Never read directly outside this file and its test: * callers use `getLocalModelCatalog()`, which validates it first. */ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { schema: 1, - updated: '2026-11-06', + updated: '2026-11-08', // The oldest macOS the model runtime supports. minMacosMajor: 14, - // Ordered by rank. The higher rank wins among models that fit equally well and are equally - // fast. `flint:40b` is left out on purpose: its plain tag is the slow build on a Mac and it - // has an open bug with tool arguments. + // Ordered by rank. The higher rank wins among models that fit and whose speed is graded + // alike (recommend.ts ranks the speed grade first). See the file header for the models + // that are left out and why. models: [ { - id: 'qwen3.5:9b', + // Small Macs (16 to 32 GB, `limited`): the base model of the row below. + id: 'qwen3.5:9b-q4_k_m', label: 'Qwen3.5 9B', license: 'Apache-2.0', paramsB: 9, - bytes: 6_550_825_373, - ...MEASURED_MEMORY, + runner: 'llamacpp', + bytes: 6_550_825_550, + // 8 attention layers x 4 K/V heads x 256 x 2 x 34/32. + heldBytesPerToken: 17_408, + // 24 linear-attention layers hold 52,690,944 bytes of f32 state. + overheadBytes: 52_690_944 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + peakExtraBytes: 0, figures: 'estimated', - contexts: [65536, 32768], + contexts: [131072, 65536], rank: 1, - // 37.1 tokens per second measured on 400 GB/s. + // Measured: 37.1 tokens per second on 400 GB/s (this Q4_K_M build, llama.cpp). speedPer100Gbps: 9.3, tested: [], }, { - id: 'qwen3.5:9b-q8_0', - label: 'Qwen3.5 9B (Q8)', - license: 'Apache-2.0', + // Small Macs (16 to 32 GB, `limited`). MiMo-V2.6-Distill-Qwen-9B, a fine-tune of + // Qwen3.5-9B for code and agents: SWE-bench Verified 61.1 and SWE-bench Pro 44.6, + // against 60.0 and 32.0 for its base (Xiaomi's card). Requires exactly the pinned + // runtime, 0.40.15. + id: 'mimo-v2.6:9b-q4_k_m', + label: 'MiMo V2.6 9B', + license: 'MIT', paramsB: 9, - bytes: 10_449_217_949, - ...MEASURED_MEMORY, - figures: 'estimated', - contexts: [65536, 32768], - rank: 2, - // Not measured: the 9.3 of the row above scaled by file size. - speedPer100Gbps: 5.8, - tested: [], - }, - { - id: 'gpt-oss:20b', - label: 'GPT-OSS 20B', - license: 'Apache-2.0', - paramsB: 20, - bytes: 13_793_441_427, - ...MEASURED_MEMORY, + runner: 'llamacpp', + bytes: 6_550_811_033, + // The architecture and the quantization of the row above, so its figures. + heldBytesPerToken: 17_408, + overheadBytes: 52_690_944 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + peakExtraBytes: 0, figures: 'estimated', contexts: [131072, 65536], - rank: 3, - // No speed figure: its speed is unknown on every chip. + rank: 2, + // Not measured: the 9.3 of the row above, the same architecture and file size. + speedPer100Gbps: 9.3, tested: [], }, { + // Cohere's model for agentic software engineering, 3 billion parameters active: + // SWE-bench Verified 67.6 (vendor). Its card asks OpenCode for interleaved reasoning, + // which Codeman's OpenCode settings do not set. + id: 'north-mini-code-1.0:q4_k_m', + label: 'North Mini Code 1.0', + license: 'Apache-2.0', + paramsB: 30, + runner: 'llamacpp', + bytes: 18_593_967_008, + // 13 full-attention layers x 4 K/V heads x 128 x 2 x 34/32. + heldBytesPerToken: 14_144, + // 36 sliding-window layers: (4,096 + 512) cells x 4 x 128 x 2 x 34/32 = 180,486,144. + overheadBytes: 180_486_144 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + peakExtraBytes: 0, + figures: 'estimated', + contexts: [262144, 131072, 65536], + rank: 3, + // Not measured: the 13.6 measured for Qwen3.6 35B-A3B on llama.cpp, which has the same + // 3 billion active parameters. + speedPer100Gbps: 13.6, + tested: [], + }, + { + // Macs of 36 GB on a slower chip: the model of the tested pair, as a GGUF build whose + // memory does not grow after it loaded. With its multi-token-prediction layer. + id: 'qwen3.6:35b-a3b-mtp-q4_k_m', + label: 'Qwen3.6 35B-A3B (GGUF)', + license: 'Apache-2.0', + paramsB: 35, + runner: 'llamacpp', + bytes: 22_621_314_381, + // (10 attention layers + 1 prediction layer) x 2 K/V heads x 256 x 2 x 34/32. + heldBytesPerToken: 11_968, + // 30 linear-attention layers hold 65,863,680 bytes of f32 state. + overheadBytes: 65_863_680 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + peakExtraBytes: 0, + figures: 'estimated', + contexts: [262144, 131072, 65536], + rank: 4, + // Measured: 54.5 tokens per second on 400 GB/s (llama.cpp, without prediction). + speedPer100Gbps: 13.6, + tested: [], + }, + { + // 48 GB: the tested pair. SWE-bench Verified 73.4, SWE-bench Pro 49.5 (Qwen's card). id: 'qwen3.6:35b-a3b', label: 'Qwen3.6 35B-A3B', license: 'Apache-2.0', paramsB: 35, - // What a Mac really downloads: an MLX build, 2% above the 22,621,314,161 of the manifest. + // The plain tag is a manifest list; a Mac takes its MLX child. + runner: 'mlx', + // What a Mac really downloads: the MLX build, 2% above the GGUF build of the row above. bytes: 23_105_798_956, ...MEASURED_MEMORY, figures: 'measured', contexts: [131072, 65536], - rank: 4, - // 54.5 tokens per second measured on 400 GB/s (3 billion parameters are active per token). - // The MLX build ran faster than this figure predicts on the one Mac we tested (67.7 - // against 41.8), so on other chips the estimate is on the low side. - speedPer100Gbps: 13.6, + rank: 5, + // Measured: 67.8 tokens per second on 400 GB/s with MLX. The one Mac we tested ran it at + // 67.7 on 307 GB/s, so on other chips the estimate is on the low side. + speedPer100Gbps: 16.9, // Run by us with OpenCode on the MacBook Pro (Mac17,13): a four-step task with tool calls. tested: [{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 }], }, { - id: 'qwen3.8:27b', - label: 'Qwen3.8 27B', - license: 'Apache-2.0', - paramsB: 27.8, - bytes: 17_741_871_939, - ...MEASURED_MEMORY, - figures: 'estimated', - contexts: [131072, 65536, 32768], - rank: 5, - // 11.8 tokens per second measured on 400 GB/s. - speedPer100Gbps: 2.95, - tested: [], - }, - { + // Large Macs that want 8-bit weights: the row below in Q8_0, half as fast. id: 'qwen3.8:27b-q8_0', label: 'Qwen3.8 27B (Q8)', license: 'Apache-2.0', paramsB: 27.8, - bytes: 29_978_241_837, - ...MEASURED_MEMORY, + runner: 'llamacpp', + bytes: 29_978_242_050, + // The architecture of the row below, its prediction layer counted (file header). + heldBytesPerToken: 36_992, + overheadBytes: 156_893_184 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + peakExtraBytes: 0, figures: 'estimated', - contexts: [262144, 131072], + contexts: [262144, 131072, 65536], rank: 6, - // Not measured: the 2.95 of the row above scaled by file size. + // Not measured: the 2.95 of the row below scaled by file size. speedPer100Gbps: 1.7, tested: [], }, { - id: 'qwen3.5:122b-a10b', - label: 'Qwen3.5 122B-A10B', + // 36 to 96 GB on a fast chip: the strongest model of this size. Terminal-Bench 2.1 73.0, + // SWE-bench Pro 61.7 (Qwen's card). Dense, so it is slow on a Pro chip. + id: 'qwen3.8:27b-mtp-q4_k_m', + label: 'Qwen3.8 27B', license: 'Apache-2.0', - paramsB: 122, - bytes: 81_370_035_877, - ...MEASURED_MEMORY, + paramsB: 27.8, + runner: 'llamacpp', + bytes: 17_741_872_154, + // (16 attention layers + 1 prediction layer) x 4 K/V heads x 256 x 2 x 34/32. + heldBytesPerToken: 36_992, + // 48 linear-attention layers hold 156,893,184 bytes of f32 state. + overheadBytes: 156_893_184 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + peakExtraBytes: 0, figures: 'estimated', - contexts: [131072], + contexts: [262144, 131072, 65536], rank: 7, - // No speed figure: its speed is unknown on every chip. + // Measured: 11.8 tokens per second on 400 GB/s (llama.cpp, Q4_K_M, without prediction). + // This build predicts several tokens at a time, so it is faster than this figure says. + speedPer100Gbps: 2.95, + tested: [], + }, + { + // 128 GB and more. A fine-tune of Qwen3.5-122B-A10B, 10 billion parameters active: + // Terminal-Bench 2.1 81.2, SWE-bench Verified 84.5, SWE-bench Pro 76.9 (Ornith's card). + id: 'ornith-1.5:122b', + label: 'Ornith 1.5 122B-A10B', + license: 'MIT', + paramsB: 122, + // The plain tag is a manifest list; a Mac takes its MLX child (with a renderer and a + // tool parser, which the GGUF builds of this family lack). + runner: 'mlx', + bytes: 73_388_069_552, + // 12 attention layers x 2 K/V heads x 256 x 2 x 2 bytes = 24,576, + a 153,649,152-byte + // recurrent snapshot per 8,192 tokens = 18,756: 43,332, x 37,000 / 28,340. + heldBytesPerToken: 56_574, + overheadBytes: MEASURED_MEMORY.overheadBytes + MLX_PREFIX_CACHE_BYTES, + peakExtraBytes: MEASURED_MEMORY.peakExtraBytes, + figures: 'estimated', + contexts: [262144, 131072], + rank: 8, + // Not measured: the 16.9 of Qwen3.6 35B-A3B on MLX x 3/10, its active parameters + // against these, rounded down. + speedPer100Gbps: 5, tested: [], }, ], diff --git a/src/local-model/recommend.ts b/src/local-model/recommend.ts index 1d3c84cf..67ff2d88 100644 --- a/src/local-model/recommend.ts +++ b/src/local-model/recommend.ts @@ -12,8 +12,9 @@ * - `needBytes()`: the PEAK, `bytes + heldBytesPerToken * ctx + overheadBytes + * peakExtraBytes` of the catalogue row: the weights, what stays held per token of context, * what the runner holds beside both, and what it takes for a moment while a long prompt is - * processed. Measured for one model on one Mac (see the catalogue header); every other row - * carries the same figures and says `estimated`. + * processed. Measured for one model on one Mac; every other row's figures are computed from + * its own model's architecture for its runner and say `estimated` (see the catalogue + * header, MEMORY). * - Fit: need over budget up to 0.85 is good, up to 0.98 is tight, above is out. The grade * decides whether a model is offered at all and what its row says. It does NOT sort: the * need is the worst moment at the full context, so a tight fit is a model that runs, and a diff --git a/src/types/local-model.ts b/src/types/local-model.ts index 16725a6e..1cc71e6c 100644 --- a/src/types/local-model.ts +++ b/src/types/local-model.ts @@ -121,6 +121,9 @@ export interface TestedOn { } /** One model of the catalogue. */ +/** The runner of the model runtime that runs a model on a Mac, by the registry's name for it. See `LocalModelEntry.runner`. */ +export type LocalModelRunner = 'mlx' | 'llamacpp'; + export interface LocalModelEntry { /** `name:tag` exactly as the model registry lists it. Lowercase, one colon, no slash (`LOCAL_MODEL_ID_PATTERN`). */ id: string; @@ -130,35 +133,55 @@ export interface LocalModelEntry { license: string; /** Parameter count in billions. When no model that fits has 12 or more, the verdict is `limited`, never `ok`. */ paramsB: number; + /** + * The runner the runtime uses for this tag ON A MAC, as the registry names it: `mlx` (a + * safetensors build, or a manifest list with an MLX child, which a Mac always takes) or + * `llamacpp` (a GGUF build, run by llama-server on Metal). The two spend memory in different + * ways, so a row's memory figures are only meaningful together with its runner: llama.cpp + * allocates the whole K/V cache for the context when the model loads (in the runtime's + * `OLLAMA_KV_CACHE_TYPE`), MLX grows it with the tokens actually used and keeps snapshots of + * it besides. + */ + runner: LocalModelRunner; /** * Size of the weights in bytes: both the download and what they take in memory. * - * It is the size of the build the runtime really downloads ON A MAC. That is not always - * the sum of the registry manifest: `qwen3.6:35b-a3b` is an MLX build of 23,105,798,956 bytes - * on Apple silicon, 2% more than the manifest's 22,621,314,161. A row whose Mac size nobody - * measured keeps the manifest sum and says so in `figures`. + * It is the size of the build the runtime really downloads ON A MAC: the build of `runner`, + * every layer of its manifest plus its config, which is the `total` a real pull reports. + * That is not always the first manifest the registry answers with: `qwen3.6:35b-a3b` is an + * MLX build of 23,105,798,956 bytes on Apple silicon, 2% more than its GGUF build of + * 22,621,314,381. */ bytes: number; /** - * Bytes that stay HELD in memory per token of context once the token was processed (measured - * on the MLX runner: 36,000 to 38,000, so about 37,000). A figure computed from a model's - * config alone came out too low by almost half, so a row that was not measured carries the - * measured row's figure and says so in `figures`. + * Bytes that stay HELD in memory per token of context. For a `llamacpp` row: the K/V cache + * of its attention layers in the runtime's cache type, which llama.cpp allocates for the + * whole context at load. For an `mlx` row: the K/V cache in bf16 plus the snapshots of the + * recurrent state the runner keeps every 8,192 tokens, which it grows with use. Measured + * on one row (36,000 to 38,000 on the MLX runner, so about 37,000); computed from the + * model's config for the others, see `figures` and the catalogue header. */ heldBytesPerToken: number; - /** What the runner holds beside the weights and the context (measured: about 3 GB). */ + /** + * What the runner holds beside the weights and the context, whatever the context: its own + * buffers, the live recurrent or sliding-window caches, and the caches it keeps of earlier + * prompts (for `llamacpp` the 1024 MiB host prompt cache `LLAMA_ARG_CACHE_RAM` allows, for + * an estimated `mlx` row the 8 GiB of prefix snapshots the runner may keep). Measured on + * one row (about 3 GB); see the catalogue header for the others. + */ overheadBytes: number; /** * The TRANSIENT extra the runner takes while it processes a long prompt, on top of - * what stays held (measured: about 5 GB). The fit is judged on the peak: + * what stays held (measured on the MLX runner: about 5 GB; 0 for `llamacpp`, which + * allocates its buffers at load). The fit is judged on the peak: * `bytes + heldBytesPerToken * context + overheadBytes + peakExtraBytes`. For * `qwen3.6:35b-a3b` at 131072 tokens that is about 36 GB, which is tight on a 48 GB Mac. */ peakExtraBytes: number; /** - * Where `bytes` and the three memory figures above come from: `measured` on a Mac by - * us, or `estimated` (the manifest size, and the per-token, overhead and peak figures of the - * measured row applied to this one). + * Where the three memory figures above come from: `measured` on a Mac by us, or + * `estimated` (computed from the model's architecture for its runner, see the catalogue + * header). `bytes` is the registry's size of the build a Mac pulls in both cases. */ figures: 'measured' | 'estimated'; /** Context lengths (tokens) the model may be run at, largest first. */ diff --git a/test/fixtures/local-model-rules-catalog.ts b/test/fixtures/local-model-rules-catalog.ts new file mode 100644 index 00000000..8fcdc4d5 --- /dev/null +++ b/test/fixtures/local-model-rules-catalog.ts @@ -0,0 +1,182 @@ +/** + * @fileoverview A frozen model catalogue for the RULE tests of src/local-model/recommend.ts: + * the stock catalogue exactly as it shipped on 2026-11-06 (seven rows that all carry the + * figures measured on one model), with the `runner` field the schema gained since. + * + * Why a frozen copy and not the stock catalogue: the stock catalogue is meant to change every + * week or so (new models, new measurements). The tests of a RULE (the ranking, the disk + * filter, the tags, the `limited` line, the harness fallback) need realistic rows whose + * answers do not move with it, or every catalogue update would rewrite a hundred rule tests + * and hide the few answers that really changed. What the STOCK catalogue answers for each Mac + * is pinned separately, in test/local-model-recommend.test.ts ("every memory size") and the + * two real-Mac blocks, and that is the table a catalogue change must update on purpose. + * + * Its figures are NOT a statement about any model: the rows borrow the measured row's figures, + * which is exactly why the stock catalogue replaced them. Validated by the real schema. + */ + +import type { LocalModelCatalog } from '../../src/types.js'; + +const MEASURED_MEMORY = { + /** 36,000 to 38,000 bytes stayed held per token of context after a prompt of 89,000 and one of 167,000 tokens. */ + heldBytesPerToken: 37_000, + /** The runner held 2.5 to 3 GB beside the weights. */ + overheadBytes: 3_000_000_000, + /** While a long prompt was processed the peak stood about 5 GB above what stayed held. */ + peakExtraBytes: 5_000_000_000, +} as const; + +/** + * The catalogue as it shipped on 2026-11-06, with the `runner` field added. Frozen: a fixture, + * not the stock catalogue. + */ +export const RULES_LOCAL_MODEL_CATALOG: LocalModelCatalog = { + schema: 1, + updated: '2026-11-06', + // The oldest macOS the model runtime supports. + minMacosMajor: 14, + // Ordered by rank. The higher rank wins among models that fit equally well and are equally + // fast. `flint:40b` is left out on purpose: its plain tag is the slow build on a Mac and it + // has an open bug with tool arguments. + models: [ + { + id: 'qwen3.5:9b', + label: 'Qwen3.5 9B', + license: 'Apache-2.0', + paramsB: 9, + runner: 'mlx', + bytes: 6_550_825_373, + ...MEASURED_MEMORY, + figures: 'estimated', + contexts: [65536, 32768], + rank: 1, + // 37.1 tokens per second measured on 400 GB/s. + speedPer100Gbps: 9.3, + tested: [], + }, + { + id: 'qwen3.5:9b-q8_0', + label: 'Qwen3.5 9B (Q8)', + license: 'Apache-2.0', + paramsB: 9, + runner: 'llamacpp', + bytes: 10_449_217_949, + ...MEASURED_MEMORY, + figures: 'estimated', + contexts: [65536, 32768], + rank: 2, + // Not measured: the 9.3 of the row above scaled by file size. + speedPer100Gbps: 5.8, + tested: [], + }, + { + id: 'gpt-oss:20b', + label: 'GPT-OSS 20B', + license: 'Apache-2.0', + paramsB: 20, + runner: 'llamacpp', + bytes: 13_793_441_427, + ...MEASURED_MEMORY, + figures: 'estimated', + contexts: [131072, 65536], + rank: 3, + // No speed figure: its speed is unknown on every chip. + tested: [], + }, + { + id: 'qwen3.6:35b-a3b', + label: 'Qwen3.6 35B-A3B', + license: 'Apache-2.0', + paramsB: 35, + runner: 'mlx', + // What a Mac really downloads: an MLX build, 2% above the 22,621,314,161 of the manifest. + bytes: 23_105_798_956, + ...MEASURED_MEMORY, + figures: 'measured', + contexts: [131072, 65536], + rank: 4, + // 54.5 tokens per second measured on 400 GB/s (3 billion parameters are active per token). + // The MLX build ran faster than this figure predicts on the one Mac we tested (67.7 + // against 41.8), so on other chips the estimate is on the low side. + speedPer100Gbps: 13.6, + // Run by us with OpenCode on the MacBook Pro (Mac17,13): a four-step task with tool calls. + tested: [{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 }], + }, + { + id: 'qwen3.8:27b', + label: 'Qwen3.8 27B', + license: 'Apache-2.0', + paramsB: 27.8, + runner: 'mlx', + bytes: 17_741_871_939, + ...MEASURED_MEMORY, + figures: 'estimated', + contexts: [131072, 65536, 32768], + rank: 5, + // 11.8 tokens per second measured on 400 GB/s. + speedPer100Gbps: 2.95, + tested: [], + }, + { + id: 'qwen3.8:27b-q8_0', + label: 'Qwen3.8 27B (Q8)', + license: 'Apache-2.0', + paramsB: 27.8, + runner: 'llamacpp', + bytes: 29_978_241_837, + ...MEASURED_MEMORY, + figures: 'estimated', + contexts: [262144, 131072], + rank: 6, + // Not measured: the 2.95 of the row above scaled by file size. + speedPer100Gbps: 1.7, + tested: [], + }, + { + id: 'qwen3.5:122b-a10b', + label: 'Qwen3.5 122B-A10B', + license: 'Apache-2.0', + paramsB: 122, + runner: 'llamacpp', + bytes: 81_370_035_877, + ...MEASURED_MEMORY, + figures: 'estimated', + contexts: [131072], + rank: 7, + // No speed figure: its speed is unknown on every chip. + tested: [], + }, + ], + // The higher rank wins when a model can run with more than one harness. + harnesses: [ + { cli: 'opencode', minContext: 65536, rank: 3 }, + // Sends a fixed prompt of about 36,000 tokens, so the first answer takes minutes. + { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true }, + // Gated: neither its 32768 minimum nor its handling of a `name:tag` model id is verified. + { cli: 'pi', minContext: 32768, rank: 1, gated: true }, + ], + // Memory bandwidth in GB/s, most specific first. Only chips that shipped and are in the source + // table are listed. Where one name covers two bandwidths (M3 Max 300 or 400, M4 Max 410 or 546, + // M5 Max 460 or 614) the LOWER one stands, so a speed computed from it is a floor. + chips: [ + { match: 'M5 Ultra', gbps: 1228 }, + { match: 'M3 Ultra', gbps: 800 }, + { match: 'M2 Ultra', gbps: 800 }, + { match: 'M1 Ultra', gbps: 800 }, + { match: 'M5 Max', gbps: 460 }, + { match: 'M4 Max', gbps: 410 }, + { match: 'M3 Max', gbps: 300 }, + { match: 'M2 Max', gbps: 400 }, + { match: 'M1 Max', gbps: 400 }, + { match: 'M5 Pro', gbps: 307 }, + { match: 'M4 Pro', gbps: 273 }, + { match: 'M3 Pro', gbps: 150 }, + { match: 'M2 Pro', gbps: 200 }, + { match: 'M1 Pro', gbps: 200 }, + { match: 'M5', gbps: 154 }, + { match: 'M4', gbps: 120 }, + { match: 'M3', gbps: 100 }, + { match: 'M2', gbps: 100 }, + { match: 'M1', gbps: 68 }, + ], +}; diff --git a/test/local-model-catalog.test.ts b/test/local-model-catalog.test.ts index 4c185e0b..f53815a4 100644 --- a/test/local-model-catalog.test.ts +++ b/test/local-model-catalog.test.ts @@ -12,9 +12,12 @@ * hides a more specific one). * 3. The chip lookup never guesses: a chip that no row names has no bandwidth, including one * whose name merely contains a known name ('Apple M10', 'Apple M6 Pro'). - * 4. Which figures were MEASURED on a Mac and which are carried over from the measured row: - * one row says `measured` (its size is what a Mac really downloads, and it alone has a - * tested entry), every other row says `estimated`. + * 4. Which figures were MEASURED on a Mac and which were computed: one row says `measured` + * (it alone has a tested entry), every other row says `estimated`, and every estimated + * row's three memory figures are recomputed here from its OWN model's architecture (a + * table read from each model's config.json) for its runner and the runtime's settings. Two + * rows may share figures only when they share the architecture and the runner, so a row + * can never again carry another model's measurement without saying so. * * Port: N/A (pure data and functions). */ @@ -28,6 +31,7 @@ import { LocalModelCatalogSchema, STOCK_LOCAL_MODEL_CATALOG, } from '../src/config/local-model-catalog.js'; +import { RUNTIME_HOST_PROMPT_CACHE_MIB, RUNTIME_KV_CACHE_TYPE, runtimeEnv } from '../src/local-model/runtime.js'; import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js'; /** A deep copy of the stock catalogue that a test may break. */ @@ -48,7 +52,101 @@ function withFirstModel(change: Partial = { + 'qwen3.5:9b-q4_k_m': { architecture: 'Qwen3.5-9B', predictionLayer: false }, + 'mimo-v2.6:9b-q4_k_m': { architecture: 'Qwen3.5-9B', predictionLayer: false }, + 'north-mini-code-1.0:q4_k_m': { architecture: 'North-Mini-Code-1.0', predictionLayer: false }, + 'qwen3.6:35b-a3b-mtp-q4_k_m': { architecture: 'Qwen3.6-35B-A3B', predictionLayer: true }, + 'qwen3.6:35b-a3b': { architecture: 'Qwen3.6-35B-A3B', predictionLayer: false }, + 'qwen3.8:27b-q8_0': { architecture: 'Qwen3.8-27B', predictionLayer: true }, + 'qwen3.8:27b-mtp-q4_k_m': { architecture: 'Qwen3.8-27B', predictionLayer: true }, + 'ornith-1.5:122b': { architecture: 'Qwen3.5-122B-A10B', predictionLayer: false }, +}; + +/** Bytes per cached value in each K/V cache type of llama.cpp: q8_0 keeps 32 values in 34 bytes. */ +const KV_BYTES_PER_VALUE: Record = { f16: 2, q8_0: 34 / 32 }; +/** The head size of a Gated DeltaNet layer's keys and values, and its convolution kernel, in every model above. */ +const LINEAR_HEAD_DIM = 128; +const LINEAR_CONV_KERNEL = 4; + +/** The recurrent state of the linear-attention layers in bytes: the convolution tail and the delta state. */ +function recurrentStateBytes(arch: Architecture, convBytes: number): number { + if (!('linear' in arch)) return 0; + const { layers, vHeads, kHeads } = arch.linear; + const convDim = vHeads * LINEAR_HEAD_DIM + 2 * kHeads * LINEAR_HEAD_DIM; + const conv = (LINEAR_CONV_KERNEL - 1) * convDim * convBytes; + const delta = vHeads * LINEAR_HEAD_DIM * LINEAR_HEAD_DIM * 4; + return layers * (conv + delta); +} + +/** What a `llamacpp` row's three figures must be, from its architecture and the runtime's settings. */ +function llamaCppFigures(arch: Architecture, predictionLayer: boolean): [number, number, number] { + const perValue = KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE]; + const layers = arch.attentionLayers + (predictionLayer ? 1 : 0); + const held = layers * arch.kvHeads * arch.headDim * 2 * perValue; + // llama.cpp keeps the recurrent state in f32 (both parts), the sliding window in the cache type. + const sliding = + 'sliding' in arch + ? arch.sliding.layers * (arch.sliding.window + 512) * arch.kvHeads * arch.headDim * 2 * perValue + : 0; + const fixed = recurrentStateBytes(arch, 4) + sliding; + return [held, fixed + 1_000_000_000 + RUNTIME_HOST_PROMPT_CACHE_MIB * MIB, 0]; +} + +/** The per-token figure of the MLX runner computed from an architecture: K/V in bf16, a recurrent snapshot per 8,192 tokens. */ +function mlxComputedPerToken(arch: Architecture): number { + return arch.attentionLayers * arch.kvHeads * arch.headDim * 2 * 2 + recurrentStateBytes(arch, 2) / 8192; +} + +/** The figures measured for `qwen3.6:35b-a3b` on the MacBook Pro. */ +const MEASURED: [number, number, number] = [37_000, 3_000_000_000, 5_000_000_000]; + +/** What an estimated `mlx` row's three figures must be: computed, scaled by what the measured row showed. */ +function mlxFigures(arch: Architecture): [number, number, number] { + const ratio = MEASURED[0] / mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B']); + return [Math.ceil(mlxComputedPerToken(arch) * ratio), MEASURED[1] + 8 * GIB, MEASURED[2]]; +} + +/** A row's three memory figures. */ +function figuresOf(model: LocalModelEntry): [number, number, number] { + return [model.heldBytesPerToken, model.overheadBytes, model.peakExtraBytes]; +} + describe('the stock catalogue', () => { + const models = STOCK_LOCAL_MODEL_CATALOG.models; + it('passes its own schema', () => { expect(issuesOf(STOCK_LOCAL_MODEL_CATALOG)).toEqual([]); }); @@ -60,94 +158,114 @@ describe('the stock catalogue', () => { expect(getLocalModelCatalog()).toBe(first); }); - it('lists the seven models of the design, without flint:40b', () => { - expect(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => m.id)).toEqual([ - 'qwen3.5:9b', - 'qwen3.5:9b-q8_0', - 'gpt-oss:20b', + it('lists eight models, by rank, and none of those set aside in the header', () => { + expect(models.map((m) => m.id)).toEqual([ + 'qwen3.5:9b-q4_k_m', + 'mimo-v2.6:9b-q4_k_m', + 'north-mini-code-1.0:q4_k_m', + 'qwen3.6:35b-a3b-mtp-q4_k_m', 'qwen3.6:35b-a3b', - 'qwen3.8:27b', 'qwen3.8:27b-q8_0', - 'qwen3.5:122b-a10b', + 'qwen3.8:27b-mtp-q4_k_m', + 'ornith-1.5:122b', ]); - expect(STOCK_LOCAL_MODEL_CATALOG.models.some((m) => m.id.startsWith('flint'))).toBe(false); + expect(models.map((m) => m.rank)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); + const setAside = + /^(bonsai-2|gpt-oss|flint|lfm2\.5|laguna|qwen3\.8-flash-next|qwen3\.5:122b)|^ornith-1\.5:(9b|35b|397b)/; + expect(models.filter((m) => setAside.test(m.id)).map((m) => m.id)).toEqual([]); + // The plain 9B and 27B tags are MLX builds on a Mac: their GGUF builds are listed instead. + for (const plain of ['qwen3.5:9b', 'mimo-v2.6:9b', 'qwen3.8:27b']) { + expect(models.some((m) => m.id === plain)).toBe(false); + } }); - it('carries the size a Mac really downloads where that was measured, else the registry manifest', () => { - expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.bytes]))).toEqual({ - // The sum of the layer sizes of each tag's manifest, as fetched on 2026-11-06. - 'qwen3.5:9b': 6_550_825_373, - 'qwen3.5:9b-q8_0': 10_449_217_949, - 'gpt-oss:20b': 13_793_441_427, - // The total of the real pull on a Mac (an MLX build). Its manifest sums to 22,621,314,161. + it('names the runner a Mac uses for each tag', () => { + expect(Object.fromEntries(models.map((m) => [m.id, m.runner]))).toEqual({ + 'qwen3.5:9b-q4_k_m': 'llamacpp', + 'mimo-v2.6:9b-q4_k_m': 'llamacpp', + 'north-mini-code-1.0:q4_k_m': 'llamacpp', + 'qwen3.6:35b-a3b-mtp-q4_k_m': 'llamacpp', + // Manifest lists: a Mac takes the MLX child. + 'qwen3.6:35b-a3b': 'mlx', + 'qwen3.8:27b-q8_0': 'llamacpp', + 'qwen3.8:27b-mtp-q4_k_m': 'llamacpp', + 'ornith-1.5:122b': 'mlx', + }); + }); + + it('carries the size of the build a Mac pulls: every layer plus the config', () => { + expect(Object.fromEntries(models.map((m) => [m.id, m.bytes]))).toEqual({ + // The registry's manifests as fetched on 2026-11-08, in lowercase. + 'qwen3.5:9b-q4_k_m': 6_550_825_550, + 'mimo-v2.6:9b-q4_k_m': 6_550_811_033, + 'north-mini-code-1.0:q4_k_m': 18_593_967_008, + 'qwen3.6:35b-a3b-mtp-q4_k_m': 22_621_314_381, + // The MLX child of the list, and the total of a real pull on a Mac, byte for byte. 'qwen3.6:35b-a3b': 23_105_798_956, - 'qwen3.8:27b': 17_741_871_939, - 'qwen3.8:27b-q8_0': 29_978_241_837, - 'qwen3.5:122b-a10b': 81_370_035_877, + 'qwen3.8:27b-q8_0': 29_978_242_050, + 'qwen3.8:27b-mtp-q4_k_m': 17_741_872_154, + // The MLX child of the list (its GGUF child is 76,756,174,997). + 'ornith-1.5:122b': 73_388_069_552, }); }); it('says which row was measured on a Mac: one, and every other row is an estimate', () => { - expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.figures]))).toEqual({ - 'qwen3.5:9b': 'estimated', - 'qwen3.5:9b-q8_0': 'estimated', - 'gpt-oss:20b': 'estimated', - 'qwen3.6:35b-a3b': 'measured', - 'qwen3.8:27b': 'estimated', - 'qwen3.8:27b-q8_0': 'estimated', - 'qwen3.5:122b-a10b': 'estimated', - }); - }); - - it('gives every row the memory figures measured on the one row: 37 KB per token, 3 GB, 5 GB', () => { - for (const model of STOCK_LOCAL_MODEL_CATALOG.models) { - expect( - [model.heldBytesPerToken, model.overheadBytes, model.peakExtraBytes], - `${model.id}: an unmeasured row carries the measured row's figures, not invented ones` - ).toEqual([37_000, 3_000_000_000, 5_000_000_000]); - } - // No row keeps the figure that was computed from the model's config. - expect(JSON.stringify(STOCK_LOCAL_MODEL_CATALOG)).not.toContain('kvBytesPerToken'); + expect(models.filter((m) => m.figures === 'measured').map((m) => m.id)).toEqual(['qwen3.6:35b-a3b']); + expect(figuresOf(models.find((m) => m.id === 'qwen3.6:35b-a3b') as LocalModelEntry)).toEqual(MEASURED); }); it('has one tested entry: the 35B-A3B with OpenCode on the MacBook Pro, with its measured speed', () => { - expect( - Object.fromEntries( - STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.tested.length > 0).map((m) => [m.id, m.tested]) - ) - ).toEqual({ + expect(Object.fromEntries(models.filter((m) => m.tested.length > 0).map((m) => [m.id, m.tested]))).toEqual({ 'qwen3.6:35b-a3b': [ { chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 }, ], }); // A row may claim a test only when its figures were measured. - for (const model of STOCK_LOCAL_MODEL_CATALOG.models) { + for (const model of models) { if (model.tested.length > 0) expect(model.figures, model.id).toBe('measured'); } }); it('gives every model a parameter count, an id of the default registry and a license', () => { - for (const model of STOCK_LOCAL_MODEL_CATALOG.models) { - expect(model.paramsB, model.id).toBeGreaterThan(0); - expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN); - expect(model.license).toBe('Apache-2.0'); - } + expect(Object.fromEntries(models.map((m) => [m.id, [m.paramsB, m.license]]))).toEqual({ + 'qwen3.5:9b-q4_k_m': [9, 'Apache-2.0'], + 'mimo-v2.6:9b-q4_k_m': [9, 'MIT'], + 'north-mini-code-1.0:q4_k_m': [30, 'Apache-2.0'], + 'qwen3.6:35b-a3b-mtp-q4_k_m': [35, 'Apache-2.0'], + 'qwen3.6:35b-a3b': [35, 'Apache-2.0'], + 'qwen3.8:27b-q8_0': [27.8, 'Apache-2.0'], + 'qwen3.8:27b-mtp-q4_k_m': [27.8, 'Apache-2.0'], + 'ornith-1.5:122b': [122, 'MIT'], + }); + for (const model of models) expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN); // The two 9B rows are the ones under the 12 billion line of the `limited` verdict. - expect(STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual([ - 'qwen3.5:9b', - 'qwen3.5:9b-q8_0', - ]); + expect(models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual(['qwen3.5:9b-q4_k_m', 'mimo-v2.6:9b-q4_k_m']); + }); + + it('offers no model below the 65536 tokens a harness needs, nor beyond its own window', () => { + for (const model of models) { + expect(Math.min(...model.contexts), model.id).toBeGreaterThanOrEqual(65536); + expect(model.contexts[0], model.id).toBeLessThanOrEqual(262144); + } }); it('has a speed figure only where one was measured or scaled from a measurement', () => { - expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({ - 'qwen3.5:9b': 9.3, - 'qwen3.5:9b-q8_0': 5.8, - 'gpt-oss:20b': null, - 'qwen3.6:35b-a3b': 13.6, - 'qwen3.8:27b': 2.95, + expect(Object.fromEntries(models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({ + // 37.1 tokens per second on 400 GB/s, and the same for the fine-tune of the same size. + 'qwen3.5:9b-q4_k_m': 9.3, + 'mimo-v2.6:9b-q4_k_m': 9.3, + // The 35B-A3B's llama.cpp figure: both have 3 billion active parameters. + 'north-mini-code-1.0:q4_k_m': 13.6, + // 54.5 on 400 GB/s with llama.cpp. + 'qwen3.6:35b-a3b-mtp-q4_k_m': 13.6, + // 67.8 on 400 GB/s with MLX. + 'qwen3.6:35b-a3b': 16.9, + // 2.95 scaled by file size. 'qwen3.8:27b-q8_0': 1.7, - 'qwen3.5:122b-a10b': null, + // 11.8 on 400 GB/s with llama.cpp, without prediction. + 'qwen3.8:27b-mtp-q4_k_m': 2.95, + // 16.9 x 3/10 active parameters, rounded down. + 'ornith-1.5:122b': 5, }); }); @@ -174,6 +292,82 @@ describe('the stock catalogue', () => { }); }); +describe('the memory figures of the stock rows: each row has its own', () => { + const models = STOCK_LOCAL_MODEL_CATALOG.models; + + it('knows the architecture of every row, and every row of that table is in the catalogue', () => { + expect(models.map((m) => m.id).sort()).toEqual(Object.keys(ROW_ARCHITECTURE).sort()); + }); + + it('computes the llamacpp rows for the K/V cache type and the prompt cache the runtime really sets', () => { + // The figures below assume these; runtimeEnv() is what the runtime gets. + const env = runtimeEnv({ source: {}, home: '/h', modelsDir: '/m', port: 11435, contextLength: 65536 }); + expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE); + expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB)); + expect(KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE]).toBeDefined(); + }); + + it("recomputes every estimated row's figures from its own model's architecture and runner", () => { + for (const model of models.filter((m) => m.figures === 'estimated')) { + const { architecture, predictionLayer } = ROW_ARCHITECTURE[model.id]; + const arch = ARCHITECTURES[architecture]; + const expected = model.runner === 'llamacpp' ? llamaCppFigures(arch, predictionLayer) : mlxFigures(arch); + expect(figuresOf(model), `${model.id} (${architecture}, ${model.runner})`).toEqual(expected); + } + }); + + it('states the numbers the header and the row comments name', () => { + expect(Object.fromEntries(models.map((m) => [m.id, figuresOf(m)]))).toEqual({ + 'qwen3.5:9b-q4_k_m': [17_408, 2_126_432_768, 0], + 'mimo-v2.6:9b-q4_k_m': [17_408, 2_126_432_768, 0], + 'north-mini-code-1.0:q4_k_m': [14_144, 2_254_227_968, 0], + 'qwen3.6:35b-a3b-mtp-q4_k_m': [11_968, 2_139_605_504, 0], + 'qwen3.6:35b-a3b': [37_000, 3_000_000_000, 5_000_000_000], + 'qwen3.8:27b-q8_0': [36_992, 2_230_635_008, 0], + 'qwen3.8:27b-mtp-q4_k_m': [36_992, 2_230_635_008, 0], + 'ornith-1.5:122b': [56_574, 11_589_934_592, 5_000_000_000], + }); + // The MLX calibration: 37,000 held against 28,340 computed for the measured row. + expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B'])).toBe(28_340); + expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.5-122B-A10B'])).toBe(43_332); + }); + + it('lets two rows share figures only when they share the architecture, the prediction layer and the runner', () => { + const byFigures = new Map(); + for (const model of models) { + const key = figuresOf(model).join('/'); + byFigures.set(key, [...(byFigures.get(key) ?? []), model]); + } + const shared = [...byFigures.values()].filter((group) => group.length > 1); + // The two 9B rows (a fine-tune and its base) and the two 27B rows (two quantizations of one model). + expect(shared.map((group) => group.map((m) => m.id))).toEqual([ + ['qwen3.5:9b-q4_k_m', 'mimo-v2.6:9b-q4_k_m'], + ['qwen3.8:27b-q8_0', 'qwen3.8:27b-mtp-q4_k_m'], + ]); + for (const group of shared) { + const identity = (m: LocalModelEntry): string => JSON.stringify([ROW_ARCHITECTURE[m.id], m.runner, m.figures]); + expect(new Set(group.map(identity)).size, group.map((m) => m.id).join(', ')).toBe(1); + } + }); + + it("gives no estimated row the measured row's per-token or overhead figure", () => { + for (const model of models.filter((m) => m.figures === 'estimated')) { + expect(model.heldBytesPerToken, model.id).not.toBe(MEASURED[0]); + expect(model.overheadBytes, model.id).not.toBe(MEASURED[1]); + expect(figuresOf(model), model.id).not.toEqual(MEASURED); + } + }); + + it('gives a llamacpp row no peak (its buffers are allocated at load) and an estimated mlx row the whole prefix budget', () => { + for (const model of models) { + if (model.runner === 'llamacpp') expect(model.peakExtraBytes, model.id).toBe(0); + if (model.runner === 'mlx' && model.figures === 'estimated') { + expect(model.overheadBytes - MEASURED[1], model.id).toBe(8 * GIB); + } + } + }); +}); + describe('the model id rule', () => { it('accepts name:tag in lowercase', () => { for (const id of ['qwen3.5:9b', 'gpt-oss:20b', 'qwen3.8:27b-q8_0', 'a:b', 'x_y.z-1:2.3_4']) { @@ -240,6 +434,19 @@ describe('the schema, field by field', () => { expect(issuesOf(withFirstModel({ peakExtraBytes: 1.5 }))).not.toEqual([]); }); + it('requires the runner on every model row, and knows two', () => { + const row: Record = { ...copy().models[0] }; + delete row.runner; + const issues = issuesOf({ ...copy(), models: [row] }); + expect(issues).toHaveLength(1); + expect(issues[0]).toContain('models.0.runner'); + expect(issuesOf(withFirstModel({ runner: 'mlx' }))).toEqual([]); + expect(issuesOf(withFirstModel({ runner: 'llamacpp' }))).toEqual([]); + for (const runner of ['ggml', 'MLX', 'llama.cpp', '']) { + expect(issuesOf(withFirstModel({ runner })), runner).not.toEqual([]); + } + }); + it('requires paramsB and tested on every model row', () => { for (const field of ['paramsB', 'tested'] as const) { const row: Record = { ...copy().models[0] }; diff --git a/test/local-model-fits-budget.test.ts b/test/local-model-fits-budget.test.ts index f66707d4..c02c053a 100644 --- a/test/local-model-fits-budget.test.ts +++ b/test/local-model-fits-budget.test.ts @@ -51,7 +51,7 @@ describe('modelFitsBudget', () => { const catalog = getLocalModelCatalog(); it('agrees with the recommendation on every model and context of the catalogue', () => { - for (const gb of [8, 16, 24, 32, 48, 64, 128]) { + for (const gb of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192, 256, 512]) { const hw = mac(gb); const offered = recommend(hw, catalog, harnessFacts()); const rows = [offered.pick, ...offered.alternatives].filter((row) => row !== undefined); @@ -85,7 +85,7 @@ describe('modelFitsBudget', () => { }); it('rules out a model that is too large, and one the catalogue does not have', () => { - expect(modelFitsBudget(mac(48), catalog, 'qwen3.5:122b-a10b', 131072)).toBe(false); + expect(modelFitsBudget(mac(48), catalog, 'ornith-1.5:122b', 131072)).toBe(false); expect(modelFitsBudget(mac(48), catalog, 'no-such:1b', 65536)).toBe(false); expect(modelFitsBudget(mac(48), catalog, 'qwen3.6:35b-a3b', 131072)).toBe(true); }); @@ -96,14 +96,16 @@ describe('modelFitsBudget', () => { expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 40_157_944_217 })).toBe(true); // A runtime that reports less than the estimate rules out what the estimate allowed. expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 33 * 1024 ** 3 })).toBe(false); - // And one that reports more admits what the estimate ruled out: the 27B on a 36 GB Mac. - expect(modelFitsBudget(mac(36), catalog, 'qwen3.8:27b', 65536)).toBe(false); - expect(modelFitsBudget(mac(36), catalog, 'qwen3.8:27b', 65536, { measuredTotalBytes: 27 * 1024 ** 3 })).toBe(true); + // And one that reports more admits what the estimate ruled out: on a 36 GB Mac the GGUF + // 35B-A3B at its full window needs 27.9 GB, 1.0003 of the 27.9 GB estimate and 0.962 of 27 GiB. + const moeGguf = 'qwen3.6:35b-a3b-mtp-q4_k_m'; + expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144)).toBe(false); + expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144, { measuredTotalBytes: 27 * 1024 ** 3 })).toBe(true); // The reserve still bounds it: 100 GiB "measured" on a 48 GB Mac is 40 GiB (42.9 GB) of - // budget, so the Q8 27B with its 42.8 GB stays out (0.997) although 100 GiB would hold it. + // budget, so the 122B with its 97.4 GB stays out although 100 GiB would hold it. const huge = { measuredTotalBytes: 100 * 1024 ** 3 }; - expect(modelFitsBudget(mac(48), catalog, 'qwen3.8:27b-q8_0', 131072, huge)).toBe(false); - expect(modelFitsBudget(mac(128), catalog, 'qwen3.8:27b-q8_0', 131072, huge)).toBe(true); + expect(modelFitsBudget(mac(48), catalog, 'ornith-1.5:122b', 131072, huge)).toBe(false); + expect(modelFitsBudget(mac(128), catalog, 'ornith-1.5:122b', 131072, huge)).toBe(true); }); it('agrees with the recommendation under a measured budget too', () => { @@ -130,7 +132,7 @@ describe('modelFitsBudget', () => { it('is false for a context that is no token count', () => { for (const context of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { - expect(modelFitsBudget(mac(128), catalog, 'qwen3.5:9b', context), String(context)).toBe(false); + expect(modelFitsBudget(mac(128), catalog, 'qwen3.5:9b-q4_k_m', context), String(context)).toBe(false); } }); }); diff --git a/test/local-model-recommend.test.ts b/test/local-model-recommend.test.ts index 6ad0eb40..c98a765c 100644 --- a/test/local-model-recommend.test.ts +++ b/test/local-model-recommend.test.ts @@ -4,11 +4,12 @@ * * Laid out like the module: * - * 1. The budget (`deviceBudget`) for every memory size Apple sells from 8 to 128 GB, estimated + * 1. The budget (`deviceBudget`) for every memory size Apple sells from 8 to 512 GB, estimated * and measured, and the need of a model at a context (`needBytes`: the peak). * 2. The two REAL Macs of the project, from their captured `sysctl` output through the real * parser: the MacBook Pro (M5 Pro, 48 GB) and the Mac mini (M4, 16 GB). Each test states the * expected answer and why. The MacBook's answer is checked against what was MEASURED on it. + * Then what the stock catalogue answers for every memory size from 8 to 512 GB. * 3. Every reason for verdict `none`, one fixture per reason, and their order. * 4. The `limited` verdict (a statement about the Mac, not about the pick), the notes that say * what the headline cannot (a small pick, a model left out for disk space), the ranking @@ -17,19 +18,22 @@ * on the word "Recommended", what each row of the chooser carries (its tag, its harnesses, * the whole download of an install), and the shape of the answer. * - * The stock catalogue is used wherever the answer for a real Mac is at stake. Small synthetic - * catalogues (validated by the real schema) isolate one rule of the ranking at a time. + * Three catalogues: + * - `STOCK`, the catalogue that ships, wherever the answer for a real Mac or a memory size is + * at stake (part 2, the sweeps that must hold for every Mac, and the end-to-end checks). A + * change of the stock catalogue changes these on purpose, and only these. + * - `RULES`, the catalogue as it shipped on 2026-11-06, frozen in + * test/fixtures/local-model-rules-catalog.ts: realistic rows for the rule tests of parts 3 + * and 4, whose answers must not move every time a model is added (see that file). + * - Small synthetic catalogues (validated by the real schema) that isolate one rule of the + * ranking at a time. * * Port: N/A (pure functions; the two hardware fixtures go through `readHardware` with every * piece of IO injected). */ import { beforeEach, describe, expect, it } from 'vitest'; -import { - getLocalModelCatalog, - LocalModelCatalogSchema, - STOCK_LOCAL_MODEL_CATALOG, -} from '../src/config/local-model-catalog.js'; +import { getLocalModelCatalog, LocalModelCatalogSchema } from '../src/config/local-model-catalog.js'; import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js'; import { readHardware, resetHardwareCache } from '../src/local-model/hardware.js'; import { @@ -56,6 +60,7 @@ import { type Recommendation, type TestedOn, } from '../src/types.js'; +import { RULES_LOCAL_MODEL_CATALOG } from './fixtures/local-model-rules-catalog.js'; // ---------- Fixtures ---------- @@ -159,9 +164,13 @@ function fact(cli: string, overrides: Partial = {}): HarnessFact { }; } +/** The catalogue that ships: for the answers a real Mac gets. */ const STOCK = getLocalModelCatalog(); -/** Every stock harness installed, the gated one included. */ +/** The frozen catalogue of 2026-11-06: for the rule tests (see the file header). */ +const RULES = LocalModelCatalogSchema.parse(RULES_LOCAL_MODEL_CATALOG); + +/** Every harness installed, the gated one included. Both catalogues list the same three. */ const ALL_INSTALLED: HarnessFact[] = STOCK.harnesses.map((row) => fact(row.cli)); /** A stock model row by id. */ @@ -171,16 +180,39 @@ function stockModel(id: string): LocalModelEntry { /** The stock catalogue with a change applied, validated by the real schema. */ function stockWith(change: (catalog: LocalModelCatalog) => void): LocalModelCatalog { - const catalog = structuredClone(STOCK_LOCAL_MODEL_CATALOG); + const catalog = structuredClone(STOCK); change(catalog); return LocalModelCatalogSchema.parse(catalog); } -/** The stock catalogue as it was before anybody tested a row: no `tested` entry anywhere. */ -const UNTESTED = stockWith((catalog) => { +/** A model row of the rules catalogue by id. */ +function rulesModel(id: string): LocalModelEntry { + return RULES.models.find((m) => m.id === id) as LocalModelEntry; +} + +/** The rules catalogue with a change applied, validated by the real schema. */ +function rulesWith(change: (catalog: LocalModelCatalog) => void): LocalModelCatalog { + const catalog = structuredClone(RULES_LOCAL_MODEL_CATALOG); + change(catalog); + return LocalModelCatalogSchema.parse(catalog); +} + +/** The stock catalogue as if nobody had tested a row: no `tested` entry anywhere. */ +const STOCK_UNTESTED = stockWith((catalog) => { for (const row of catalog.models) row.tested = []; }); +/** The rules catalogue as it was before anybody tested a row: no `tested` entry anywhere. */ +const UNTESTED = rulesWith((catalog) => { + for (const row of catalog.models) row.tested = []; +}); + +/** Both real catalogues, for the sweeps whose rule must hold whatever the rows are. */ +const BOTH: [name: string, catalog: LocalModelCatalog][] = [ + ['stock', STOCK], + ['rules', RULES], +]; + /** * A model row for a synthetic catalogue: 20 billion parameters, 10 GiB, one small context, one * byte held per token and nothing else on top, so its need is its weights plus its context. @@ -191,6 +223,7 @@ function model(id: string, overrides: Partial = {}): LocalModel label: id, license: 'Apache-2.0', paramsB: 20, + runner: 'llamacpp', bytes: 10 * GIB, heldBytesPerToken: 1, overheadBytes: 0, @@ -256,11 +289,24 @@ function deepFreeze(value: T): T { return value; } +/** Every memory size Apple sells or sold an Apple silicon Mac with. */ +const MEMORY_SIZES = [8, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192, 256, 512]; + /** The Macs the sweeps run over: every memory size, with a slow, a fast and an unknown chip. */ -const SWEEP: HardwareFacts[] = [8, 16, 18, 24, 32, 36, 48, 64, 96, 128].flatMap((soldGb) => +const SWEEP: HardwareFacts[] = MEMORY_SIZES.flatMap((soldGb) => ['Apple M4', 'Apple M4 Max', 'Apple M5 Ultra', 'Apple M6'].map((chip) => mac(soldGb, chip)) ); +/** Every Mac of the sweep with each of the two real catalogues. */ +const SWEEP_BOTH: [HardwareFacts, LocalModelCatalog][] = BOTH.flatMap(([, catalog]) => + SWEEP.map((hw): [HardwareFacts, LocalModelCatalog] => [hw, catalog]) +); + +/** The same plus the tested MacBook, with each of the two real catalogues. */ +const SWEEP_BOTH_AND_MACBOOK: [HardwareFacts, LocalModelCatalog][] = BOTH.flatMap(([, catalog]) => + [...SWEEP, mac(48, 'Apple M5 Pro')].map((hw): [HardwareFacts, LocalModelCatalog] => [hw, catalog]) +); + beforeEach(() => { resetHardwareCache(); }); @@ -281,6 +327,9 @@ describe('deviceBudget, estimated', () => { [64, 48, 47], [96, 72, 71], [128, 96, 95], + [192, 144, 143], + [256, 192, 191], + [512, 384, 383], ]; it.each(TIERS)('gives a %i GB Mac a limit of %f GiB and a budget of %f GiB', (soldGb, limitGib, budgetGib) => { @@ -309,6 +358,9 @@ describe('deviceBudget, estimated', () => { [64, '47.0'], [96, '71.0'], [128, '95.0'], + [192, '143.0'], + [256, '191.0'], + [512, '383.0'], ]); }); @@ -320,7 +372,7 @@ describe('deviceBudget, estimated', () => { }; expect([8, 16, 18].map(reserveOf)).toEqual([6, 6, 6]); expect([24, 32, 36, 48].map(reserveOf)).toEqual([8, 8, 8, 8]); - expect([64, 96, 128].map(reserveOf)).toEqual([12, 12, 12]); + expect([64, 96, 128, 192, 256, 512].map(reserveOf)).toEqual([12, 12, 12, 12, 12, 12]); }); it('uses the wired limit when it is above 0', () => { @@ -412,11 +464,12 @@ describe('needBytes', () => { }); it('grows with the context by what one token holds', () => { - const nineB = stockModel('qwen3.5:9b'); - expect(needBytes(nineB, 65536)).toBe(16_975_657_373); - expect(needBytes(nineB, 32768)).toBe(15_763_241_373); - // Half the context, half of what the context holds: 37,000 x 32,768. - expect(needBytes(nineB, 65536) - needBytes(nineB, 32768)).toBe(1_212_416_000); + // A llama.cpp row: 6,550,811,033 + 17,408 x 65,536 + 2,126,432,768, and no peak. + const nineB = stockModel('mimo-v2.6:9b-q4_k_m'); + expect(needBytes(nineB, 65536)).toBe(9_818_094_489); + expect(needBytes(nineB, 131072)).toBe(10_958_945_177); + // Twice the context, twice what the context holds: 17,408 x 65,536 more. + expect(needBytes(nineB, 131072) - needBytes(nineB, 65536)).toBe(1_140_850_688); }); it('reads every figure from the row it is given', () => { @@ -432,7 +485,20 @@ describe('needBytes', () => { }); it('states the need of every stock row at its largest context', () => { - expect(Object.fromEntries(STOCK.models.map((m) => [m.id, formatGb(needBytes(m, m.contexts[0]))]))).toEqual({ + expect(Object.fromEntries(STOCK.models.map((m) => [m.id, needBytes(m, m.contexts[0])]))).toEqual({ + 'qwen3.5:9b-q4_k_m': 10_958_959_694, + 'mimo-v2.6:9b-q4_k_m': 10_958_945_177, + 'north-mini-code-1.0:q4_k_m': 24_555_959_712, + 'qwen3.6:35b-a3b-mtp-q4_k_m': 27_898_259_277, + 'qwen3.6:35b-a3b': 35_955_462_956, + 'qwen3.8:27b-q8_0': 41_906_107_906, + 'qwen3.8:27b-mtp-q4_k_m': 29_669_738_010, + 'ornith-1.5:122b': 104_808_538_800, + }); + }); + + it('states the need of every rules row at its largest context', () => { + expect(Object.fromEntries(RULES.models.map((m) => [m.id, formatGb(needBytes(m, m.contexts[0]))]))).toEqual({ 'qwen3.5:9b': '17.0', 'qwen3.5:9b-q8_0': '20.9', 'gpt-oss:20b': '26.6', @@ -484,7 +550,7 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { * * Why. This is the Mac the model was run on (2026-11-06), so its row has a `tested` entry for * exactly this chip, memory size and harness. A tested row ranks first, and it reports the - * speed that was measured (67.7) in place of the estimate from the chip (13.6 x 3.07 = 41.8). + * speed that was measured (67.7) in place of the estimate from the chip (16.9 x 3.07 = 51.9). * * The need is the peak at the full context: 23.1 GB of weights + 4.8 GB held for 131,072 * tokens + 3 GB of runner + 5 GB for a moment while a long prompt is processed = 36.0 GB. @@ -517,14 +583,14 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { needBytes: 35_955_462_956, needGb: '36.0', memory: { modelBytes: 26_105_798_956, contextBytes: 4_849_664_000, peakBytes: 5_000_000_000 }, - // What a Mac really downloads (the MLX build), not the 22.6 GB of the registry manifest. + // What a Mac really downloads (the MLX build), not the 22.6 GB of the GGUF build. downloadBytes: 23_105_798_956, downloadGb: '23.1', budgetShare: 0.957, fit: 'tight', fitLabel: 'Tight', speed: 'ok', - // Measured on this Mac, not the 41.8 the chip's bandwidth predicts. + // Measured on this Mac, not the 51.9 the chip's bandwidth predicts. tokensPerSecond: 67.7, tested: true, needSource: 'measured', @@ -576,35 +642,36 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { expect(result.notes).toEqual([]); }); - it('lists the other four models that fit: the fast ones, the one of unknown speed, the slow one', async () => { + it('lists four other models that fit, each at the largest context it fits at, the fast ones first', async () => { const result = recommend(await macbook(), STOCK, ALL_INSTALLED); expect( result.alternatives.map((c) => [c.modelId, c.context, c.fit, c.speed, c.tokensPerSecond ?? null, c.tag?.kind]) ).toEqual([ - ['qwen3.5:9b-q8_0', 65536, 'good', 'ok', 17.8, 'small'], - ['qwen3.5:9b', 65536, 'good', 'ok', 28.6, 'small'], - ['gpt-oss:20b', 131072, 'good', 'unknown', null, undefined], - // The dense 27B computes to 2.95 x 3.07 = 9.1 tokens per second on this chip. - ['qwen3.8:27b', 131072, 'good', 'slow', 9.1, 'slow'], + // The same model as a GGUF build: its fixed footprint holds the full window, 27.9 GB. + ['qwen3.6:35b-a3b-mtp-q4_k_m', 262144, 'good', 'ok', 41.8, undefined], + ['north-mini-code-1.0:q4_k_m', 262144, 'good', 'ok', 41.8, undefined], + ['mimo-v2.6:9b-q4_k_m', 131072, 'good', 'ok', 28.6, 'small'], + ['qwen3.5:9b-q4_k_m', 131072, 'good', 'ok', 28.6, 'small'], ]); - expect(result.alternatives[2].speedLabel).toBe('Speed unknown'); - expect(result.alternatives[3].speedLabel).toBe('Slow'); // Only the tested row rests on a measurement of that model. expect(result.alternatives.map((c) => c.needSource)).toEqual(['estimated', 'estimated', 'estimated', 'estimated']); - // The two that do not fit at any context are not listed. - expect(order(result).some((entry) => entry.startsWith('qwen3.8:27b-q8_0'))).toBe(false); - expect(order(result).some((entry) => entry.startsWith('qwen3.5:122b'))).toBe(false); + // The dense 27B fits too (29.7 GB at 262144) but computes to 2.95 x 3.07 = 9.1 tokens per + // second here: slow, so it ranks after four fast models and is not listed. + expect(modelFitsBudget(await macbook(), STOCK, 'qwen3.8:27b-mtp-q4_k_m', 262144)).toBe(true); + expect(order(result).some((entry) => entry.startsWith('qwen3.8:27b'))).toBe(false); + // The 122B fits at no context. + expect(order(result).some((entry) => entry.startsWith('ornith'))).toBe(false); }); it('would get the same model as "Best fit", with the estimated speed, had nobody tested it', async () => { - const result = recommend(await macbook(), UNTESTED, ALL_INSTALLED); + const result = recommend(await macbook(), STOCK_UNTESTED, ALL_INSTALLED); expect(result.pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', context: 131072, fit: 'tight', tested: false, - // 13.6 x 307 / 100. - tokensPerSecond: 41.8, + // 16.9 x 307 / 100. + tokensPerSecond: 51.9, // It only just fits, and the tag says so. tag: { kind: 'best-tight', label: 'Best fit, tight' }, }); @@ -622,122 +689,131 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { expect(result.pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', speed: 'ok', tokensPerSecond: 67.7, tested: true }); // The others have no speed, so the model rank orders them. expect(result.alternatives.map((c) => [c.modelId, c.speed])).toEqual([ - ['qwen3.8:27b', 'unknown'], - ['gpt-oss:20b', 'unknown'], - ['qwen3.5:9b-q8_0', 'unknown'], - ['qwen3.5:9b', 'unknown'], + ['qwen3.8:27b-mtp-q4_k_m', 'unknown'], + ['qwen3.8:27b-q8_0', 'unknown'], + ['qwen3.6:35b-a3b-mtp-q4_k_m', 'unknown'], + ['north-mini-code-1.0:q4_k_m', 'unknown'], ]); }); }); describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { /** - * Expected: no recommendation, reason `memory`, in the plain sentence. + * Expected: verdict `limited`, MiMo V2.6 9B (a GGUF build) at 65,536 tokens with OpenCode, + * a TIGHT fit, about 11 tokens per second. * - * Why. The budget is 9.67 GiB (10.4 GB: 2/3 of 16 GiB minus 1). The smallest model, the 9B, - * needs 6.6 GB of weights + 1.2 GB held for its smallest context of 32,768 tokens + 3 GB of - * runner + 5 GB at the peak = 15.8 GB: one and a half times the budget. Nothing fits at any - * context, so this is no longer a miss by a hair that deserves the sentence with both numbers. + * Why. The budget is 9.67 GiB (10.4 GB: 2/3 of 16 GiB minus 1). The 9B on llama.cpp needs + * 6.55 GB of weights + 1.14 GB for a q8_0 K/V cache of 65,536 tokens + 2.13 GB beside them + * (its recurrent state, the runner, and the 1 GiB prompt cache the runtime allows) = 9.8 GB: + * 0.946 of the budget. At 131,072 tokens it would need 11.0 GB, so it steps down. Nothing of + * 12 billion parameters or more fits, so the verdict is `limited`. * - * The 3 GB and the 5 GB were measured on a 35B model and are applied to the 9B unmeasured - * (`needSource: 'estimated'`), so this answer is careful, not proven. + * Every figure of this row is computed from the model's architecture, none was measured + * (`needSource: 'estimated'`); the check after the download (the whole model on the GPU) + * is what catches it if llama.cpp holds more. */ - it('gets no recommendation: no model fits at any context', async () => { + it('gets a small model: the 9B at 65536 tokens, tight', async () => { const hw = await macMini(); expect(hw.chip).toBe('Apple M4'); expect(hw.memClassGb).toBe(16); const result = recommend(hw, STOCK, ALL_INSTALLED); - expect(result).toEqual({ - verdict: 'none', - reason: 'memory', - headline: { - text: 'With {memory} GB of memory, this Mac cannot run a model that can drive a coding agent.', - vars: { memory: '16' }, - }, - notes: [], - alternatives: [], - // floor(17,179,869,184 x 2/3) - 1 GiB. - budgetBytes: 10_379_504_298, - budgetGb: '10.4', - budgetSource: 'estimated', + expect(result.verdict).toBe('limited'); + expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.headlineLimited }); + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + // floor(17,179,869,184 x 2/3) - 1 GiB. + expect(result.budgetBytes).toBe(10_379_504_298); + expect(result.budgetGb).toBe('10.4'); + expect(result.pick).toMatchObject({ + modelId: 'mimo-v2.6:9b-q4_k_m', + modelLabel: 'MiMo V2.6 9B', + license: 'MIT', + context: 65536, + cli: 'opencode', + // 6,550,811,033 + 17,408 x 65,536 + 2,126,432,768. + needBytes: 9_818_094_489, + needGb: '9.8', + memory: { modelBytes: 8_677_243_801, contextBytes: 1_140_850_688, peakBytes: 0 }, + budgetShare: 0.946, + fit: 'tight', + // 9.3 x 120 / 100. + speed: 'ok', + tokensPerSecond: 11.2, + tested: false, + needSource: 'estimated', + tag: { kind: 'best-tight', label: 'Best fit, tight' }, }); + // Its base model, the same figures, is the one alternative. + expect(order(result)).toEqual(['mimo-v2.6:9b-q4_k_m@65536', 'qwen3.5:9b-q4_k_m@65536']); }); - it('is far from fitting: the smallest model at its smallest context needs 1.5 times the budget', async () => { + it('would need 11.0 GB at 131072 tokens, more than the whole budget', async () => { const hw = await macMini(); - const nineB = stockModel('qwen3.5:9b'); + const nineB = stockModel('mimo-v2.6:9b-q4_k_m'); const budget = deviceBudget(hw).budgetBytes; - expect(formatGb(needBytes(nineB, 32768))).toBe('15.8'); - expect(needBytes(nineB, 32768) / budget).toBeCloseTo(1.5187, 4); - expect(needBytes(nineB, 65536) / budget).toBeCloseTo(1.6355, 4); - // Every model needs more than the whole budget at every context. - for (const row of STOCK.models) { - for (const ctx of row.contexts) expect(needBytes(row, ctx), `${row.id}@${ctx}`).toBeGreaterThan(budget); - } + expect(formatGb(needBytes(nineB, 131072))).toBe('11.0'); + expect(needBytes(nineB, 131072) / budget).toBeGreaterThan(1); + expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 131072)).toBe(false); + expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 65536)).toBe(true); }); - it('gets none with the budget its runtime measured, too', async () => { + it('gets the same with the budget its runtime measured, with a little more room', async () => { // The runtime reported 11.8 GiB here; the 6 GiB reserve leaves 10 GiB (10.7 GB) of it. const result = recommend(await macMini(), STOCK, ALL_INSTALLED, { measuredTotalBytes: MACMINI_MEASURED_TOTAL }); expect(result).toMatchObject({ - verdict: 'none', - reason: 'memory', - headline: { text: LOCAL_MODEL_LABELS.noneMemory, vars: { memory: '16' } }, + verdict: 'limited', budgetBytes: 10 * GIB, budgetGb: '10.7', budgetSource: 'measured', }); + expect(result.pick).toMatchObject({ modelId: 'mimo-v2.6:9b-q4_k_m', context: 65536, budgetShare: 0.914 }); }); - it('gets none with a wired limit or with the gated harness opened: memory is what is missing', async () => { - // iogpu.wired_limit_mb = 14000 lifts the budget to 10 GiB (memory minus the 6 GiB reserve). - const wired = recommend({ ...(await macMini()), wiredLimitMb: 14000 }, STOCK, ALL_INSTALLED); - expect(wired).toMatchObject({ verdict: 'none', reason: 'memory', budgetSource: 'wired-limit' }); - expect(wired.budgetBytes).toBe(10 * GIB); - - const ungated = stockWith((catalog) => { - catalog.harnesses = catalog.harnesses.map((row) => ({ - cli: row.cli, - minContext: row.minContext, - rank: row.rank, - })); - }); - const opened = recommend(await macMini(), ungated, ALL_INSTALLED); - expect(opened.reason).toBe('memory'); - expect(opened.headline.text).toBe(LOCAL_MODEL_LABELS.noneMemory); + it('gets the same with Claude Code as its only coding agent, which also needs 65536 tokens', async () => { + const result = recommend(await macMini(), STOCK, [fact('claude')]); + expect(result.verdict).toBe('limited'); + expect(result.pick).toMatchObject({ modelId: 'mimo-v2.6:9b-q4_k_m', context: 65536, cli: 'claude' }); }); }); describe('every memory size, with the stock catalogue', () => { - // What the rules compute today. A change of a catalogue figure or of a threshold that moves - // one of these answers has to change this table on purpose. + // What the stock catalogue answers today, from 8 to 512 GB, on chips that ship with each + // size. A change of a catalogue row or of a threshold that moves one of these answers has to + // change this table on purpose. docs/local-models.md has the same table for people. const EXPECTED: [soldGb: number, chip: string, answer: string][] = [ + // The smallest need at 65536 tokens is 9.8 GB; the budget is 2.1 GB. [8, 'Apple M2', 'none memory'], - [16, 'Apple M4', 'none memory'], - // The 9B at its smallest context needs 15.8 GB of the 11.8 GB budget. - [18, 'Apple M3 Pro', 'none memory'], - // The 9B fits only at 32768 tokens (0.979), and the one harness that works there is gated. - [24, 'Apple M4', 'none memory'], - // Only the two 9B rows fit (0.778, and 0.956 for the Q8, which is slow on a base M4), so - // this is a small-model Mac: gpt-oss:20b needs 24.2 GB of the 21.8 GB budget. - [32, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], - // Both 9B rows are fast enough here, and the higher rank wins although it is the tight one. - [32, 'Apple M1 Max', 'limited qwen3.5:9b-q8_0@65536 tight ok'], - // gpt-oss:20b fits as Tight (0.954), so the verdict is `ok`; it has no speed figure, and a - // known speed ranks before an unknown one. - [36, 'Apple M4 Max', 'ok qwen3.5:9b-q8_0@65536 good ok'], - // On a Pro chip the dense 27B is slow (8.1 tokens per second), the 35B-A3B is not. - [48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 tight ok'], + // The 9B fits only at 65536 tokens (0.946), and nothing of 12 billion fits: `limited`. + [16, 'Apple M4', 'limited mimo-v2.6:9b-q4_k_m@65536 tight ok'], + // On the slowest chip the pick is the same, and slow (6.3 tokens per second). + [16, 'Apple M1', 'limited mimo-v2.6:9b-q4_k_m@65536 tight slow'], + [18, 'Apple M3 Pro', 'limited mimo-v2.6:9b-q4_k_m@131072 tight ok'], + [24, 'Apple M4', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'], + [24, 'Apple M4 Pro', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'], + // North Mini Code needs 21.8 GB at 65536 tokens: 0.997 of the 21.8 GB estimate, out by a + // hair. A runtime that measures more than the estimate admits it (see below). + [32, 'Apple M4', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'], + [32, 'Apple M1 Max', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'], + // A Max chip runs the dense 27B fast enough (12.1); an M3 Max (300 GB/s) does not (8.9), + // and takes the 35B-A3B as a GGUF build, whose memory does not grow after it loaded. + [36, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@131072 tight ok'], + [36, 'Apple M3 Max', 'ok qwen3.6:35b-a3b-mtp-q4_k_m@131072 tight ok'], + // The tested pair on the tested Mac; on a Pro chip the 27B is slow (8.1), so the same pick. [48, 'Apple M5 Pro', 'ok qwen3.6:35b-a3b@131072 tight ok'], - // On a Max chip the 27B is fast enough (12.1) and has the higher rank. - [48, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], - // The Q8 27B fits 47 GiB but computes to 7 tokens per second on this chip. - [64, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], - [96, 'Apple M3 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], - // The 122B fits 95 GiB only as Tight (0.924) and has no speed figure. - [128, 'Apple M5 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], + [48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 tight ok'], + // On a Max chip the 27B has the higher rank and holds its full window, 29.7 GB. + [48, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@262144 good ok'], + [64, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'], + [64, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@262144 good ok'], + [96, 'Apple M3 Ultra', 'ok qwen3.8:27b-mtp-q4_k_m@262144 good ok'], + // The 122B needs 97.4 GB at 131072 tokens (8 GiB of MLX prefix snapshots included): tight. + [128, 'Apple M4 Max', 'ok ornith-1.5:122b@131072 tight ok'], + [128, 'Apple M1 Ultra', 'ok ornith-1.5:122b@131072 tight ok'], + [192, 'Apple M2 Ultra', 'ok ornith-1.5:122b@262144 good ok'], + [256, 'Apple M3 Ultra', 'ok ornith-1.5:122b@262144 good ok'], + // Nothing better than the 122B is listed (see the catalogue header for the 397B). + [512, 'Apple M3 Ultra', 'ok ornith-1.5:122b@262144 good ok'], ]; it.each(EXPECTED)('%i GB, %s: %s', (soldGb, chip, answer) => { @@ -748,21 +824,44 @@ describe('every memory size, with the stock catalogue', () => { expect(actual).toBe(answer); }); - it('tells the four sizes without a model which kind of "too little" it is', () => { - const headlineOf = (soldGb: number): Recommendation['headline'] => - recommend(mac(soldGb, 'Apple M4'), STOCK, ALL_INSTALLED).headline; - // Nothing fits at any context: the plain sentence with the size the Mac is sold as. - for (const soldGb of [8, 16, 18]) { - expect(headlineOf(soldGb), `${soldGb} GB`).toEqual({ - text: LOCAL_MODEL_LABELS.noneMemory, - vars: { memory: String(soldGb) }, - }); + it('covers every memory size', () => { + expect([...new Set(EXPECTED.map(([soldGb]) => soldGb))]).toEqual(MEMORY_SIZES); + }); + + it('tells an 8 GB Mac why in the plain sentence, with its memory size', () => { + const result = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); + expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.noneMemory, vars: { memory: '8' } }); + // The smallest model a harness can work with needs more than four times the budget. + const smallest = Math.min(...STOCK.models.map((m) => needBytes(m, 65536))); + expect(formatGb(smallest)).toBe('9.8'); + expect(smallest / result.budgetBytes).toBeGreaterThan(4); + }); + + it('gives every Mac from 16 to 32 GB a small model, and every larger one a model of 27 billion or more', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + const label = `${hw.memClassGb} GB ${hw.chip}`; + if (hw.memClassGb === 8) expect(result.reason, label).toBe('memory'); + else if (hw.memClassGb <= 32) { + expect(result.verdict, label).toBe('limited'); + expect(result.pick?.paramsB, label).toBe(9); + } else { + expect(result.verdict, label).toBe('ok'); + expect(result.pick?.paramsB, label).toBeGreaterThanOrEqual(27); + } } - // A model fits, but only below the context a harness needs: both numbers. - expect(headlineOf(24)).toEqual({ - text: LOCAL_MODEL_LABELS.noneMemoryContext, - vars: { budget: '16.1', need: '17.0' }, - }); + }); + + it('gives a 32 GB Mac a model of 30 billion once its runtime measures what other Macs measured', () => { + // The 16 GB and the 48 GB Mac both measured about 3/4 of their memory, more than the 2/3 + // the estimate assumes below 36 GB. 0.74 x 32 GiB lifts the budget to 25.4 GB. + const measured = { measuredTotalBytes: Math.floor(0.74 * 32 * GIB) }; + const base = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED, measured); + expect(base.verdict).toBe('ok'); + expect(base.pick).toMatchObject({ modelId: 'north-mini-code-1.0:q4_k_m', context: 262144, fit: 'tight' }); + // On a Max chip the 27B is fast enough, and it is the higher rank. + const max = recommend(mac(32, 'Apple M1 Max'), STOCK, ALL_INSTALLED, measured); + expect(max.pick).toMatchObject({ modelId: 'qwen3.8:27b-mtp-q4_k_m', context: 131072, fit: 'tight' }); }); it('always pairs the pick with the top-ranked harness, OpenCode', () => { @@ -771,6 +870,15 @@ describe('every memory size, with the stock catalogue', () => { if (result.pick) expect(result.pick.cli).toBe('opencode'); } }); + + it('never offers an MLX build below 48 GB: its prefix snapshots alone are 8 GiB', () => { + const mlx = new Set(STOCK.models.filter((m) => m.runner === 'mlx').map((m) => m.id)); + for (const hw of SWEEP.filter((swept) => swept.memClassGb < 48)) { + for (const candidate of rows(recommend(hw, STOCK, ALL_INSTALLED))) { + expect(mlx.has(candidate.modelId), `${hw.memClassGb} GB ${candidate.modelId}`).toBe(false); + } + } + }); }); // ---------- 3. Verdict none ---------- @@ -798,7 +906,7 @@ describe('verdict none', () => { memClassGb: 64, source: 'not-probed', }; - const result = recommend(linux, STOCK, ALL_INSTALLED); + const result = recommend(linux, RULES, ALL_INSTALLED); expectNone(result, 'not-macos', LOCAL_MODEL_LABELS.noneNotMacos); expect(result.headline.vars).toBeUndefined(); expect(result.budgetBytes).toBe(0); @@ -809,7 +917,7 @@ describe('verdict none', () => { it('comes before every other reason', () => { for (const platform of ['linux', 'win32', 'freebsd']) { const hw = mac(8, 'Intel(R) Core(TM) i7', { platform, arch: 'x64', arm64: false, macosMajor: 12 }); - expect(recommend(hw, STOCK, []).reason, platform).toBe('not-macos'); + expect(recommend(hw, RULES, []).reason, platform).toBe('not-macos'); } }); }); @@ -819,7 +927,7 @@ describe('verdict none', () => { it('answers so for an Intel Mac, whose sysctl has no arm64 flag', () => { const hw = without(mac(32, intelChip, { arch: 'x64' }), 'arm64', 'translated', 'wiredLimitMb'); - const result = recommend(hw, STOCK, ALL_INSTALLED); + const result = recommend(hw, RULES, ALL_INSTALLED); expectNone(result, 'intel', 'Local coding models need Apple silicon, and this Mac has an Intel processor.'); expect(result.budgetBytes).toBe(0); expect(result.budgetGb).toBe('0.0'); @@ -827,20 +935,20 @@ describe('verdict none', () => { it('answers so when the arm64 flag reads 0', () => { const hw = without(mac(32, intelChip, { arch: 'x64', arm64: false }), 'translated'); - expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('intel'); + expect(recommend(hw, RULES, ALL_INSTALLED).reason).toBe('intel'); }); it('comes before the macOS version and the memory', () => { const hw = without(mac(8, intelChip, { arch: 'x64', macosMajor: 12, macosVersion: '12.7' }), 'arm64'); - expect(recommend(hw, STOCK, []).reason).toBe('intel'); + expect(recommend(hw, RULES, []).reason).toBe('intel'); }); it('does not mistake Node under Rosetta for an Intel Mac', () => { // An x64 process on Apple silicon: the machine flag says arm64, the process is translated. const rosetta = mac(48, 'Apple M4 Max', { arch: 'x64', translated: true }); - expect(recommend(rosetta, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + expect(recommend(rosetta, RULES, ALL_INSTALLED).verdict).toBe('ok'); // The same when only the translation flag was read. - expect(recommend(without(rosetta, 'arm64'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + expect(recommend(without(rosetta, 'arm64'), RULES, ALL_INSTALLED).verdict).toBe('ok'); }); it('decides from node:os alone when the probe failed', () => { @@ -854,12 +962,12 @@ describe('verdict none', () => { source: 'os-fallback', }); // An arm64 process can only run on Apple silicon. - expect(recommend(fallback('arm64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + expect(recommend(fallback('arm64', 'Apple M4 Max'), RULES, ALL_INSTALLED).verdict).toBe('ok'); // An x64 process with an Apple chip name is Rosetta. - expect(recommend(fallback('x64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + expect(recommend(fallback('x64', 'Apple M4 Max'), RULES, ALL_INSTALLED).verdict).toBe('ok'); // An x64 process with any other chip name is taken for an Intel Mac. - expect(recommend(fallback('x64', intelChip), STOCK, ALL_INSTALLED).reason).toBe('intel'); - expect(recommend(without(fallback('x64', intelChip), 'chip'), STOCK, ALL_INSTALLED).reason).toBe('intel'); + expect(recommend(fallback('x64', intelChip), RULES, ALL_INSTALLED).reason).toBe('intel'); + expect(recommend(without(fallback('x64', intelChip), 'chip'), RULES, ALL_INSTALLED).reason).toBe('intel'); }); }); @@ -867,7 +975,7 @@ describe('verdict none', () => { it('answers so below the catalogue minimum, naming both versions', () => { const result = recommend( mac(48, 'Apple M1 Max', { macosVersion: '13.6.7', macosMajor: 13 }), - STOCK, + RULES, ALL_INSTALLED ); expectNone(result, 'macos', 'Local models need macOS {minimum} or newer, and this Mac runs macOS {version}.'); @@ -879,24 +987,24 @@ describe('verdict none', () => { it('accepts the minimum itself and anything newer', () => { for (const macosMajor of [14, 15, 26, 27]) { const hw = mac(48, 'Apple M4 Max', { macosMajor, macosVersion: `${macosMajor}.0` }); - expect(recommend(hw, STOCK, ALL_INSTALLED).verdict, String(macosMajor)).toBe('ok'); + expect(recommend(hw, RULES, ALL_INSTALLED).verdict, String(macosMajor)).toBe('ok'); } }); it('does not block on a version that could not be read', () => { const hw = without(mac(48, 'Apple M4 Max'), 'macosMajor', 'macosVersion'); - expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + expect(recommend(hw, RULES, ALL_INSTALLED).verdict).toBe('ok'); }); it('comes before the memory', () => { const hw = mac(8, 'Apple M1', { macosVersion: '12.7', macosMajor: 12 }); - expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('macos'); + expect(recommend(hw, RULES, ALL_INSTALLED).reason).toBe('macos'); }); }); describe('memory', () => { it('answers so at 8 GB, where nothing fits at any context', () => { - const result = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); + const result = recommend(mac(8, 'Apple M2'), RULES, ALL_INSTALLED); expectNone( result, 'memory', @@ -924,11 +1032,11 @@ describe('verdict none', () => { it('says both numbers when models fit, but only below the context a harness needs', () => { // The 24 GB case, with harnesses that are all present: the 9B fits at 32768 tokens. const hw = mac(24, 'Apple M4'); - const nineB = stockModel('qwen3.5:9b'); + const nineB = rulesModel('qwen3.5:9b'); expect(needBytes(nineB, 32768) / deviceBudget(hw).budgetBytes).toBeCloseTo(0.9787, 4); expect(needBytes(nineB, 65536) / deviceBudget(hw).budgetBytes).toBeCloseTo(1.054, 4); - const result = recommend(hw, STOCK, ALL_INSTALLED); + const result = recommend(hw, RULES, ALL_INSTALLED); expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemoryContext); // The need is the smallest one at a context a usable harness works at (the 9B at 65536), // not the 9B at 32768, which fits and which only the gated harness could use. @@ -941,7 +1049,7 @@ describe('verdict none', () => { }); it('would give that Mac the 9B at 32768 tokens, as a limited answer, once the gated harness is opened', () => { - const ungated = stockWith((catalog) => { + const ungated = rulesWith((catalog) => { catalog.harnesses = catalog.harnesses.map((row) => ({ cli: row.cli, minContext: row.minContext, @@ -986,7 +1094,7 @@ describe('verdict none', () => { }); it('comes before no-harness when nothing fits at all', () => { - expect(recommend(mac(8, 'Apple M2'), STOCK, []).reason).toBe('memory'); + expect(recommend(mac(8, 'Apple M2'), RULES, []).reason).toBe('memory'); }); it('answers so for an empty catalogue', () => { @@ -995,7 +1103,7 @@ describe('verdict none', () => { it('answers so when the memory size is unknown, without stating a size', () => { const hw = { ...mac(48, 'Apple M4 Max'), memBytes: 0, memGb: '0.0', memClassGb: 0 }; - const result = recommend(hw, STOCK, ALL_INSTALLED); + const result = recommend(hw, RULES, ALL_INSTALLED); expectNone( result, 'memory', @@ -1012,24 +1120,24 @@ describe('verdict none', () => { const headline = 'No coding agent that Codeman can connect to a local model is available on this Mac.'; it('answers so when the caller knows no catalogue harness', () => { - const result = recommend(bigMac, STOCK, []); + const result = recommend(bigMac, RULES, []); expectNone(result, 'no-harness', headline); expect(result.headline.vars).toBeUndefined(); expect(result.budgetBytes).toBe(35 * GIB); }); it('answers so when no harness is installed or installable', () => { - const facts = STOCK.harnesses.map((row) => fact(row.cli, { installed: false, installable: false })); - expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); + const facts = RULES.harnesses.map((row) => fact(row.cli, { installed: false, installable: false })); + expect(recommend(bigMac, RULES, facts).reason).toBe('no-harness'); }); it('answers so when only the gated harness is on the machine', () => { - const gatedRows = STOCK.harnesses.filter((row) => row.gated); + const gatedRows = RULES.harnesses.filter((row) => row.gated); expect(gatedRows).toHaveLength(1); expect( recommend( bigMac, - STOCK, + RULES, gatedRows.map((row) => fact(row.cli)) ).reason ).toBe('no-harness'); @@ -1037,11 +1145,11 @@ describe('verdict none', () => { it('answers so when a harness is there but accepts no model id', () => { const facts = ALL_INSTALLED.map((entry) => ({ ...entry, acceptsModelId: () => false })); - expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); + expect(recommend(bigMac, RULES, facts).reason).toBe('no-harness'); }); it('ignores a fact for a CLI that is not a catalogue harness', () => { - expect(recommend(bigMac, STOCK, [fact('codex'), fact('some-other-cli')]).reason).toBe('no-harness'); + expect(recommend(bigMac, RULES, [fact('codex'), fact('some-other-cli')]).reason).toBe('no-harness'); }); }); @@ -1049,7 +1157,7 @@ describe('verdict none', () => { it('answers so when every fitting model would leave under 10 GiB free', () => { // 12 GiB free: even the smallest download (6.55 GB) leaves 5.9 GiB. const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); - const result = recommend(hw, STOCK, ALL_INSTALLED); + const result = recommend(hw, RULES, ALL_INSTALLED); // One sentence with the one number that says what to do. expectNone( result, @@ -1064,7 +1172,7 @@ describe('verdict none', () => { it('says what to free, counting the reserve: download + 10 GiB - free', () => { // 12 GB free is more than the 6.6 GB download: what is missing is the reserve on top of it. const hw = mac(32, 'Apple M4', { freeDiskBytes: 12 * GB, freeDiskGb: formatGb(12 * GB) }); - const vars = recommend(hw, STOCK, ALL_INSTALLED).headline.vars as Record; + const vars = recommend(hw, RULES, ALL_INSTALLED).headline.vars as Record; // 6,550,825,373 + 10,737,418,240 - 12,000,000,000 bytes. expect(vars).toEqual({ missing: '5.3' }); expect(Number(vars.missing)).toBeCloseTo((6_550_825_373 + DISK_HEADROOM_BYTES - 12 * GB) / GB, 1); @@ -1077,7 +1185,7 @@ describe('verdict none', () => { acceptsModelId: (id: string) => !id.startsWith('qwen3.5:9b'), })); const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); - const result = recommend(hw, STOCK, facts); + const result = recommend(hw, RULES, facts); expect(result.reason).toBe('disk'); // 13,793,441,427 + 10 GiB - 12 GiB: the 20B's download, not the 9B's. expect(result.headline.vars).toEqual({ missing: '11.6' }); @@ -1086,7 +1194,7 @@ describe('verdict none', () => { it('drops only the models that do not leave 10 GiB, picks among the rest, and says what was left out', () => { // 20 GiB free: the 9B downloads (6.1 and 9.7 GiB) leave 10 GiB or more, the others do not. const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 20 * GIB, freeDiskGb: formatGb(20 * GIB) }); - const result = recommend(hw, STOCK, ALL_INSTALLED); + const result = recommend(hw, RULES, ALL_INSTALLED); expect(order(result)).toEqual(['qwen3.5:9b-q8_0@65536', 'qwen3.5:9b@65536']); // A 48 GB Mac is not a small-model Mac because its disk is full: the verdict stays `ok`, // the pick is called small in a note, and the model it would have been is named with @@ -1114,12 +1222,12 @@ describe('verdict none', () => { freeDiskGb: formatGb(bytes), }); - const enough = recommend(withFree(34 * GB), STOCK, ALL_INSTALLED); + const enough = recommend(withFree(34 * GB), RULES, ALL_INSTALLED); expect(enough.verdict).toBe('ok'); expect(enough.pick?.modelId).toBe('qwen3.6:35b-a3b'); expect(enough.notes).toEqual([]); - const short = recommend(withFree(33 * GB), STOCK, ALL_INSTALLED); + const short = recommend(withFree(33 * GB), RULES, ALL_INSTALLED); expect(short.verdict).toBe('ok'); // The tested model has no room, so nothing here is "Recommended". expect(short.headline.text).toBe(LOCAL_MODEL_LABELS.headlineOk); @@ -1133,14 +1241,39 @@ describe('verdict none', () => { // The exact cut: one byte decides, and the note appears with the first missing byte. const cut = 23_105_798_956 + DISK_HEADROOM_BYTES; expect(cut).toBe(33_843_217_196); + expect(recommend(withFree(cut), RULES, ALL_INSTALLED).pick?.modelId).toBe('qwen3.6:35b-a3b'); + expect(recommend(withFree(cut - 1), RULES, ALL_INSTALLED).pick?.modelId).toBe('qwen3.5:9b-q8_0'); + }); + + it('on the real MacBook with the stock catalogue, falls back to the same model as a GGUF build', async () => { + const hw = await macbook(); + const withFree = (bytes: number): HardwareFacts => ({ ...hw, freeDiskBytes: bytes, freeDiskGb: formatGb(bytes) }); + // Enough for the tested pair; the Q8 27B (30.0 GB) fits the memory and not the disk, and says so. + const enough = recommend(withFree(34 * GB), STOCK, ALL_INSTALLED); + expect(enough.pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', tested: true }); + expect(enough.notes).toEqual([ + { text: LOCAL_MODEL_LABELS.noteDiskDropped, vars: { model: 'Qwen3.8 27B (Q8)', missing: '6.7' } }, + ]); + // One byte short of the tested pair: its GGUF build (22.6 GB) still has room. + const cut = 23_105_798_956 + DISK_HEADROOM_BYTES; expect(recommend(withFree(cut), STOCK, ALL_INSTALLED).pick?.modelId).toBe('qwen3.6:35b-a3b'); - expect(recommend(withFree(cut - 1), STOCK, ALL_INSTALLED).pick?.modelId).toBe('qwen3.5:9b-q8_0'); + const oneShort = recommend(withFree(cut - 1), STOCK, ALL_INSTALLED); + expect(oneShort.pick?.modelId).toBe('qwen3.6:35b-a3b-mtp-q4_k_m'); + expect(oneShort.headline.text).toBe(LOCAL_MODEL_LABELS.headlineOk); + // At 33 GB neither 35B-A3B build has room: North Mini Code, and the tested pair is named. + const short = recommend(withFree(33 * GB), STOCK, ALL_INSTALLED); + expect(short.verdict).toBe('ok'); + expect(short.pick?.modelId).toBe('north-mini-code-1.0:q4_k_m'); + expect(short.notes).toEqual([ + { text: LOCAL_MODEL_LABELS.noteDiskDropped, vars: { model: 'Qwen3.6 35B-A3B', missing: '0.8' } }, + { text: LOCAL_MODEL_LABELS.noteUntested }, + ]); }); it('names the best-ranked model left out, also when the pick is not affected', () => { // 64 GB, M4 Max, 35 GB free: only the Q8 27B (30 GB) has no room. The pick stays the 27B. const hw = mac(64, 'Apple M4 Max', { freeDiskBytes: 35 * GB, freeDiskGb: '35.0' }); - const result = recommend(hw, STOCK, ALL_INSTALLED); + const result = recommend(hw, RULES, ALL_INSTALLED); expect(result.pick?.modelId).toBe('qwen3.8:27b'); expect(order(result).some((entry) => entry.startsWith('qwen3.8:27b-q8_0'))).toBe(false); // 29,978,241,837 + 10 GiB - 35 GB. @@ -1152,36 +1285,36 @@ describe('verdict none', () => { it('adds no disk note when every model that fits has room, or the free disk is unknown', () => { for (const hw of [mac(64, 'Apple M4 Max'), without(mac(64, 'Apple M4 Max'), 'freeDiskBytes', 'freeDiskGb')]) { - const texts = recommend(hw, STOCK, ALL_INSTALLED).notes.map((note) => note.text); + const texts = recommend(hw, RULES, ALL_INSTALLED).notes.map((note) => note.text); expect(texts).not.toContain(LOCAL_MODEL_LABELS.noteDiskDropped); } }); it('allows a download that leaves exactly 10 GiB', () => { - const exact = STOCK.models[0].bytes + DISK_HEADROOM_BYTES; + const exact = RULES.models[0].bytes + DISK_HEADROOM_BYTES; expect(DISK_HEADROOM_BYTES).toBe(10 * GIB); - const fits = recommend(mac(32, 'Apple M4', { freeDiskBytes: exact }), STOCK, ALL_INSTALLED); + const fits = recommend(mac(32, 'Apple M4', { freeDiskBytes: exact }), RULES, ALL_INSTALLED); expect(fits.pick?.modelId).toBe('qwen3.5:9b'); - const short = recommend(mac(32, 'Apple M4', { freeDiskBytes: exact - 1 }), STOCK, ALL_INSTALLED); + const short = recommend(mac(32, 'Apple M4', { freeDiskBytes: exact - 1 }), RULES, ALL_INSTALLED); expect(short.reason).toBe('disk'); }); it('does not block when the free disk is unknown', () => { const hw = without(mac(48, 'Apple M4 Max'), 'freeDiskBytes', 'freeDiskGb'); - expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + expect(recommend(hw, RULES, ALL_INSTALLED).verdict).toBe('ok'); }); it('needs only the bytes of the free disk, not its text', () => { const hw = without(mac(48, 'Apple M4 Max', { freeDiskBytes: 3 * GB }), 'freeDiskGb'); // 6,550,825,373 + 10 GiB - 3 GB. - expect(recommend(hw, STOCK, ALL_INSTALLED).headline.vars).toEqual({ missing: '14.3' }); + expect(recommend(hw, RULES, ALL_INSTALLED).headline.vars).toEqual({ missing: '14.3' }); }); it('comes after the memory and the harness', () => { const fullDisk = { freeDiskBytes: 0, freeDiskGb: '0.0' }; - expect(recommend(mac(16, 'Apple M4', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('memory'); - expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, []).reason).toBe('no-harness'); - expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('disk'); + expect(recommend(mac(16, 'Apple M4', fullDisk), RULES, ALL_INSTALLED).reason).toBe('memory'); + expect(recommend(mac(48, 'Apple M4 Max', fullDisk), RULES, []).reason).toBe('no-harness'); + expect(recommend(mac(48, 'Apple M4 Max', fullDisk), RULES, ALL_INSTALLED).reason).toBe('disk'); }); }); }); @@ -1191,7 +1324,7 @@ describe('verdict none', () => { describe('verdict limited', () => { it('is the answer when no model that fits has 12 billion parameters or more', () => { expect(LIMITED_BELOW_PARAMS_B).toBe(12); - const result = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED); + const result = recommend(mac(32, 'Apple M4'), RULES, ALL_INSTALLED); expect(result.verdict).toBe('limited'); expect(result.reason).toBeUndefined(); expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', paramsB: 9, context: 65536, fit: 'good' }); @@ -1202,7 +1335,7 @@ describe('verdict limited', () => { }); it('holds when the pick is a tight fit too (32 GB with a fast chip)', () => { - const result = recommend(mac(32, 'Apple M1 Max'), STOCK, ALL_INSTALLED); + const result = recommend(mac(32, 'Apple M1 Max'), RULES, ALL_INSTALLED); expect(result.verdict).toBe('limited'); expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b-q8_0', @@ -1225,7 +1358,7 @@ describe('verdict limited', () => { it('is about the MAC, not the pick: a larger model further down the list lifts it', () => { // 36 GB: the 20B fits as Tight and has no speed figure, so a 9B is the pick. The Mac can // still run a 20B, so "This Mac can run a small model" would be false of it. - const result = recommend(mac(36, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const result = recommend(mac(36, 'Apple M4 Max'), RULES, ALL_INSTALLED); expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b-q8_0', paramsB: 9, fit: 'good', speed: 'ok' }); expect(result.alternatives.map((c) => [c.modelId, c.fit, c.speed])).toEqual([ ['qwen3.5:9b', 'good', 'ok'], @@ -1245,8 +1378,8 @@ describe('verdict limited', () => { it('does not depend on whether the chip is in the chips table', () => { // Knowing the chip must not change the verdict: it is about what fits, not about speed. const verdicts = [32, 36].map((soldGb) => [ - recommend(mac(soldGb, 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict, - recommend(mac(soldGb, 'Apple M6'), STOCK, ALL_INSTALLED).verdict, + recommend(mac(soldGb, 'Apple M4 Max'), RULES, ALL_INSTALLED).verdict, + recommend(mac(soldGb, 'Apple M6'), RULES, ALL_INSTALLED).verdict, ]); expect(verdicts).toEqual([ ['limited', 'limited'], @@ -1255,19 +1388,26 @@ describe('verdict limited', () => { }); it('is limited exactly when nothing with 12 billion parameters fits, on every Mac of the sweep', () => { - for (const hw of SWEEP) { - const result = recommend(hw, STOCK, ALL_INSTALLED); - if (result.verdict === 'none') continue; - const budget = deviceBudget(hw).budgetBytes; - const largeFits = STOCK.models.some( - (m) => - m.paramsB >= LIMITED_BELOW_PARAMS_B && - m.contexts.some((ctx) => ctx >= 65536 && needBytes(m, ctx) / budget <= FIT_TIGHT_MAX_SHARE) - ); - expect([hw.memClassGb, hw.chip, result.verdict]).toEqual([hw.memClassGb, hw.chip, largeFits ? 'ok' : 'limited']); - // The small-pick note: under `ok` only, and exactly when the pick is small. - const small = result.notes.some((note) => note.text === LOCAL_MODEL_LABELS.noteSmallPick); - expect(small).toBe(result.verdict === 'ok' && (result.pick as Candidate).paramsB < LIMITED_BELOW_PARAMS_B); + for (const [name, catalog] of BOTH) { + for (const hw of SWEEP) { + const result = recommend(hw, catalog, ALL_INSTALLED); + if (result.verdict === 'none') continue; + const budget = deviceBudget(hw).budgetBytes; + const largeFits = catalog.models.some( + (m) => + m.paramsB >= LIMITED_BELOW_PARAMS_B && + m.contexts.some((ctx) => ctx >= 65536 && needBytes(m, ctx) / budget <= FIT_TIGHT_MAX_SHARE) + ); + expect([name, hw.memClassGb, hw.chip, result.verdict]).toEqual([ + name, + hw.memClassGb, + hw.chip, + largeFits ? 'ok' : 'limited', + ]); + // The small-pick note: under `ok` only, and exactly when the pick is small. + const small = result.notes.some((note) => note.text === LOCAL_MODEL_LABELS.noteSmallPick); + expect(small).toBe(result.verdict === 'ok' && (result.pick as Candidate).paramsB < LIMITED_BELOW_PARAMS_B); + } } }); @@ -1277,7 +1417,7 @@ describe('verdict limited', () => { ...entry, acceptsModelId: (id: string) => id.startsWith('qwen3.5:9b'), })); - const result = recommend(mac(48, 'Apple M4 Max'), STOCK, facts); + const result = recommend(mac(48, 'Apple M4 Max'), RULES, facts); expect(result.verdict).toBe('limited'); expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); // The headline says it, so the note does not repeat it. @@ -1285,11 +1425,13 @@ describe('verdict limited', () => { }); it('never says "Recommended", for any Mac', () => { - for (const hw of SWEEP) { - const result = recommend(hw, STOCK, ALL_INSTALLED); - if (result.verdict === 'limited') { - expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); - expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + for (const [, catalog] of BOTH) { + for (const hw of SWEEP) { + const result = recommend(hw, catalog, ALL_INSTALLED); + if (result.verdict === 'limited') { + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + } } } }); @@ -1297,7 +1439,7 @@ describe('verdict limited', () => { describe('speed', () => { it('is the model figure times the chip bandwidth, rounded to one decimal', () => { - const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const result = recommend(mac(48, 'Apple M4 Max'), RULES, ALL_INSTALLED); // 2.95 x 410 / 100. expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', speed: 'ok', tokensPerSecond: 12.1 }); expect(result.pick).not.toHaveProperty('speedLabel'); @@ -1327,7 +1469,7 @@ describe('speed', () => { it('is unknown for a model without a figure, on any chip', () => { // 48 GB: five models fit, so all of them are listed and gpt-oss:20b is among them. for (const chip of ['Apple M1', 'Apple M4 Max', 'Apple M5 Ultra']) { - const result = recommend(mac(48, chip), STOCK, ALL_INSTALLED); + const result = recommend(mac(48, chip), RULES, ALL_INSTALLED); const gptOss = rows(result).find((c) => c.modelId === 'gpt-oss:20b'); expect(gptOss, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); expect(gptOss).not.toHaveProperty('tokensPerSecond'); @@ -1336,14 +1478,14 @@ describe('speed', () => { it('is unknown for every model on a chip that is in no chips row: no bandwidth is guessed', () => { for (const chip of ['Apple M6', 'Apple M6 Pro', 'Apple M10', 'Some Future Chip']) { - const result = recommend(mac(64, chip), STOCK, ALL_INSTALLED); + const result = recommend(mac(64, chip), RULES, ALL_INSTALLED); for (const candidate of rows(result)) { expect(candidate, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); expect(candidate).not.toHaveProperty('tokensPerSecond'); } } // And when the chip name itself is missing. - const result = recommend(without(mac(64, 'x'), 'chip'), STOCK, ALL_INSTALLED); + const result = recommend(without(mac(64, 'x'), 'chip'), RULES, ALL_INSTALLED); expect(result.pick?.speed).toBe('unknown'); }); @@ -1384,7 +1526,7 @@ describe('the ranking', () => { it('ranks by model rank alone when every speed is unknown', () => { // 'Apple M6' is in no chips row. - const result = recommend(mac(48, 'Apple M6'), STOCK, ALL_INSTALLED); + const result = recommend(mac(48, 'Apple M6'), RULES, ALL_INSTALLED); expect(order(result)).toEqual([ 'qwen3.8:27b@131072', 'qwen3.6:35b-a3b@131072', @@ -1396,7 +1538,7 @@ describe('the ranking', () => { it('ranks an unknown speed below a known ok one and above a slow one, on a real tier', () => { // 48 GB, M4 Pro: three ok models, then gpt-oss:20b (unknown), then the 27B (slow). - const result = recommend(mac(48, 'Apple M4 Pro'), STOCK, ALL_INSTALLED); + const result = recommend(mac(48, 'Apple M4 Pro'), RULES, ALL_INSTALLED); expect(rows(result).map((c) => [c.modelId, c.speed])).toEqual([ ['qwen3.6:35b-a3b', 'ok'], ['qwen3.5:9b-q8_0', 'ok'], @@ -1467,15 +1609,17 @@ describe('the ranking', () => { it('lists each model once, never the pick again, and at most four alternatives', () => { expect(MAX_ALTERNATIVES).toBe(4); // Seven models fit a 128 GB Mac; the pick plus four are shown. - const result = recommend(mac(128, 'Apple M5 Ultra'), STOCK, ALL_INSTALLED); + const result = recommend(mac(128, 'Apple M5 Ultra'), RULES, ALL_INSTALLED); expect(result.alternatives).toHaveLength(4); const ids = rows(result).map((c) => c.modelId); expect(new Set(ids).size).toBe(ids.length); - for (const hw of SWEEP) { - const swept = recommend(hw, STOCK, ALL_INSTALLED); - const sweptIds = rows(swept).map((c) => c.modelId); - expect(new Set(sweptIds).size).toBe(sweptIds.length); - expect(swept.alternatives.length).toBeLessThanOrEqual(MAX_ALTERNATIVES); + for (const [, catalog] of BOTH) { + for (const hw of SWEEP) { + const swept = recommend(hw, catalog, ALL_INSTALLED); + const sweptIds = rows(swept).map((c) => c.modelId); + expect(new Set(sweptIds).size).toBe(sweptIds.length); + expect(swept.alternatives.length).toBeLessThanOrEqual(MAX_ALTERNATIVES); + } } }); }); @@ -1484,40 +1628,44 @@ describe('harnesses', () => { const hw = mac(48, 'Apple M4 Max'); it('never chooses a gated harness, even when it is the only one that could run a model', () => { - const gated = STOCK.harnesses.filter((row) => row.gated).map((row) => row.cli); - expect(gated).toEqual(['pi']); - for (const swept of SWEEP) { - const result = recommend(swept, STOCK, ALL_INSTALLED); - for (const candidate of rows(result)) { - expect(gated).not.toContain(candidate.cli); - expect(candidate.harnesses.map((h) => h.cli).some((cli) => gated.includes(cli))).toBe(false); + for (const [, catalog] of BOTH) { + const gated = catalog.harnesses.filter((row) => row.gated).map((row) => row.cli); + expect(gated).toEqual(['pi']); + for (const swept of SWEEP) { + const result = recommend(swept, catalog, ALL_INSTALLED); + for (const candidate of rows(result)) { + expect(gated).not.toContain(candidate.cli); + expect(candidate.harnesses.map((h) => h.cli).some((cli) => gated.includes(cli))).toBe(false); + } } } // 24 GB: the gated harness alone works at 32768 tokens, where the 9B would fit. Still none. - expect(recommend(mac(24, 'Apple M4'), STOCK, ALL_INSTALLED).verdict).toBe('none'); + expect(recommend(mac(24, 'Apple M4'), RULES, ALL_INSTALLED).verdict).toBe('none'); }); it('never offers a context below the minimum of the harness', () => { - for (const swept of SWEEP) { - const result = recommend(swept, STOCK, ALL_INSTALLED); - for (const candidate of rows(result)) { - for (const harness of candidate.harnesses) { - const row = STOCK.harnesses.find((entry) => entry.cli === harness.cli); - expect(candidate.context).toBeGreaterThanOrEqual(row?.minContext ?? Number.POSITIVE_INFINITY); + for (const [, catalog] of BOTH) { + for (const swept of SWEEP) { + const result = recommend(swept, catalog, ALL_INSTALLED); + for (const candidate of rows(result)) { + for (const harness of candidate.harnesses) { + const row = catalog.harnesses.find((entry) => entry.cli === harness.cli); + expect(candidate.context).toBeGreaterThanOrEqual(row?.minContext ?? Number.POSITIVE_INFINITY); + } } } } }); it('takes the higher-ranked harness and reports its label and that it is installed', () => { - const result = recommend(hw, STOCK, ALL_INSTALLED); + const result = recommend(hw, RULES, ALL_INSTALLED); expect(result.pick).toMatchObject({ cli: 'opencode', cliLabel: 'OpenCode', cliInstalled: true }); expect(result.pick).not.toHaveProperty('slowFirstTurn'); }); it('falls back to the next harness when the first is not on the machine, and flags its slow first turn', () => { const facts = [fact('opencode', { installed: false, installable: false }), fact('claude'), fact('pi')]; - const result = recommend(hw, STOCK, facts); + const result = recommend(hw, RULES, facts); expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'claude', @@ -1526,24 +1674,24 @@ describe('harnesses', () => { slowFirstTurn: true, }); // The same when the caller has no fact for it at all. - expect(recommend(hw, STOCK, [fact('claude')]).pick?.cli).toBe('claude'); + expect(recommend(hw, RULES, [fact('claude')]).pick?.cli).toBe('claude'); }); it('offers a harness that is not installed but can be, and says it is not installed', () => { const facts = [fact('opencode', { installed: false, installable: true }), fact('claude')]; - const result = recommend(hw, STOCK, facts); + const result = recommend(hw, RULES, facts); expect(result.pick).toMatchObject({ cli: 'opencode', cliInstalled: false }); }); it('pairs a model only with a harness that accepts its id', () => { const refusing27b = (id: string): boolean => id !== 'qwen3.8:27b'; // OpenCode refuses the 27B: it goes with the next harness. - const one = recommend(hw, STOCK, [fact('opencode', { acceptsModelId: refusing27b }), fact('claude')]); + const one = recommend(hw, RULES, [fact('opencode', { acceptsModelId: refusing27b }), fact('claude')]); expect(one.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'claude' }); expect(one.pick?.harnesses.map((h) => h.cli)).toEqual(['claude']); expect(one.alternatives[0]).toMatchObject({ modelId: 'qwen3.6:35b-a3b', cli: 'opencode' }); // Both refuse it: the model is not offered. - const both = recommend(hw, STOCK, [ + const both = recommend(hw, RULES, [ fact('opencode', { acceptsModelId: refusing27b }), fact('claude', { acceptsModelId: refusing27b }), ]); @@ -1557,10 +1705,10 @@ describe('harnesses', () => { throw new Error('registry not loaded'); }, }); - const result = recommend(hw, STOCK, [throwing, fact('claude')]); + const result = recommend(hw, RULES, [throwing, fact('claude')]); expect(result.verdict).toBe('ok'); expect(result.pick?.cli).toBe('claude'); - expect(recommend(hw, STOCK, [throwing]).reason).toBe('no-harness'); + expect(recommend(hw, RULES, [throwing]).reason).toBe('no-harness'); }); it('puts the model before the harness rank, and pairs it with the best harness that can run it', () => { @@ -1608,7 +1756,7 @@ describe('the tested flag and the word "Recommended"', () => { const testedOn = (chip: string, memGb: number, cli: string): TestedOn => ({ chip, memGb, cli, date: '2026-11-06' }); /** The untested catalogue with one tested entry on the dense 27B. */ const with27bTested = (chip: string, memGb: number, cli: string): LocalModelCatalog => - stockWith((catalog) => { + rulesWith((catalog) => { for (const row of catalog.models) row.tested = []; (catalog.models.find((m) => m.id === 'qwen3.8:27b') as LocalModelEntry).tested = [testedOn(chip, memGb, cli)]; }); @@ -1644,7 +1792,7 @@ describe('the tested flag and the word "Recommended"', () => { it('is off for the tested Mac when its harness is not the tested one', () => { // The MacBook without OpenCode: the model is the same, the pair was not run by us. - const result = recommend(mac(48, 'Apple M5 Pro'), STOCK, [fact('claude')]); + const result = recommend(mac(48, 'Apple M5 Pro'), RULES, [fact('claude')]); expect(result.pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', cli: 'claude', tested: false }); // So the speed is the estimate again, and nothing says "Recommended". expect(result.pick?.tokensPerSecond).toBe(41.8); @@ -1704,7 +1852,7 @@ describe('the tested flag and the word "Recommended"', () => { }); it('never says "Recommended" for a limited answer, tested or not', () => { - const catalog = stockWith((stock) => { + const catalog = rulesWith((stock) => { const row = stock.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; row.tested = [testedOn('Apple M4', 32, 'opencode')]; }); @@ -1723,7 +1871,7 @@ describe('what each row of the chooser carries', () => { it('is "Best fit" on an untested pick and the first kind that applies on the others', () => { // 48 GB, M4 Max: the 27B is the pick; the 35B-A3B is tight; the two 9B rows are small; // gpt-oss:20b is none of these and has no tag. - const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const result = recommend(mac(48, 'Apple M4 Max'), RULES, ALL_INSTALLED); expect(rows(result).map((c) => [c.modelId, c.tag ?? null])).toEqual([ ['qwen3.8:27b', { kind: 'best', label: 'Best fit' }], ['qwen3.6:35b-a3b', { kind: 'tight', label: 'Tight' }], @@ -1735,16 +1883,16 @@ describe('what each row of the chooser carries', () => { }); it('says "Best fit, tight" on a pick that only just fits: the words must not say less than the bar', () => { - const result = recommend(mac(48, 'Apple M4 Pro'), STOCK, ALL_INSTALLED); + const result = recommend(mac(48, 'Apple M4 Pro'), RULES, ALL_INSTALLED); expect(result.pick).toMatchObject({ fit: 'tight', tag: { kind: 'best-tight', label: 'Best fit, tight' } }); // A pick with room says "Best fit" and nothing more. - const roomy = recommend(mac(64, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const roomy = recommend(mac(64, 'Apple M4 Max'), RULES, ALL_INSTALLED); expect(roomy.pick).toMatchObject({ fit: 'good', tag: { kind: 'best', label: 'Best fit' } }); }); it('says "Tested on a Mac like this" in place of every other tag', () => { // The MacBook's pick is tight and the pick, and says neither. - expect(recommend(mac(48, 'Apple M5 Pro'), STOCK, ALL_INSTALLED).pick?.tag).toEqual({ + expect(recommend(mac(48, 'Apple M5 Pro'), RULES, ALL_INSTALLED).pick?.tag).toEqual({ kind: 'tested', label: 'Tested on a Mac like this', }); @@ -1782,8 +1930,8 @@ describe('what each row of the chooser carries', () => { it('takes every tag text from the fixed list', () => { const texts = new Set(Object.values(LOCAL_MODEL_TAG_LABELS)); - for (const hw of [...SWEEP, mac(48, 'Apple M5 Pro')]) { - for (const candidate of rows(recommend(hw, STOCK, ALL_INSTALLED))) { + for (const [hw, catalog] of SWEEP_BOTH_AND_MACBOOK) { + for (const candidate of rows(recommend(hw, catalog, ALL_INSTALLED))) { if (!candidate.tag) continue; expect(texts.has(candidate.tag.label)).toBe(true); expect(candidate.tag.label).toBe(LOCAL_MODEL_TAG_LABELS[candidate.tag.kind]); @@ -1792,8 +1940,8 @@ describe('what each row of the chooser carries', () => { }); it('gives exactly one row the pick tag, and it is the first', () => { - for (const hw of SWEEP) { - const result = recommend(hw, STOCK, ALL_INSTALLED); + for (const [hw, catalog] of SWEEP_BOTH) { + const result = recommend(hw, catalog, ALL_INSTALLED); if (!result.pick) continue; // The pick's tag is the pick's alone: "Best fit", with ", tight" when it only just fits. expect(result.pick.tag?.kind).toBe(result.pick.fit === 'tight' ? 'best-tight' : 'best'); @@ -1806,7 +1954,7 @@ describe('what each row of the chooser carries', () => { const hw = mac(48, 'Apple M4 Max'); it('lists every harness that can run the model at its context, best first, the flat one first', () => { - const result = recommend(hw, STOCK, ALL_INSTALLED); + const result = recommend(hw, RULES, ALL_INSTALLED); for (const candidate of rows(result)) { expect(candidate.harnesses).toEqual([ { cli: 'opencode', cliLabel: 'OpenCode', installed: true }, @@ -1827,12 +1975,12 @@ describe('what each row of the chooser carries', () => { // Installed: a command the caller knows is not shown. fact('claude', { installCommand: 'npm install -g something' }), ]; - expect(recommend(hw, STOCK, facts).pick?.harnesses).toEqual([ + expect(recommend(hw, RULES, facts).pick?.harnesses).toEqual([ { cli: 'opencode', cliLabel: 'OpenCode', installed: false, installCommand: command }, { cli: 'claude', cliLabel: 'Claude Code', installed: true, slowFirstTurn: true }, ]); // Installable without a command the caller could name: listed, without the field. - const bare = recommend(hw, STOCK, [fact('opencode', { installed: false, installable: true })]); + const bare = recommend(hw, RULES, [fact('opencode', { installed: false, installable: true })]); expect(bare.pick?.harnesses).toEqual([{ cli: 'opencode', cliLabel: 'OpenCode', installed: false }]); }); @@ -1841,14 +1989,14 @@ describe('what each row of the chooser carries', () => { fact('opencode', { acceptsModelId: (id: string) => id !== 'qwen3.8:27b' }), fact('claude', { installed: false, installable: false }), ]; - const result = recommend(hw, STOCK, facts); + const result = recommend(hw, RULES, facts); for (const candidate of rows(result)) expect(candidate.harnesses.map((h) => h.cli)).toEqual(['opencode']); expect(order(result)).not.toContain('qwen3.8:27b@131072'); }); it('is never empty', () => { - for (const swept of [...SWEEP, mac(48, 'Apple M5 Pro')]) { - for (const candidate of rows(recommend(swept, STOCK, ALL_INSTALLED))) { + for (const [swept, catalog] of SWEEP_BOTH_AND_MACBOOK) { + for (const candidate of rows(recommend(swept, catalog, ALL_INSTALLED))) { expect(candidate.harnesses.length).toBeGreaterThan(0); expect(new Set(candidate.harnesses.map((h) => h.cli)).size).toBe(candidate.harnesses.length); } @@ -1860,14 +2008,14 @@ describe('what each row of the chooser carries', () => { const hw = mac(48, 'Apple M4 Max'); it('is the model alone once the runtime is installed', () => { - for (const candidate of rows(recommend(hw, STOCK, ALL_INSTALLED))) { + for (const candidate of rows(recommend(hw, RULES, ALL_INSTALLED))) { expect(candidate.installBytes).toBe(candidate.downloadBytes); expect(candidate.installGb).toBe(candidate.downloadGb); } }); it('adds the runtime archive to every row while the runtime is not installed', () => { - const result = recommend(hw, STOCK, ALL_INSTALLED, { runtimeDownloadBytes: RUNTIME_ARCHIVE_BYTES }); + const result = recommend(hw, RULES, ALL_INSTALLED, { runtimeDownloadBytes: RUNTIME_ARCHIVE_BYTES }); for (const candidate of rows(result)) { expect(candidate.installBytes).toBe(candidate.downloadBytes + 185_971_671); expect(candidate.installGb).toBe(formatGb(candidate.installBytes)); @@ -1877,8 +2025,8 @@ describe('what each row of the chooser carries', () => { }); it('does not let it change which model is the pick, what fits, or the need', () => { - const plain = recommend(hw, STOCK, ALL_INSTALLED); - const withRuntime = recommend(hw, STOCK, ALL_INSTALLED, { runtimeDownloadBytes: RUNTIME_ARCHIVE_BYTES }); + const plain = recommend(hw, RULES, ALL_INSTALLED); + const withRuntime = recommend(hw, RULES, ALL_INSTALLED, { runtimeDownloadBytes: RUNTIME_ARCHIVE_BYTES }); const strip = (candidate: Candidate): Omit => { const { installBytes: _bytes, installGb: _gb, ...rest } = candidate; return rest; @@ -1888,14 +2036,14 @@ describe('what each row of the chooser carries', () => { it('ignores a runtime size that is no byte count', () => { for (const runtimeDownloadBytes of [0, -5, Number.NaN, Number.POSITIVE_INFINITY]) { - const pick = recommend(hw, STOCK, ALL_INSTALLED, { runtimeDownloadBytes }).pick as Candidate; + const pick = recommend(hw, RULES, ALL_INSTALLED, { runtimeDownloadBytes }).pick as Candidate; expect(pick.installBytes, String(runtimeDownloadBytes)).toBe(pick.downloadBytes); } }); }); it('says whether the need of a row rests on a measurement of that model', () => { - const result = recommend(mac(64, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const result = recommend(mac(64, 'Apple M4 Max'), RULES, ALL_INSTALLED); expect(Object.fromEntries(rows(result).map((c) => [c.modelId, c.needSource]))).toEqual({ 'qwen3.8:27b': 'estimated', 'qwen3.6:35b-a3b': 'measured', @@ -1915,8 +2063,8 @@ describe('the measured budget, end to end', () => { expect([measured.budgetSource, measured.budgetGb]).toEqual(['measured', '40.2']); // The same rows in the same order; only the share of each changes. expect(order(measured)).toEqual(order(estimated)); - expect(rows(estimated).map((c) => c.budgetShare)).toEqual([0.957, 0.555, 0.452, 0.709, 0.814]); - expect(rows(measured).map((c) => c.budgetShare)).toEqual([0.895, 0.52, 0.423, 0.663, 0.762]); + expect(rows(estimated).map((c) => c.budgetShare)).toEqual([0.957, 0.742, 0.653, 0.292, 0.292]); + expect(rows(measured).map((c) => c.budgetShare)).toEqual([0.895, 0.695, 0.611, 0.273, 0.273]); for (const candidate of rows(measured)) { expect(candidate.budgetShare).toBeCloseTo(candidate.needBytes / MACBOOK_MEASURED_TOTAL, 3); } @@ -1925,26 +2073,26 @@ describe('the measured budget, end to end', () => { it('can rule a model out that the estimate offered, and offer one the estimate ruled out', () => { const hw = mac(48, 'Apple M4 Pro'); // The estimate (35 GiB) offers the 35B-A3B at 131072 tokens (0.957). - expect(order(recommend(hw, STOCK, ALL_INSTALLED))[0]).toBe('qwen3.6:35b-a3b@131072'); - expect(modelFitsBudget(hw, STOCK, 'qwen3.6:35b-a3b', 131072)).toBe(true); + expect(order(recommend(hw, RULES, ALL_INSTALLED))[0]).toBe('qwen3.6:35b-a3b@131072'); + expect(modelFitsBudget(hw, RULES, 'qwen3.6:35b-a3b', 131072)).toBe(true); // A runtime that reports only 33 GiB rules that context out (1.015), so the install job // stops before the download, and the recommendation steps the model down to 65536 (0.946). const lower = { measuredTotalBytes: 33 * GIB }; - expect(modelFitsBudget(hw, STOCK, 'qwen3.6:35b-a3b', 131072, lower)).toBe(false); - expect(modelFitsBudget(hw, STOCK, 'qwen3.6:35b-a3b', 65536, lower)).toBe(true); - expect(recommend(hw, STOCK, ALL_INSTALLED, lower).pick).toMatchObject({ + expect(modelFitsBudget(hw, RULES, 'qwen3.6:35b-a3b', 131072, lower)).toBe(false); + expect(modelFitsBudget(hw, RULES, 'qwen3.6:35b-a3b', 65536, lower)).toBe(true); + expect(recommend(hw, RULES, ALL_INSTALLED, lower).pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', context: 65536, budgetShare: 0.946, }); // At 31 GiB it fits at no context and is not listed at all. const lowest = { measuredTotalBytes: 31 * GIB }; - expect(modelFitsBudget(hw, STOCK, 'qwen3.6:35b-a3b', 65536, lowest)).toBe(false); - expect(order(recommend(hw, STOCK, ALL_INSTALLED, lowest)).some((entry) => entry.startsWith('qwen3.6'))).toBe(false); + expect(modelFitsBudget(hw, RULES, 'qwen3.6:35b-a3b', 65536, lowest)).toBe(false); + expect(order(recommend(hw, RULES, ALL_INSTALLED, lowest)).some((entry) => entry.startsWith('qwen3.6'))).toBe(false); // On a 36 GB Mac the estimate (26 GiB) rules the 27B out at 65536 tokens; 27 GiB admits it. const small = mac(36, 'Apple M4 Max'); - expect(modelFitsBudget(small, STOCK, 'qwen3.8:27b', 65536)).toBe(false); - expect(modelFitsBudget(small, STOCK, 'qwen3.8:27b', 65536, { measuredTotalBytes: 27 * GIB })).toBe(true); + expect(modelFitsBudget(small, RULES, 'qwen3.8:27b', 65536)).toBe(false); + expect(modelFitsBudget(small, RULES, 'qwen3.8:27b', 65536, { measuredTotalBytes: 27 * GIB })).toBe(true); }); }); @@ -1952,8 +2100,8 @@ describe('the shape of the answer', () => { const labels = new Set(Object.values(LOCAL_MODEL_LABELS)); it('takes every sentence and label from the fixed list', () => { - for (const hw of [...SWEEP, mac(48, 'Apple M5 Pro')]) { - const result = recommend(hw, STOCK, ALL_INSTALLED); + for (const [hw, catalog] of SWEEP_BOTH_AND_MACBOOK) { + const result = recommend(hw, catalog, ALL_INSTALLED); expect(labels.has(result.headline.text)).toBe(true); for (const note of result.notes) expect(labels.has(note.text)).toBe(true); for (const candidate of rows(result)) { @@ -1967,11 +2115,11 @@ describe('the shape of the answer', () => { it('fills every placeholder of a headline and nothing else', () => { const answers = [ - recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED), - recommend(mac(48, 'Apple M4', { macosMajor: 13, macosVersion: '13.0' }), STOCK, ALL_INSTALLED), - recommend(mac(48, 'Apple M4', { freeDiskBytes: GB, freeDiskGb: '1.0' }), STOCK, ALL_INSTALLED), - recommend(mac(48, 'Apple M4'), STOCK, []), - recommend(mac(48, 'Apple M4'), STOCK, ALL_INSTALLED), + recommend(mac(8, 'Apple M2'), RULES, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { macosMajor: 13, macosVersion: '13.0' }), RULES, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { freeDiskBytes: GB, freeDiskGb: '1.0' }), RULES, ALL_INSTALLED), + recommend(mac(48, 'Apple M4'), RULES, []), + recommend(mac(48, 'Apple M4'), RULES, ALL_INSTALLED), ]; expect(answers.map((answer) => answer.reason ?? answer.verdict)).toEqual([ 'memory', @@ -1982,9 +2130,9 @@ describe('the shape of the answer', () => { ]); // The two other sentences of the `memory` reason, and an answer with every kind of note. answers.push( - recommend(mac(24, 'Apple M4'), STOCK, ALL_INSTALLED), - recommend({ ...mac(48, 'Apple M4'), memBytes: 0, memGb: '0.0', memClassGb: 0 }, STOCK, ALL_INSTALLED), - recommend(mac(48, 'Apple M4', { freeDiskBytes: 20 * GIB, freeDiskGb: '21.5' }), STOCK, ALL_INSTALLED) + recommend(mac(24, 'Apple M4'), RULES, ALL_INSTALLED), + recommend({ ...mac(48, 'Apple M4'), memBytes: 0, memGb: '0.0', memClassGb: 0 }, RULES, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { freeDiskBytes: 20 * GIB, freeDiskGb: '21.5' }), RULES, ALL_INSTALLED) ); expect(answers.slice(5).map((answer) => answer.headline.text)).toEqual([ LOCAL_MODEL_LABELS.noneMemoryContext, @@ -2008,8 +2156,8 @@ describe('the shape of the answer', () => { }); it('prints every size in one unit: decimal GB from the one formatter, never GiB', () => { - for (const hw of [...SWEEP, mac(48, 'Apple M5 Pro')]) { - const result = recommend(hw, STOCK, ALL_INSTALLED, { runtimeDownloadBytes: RUNTIME_ARCHIVE_BYTES }); + for (const [hw, catalog] of SWEEP_BOTH_AND_MACBOOK) { + const result = recommend(hw, catalog, ALL_INSTALLED, { runtimeDownloadBytes: RUNTIME_ARCHIVE_BYTES }); expect(result.budgetGb).toBe(formatGb(result.budgetBytes)); for (const candidate of rows(result)) { expect(candidate.needGb).toBe(formatGb(candidate.needBytes)); @@ -2022,7 +2170,7 @@ describe('the shape of the answer', () => { }); it('adds up for a person: download, plus what the context and the runner take, is the need', () => { - const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const result = recommend(mac(48, 'Apple M4 Max'), RULES, ALL_INSTALLED); const pick = result.pick as Candidate; // 17.7 GB of weights + 4.8 GB held for 131,072 tokens + 3 GB + 5 GB, all in decimal GB. expect(pick.downloadGb).toBe('17.7'); @@ -2033,12 +2181,12 @@ describe('the shape of the answer', () => { }); it('is plain JSON and has exactly the fields of its verdict', () => { - const ok = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const ok = recommend(mac(48, 'Apple M4 Max'), RULES, ALL_INSTALLED); expect(JSON.parse(JSON.stringify(ok))).toEqual(ok); expect(Object.keys(ok).sort()).toEqual( ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'pick', 'verdict'].sort() ); - const none = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); + const none = recommend(mac(8, 'Apple M2'), RULES, ALL_INSTALLED); expect(JSON.parse(JSON.stringify(none))).toEqual(none); expect(Object.keys(none).sort()).toEqual( ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'reason', 'verdict'].sort() @@ -2071,8 +2219,8 @@ describe('the shape of the answer', () => { 'tested', ]; const sometimes = ['slowFirstTurn', 'speedLabel', 'tag', 'tokensPerSecond']; - for (const hw of [...SWEEP, mac(48, 'Apple M5 Pro')]) { - for (const candidate of rows(recommend(hw, STOCK, ALL_INSTALLED))) { + for (const [hw, catalog] of SWEEP_BOTH_AND_MACBOOK) { + for (const candidate of rows(recommend(hw, catalog, ALL_INSTALLED))) { const keys = Object.keys(candidate); expect(keys.filter((key) => !sometimes.includes(key)).sort()).toEqual(always); for (const key of keys) expect((candidate as unknown as Record)[key]).not.toBeUndefined(); @@ -2082,7 +2230,7 @@ describe('the shape of the answer', () => { it('is pure: frozen inputs are enough, and the same inputs give the same answer', () => { const hw = deepFreeze(mac(48, 'Apple M4 Pro')); - const catalog = deepFreeze(structuredClone(STOCK_LOCAL_MODEL_CATALOG)); + const catalog = deepFreeze(structuredClone(RULES_LOCAL_MODEL_CATALOG)); const facts = deepFreeze(ALL_INSTALLED.map((entry) => ({ ...entry }))); const options = deepFreeze({ measuredTotalBytes: MACBOOK_MEASURED_TOTAL, runtimeDownloadBytes: 1 }); const first = recommend(hw, catalog, facts, options);