feat(local-model): a catalogue for every Mac from 8 to 512 GB, each row with its own figures

The catalogue had seven Qwen-family rows that all carried the memory
figures measured on one model (Qwen3.6 35B-A3B on the MLX runner), so a
9B or a dense 27B was judged with figures that do not describe it, two
rows had the wrong Mac download size, and every Mac below 32 GB got
"none".

Now eight rows, each with the runner a Mac really uses for its tag (a new
`runner` field: `mlx` or `llamacpp`) and memory figures of its own:

- the 35B-A3B MLX row keeps its measured figures and the tested pair;
- every llama.cpp row is computed from its model's config.json: a q8_0
  K/V cache for the whole context (what the runtime now sets), its
  recurrent or sliding-window state, 1.0 GB of runner and the 1 GiB host
  prompt cache;
- the one other MLX row (Ornith 1.5 122B) is computed the same way,
  scaled by what the measured row showed, plus the runner's 8 GiB of
  prefix snapshots.

Sizes are the registry's Mac pull totals (2026-11-08, lowercase ids
verified there). What each memory size gets: 8 GB none; 16 to 32 GB a 9B
(limited); 36 GB Qwen3.8 27B or the 35B-A3B as GGUF; 48 GB the tested
pair (or the 27B on a Max chip); 64 and 96 GB the 27B or the 35B-A3B;
128 to 512 GB Ornith 1.5 122B. Nothing but the one row was measured.

The catalogue test recomputes every computed figure from a table of
architectures and lets two rows share figures only when they share the
architecture and the runner. The rule tests of recommend() now run on a
frozen copy of the 2026-11-06 catalogue, so a weekly catalogue update
changes only the per-memory table and the real-Mac tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codeman maintainer committed 2026-11-08 09:33:48 +01:00
1 parent f60b63c005
commit 989fcd778a
7 files changed
+1176 -465

No files matched your search

+182
View File
@@ -0,0 +1,182 @@
/**
* @fileoverview A frozen model catalogue for the RULE tests of src/local-model/recommend.ts:
* the stock catalogue exactly as it shipped on 2026-11-06 (seven rows that all carry the
* figures measured on one model), with the `runner` field the schema gained since.
*
* Why a frozen copy and not the stock catalogue: the stock catalogue is meant to change every
* week or so (new models, new measurements). The tests of a RULE (the ranking, the disk
* filter, the tags, the `limited` line, the harness fallback) need realistic rows whose
* answers do not move with it, or every catalogue update would rewrite a hundred rule tests
* and hide the few answers that really changed. What the STOCK catalogue answers for each Mac
* is pinned separately, in test/local-model-recommend.test.ts ("every memory size") and the
* two real-Mac blocks, and that is the table a catalogue change must update on purpose.
*
* Its figures are NOT a statement about any model: the rows borrow the measured row's figures,
* which is exactly why the stock catalogue replaced them. Validated by the real schema.
*/
import type { LocalModelCatalog } from '../../src/types.js';
const MEASURED_MEMORY = {
/** 36,000 to 38,000 bytes stayed held per token of context after a prompt of 89,000 and one of 167,000 tokens. */
heldBytesPerToken: 37_000,
/** The runner held 2.5 to 3 GB beside the weights. */
overheadBytes: 3_000_000_000,
/** While a long prompt was processed the peak stood about 5 GB above what stayed held. */
peakExtraBytes: 5_000_000_000,
} as const;
/**
* The catalogue as it shipped on 2026-11-06, with the `runner` field added. Frozen: a fixture,
* not the stock catalogue.
*/
export const RULES_LOCAL_MODEL_CATALOG: LocalModelCatalog = {
schema: 1,
updated: '2026-11-06',
// The oldest macOS the model runtime supports.
minMacosMajor: 14,
// Ordered by rank. The higher rank wins among models that fit equally well and are equally
// fast. `flint:40b` is left out on purpose: its plain tag is the slow build on a Mac and it
// has an open bug with tool arguments.
models: [
{
id: 'qwen3.5:9b',
label: 'Qwen3.5 9B',
license: 'Apache-2.0',
paramsB: 9,
runner: 'mlx',
bytes: 6_550_825_373,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [65536, 32768],
rank: 1,
// 37.1 tokens per second measured on 400 GB/s.
speedPer100Gbps: 9.3,
tested: [],
},
{
id: 'qwen3.5:9b-q8_0',
label: 'Qwen3.5 9B (Q8)',
license: 'Apache-2.0',
paramsB: 9,
runner: 'llamacpp',
bytes: 10_449_217_949,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [65536, 32768],
rank: 2,
// Not measured: the 9.3 of the row above scaled by file size.
speedPer100Gbps: 5.8,
tested: [],
},
{
id: 'gpt-oss:20b',
label: 'GPT-OSS 20B',
license: 'Apache-2.0',
paramsB: 20,
runner: 'llamacpp',
bytes: 13_793_441_427,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [131072, 65536],
rank: 3,
// No speed figure: its speed is unknown on every chip.
tested: [],
},
{
id: 'qwen3.6:35b-a3b',
label: 'Qwen3.6 35B-A3B',
license: 'Apache-2.0',
paramsB: 35,
runner: 'mlx',
// What a Mac really downloads: an MLX build, 2% above the 22,621,314,161 of the manifest.
bytes: 23_105_798_956,
...MEASURED_MEMORY,
figures: 'measured',
contexts: [131072, 65536],
rank: 4,
// 54.5 tokens per second measured on 400 GB/s (3 billion parameters are active per token).
// The MLX build ran faster than this figure predicts on the one Mac we tested (67.7
// against 41.8), so on other chips the estimate is on the low side.
speedPer100Gbps: 13.6,
// Run by us with OpenCode on the MacBook Pro (Mac17,13): a four-step task with tool calls.
tested: [{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 }],
},
{
id: 'qwen3.8:27b',
label: 'Qwen3.8 27B',
license: 'Apache-2.0',
paramsB: 27.8,
runner: 'mlx',
bytes: 17_741_871_939,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [131072, 65536, 32768],
rank: 5,
// 11.8 tokens per second measured on 400 GB/s.
speedPer100Gbps: 2.95,
tested: [],
},
{
id: 'qwen3.8:27b-q8_0',
label: 'Qwen3.8 27B (Q8)',
license: 'Apache-2.0',
paramsB: 27.8,
runner: 'llamacpp',
bytes: 29_978_241_837,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [262144, 131072],
rank: 6,
// Not measured: the 2.95 of the row above scaled by file size.
speedPer100Gbps: 1.7,
tested: [],
},
{
id: 'qwen3.5:122b-a10b',
label: 'Qwen3.5 122B-A10B',
license: 'Apache-2.0',
paramsB: 122,
runner: 'llamacpp',
bytes: 81_370_035_877,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [131072],
rank: 7,
// No speed figure: its speed is unknown on every chip.
tested: [],
},
],
// The higher rank wins when a model can run with more than one harness.
harnesses: [
{ cli: 'opencode', minContext: 65536, rank: 3 },
// Sends a fixed prompt of about 36,000 tokens, so the first answer takes minutes.
{ cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true },
// Gated: neither its 32768 minimum nor its handling of a `name:tag` model id is verified.
{ cli: 'pi', minContext: 32768, rank: 1, gated: true },
],
// Memory bandwidth in GB/s, most specific first. Only chips that shipped and are in the source
// table are listed. Where one name covers two bandwidths (M3 Max 300 or 400, M4 Max 410 or 546,
// M5 Max 460 or 614) the LOWER one stands, so a speed computed from it is a floor.
chips: [
{ match: 'M5 Ultra', gbps: 1228 },
{ match: 'M3 Ultra', gbps: 800 },
{ match: 'M2 Ultra', gbps: 800 },
{ match: 'M1 Ultra', gbps: 800 },
{ match: 'M5 Max', gbps: 460 },
{ match: 'M4 Max', gbps: 410 },
{ match: 'M3 Max', gbps: 300 },
{ match: 'M2 Max', gbps: 400 },
{ match: 'M1 Max', gbps: 400 },
{ match: 'M5 Pro', gbps: 307 },
{ match: 'M4 Pro', gbps: 273 },
{ match: 'M3 Pro', gbps: 150 },
{ match: 'M2 Pro', gbps: 200 },
{ match: 'M1 Pro', gbps: 200 },
{ match: 'M5', gbps: 154 },
{ match: 'M4', gbps: 120 },
{ match: 'M3', gbps: 100 },
{ match: 'M2', gbps: 100 },
{ match: 'M1', gbps: 68 },
],
};
+270 -63
View File
@@ -12,9 +12,12 @@
* hides a more specific one).
* 3. The chip lookup never guesses: a chip that no row names has no bandwidth, including one
* whose name merely contains a known name ('Apple M10', 'Apple M6 Pro').
* 4. Which figures were MEASURED on a Mac and which are carried over from the measured row:
* one row says `measured` (its size is what a Mac really downloads, and it alone has a
* tested entry), every other row says `estimated`.
* 4. Which figures were MEASURED on a Mac and which were computed: one row says `measured`
* (it alone has a tested entry), every other row says `estimated`, and every estimated
* row's three memory figures are recomputed here from its OWN model's architecture (a
* table read from each model's config.json) for its runner and the runtime's settings. Two
* rows may share figures only when they share the architecture and the runner, so a row
* can never again carry another model's measurement without saying so.
*
* Port: N/A (pure data and functions).
*/
@@ -28,6 +31,7 @@ import {
LocalModelCatalogSchema,
STOCK_LOCAL_MODEL_CATALOG,
} from '../src/config/local-model-catalog.js';
import { RUNTIME_HOST_PROMPT_CACHE_MIB, RUNTIME_KV_CACHE_TYPE, runtimeEnv } from '../src/local-model/runtime.js';
import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js';
/** A deep copy of the stock catalogue that a test may break. */
@@ -48,7 +52,101 @@ function withFirstModel(change: Partial<Record<keyof LocalModelEntry | 'url' | '
return { ...catalog, models: [{ ...catalog.models[0], ...change }, ...catalog.models.slice(1)] };
}
/** One MiB and one GiB, in bytes. */
const MIB = 1024 * 1024;
const GIB = 1024 * MIB;
/**
* The architecture of every model the stock catalogue lists, read from its config.json on
* Hugging Face (2026-11-08): full-attention layers, their K/V heads and head size, and the
* layers whose cache does not grow with the context (Gated DeltaNet layers with their value
* and key heads, or sliding-window layers with their window). Fine-tunes share their base's.
*/
const ARCHITECTURES = {
'Qwen3.5-9B': { attentionLayers: 8, kvHeads: 4, headDim: 256, linear: { layers: 24, vHeads: 32, kHeads: 16 } },
'Qwen3.8-27B': { attentionLayers: 16, kvHeads: 4, headDim: 256, linear: { layers: 48, vHeads: 48, kHeads: 16 } },
'Qwen3.6-35B-A3B': { attentionLayers: 10, kvHeads: 2, headDim: 256, linear: { layers: 30, vHeads: 32, kHeads: 16 } },
'Qwen3.5-122B-A10B': {
attentionLayers: 12,
kvHeads: 2,
headDim: 256,
linear: { layers: 36, vHeads: 64, kHeads: 16 },
},
'North-Mini-Code-1.0': { attentionLayers: 13, kvHeads: 4, headDim: 128, sliding: { layers: 36, window: 4096 } },
} as const;
type ArchitectureName = keyof typeof ARCHITECTURES;
type Architecture = (typeof ARCHITECTURES)[ArchitectureName];
/**
* Which architecture each stock row runs, and whether its GGUF carries the one
* multi-token-prediction layer (counted as one more attention layer, see the catalogue
* header). A row the catalogue gains must be added here, or the test below fails: its figures
* cannot be checked without it.
*/
const ROW_ARCHITECTURE: Record<string, { architecture: ArchitectureName; predictionLayer: boolean }> = {
'qwen3.5:9b-q4_k_m': { architecture: 'Qwen3.5-9B', predictionLayer: false },
'mimo-v2.6:9b-q4_k_m': { architecture: 'Qwen3.5-9B', predictionLayer: false },
'north-mini-code-1.0:q4_k_m': { architecture: 'North-Mini-Code-1.0', predictionLayer: false },
'qwen3.6:35b-a3b-mtp-q4_k_m': { architecture: 'Qwen3.6-35B-A3B', predictionLayer: true },
'qwen3.6:35b-a3b': { architecture: 'Qwen3.6-35B-A3B', predictionLayer: false },
'qwen3.8:27b-q8_0': { architecture: 'Qwen3.8-27B', predictionLayer: true },
'qwen3.8:27b-mtp-q4_k_m': { architecture: 'Qwen3.8-27B', predictionLayer: true },
'ornith-1.5:122b': { architecture: 'Qwen3.5-122B-A10B', predictionLayer: false },
};
/** Bytes per cached value in each K/V cache type of llama.cpp: q8_0 keeps 32 values in 34 bytes. */
const KV_BYTES_PER_VALUE: Record<string, number> = { f16: 2, q8_0: 34 / 32 };
/** The head size of a Gated DeltaNet layer's keys and values, and its convolution kernel, in every model above. */
const LINEAR_HEAD_DIM = 128;
const LINEAR_CONV_KERNEL = 4;
/** The recurrent state of the linear-attention layers in bytes: the convolution tail and the delta state. */
function recurrentStateBytes(arch: Architecture, convBytes: number): number {
if (!('linear' in arch)) return 0;
const { layers, vHeads, kHeads } = arch.linear;
const convDim = vHeads * LINEAR_HEAD_DIM + 2 * kHeads * LINEAR_HEAD_DIM;
const conv = (LINEAR_CONV_KERNEL - 1) * convDim * convBytes;
const delta = vHeads * LINEAR_HEAD_DIM * LINEAR_HEAD_DIM * 4;
return layers * (conv + delta);
}
/** What a `llamacpp` row's three figures must be, from its architecture and the runtime's settings. */
function llamaCppFigures(arch: Architecture, predictionLayer: boolean): [number, number, number] {
const perValue = KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE];
const layers = arch.attentionLayers + (predictionLayer ? 1 : 0);
const held = layers * arch.kvHeads * arch.headDim * 2 * perValue;
// llama.cpp keeps the recurrent state in f32 (both parts), the sliding window in the cache type.
const sliding =
'sliding' in arch
? arch.sliding.layers * (arch.sliding.window + 512) * arch.kvHeads * arch.headDim * 2 * perValue
: 0;
const fixed = recurrentStateBytes(arch, 4) + sliding;
return [held, fixed + 1_000_000_000 + RUNTIME_HOST_PROMPT_CACHE_MIB * MIB, 0];
}
/** The per-token figure of the MLX runner computed from an architecture: K/V in bf16, a recurrent snapshot per 8,192 tokens. */
function mlxComputedPerToken(arch: Architecture): number {
return arch.attentionLayers * arch.kvHeads * arch.headDim * 2 * 2 + recurrentStateBytes(arch, 2) / 8192;
}
/** The figures measured for `qwen3.6:35b-a3b` on the MacBook Pro. */
const MEASURED: [number, number, number] = [37_000, 3_000_000_000, 5_000_000_000];
/** What an estimated `mlx` row's three figures must be: computed, scaled by what the measured row showed. */
function mlxFigures(arch: Architecture): [number, number, number] {
const ratio = MEASURED[0] / mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B']);
return [Math.ceil(mlxComputedPerToken(arch) * ratio), MEASURED[1] + 8 * GIB, MEASURED[2]];
}
/** A row's three memory figures. */
function figuresOf(model: LocalModelEntry): [number, number, number] {
return [model.heldBytesPerToken, model.overheadBytes, model.peakExtraBytes];
}
describe('the stock catalogue', () => {
const models = STOCK_LOCAL_MODEL_CATALOG.models;
it('passes its own schema', () => {
expect(issuesOf(STOCK_LOCAL_MODEL_CATALOG)).toEqual([]);
});
@@ -60,94 +158,114 @@ describe('the stock catalogue', () => {
expect(getLocalModelCatalog()).toBe(first);
});
it('lists the seven models of the design, without flint:40b', () => {
expect(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => m.id)).toEqual([
'qwen3.5:9b',
'qwen3.5:9b-q8_0',
'gpt-oss:20b',
it('lists eight models, by rank, and none of those set aside in the header', () => {
expect(models.map((m) => m.id)).toEqual([
'qwen3.5:9b-q4_k_m',
'mimo-v2.6:9b-q4_k_m',
'north-mini-code-1.0:q4_k_m',
'qwen3.6:35b-a3b-mtp-q4_k_m',
'qwen3.6:35b-a3b',
'qwen3.8:27b',
'qwen3.8:27b-q8_0',
'qwen3.5:122b-a10b',
'qwen3.8:27b-mtp-q4_k_m',
'ornith-1.5:122b',
]);
expect(STOCK_LOCAL_MODEL_CATALOG.models.some((m) => m.id.startsWith('flint'))).toBe(false);
expect(models.map((m) => m.rank)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
const setAside =
/^(bonsai-2|gpt-oss|flint|lfm2\.5|laguna|qwen3\.8-flash-next|qwen3\.5:122b)|^ornith-1\.5:(9b|35b|397b)/;
expect(models.filter((m) => setAside.test(m.id)).map((m) => m.id)).toEqual([]);
// The plain 9B and 27B tags are MLX builds on a Mac: their GGUF builds are listed instead.
for (const plain of ['qwen3.5:9b', 'mimo-v2.6:9b', 'qwen3.8:27b']) {
expect(models.some((m) => m.id === plain)).toBe(false);
}
});
it('carries the size a Mac really downloads where that was measured, else the registry manifest', () => {
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.bytes]))).toEqual({
// The sum of the layer sizes of each tag's manifest, as fetched on 2026-11-06.
'qwen3.5:9b': 6_550_825_373,
'qwen3.5:9b-q8_0': 10_449_217_949,
'gpt-oss:20b': 13_793_441_427,
// The total of the real pull on a Mac (an MLX build). Its manifest sums to 22,621,314,161.
it('names the runner a Mac uses for each tag', () => {
expect(Object.fromEntries(models.map((m) => [m.id, m.runner]))).toEqual({
'qwen3.5:9b-q4_k_m': 'llamacpp',
'mimo-v2.6:9b-q4_k_m': 'llamacpp',
'north-mini-code-1.0:q4_k_m': 'llamacpp',
'qwen3.6:35b-a3b-mtp-q4_k_m': 'llamacpp',
// Manifest lists: a Mac takes the MLX child.
'qwen3.6:35b-a3b': 'mlx',
'qwen3.8:27b-q8_0': 'llamacpp',
'qwen3.8:27b-mtp-q4_k_m': 'llamacpp',
'ornith-1.5:122b': 'mlx',
});
});
it('carries the size of the build a Mac pulls: every layer plus the config', () => {
expect(Object.fromEntries(models.map((m) => [m.id, m.bytes]))).toEqual({
// The registry's manifests as fetched on 2026-11-08, in lowercase.
'qwen3.5:9b-q4_k_m': 6_550_825_550,
'mimo-v2.6:9b-q4_k_m': 6_550_811_033,
'north-mini-code-1.0:q4_k_m': 18_593_967_008,
'qwen3.6:35b-a3b-mtp-q4_k_m': 22_621_314_381,
// The MLX child of the list, and the total of a real pull on a Mac, byte for byte.
'qwen3.6:35b-a3b': 23_105_798_956,
'qwen3.8:27b': 17_741_871_939,
'qwen3.8:27b-q8_0': 29_978_241_837,
'qwen3.5:122b-a10b': 81_370_035_877,
'qwen3.8:27b-q8_0': 29_978_242_050,
'qwen3.8:27b-mtp-q4_k_m': 17_741_872_154,
// The MLX child of the list (its GGUF child is 76,756,174,997).
'ornith-1.5:122b': 73_388_069_552,
});
});
it('says which row was measured on a Mac: one, and every other row is an estimate', () => {
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.figures]))).toEqual({
'qwen3.5:9b': 'estimated',
'qwen3.5:9b-q8_0': 'estimated',
'gpt-oss:20b': 'estimated',
'qwen3.6:35b-a3b': 'measured',
'qwen3.8:27b': 'estimated',
'qwen3.8:27b-q8_0': 'estimated',
'qwen3.5:122b-a10b': 'estimated',
});
});
it('gives every row the memory figures measured on the one row: 37 KB per token, 3 GB, 5 GB', () => {
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
expect(
[model.heldBytesPerToken, model.overheadBytes, model.peakExtraBytes],
`${model.id}: an unmeasured row carries the measured row's figures, not invented ones`
).toEqual([37_000, 3_000_000_000, 5_000_000_000]);
}
// No row keeps the figure that was computed from the model's config.
expect(JSON.stringify(STOCK_LOCAL_MODEL_CATALOG)).not.toContain('kvBytesPerToken');
expect(models.filter((m) => m.figures === 'measured').map((m) => m.id)).toEqual(['qwen3.6:35b-a3b']);
expect(figuresOf(models.find((m) => m.id === 'qwen3.6:35b-a3b') as LocalModelEntry)).toEqual(MEASURED);
});
it('has one tested entry: the 35B-A3B with OpenCode on the MacBook Pro, with its measured speed', () => {
expect(
Object.fromEntries(
STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.tested.length > 0).map((m) => [m.id, m.tested])
)
).toEqual({
expect(Object.fromEntries(models.filter((m) => m.tested.length > 0).map((m) => [m.id, m.tested]))).toEqual({
'qwen3.6:35b-a3b': [
{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 },
],
});
// A row may claim a test only when its figures were measured.
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
for (const model of models) {
if (model.tested.length > 0) expect(model.figures, model.id).toBe('measured');
}
});
it('gives every model a parameter count, an id of the default registry and a license', () => {
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
expect(model.paramsB, model.id).toBeGreaterThan(0);
expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN);
expect(model.license).toBe('Apache-2.0');
}
expect(Object.fromEntries(models.map((m) => [m.id, [m.paramsB, m.license]]))).toEqual({
'qwen3.5:9b-q4_k_m': [9, 'Apache-2.0'],
'mimo-v2.6:9b-q4_k_m': [9, 'MIT'],
'north-mini-code-1.0:q4_k_m': [30, 'Apache-2.0'],
'qwen3.6:35b-a3b-mtp-q4_k_m': [35, 'Apache-2.0'],
'qwen3.6:35b-a3b': [35, 'Apache-2.0'],
'qwen3.8:27b-q8_0': [27.8, 'Apache-2.0'],
'qwen3.8:27b-mtp-q4_k_m': [27.8, 'Apache-2.0'],
'ornith-1.5:122b': [122, 'MIT'],
});
for (const model of models) expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN);
// The two 9B rows are the ones under the 12 billion line of the `limited` verdict.
expect(STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual([
'qwen3.5:9b',
'qwen3.5:9b-q8_0',
]);
expect(models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual(['qwen3.5:9b-q4_k_m', 'mimo-v2.6:9b-q4_k_m']);
});
it('offers no model below the 65536 tokens a harness needs, nor beyond its own window', () => {
for (const model of models) {
expect(Math.min(...model.contexts), model.id).toBeGreaterThanOrEqual(65536);
expect(model.contexts[0], model.id).toBeLessThanOrEqual(262144);
}
});
it('has a speed figure only where one was measured or scaled from a measurement', () => {
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({
'qwen3.5:9b': 9.3,
'qwen3.5:9b-q8_0': 5.8,
'gpt-oss:20b': null,
'qwen3.6:35b-a3b': 13.6,
'qwen3.8:27b': 2.95,
expect(Object.fromEntries(models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({
// 37.1 tokens per second on 400 GB/s, and the same for the fine-tune of the same size.
'qwen3.5:9b-q4_k_m': 9.3,
'mimo-v2.6:9b-q4_k_m': 9.3,
// The 35B-A3B's llama.cpp figure: both have 3 billion active parameters.
'north-mini-code-1.0:q4_k_m': 13.6,
// 54.5 on 400 GB/s with llama.cpp.
'qwen3.6:35b-a3b-mtp-q4_k_m': 13.6,
// 67.8 on 400 GB/s with MLX.
'qwen3.6:35b-a3b': 16.9,
// 2.95 scaled by file size.
'qwen3.8:27b-q8_0': 1.7,
'qwen3.5:122b-a10b': null,
// 11.8 on 400 GB/s with llama.cpp, without prediction.
'qwen3.8:27b-mtp-q4_k_m': 2.95,
// 16.9 x 3/10 active parameters, rounded down.
'ornith-1.5:122b': 5,
});
});
@@ -174,6 +292,82 @@ describe('the stock catalogue', () => {
});
});
describe('the memory figures of the stock rows: each row has its own', () => {
const models = STOCK_LOCAL_MODEL_CATALOG.models;
it('knows the architecture of every row, and every row of that table is in the catalogue', () => {
expect(models.map((m) => m.id).sort()).toEqual(Object.keys(ROW_ARCHITECTURE).sort());
});
it('computes the llamacpp rows for the K/V cache type and the prompt cache the runtime really sets', () => {
// The figures below assume these; runtimeEnv() is what the runtime gets.
const env = runtimeEnv({ source: {}, home: '/h', modelsDir: '/m', port: 11435, contextLength: 65536 });
expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE);
expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB));
expect(KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE]).toBeDefined();
});
it("recomputes every estimated row's figures from its own model's architecture and runner", () => {
for (const model of models.filter((m) => m.figures === 'estimated')) {
const { architecture, predictionLayer } = ROW_ARCHITECTURE[model.id];
const arch = ARCHITECTURES[architecture];
const expected = model.runner === 'llamacpp' ? llamaCppFigures(arch, predictionLayer) : mlxFigures(arch);
expect(figuresOf(model), `${model.id} (${architecture}, ${model.runner})`).toEqual(expected);
}
});
it('states the numbers the header and the row comments name', () => {
expect(Object.fromEntries(models.map((m) => [m.id, figuresOf(m)]))).toEqual({
'qwen3.5:9b-q4_k_m': [17_408, 2_126_432_768, 0],
'mimo-v2.6:9b-q4_k_m': [17_408, 2_126_432_768, 0],
'north-mini-code-1.0:q4_k_m': [14_144, 2_254_227_968, 0],
'qwen3.6:35b-a3b-mtp-q4_k_m': [11_968, 2_139_605_504, 0],
'qwen3.6:35b-a3b': [37_000, 3_000_000_000, 5_000_000_000],
'qwen3.8:27b-q8_0': [36_992, 2_230_635_008, 0],
'qwen3.8:27b-mtp-q4_k_m': [36_992, 2_230_635_008, 0],
'ornith-1.5:122b': [56_574, 11_589_934_592, 5_000_000_000],
});
// The MLX calibration: 37,000 held against 28,340 computed for the measured row.
expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B'])).toBe(28_340);
expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.5-122B-A10B'])).toBe(43_332);
});
it('lets two rows share figures only when they share the architecture, the prediction layer and the runner', () => {
const byFigures = new Map<string, LocalModelEntry[]>();
for (const model of models) {
const key = figuresOf(model).join('/');
byFigures.set(key, [...(byFigures.get(key) ?? []), model]);
}
const shared = [...byFigures.values()].filter((group) => group.length > 1);
// The two 9B rows (a fine-tune and its base) and the two 27B rows (two quantizations of one model).
expect(shared.map((group) => group.map((m) => m.id))).toEqual([
['qwen3.5:9b-q4_k_m', 'mimo-v2.6:9b-q4_k_m'],
['qwen3.8:27b-q8_0', 'qwen3.8:27b-mtp-q4_k_m'],
]);
for (const group of shared) {
const identity = (m: LocalModelEntry): string => JSON.stringify([ROW_ARCHITECTURE[m.id], m.runner, m.figures]);
expect(new Set(group.map(identity)).size, group.map((m) => m.id).join(', ')).toBe(1);
}
});
it("gives no estimated row the measured row's per-token or overhead figure", () => {
for (const model of models.filter((m) => m.figures === 'estimated')) {
expect(model.heldBytesPerToken, model.id).not.toBe(MEASURED[0]);
expect(model.overheadBytes, model.id).not.toBe(MEASURED[1]);
expect(figuresOf(model), model.id).not.toEqual(MEASURED);
}
});
it('gives a llamacpp row no peak (its buffers are allocated at load) and an estimated mlx row the whole prefix budget', () => {
for (const model of models) {
if (model.runner === 'llamacpp') expect(model.peakExtraBytes, model.id).toBe(0);
if (model.runner === 'mlx' && model.figures === 'estimated') {
expect(model.overheadBytes - MEASURED[1], model.id).toBe(8 * GIB);
}
}
});
});
describe('the model id rule', () => {
it('accepts name:tag in lowercase', () => {
for (const id of ['qwen3.5:9b', 'gpt-oss:20b', 'qwen3.8:27b-q8_0', 'a:b', 'x_y.z-1:2.3_4']) {
@@ -240,6 +434,19 @@ describe('the schema, field by field', () => {
expect(issuesOf(withFirstModel({ peakExtraBytes: 1.5 }))).not.toEqual([]);
});
it('requires the runner on every model row, and knows two', () => {
const row: Record<string, unknown> = { ...copy().models[0] };
delete row.runner;
const issues = issuesOf({ ...copy(), models: [row] });
expect(issues).toHaveLength(1);
expect(issues[0]).toContain('models.0.runner');
expect(issuesOf(withFirstModel({ runner: 'mlx' }))).toEqual([]);
expect(issuesOf(withFirstModel({ runner: 'llamacpp' }))).toEqual([]);
for (const runner of ['ggml', 'MLX', 'llama.cpp', '']) {
expect(issuesOf(withFirstModel({ runner })), runner).not.toEqual([]);
}
});
it('requires paramsB and tested on every model row', () => {
for (const field of ['paramsB', 'tested'] as const) {
const row: Record<string, unknown> = { ...copy().models[0] };
+11 -9
View File
@@ -51,7 +51,7 @@ describe('modelFitsBudget', () => {
const catalog = getLocalModelCatalog();
it('agrees with the recommendation on every model and context of the catalogue', () => {
for (const gb of [8, 16, 24, 32, 48, 64, 128]) {
for (const gb of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192, 256, 512]) {
const hw = mac(gb);
const offered = recommend(hw, catalog, harnessFacts());
const rows = [offered.pick, ...offered.alternatives].filter((row) => row !== undefined);
@@ -85,7 +85,7 @@ describe('modelFitsBudget', () => {
});
it('rules out a model that is too large, and one the catalogue does not have', () => {
expect(modelFitsBudget(mac(48), catalog, 'qwen3.5:122b-a10b', 131072)).toBe(false);
expect(modelFitsBudget(mac(48), catalog, 'ornith-1.5:122b', 131072)).toBe(false);
expect(modelFitsBudget(mac(48), catalog, 'no-such:1b', 65536)).toBe(false);
expect(modelFitsBudget(mac(48), catalog, 'qwen3.6:35b-a3b', 131072)).toBe(true);
});
@@ -96,14 +96,16 @@ describe('modelFitsBudget', () => {
expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 40_157_944_217 })).toBe(true);
// A runtime that reports less than the estimate rules out what the estimate allowed.
expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 33 * 1024 ** 3 })).toBe(false);
// And one that reports more admits what the estimate ruled out: the 27B on a 36 GB Mac.
expect(modelFitsBudget(mac(36), catalog, 'qwen3.8:27b', 65536)).toBe(false);
expect(modelFitsBudget(mac(36), catalog, 'qwen3.8:27b', 65536, { measuredTotalBytes: 27 * 1024 ** 3 })).toBe(true);
// And one that reports more admits what the estimate ruled out: on a 36 GB Mac the GGUF
// 35B-A3B at its full window needs 27.9 GB, 1.0003 of the 27.9 GB estimate and 0.962 of 27 GiB.
const moeGguf = 'qwen3.6:35b-a3b-mtp-q4_k_m';
expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144)).toBe(false);
expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144, { measuredTotalBytes: 27 * 1024 ** 3 })).toBe(true);
// The reserve still bounds it: 100 GiB "measured" on a 48 GB Mac is 40 GiB (42.9 GB) of
// budget, so the Q8 27B with its 42.8 GB stays out (0.997) although 100 GiB would hold it.
// budget, so the 122B with its 97.4 GB stays out although 100 GiB would hold it.
const huge = { measuredTotalBytes: 100 * 1024 ** 3 };
expect(modelFitsBudget(mac(48), catalog, 'qwen3.8:27b-q8_0', 131072, huge)).toBe(false);
expect(modelFitsBudget(mac(128), catalog, 'qwen3.8:27b-q8_0', 131072, huge)).toBe(true);
expect(modelFitsBudget(mac(48), catalog, 'ornith-1.5:122b', 131072, huge)).toBe(false);
expect(modelFitsBudget(mac(128), catalog, 'ornith-1.5:122b', 131072, huge)).toBe(true);
});
it('agrees with the recommendation under a measured budget too', () => {
@@ -130,7 +132,7 @@ describe('modelFitsBudget', () => {
it('is false for a context that is no token count', () => {
for (const context of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
expect(modelFitsBudget(mac(128), catalog, 'qwen3.5:9b', context), String(context)).toBe(false);
expect(modelFitsBudget(mac(128), catalog, 'qwen3.5:9b-q4_k_m', context), String(context)).toBe(false);
}
});
});
File diff suppressed because it is too large. Load diff