mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-17 19:59:33 +01:00
The catalogue had seven Qwen-family rows that all carried the memory figures measured on one model (Qwen3.6 35B-A3B on the MLX runner), so a 9B or a dense 27B was judged with figures that do not describe it, two rows had the wrong Mac download size, and every Mac below 32 GB got "none". Now eight rows, each with the runner a Mac really uses for its tag (a new `runner` field: `mlx` or `llamacpp`) and memory figures of its own: - the 35B-A3B MLX row keeps its measured figures and the tested pair; - every llama.cpp row is computed from its model's config.json: a q8_0 K/V cache for the whole context (what the runtime now sets), its recurrent or sliding-window state, 1.0 GB of runner and the 1 GiB host prompt cache; - the one other MLX row (Ornith 1.5 122B) is computed the same way, scaled by what the measured row showed, plus the runner's 8 GiB of prefix snapshots. Sizes are the registry's Mac pull totals (2026-11-08, lowercase ids verified there). What each memory size gets: 8 GB none; 16 to 32 GB a 9B (limited); 36 GB Qwen3.8 27B or the 35B-A3B as GGUF; 48 GB the tested pair (or the 27B on a Max chip); 64 and 96 GB the 27B or the 35B-A3B; 128 to 512 GB Ornith 1.5 122B. Nothing but the one row was measured. The catalogue test recomputes every computed figure from a table of architectures and lets two rows share figures only when they share the architecture and the runner. The rule tests of recommend() now run on a frozen copy of the 2026-11-06 catalogue, so a weekly catalogue update changes only the per-memory table and the real-Mac tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
139 lines
6.5 KiB
TypeScript
139 lines
6.5 KiB
TypeScript
/**
|
|
* @fileoverview Tests for `modelFitsBudget()` of src/local-model/recommend.ts: the judgement
|
|
* the install job makes before it downloads a model.
|
|
*
|
|
* Pinned: it agrees with what `recommend()` offers (a model that is the pick or listed fits,
|
|
* a model that is in neither and is not left out for its rank does not), it does not depend
|
|
* on the ranking, a model the catalogue does not have never fits, and it judges against the
|
|
* budget the runtime measured when the caller passes one (the whole point of asking again
|
|
* after the runtime started).
|
|
*
|
|
* Port: N/A (pure functions).
|
|
*/
|
|
|
|
import { describe, expect, it } from 'vitest';
|
|
import { getLocalModelCatalog } from '../src/config/local-model-catalog.js';
|
|
import { formatGb, memClassGb } from '../src/local-model/format.js';
|
|
import { modelFitsBudget, recommend } from '../src/local-model/recommend.js';
|
|
import type { HardwareFacts, HarnessFact } from '../src/types.js';
|
|
|
|
/** A Mac with Apple silicon and `gb` GB of memory as sold. */
|
|
function mac(gb: number, chip = 'Apple M5 Pro'): HardwareFacts {
|
|
const memBytes = gb * 1024 ** 3;
|
|
return {
|
|
platform: 'darwin',
|
|
arch: 'arm64',
|
|
arm64: true,
|
|
translated: false,
|
|
memBytes,
|
|
memGb: formatGb(memBytes),
|
|
memClassGb: memClassGb(memBytes),
|
|
chip,
|
|
macosMajor: 27,
|
|
macosVersion: '27.2',
|
|
wiredLimitMb: 0,
|
|
source: 'sysctl',
|
|
};
|
|
}
|
|
|
|
/** Every harness of the catalogue, installed and accepting every model id. */
|
|
function harnessFacts(): HarnessFact[] {
|
|
return getLocalModelCatalog().harnesses.map((row) => ({
|
|
cli: row.cli,
|
|
label: row.cli,
|
|
installed: true,
|
|
installable: false,
|
|
acceptsModelId: () => true,
|
|
}));
|
|
}
|
|
|
|
describe('modelFitsBudget', () => {
|
|
const catalog = getLocalModelCatalog();
|
|
|
|
it('agrees with the recommendation on every model and context of the catalogue', () => {
|
|
for (const gb of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192, 256, 512]) {
|
|
const hw = mac(gb);
|
|
const offered = recommend(hw, catalog, harnessFacts());
|
|
const rows = [offered.pick, ...offered.alternatives].filter((row) => row !== undefined);
|
|
// Whatever the recommendation offers fits at the context it offers it at.
|
|
for (const row of rows) {
|
|
expect(modelFitsBudget(hw, catalog, row.modelId, row.context), `${gb} GB ${row.modelId}`).toBe(true);
|
|
}
|
|
// A Mac that gets no recommendation for lack of memory has no model that fits at a
|
|
// context a harness works at.
|
|
if (offered.verdict === 'none') {
|
|
const smallestHarnessContext = Math.min(...catalog.harnesses.filter((h) => !h.gated).map((h) => h.minContext));
|
|
for (const model of catalog.models) {
|
|
for (const context of model.contexts.filter((ctx) => ctx >= smallestHarnessContext)) {
|
|
expect(modelFitsBudget(hw, catalog, model.id, context), `${gb} GB ${model.id}`).toBe(false);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
it('does not depend on the ranking: a model that fits and is listed nowhere still fits', () => {
|
|
// On a 128 GB Mac more models fit than the recommendation lists.
|
|
const hw = mac(128, 'Apple M5 Max');
|
|
const offered = recommend(hw, catalog, harnessFacts());
|
|
const listed = new Set([offered.pick, ...offered.alternatives].map((row) => row?.modelId));
|
|
const unlisted = catalog.models.filter((model) => !listed.has(model.id));
|
|
expect(unlisted.length).toBeGreaterThan(0);
|
|
for (const model of unlisted) {
|
|
expect(modelFitsBudget(hw, catalog, model.id, model.contexts.at(-1) as number), model.id).toBe(true);
|
|
}
|
|
});
|
|
|
|
it('rules out a model that is too large, and one the catalogue does not have', () => {
|
|
expect(modelFitsBudget(mac(48), catalog, 'ornith-1.5:122b', 131072)).toBe(false);
|
|
expect(modelFitsBudget(mac(48), catalog, 'no-such:1b', 65536)).toBe(false);
|
|
expect(modelFitsBudget(mac(48), catalog, 'qwen3.6:35b-a3b', 131072)).toBe(true);
|
|
});
|
|
|
|
it('judges against the budget the runtime measured, when it is given one', () => {
|
|
const id = 'qwen3.6:35b-a3b';
|
|
// The MacBook: the runtime reported 37.4 GiB, and the model needs 36.0 of those 40.2 GB.
|
|
expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 40_157_944_217 })).toBe(true);
|
|
// A runtime that reports less than the estimate rules out what the estimate allowed.
|
|
expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 33 * 1024 ** 3 })).toBe(false);
|
|
// And one that reports more admits what the estimate ruled out: on a 36 GB Mac the GGUF
|
|
// 35B-A3B at its full window needs 27.9 GB, 1.0003 of the 27.9 GB estimate and 0.962 of 27 GiB.
|
|
const moeGguf = 'qwen3.6:35b-a3b-mtp-q4_k_m';
|
|
expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144)).toBe(false);
|
|
expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144, { measuredTotalBytes: 27 * 1024 ** 3 })).toBe(true);
|
|
// The reserve still bounds it: 100 GiB "measured" on a 48 GB Mac is 40 GiB (42.9 GB) of
|
|
// budget, so the 122B with its 97.4 GB stays out although 100 GiB would hold it.
|
|
const huge = { measuredTotalBytes: 100 * 1024 ** 3 };
|
|
expect(modelFitsBudget(mac(48), catalog, 'ornith-1.5:122b', 131072, huge)).toBe(false);
|
|
expect(modelFitsBudget(mac(128), catalog, 'ornith-1.5:122b', 131072, huge)).toBe(true);
|
|
});
|
|
|
|
it('agrees with the recommendation under a measured budget too', () => {
|
|
for (const [gb, measuredGib] of [
|
|
[16, 11.8],
|
|
[32, 20],
|
|
[48, 37.4],
|
|
[48, 30],
|
|
[64, 50],
|
|
]) {
|
|
const hw = mac(gb);
|
|
const options = { measuredTotalBytes: Math.floor(measuredGib * 1024 ** 3) };
|
|
const offered = recommend(hw, catalog, harnessFacts(), options);
|
|
expect(offered.verdict === 'none' || offered.budgetSource === 'measured').toBe(true);
|
|
for (const row of [offered.pick, ...offered.alternatives].filter((entry) => entry !== undefined)) {
|
|
expect(modelFitsBudget(hw, catalog, row.modelId, row.context, options), `${gb} GB ${row.modelId}`).toBe(true);
|
|
// The next larger context of the same model does not fit, or the row would be at it.
|
|
const model = catalog.models.find((entry) => entry.id === row.modelId);
|
|
const larger = model?.contexts.filter((ctx) => ctx > row.context) ?? [];
|
|
for (const ctx of larger) expect(modelFitsBudget(hw, catalog, row.modelId, ctx, options)).toBe(false);
|
|
}
|
|
}
|
|
});
|
|
|
|
it('is false for a context that is no token count', () => {
|
|
for (const context of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
|
|
expect(modelFitsBudget(mac(128), catalog, 'qwen3.5:9b-q4_k_m', context), String(context)).toBe(false);
|
|
}
|
|
});
|
|
});
|