mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-18 04:09:33 +01:00
feat(local-model): a catalogue for every Mac from 8 to 512 GB, each row with its own figures
The catalogue had seven Qwen-family rows that all carried the memory figures measured on one model (Qwen3.6 35B-A3B on the MLX runner), so a 9B or a dense 27B was judged with figures that do not describe it, two rows had the wrong Mac download size, and every Mac below 32 GB got "none". Now eight rows, each with the runner a Mac really uses for its tag (a new `runner` field: `mlx` or `llamacpp`) and memory figures of its own: - the 35B-A3B MLX row keeps its measured figures and the tested pair; - every llama.cpp row is computed from its model's config.json: a q8_0 K/V cache for the whole context (what the runtime now sets), its recurrent or sliding-window state, 1.0 GB of runner and the 1 GiB host prompt cache; - the one other MLX row (Ornith 1.5 122B) is computed the same way, scaled by what the measured row showed, plus the runner's 8 GiB of prefix snapshots. Sizes are the registry's Mac pull totals (2026-11-08, lowercase ids verified there). What each memory size gets: 8 GB none; 16 to 32 GB a 9B (limited); 36 GB Qwen3.8 27B or the 35B-A3B as GGUF; 48 GB the tested pair (or the 27B on a Max chip); 64 and 96 GB the 27B or the 35B-A3B; 128 to 512 GB Ornith 1.5 122B. Nothing but the one row was measured. The catalogue test recomputes every computed figure from a table of architectures and lets two rows share figures only when they share the architecture and the runner. The rule tests of recommend() now run on a frozen copy of the 2026-11-06 catalogue, so a weekly catalogue update changes only the per-memory table and the real-Mac tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f60b63c005
commit
989fcd778a
7 files changed
+1176
-465
No files matched your search
@@ -12,9 +12,12 @@
|
||||
* hides a more specific one).
|
||||
* 3. The chip lookup never guesses: a chip that no row names has no bandwidth, including one
|
||||
* whose name merely contains a known name ('Apple M10', 'Apple M6 Pro').
|
||||
* 4. Which figures were MEASURED on a Mac and which are carried over from the measured row:
|
||||
* one row says `measured` (its size is what a Mac really downloads, and it alone has a
|
||||
* tested entry), every other row says `estimated`.
|
||||
* 4. Which figures were MEASURED on a Mac and which were computed: one row says `measured`
|
||||
* (it alone has a tested entry), every other row says `estimated`, and every estimated
|
||||
* row's three memory figures are recomputed here from its OWN model's architecture (a
|
||||
* table read from each model's config.json) for its runner and the runtime's settings. Two
|
||||
* rows may share figures only when they share the architecture and the runner, so a row
|
||||
* can never again carry another model's measurement without saying so.
|
||||
*
|
||||
* Port: N/A (pure data and functions).
|
||||
*/
|
||||
@@ -28,6 +31,7 @@ import {
|
||||
LocalModelCatalogSchema,
|
||||
STOCK_LOCAL_MODEL_CATALOG,
|
||||
} from '../src/config/local-model-catalog.js';
|
||||
import { RUNTIME_HOST_PROMPT_CACHE_MIB, RUNTIME_KV_CACHE_TYPE, runtimeEnv } from '../src/local-model/runtime.js';
|
||||
import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js';
|
||||
|
||||
/** A deep copy of the stock catalogue that a test may break. */
|
||||
@@ -48,7 +52,101 @@ function withFirstModel(change: Partial<Record<keyof LocalModelEntry | 'url' | '
|
||||
return { ...catalog, models: [{ ...catalog.models[0], ...change }, ...catalog.models.slice(1)] };
|
||||
}
|
||||
|
||||
/** One MiB and one GiB, in bytes. */
|
||||
const MIB = 1024 * 1024;
|
||||
const GIB = 1024 * MIB;
|
||||
|
||||
/**
|
||||
* The architecture of every model the stock catalogue lists, read from its config.json on
|
||||
* Hugging Face (2026-11-08): full-attention layers, their K/V heads and head size, and the
|
||||
* layers whose cache does not grow with the context (Gated DeltaNet layers with their value
|
||||
* and key heads, or sliding-window layers with their window). Fine-tunes share their base's.
|
||||
*/
|
||||
const ARCHITECTURES = {
|
||||
'Qwen3.5-9B': { attentionLayers: 8, kvHeads: 4, headDim: 256, linear: { layers: 24, vHeads: 32, kHeads: 16 } },
|
||||
'Qwen3.8-27B': { attentionLayers: 16, kvHeads: 4, headDim: 256, linear: { layers: 48, vHeads: 48, kHeads: 16 } },
|
||||
'Qwen3.6-35B-A3B': { attentionLayers: 10, kvHeads: 2, headDim: 256, linear: { layers: 30, vHeads: 32, kHeads: 16 } },
|
||||
'Qwen3.5-122B-A10B': {
|
||||
attentionLayers: 12,
|
||||
kvHeads: 2,
|
||||
headDim: 256,
|
||||
linear: { layers: 36, vHeads: 64, kHeads: 16 },
|
||||
},
|
||||
'North-Mini-Code-1.0': { attentionLayers: 13, kvHeads: 4, headDim: 128, sliding: { layers: 36, window: 4096 } },
|
||||
} as const;
|
||||
|
||||
type ArchitectureName = keyof typeof ARCHITECTURES;
|
||||
type Architecture = (typeof ARCHITECTURES)[ArchitectureName];
|
||||
|
||||
/**
|
||||
* Which architecture each stock row runs, and whether its GGUF carries the one
|
||||
* multi-token-prediction layer (counted as one more attention layer, see the catalogue
|
||||
* header). A row the catalogue gains must be added here, or the test below fails: its figures
|
||||
* cannot be checked without it.
|
||||
*/
|
||||
const ROW_ARCHITECTURE: Record<string, { architecture: ArchitectureName; predictionLayer: boolean }> = {
|
||||
'qwen3.5:9b-q4_k_m': { architecture: 'Qwen3.5-9B', predictionLayer: false },
|
||||
'mimo-v2.6:9b-q4_k_m': { architecture: 'Qwen3.5-9B', predictionLayer: false },
|
||||
'north-mini-code-1.0:q4_k_m': { architecture: 'North-Mini-Code-1.0', predictionLayer: false },
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': { architecture: 'Qwen3.6-35B-A3B', predictionLayer: true },
|
||||
'qwen3.6:35b-a3b': { architecture: 'Qwen3.6-35B-A3B', predictionLayer: false },
|
||||
'qwen3.8:27b-q8_0': { architecture: 'Qwen3.8-27B', predictionLayer: true },
|
||||
'qwen3.8:27b-mtp-q4_k_m': { architecture: 'Qwen3.8-27B', predictionLayer: true },
|
||||
'ornith-1.5:122b': { architecture: 'Qwen3.5-122B-A10B', predictionLayer: false },
|
||||
};
|
||||
|
||||
/** Bytes per cached value in each K/V cache type of llama.cpp: q8_0 keeps 32 values in 34 bytes. */
|
||||
const KV_BYTES_PER_VALUE: Record<string, number> = { f16: 2, q8_0: 34 / 32 };
|
||||
/** The head size of a Gated DeltaNet layer's keys and values, and its convolution kernel, in every model above. */
|
||||
const LINEAR_HEAD_DIM = 128;
|
||||
const LINEAR_CONV_KERNEL = 4;
|
||||
|
||||
/** The recurrent state of the linear-attention layers in bytes: the convolution tail and the delta state. */
|
||||
function recurrentStateBytes(arch: Architecture, convBytes: number): number {
|
||||
if (!('linear' in arch)) return 0;
|
||||
const { layers, vHeads, kHeads } = arch.linear;
|
||||
const convDim = vHeads * LINEAR_HEAD_DIM + 2 * kHeads * LINEAR_HEAD_DIM;
|
||||
const conv = (LINEAR_CONV_KERNEL - 1) * convDim * convBytes;
|
||||
const delta = vHeads * LINEAR_HEAD_DIM * LINEAR_HEAD_DIM * 4;
|
||||
return layers * (conv + delta);
|
||||
}
|
||||
|
||||
/** What a `llamacpp` row's three figures must be, from its architecture and the runtime's settings. */
|
||||
function llamaCppFigures(arch: Architecture, predictionLayer: boolean): [number, number, number] {
|
||||
const perValue = KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE];
|
||||
const layers = arch.attentionLayers + (predictionLayer ? 1 : 0);
|
||||
const held = layers * arch.kvHeads * arch.headDim * 2 * perValue;
|
||||
// llama.cpp keeps the recurrent state in f32 (both parts), the sliding window in the cache type.
|
||||
const sliding =
|
||||
'sliding' in arch
|
||||
? arch.sliding.layers * (arch.sliding.window + 512) * arch.kvHeads * arch.headDim * 2 * perValue
|
||||
: 0;
|
||||
const fixed = recurrentStateBytes(arch, 4) + sliding;
|
||||
return [held, fixed + 1_000_000_000 + RUNTIME_HOST_PROMPT_CACHE_MIB * MIB, 0];
|
||||
}
|
||||
|
||||
/** The per-token figure of the MLX runner computed from an architecture: K/V in bf16, a recurrent snapshot per 8,192 tokens. */
|
||||
function mlxComputedPerToken(arch: Architecture): number {
|
||||
return arch.attentionLayers * arch.kvHeads * arch.headDim * 2 * 2 + recurrentStateBytes(arch, 2) / 8192;
|
||||
}
|
||||
|
||||
/** The figures measured for `qwen3.6:35b-a3b` on the MacBook Pro. */
|
||||
const MEASURED: [number, number, number] = [37_000, 3_000_000_000, 5_000_000_000];
|
||||
|
||||
/** What an estimated `mlx` row's three figures must be: computed, scaled by what the measured row showed. */
|
||||
function mlxFigures(arch: Architecture): [number, number, number] {
|
||||
const ratio = MEASURED[0] / mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B']);
|
||||
return [Math.ceil(mlxComputedPerToken(arch) * ratio), MEASURED[1] + 8 * GIB, MEASURED[2]];
|
||||
}
|
||||
|
||||
/** A row's three memory figures. */
|
||||
function figuresOf(model: LocalModelEntry): [number, number, number] {
|
||||
return [model.heldBytesPerToken, model.overheadBytes, model.peakExtraBytes];
|
||||
}
|
||||
|
||||
describe('the stock catalogue', () => {
|
||||
const models = STOCK_LOCAL_MODEL_CATALOG.models;
|
||||
|
||||
it('passes its own schema', () => {
|
||||
expect(issuesOf(STOCK_LOCAL_MODEL_CATALOG)).toEqual([]);
|
||||
});
|
||||
@@ -60,94 +158,114 @@ describe('the stock catalogue', () => {
|
||||
expect(getLocalModelCatalog()).toBe(first);
|
||||
});
|
||||
|
||||
it('lists the seven models of the design, without flint:40b', () => {
|
||||
expect(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => m.id)).toEqual([
|
||||
'qwen3.5:9b',
|
||||
'qwen3.5:9b-q8_0',
|
||||
'gpt-oss:20b',
|
||||
it('lists eight models, by rank, and none of those set aside in the header', () => {
|
||||
expect(models.map((m) => m.id)).toEqual([
|
||||
'qwen3.5:9b-q4_k_m',
|
||||
'mimo-v2.6:9b-q4_k_m',
|
||||
'north-mini-code-1.0:q4_k_m',
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m',
|
||||
'qwen3.6:35b-a3b',
|
||||
'qwen3.8:27b',
|
||||
'qwen3.8:27b-q8_0',
|
||||
'qwen3.5:122b-a10b',
|
||||
'qwen3.8:27b-mtp-q4_k_m',
|
||||
'ornith-1.5:122b',
|
||||
]);
|
||||
expect(STOCK_LOCAL_MODEL_CATALOG.models.some((m) => m.id.startsWith('flint'))).toBe(false);
|
||||
expect(models.map((m) => m.rank)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
|
||||
const setAside =
|
||||
/^(bonsai-2|gpt-oss|flint|lfm2\.5|laguna|qwen3\.8-flash-next|qwen3\.5:122b)|^ornith-1\.5:(9b|35b|397b)/;
|
||||
expect(models.filter((m) => setAside.test(m.id)).map((m) => m.id)).toEqual([]);
|
||||
// The plain 9B and 27B tags are MLX builds on a Mac: their GGUF builds are listed instead.
|
||||
for (const plain of ['qwen3.5:9b', 'mimo-v2.6:9b', 'qwen3.8:27b']) {
|
||||
expect(models.some((m) => m.id === plain)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it('carries the size a Mac really downloads where that was measured, else the registry manifest', () => {
|
||||
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.bytes]))).toEqual({
|
||||
// The sum of the layer sizes of each tag's manifest, as fetched on 2026-11-06.
|
||||
'qwen3.5:9b': 6_550_825_373,
|
||||
'qwen3.5:9b-q8_0': 10_449_217_949,
|
||||
'gpt-oss:20b': 13_793_441_427,
|
||||
// The total of the real pull on a Mac (an MLX build). Its manifest sums to 22,621,314,161.
|
||||
it('names the runner a Mac uses for each tag', () => {
|
||||
expect(Object.fromEntries(models.map((m) => [m.id, m.runner]))).toEqual({
|
||||
'qwen3.5:9b-q4_k_m': 'llamacpp',
|
||||
'mimo-v2.6:9b-q4_k_m': 'llamacpp',
|
||||
'north-mini-code-1.0:q4_k_m': 'llamacpp',
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': 'llamacpp',
|
||||
// Manifest lists: a Mac takes the MLX child.
|
||||
'qwen3.6:35b-a3b': 'mlx',
|
||||
'qwen3.8:27b-q8_0': 'llamacpp',
|
||||
'qwen3.8:27b-mtp-q4_k_m': 'llamacpp',
|
||||
'ornith-1.5:122b': 'mlx',
|
||||
});
|
||||
});
|
||||
|
||||
it('carries the size of the build a Mac pulls: every layer plus the config', () => {
|
||||
expect(Object.fromEntries(models.map((m) => [m.id, m.bytes]))).toEqual({
|
||||
// The registry's manifests as fetched on 2026-11-08, in lowercase.
|
||||
'qwen3.5:9b-q4_k_m': 6_550_825_550,
|
||||
'mimo-v2.6:9b-q4_k_m': 6_550_811_033,
|
||||
'north-mini-code-1.0:q4_k_m': 18_593_967_008,
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': 22_621_314_381,
|
||||
// The MLX child of the list, and the total of a real pull on a Mac, byte for byte.
|
||||
'qwen3.6:35b-a3b': 23_105_798_956,
|
||||
'qwen3.8:27b': 17_741_871_939,
|
||||
'qwen3.8:27b-q8_0': 29_978_241_837,
|
||||
'qwen3.5:122b-a10b': 81_370_035_877,
|
||||
'qwen3.8:27b-q8_0': 29_978_242_050,
|
||||
'qwen3.8:27b-mtp-q4_k_m': 17_741_872_154,
|
||||
// The MLX child of the list (its GGUF child is 76,756,174,997).
|
||||
'ornith-1.5:122b': 73_388_069_552,
|
||||
});
|
||||
});
|
||||
|
||||
it('says which row was measured on a Mac: one, and every other row is an estimate', () => {
|
||||
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.figures]))).toEqual({
|
||||
'qwen3.5:9b': 'estimated',
|
||||
'qwen3.5:9b-q8_0': 'estimated',
|
||||
'gpt-oss:20b': 'estimated',
|
||||
'qwen3.6:35b-a3b': 'measured',
|
||||
'qwen3.8:27b': 'estimated',
|
||||
'qwen3.8:27b-q8_0': 'estimated',
|
||||
'qwen3.5:122b-a10b': 'estimated',
|
||||
});
|
||||
});
|
||||
|
||||
it('gives every row the memory figures measured on the one row: 37 KB per token, 3 GB, 5 GB', () => {
|
||||
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
|
||||
expect(
|
||||
[model.heldBytesPerToken, model.overheadBytes, model.peakExtraBytes],
|
||||
`${model.id}: an unmeasured row carries the measured row's figures, not invented ones`
|
||||
).toEqual([37_000, 3_000_000_000, 5_000_000_000]);
|
||||
}
|
||||
// No row keeps the figure that was computed from the model's config.
|
||||
expect(JSON.stringify(STOCK_LOCAL_MODEL_CATALOG)).not.toContain('kvBytesPerToken');
|
||||
expect(models.filter((m) => m.figures === 'measured').map((m) => m.id)).toEqual(['qwen3.6:35b-a3b']);
|
||||
expect(figuresOf(models.find((m) => m.id === 'qwen3.6:35b-a3b') as LocalModelEntry)).toEqual(MEASURED);
|
||||
});
|
||||
|
||||
it('has one tested entry: the 35B-A3B with OpenCode on the MacBook Pro, with its measured speed', () => {
|
||||
expect(
|
||||
Object.fromEntries(
|
||||
STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.tested.length > 0).map((m) => [m.id, m.tested])
|
||||
)
|
||||
).toEqual({
|
||||
expect(Object.fromEntries(models.filter((m) => m.tested.length > 0).map((m) => [m.id, m.tested]))).toEqual({
|
||||
'qwen3.6:35b-a3b': [
|
||||
{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 },
|
||||
],
|
||||
});
|
||||
// A row may claim a test only when its figures were measured.
|
||||
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
|
||||
for (const model of models) {
|
||||
if (model.tested.length > 0) expect(model.figures, model.id).toBe('measured');
|
||||
}
|
||||
});
|
||||
|
||||
it('gives every model a parameter count, an id of the default registry and a license', () => {
|
||||
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
|
||||
expect(model.paramsB, model.id).toBeGreaterThan(0);
|
||||
expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN);
|
||||
expect(model.license).toBe('Apache-2.0');
|
||||
}
|
||||
expect(Object.fromEntries(models.map((m) => [m.id, [m.paramsB, m.license]]))).toEqual({
|
||||
'qwen3.5:9b-q4_k_m': [9, 'Apache-2.0'],
|
||||
'mimo-v2.6:9b-q4_k_m': [9, 'MIT'],
|
||||
'north-mini-code-1.0:q4_k_m': [30, 'Apache-2.0'],
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': [35, 'Apache-2.0'],
|
||||
'qwen3.6:35b-a3b': [35, 'Apache-2.0'],
|
||||
'qwen3.8:27b-q8_0': [27.8, 'Apache-2.0'],
|
||||
'qwen3.8:27b-mtp-q4_k_m': [27.8, 'Apache-2.0'],
|
||||
'ornith-1.5:122b': [122, 'MIT'],
|
||||
});
|
||||
for (const model of models) expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN);
|
||||
// The two 9B rows are the ones under the 12 billion line of the `limited` verdict.
|
||||
expect(STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual([
|
||||
'qwen3.5:9b',
|
||||
'qwen3.5:9b-q8_0',
|
||||
]);
|
||||
expect(models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual(['qwen3.5:9b-q4_k_m', 'mimo-v2.6:9b-q4_k_m']);
|
||||
});
|
||||
|
||||
it('offers no model below the 65536 tokens a harness needs, nor beyond its own window', () => {
|
||||
for (const model of models) {
|
||||
expect(Math.min(...model.contexts), model.id).toBeGreaterThanOrEqual(65536);
|
||||
expect(model.contexts[0], model.id).toBeLessThanOrEqual(262144);
|
||||
}
|
||||
});
|
||||
|
||||
it('has a speed figure only where one was measured or scaled from a measurement', () => {
|
||||
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({
|
||||
'qwen3.5:9b': 9.3,
|
||||
'qwen3.5:9b-q8_0': 5.8,
|
||||
'gpt-oss:20b': null,
|
||||
'qwen3.6:35b-a3b': 13.6,
|
||||
'qwen3.8:27b': 2.95,
|
||||
expect(Object.fromEntries(models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({
|
||||
// 37.1 tokens per second on 400 GB/s, and the same for the fine-tune of the same size.
|
||||
'qwen3.5:9b-q4_k_m': 9.3,
|
||||
'mimo-v2.6:9b-q4_k_m': 9.3,
|
||||
// The 35B-A3B's llama.cpp figure: both have 3 billion active parameters.
|
||||
'north-mini-code-1.0:q4_k_m': 13.6,
|
||||
// 54.5 on 400 GB/s with llama.cpp.
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': 13.6,
|
||||
// 67.8 on 400 GB/s with MLX.
|
||||
'qwen3.6:35b-a3b': 16.9,
|
||||
// 2.95 scaled by file size.
|
||||
'qwen3.8:27b-q8_0': 1.7,
|
||||
'qwen3.5:122b-a10b': null,
|
||||
// 11.8 on 400 GB/s with llama.cpp, without prediction.
|
||||
'qwen3.8:27b-mtp-q4_k_m': 2.95,
|
||||
// 16.9 x 3/10 active parameters, rounded down.
|
||||
'ornith-1.5:122b': 5,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -174,6 +292,82 @@ describe('the stock catalogue', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('the memory figures of the stock rows: each row has its own', () => {
|
||||
const models = STOCK_LOCAL_MODEL_CATALOG.models;
|
||||
|
||||
it('knows the architecture of every row, and every row of that table is in the catalogue', () => {
|
||||
expect(models.map((m) => m.id).sort()).toEqual(Object.keys(ROW_ARCHITECTURE).sort());
|
||||
});
|
||||
|
||||
it('computes the llamacpp rows for the K/V cache type and the prompt cache the runtime really sets', () => {
|
||||
// The figures below assume these; runtimeEnv() is what the runtime gets.
|
||||
const env = runtimeEnv({ source: {}, home: '/h', modelsDir: '/m', port: 11435, contextLength: 65536 });
|
||||
expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE);
|
||||
expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB));
|
||||
expect(KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE]).toBeDefined();
|
||||
});
|
||||
|
||||
it("recomputes every estimated row's figures from its own model's architecture and runner", () => {
|
||||
for (const model of models.filter((m) => m.figures === 'estimated')) {
|
||||
const { architecture, predictionLayer } = ROW_ARCHITECTURE[model.id];
|
||||
const arch = ARCHITECTURES[architecture];
|
||||
const expected = model.runner === 'llamacpp' ? llamaCppFigures(arch, predictionLayer) : mlxFigures(arch);
|
||||
expect(figuresOf(model), `${model.id} (${architecture}, ${model.runner})`).toEqual(expected);
|
||||
}
|
||||
});
|
||||
|
||||
it('states the numbers the header and the row comments name', () => {
|
||||
expect(Object.fromEntries(models.map((m) => [m.id, figuresOf(m)]))).toEqual({
|
||||
'qwen3.5:9b-q4_k_m': [17_408, 2_126_432_768, 0],
|
||||
'mimo-v2.6:9b-q4_k_m': [17_408, 2_126_432_768, 0],
|
||||
'north-mini-code-1.0:q4_k_m': [14_144, 2_254_227_968, 0],
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': [11_968, 2_139_605_504, 0],
|
||||
'qwen3.6:35b-a3b': [37_000, 3_000_000_000, 5_000_000_000],
|
||||
'qwen3.8:27b-q8_0': [36_992, 2_230_635_008, 0],
|
||||
'qwen3.8:27b-mtp-q4_k_m': [36_992, 2_230_635_008, 0],
|
||||
'ornith-1.5:122b': [56_574, 11_589_934_592, 5_000_000_000],
|
||||
});
|
||||
// The MLX calibration: 37,000 held against 28,340 computed for the measured row.
|
||||
expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B'])).toBe(28_340);
|
||||
expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.5-122B-A10B'])).toBe(43_332);
|
||||
});
|
||||
|
||||
it('lets two rows share figures only when they share the architecture, the prediction layer and the runner', () => {
|
||||
const byFigures = new Map<string, LocalModelEntry[]>();
|
||||
for (const model of models) {
|
||||
const key = figuresOf(model).join('/');
|
||||
byFigures.set(key, [...(byFigures.get(key) ?? []), model]);
|
||||
}
|
||||
const shared = [...byFigures.values()].filter((group) => group.length > 1);
|
||||
// The two 9B rows (a fine-tune and its base) and the two 27B rows (two quantizations of one model).
|
||||
expect(shared.map((group) => group.map((m) => m.id))).toEqual([
|
||||
['qwen3.5:9b-q4_k_m', 'mimo-v2.6:9b-q4_k_m'],
|
||||
['qwen3.8:27b-q8_0', 'qwen3.8:27b-mtp-q4_k_m'],
|
||||
]);
|
||||
for (const group of shared) {
|
||||
const identity = (m: LocalModelEntry): string => JSON.stringify([ROW_ARCHITECTURE[m.id], m.runner, m.figures]);
|
||||
expect(new Set(group.map(identity)).size, group.map((m) => m.id).join(', ')).toBe(1);
|
||||
}
|
||||
});
|
||||
|
||||
it("gives no estimated row the measured row's per-token or overhead figure", () => {
|
||||
for (const model of models.filter((m) => m.figures === 'estimated')) {
|
||||
expect(model.heldBytesPerToken, model.id).not.toBe(MEASURED[0]);
|
||||
expect(model.overheadBytes, model.id).not.toBe(MEASURED[1]);
|
||||
expect(figuresOf(model), model.id).not.toEqual(MEASURED);
|
||||
}
|
||||
});
|
||||
|
||||
it('gives a llamacpp row no peak (its buffers are allocated at load) and an estimated mlx row the whole prefix budget', () => {
|
||||
for (const model of models) {
|
||||
if (model.runner === 'llamacpp') expect(model.peakExtraBytes, model.id).toBe(0);
|
||||
if (model.runner === 'mlx' && model.figures === 'estimated') {
|
||||
expect(model.overheadBytes - MEASURED[1], model.id).toBe(8 * GIB);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('the model id rule', () => {
|
||||
it('accepts name:tag in lowercase', () => {
|
||||
for (const id of ['qwen3.5:9b', 'gpt-oss:20b', 'qwen3.8:27b-q8_0', 'a:b', 'x_y.z-1:2.3_4']) {
|
||||
@@ -240,6 +434,19 @@ describe('the schema, field by field', () => {
|
||||
expect(issuesOf(withFirstModel({ peakExtraBytes: 1.5 }))).not.toEqual([]);
|
||||
});
|
||||
|
||||
it('requires the runner on every model row, and knows two', () => {
|
||||
const row: Record<string, unknown> = { ...copy().models[0] };
|
||||
delete row.runner;
|
||||
const issues = issuesOf({ ...copy(), models: [row] });
|
||||
expect(issues).toHaveLength(1);
|
||||
expect(issues[0]).toContain('models.0.runner');
|
||||
expect(issuesOf(withFirstModel({ runner: 'mlx' }))).toEqual([]);
|
||||
expect(issuesOf(withFirstModel({ runner: 'llamacpp' }))).toEqual([]);
|
||||
for (const runner of ['ggml', 'MLX', 'llama.cpp', '']) {
|
||||
expect(issuesOf(withFirstModel({ runner })), runner).not.toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it('requires paramsB and tested on every model row', () => {
|
||||
for (const field of ['paramsB', 'tested'] as const) {
|
||||
const row: Record<string, unknown> = { ...copy().models[0] };
|
||||
|
||||
Reference in new issue
Block a user