mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-15 10:59:32 +01:00
fix(local-model): count what llama-server keeps beside the K/V cache, and bound its checkpoints
The llama.cpp memory rule of the catalogue left out two allocations that the llama.cpp the runtime bundles (b12111, the runtime's LLAMA_CPP_VERSION) makes for every GGUF row: - Context checkpoints. For a model whose state cannot be rolled back (the recurrent state of the Qwen-family rows, the sliding window of North Mini Code), llama-server keeps host copies of that state, two or three per request, up to 32 per slot by default (tools/server/server-context.cpp create_checkpoint). For the 27B that is 32 x 157 MB = 5 GB, none of which was counted. The runtime now sets LLAMA_ARG_CTX_CHECKPOINTS=8 (read by llama-server from its environment like LLAMA_ARG_CACHE_RAM; 8 was llama.cpp's own default before it went to 32), and every llamacpp row counts 8 copies of its state. - Multi-token prediction. The two -mtp tags and qwen3.8:27b-q8_0 carry a prediction layer (nextn_predict_layers 1, read from the first bytes of each GGUF; the 9B tags and North Mini Code have none). The runtime then always passes --spec-type draft-mtp --spec-draft-n-max 4, so llama.cpp keeps 4 rollback copies of the recurrent state (need_n_rs_seq(), llama-memory-recurrent.cpp) and the prediction layer's cache in the draft context's cache type, f16 by default. The rows counted one state and that layer at q8_0. What moves (test/local-model-recommend.test.ts, "every memory size", re-pinned): - 16 GB: limited -> none. MiMo 9B at 65,536 tokens needs 10.2 GB, 0.987 of the 10.4 GB estimate. With the budget the Mac mini measured (10.7 GB after the reserve) it is offered, 0.954, tight; that test is kept. With 4 checkpoints instead of 8 it would be 0.966 on the estimate: this size sits on the edge, and only a run on such a Mac settles it. - 18 GB: the same pick, 0.964 instead of 0.928. - 36 GB: the same picks, 0.966 (27B on M4/M5 Max) and 0.976 (35B-A3B GGUF on M3). - 48 GB on a Max chip: the 27B at 262,144 tokens is tight (0.853) instead of good. - 32 GB with a measured budget of 0.74: North Mini Code at 131,072 tokens (was 262,144), the 27B at 65,536 on a Max chip (was 131,072). - Every MLX row, and everything from 64 GB up, is unchanged. test/local-model-catalog.test.ts recomputes the new rule from the architecture table and holds the catalogue to LLAMA_ARG_CTX_CHECKPOINTS. The catalogue header, runtime.ts, the types, docs/local-models.md, CLAUDE.md and architecture-invariants say the same. Read in the source, not measured. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codeman maintainer committed
1 parent
bc9c3c1639
commit
2cd3cbd6a0
11 files changed
+286
-168
No files matched your search
@@ -31,7 +31,12 @@ import {
|
||||
LocalModelCatalogSchema,
|
||||
STOCK_LOCAL_MODEL_CATALOG,
|
||||
} from '../src/config/local-model-catalog.js';
|
||||
import { RUNTIME_HOST_PROMPT_CACHE_MIB, RUNTIME_KV_CACHE_TYPE, runtimeEnv } from '../src/local-model/runtime.js';
|
||||
import {
|
||||
RUNTIME_CTX_CHECKPOINTS,
|
||||
RUNTIME_HOST_PROMPT_CACHE_MIB,
|
||||
RUNTIME_KV_CACHE_TYPE,
|
||||
runtimeEnv,
|
||||
} from '../src/local-model/runtime.js';
|
||||
import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js';
|
||||
|
||||
/** A deep copy of the stock catalogue that a test may break. */
|
||||
@@ -79,10 +84,12 @@ type ArchitectureName = keyof typeof ARCHITECTURES;
|
||||
type Architecture = (typeof ARCHITECTURES)[ArchitectureName];
|
||||
|
||||
/**
|
||||
* Which architecture each stock row runs, and whether its GGUF carries the one
|
||||
* multi-token-prediction layer (counted as one more attention layer, see the catalogue
|
||||
* header). A row the catalogue gains must be added here, or the test below fails: its figures
|
||||
* cannot be checked without it.
|
||||
* Which architecture each stock row runs, and whether its GGUF carries a
|
||||
* multi-token-prediction layer (`nextn_predict_layers` 1 in the GGUF header, read on
|
||||
* 2026-11-08 from the first bytes of each model layer). For such a model the runtime turns on
|
||||
* prediction, which costs a cache for that layer and rollback copies of the recurrent state
|
||||
* (see the catalogue header). A row the catalogue gains must be added here, or the test below
|
||||
* fails: its figures cannot be checked without it.
|
||||
*/
|
||||
const ROW_ARCHITECTURE: Record<string, { architecture: ArchitectureName; predictionLayer: boolean }> = {
|
||||
'qwen3.5:9b-q4_k_m': { architecture: 'Qwen3.5-9B', predictionLayer: false },
|
||||
@@ -97,6 +104,19 @@ const ROW_ARCHITECTURE: Record<string, { architecture: ArchitectureName; predict
|
||||
|
||||
/** Bytes per cached value in each K/V cache type of llama.cpp: q8_0 keeps 32 values in 34 bytes. */
|
||||
const KV_BYTES_PER_VALUE: Record<string, number> = { f16: 2, q8_0: 34 / 32 };
|
||||
/**
|
||||
* The cache type of llama-server's draft context, which holds the prediction layer's cache: its
|
||||
* default, because the runtime sets only the main context's (`--cache-type-k`).
|
||||
*/
|
||||
const DRAFT_KV_CACHE_TYPE = 'f16';
|
||||
/**
|
||||
* The recurrent states llama.cpp keeps beside the live one for a model with prediction: the
|
||||
* runtime's draft length (`draft_num_predict` 4, passed as `--spec-draft-n-max`), which
|
||||
* `need_n_rs_seq()` turns into that many rollback copies.
|
||||
*/
|
||||
const MTP_ROLLBACK_STATES = 4;
|
||||
/** The sliding-window cache holds the window plus one batch of 512 tokens (`n_swa + n_ubatch`). */
|
||||
const SLIDING_EXTRA_CELLS = 512;
|
||||
/** The head size of a Gated DeltaNet layer's keys and values, and its convolution kernel, in every model above. */
|
||||
const LINEAR_HEAD_DIM = 128;
|
||||
const LINEAR_CONV_KERNEL = 4;
|
||||
@@ -114,15 +134,20 @@ function recurrentStateBytes(arch: Architecture, convBytes: number): number {
|
||||
/** What a `llamacpp` row's three figures must be, from its architecture and the runtime's settings. */
|
||||
function llamaCppFigures(arch: Architecture, predictionLayer: boolean): [number, number, number] {
|
||||
const perValue = KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE];
|
||||
const layers = arch.attentionLayers + (predictionLayer ? 1 : 0);
|
||||
const held = layers * arch.kvHeads * arch.headDim * 2 * perValue;
|
||||
const perLayer = arch.kvHeads * arch.headDim * 2;
|
||||
// The prediction layer's cache sits in the draft context, in the draft's cache type.
|
||||
const held =
|
||||
arch.attentionLayers * perLayer * perValue +
|
||||
(predictionLayer ? perLayer * KV_BYTES_PER_VALUE[DRAFT_KV_CACHE_TYPE] : 0);
|
||||
// llama.cpp keeps the recurrent state in f32 (both parts), the sliding window in the cache type.
|
||||
const recurrent = recurrentStateBytes(arch, 4);
|
||||
const sliding =
|
||||
'sliding' in arch
|
||||
? arch.sliding.layers * (arch.sliding.window + 512) * arch.kvHeads * arch.headDim * 2 * perValue
|
||||
: 0;
|
||||
const fixed = recurrentStateBytes(arch, 4) + sliding;
|
||||
return [held, fixed + 1_000_000_000 + RUNTIME_HOST_PROMPT_CACHE_MIB * MIB, 0];
|
||||
'sliding' in arch ? arch.sliding.layers * (arch.sliding.window + SLIDING_EXTRA_CELLS) * perLayer * perValue : 0;
|
||||
// Live: the sliding window, and the recurrent state with its rollback copies under prediction.
|
||||
const live = sliding + recurrent * (1 + (predictionLayer ? MTP_ROLLBACK_STATES : 0));
|
||||
// A context checkpoint is a copy of what cannot be rolled back: the recurrent state or the window.
|
||||
const checkpoints = RUNTIME_CTX_CHECKPOINTS * (recurrent + sliding);
|
||||
return [held, live + checkpoints + 1_000_000_000 + RUNTIME_HOST_PROMPT_CACHE_MIB * MIB, 0];
|
||||
}
|
||||
|
||||
/** The per-token figure of the MLX runner computed from an architecture: K/V in bf16, a recurrent snapshot per 8,192 tokens. */
|
||||
@@ -304,7 +329,10 @@ describe('the memory figures of the stock rows: each row has its own', () => {
|
||||
const env = runtimeEnv({ source: {}, home: '/h', modelsDir: '/m', port: 11435, contextLength: 65536 });
|
||||
expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE);
|
||||
expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB));
|
||||
expect(env.LLAMA_ARG_CTX_CHECKPOINTS).toBe(String(RUNTIME_CTX_CHECKPOINTS));
|
||||
expect(KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE]).toBeDefined();
|
||||
// The runtime sets no cache type for the draft context, so the prediction layer's stays f16.
|
||||
expect(Object.keys(env).filter((name) => /DRAFT|SPEC/.test(name))).toEqual([]);
|
||||
});
|
||||
|
||||
it("recomputes every estimated row's figures from its own model's architecture and runner", () => {
|
||||
@@ -318,15 +346,22 @@ describe('the memory figures of the stock rows: each row has its own', () => {
|
||||
|
||||
it('states the numbers the header and the row comments name', () => {
|
||||
expect(Object.fromEntries(models.map((m) => [m.id, figuresOf(m)]))).toEqual({
|
||||
'qwen3.5:9b-q4_k_m': [17_408, 2_126_432_768, 0],
|
||||
'mimo-v2.6:9b-q4_k_m': [17_408, 2_126_432_768, 0],
|
||||
'north-mini-code-1.0:q4_k_m': [14_144, 2_254_227_968, 0],
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': [11_968, 2_139_605_504, 0],
|
||||
// State x (1 live + 8 checkpoints) + 1.0 GB + 1 GiB.
|
||||
'qwen3.5:9b-q4_k_m': [17_408, 2_547_960_320, 0],
|
||||
'mimo-v2.6:9b-q4_k_m': [17_408, 2_547_960_320, 0],
|
||||
// The sliding window x (1 live + 8 checkpoints) + 1.0 GB + 1 GiB.
|
||||
'north-mini-code-1.0:q4_k_m': [14_144, 3_698_117_120, 0],
|
||||
// State x (1 live + 4 rollback copies + 8 checkpoints) + 1.0 GB + 1 GiB; the prediction
|
||||
// layer's cache in f16 per token.
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': [12_928, 2_929_969_664, 0],
|
||||
'qwen3.6:35b-a3b': [37_000, 3_000_000_000, 5_000_000_000],
|
||||
'qwen3.8:27b-q8_0': [36_992, 2_230_635_008, 0],
|
||||
'qwen3.8:27b-mtp-q4_k_m': [36_992, 2_230_635_008, 0],
|
||||
'qwen3.8:27b-q8_0': [38_912, 4_113_353_216, 0],
|
||||
'qwen3.8:27b-mtp-q4_k_m': [38_912, 4_113_353_216, 0],
|
||||
'ornith-1.5:122b': [56_574, 11_589_934_592, 5_000_000_000],
|
||||
});
|
||||
// The two parts of a llamacpp overhead that are not computed from the architecture.
|
||||
expect(RUNTIME_CTX_CHECKPOINTS).toBe(8);
|
||||
expect(RUNTIME_HOST_PROMPT_CACHE_MIB).toBe(1024);
|
||||
// The MLX calibration: 37,000 held against 28,340 computed for the measured row.
|
||||
expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B'])).toBe(28_340);
|
||||
expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.5-122B-A10B'])).toBe(43_332);
|
||||
|
||||
@@ -97,10 +97,10 @@ describe('modelFitsBudget', () => {
|
||||
// A runtime that reports less than the estimate rules out what the estimate allowed.
|
||||
expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 33 * 1024 ** 3 })).toBe(false);
|
||||
// And one that reports more admits what the estimate ruled out: on a 36 GB Mac the GGUF
|
||||
// 35B-A3B at its full window needs 27.9 GB, 1.0003 of the 27.9 GB estimate and 0.962 of 27 GiB.
|
||||
// 35B-A3B at its full window needs 28.9 GB, 1.037 of the 27.9 GB estimate and 0.963 of 28 GiB.
|
||||
const moeGguf = 'qwen3.6:35b-a3b-mtp-q4_k_m';
|
||||
expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144)).toBe(false);
|
||||
expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144, { measuredTotalBytes: 27 * 1024 ** 3 })).toBe(true);
|
||||
expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144, { measuredTotalBytes: 28 * 1024 ** 3 })).toBe(true);
|
||||
// The reserve still bounds it: 100 GiB "measured" on a 48 GB Mac is 40 GiB (42.9 GB) of
|
||||
// budget, so the 122B with its 97.4 GB stays out although 100 GiB would hold it.
|
||||
const huge = { measuredTotalBytes: 100 * 1024 ** 3 };
|
||||
|
||||
@@ -464,10 +464,10 @@ describe('needBytes', () => {
|
||||
});
|
||||
|
||||
it('grows with the context by what one token holds', () => {
|
||||
// A llama.cpp row: 6,550,811,033 + 17,408 x 65,536 + 2,126,432,768, and no peak.
|
||||
// A llama.cpp row: 6,550,811,033 + 17,408 x 65,536 + 2,547,960,320, and no peak.
|
||||
const nineB = stockModel('mimo-v2.6:9b-q4_k_m');
|
||||
expect(needBytes(nineB, 65536)).toBe(9_818_094_489);
|
||||
expect(needBytes(nineB, 131072)).toBe(10_958_945_177);
|
||||
expect(needBytes(nineB, 65536)).toBe(10_239_622_041);
|
||||
expect(needBytes(nineB, 131072)).toBe(11_380_472_729);
|
||||
// Twice the context, twice what the context holds: 17,408 x 65,536 more.
|
||||
expect(needBytes(nineB, 131072) - needBytes(nineB, 65536)).toBe(1_140_850_688);
|
||||
});
|
||||
@@ -486,13 +486,13 @@ describe('needBytes', () => {
|
||||
|
||||
it('states the need of every stock row at its largest context', () => {
|
||||
expect(Object.fromEntries(STOCK.models.map((m) => [m.id, needBytes(m, m.contexts[0])]))).toEqual({
|
||||
'qwen3.5:9b-q4_k_m': 10_958_959_694,
|
||||
'mimo-v2.6:9b-q4_k_m': 10_958_945_177,
|
||||
'north-mini-code-1.0:q4_k_m': 24_555_959_712,
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': 27_898_259_277,
|
||||
'qwen3.5:9b-q4_k_m': 11_380_487_246,
|
||||
'mimo-v2.6:9b-q4_k_m': 11_380_472_729,
|
||||
'north-mini-code-1.0:q4_k_m': 25_999_848_864,
|
||||
'qwen3.6:35b-a3b-mtp-q4_k_m': 28_940_281_677,
|
||||
'qwen3.6:35b-a3b': 35_955_462_956,
|
||||
'qwen3.8:27b-q8_0': 41_906_107_906,
|
||||
'qwen3.8:27b-mtp-q4_k_m': 29_669_738_010,
|
||||
'qwen3.8:27b-q8_0': 44_292_142_594,
|
||||
'qwen3.8:27b-mtp-q4_k_m': 32_055_772_698,
|
||||
'ornith-1.5:122b': 104_808_538_800,
|
||||
});
|
||||
});
|
||||
@@ -647,7 +647,7 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => {
|
||||
expect(
|
||||
result.alternatives.map((c) => [c.modelId, c.context, c.fit, c.speed, c.tokensPerSecond ?? null, c.tag?.kind])
|
||||
).toEqual([
|
||||
// The same model as a GGUF build: its fixed footprint holds the full window, 27.9 GB.
|
||||
// The same model as a GGUF build: its fixed footprint holds the full window, 28.9 GB.
|
||||
['qwen3.6:35b-a3b-mtp-q4_k_m', 262144, 'good', 'ok', 41.8, undefined],
|
||||
['north-mini-code-1.0:q4_k_m', 262144, 'good', 'ok', 41.8, undefined],
|
||||
['mimo-v2.6:9b-q4_k_m', 131072, 'good', 'ok', 28.6, 'small'],
|
||||
@@ -655,7 +655,7 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => {
|
||||
]);
|
||||
// Only the tested row rests on a measurement of that model.
|
||||
expect(result.alternatives.map((c) => c.needSource)).toEqual(['estimated', 'estimated', 'estimated', 'estimated']);
|
||||
// The dense 27B fits too (29.7 GB at 262144) but computes to 2.95 x 3.07 = 9.1 tokens per
|
||||
// The dense 27B fits too (32.1 GB at 262144) but computes to 2.95 x 3.07 = 9.1 tokens per
|
||||
// second here: slow, so it ranks after four fast models and is not listed.
|
||||
expect(modelFitsBudget(await macbook(), STOCK, 'qwen3.8:27b-mtp-q4_k_m', 262144)).toBe(true);
|
||||
expect(order(result).some((entry) => entry.startsWith('qwen3.8:27b'))).toBe(false);
|
||||
@@ -699,43 +699,72 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => {
|
||||
|
||||
describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => {
|
||||
/**
|
||||
* Expected: verdict `limited`, MiMo V2.6 9B (a GGUF build) at 65,536 tokens with OpenCode,
|
||||
* a TIGHT fit, about 11 tokens per second.
|
||||
* Expected: verdict `none` (reason `memory`) on the estimated budget, and `limited` with
|
||||
* MiMo V2.6 9B (a GGUF build) at 65,536 tokens, a TIGHT fit, once the runtime measured its
|
||||
* budget.
|
||||
*
|
||||
* Why. The budget is 9.67 GiB (10.4 GB: 2/3 of 16 GiB minus 1). The 9B on llama.cpp needs
|
||||
* 6.55 GB of weights + 1.14 GB for a q8_0 K/V cache of 65,536 tokens + 2.13 GB beside them
|
||||
* (its recurrent state, the runner, and the 1 GiB prompt cache the runtime allows) = 9.8 GB:
|
||||
* 0.946 of the budget. At 131,072 tokens it would need 11.0 GB, so it steps down. Nothing of
|
||||
* 12 billion parameters or more fits, so the verdict is `limited`.
|
||||
* Why. The estimated budget is 9.67 GiB (10.4 GB: 2/3 of 16 GiB minus 1). The 9B on
|
||||
* llama.cpp needs 6.55 GB of weights + 1.14 GB for a q8_0 K/V cache of 65,536 tokens +
|
||||
* 2.55 GB beside them (its recurrent state live and in 8 context checkpoints, the runner, and
|
||||
* the 1 GiB prompt cache the runtime allows) = 10.2 GB: 0.987 of the estimate, just past the
|
||||
* 0.98 a tight fit may reach. The runtime measured 11.8 GiB on this Mac; the 6 GiB reserve
|
||||
* leaves 10 GiB (10.7 GB) of it, and the same 10.2 GB is 0.954 of that.
|
||||
*
|
||||
* Every figure of this row is computed from the model's architecture, none was measured
|
||||
* (`needSource: 'estimated'`); the check after the download (the whole model on the GPU)
|
||||
* is what catches it if llama.cpp holds more.
|
||||
* (`needSource: 'estimated'`), so this Mac is on the edge either way. Only a run of the 9B on
|
||||
* it settles which side.
|
||||
*/
|
||||
it('gets a small model: the 9B at 65536 tokens, tight', async () => {
|
||||
it('gets no model on the estimated budget: the 9B needs 0.987 of it', async () => {
|
||||
const hw = await macMini();
|
||||
expect(hw.chip).toBe('Apple M4');
|
||||
expect(hw.memClassGb).toBe(16);
|
||||
|
||||
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||||
|
||||
expect(result.verdict).toBe('limited');
|
||||
expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.headlineLimited });
|
||||
expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]);
|
||||
expect(result).toMatchObject({ verdict: 'none', reason: 'memory', budgetSource: 'estimated' });
|
||||
expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.noneMemory, vars: { memory: '16' } });
|
||||
expect(result.pick).toBeUndefined();
|
||||
// floor(17,179,869,184 x 2/3) - 1 GiB.
|
||||
expect(result.budgetBytes).toBe(10_379_504_298);
|
||||
expect(result.budgetGb).toBe('10.4');
|
||||
const nineB = stockModel('mimo-v2.6:9b-q4_k_m');
|
||||
// 6,550,811,033 + 17,408 x 65,536 + 2,547,960,320.
|
||||
expect(needBytes(nineB, 65536)).toBe(10_239_622_041);
|
||||
expect(needBytes(nineB, 65536) / result.budgetBytes).toBeGreaterThan(FIT_TIGHT_MAX_SHARE);
|
||||
expect(needBytes(nineB, 65536) / result.budgetBytes).toBeLessThan(0.99);
|
||||
});
|
||||
|
||||
it('would need 11.4 GB at 131072 tokens, more than the whole budget', async () => {
|
||||
const hw = await macMini();
|
||||
const nineB = stockModel('mimo-v2.6:9b-q4_k_m');
|
||||
const budget = deviceBudget(hw).budgetBytes;
|
||||
expect(formatGb(needBytes(nineB, 131072))).toBe('11.4');
|
||||
expect(needBytes(nineB, 131072) / budget).toBeGreaterThan(1);
|
||||
expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 131072)).toBe(false);
|
||||
expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 65536)).toBe(false);
|
||||
});
|
||||
|
||||
it('gets a small model with the budget its runtime measured: the 9B at 65536 tokens, tight', async () => {
|
||||
// The runtime reported 11.8 GiB here; the 6 GiB reserve leaves 10 GiB (10.7 GB) of it.
|
||||
const result = recommend(await macMini(), STOCK, ALL_INSTALLED, { measuredTotalBytes: MACMINI_MEASURED_TOTAL });
|
||||
expect(result).toMatchObject({
|
||||
verdict: 'limited',
|
||||
budgetBytes: 10 * GIB,
|
||||
budgetGb: '10.7',
|
||||
budgetSource: 'measured',
|
||||
});
|
||||
expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.headlineLimited });
|
||||
expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]);
|
||||
expect(result.pick).toMatchObject({
|
||||
modelId: 'mimo-v2.6:9b-q4_k_m',
|
||||
modelLabel: 'MiMo V2.6 9B',
|
||||
license: 'MIT',
|
||||
context: 65536,
|
||||
cli: 'opencode',
|
||||
// 6,550,811,033 + 17,408 x 65,536 + 2,126,432,768.
|
||||
needBytes: 9_818_094_489,
|
||||
needGb: '9.8',
|
||||
memory: { modelBytes: 8_677_243_801, contextBytes: 1_140_850_688, peakBytes: 0 },
|
||||
budgetShare: 0.946,
|
||||
needBytes: 10_239_622_041,
|
||||
needGb: '10.2',
|
||||
memory: { modelBytes: 9_098_771_353, contextBytes: 1_140_850_688, peakBytes: 0 },
|
||||
budgetShare: 0.954,
|
||||
fit: 'tight',
|
||||
// 9.3 x 120 / 100.
|
||||
speed: 'ok',
|
||||
@@ -748,32 +777,12 @@ describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => {
|
||||
expect(order(result)).toEqual(['mimo-v2.6:9b-q4_k_m@65536', 'qwen3.5:9b-q4_k_m@65536']);
|
||||
});
|
||||
|
||||
it('would need 11.0 GB at 131072 tokens, more than the whole budget', async () => {
|
||||
const hw = await macMini();
|
||||
const nineB = stockModel('mimo-v2.6:9b-q4_k_m');
|
||||
const budget = deviceBudget(hw).budgetBytes;
|
||||
expect(formatGb(needBytes(nineB, 131072))).toBe('11.0');
|
||||
expect(needBytes(nineB, 131072) / budget).toBeGreaterThan(1);
|
||||
expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 131072)).toBe(false);
|
||||
expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 65536)).toBe(true);
|
||||
});
|
||||
|
||||
it('gets the same with the budget its runtime measured, with a little more room', async () => {
|
||||
// The runtime reported 11.8 GiB here; the 6 GiB reserve leaves 10 GiB (10.7 GB) of it.
|
||||
const result = recommend(await macMini(), STOCK, ALL_INSTALLED, { measuredTotalBytes: MACMINI_MEASURED_TOTAL });
|
||||
expect(result).toMatchObject({
|
||||
verdict: 'limited',
|
||||
budgetBytes: 10 * GIB,
|
||||
budgetGb: '10.7',
|
||||
budgetSource: 'measured',
|
||||
});
|
||||
expect(result.pick).toMatchObject({ modelId: 'mimo-v2.6:9b-q4_k_m', context: 65536, budgetShare: 0.914 });
|
||||
});
|
||||
|
||||
it('gets the same with Claude Code as its only coding agent, which also needs 65536 tokens', async () => {
|
||||
const result = recommend(await macMini(), STOCK, [fact('claude')]);
|
||||
expect(result.verdict).toBe('limited');
|
||||
expect(result.pick).toMatchObject({ modelId: 'mimo-v2.6:9b-q4_k_m', context: 65536, cli: 'claude' });
|
||||
const hw = await macMini();
|
||||
expect(recommend(hw, STOCK, [fact('claude')])).toMatchObject({ verdict: 'none', reason: 'memory' });
|
||||
const measured = recommend(hw, STOCK, [fact('claude')], { measuredTotalBytes: MACMINI_MEASURED_TOTAL });
|
||||
expect(measured.verdict).toBe('limited');
|
||||
expect(measured.pick).toMatchObject({ modelId: 'mimo-v2.6:9b-q4_k_m', context: 65536, cli: 'claude' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -782,28 +791,32 @@ describe('every memory size, with the stock catalogue', () => {
|
||||
// size. A change of a catalogue row or of a threshold that moves one of these answers has to
|
||||
// change this table on purpose. docs/local-models.md has the same table for people.
|
||||
const EXPECTED: [soldGb: number, chip: string, answer: string][] = [
|
||||
// The smallest need at 65536 tokens is 9.8 GB; the budget is 2.1 GB.
|
||||
// The smallest need at 65536 tokens is 10.2 GB; the budget is 2.1 GB.
|
||||
[8, 'Apple M2', 'none memory'],
|
||||
// The 9B fits only at 65536 tokens (0.946), and nothing of 12 billion fits: `limited`.
|
||||
[16, 'Apple M4', 'limited mimo-v2.6:9b-q4_k_m@65536 tight ok'],
|
||||
// On the slowest chip the pick is the same, and slow (6.3 tokens per second).
|
||||
[16, 'Apple M1', 'limited mimo-v2.6:9b-q4_k_m@65536 tight slow'],
|
||||
// The 9B needs 10.2 GB at 65536 tokens, 0.987 of the 10.4 GB estimate: just out. A runtime
|
||||
// that measured the budget a real 16 GB Mac measured admits it (see the Mac mini above).
|
||||
[16, 'Apple M4', 'none memory'],
|
||||
[16, 'Apple M1', 'none memory'],
|
||||
// 11.4 GB at 131072 tokens, 0.964 of 11.8 GB, and nothing of 12 billion fits: `limited`.
|
||||
[18, 'Apple M3 Pro', 'limited mimo-v2.6:9b-q4_k_m@131072 tight ok'],
|
||||
// On a base chip with 100 GB/s the 9B computes to 9.3 tokens per second: slow.
|
||||
[24, 'Apple M3', 'limited mimo-v2.6:9b-q4_k_m@131072 good slow'],
|
||||
[24, 'Apple M4', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'],
|
||||
[24, 'Apple M4 Pro', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'],
|
||||
// North Mini Code needs 21.8 GB at 65536 tokens: 0.997 of the 21.8 GB estimate, out by a
|
||||
// hair. A runtime that measures more than the estimate admits it (see below).
|
||||
// North Mini Code needs 23.2 GB at 65536 tokens: 1.06 of the 21.8 GB estimate. A runtime
|
||||
// that measures about 3/4 of the memory admits it (see below).
|
||||
[32, 'Apple M4', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'],
|
||||
[32, 'Apple M1 Max', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'],
|
||||
// A Max chip runs the dense 27B fast enough (12.1); an M3 Max (300 GB/s) does not (8.9),
|
||||
// and takes the 35B-A3B as a GGUF build, whose memory does not grow after it loaded.
|
||||
// and takes the 35B-A3B as a GGUF build, whose memory does not grow after it loaded. Both
|
||||
// at 131072 tokens, 0.966 and 0.976 of the budget.
|
||||
[36, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@131072 tight ok'],
|
||||
[36, 'Apple M3 Max', 'ok qwen3.6:35b-a3b-mtp-q4_k_m@131072 tight ok'],
|
||||
// The tested pair on the tested Mac; on a Pro chip the 27B is slow (8.1), so the same pick.
|
||||
[48, 'Apple M5 Pro', 'ok qwen3.6:35b-a3b@131072 tight ok'],
|
||||
[48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 tight ok'],
|
||||
// On a Max chip the 27B has the higher rank and holds its full window, 29.7 GB.
|
||||
[48, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@262144 good ok'],
|
||||
// On a Max chip the 27B has the higher rank and holds its full window, 32.1 GB: 0.853.
|
||||
[48, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@262144 tight ok'],
|
||||
[64, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'],
|
||||
[64, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@262144 good ok'],
|
||||
[96, 'Apple M3 Ultra', 'ok qwen3.8:27b-mtp-q4_k_m@262144 good ok'],
|
||||
@@ -833,15 +846,15 @@ describe('every memory size, with the stock catalogue', () => {
|
||||
expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.noneMemory, vars: { memory: '8' } });
|
||||
// The smallest model a harness can work with needs more than four times the budget.
|
||||
const smallest = Math.min(...STOCK.models.map((m) => needBytes(m, 65536)));
|
||||
expect(formatGb(smallest)).toBe('9.8');
|
||||
expect(formatGb(smallest)).toBe('10.2');
|
||||
expect(smallest / result.budgetBytes).toBeGreaterThan(4);
|
||||
});
|
||||
|
||||
it('gives every Mac from 16 to 32 GB a small model, and every larger one a model of 27 billion or more', () => {
|
||||
it('gives no model to 8 and 16 GB, a small model to every Mac from 18 to 32 GB, and every larger one a model of 27 billion or more', () => {
|
||||
for (const hw of SWEEP) {
|
||||
const result = recommend(hw, STOCK, ALL_INSTALLED);
|
||||
const label = `${hw.memClassGb} GB ${hw.chip}`;
|
||||
if (hw.memClassGb === 8) expect(result.reason, label).toBe('memory');
|
||||
if (hw.memClassGb <= 16) expect(result.reason, label).toBe('memory');
|
||||
else if (hw.memClassGb <= 32) {
|
||||
expect(result.verdict, label).toBe('limited');
|
||||
expect(result.pick?.paramsB, label).toBe(9);
|
||||
@@ -858,10 +871,11 @@ describe('every memory size, with the stock catalogue', () => {
|
||||
const measured = { measuredTotalBytes: Math.floor(0.74 * 32 * GIB) };
|
||||
const base = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED, measured);
|
||||
expect(base.verdict).toBe('ok');
|
||||
expect(base.pick).toMatchObject({ modelId: 'north-mini-code-1.0:q4_k_m', context: 262144, fit: 'tight' });
|
||||
// On a Max chip the 27B is fast enough, and it is the higher rank.
|
||||
// 24.1 GB at 131072 tokens, 0.950 of it; at 262144 tokens 26.0 GB would not fit.
|
||||
expect(base.pick).toMatchObject({ modelId: 'north-mini-code-1.0:q4_k_m', context: 131072, fit: 'tight' });
|
||||
// On a Max chip the 27B is fast enough, and it is the higher rank: 24.4 GB at 65536 tokens.
|
||||
const max = recommend(mac(32, 'Apple M1 Max'), STOCK, ALL_INSTALLED, measured);
|
||||
expect(max.pick).toMatchObject({ modelId: 'qwen3.8:27b-mtp-q4_k_m', context: 131072, fit: 'tight' });
|
||||
expect(max.pick).toMatchObject({ modelId: 'qwen3.8:27b-mtp-q4_k_m', context: 65536, fit: 'tight' });
|
||||
});
|
||||
|
||||
it('always pairs the pick with the top-ranked harness, OpenCode', () => {
|
||||
@@ -2063,8 +2077,8 @@ describe('the measured budget, end to end', () => {
|
||||
expect([measured.budgetSource, measured.budgetGb]).toEqual(['measured', '40.2']);
|
||||
// The same rows in the same order; only the share of each changes.
|
||||
expect(order(measured)).toEqual(order(estimated));
|
||||
expect(rows(estimated).map((c) => c.budgetShare)).toEqual([0.957, 0.742, 0.653, 0.292, 0.292]);
|
||||
expect(rows(measured).map((c) => c.budgetShare)).toEqual([0.895, 0.695, 0.611, 0.273, 0.273]);
|
||||
expect(rows(estimated).map((c) => c.budgetShare)).toEqual([0.957, 0.77, 0.692, 0.303, 0.303]);
|
||||
expect(rows(measured).map((c) => c.budgetShare)).toEqual([0.895, 0.721, 0.647, 0.283, 0.283]);
|
||||
for (const candidate of rows(measured)) {
|
||||
expect(candidate.budgetShare).toBeCloseTo(candidate.needBytes / MACBOOK_MEASURED_TOTAL, 3);
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ import {
|
||||
parseInferenceComputeTotal,
|
||||
removeLocalModelFiles,
|
||||
removePartialModelBlobs,
|
||||
RUNTIME_CTX_CHECKPOINTS,
|
||||
RUNTIME_DOWNLOAD_RETRIES,
|
||||
RUNTIME_HOST_PROMPT_CACHE_MIB,
|
||||
RUNTIME_KEEP_ALIVE,
|
||||
@@ -262,6 +263,7 @@ describe('runtimeEnv', () => {
|
||||
OLLAMA_REMOTES: 'evil.example',
|
||||
OLLAMA_AUTH: '1',
|
||||
LLAMA_ARG_CACHE_RAM: '0',
|
||||
LLAMA_ARG_CTX_CHECKPOINTS: '64',
|
||||
// Other secrets a server environment carries.
|
||||
ANTHROPIC_API_KEY: 'sk-ant-secret',
|
||||
OPENAI_API_KEY: 'sk-secret',
|
||||
@@ -275,7 +277,7 @@ describe('runtimeEnv', () => {
|
||||
DYLD_INSERT_LIBRARIES: '/tmp/evil.dylib',
|
||||
};
|
||||
|
||||
it('holds exactly the allowlist: what is copied, the private HOME, and ten settings', () => {
|
||||
it('holds exactly the allowlist: what is copied, the private HOME, and eleven settings', () => {
|
||||
expect(runtimeEnv(input(SERVER_ENV))).toEqual({
|
||||
PATH: '/usr/local/bin:/usr/bin:/bin',
|
||||
TMPDIR: '/var/folders/xy/T/',
|
||||
@@ -298,22 +300,27 @@ describe('runtimeEnv', () => {
|
||||
OLLAMA_KV_CACHE_TYPE: 'q8_0',
|
||||
OLLAMA_FLASH_ATTENTION: '1',
|
||||
LLAMA_ARG_CACHE_RAM: '1024',
|
||||
LLAMA_ARG_CTX_CHECKPOINTS: '8',
|
||||
});
|
||||
});
|
||||
|
||||
it('sets the llama.cpp memory settings from the exported constants, never from the server', () => {
|
||||
expect(RUNTIME_KV_CACHE_TYPE).toBe('q8_0');
|
||||
expect(RUNTIME_HOST_PROMPT_CACHE_MIB).toBe(1024);
|
||||
expect(RUNTIME_CTX_CHECKPOINTS).toBe(8);
|
||||
const env = runtimeEnv(input(SERVER_ENV));
|
||||
expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE);
|
||||
expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB));
|
||||
// The server's own `LLAMA_ARG_CACHE_RAM=0` and a user's q4_0 cache lose against ours.
|
||||
expect(env.LLAMA_ARG_CTX_CHECKPOINTS).toBe(String(RUNTIME_CTX_CHECKPOINTS));
|
||||
// The server's own `LLAMA_ARG_CACHE_RAM=0` and `LLAMA_ARG_CTX_CHECKPOINTS=64`, and a user's
|
||||
// q4_0 cache, lose against ours.
|
||||
const env2 = runtimeEnv(input({ ...SERVER_ENV, OLLAMA_KV_CACHE_TYPE: 'q4_0', OLLAMA_FLASH_ATTENTION: '0' }));
|
||||
expect([env2.OLLAMA_KV_CACHE_TYPE, env2.OLLAMA_FLASH_ATTENTION, env2.LLAMA_ARG_CACHE_RAM]).toEqual([
|
||||
'q8_0',
|
||||
'1',
|
||||
'1024',
|
||||
]);
|
||||
expect([
|
||||
env2.OLLAMA_KV_CACHE_TYPE,
|
||||
env2.OLLAMA_FLASH_ATTENTION,
|
||||
env2.LLAMA_ARG_CACHE_RAM,
|
||||
env2.LLAMA_ARG_CTX_CHECKPOINTS,
|
||||
]).toEqual(['q8_0', '1', '1024', '8']);
|
||||
});
|
||||
|
||||
it('lets no CODEMAN_ variable through, the password least of all', () => {
|
||||
@@ -390,6 +397,7 @@ describe('runtimeEnv', () => {
|
||||
expect(Object.keys(runtimeEnv(input({}))).sort()).toEqual([
|
||||
'HOME',
|
||||
'LLAMA_ARG_CACHE_RAM',
|
||||
'LLAMA_ARG_CTX_CHECKPOINTS',
|
||||
'OLLAMA_CONTEXT_LENGTH',
|
||||
'OLLAMA_FLASH_ATTENTION',
|
||||
'OLLAMA_HOST',
|
||||
|
||||
@@ -209,6 +209,7 @@ describe('start', () => {
|
||||
OLLAMA_KV_CACHE_TYPE: 'q8_0',
|
||||
OLLAMA_FLASH_ATTENTION: '1',
|
||||
LLAMA_ARG_CACHE_RAM: '1024',
|
||||
LLAMA_ARG_CTX_CHECKPOINTS: '8',
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user