From 2cd3cbd6a01c7d2999c6b776b1363cfc22f6a39a Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Sun, 8 Nov 2026 10:26:57 +0100 Subject: [PATCH] fix(local-model): count what llama-server keeps beside the K/V cache, and bound its checkpoints The llama.cpp memory rule of the catalogue left out two allocations that the llama.cpp the runtime bundles (b12111, the runtime's LLAMA_CPP_VERSION) makes for every GGUF row: - Context checkpoints. For a model whose state cannot be rolled back (the recurrent state of the Qwen-family rows, the sliding window of North Mini Code), llama-server keeps host copies of that state, two or three per request, up to 32 per slot by default (tools/server/server-context.cpp create_checkpoint). For the 27B that is 32 x 157 MB = 5 GB, none of which was counted. The runtime now sets LLAMA_ARG_CTX_CHECKPOINTS=8 (read by llama-server from its environment like LLAMA_ARG_CACHE_RAM; 8 was llama.cpp's own default before it went to 32), and every llamacpp row counts 8 copies of its state. - Multi-token prediction. The two -mtp tags and qwen3.8:27b-q8_0 carry a prediction layer (nextn_predict_layers 1, read from the first bytes of each GGUF; the 9B tags and North Mini Code have none). The runtime then always passes --spec-type draft-mtp --spec-draft-n-max 4, so llama.cpp keeps 4 rollback copies of the recurrent state (need_n_rs_seq(), llama-memory-recurrent.cpp) and the prediction layer's cache in the draft context's cache type, f16 by default. The rows counted one state and that layer at q8_0. What moves (test/local-model-recommend.test.ts, "every memory size", re-pinned): - 16 GB: limited -> none. MiMo 9B at 65,536 tokens needs 10.2 GB, 0.987 of the 10.4 GB estimate. With the budget the Mac mini measured (10.7 GB after the reserve) it is offered, 0.954, tight; that test is kept. With 4 checkpoints instead of 8 it would be 0.966 on the estimate: this size sits on the edge, and only a run on such a Mac settles it. - 18 GB: the same pick, 0.964 instead of 0.928. - 36 GB: the same picks, 0.966 (27B on M4/M5 Max) and 0.976 (35B-A3B GGUF on M3). - 48 GB on a Max chip: the 27B at 262,144 tokens is tight (0.853) instead of good. - 32 GB with a measured budget of 0.74: North Mini Code at 131,072 tokens (was 262,144), the 27B at 65,536 on a Max chip (was 131,072). - Every MLX row, and everything from 64 GB up, is unchanged. test/local-model-catalog.test.ts recomputes the new rule from the architecture table and holds the catalogue to LLAMA_ARG_CTX_CHECKPOINTS. The catalogue header, runtime.ts, the types, docs/local-models.md, CLAUDE.md and architecture-invariants say the same. Read in the source, not measured. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- docs/architecture-invariants.md | 2 +- docs/local-models.md | 64 ++++++----- src/config/local-model-catalog.ts | 97 ++++++++++------ src/local-model/runtime.ts | 22 +++- src/types/local-model.ts | 11 +- test/local-model-catalog.test.ts | 71 +++++++++--- test/local-model-fits-budget.test.ts | 4 +- test/local-model-recommend.test.ts | 158 +++++++++++++++------------ test/local-model-runtime.test.ts | 22 ++-- test/local-model-supervisor.test.ts | 1 + 11 files changed, 286 insertions(+), 168 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4500c408..d6faacea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -191,7 +191,7 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph ⚠️ **llama-swap endpoints** (one model at a time): the apply routes check `GET /running` and return `requiresConfirmation` before evicting a model another live session uses; `confirmedSwap` and `confirmedContext` are SEPARATE flags and must stay so. Claude alone gets a context floor (`CLAUDE_MIN_SAFE_CONTEXT_TOKENS`); context is parsed from `/running`'s `cmd`, never trusted from `/props`. The floor also divides that context by the model's parallel slot count (`modelSlotCounts`: `--parallel`/`-np` from `cmd`, else `/props` `total_slots`; `perSlotContext()`), and `/props` only ever RAISES that warning: it supplies the context only when `cmd` states none and the count only when `cmd` has no `--parallel`, never a larger pool. `/props` is probed once per launch command (it refreshes llama-swap's last-use time, so a `ttl` would never expire), and a `/props` context beside an explicit `--parallel N` is already per slot, so it is not divided again. Backend log lines come from llama-swap's `/api/events` `upstream` source, never `/logs`. → [architecture-invariants#custom-model-endpoint-profiles](docs/architecture-invariants.md#custom-model-endpoint-profiles) -**Local model recommendation** (`GET /api/local-model`, `src/local-model/`, `src/config/local-model-catalog.ts`, `src/types/local-model.ts`; EXPERIMENTAL): the server looks at the Mac it runs on and says which local coding model fits it and with which agent CLI. ⚠️ The READ side stays read-only: the probe, the catalogue and `recommend()` download nothing, install nothing and write nothing; everything that changes the machine is the install below, behind an explicit request. Apple silicon macOS only: anywhere else the answer is a `200` with verdict `none` (reasons `not-macos`, `intel`, `memory`, `disk`, `macos`, `no-harness`), and a failed probe is an answer, never a 500. The probe (`hardware.ts`) is ONE bounded async spawn of `/usr/sbin/sysctl` WITHOUT `-n`, parsed as `key: value` lines by the pure `parseSysctl()`, plus `statfs` of the data dir (probed bounded first, #516) and `node:os` fallbacks, cached 60 s; ⚠️ never `system_profiler`, never a second spawn, and `iogpu.wired_limit_mb` of 0 means not set. The catalogue is a compiled constant validated by a Zod schema: ⚠️ no override file and no runtime data, and a model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`, so an id can never name another registry host). ⚠️ A harness is a catalogue ROW (`{ cli, minContext, rank, gated?, slowFirstTurn? }`) plus a `HarnessFact` built from the CLI registry at call time (`harness-facts.ts`); no code compares a CLI id (both id-branching guards scan these files). `recommend()` (`recommend.ts`) is pure. ⚠️ A model's need is its PEAK (`bytes + heldBytesPerToken * context + overheadBytes + peakExtraBytes`, measured on a Mac for one row and marked `estimated` on the others), and every row names its `runner` (`mlx` or `llamacpp`, the build a Mac really pulls) and carries figures of its OWN: computed from its model's config.json by that runner's rule (catalogue header, MEMORY), never borrowed from another row; `test/local-model-catalog.test.ts` recomputes them, and the `llamacpp` rows assume the runtime's `OLLAMA_KV_CACHE_TYPE=q8_0` and `LLAMA_ARG_CACHE_RAM=1024` (`runtimeEnv()`), so change the two together. The need is judged against a budget that is `estimated` until the runtime ran and `measured` from then on (`RecommendOptions.measuredTotalBytes`). ⚠️ Ranking, lexicographic: tested on a Mac like this first, then speed `ok` before `unknown` before `slow`, then model rank, harness rank, larger context; the fit grade decides whether a row is listed and what it says, it does NOT sort, so a model is offered at the largest context it fits at. ⚠️ `limited` is a statement about the MAC (no model with 12 billion parameters or more fits), judged before the disk filter and whatever the pick is; a small pick under `ok` carries `noteSmallPick`, and a model left out for disk space is named in `noteDiskDropped`, never dropped in silence. ⚠️ The word "Recommended" exists only in `headlineTested`, and the tag "Tested on a Mac like this" only on a row whose model has a `tested` entry for exactly this chip, memory size and harness. ⚠️ Speed is `ok`/`slow`/`unknown`, graded on the unrounded figure, and a chip in no chips row is `unknown`: never guess a bandwidth. ⚠️ ONE unit for every computed size a person reads: decimal GB text from `formatGb()` (`format.ts`), sent beside the bytes, and the browser never divides bytes; the one exception is the installed memory, shown as `memClassGb` (the size the Mac is sold as, 48 and not 51.5). ⚠️ Every sentence the server chooses is a `LOCAL_MODEL_LABELS`, `LOCAL_MODEL_TAG_LABELS` or `LOCAL_MODEL_INSTALL_LABELS` text with `{placeholder}` slots filled from `vars` (no apostrophe in a text: the zh-CN key is single-quoted), and each needs a zh-CN entry. Admin only in multi-user mode (403 before anything is probed). Tests: `test/local-model-*.test.ts`, `test/routes/local-model-routes.test.ts`. Window rules: `src/web/CLAUDE.md` (Run a model on this Mac). → `docs/api-reference.md` (Local model), `docs/versioning-policy.md` +**Local model recommendation** (`GET /api/local-model`, `src/local-model/`, `src/config/local-model-catalog.ts`, `src/types/local-model.ts`; EXPERIMENTAL): the server looks at the Mac it runs on and says which local coding model fits it and with which agent CLI. ⚠️ The READ side stays read-only: the probe, the catalogue and `recommend()` download nothing, install nothing and write nothing; everything that changes the machine is the install below, behind an explicit request. Apple silicon macOS only: anywhere else the answer is a `200` with verdict `none` (reasons `not-macos`, `intel`, `memory`, `disk`, `macos`, `no-harness`), and a failed probe is an answer, never a 500. The probe (`hardware.ts`) is ONE bounded async spawn of `/usr/sbin/sysctl` WITHOUT `-n`, parsed as `key: value` lines by the pure `parseSysctl()`, plus `statfs` of the data dir (probed bounded first, #516) and `node:os` fallbacks, cached 60 s; ⚠️ never `system_profiler`, never a second spawn, and `iogpu.wired_limit_mb` of 0 means not set. The catalogue is a compiled constant validated by a Zod schema: ⚠️ no override file and no runtime data, and a model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`, so an id can never name another registry host). ⚠️ A harness is a catalogue ROW (`{ cli, minContext, rank, gated?, slowFirstTurn? }`) plus a `HarnessFact` built from the CLI registry at call time (`harness-facts.ts`); no code compares a CLI id (both id-branching guards scan these files). `recommend()` (`recommend.ts`) is pure. ⚠️ A model's need is its PEAK (`bytes + heldBytesPerToken * context + overheadBytes + peakExtraBytes`, measured on a Mac for one row and marked `estimated` on the others), and every row names its `runner` (`mlx` or `llamacpp`, the build a Mac really pulls) and carries figures of its OWN: computed from its model's config.json by that runner's rule (catalogue header, MEMORY), never borrowed from another row; `test/local-model-catalog.test.ts` recomputes them, and the `llamacpp` rows assume the runtime's `OLLAMA_KV_CACHE_TYPE=q8_0`, `LLAMA_ARG_CACHE_RAM=1024` and `LLAMA_ARG_CTX_CHECKPOINTS=8` (`runtimeEnv()`) and count what llama-server keeps beside the cache (8 context checkpoints of the recurrent or sliding-window state, and for a GGUF with a prediction layer 4 rollback copies of the recurrent state and that layer's cache in f16), so change the two together. The need is judged against a budget that is `estimated` until the runtime ran and `measured` from then on (`RecommendOptions.measuredTotalBytes`). ⚠️ Ranking, lexicographic: tested on a Mac like this first, then speed `ok` before `unknown` before `slow`, then model rank, harness rank, larger context; the fit grade decides whether a row is listed and what it says, it does NOT sort, so a model is offered at the largest context it fits at. ⚠️ `limited` is a statement about the MAC (no model with 12 billion parameters or more fits), judged before the disk filter and whatever the pick is; a small pick under `ok` carries `noteSmallPick`, and a model left out for disk space is named in `noteDiskDropped`, never dropped in silence. ⚠️ The word "Recommended" exists only in `headlineTested`, and the tag "Tested on a Mac like this" only on a row whose model has a `tested` entry for exactly this chip, memory size and harness. ⚠️ Speed is `ok`/`slow`/`unknown`, graded on the unrounded figure, and a chip in no chips row is `unknown`: never guess a bandwidth. ⚠️ ONE unit for every computed size a person reads: decimal GB text from `formatGb()` (`format.ts`), sent beside the bytes, and the browser never divides bytes; the one exception is the installed memory, shown as `memClassGb` (the size the Mac is sold as, 48 and not 51.5). ⚠️ Every sentence the server chooses is a `LOCAL_MODEL_LABELS`, `LOCAL_MODEL_TAG_LABELS` or `LOCAL_MODEL_INSTALL_LABELS` text with `{placeholder}` slots filled from `vars` (no apostrophe in a text: the zh-CN key is single-quoted), and each needs a zh-CN entry. Admin only in multi-user mode (403 before anything is probed). Tests: `test/local-model-*.test.ts`, `test/routes/local-model-routes.test.ts`. Window rules: `src/web/CLAUDE.md` (Run a model on this Mac). → `docs/api-reference.md` (Local model), `docs/versioning-policy.md` **Local model install** (`POST /api/local-model/install|cancel|resume|runtime`, `DELETE /api/local-model`; `src/local-model/{install-job,runtime,ollama-client,connect,state-store,paths,developer-tools}.ts`; EXPERIMENTAL, admin only in multi-user mode on EVERY verb; guide `docs/local-models.md`): installs the chosen model on the Mac the server runs on and ends on a session of the coding agent running on it. ONE job of five idempotent steps (`runtime`, `model`, `verify`, `agent`, `connect`; `agent` only when the coding agent is missing), a pure reducer (`reduceInstallJob()`) over three tables in `install-job.ts` driven by one service per process (`getLocalModelService()`); the window polls the GET once a second, no SSE event. ⚠️ Layout only through `localModelPaths()` (all `dataPath()`): `local-model/runtime/ollama-/`, `local-model/models/`, `local-model/home/` (the runtime's PRIVATE HOME, so the user's `~/.ollama` is never touched) and the state file `local-model.json` (0600, ONE serialized writer, total reader, holds NO path); nothing is written or deleted outside `local-model/` except that file and the two writes of the connect step. ⚠️ The runtime pin (`OLLAMA_RUNTIME_PIN`: version, URL, size, sha256) is a COMPILED constant no file, setting or request may supply; the hash is compared BEFORE anything is unpacked. ⚠️ The runtime is a NON-detached child whose environment comes from the `runtimeEnv()` ALLOWLIST, never `process.env` (no `CODEMAN_*`, none of the user's `OLLAMA_*`); the port is chosen ONCE by binding (from 11435, never 11434) and stored; a listener on it is adopted or replaced only when its stored pid runs a binary under OUR runtime folder, anything else is `port-in-use`; started at boot when installed and enabled, max 3 restarts, SIGTERM at shutdown beside `stopDeepSeekWeb()`; under vitest nothing spawns or fetches. ⚠️ Consent is bound to the size: `acknowledgedBytes` must equal the server's own total (`Candidate.installBytes`) or the answer is 409 `size-changed`; the model, context and CLI must be a row the current recommendation offers (400). ⚠️ ONE job at a time (409 `job-running`); a job a restart interrupted comes back `paused` and NEVER continues without a click (so `shutdown()` writes nothing), and an interrupted step is never recorded as failed. ⚠️ An event the transition table does not list is refused (same object back, 409 `wrong-state`): add a row to the table, never a special case. ⚠️ The budget is MEASURED from the runtime's "inference compute" log line and the model judged against it BEFORE it is downloaded. ⚠️ The pull stream is parsed incrementally (about 1.4 million lines for 23 GB): never buffer it, never write state per line. ⚠️ A failure is a code with ONE next step (`LOCAL_MODEL_ERROR_NEXT`), never a bare "failed"; a probe that ended on its token limit is `probe-unfinished`, not `no-tool-call`. ⚠️ The coding agent is installed through the shared runner `src/cli-install-runner.ts` (extracted from `cli-registry-routes.ts`, ONE install slot per CLI for both callers) with `stockInstallCommand()`, the COMPILED stock command, never the `clis.json`-merged one (this path has no `cliManagementEnabled` gate); only for a catalogue harness row, after the consent click that showed the literal command, audited (`cli_install`, `via: 'local-model'`); afterwards `forgetResolvedCli()` and a fresh lookup, and still missing is `harness-failed`. ⚠️ Connect writes ONE endpoint record under the RESERVED id `local-ollama` through `mutateCustomModelHosts()` and `customModelEndpointsEnabled: true` through `mutateSettingsFile()`, remembering the flag's first value; a record with that id that is not ours is never overwritten, and while it is ours it is MANAGED: the endpoint routes refuse to edit or delete it and the discovery sweep skips it (the runtime's port has no login, so its `models` must stay the one model that was checked). ⚠️ The model digest is stored and read again at boot and before every START of a coding agent on that endpoint (`launchRefusalMessage()`): the two custom-model launch routes in `session-routes.ts` ask it before anything is written, the reboot-restore rebuild asks it before it builds (reason `local-model-not-ready`, left on offer), and `Session`'s launch guard (`setLaunchGuard()`, set in `setupSessionListeners()`) asks it right before a NEW agent process starts (a fresh pane, the respawn of a dead one, `restartCli()`), never for an attach to an agent that runs; a refusal is a `LaunchRefusedError` that never falls back to a direct PTY. A changed or missing model, a runtime Codeman did not start or that did not start, or another model id refuses the launch with one sentence. ⚠️ The connection is CLAIMED in the state file (`connection`, with the flag's value read by `inspectLocalEndpoint()`) BEFORE the endpoint record and the settings flag are written, so a state write that fails or a kill between the two never leaves a record the state file does not know; a record under the reserved id that says exactly what ours would say is ours (`isSameLocalEndpoint()`). ⚠️ A job whose end could not be written (a full disk) is kept in memory (`unrecorded`): the view shows it, the next verb writes it, and `installing` in the file while this process runs no job always reads as `paused` or that failure, never as a running job. ⚠️ The runtime's pid is stored the moment it is spawned, and Stop, Remove and the boot of anything that is not a running installation end a runtime an earlier server left behind (`supervisor.stop({ previousPid })`, ours by its command line) before any file is deleted; an installed model keeps the runtime VERSION it was installed with while that folder is unpacked, and a start that fails shows as `runtime-start-failed` with Start as the next step. Stop and "Use another port" answer 409 `sessions-open` while a live session uses the model. ⚠️ Remove (409 `sessions-open` with the names while a live session uses the endpoint) undoes the connection FIRST, restores the flag only when this feature switched it on AND no other endpoint is saved, deletes only inside `local-model/` (the models only when asked), and leaves the coding agent. ⚠️ Developer tools: only `/usr/bin/xcode-select -p`, bounded; never `git`/`python3`/`xcrun` from the server (each opens a system window on such a Mac). Tests: `test/local-model-install-job.test.ts` (every state with every event), `-install-service`, `-install-e2e` (real modules over `test/fixtures/fake-ollama.ts`), `-state-store`, `-connect`, `test/cli-install-runner.test.ts`, `test/custom-model-hosts-writer.test.ts`, `test/routes/local-model-routes.test.ts`, `test/routes/local-model-launch-guard.test.ts`. → [architecture-invariants#local-model-install](docs/architecture-invariants.md#local-model-install), `docs/local-models.md`, `docs/api-reference.md` (Local model) diff --git a/docs/architecture-invariants.md b/docs/architecture-invariants.md index 7aa02a57..83c39d8a 100644 --- a/docs/architecture-invariants.md +++ b/docs/architecture-invariants.md @@ -702,7 +702,7 @@ Either way the actual launch (`runCustomModelEntry`) routes through `run()` itse **The pin** (`OLLAMA_RUNTIME_PIN`, `runtime.ts`) is a COMPILED constant: version, URL, size, sha256. ⚠️ No file, setting, env var or request may supply any part of it: a hash that comes from the same writable place as the URL pins nothing. The archive is streamed to `*.partial` while it is hashed, continued with a Range request (3 bounded retries), refused when it grows past the pinned size, and the hash is compared BEFORE `/usr/bin/tar` unpacks anything into a temporary folder that is then renamed. -**Supervision** (`createRuntimeSupervisor()`): the runtime is a NON-detached child, started at boot once a model is installed and enabled, restarted at most 3 times in a row, SIGTERMed at shutdown beside `stopDeepSeekWeb()`. ⚠️ Its environment comes from the pure `runtimeEnv()` ALLOWLIST (PATH, the private HOME, TMPDIR, LANG, the proxy variables, and exactly `OLLAMA_HOST=127.0.0.1:`, `OLLAMA_MODELS`, `OLLAMA_CONTEXT_LENGTH`, `OLLAMA_NO_CLOUD=1`, `OLLAMA_KEEP_ALIVE=10m`, `OLLAMA_NUM_PARALLEL=1`, `OLLAMA_MAX_LOADED_MODELS=1`, plus the llama.cpp memory settings `OLLAMA_KV_CACHE_TYPE=q8_0`, `OLLAMA_FLASH_ATTENTION=1`, `LLAMA_ARG_CACHE_RAM=1024`, which the catalogue's `llamacpp` rows are computed for, so the two change together): never `process.env` whole (it carries `CODEMAN_PASSWORD`, and a user's own `OLLAMA_ORIGINS=*` would open the port to every web page). ⚠️ The port is chosen ONCE by binding (11435 across 20; 11434 belongs to a runtime the user may run) and stored; it is never searched again on a later start, because it is part of the saved endpoint's URL. ⚠️ Pid and port are in the state file: a listener on the port that answers `/api/version` and whose pid runs a binary under OUR runtime folder is our own orphan (a server that was killed leaves one: a child is not ended by its parent's death) and is adopted or replaced; ANY other listener is `port-in-use`, never something to talk to. ⚠️ The supervisor never writes the state file: it reports through `onChange`, and the job stores what it reports. ⚠️ Under vitest nothing spawns and nothing is fetched unless a test injects it. +**Supervision** (`createRuntimeSupervisor()`): the runtime is a NON-detached child, started at boot once a model is installed and enabled, restarted at most 3 times in a row, SIGTERMed at shutdown beside `stopDeepSeekWeb()`. ⚠️ Its environment comes from the pure `runtimeEnv()` ALLOWLIST (PATH, the private HOME, TMPDIR, LANG, the proxy variables, and exactly `OLLAMA_HOST=127.0.0.1:`, `OLLAMA_MODELS`, `OLLAMA_CONTEXT_LENGTH`, `OLLAMA_NO_CLOUD=1`, `OLLAMA_KEEP_ALIVE=10m`, `OLLAMA_NUM_PARALLEL=1`, `OLLAMA_MAX_LOADED_MODELS=1`, plus the llama.cpp memory settings `OLLAMA_KV_CACHE_TYPE=q8_0`, `OLLAMA_FLASH_ATTENTION=1`, `LLAMA_ARG_CACHE_RAM=1024`, `LLAMA_ARG_CTX_CHECKPOINTS=8`, which the catalogue's `llamacpp` rows are computed for, so the two change together): never `process.env` whole (it carries `CODEMAN_PASSWORD`, and a user's own `OLLAMA_ORIGINS=*` would open the port to every web page). ⚠️ The port is chosen ONCE by binding (11435 across 20; 11434 belongs to a runtime the user may run) and stored; it is never searched again on a later start, because it is part of the saved endpoint's URL. ⚠️ Pid and port are in the state file: a listener on the port that answers `/api/version` and whose pid runs a binary under OUR runtime folder is our own orphan (a server that was killed leaves one: a child is not ended by its parent's death) and is adopted or replaced; ANY other listener is `port-in-use`, never something to talk to. ⚠️ The supervisor never writes the state file: it reports through `onChange`, and the job stores what it reports. ⚠️ Under vitest nothing spawns and nothing is fetched unless a test injects it. **The budget is measured.** The runtime prints the device total in its log line "inference compute" (`total="37.4 GiB"`); `parseInferenceComputeTotal()` reads it, the job stores it and judges the chosen model against it (`modelFitsBudget()`) BEFORE the model is downloaded; from then on the GET recommends with `budgetSource: 'measured'`. One consent covers the whole install: the sizes that were confirmed do not change, so a model that no longer fits is a failure with the next model, not a second question. diff --git a/docs/local-models.md b/docs/local-models.md index 48d61634..a3c6d068 100644 --- a/docs/local-models.md +++ b/docs/local-models.md @@ -22,19 +22,19 @@ the catalogue answers today, before the runtime measured anything (the budget is estimate: 2/3 of the memory below 36 GB, 3/4 from 36 GB, less 1 GiB, and never more than the memory less 6, 8 or 12 GiB for macOS, Codeman, a browser and the agent): -| Memory | Budget | What it gets | -| ---------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 8 GB | 2.1 GB | Nothing. The smallest model a coding agent can work with needs 9.8 GB at 65,536 tokens | -| 16 GB | 10.4 GB | A small model (`limited`): MiMo V2.6 9B at 65,536 tokens, 9.8 GB, tight. Its base, Qwen3.5 9B, is the one alternative | -| 18 GB | 11.8 GB | The same at 131,072 tokens, 11.0 GB, tight | -| 24 GB | 16.1 GB | The same at 131,072 tokens, with room | -| 32 GB | 21.8 GB | The same. North Mini Code 1.0 needs 21.8 GB and misses the estimate by a hair; once the runtime measures about 3/4 of the memory, as it did on a 16 and a 48 GB Mac, the Mac gets it, or Qwen3.8 27B on a Max chip | -| 36 GB | 27.9 GB | Qwen3.8 27B at 131,072 tokens on an M4 or M5 Max; on a slower chip, where the dense 27B computes to under 10 tokens per second, Qwen3.6 35B-A3B as a GGUF build | -| 48 GB | 37.6 GB | Qwen3.6 35B-A3B (MLX) at 131,072 tokens: the pair we tested, "Recommended" on an Apple M5 Pro with 48 GB. On a Max chip Qwen3.8 27B at 262,144 tokens. The test was a short task: in a long session MLX may keep up to 8 GiB more, which brings this pair to the edge of the budget, and past it while a long prompt is processed | -| 64 GB | 50.5 GB | Qwen3.8 27B at 262,144 tokens on a Max chip; Qwen3.6 35B-A3B (MLX) on a Pro chip | -| 96 GB | 76.2 GB | Qwen3.8 27B at 262,144 tokens; on an M3 Max, Qwen3.6 35B-A3B (MLX) | -| 128 GB | 102 GB | Ornith 1.5 122B-A10B (MLX) at 131,072 tokens, 97.4 GB, tight | -| 192, 256, 512 GB | 154 GB+ | Ornith 1.5 122B-A10B at 262,144 tokens | +| Memory | Budget | What it gets | +| ---------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 8 GB | 2.1 GB | Nothing. The smallest model a coding agent can work with needs 10.2 GB at 65,536 tokens | +| 16 GB | 10.4 GB | Nothing. MiMo V2.6 9B at 65,536 tokens needs 10.2 GB, 98.7% of the budget, just past the 98% a tight fit may reach (it is 95% of the 10.7 GB a 16 GB Mac mini measured). On the edge either way: only a run on such a Mac settles it | +| 18 GB | 11.8 GB | A small model (`limited`): MiMo V2.6 9B at 131,072 tokens, 11.4 GB, tight. Its base, Qwen3.5 9B, is the one alternative | +| 24 GB | 16.1 GB | The same, with room. On a base M2 or M3 (100 GB/s) it computes to 9.3 tokens per second: slow | +| 32 GB | 21.8 GB | The same. North Mini Code 1.0 needs 23.2 GB at 65,536 tokens; once the runtime measures about 3/4 of the memory, as it did on a 16 and a 48 GB Mac, the Mac gets it at 131,072 tokens, or Qwen3.8 27B at 65,536 tokens on a Max chip | +| 36 GB | 27.9 GB | Qwen3.8 27B at 131,072 tokens on an M4 or M5 Max (27.0 GB, 97%); on a slower chip, where the dense 27B computes to under 10 tokens per second, Qwen3.6 35B-A3B as a GGUF build (27.2 GB, 98%). Both tight | +| 48 GB | 37.6 GB | Qwen3.6 35B-A3B (MLX) at 131,072 tokens: the pair we tested, "Recommended" on an Apple M5 Pro with 48 GB. On a Max chip Qwen3.8 27B at 262,144 tokens (32.1 GB, tight). The test was a short task: in a long session MLX may keep up to 8 GiB more, which brings this pair to the edge of the budget, and past it while a long prompt is processed | +| 64 GB | 50.5 GB | Qwen3.8 27B at 262,144 tokens on a Max chip; Qwen3.6 35B-A3B (MLX) on a Pro chip | +| 96 GB | 76.2 GB | Qwen3.8 27B at 262,144 tokens; on an M3 Max, Qwen3.6 35B-A3B (MLX) | +| 128 GB | 102 GB | Ornith 1.5 122B-A10B (MLX) at 131,072 tokens, 97.4 GB, tight | +| 192, 256, 512 GB | 154 GB+ | Ornith 1.5 122B-A10B at 262,144 tokens | "Tight" means the need is between 85% and 98% of the budget; above 98% a model is not offered. A model that computes to under 10 tokens per second on the chip is "slow" and ranks @@ -54,10 +54,18 @@ names its runner (`runner`) and its figures follow that runner's rule: model loads and does not grow after that. The runtime is started with `OLLAMA_KV_CACHE_TYPE=q8_0` (the cache in 8 bits, 17/32 of the default f16; the runtime's own documentation calls the loss very small), `OLLAMA_FLASH_ATTENTION=1` (an 8-bit cache - needs it) and `LLAMA_ARG_CACHE_RAM=1024` (llama-server keeps copies of earlier prompts, and - this bounds them to 1 GiB). A row's figures are computed from its model's `config.json`: the - K/V cache of its attention layers, its recurrent or sliding-window state, 1.0 GB for the - runner and the 1 GiB prompt cache. No peak: the buffers are allocated at load. + needs it), `LLAMA_ARG_CACHE_RAM=1024` (llama-server keeps copies of earlier prompts, and + this bounds them to 1 GiB) and `LLAMA_ARG_CTX_CHECKPOINTS=8` (llama-server keeps copies of + the state it cannot roll back, two or three per request, up to 32 by default: 5 GB for the + 27B; 8 keeps the last few requests, which is what an agent's next request restores from). A + row's figures are computed from its model's `config.json`: the K/V cache of its attention + layers, its recurrent or sliding-window state, live and in each of the 8 checkpoints, 1.0 + GB for the runner and the 1 GiB prompt cache. A build with a multi-token-prediction layer + (the two `-mtp` tags and `qwen3.8:27b-q8_0`) costs more: the runtime turns prediction on, + so llama-server keeps that layer's cache in f16 and 4 more copies of the recurrent state + for rollback. No peak: the buffers are allocated at load. The checkpoints and the + prediction costs were read in the llama.cpp source the runtime bundles (b12111), not + measured. - **MLX** grows its cache with the tokens actually used, keeps snapshots of the recurrent state, and may keep up to 8 GiB of snapshots of earlier prompts, which no setting bounds. A Mac always takes the MLX build of a tag that has one. Below 48 GB that 8 GiB does not fit @@ -70,17 +78,19 @@ Where every figure comes from: | `qwen3.5:9b-q4_k_m` | llama.cpp | computed from the Qwen3.5-9B config | 9.3: a published run on an M1 Max | | `mimo-v2.6:9b-q4_k_m` | llama.cpp | the same (a fine-tune of Qwen3.5-9B) | 9.3: the same architecture and size | | `north-mini-code-1.0:q4_k_m` | llama.cpp | computed from its config | 13.6: the 35B-A3B's, the same 3 billion active parameters | -| `qwen3.6:35b-a3b-mtp-q4_k_m` | llama.cpp | computed from the Qwen3.6-35B-A3B config | 13.6: a published run on an M1 Max | +| `qwen3.6:35b-a3b-mtp-q4_k_m` | llama.cpp | computed from the Qwen3.6-35B-A3B config, prediction layer included | 13.6: a published run on an M1 Max | | `qwen3.6:35b-a3b` | MLX | **measured** on an Apple M5 Pro with 48 GB | 16.9: a published run; 67.7 measured on the tested Mac | -| `qwen3.8:27b-q8_0` | llama.cpp | computed from the Qwen3.8-27B config | 1.7: the row below scaled by file size | -| `qwen3.8:27b-mtp-q4_k_m` | llama.cpp | computed from the Qwen3.8-27B config | 2.95: a published run without prediction (a floor) | +| `qwen3.8:27b-q8_0` | llama.cpp | computed from the Qwen3.8-27B config, prediction layer included | 1.7: the row below scaled by file size | +| `qwen3.8:27b-mtp-q4_k_m` | llama.cpp | computed from the Qwen3.8-27B config, prediction layer included | 2.95: a published run without prediction (a floor) | | `ornith-1.5:122b` | MLX | computed from the Qwen3.5-122B-A10B config, scaled by the measured MLX row, + 8 GiB | 5.0: the 35B-A3B's, x 3/10 active parameters | Sizes are what the registry serves a Mac for each tag (every layer plus the config), read on 2026-11-08. Only one row was ever run on a Mac, so every other answer in the table above is a -calculation. For a llama.cpp build the check after the download catches one that was judged -too kindly; for an MLX build it cannot (see "The check" below). The measured row's figures come from single long prompts, not a session of many requests, in -which MLX may keep up to 8 GiB more. The catalogue header lists the models left out on +calculation. For a llama.cpp build the check after the download catches a load that was +judged too kindly, but not what grows after it in host memory (the checkpoints and the prompt +cache, which are counted, not checked); for an MLX build it catches neither (see "The check" +below). The measured row's figures come from single long prompts, not a session of many +requests, in which MLX may keep up to 8 GiB more. The catalogue header lists the models left out on purpose (no evidence of working tool calls, a restrictive licence, a known bug) and why. A new row needs its model's architecture (attention layers, K/V heads, head size, and its @@ -124,10 +134,10 @@ Every step can be repeated: Resume and "Try again" enter the step the job stoppe copied from the server. `HOME` is a private folder. Exactly these are set: `OLLAMA_HOST=127.0.0.1:`, `OLLAMA_MODELS`, `OLLAMA_CONTEXT_LENGTH`, `OLLAMA_NO_CLOUD=1`, `OLLAMA_KEEP_ALIVE=10m`, `OLLAMA_NUM_PARALLEL=1`, - `OLLAMA_MAX_LOADED_MODELS=1`, and three memory settings of the llama.cpp runner: - `OLLAMA_KV_CACHE_TYPE=q8_0`, `OLLAMA_FLASH_ATTENTION=1` and `LLAMA_ARG_CACHE_RAM=1024` - (see "How much memory a model needs" above). No `CODEMAN_*` variable and none of the user's - own `OLLAMA_*` settings reach it. + `OLLAMA_MAX_LOADED_MODELS=1`, and four memory settings of the llama.cpp runner: + `OLLAMA_KV_CACHE_TYPE=q8_0`, `OLLAMA_FLASH_ATTENTION=1`, `LLAMA_ARG_CACHE_RAM=1024` and + `LLAMA_ARG_CTX_CHECKPOINTS=8` (see "How much memory a model needs" above). No `CODEMAN_*` + variable and none of the user's own `OLLAMA_*` or `LLAMA_ARG_*` settings reach it. - **The port is chosen once**, by binding, from 11435 across 20 ports, and stored. 11434 is left alone: it is the default of a runtime the user may run themselves, which is never touched. diff --git a/src/config/local-model-catalog.ts b/src/config/local-model-catalog.ts index 3821534f..9003d5bf 100644 --- a/src/config/local-model-catalog.ts +++ b/src/config/local-model-catalog.ts @@ -68,23 +68,35 @@ * (src/local-model/recommend.ts). The two runners spend memory differently, so a row's three * figures are filled by the rule of its runner: * - * - `llamacpp` (llama-server on Metal; runtime v0.40.15 source, llm/llama_server.go). It - * allocates the K/V cache for the WHOLE context when the model loads, in the cache type the - * runtime sets (`RUNTIME_KV_CACHE_TYPE` in src/local-model/runtime.ts, q8_0: 34 bytes per - * 32 values), and does not grow after that. - * - held = attention layers x K/V heads x head dim x 2 (K and V) x 34/32. The one - * multi-token-prediction layer of the `-mtp` tags is counted as one more attention - * layer, and so it is for `qwen3.8:27b-q8_0`, whose GGUF may carry it too: neither - * whether that GGUF has it nor whether llama.cpp gives it a cache was checked, so this - * errs on the careful side. + * - `llamacpp` (llama-server on Metal; runtime v0.40.15 source, llm/llama_server.go, and the + * llama.cpp it bundles, b12111). It allocates the K/V cache for the WHOLE context when the + * model loads, in the cache type the runtime sets (`RUNTIME_KV_CACHE_TYPE` in + * src/local-model/runtime.ts, q8_0: 34 bytes per 32 values), and does not grow after that. + * "MTP" below is a GGUF with a multi-token-prediction layer (`nextn_predict_layers` 1 in + * its header, read on 2026-11-08 from the first bytes of each model layer: the two `-mtp` + * tags and `qwen3.8:27b-q8_0` have it, the two 9B tags and North Mini Code do not). For + * such a model the runtime always passes `--spec-type draft-mtp --spec-draft-n-max 4` + * (its default `draft_num_predict` is 4). + * - held = attention layers x K/V heads x head dim x 2 (K and V) x 34/32. An MTP model + * adds its prediction layer, whose cache llama-server keeps in a draft context of the + * same length in the draft's own cache type, f16 by default (2 bytes per value). * - overhead = the fixed caches (the f32 recurrent state of the linear-attention layers, - * or the sliding-window cache of `n_swa + 512` cells), + 1.0 GB for the runner's own - * buffers (calibrated on a third-party table of a 27B model on llama.cpp: 8.4 GB at - * 4,096 tokens with 7.15 GB of weights), + the 1024 MiB host prompt cache that - * `LLAMA_ARG_CACHE_RAM` allows (`RUNTIME_HOST_PROMPT_CACHE_MIB`). + * or the sliding-window cache of `n_swa + 512` cells), held 1 + 4 times for an MTP + * model (llama.cpp keeps one recurrent state per drafted token for rollback, + * `need_n_rs_seq()`), + the context checkpoints: `LLAMA_ARG_CTX_CHECKPOINTS` host copies + * (`RUNTIME_CTX_CHECKPOINTS`, 8) of the state that cannot be rolled back (the recurrent + * state, or the sliding-window cache), + 1.0 GB for the runner's own buffers (calibrated + * on a third-party table of a 27B model on llama.cpp: 8.4 GB at 4,096 tokens with + * 7.15 GB of weights), + the 1024 MiB host prompt cache that `LLAMA_ARG_CACHE_RAM` + * allows (`RUNTIME_HOST_PROMPT_CACHE_MIB`). The checkpoints and the prompt cache are + * host memory, not Metal buffers; on a Mac both come out of the same memory, so they + * are counted like the rest. * - peak = 0: its compute buffers are allocated at load and are inside the 1.0 GB. * The one outside check of this rule: a third party measured 65,600 bytes per token of * growth at f16 for a model of the Qwen3.8-27B architecture, where its config gives 65,536. + * The checkpoints and the rollback copies are read in the llama.cpp source + * (tools/server/server-context.cpp `create_checkpoint`, src/llama-memory-recurrent.cpp), + * not measured. * - `mlx` (runtime v0.40.15 source, mlxrunner/). It grows the K/V cache in bf16 with the * tokens used, keeps the recurrent state in f32 and a snapshot of it every 8,192 tokens, * and keeps up to 8 GiB of paged-out prefix snapshots (prefix_cache.go `maxPagedOutBytes`, @@ -106,9 +118,10 @@ * After the download the install checks that the runtime holds the whole model on the GPU. * For a `llamacpp` row both of the runtime's figures are the buffers llama-server reports * allocating (weights, K/V cache, compute; `MemorySize()` in llm/llama_server.go), so the - * check proves the whole load is on the GPU, whatever the cache type. For an `mlx` row it - * proves only the weights (issue 19417): the figures above are all that stands between such - * a row and swap. + * check proves the whole load is on the GPU, whatever the cache type; what grows after the + * load in host memory (the context checkpoints and the prompt cache) is counted above, not + * checked. For an `mlx` row it proves only the weights (issue 19417): the figures above are + * all that stands between such a row and swap. * * Models set aside on purpose (a weekly review should look at them again): * - `bonsai-2:27b`, `bonsai-2:9b` (ternary builds of Qwen3.8-27B and Qwen3.5-9B; the 27B @@ -308,6 +321,10 @@ const MEASURED_MEMORY = { const LLAMACPP_RUNNER_BYTES = 1_000_000_000; /** The host prompt cache llama-server may keep: `LLAMA_ARG_CACHE_RAM`, 1024 MiB (`RUNTIME_HOST_PROMPT_CACHE_MIB`). */ const LLAMACPP_PROMPT_CACHE_BYTES = 1024 * 1024 * 1024; +/** The context checkpoints llama-server may keep: `LLAMA_ARG_CTX_CHECKPOINTS`, 8 (`RUNTIME_CTX_CHECKPOINTS`). */ +const LLAMACPP_CTX_CHECKPOINTS = 8; +/** The recurrent states llama.cpp keeps beside the live one for an MTP model: the runtime's draft length, 4. */ +const LLAMACPP_MTP_ROLLBACK_STATES = 4; /** The prefix snapshots the MLX runner may keep: `maxPagedOutBytes`, 8 GiB. */ const MLX_PREFIX_CACHE_BYTES = 8 * 1024 * 1024 * 1024; @@ -334,8 +351,9 @@ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { bytes: 6_550_825_550, // 8 attention layers x 4 K/V heads x 256 x 2 x 34/32. heldBytesPerToken: 17_408, - // 24 linear-attention layers hold 52,690,944 bytes of f32 state. - overheadBytes: 52_690_944 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + // 24 linear-attention layers hold 52,690,944 bytes of f32 state: once live, and once in + // each context checkpoint. + overheadBytes: 52_690_944 * (1 + LLAMACPP_CTX_CHECKPOINTS) + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, peakExtraBytes: 0, figures: 'estimated', contexts: [131072, 65536], @@ -357,7 +375,7 @@ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { bytes: 6_550_811_033, // The architecture and the quantization of the row above, so its figures. heldBytesPerToken: 17_408, - overheadBytes: 52_690_944 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + overheadBytes: 52_690_944 * (1 + LLAMACPP_CTX_CHECKPOINTS) + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, peakExtraBytes: 0, figures: 'estimated', contexts: [131072, 65536], @@ -378,8 +396,9 @@ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { bytes: 18_593_967_008, // 13 full-attention layers x 4 K/V heads x 128 x 2 x 34/32. heldBytesPerToken: 14_144, - // 36 sliding-window layers: (4,096 + 512) cells x 4 x 128 x 2 x 34/32 = 180,486,144. - overheadBytes: 180_486_144 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + // 36 sliding-window layers: (4,096 + 512) cells x 4 x 128 x 2 x 34/32 = 180,486,144, live + // and in each context checkpoint. + overheadBytes: 180_486_144 * (1 + LLAMACPP_CTX_CHECKPOINTS) + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, peakExtraBytes: 0, figures: 'estimated', contexts: [262144, 131072, 65536], @@ -398,10 +417,15 @@ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { paramsB: 35, runner: 'llamacpp', bytes: 22_621_314_381, - // (10 attention layers + 1 prediction layer) x 2 K/V heads x 256 x 2 x 34/32. - heldBytesPerToken: 11_968, - // 30 linear-attention layers hold 65,863,680 bytes of f32 state. - overheadBytes: 65_863_680 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + // 10 attention layers x 2 K/V heads x 256 x 2 x 34/32 = 10,880, + the prediction layer in + // f16: 2 x 256 x 2 x 2 = 2,048. + heldBytesPerToken: 12_928, + // 30 linear-attention layers hold 65,863,680 bytes of f32 state: live, in the 4 rollback + // copies of prediction, and in each context checkpoint. + overheadBytes: + 65_863_680 * (1 + LLAMACPP_MTP_ROLLBACK_STATES + LLAMACPP_CTX_CHECKPOINTS) + + LLAMACPP_RUNNER_BYTES + + LLAMACPP_PROMPT_CACHE_BYTES, peakExtraBytes: 0, figures: 'estimated', contexts: [262144, 131072, 65536], @@ -438,9 +462,13 @@ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { paramsB: 27.8, runner: 'llamacpp', bytes: 29_978_242_050, - // The architecture of the row below, its prediction layer counted (file header). - heldBytesPerToken: 36_992, - overheadBytes: 156_893_184 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + // The architecture of the row below, prediction layer included (its GGUF has one), so its + // figures. + heldBytesPerToken: 38_912, + overheadBytes: + 156_893_184 * (1 + LLAMACPP_MTP_ROLLBACK_STATES + LLAMACPP_CTX_CHECKPOINTS) + + LLAMACPP_RUNNER_BYTES + + LLAMACPP_PROMPT_CACHE_BYTES, peakExtraBytes: 0, figures: 'estimated', contexts: [262144, 131072, 65536], @@ -458,10 +486,15 @@ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { paramsB: 27.8, runner: 'llamacpp', bytes: 17_741_872_154, - // (16 attention layers + 1 prediction layer) x 4 K/V heads x 256 x 2 x 34/32. - heldBytesPerToken: 36_992, - // 48 linear-attention layers hold 156,893,184 bytes of f32 state. - overheadBytes: 156_893_184 + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES, + // 16 attention layers x 4 K/V heads x 256 x 2 x 34/32 = 34,816, + the prediction layer in + // f16: 4 x 256 x 2 x 2 = 4,096. + heldBytesPerToken: 38_912, + // 48 linear-attention layers hold 156,893,184 bytes of f32 state: live, in the 4 rollback + // copies of prediction, and in each context checkpoint. + overheadBytes: + 156_893_184 * (1 + LLAMACPP_MTP_ROLLBACK_STATES + LLAMACPP_CTX_CHECKPOINTS) + + LLAMACPP_RUNNER_BYTES + + LLAMACPP_PROMPT_CACHE_BYTES, peakExtraBytes: 0, figures: 'estimated', contexts: [262144, 131072, 65536], diff --git a/src/local-model/runtime.ts b/src/local-model/runtime.ts index 08f15d1a..a67ce5b1 100644 --- a/src/local-model/runtime.ts +++ b/src/local-model/runtime.ts @@ -16,9 +16,9 @@ * either case) are copied from the server's; HOME is the private one; and exactly these are * set: `OLLAMA_HOST=127.0.0.1:`, `OLLAMA_MODELS`, `OLLAMA_CONTEXT_LENGTH`, * `OLLAMA_NO_CLOUD=1`, `OLLAMA_KEEP_ALIVE=10m`, `OLLAMA_NUM_PARALLEL=1`, - * `OLLAMA_MAX_LOADED_MODELS=1`, and the three memory settings of the llama.cpp runner (see + * `OLLAMA_MAX_LOADED_MODELS=1`, and the four memory settings of the llama.cpp runner (see * MEMORY below). Nothing else gets through: no `CODEMAN_*` and none of the user's own - * `OLLAMA_*`. + * `OLLAMA_*` or `LLAMA_ARG_*`. * - MEMORY (what the catalogue's figures for its `llamacpp` rows assume, so change both * together: test/local-model-catalog.test.ts holds them to each other). Read in the * runtime's v0.40.15 source, not yet measured on a Mac: @@ -34,7 +34,20 @@ * for a coding model, and llama-server reads the variable from the environment it * inherits (`cmd.Env = os.Environ()`). Unbounded, that cache can grow by gigabytes in a * session; 1024 MiB is counted in every `llamacpp` row's `overheadBytes`. - * The MLX runner reads none of the three (nothing under mlxrunner/ does): an `mlx` row's + * - `LLAMA_ARG_CTX_CHECKPOINTS=8` bounds the context checkpoints llama-server keeps per + * slot (`--ctx-checkpoints`, read from the environment the same way). The llama.cpp the + * runtime bundles (b12111, the runtime's LLAMA_CPP_VERSION) makes them for every model + * of the catalogue's `llamacpp` rows: a checkpoint is a host copy of the state that + * cannot be rolled back (the recurrent state of the linear-attention layers, or the + * sliding-window cache), two or three per request, and llama-server keeps up to 32 by + * default. 32 copies of the 27B's 157 MB state are 5 GB that the session holds for + * nothing: an agent's next request restores the newest checkpoint of the previous one. + * 8 (llama.cpp's own default before it went to 32 to cover a 262,144-token window in + * 8,192-token steps) keeps the last few requests. The cost: a prompt that differs from + * the last checkpoints further back (a compaction, a cleared old tool result) is + * processed again from the newest checkpoint before that point, or from its start. + * 8 checkpoints are counted in every `llamacpp` row's `overheadBytes`. + * The MLX runner reads none of the four (nothing under mlxrunner/ does): an `mlx` row's * figures do not depend on them. * - PORT (`chooseRuntimePort()`). Chosen ONCE by binding, from `RUNTIME_PORT_FIRST` across * `RUNTIME_PORT_SPAN`, then stored by the caller. Never searched again on a later start. @@ -105,6 +118,8 @@ export const RUNTIME_KEEP_ALIVE = '10m'; export const RUNTIME_KV_CACHE_TYPE = 'q8_0'; /** `LLAMA_ARG_CACHE_RAM`, in MiB: the bound of llama-server's host prompt cache. Counted in every `llamacpp` row's overhead. */ export const RUNTIME_HOST_PROMPT_CACHE_MIB = 1024; +/** `LLAMA_ARG_CTX_CHECKPOINTS`: the context checkpoints llama-server keeps per slot. Counted in every `llamacpp` row's overhead. */ +export const RUNTIME_CTX_CHECKPOINTS = 8; /** The name of the runtime's binary inside its folder and inside the archive. */ const RUNTIME_BINARY_NAME = 'ollama'; @@ -279,6 +294,7 @@ export function runtimeEnv(input: RuntimeEnvInput): NodeJS.ProcessEnv { env.OLLAMA_KV_CACHE_TYPE = RUNTIME_KV_CACHE_TYPE; env.OLLAMA_FLASH_ATTENTION = '1'; env.LLAMA_ARG_CACHE_RAM = String(RUNTIME_HOST_PROMPT_CACHE_MIB); + env.LLAMA_ARG_CTX_CHECKPOINTS = String(RUNTIME_CTX_CHECKPOINTS); return env; } diff --git a/src/types/local-model.ts b/src/types/local-model.ts index 1cc71e6c..bc1b5684 100644 --- a/src/types/local-model.ts +++ b/src/types/local-model.ts @@ -120,10 +120,10 @@ export interface TestedOn { tokensPerSecond?: number; } -/** One model of the catalogue. */ /** The runner of the model runtime that runs a model on a Mac, by the registry's name for it. See `LocalModelEntry.runner`. */ export type LocalModelRunner = 'mlx' | 'llamacpp'; +/** One model of the catalogue. */ export interface LocalModelEntry { /** `name:tag` exactly as the model registry lists it. Lowercase, one colon, no slash (`LOCAL_MODEL_ID_PATTERN`). */ id: string; @@ -156,7 +156,7 @@ export interface LocalModelEntry { /** * Bytes that stay HELD in memory per token of context. For a `llamacpp` row: the K/V cache * of its attention layers in the runtime's cache type, which llama.cpp allocates for the - * whole context at load. For an `mlx` row: the K/V cache in bf16 plus the snapshots of the + * whole context at load (a multi-token-prediction layer's in f16). For an `mlx` row: the K/V cache in bf16 plus the snapshots of the * recurrent state the runner keeps every 8,192 tokens, which it grows with use. Measured * on one row (36,000 to 38,000 on the MLX runner, so about 37,000); computed from the * model's config for the others, see `figures` and the catalogue header. @@ -165,9 +165,10 @@ export interface LocalModelEntry { /** * What the runner holds beside the weights and the context, whatever the context: its own * buffers, the live recurrent or sliding-window caches, and the caches it keeps of earlier - * prompts (for `llamacpp` the 1024 MiB host prompt cache `LLAMA_ARG_CACHE_RAM` allows, for - * an estimated `mlx` row the 8 GiB of prefix snapshots the runner may keep). Measured on - * one row (about 3 GB); see the catalogue header for the others. + * prompts (for `llamacpp` the 1024 MiB host prompt cache `LLAMA_ARG_CACHE_RAM` allows and + * the copies of the recurrent or sliding-window state in its `LLAMA_ARG_CTX_CHECKPOINTS` + * context checkpoints, for an estimated `mlx` row the 8 GiB of prefix snapshots the runner + * may keep). Measured on one row (about 3 GB); see the catalogue header for the others. */ overheadBytes: number; /** diff --git a/test/local-model-catalog.test.ts b/test/local-model-catalog.test.ts index f53815a4..a3760f68 100644 --- a/test/local-model-catalog.test.ts +++ b/test/local-model-catalog.test.ts @@ -31,7 +31,12 @@ import { LocalModelCatalogSchema, STOCK_LOCAL_MODEL_CATALOG, } from '../src/config/local-model-catalog.js'; -import { RUNTIME_HOST_PROMPT_CACHE_MIB, RUNTIME_KV_CACHE_TYPE, runtimeEnv } from '../src/local-model/runtime.js'; +import { + RUNTIME_CTX_CHECKPOINTS, + RUNTIME_HOST_PROMPT_CACHE_MIB, + RUNTIME_KV_CACHE_TYPE, + runtimeEnv, +} from '../src/local-model/runtime.js'; import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js'; /** A deep copy of the stock catalogue that a test may break. */ @@ -79,10 +84,12 @@ type ArchitectureName = keyof typeof ARCHITECTURES; type Architecture = (typeof ARCHITECTURES)[ArchitectureName]; /** - * Which architecture each stock row runs, and whether its GGUF carries the one - * multi-token-prediction layer (counted as one more attention layer, see the catalogue - * header). A row the catalogue gains must be added here, or the test below fails: its figures - * cannot be checked without it. + * Which architecture each stock row runs, and whether its GGUF carries a + * multi-token-prediction layer (`nextn_predict_layers` 1 in the GGUF header, read on + * 2026-11-08 from the first bytes of each model layer). For such a model the runtime turns on + * prediction, which costs a cache for that layer and rollback copies of the recurrent state + * (see the catalogue header). A row the catalogue gains must be added here, or the test below + * fails: its figures cannot be checked without it. */ const ROW_ARCHITECTURE: Record = { 'qwen3.5:9b-q4_k_m': { architecture: 'Qwen3.5-9B', predictionLayer: false }, @@ -97,6 +104,19 @@ const ROW_ARCHITECTURE: Record = { f16: 2, q8_0: 34 / 32 }; +/** + * The cache type of llama-server's draft context, which holds the prediction layer's cache: its + * default, because the runtime sets only the main context's (`--cache-type-k`). + */ +const DRAFT_KV_CACHE_TYPE = 'f16'; +/** + * The recurrent states llama.cpp keeps beside the live one for a model with prediction: the + * runtime's draft length (`draft_num_predict` 4, passed as `--spec-draft-n-max`), which + * `need_n_rs_seq()` turns into that many rollback copies. + */ +const MTP_ROLLBACK_STATES = 4; +/** The sliding-window cache holds the window plus one batch of 512 tokens (`n_swa + n_ubatch`). */ +const SLIDING_EXTRA_CELLS = 512; /** The head size of a Gated DeltaNet layer's keys and values, and its convolution kernel, in every model above. */ const LINEAR_HEAD_DIM = 128; const LINEAR_CONV_KERNEL = 4; @@ -114,15 +134,20 @@ function recurrentStateBytes(arch: Architecture, convBytes: number): number { /** What a `llamacpp` row's three figures must be, from its architecture and the runtime's settings. */ function llamaCppFigures(arch: Architecture, predictionLayer: boolean): [number, number, number] { const perValue = KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE]; - const layers = arch.attentionLayers + (predictionLayer ? 1 : 0); - const held = layers * arch.kvHeads * arch.headDim * 2 * perValue; + const perLayer = arch.kvHeads * arch.headDim * 2; + // The prediction layer's cache sits in the draft context, in the draft's cache type. + const held = + arch.attentionLayers * perLayer * perValue + + (predictionLayer ? perLayer * KV_BYTES_PER_VALUE[DRAFT_KV_CACHE_TYPE] : 0); // llama.cpp keeps the recurrent state in f32 (both parts), the sliding window in the cache type. + const recurrent = recurrentStateBytes(arch, 4); const sliding = - 'sliding' in arch - ? arch.sliding.layers * (arch.sliding.window + 512) * arch.kvHeads * arch.headDim * 2 * perValue - : 0; - const fixed = recurrentStateBytes(arch, 4) + sliding; - return [held, fixed + 1_000_000_000 + RUNTIME_HOST_PROMPT_CACHE_MIB * MIB, 0]; + 'sliding' in arch ? arch.sliding.layers * (arch.sliding.window + SLIDING_EXTRA_CELLS) * perLayer * perValue : 0; + // Live: the sliding window, and the recurrent state with its rollback copies under prediction. + const live = sliding + recurrent * (1 + (predictionLayer ? MTP_ROLLBACK_STATES : 0)); + // A context checkpoint is a copy of what cannot be rolled back: the recurrent state or the window. + const checkpoints = RUNTIME_CTX_CHECKPOINTS * (recurrent + sliding); + return [held, live + checkpoints + 1_000_000_000 + RUNTIME_HOST_PROMPT_CACHE_MIB * MIB, 0]; } /** The per-token figure of the MLX runner computed from an architecture: K/V in bf16, a recurrent snapshot per 8,192 tokens. */ @@ -304,7 +329,10 @@ describe('the memory figures of the stock rows: each row has its own', () => { const env = runtimeEnv({ source: {}, home: '/h', modelsDir: '/m', port: 11435, contextLength: 65536 }); expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE); expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB)); + expect(env.LLAMA_ARG_CTX_CHECKPOINTS).toBe(String(RUNTIME_CTX_CHECKPOINTS)); expect(KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE]).toBeDefined(); + // The runtime sets no cache type for the draft context, so the prediction layer's stays f16. + expect(Object.keys(env).filter((name) => /DRAFT|SPEC/.test(name))).toEqual([]); }); it("recomputes every estimated row's figures from its own model's architecture and runner", () => { @@ -318,15 +346,22 @@ describe('the memory figures of the stock rows: each row has its own', () => { it('states the numbers the header and the row comments name', () => { expect(Object.fromEntries(models.map((m) => [m.id, figuresOf(m)]))).toEqual({ - 'qwen3.5:9b-q4_k_m': [17_408, 2_126_432_768, 0], - 'mimo-v2.6:9b-q4_k_m': [17_408, 2_126_432_768, 0], - 'north-mini-code-1.0:q4_k_m': [14_144, 2_254_227_968, 0], - 'qwen3.6:35b-a3b-mtp-q4_k_m': [11_968, 2_139_605_504, 0], + // State x (1 live + 8 checkpoints) + 1.0 GB + 1 GiB. + 'qwen3.5:9b-q4_k_m': [17_408, 2_547_960_320, 0], + 'mimo-v2.6:9b-q4_k_m': [17_408, 2_547_960_320, 0], + // The sliding window x (1 live + 8 checkpoints) + 1.0 GB + 1 GiB. + 'north-mini-code-1.0:q4_k_m': [14_144, 3_698_117_120, 0], + // State x (1 live + 4 rollback copies + 8 checkpoints) + 1.0 GB + 1 GiB; the prediction + // layer's cache in f16 per token. + 'qwen3.6:35b-a3b-mtp-q4_k_m': [12_928, 2_929_969_664, 0], 'qwen3.6:35b-a3b': [37_000, 3_000_000_000, 5_000_000_000], - 'qwen3.8:27b-q8_0': [36_992, 2_230_635_008, 0], - 'qwen3.8:27b-mtp-q4_k_m': [36_992, 2_230_635_008, 0], + 'qwen3.8:27b-q8_0': [38_912, 4_113_353_216, 0], + 'qwen3.8:27b-mtp-q4_k_m': [38_912, 4_113_353_216, 0], 'ornith-1.5:122b': [56_574, 11_589_934_592, 5_000_000_000], }); + // The two parts of a llamacpp overhead that are not computed from the architecture. + expect(RUNTIME_CTX_CHECKPOINTS).toBe(8); + expect(RUNTIME_HOST_PROMPT_CACHE_MIB).toBe(1024); // The MLX calibration: 37,000 held against 28,340 computed for the measured row. expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B'])).toBe(28_340); expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.5-122B-A10B'])).toBe(43_332); diff --git a/test/local-model-fits-budget.test.ts b/test/local-model-fits-budget.test.ts index c02c053a..3b112150 100644 --- a/test/local-model-fits-budget.test.ts +++ b/test/local-model-fits-budget.test.ts @@ -97,10 +97,10 @@ describe('modelFitsBudget', () => { // A runtime that reports less than the estimate rules out what the estimate allowed. expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 33 * 1024 ** 3 })).toBe(false); // And one that reports more admits what the estimate ruled out: on a 36 GB Mac the GGUF - // 35B-A3B at its full window needs 27.9 GB, 1.0003 of the 27.9 GB estimate and 0.962 of 27 GiB. + // 35B-A3B at its full window needs 28.9 GB, 1.037 of the 27.9 GB estimate and 0.963 of 28 GiB. const moeGguf = 'qwen3.6:35b-a3b-mtp-q4_k_m'; expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144)).toBe(false); - expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144, { measuredTotalBytes: 27 * 1024 ** 3 })).toBe(true); + expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144, { measuredTotalBytes: 28 * 1024 ** 3 })).toBe(true); // The reserve still bounds it: 100 GiB "measured" on a 48 GB Mac is 40 GiB (42.9 GB) of // budget, so the 122B with its 97.4 GB stays out although 100 GiB would hold it. const huge = { measuredTotalBytes: 100 * 1024 ** 3 }; diff --git a/test/local-model-recommend.test.ts b/test/local-model-recommend.test.ts index c98a765c..59139d03 100644 --- a/test/local-model-recommend.test.ts +++ b/test/local-model-recommend.test.ts @@ -464,10 +464,10 @@ describe('needBytes', () => { }); it('grows with the context by what one token holds', () => { - // A llama.cpp row: 6,550,811,033 + 17,408 x 65,536 + 2,126,432,768, and no peak. + // A llama.cpp row: 6,550,811,033 + 17,408 x 65,536 + 2,547,960,320, and no peak. const nineB = stockModel('mimo-v2.6:9b-q4_k_m'); - expect(needBytes(nineB, 65536)).toBe(9_818_094_489); - expect(needBytes(nineB, 131072)).toBe(10_958_945_177); + expect(needBytes(nineB, 65536)).toBe(10_239_622_041); + expect(needBytes(nineB, 131072)).toBe(11_380_472_729); // Twice the context, twice what the context holds: 17,408 x 65,536 more. expect(needBytes(nineB, 131072) - needBytes(nineB, 65536)).toBe(1_140_850_688); }); @@ -486,13 +486,13 @@ describe('needBytes', () => { it('states the need of every stock row at its largest context', () => { expect(Object.fromEntries(STOCK.models.map((m) => [m.id, needBytes(m, m.contexts[0])]))).toEqual({ - 'qwen3.5:9b-q4_k_m': 10_958_959_694, - 'mimo-v2.6:9b-q4_k_m': 10_958_945_177, - 'north-mini-code-1.0:q4_k_m': 24_555_959_712, - 'qwen3.6:35b-a3b-mtp-q4_k_m': 27_898_259_277, + 'qwen3.5:9b-q4_k_m': 11_380_487_246, + 'mimo-v2.6:9b-q4_k_m': 11_380_472_729, + 'north-mini-code-1.0:q4_k_m': 25_999_848_864, + 'qwen3.6:35b-a3b-mtp-q4_k_m': 28_940_281_677, 'qwen3.6:35b-a3b': 35_955_462_956, - 'qwen3.8:27b-q8_0': 41_906_107_906, - 'qwen3.8:27b-mtp-q4_k_m': 29_669_738_010, + 'qwen3.8:27b-q8_0': 44_292_142_594, + 'qwen3.8:27b-mtp-q4_k_m': 32_055_772_698, 'ornith-1.5:122b': 104_808_538_800, }); }); @@ -647,7 +647,7 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { expect( result.alternatives.map((c) => [c.modelId, c.context, c.fit, c.speed, c.tokensPerSecond ?? null, c.tag?.kind]) ).toEqual([ - // The same model as a GGUF build: its fixed footprint holds the full window, 27.9 GB. + // The same model as a GGUF build: its fixed footprint holds the full window, 28.9 GB. ['qwen3.6:35b-a3b-mtp-q4_k_m', 262144, 'good', 'ok', 41.8, undefined], ['north-mini-code-1.0:q4_k_m', 262144, 'good', 'ok', 41.8, undefined], ['mimo-v2.6:9b-q4_k_m', 131072, 'good', 'ok', 28.6, 'small'], @@ -655,7 +655,7 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { ]); // Only the tested row rests on a measurement of that model. expect(result.alternatives.map((c) => c.needSource)).toEqual(['estimated', 'estimated', 'estimated', 'estimated']); - // The dense 27B fits too (29.7 GB at 262144) but computes to 2.95 x 3.07 = 9.1 tokens per + // The dense 27B fits too (32.1 GB at 262144) but computes to 2.95 x 3.07 = 9.1 tokens per // second here: slow, so it ranks after four fast models and is not listed. expect(modelFitsBudget(await macbook(), STOCK, 'qwen3.8:27b-mtp-q4_k_m', 262144)).toBe(true); expect(order(result).some((entry) => entry.startsWith('qwen3.8:27b'))).toBe(false); @@ -699,43 +699,72 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { /** - * Expected: verdict `limited`, MiMo V2.6 9B (a GGUF build) at 65,536 tokens with OpenCode, - * a TIGHT fit, about 11 tokens per second. + * Expected: verdict `none` (reason `memory`) on the estimated budget, and `limited` with + * MiMo V2.6 9B (a GGUF build) at 65,536 tokens, a TIGHT fit, once the runtime measured its + * budget. * - * Why. The budget is 9.67 GiB (10.4 GB: 2/3 of 16 GiB minus 1). The 9B on llama.cpp needs - * 6.55 GB of weights + 1.14 GB for a q8_0 K/V cache of 65,536 tokens + 2.13 GB beside them - * (its recurrent state, the runner, and the 1 GiB prompt cache the runtime allows) = 9.8 GB: - * 0.946 of the budget. At 131,072 tokens it would need 11.0 GB, so it steps down. Nothing of - * 12 billion parameters or more fits, so the verdict is `limited`. + * Why. The estimated budget is 9.67 GiB (10.4 GB: 2/3 of 16 GiB minus 1). The 9B on + * llama.cpp needs 6.55 GB of weights + 1.14 GB for a q8_0 K/V cache of 65,536 tokens + + * 2.55 GB beside them (its recurrent state live and in 8 context checkpoints, the runner, and + * the 1 GiB prompt cache the runtime allows) = 10.2 GB: 0.987 of the estimate, just past the + * 0.98 a tight fit may reach. The runtime measured 11.8 GiB on this Mac; the 6 GiB reserve + * leaves 10 GiB (10.7 GB) of it, and the same 10.2 GB is 0.954 of that. * * Every figure of this row is computed from the model's architecture, none was measured - * (`needSource: 'estimated'`); the check after the download (the whole model on the GPU) - * is what catches it if llama.cpp holds more. + * (`needSource: 'estimated'`), so this Mac is on the edge either way. Only a run of the 9B on + * it settles which side. */ - it('gets a small model: the 9B at 65536 tokens, tight', async () => { + it('gets no model on the estimated budget: the 9B needs 0.987 of it', async () => { const hw = await macMini(); expect(hw.chip).toBe('Apple M4'); expect(hw.memClassGb).toBe(16); const result = recommend(hw, STOCK, ALL_INSTALLED); - expect(result.verdict).toBe('limited'); - expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.headlineLimited }); - expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + expect(result).toMatchObject({ verdict: 'none', reason: 'memory', budgetSource: 'estimated' }); + expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.noneMemory, vars: { memory: '16' } }); + expect(result.pick).toBeUndefined(); // floor(17,179,869,184 x 2/3) - 1 GiB. expect(result.budgetBytes).toBe(10_379_504_298); expect(result.budgetGb).toBe('10.4'); + const nineB = stockModel('mimo-v2.6:9b-q4_k_m'); + // 6,550,811,033 + 17,408 x 65,536 + 2,547,960,320. + expect(needBytes(nineB, 65536)).toBe(10_239_622_041); + expect(needBytes(nineB, 65536) / result.budgetBytes).toBeGreaterThan(FIT_TIGHT_MAX_SHARE); + expect(needBytes(nineB, 65536) / result.budgetBytes).toBeLessThan(0.99); + }); + + it('would need 11.4 GB at 131072 tokens, more than the whole budget', async () => { + const hw = await macMini(); + const nineB = stockModel('mimo-v2.6:9b-q4_k_m'); + const budget = deviceBudget(hw).budgetBytes; + expect(formatGb(needBytes(nineB, 131072))).toBe('11.4'); + expect(needBytes(nineB, 131072) / budget).toBeGreaterThan(1); + expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 131072)).toBe(false); + expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 65536)).toBe(false); + }); + + it('gets a small model with the budget its runtime measured: the 9B at 65536 tokens, tight', async () => { + // The runtime reported 11.8 GiB here; the 6 GiB reserve leaves 10 GiB (10.7 GB) of it. + const result = recommend(await macMini(), STOCK, ALL_INSTALLED, { measuredTotalBytes: MACMINI_MEASURED_TOTAL }); + expect(result).toMatchObject({ + verdict: 'limited', + budgetBytes: 10 * GIB, + budgetGb: '10.7', + budgetSource: 'measured', + }); + expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.headlineLimited }); + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); expect(result.pick).toMatchObject({ modelId: 'mimo-v2.6:9b-q4_k_m', modelLabel: 'MiMo V2.6 9B', license: 'MIT', context: 65536, cli: 'opencode', - // 6,550,811,033 + 17,408 x 65,536 + 2,126,432,768. - needBytes: 9_818_094_489, - needGb: '9.8', - memory: { modelBytes: 8_677_243_801, contextBytes: 1_140_850_688, peakBytes: 0 }, - budgetShare: 0.946, + needBytes: 10_239_622_041, + needGb: '10.2', + memory: { modelBytes: 9_098_771_353, contextBytes: 1_140_850_688, peakBytes: 0 }, + budgetShare: 0.954, fit: 'tight', // 9.3 x 120 / 100. speed: 'ok', @@ -748,32 +777,12 @@ describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { expect(order(result)).toEqual(['mimo-v2.6:9b-q4_k_m@65536', 'qwen3.5:9b-q4_k_m@65536']); }); - it('would need 11.0 GB at 131072 tokens, more than the whole budget', async () => { - const hw = await macMini(); - const nineB = stockModel('mimo-v2.6:9b-q4_k_m'); - const budget = deviceBudget(hw).budgetBytes; - expect(formatGb(needBytes(nineB, 131072))).toBe('11.0'); - expect(needBytes(nineB, 131072) / budget).toBeGreaterThan(1); - expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 131072)).toBe(false); - expect(modelFitsBudget(hw, STOCK, 'mimo-v2.6:9b-q4_k_m', 65536)).toBe(true); - }); - - it('gets the same with the budget its runtime measured, with a little more room', async () => { - // The runtime reported 11.8 GiB here; the 6 GiB reserve leaves 10 GiB (10.7 GB) of it. - const result = recommend(await macMini(), STOCK, ALL_INSTALLED, { measuredTotalBytes: MACMINI_MEASURED_TOTAL }); - expect(result).toMatchObject({ - verdict: 'limited', - budgetBytes: 10 * GIB, - budgetGb: '10.7', - budgetSource: 'measured', - }); - expect(result.pick).toMatchObject({ modelId: 'mimo-v2.6:9b-q4_k_m', context: 65536, budgetShare: 0.914 }); - }); - it('gets the same with Claude Code as its only coding agent, which also needs 65536 tokens', async () => { - const result = recommend(await macMini(), STOCK, [fact('claude')]); - expect(result.verdict).toBe('limited'); - expect(result.pick).toMatchObject({ modelId: 'mimo-v2.6:9b-q4_k_m', context: 65536, cli: 'claude' }); + const hw = await macMini(); + expect(recommend(hw, STOCK, [fact('claude')])).toMatchObject({ verdict: 'none', reason: 'memory' }); + const measured = recommend(hw, STOCK, [fact('claude')], { measuredTotalBytes: MACMINI_MEASURED_TOTAL }); + expect(measured.verdict).toBe('limited'); + expect(measured.pick).toMatchObject({ modelId: 'mimo-v2.6:9b-q4_k_m', context: 65536, cli: 'claude' }); }); }); @@ -782,28 +791,32 @@ describe('every memory size, with the stock catalogue', () => { // size. A change of a catalogue row or of a threshold that moves one of these answers has to // change this table on purpose. docs/local-models.md has the same table for people. const EXPECTED: [soldGb: number, chip: string, answer: string][] = [ - // The smallest need at 65536 tokens is 9.8 GB; the budget is 2.1 GB. + // The smallest need at 65536 tokens is 10.2 GB; the budget is 2.1 GB. [8, 'Apple M2', 'none memory'], - // The 9B fits only at 65536 tokens (0.946), and nothing of 12 billion fits: `limited`. - [16, 'Apple M4', 'limited mimo-v2.6:9b-q4_k_m@65536 tight ok'], - // On the slowest chip the pick is the same, and slow (6.3 tokens per second). - [16, 'Apple M1', 'limited mimo-v2.6:9b-q4_k_m@65536 tight slow'], + // The 9B needs 10.2 GB at 65536 tokens, 0.987 of the 10.4 GB estimate: just out. A runtime + // that measured the budget a real 16 GB Mac measured admits it (see the Mac mini above). + [16, 'Apple M4', 'none memory'], + [16, 'Apple M1', 'none memory'], + // 11.4 GB at 131072 tokens, 0.964 of 11.8 GB, and nothing of 12 billion fits: `limited`. [18, 'Apple M3 Pro', 'limited mimo-v2.6:9b-q4_k_m@131072 tight ok'], + // On a base chip with 100 GB/s the 9B computes to 9.3 tokens per second: slow. + [24, 'Apple M3', 'limited mimo-v2.6:9b-q4_k_m@131072 good slow'], [24, 'Apple M4', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'], [24, 'Apple M4 Pro', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'], - // North Mini Code needs 21.8 GB at 65536 tokens: 0.997 of the 21.8 GB estimate, out by a - // hair. A runtime that measures more than the estimate admits it (see below). + // North Mini Code needs 23.2 GB at 65536 tokens: 1.06 of the 21.8 GB estimate. A runtime + // that measures about 3/4 of the memory admits it (see below). [32, 'Apple M4', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'], [32, 'Apple M1 Max', 'limited mimo-v2.6:9b-q4_k_m@131072 good ok'], // A Max chip runs the dense 27B fast enough (12.1); an M3 Max (300 GB/s) does not (8.9), - // and takes the 35B-A3B as a GGUF build, whose memory does not grow after it loaded. + // and takes the 35B-A3B as a GGUF build, whose memory does not grow after it loaded. Both + // at 131072 tokens, 0.966 and 0.976 of the budget. [36, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@131072 tight ok'], [36, 'Apple M3 Max', 'ok qwen3.6:35b-a3b-mtp-q4_k_m@131072 tight ok'], // The tested pair on the tested Mac; on a Pro chip the 27B is slow (8.1), so the same pick. [48, 'Apple M5 Pro', 'ok qwen3.6:35b-a3b@131072 tight ok'], [48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 tight ok'], - // On a Max chip the 27B has the higher rank and holds its full window, 29.7 GB. - [48, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@262144 good ok'], + // On a Max chip the 27B has the higher rank and holds its full window, 32.1 GB: 0.853. + [48, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@262144 tight ok'], [64, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'], [64, 'Apple M4 Max', 'ok qwen3.8:27b-mtp-q4_k_m@262144 good ok'], [96, 'Apple M3 Ultra', 'ok qwen3.8:27b-mtp-q4_k_m@262144 good ok'], @@ -833,15 +846,15 @@ describe('every memory size, with the stock catalogue', () => { expect(result.headline).toEqual({ text: LOCAL_MODEL_LABELS.noneMemory, vars: { memory: '8' } }); // The smallest model a harness can work with needs more than four times the budget. const smallest = Math.min(...STOCK.models.map((m) => needBytes(m, 65536))); - expect(formatGb(smallest)).toBe('9.8'); + expect(formatGb(smallest)).toBe('10.2'); expect(smallest / result.budgetBytes).toBeGreaterThan(4); }); - it('gives every Mac from 16 to 32 GB a small model, and every larger one a model of 27 billion or more', () => { + it('gives no model to 8 and 16 GB, a small model to every Mac from 18 to 32 GB, and every larger one a model of 27 billion or more', () => { for (const hw of SWEEP) { const result = recommend(hw, STOCK, ALL_INSTALLED); const label = `${hw.memClassGb} GB ${hw.chip}`; - if (hw.memClassGb === 8) expect(result.reason, label).toBe('memory'); + if (hw.memClassGb <= 16) expect(result.reason, label).toBe('memory'); else if (hw.memClassGb <= 32) { expect(result.verdict, label).toBe('limited'); expect(result.pick?.paramsB, label).toBe(9); @@ -858,10 +871,11 @@ describe('every memory size, with the stock catalogue', () => { const measured = { measuredTotalBytes: Math.floor(0.74 * 32 * GIB) }; const base = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED, measured); expect(base.verdict).toBe('ok'); - expect(base.pick).toMatchObject({ modelId: 'north-mini-code-1.0:q4_k_m', context: 262144, fit: 'tight' }); - // On a Max chip the 27B is fast enough, and it is the higher rank. + // 24.1 GB at 131072 tokens, 0.950 of it; at 262144 tokens 26.0 GB would not fit. + expect(base.pick).toMatchObject({ modelId: 'north-mini-code-1.0:q4_k_m', context: 131072, fit: 'tight' }); + // On a Max chip the 27B is fast enough, and it is the higher rank: 24.4 GB at 65536 tokens. const max = recommend(mac(32, 'Apple M1 Max'), STOCK, ALL_INSTALLED, measured); - expect(max.pick).toMatchObject({ modelId: 'qwen3.8:27b-mtp-q4_k_m', context: 131072, fit: 'tight' }); + expect(max.pick).toMatchObject({ modelId: 'qwen3.8:27b-mtp-q4_k_m', context: 65536, fit: 'tight' }); }); it('always pairs the pick with the top-ranked harness, OpenCode', () => { @@ -2063,8 +2077,8 @@ describe('the measured budget, end to end', () => { expect([measured.budgetSource, measured.budgetGb]).toEqual(['measured', '40.2']); // The same rows in the same order; only the share of each changes. expect(order(measured)).toEqual(order(estimated)); - expect(rows(estimated).map((c) => c.budgetShare)).toEqual([0.957, 0.742, 0.653, 0.292, 0.292]); - expect(rows(measured).map((c) => c.budgetShare)).toEqual([0.895, 0.695, 0.611, 0.273, 0.273]); + expect(rows(estimated).map((c) => c.budgetShare)).toEqual([0.957, 0.77, 0.692, 0.303, 0.303]); + expect(rows(measured).map((c) => c.budgetShare)).toEqual([0.895, 0.721, 0.647, 0.283, 0.283]); for (const candidate of rows(measured)) { expect(candidate.budgetShare).toBeCloseTo(candidate.needBytes / MACBOOK_MEASURED_TOTAL, 3); } diff --git a/test/local-model-runtime.test.ts b/test/local-model-runtime.test.ts index e61c7434..46c68908 100644 --- a/test/local-model-runtime.test.ts +++ b/test/local-model-runtime.test.ts @@ -38,6 +38,7 @@ import { parseInferenceComputeTotal, removeLocalModelFiles, removePartialModelBlobs, + RUNTIME_CTX_CHECKPOINTS, RUNTIME_DOWNLOAD_RETRIES, RUNTIME_HOST_PROMPT_CACHE_MIB, RUNTIME_KEEP_ALIVE, @@ -262,6 +263,7 @@ describe('runtimeEnv', () => { OLLAMA_REMOTES: 'evil.example', OLLAMA_AUTH: '1', LLAMA_ARG_CACHE_RAM: '0', + LLAMA_ARG_CTX_CHECKPOINTS: '64', // Other secrets a server environment carries. ANTHROPIC_API_KEY: 'sk-ant-secret', OPENAI_API_KEY: 'sk-secret', @@ -275,7 +277,7 @@ describe('runtimeEnv', () => { DYLD_INSERT_LIBRARIES: '/tmp/evil.dylib', }; - it('holds exactly the allowlist: what is copied, the private HOME, and ten settings', () => { + it('holds exactly the allowlist: what is copied, the private HOME, and eleven settings', () => { expect(runtimeEnv(input(SERVER_ENV))).toEqual({ PATH: '/usr/local/bin:/usr/bin:/bin', TMPDIR: '/var/folders/xy/T/', @@ -298,22 +300,27 @@ describe('runtimeEnv', () => { OLLAMA_KV_CACHE_TYPE: 'q8_0', OLLAMA_FLASH_ATTENTION: '1', LLAMA_ARG_CACHE_RAM: '1024', + LLAMA_ARG_CTX_CHECKPOINTS: '8', }); }); it('sets the llama.cpp memory settings from the exported constants, never from the server', () => { expect(RUNTIME_KV_CACHE_TYPE).toBe('q8_0'); expect(RUNTIME_HOST_PROMPT_CACHE_MIB).toBe(1024); + expect(RUNTIME_CTX_CHECKPOINTS).toBe(8); const env = runtimeEnv(input(SERVER_ENV)); expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE); expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB)); - // The server's own `LLAMA_ARG_CACHE_RAM=0` and a user's q4_0 cache lose against ours. + expect(env.LLAMA_ARG_CTX_CHECKPOINTS).toBe(String(RUNTIME_CTX_CHECKPOINTS)); + // The server's own `LLAMA_ARG_CACHE_RAM=0` and `LLAMA_ARG_CTX_CHECKPOINTS=64`, and a user's + // q4_0 cache, lose against ours. const env2 = runtimeEnv(input({ ...SERVER_ENV, OLLAMA_KV_CACHE_TYPE: 'q4_0', OLLAMA_FLASH_ATTENTION: '0' })); - expect([env2.OLLAMA_KV_CACHE_TYPE, env2.OLLAMA_FLASH_ATTENTION, env2.LLAMA_ARG_CACHE_RAM]).toEqual([ - 'q8_0', - '1', - '1024', - ]); + expect([ + env2.OLLAMA_KV_CACHE_TYPE, + env2.OLLAMA_FLASH_ATTENTION, + env2.LLAMA_ARG_CACHE_RAM, + env2.LLAMA_ARG_CTX_CHECKPOINTS, + ]).toEqual(['q8_0', '1', '1024', '8']); }); it('lets no CODEMAN_ variable through, the password least of all', () => { @@ -390,6 +397,7 @@ describe('runtimeEnv', () => { expect(Object.keys(runtimeEnv(input({}))).sort()).toEqual([ 'HOME', 'LLAMA_ARG_CACHE_RAM', + 'LLAMA_ARG_CTX_CHECKPOINTS', 'OLLAMA_CONTEXT_LENGTH', 'OLLAMA_FLASH_ATTENTION', 'OLLAMA_HOST', diff --git a/test/local-model-supervisor.test.ts b/test/local-model-supervisor.test.ts index 6924b065..8122eb98 100644 --- a/test/local-model-supervisor.test.ts +++ b/test/local-model-supervisor.test.ts @@ -209,6 +209,7 @@ describe('start', () => { OLLAMA_KV_CACHE_TYPE: 'q8_0', OLLAMA_FLASH_ATTENTION: '1', LLAMA_ARG_CACHE_RAM: '1024', + LLAMA_ARG_CTX_CHECKPOINTS: '8', }); });