BitSquare/Codeman
2
0 Fork 0
mirror of https://github.com/Ark0N/Codeman.git synced 2026-11-14 10:39:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge #784: a local model catalogue for every Mac memory size, and its weekly check

Browse Source
This commit is contained in:
Codeman maintainer committed 2026-11-08 11:10:13 +01:00
2 parents 8fa2884f48 04c4e43913
commit 8a54ded78b
21 files changed
+3199 -476
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/local-model-catalog.yml
+128
View File
@@ -0,0 +1,128 @@
name: Local model catalogue
# Checks every Monday that the local model catalogue (src/config/local-model-catalog.ts) and
# the pinned model runtime (OLLAMA_RUNTIME_PIN in src/local-model/runtime.ts) still match what
# the registry and Ollama's releases serve: a tag can be pushed again or removed, the build a
# Mac downloads can change size, and Ollama ships new versions. The catalogue is compiled into
# Codeman on purpose, so this job only reports; a change ships in a release like any other.
# What it checks, and what a maintainer does with its issue: docs/local-models.md, "Keeping
# the catalogue current".
#
# By hand: gh workflow run local-model-catalog.yml (the file must be on master for that,
# and the schedule only ever runs master's copy). Locally: npm run check:local-model-catalog
#
# Two jobs, so the token that can write issues never sits next to `npm ci`:
# - check installs the dependencies and runs scripts/check-local-model-catalog.mjs with a
# read-only token. Exit 0 clean, 1 drift, 2 the check failed. On 2 the job fails and the
# issue is not touched: an incomplete report must not open or update it.
# - issue runs no code from the repository, only gh. On drift it opens the one issue titled
# "Local model catalogue: weekly check", or puts this week's report into it when it is
# already open. On a clean week it comments on that issue only when one is open.
on:
schedule:
- cron: '0 6 * * 1'
workflow_dispatch:
permissions:
contents: read
# A run by hand while the scheduled one is going must not open a second issue.
concurrency: ${{ github.workflow }}
jobs:
check:
name: Check the catalogue and the runtime pin
runs-on: ubuntu-latest
timeout-minutes: 15
outputs:
status: ${{ steps.check.outputs.status }}
report: ${{ steps.check.outputs.report }}
steps:
- name: Checkout repo
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Setup Node.js
uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
with:
node-version: 22
cache: 'npm'
# The check needs tsx and zod, both plain JavaScript, so no install script runs here
# (node-pty's build and the repository's postinstall included).
- name: Install dependencies
run: npm ci --ignore-scripts
- name: Run the check
id: check
env:
# Read-only here. Sent to api.github.com only, so the release lookups are not rate limited.
GITHUB_TOKEN: ${{ github.token }}
run: |
set +e
npx tsx scripts/check-local-model-catalog.mjs > "$RUNNER_TEMP/report.md"
code=$?
set -e
cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY"
case "$code" in
0) echo "status=clean" >> "$GITHUB_OUTPUT" ;;
1) echo "status=drift" >> "$GITHUB_OUTPUT" ;;
*)
echo "::error::The check itself failed (exit $code), so the issue is left alone. The summary holds what it found."
exit 1
;;
esac
# One line of base64, so nothing in the report can end the output early. An issue
# body holds at most 65,536 characters; the report is a few KB.
echo "report=$(head -c 60000 "$RUNNER_TEMP/report.md" | base64 -w0)" >> "$GITHUB_OUTPUT"
issue:
name: Keep the weekly issue current
needs: check
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
issues: write
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
ISSUE_TITLE: 'Local model catalogue: weekly check'
STATUS: ${{ needs.check.outputs.status }}
REPORT_B64: ${{ needs.check.outputs.report }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- name: Open, update or comment on the issue
run: |
set -euo pipefail
# The issues API, not the search index, so an issue opened a minute ago is found too,
# and an exact title match, so a similar title never counts. The oldest open one wins.
number=$(gh issue list --state open --limit 1000 --json number,title \
| jq -r --arg title "$ISSUE_TITLE" '[.[] | select(.title == $title)] | sort_by(.number) | first | .number // empty')
today=$(date -u +%Y-%m-%d)
case "$STATUS" in
drift)
printf '%s' "$REPORT_B64" | base64 -d > "$RUNNER_TEMP/report.md"
printf '\n---\n\nWritten by the [weekly check](%s). Fix the catalogue or the pin in a pull request and close this issue once that is merged; the next drift opens it again.\n' \
"$RUN_URL" >> "$RUNNER_TEMP/report.md"
if [ -n "$number" ]; then
gh issue edit "$number" --body-file "$RUNNER_TEMP/report.md"
gh issue comment "$number" --body "Week of $today: drift found again. The description now holds this week's report ([run]($RUN_URL))."
else
gh issue create --title "$ISSUE_TITLE" --body-file "$RUNNER_TEMP/report.md"
fi
;;
clean)
if [ -n "$number" ]; then
gh issue comment "$number" --body "Week of $today: the check was clean ([run]($RUN_URL))."
else
echo "Clean, and no issue is open: nothing to do."
fi
;;
*)
echo "::error::The check job reported no status ('$STATUS')."
exit 1
;;
esac
CLAUDE.md
+1 -1
View File
@@ -192,7 +192,7 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph
⚠️ **llama-swap endpoints** (one model at a time): the apply routes check `GET /running` and return `requiresConfirmation` before evicting a model another live session uses; `confirmedSwap` and `confirmedContext` are SEPARATE flags and must stay so. Claude alone gets a context floor (`CLAUDE_MIN_SAFE_CONTEXT_TOKENS`); context is parsed from `/running`'s `cmd`, never trusted from `/props`. The floor also divides that context by the model's parallel slot count (`modelSlotCounts`: `--parallel`/`-np` from `cmd`, else `/props` `total_slots`; `perSlotContext()`), and `/props` only ever RAISES that warning: it supplies the context only when `cmd` states none and the count only when `cmd` has no `--parallel`, never a larger pool. `/props` is probed once per launch command (it refreshes llama-swap's last-use time, so a `ttl` would never expire), and a `/props` context beside an explicit `--parallel N` is already per slot, so it is not divided again. Backend log lines come from llama-swap's `/api/events` `upstream` source, never `/logs`. → [architecture-invariants#custom-model-endpoint-profiles](docs/architecture-invariants.md#custom-model-endpoint-profiles)
**Local model recommendation** (`GET /api/local-model`, `src/local-model/`, `src/config/local-model-catalog.ts`, `src/types/local-model.ts`; EXPERIMENTAL): the server looks at the Mac it runs on and says which local coding model fits it and with which agent CLI. ⚠️ The READ side stays read-only: the probe, the catalogue and `recommend()` download nothing, install nothing and write nothing; everything that changes the machine is the install below, behind an explicit request. Apple silicon macOS only: anywhere else the answer is a `200` with verdict `none` (reasons `not-macos`, `intel`, `memory`, `disk`, `macos`, `no-harness`), and a failed probe is an answer, never a 500. The probe (`hardware.ts`) is ONE bounded async spawn of `/usr/sbin/sysctl` WITHOUT `-n`, parsed as `key: value` lines by the pure `parseSysctl()`, plus `statfs` of the data dir (probed bounded first, #516) and `node:os` fallbacks, cached 60 s; ⚠️ never `system_profiler`, never a second spawn, and `iogpu.wired_limit_mb` of 0 means not set. The catalogue is a compiled constant validated by a Zod schema: ⚠️ no override file and no runtime data, and a model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`, so an id can never name another registry host). ⚠️ A harness is a catalogue ROW (`{ cli, minContext, rank, gated?, slowFirstTurn? }`) plus a `HarnessFact` built from the CLI registry at call time (`harness-facts.ts`); no code compares a CLI id (both id-branching guards scan these files). `recommend()` (`recommend.ts`) is pure. ⚠️ A model's need is its PEAK (`bytes + heldBytesPerToken * context + overheadBytes + peakExtraBytes`, measured on a Mac for one row and marked `estimated` on the others), judged against a budget that is `estimated` until the runtime ran and `measured` from then on (`RecommendOptions.measuredTotalBytes`). ⚠️ Ranking, lexicographic: tested on a Mac like this first, then speed `ok` before `unknown` before `slow`, then model rank, harness rank, larger context; the fit grade decides whether a row is listed and what it says, it does NOT sort, so a model is offered at the largest context it fits at. ⚠️ `limited` is a statement about the MAC (no model with 12 billion parameters or more fits), judged before the disk filter and whatever the pick is; a small pick under `ok` carries `noteSmallPick`, and a model left out for disk space is named in `noteDiskDropped`, never dropped in silence. ⚠️ The word "Recommended" exists only in `headlineTested`, and the tag "Tested on a Mac like this" only on a row whose model has a `tested` entry for exactly this chip, memory size and harness. ⚠️ Speed is `ok`/`slow`/`unknown`, graded on the unrounded figure, and a chip in no chips row is `unknown`: never guess a bandwidth. ⚠️ ONE unit for every computed size a person reads: decimal GB text from `formatGb()` (`format.ts`), sent beside the bytes, and the browser never divides bytes; the one exception is the installed memory, shown as `memClassGb` (the size the Mac is sold as, 48 and not 51.5). ⚠️ Every sentence the server chooses is a `LOCAL_MODEL_LABELS`, `LOCAL_MODEL_TAG_LABELS` or `LOCAL_MODEL_INSTALL_LABELS` text with `{placeholder}` slots filled from `vars` (no apostrophe in a text: the zh-CN key is single-quoted), and each needs a zh-CN entry. Admin only in multi-user mode (403 before anything is probed). Tests: `test/local-model-*.test.ts`, `test/routes/local-model-routes.test.ts`. Window rules: `src/web/CLAUDE.md` (Run a model on this Mac). → `docs/api-reference.md` (Local model), `docs/versioning-policy.md`
**Local model recommendation** (`GET /api/local-model`, `src/local-model/`, `src/config/local-model-catalog.ts`, `src/types/local-model.ts`; EXPERIMENTAL): the server looks at the Mac it runs on and says which local coding model fits it and with which agent CLI. ⚠️ The READ side stays read-only: the probe, the catalogue and `recommend()` download nothing, install nothing and write nothing; everything that changes the machine is the install below, behind an explicit request. Apple silicon macOS only: anywhere else the answer is a `200` with verdict `none` (reasons `not-macos`, `intel`, `memory`, `disk`, `macos`, `no-harness`), and a failed probe is an answer, never a 500. The probe (`hardware.ts`) is ONE bounded async spawn of `/usr/sbin/sysctl` WITHOUT `-n`, parsed as `key: value` lines by the pure `parseSysctl()`, plus `statfs` of the data dir (probed bounded first, #516) and `node:os` fallbacks, cached 60 s; ⚠️ never `system_profiler`, never a second spawn, and `iogpu.wired_limit_mb` of 0 means not set. The catalogue is a compiled constant validated by a Zod schema: ⚠️ no override file and no runtime data, and a model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`, so an id can never name another registry host). ⚠️ A harness is a catalogue ROW (`{ cli, minContext, rank, gated?, slowFirstTurn? }`) plus a `HarnessFact` built from the CLI registry at call time (`harness-facts.ts`); no code compares a CLI id (both id-branching guards scan these files). `recommend()` (`recommend.ts`) is pure. ⚠️ A model's need is its PEAK (`bytes + heldBytesPerToken * context + overheadBytes + peakExtraBytes`, measured on a Mac for one row and marked `estimated` on the others), and every row names its `runner` (`mlx` or `llamacpp`, the build a Mac really pulls) and carries figures of its OWN: computed from its model's config.json by that runner's rule (catalogue header, MEMORY), never borrowed from another row; `test/local-model-catalog.test.ts` recomputes them, and the `llamacpp` rows assume the runtime's `OLLAMA_KV_CACHE_TYPE=q8_0`, `LLAMA_ARG_CACHE_RAM=1024` and `LLAMA_ARG_CTX_CHECKPOINTS=8` (`runtimeEnv()`) and count what llama-server keeps beside the cache (8 context checkpoints of the recurrent or sliding-window state, and for a GGUF with a prediction layer one rollback copy of the recurrent state per drafted token and that layer's cache in f16), so change the two together. The need is judged against a budget that is `estimated` until the runtime ran and `measured` from then on (`RecommendOptions.measuredTotalBytes`). ⚠️ Ranking, lexicographic: tested on a Mac like this first, then speed `ok` before `unknown` before `slow`, then model rank, harness rank, larger context; the fit grade decides whether a row is listed and what it says, it does NOT sort, so a model is offered at the largest context it fits at. ⚠️ `limited` is a statement about the MAC (no model with 12 billion parameters or more fits), judged before the disk filter and whatever the pick is; a small pick under `ok` carries `noteSmallPick`, and a model left out for disk space is named in `noteDiskDropped`, never dropped in silence. ⚠️ The word "Recommended" exists only in `headlineTested`, and the tag "Tested on a Mac like this" only on a row whose model has a `tested` entry for exactly this chip, memory size and harness. ⚠️ Speed is `ok`/`slow`/`unknown`, graded on the unrounded figure, and a chip in no chips row is `unknown`: never guess a bandwidth. ⚠️ ONE unit for every computed size a person reads: decimal GB text from `formatGb()` (`format.ts`), sent beside the bytes, and the browser never divides bytes; the one exception is the installed memory, shown as `memClassGb` (the size the Mac is sold as, 48 and not 51.5). ⚠️ Every sentence the server chooses is a `LOCAL_MODEL_LABELS`, `LOCAL_MODEL_TAG_LABELS` or `LOCAL_MODEL_INSTALL_LABELS` text with `{placeholder}` slots filled from `vars` (no apostrophe in a text: the zh-CN key is single-quoted), and each needs a zh-CN entry. Admin only in multi-user mode (403 before anything is probed). Tests: `test/local-model-*.test.ts`, `test/routes/local-model-routes.test.ts`. Window rules: `src/web/CLAUDE.md` (Run a model on this Mac). → `docs/api-reference.md` (Local model), `docs/versioning-policy.md`
**Local model install** (`POST /api/local-model/install|cancel|resume|runtime`, `DELETE /api/local-model`; `src/local-model/{install-job,runtime,ollama-client,connect,state-store,paths,developer-tools}.ts`; EXPERIMENTAL, admin only in multi-user mode on EVERY verb; guide `docs/local-models.md`): installs the chosen model on the Mac the server runs on and ends on a session of the coding agent running on it. ONE job of five idempotent steps (`runtime`, `model`, `verify`, `agent`, `connect`; `agent` only when the coding agent is missing), a pure reducer (`reduceInstallJob()`) over three tables in `install-job.ts` driven by one service per process (`getLocalModelService()`); the window polls the GET once a second, no SSE event. ⚠️ Layout only through `localModelPaths()` (all `dataPath()`): `local-model/runtime/ollama-<version>/`, `local-model/models/`, `local-model/home/` (the runtime's PRIVATE HOME, so the user's `~/.ollama` is never touched) and the state file `local-model.json` (0600, ONE serialized writer, total reader, holds NO path); nothing is written or deleted outside `local-model/` except that file and the two writes of the connect step. ⚠️ The runtime pin (`OLLAMA_RUNTIME_PIN`: version, URL, size, sha256) is a COMPILED constant no file, setting or request may supply; the hash is compared BEFORE anything is unpacked. ⚠️ The runtime is a NON-detached child whose environment comes from the `runtimeEnv()` ALLOWLIST, never `process.env` (no `CODEMAN_*`, none of the user's `OLLAMA_*`); the port is chosen ONCE by binding (from 11435, never 11434) and stored; a listener on it is adopted or replaced only when its stored pid runs a binary under OUR runtime folder, anything else is `port-in-use`; started at boot when installed and enabled, max 3 restarts, SIGTERM at shutdown beside `stopDeepSeekWeb()`; under vitest nothing spawns or fetches. ⚠️ Consent is bound to the size: `acknowledgedBytes` must equal the server's own total (`Candidate.installBytes`) or the answer is 409 `size-changed`; the model, context and CLI must be a row the current recommendation offers (400). ⚠️ ONE job at a time (409 `job-running`); a job a restart interrupted comes back `paused` and NEVER continues without a click (so `shutdown()` writes nothing), and an interrupted step is never recorded as failed. ⚠️ An event the transition table does not list is refused (same object back, 409 `wrong-state`): add a row to the table, never a special case. ⚠️ The budget is MEASURED from the runtime's "inference compute" log line and the model judged against it BEFORE it is downloaded. ⚠️ The pull stream is parsed incrementally (about 1.4 million lines for 23 GB): never buffer it, never write state per line. ⚠️ A failure is a code with ONE next step (`LOCAL_MODEL_ERROR_NEXT`), never a bare "failed"; a probe that ended on its token limit is `probe-unfinished`, not `no-tool-call`. ⚠️ The coding agent is installed through the shared runner `src/cli-install-runner.ts` (extracted from `cli-registry-routes.ts`, ONE install slot per CLI for both callers) with `stockInstallCommand()`, the COMPILED stock command, never the `clis.json`-merged one (this path has no `cliManagementEnabled` gate); only for a catalogue harness row, after the consent click that showed the literal command, audited (`cli_install`, `via: 'local-model'`); afterwards `forgetResolvedCli()` and a fresh lookup, and still missing is `harness-failed`. ⚠️ Connect writes ONE endpoint record under the RESERVED id `local-ollama` through `mutateCustomModelHosts()` and `customModelEndpointsEnabled: true` through `mutateSettingsFile()`, remembering the flag's first value; a record with that id that is not ours is never overwritten, and while it is ours it is MANAGED: the endpoint routes refuse to edit or delete it and the discovery sweep skips it (the runtime's port has no login, so its `models` must stay the one model that was checked). ⚠️ The model digest is stored and read again at boot and before every START of a coding agent on that endpoint (`launchRefusalMessage()`): the two custom-model launch routes in `session-routes.ts` ask it before anything is written, the reboot-restore rebuild asks it before it builds (reason `local-model-not-ready`, left on offer), and `Session`'s launch guard (`setLaunchGuard()`, set in `setupSessionListeners()`) asks it right before a NEW agent process starts (a fresh pane, the respawn of a dead one, `restartCli()`), never for an attach to an agent that runs; a refusal is a `LaunchRefusedError` that never falls back to a direct PTY. A changed or missing model, a runtime Codeman did not start or that did not start, or another model id refuses the launch with one sentence. ⚠️ The connection is CLAIMED in the state file (`connection`, with the flag's value read by `inspectLocalEndpoint()`) BEFORE the endpoint record and the settings flag are written, so a state write that fails or a kill between the two never leaveLine truncated
docs/architecture-invariants.md
+1 -1
View File
@@ -702,7 +702,7 @@ Either way the actual launch (`runCustomModelEntry`) routes through `run()` itse
**The pin** (`OLLAMA_RUNTIME_PIN`, `runtime.ts`) is a COMPILED constant: version, URL, size, sha256. ⚠️ No file, setting, env var or request may supply any part of it: a hash that comes from the same writable place as the URL pins nothing. The archive is streamed to `*.partial` while it is hashed, continued with a Range request (3 bounded retries), refused when it grows past the pinned size, and the hash is compared BEFORE `/usr/bin/tar` unpacks anything into a temporary folder that is then renamed.
**Supervision** (`createRuntimeSupervisor()`): the runtime is a NON-detached child, started at boot once a model is installed and enabled, restarted at most 3 times in a row, SIGTERMed at shutdown beside `stopDeepSeekWeb()`. ⚠️ Its environment comes from the pure `runtimeEnv()` ALLOWLIST (PATH, the private HOME, TMPDIR, LANG, the proxy variables, and exactly `OLLAMA_HOST=127.0.0.1:<port>`, `OLLAMA_MODELS`, `OLLAMA_CONTEXT_LENGTH`, `OLLAMA_NO_CLOUD=1`, `OLLAMA_KEEP_ALIVE=10m`, `OLLAMA_NUM_PARALLEL=1`, `OLLAMA_MAX_LOADED_MODELS=1`): never `process.env` whole (it carries `CODEMAN_PASSWORD`, and a user's own `OLLAMA_ORIGINS=*` would open the port to every web page). ⚠️ The port is chosen ONCE by binding (11435 across 20; 11434 belongs to a runtime the user may run) and stored; it is never searched again on a later start, because it is part of the saved endpoint's URL. ⚠️ Pid and port are in the state file: a listener on the port that answers `/api/version` and whose pid runs a binary under OUR runtime folder is our own orphan (a server that was killed leaves one: a child is not ended by its parent's death) and is adopted or replaced; ANY other listener is `port-in-use`, never something to talk to. ⚠️ The supervisor never writes the state file: it reports through `onChange`, and the job stores what it reports. ⚠️ Under vitest nothing spawns and nothing is fetched unless a test injects it.
**Supervision** (`createRuntimeSupervisor()`): the runtime is a NON-detached child, started at boot once a model is installed and enabled, restarted at most 3 times in a row, SIGTERMed at shutdown beside `stopDeepSeekWeb()`. ⚠️ Its environment comes from the pure `runtimeEnv()` ALLOWLIST (PATH, the private HOME, TMPDIR, LANG, the proxy variables, and exactly `OLLAMA_HOST=127.0.0.1:<port>`, `OLLAMA_MODELS`, `OLLAMA_CONTEXT_LENGTH`, `OLLAMA_NO_CLOUD=1`, `OLLAMA_KEEP_ALIVE=10m`, `OLLAMA_NUM_PARALLEL=1`, `OLLAMA_MAX_LOADED_MODELS=1`, plus the llama.cpp memory settings `OLLAMA_KV_CACHE_TYPE=q8_0`, `OLLAMA_FLASH_ATTENTION=1`, `LLAMA_ARG_CACHE_RAM=1024`, `LLAMA_ARG_CTX_CHECKPOINTS=8`, which the catalogue's `llamacpp` rows are computed for, so the two change together): never `process.env` whole (it carries `CODEMAN_PASSWORD`, and a user's own `OLLAMA_ORIGINS=*` would open the port to every web page). ⚠️ The port is chosen ONCE by binding (11435 across 20; 11434 belongs to a runtime the user may run) and stored; it is never searched again on a later start, because it is part of the saved endpoint's URL. ⚠️ Pid and port are in the state file: a listener on the port that answers `/api/version` and whose pid runs a binary under OUR runtime folder is our own orphan (a server that was killed leaves one: a child is not ended by its parent's death) and is adopted or replaced; ANY other listener is `port-in-use`, never something to talk to. ⚠️ The supervisor never writes the state file: it reports through `onChange`, and the job stores what it reports. ⚠️ Under vitest nothing spawns and nothing is fetched unless a test injects it.
**The budget is measured.** The runtime prints the device total in its log line "inference compute" (`total="37.4 GiB"`); `parseInferenceComputeTotal()` reads it, the job stores it and judges the chosen model against it (`modelFitsBudget()`) BEFORE the model is downloaded; from then on the GET recommends with `budgetSource: 'measured'`. One consent covers the whole install: the sizes that were confirmed do not change, so a model that no longer fits is a failure with the next model, not a second question.
docs/local-models.md
+166 -5
View File
@@ -14,6 +14,92 @@ This page is the reference for how it works. The HTTP contract is in
[api-reference.md](api-reference.md) ("Local model"). The rules that must not be broken are
in [architecture-invariants.md](architecture-invariants.md#local-model-install).
## Which model fits which Mac
The window offers the best model of the catalogue (`src/config/local-model-catalog.ts`) that
fits this Mac's memory at a context a coding agent can work with, and up to four more. What
the catalogue answers today, before the runtime measured anything (the budget is then an
estimate: 2/3 of the memory below 36 GB, 3/4 from 36 GB, less 1 GiB, and never more than the
memory less 6, 8 or 12 GiB for macOS, Codeman, a browser and the agent):
| Memory | Budget | What it gets |
| ---------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 8 GB | 2.1 GB | Nothing. The smallest model a coding agent can work with needs 10.2 GB at 65,536 tokens |
| 16 GB | 10.4 GB | Nothing. MiMo V2.6 9B at 65,536 tokens needs 10.2 GB, 98.7% of the budget, just past the 98% a tight fit may reach (it is 95% of the 10.7 GB a 16 GB Mac mini measured). On the edge either way: only a run on such a Mac settles it |
| 18 GB | 11.8 GB | A small model (`limited`): MiMo V2.6 9B at 131,072 tokens, 11.4 GB, tight. Its base, Qwen3.5 9B, is the one alternative |
| 24 GB | 16.1 GB | The same, with room. On a base M2 or M3 (100 GB/s) it computes to 9.3 tokens per second: slow |
| 32 GB | 21.8 GB | The same. North Mini Code 1.0 needs 23.2 GB at 65,536 tokens; once the runtime measures about 3/4 of the memory, as it did on a 16 and a 48 GB Mac, the Mac gets it at 131,072 tokens, or Qwen3.8 27B at 65,536 tokens on a Max chip |
| 36 GB | 27.9 GB | Qwen3.8 27B at 131,072 tokens on an M4 or M5 Max (27.0 GB, 97%); on a slower chip, where the dense 27B computes to under 10 tokens per second, Qwen3.6 35B-A3B as a GGUF build (27.1 GB, 97%). Both tight |
| 48 GB | 37.6 GB | Qwen3.6 35B-A3B (MLX) at 131,072 tokens: the pair we tested, "Recommended" on an Apple M5 Pro with 48 GB. On a Max chip Qwen3.8 27B at 262,144 tokens (32.1 GB, tight). The test was a short task: in a long session MLX may keep up to 8 GiB more, which brings this pair to the edge of the budget, and past it while a long prompt is processed |
| 64 GB | 50.5 GB | Qwen3.8 27B at 262,144 tokens on a Max chip; Qwen3.6 35B-A3B (MLX) on a Pro chip |
| 96 GB | 76.2 GB | Qwen3.8 27B at 262,144 tokens; on an M3 Max, Qwen3.6 35B-A3B (MLX) |
| 128 GB | 102 GB | Ornith 1.5 122B-A10B (MLX) at 131,072 tokens, 97.4 GB, tight |
| 192, 256, 512 GB | 154 GB+ | Ornith 1.5 122B-A10B at 262,144 tokens |
"Tight" means the need is between 85% and 98% of the budget; above 98% a model is not
offered. A model that computes to under 10 tokens per second on the chip is "slow" and ranks
after the fast ones, which is why the chip changes the pick at 36 to 96 GB. Every model is
offered at the largest context it fits at, and never below 65,536 tokens: OpenCode's own
prompt is 8,000 to 10,000 tokens, and at 32,768 it would compact after less than one default
file read. `test/local-model-recommend.test.ts` ("every memory size") pins this table.
### How much memory a model needs
The need of a model is its peak: the weights, what stays held per token of context, what the
runner holds beside both, and what it takes for a moment while a long prompt is processed. The
runtime runs a model with one of two runners, which spend memory differently, so every row
names its runner (`runner`) and its figures follow that runner's rule:
- **llama.cpp** (GGUF builds, on Metal) allocates the K/V cache for the whole context when the
model loads and does not grow after that. The runtime is started with
`OLLAMA_KV_CACHE_TYPE=q8_0` (the cache in 8 bits, 17/32 of the default f16; the runtime's
own documentation calls the loss very small), `OLLAMA_FLASH_ATTENTION=1` (an 8-bit cache
needs it), `LLAMA_ARG_CACHE_RAM=1024` (llama-server keeps copies of earlier prompts, and
this bounds them to 1 GiB) and `LLAMA_ARG_CTX_CHECKPOINTS=8` (llama-server keeps copies of
the state it cannot roll back, two or three per request, up to 32 by default: 5 GB for the
27B; 8 keeps the last few requests, which is what an agent's next request restores from). A
row's figures are computed from its model's `config.json`: the K/V cache of its attention
layers, its recurrent or sliding-window state, live and in each of the 8 checkpoints, 1.0
GB for the runner and the 1 GiB prompt cache. A build with a multi-token-prediction layer
(the two `-mtp` tags and `qwen3.8:27b-q8_0`) costs more: the runtime turns prediction on,
so llama-server keeps that layer's cache in f16 and one more copy of the recurrent state for
rollback per drafted token (the tag's `draft_num_predict`: 2 for the 35B-A3B, 4 for the
27B). No peak: the buffers are allocated at load. The checkpoints and the
prediction costs were read in the llama.cpp source the runtime bundles (b12111), not
measured.
- **MLX** grows its cache with the tokens actually used, keeps snapshots of the recurrent
state, and may keep up to 8 GiB of snapshots of earlier prompts, which no setting bounds. A
Mac always takes the MLX build of a tag that has one. Below 48 GB that 8 GiB does not fit
beside a useful model, so the catalogue lists GGUF builds there.
Where every figure comes from:
| Model | Runner | Memory figures | Speed (tokens per second per 100 GB/s) |
| ---------------------------- | --------- | ----------------------------------------------------------------------------------- | --------------------------------------------------------- |
| `qwen3.5:9b-q4_k_m` | llama.cpp | computed from the Qwen3.5-9B config | 9.3: a published run on an M1 Max |
| `mimo-v2.6:9b-q4_k_m` | llama.cpp | the same (a fine-tune of Qwen3.5-9B) | 9.3: the same architecture and size |
| `north-mini-code-1.0:q4_k_m` | llama.cpp | computed from its config | 13.6: the 35B-A3B's, the same 3 billion active parameters |
| `qwen3.6:35b-a3b-mtp-q4_k_m` | llama.cpp | computed from the Qwen3.6-35B-A3B config, prediction layer included | 13.6: a published run on an M1 Max |
| `qwen3.6:35b-a3b` | MLX | **measured** on an Apple M5 Pro with 48 GB | 16.9: a published run; 67.7 measured on the tested Mac |
| `qwen3.8:27b-q8_0` | llama.cpp | computed from the Qwen3.8-27B config, prediction layer included | 1.7: the row below scaled by file size |
| `qwen3.8:27b-mtp-q4_k_m` | llama.cpp | computed from the Qwen3.8-27B config, prediction layer included | 2.95: a published run without prediction (a floor) |
| `ornith-1.5:122b` | MLX | computed from the Qwen3.5-122B-A10B config, scaled by the measured MLX row, + 8 GiB | 5.0: the 35B-A3B's, x 3/10 active parameters |
Sizes are what the registry serves a Mac for each tag (every layer plus the config), read on
2026-11-08. Only one row was ever run on a Mac, so every other answer in the table above is a
calculation. For a llama.cpp build the check after the download catches a load that was
judged too kindly, but not what grows after it in host memory (the checkpoints and the prompt
cache, which are counted, not checked); for an MLX build it catches neither (see "The check"
below). The measured row's figures come from single long prompts, not a session of many
requests, in which MLX may keep up to 8 GiB more. The catalogue header lists the models left out on
purpose (no evidence of working tool calls, a restrictive licence, a known bug) and why.
A new row needs its model's architecture (attention layers, K/V heads, head size, and its
linear-attention or sliding-window layers, from its `config.json`) in the table of
`test/local-model-catalog.test.ts`, which recomputes every computed figure from it and fails
for a row it does not know. Its answer for each memory size is then pinned in
`test/local-model-recommend.test.ts`.
## What an install does, in order
The person picks a model, presses "Install on this Mac" and confirms the sizes once. The
@@ -49,8 +135,10 @@ Every step can be repeated: Resume and "Try again" enter the step the job stoppe
copied from the server. `HOME` is a private folder. Exactly these are set:
`OLLAMA_HOST=127.0.0.1:<port>`, `OLLAMA_MODELS`, `OLLAMA_CONTEXT_LENGTH`,
`OLLAMA_NO_CLOUD=1`, `OLLAMA_KEEP_ALIVE=10m`, `OLLAMA_NUM_PARALLEL=1`,
`OLLAMA_MAX_LOADED_MODELS=1`. No `CODEMAN_*` variable and none of the user's own
`OLLAMA_*` settings reach it.
`OLLAMA_MAX_LOADED_MODELS=1`, and four memory settings of the llama.cpp runner:
`OLLAMA_KV_CACHE_TYPE=q8_0`, `OLLAMA_FLASH_ATTENTION=1`, `LLAMA_ARG_CACHE_RAM=1024` and
`LLAMA_ARG_CTX_CHECKPOINTS=8` (see "How much memory a model needs" above). No `CODEMAN_*`
variable and none of the user's own `OLLAMA_*` or `LLAMA_ARG_*` settings reach it.
- **The port is chosen once**, by binding, from 11435 across 20 ports, and stored. 11434 is
left alone: it is the default of a runtime the user may run themselves, which is never
touched.
@@ -69,8 +157,7 @@ judges the chosen model against it before a byte of the model is downloaded. A m
longer fits ends the job there with "does not fit" and the next model to try. From then on
the window shows the measured budget and the word "estimated" is gone.
The need of a model is its peak: the weights, what stays held per token of context, what the
runner holds beside both, and what it takes for a moment while a long prompt is processed.
The need of a model is its peak (see "How much memory a model needs" above).
### The model download
@@ -86,7 +173,10 @@ After the pull, in this order:
1. One tiny chat with `reasoning_effort: "none"`. It loads the model.
2. `GET /api/ps` must show the whole model on the GPU (`size_vram` equal to `size`) at the
chosen context length.
chosen context length. For a llama.cpp build both figures are the buffers llama-server
reports allocating (weights, K/V cache, compute), so this proves the whole load is on the
GPU. For an MLX build it proves only the weights: the runtime books an MLX model at its
weights (runtime issue 19417), so for those rows the catalogue's figures are all there is.
3. One tool probe on `/v1/chat/completions` (a `tools` array, `reasoning_effort: "none"`,
`max_tokens` 1024). It passes on a tool call whose arguments parse.
@@ -261,9 +351,79 @@ or `python3`.
a session uses the model: close those sessions first.
- **The runtime does not enforce the context length.** A prompt longer than the context was
accepted in testing. Do not rely on the runtime to stop one.
- **Most memory figures are calculated, not measured.** One model was run on one Mac; the
others are computed from their architecture, and the runtime settings they assume (an 8-bit
K/V cache, flash attention, a 1 GiB prompt cache) were read in its source, not yet observed
on a Mac. "Recommended" appears only for the pair we ran, on the Mac we ran it on.
- **The model stays in memory for ten minutes** after the last request, then the runtime
unloads it.
## Keeping the catalogue current
The catalogue (`src/config/local-model-catalog.ts`) and the runtime pin are compiled into
Codeman on purpose: no file or download decides what is recommended or run. What the
registry serves can still move away from them: a tag is pushed again or removed, the build
a Mac downloads changes size, Ollama ships a new version. The workflow
`.github/workflows/local-model-catalog.yml` looks for that every Monday at 06:00 UTC. It
runs `scripts/check-local-model-catalog.mjs`, which also runs locally:
`npm run check:local-model-catalog` prints the report and exits 0 (clean), 1 (drift) or 2
(the check itself failed).
For every model row it checks:
- **The tag still exists** in the registry.
- **The size of what a Mac downloads** is within 0.1% of the row's `bytes`. A tag can be a
manifest list with one build per runner, and Ollama on Apple silicon takes the `mlx` build
when there is one, else `llamacpp`, else `ggml`. So the check asks for the tag the way
Ollama asks, picks the build a Mac picks and sizes it the way a pull counts it (every layer
plus the config). The plain manifest other tools get for the same tag is the `llamacpp`
build, which is not what a Mac downloads.
- **The runner of that build**, once a row names its `runner`: a Mac still pulls an `mlx`
build for an `mlx` row and a GGUF build (`llamacpp`, or a manifest that names no runner)
for a `llamacpp` row. A row's memory figures follow its runner, so a tag that gains or
loses an mlx build is drift even at about the same size.
- **The build a Mac pulls runs on the pinned runtime**: its config declares no newer
Ollama (`requires`) than `OLLAMA_RUNTIME_PIN`.
- **The pinned digest**, once a row carries one: a Mac would still pull that manifest. The
field is `digest` and its value is what `/api/tags` reports after the pull. The catalogue
schema does not have it yet: `LocalModelEntrySchema` must gain an optional `digest` under
exactly that name, or the check never compares it. Until then the report lists today's
digests, rows with measured figures first, because without a digest a rebuild of about the
same size goes unnoticed (a switch between the mlx and the gguf build does not: their sizes
differ by 2% or more). Pinning digests is a decision, not a quick fix: today the catalogue
holds no digest on purpose, and `test/local-model-catalog.test.ts` ("holds no download
data") refuses a `sha256` anywhere in it, so that rule changes with it.
For the runtime pin it checks whether a newer Ollama release exists, and whether the pinned
`ollama-darwin.tgz` is still on GitHub with the pinned size and sha256.
It reads manifests and configs only, never a model layer. It does not check the memory
figures, the speeds or whether a coding agent works with a model: no manifest says anything
about those, and they change only when somebody measures on a Mac.
**The issue.** Drift opens one issue, "Local model catalogue: weekly check", or, while that
issue is open, replaces its description with the new report and adds a comment. A clean week
comments on the open issue and does nothing when there is none. A check that failed (the
network, a rate limit, an answer that did not parse) fails the run and leaves the issue
alone; what it found is in the run's summary. The schedule runs the copy of the workflow on
`master`, and so does `gh workflow run local-model-catalog.yml`.
**What a maintainer does with it**, in a pull request each time:
- **Size drift or a changed digest**: correct the row. When the build itself changed (another
runner or quantization), its memory figures and speed no longer describe it: mark the row
`estimated` until somebody measures it again.
- **A tag that is gone, or a build that needs a newer runtime**: replace or drop the row, or
move the runtime pin first.
- **A newer Ollama**: read its release notes, then move `OLLAMA_RUNTIME_PIN` (version, URL,
size and sha256, checked against the release's `sha256sum.txt`) and run an install on a
Mac before merging.
- **A pinned archive that changed**: a security question first. Every install refuses it at
the hash compare, so nobody gets it, but find out why before moving the pin.
Close the issue once the fix is merged. A catalogue or pin change reaches users only in a
release: an installed Codeman keeps the catalogue it was built with until it is updated.
## Source
| What | Where |
@@ -279,3 +439,4 @@ or `python3`.
| Routes | `src/web/routes/local-model-routes.ts` |
| Shared CLI install runner | `src/cli-install-runner.ts` |
| Serialized writer of the endpoint store | `src/custom-model-hosts-writer.ts` |
| Weekly catalogue check | `scripts/check-local-model-catalog.mjs`, `.github/workflows/local-model-catalog.yml` |
package.json
+1
View File
@@ -30,6 +30,7 @@
"test:mobile": "vitest run --config test/mobile/vitest.config.ts",
"check:frontend-syntax": "node scripts/check-frontend-syntax.mjs",
"check:browser-excludes": "node scripts/check-browser-test-excludes.mjs",
"check:local-model-catalog": "tsx scripts/check-local-model-catalog.mjs",
"fix:node-pty": "node scripts/fix-node-pty.mjs",
"typecheck": "tsc --noEmit && tsc -p config/tsconfig.scripts.json",
"lint": "eslint --config config/eslint.config.js 'src/**/*.ts'",
scripts/check-local-model-catalog.mjs
+791
View File
@@ -0,0 +1,791 @@
#!/usr/bin/env -S npx tsx
/**
* The weekly check of the local model catalogue (src/config/local-model-catalog.ts) and of the
* pinned model runtime (OLLAMA_RUNTIME_PIN in src/local-model/runtime.ts) against what the
* registry and Ollama's releases serve today.
*
* npx tsx scripts/check-local-model-catalog.mjs # markdown report on stdout
* npm run check:local-model-catalog # the same
*
* Exit code: 0 nothing drifted, 1 drift found, 2 the check itself failed (network, an answer
* that does not parse, a catalogue that does not validate). On 2 the report is still printed
* with everything that was found, but it is incomplete, so nothing should act on it.
* `.github/workflows/local-model-catalog.yml` runs it every Monday and keeps one issue current.
*
* Runs under tsx because it imports the TypeScript catalogue and pin, and imports them only in
* main(), so the test imports the pure parts without them.
*
* WHAT A MAC DOWNLOADS. A tag is not one build. Asked with the Accept header Ollama 0.40.15
* sends (`pullModelManifest` in server/images.go), the registry answers a manifest LIST for
* some tags, one child per runner, and Ollama on Apple silicon takes the first child whose
* runner is `mlx`, else `llamacpp`, else `ggml` (`runnerPreferences()` in manifest/manifest.go).
* Asked with the plain Docker Accept header the registry answers the llamacpp child instead,
* which is NOT what a Mac pulls: that is how `qwen3.6:35b-a3b` came to be "2% above its
* manifest" in the catalogue header. So this check asks the way Ollama asks, picks the child
* the way a Mac picks it, and compares the row's `bytes` with that child's pull total (every
* layer plus the config, which is exactly the `total` a real pull reported on a Mac). A row
* that names its `runner` is also held to that child's runner (`catalogueRunner()`), because
* the memory figures of a row follow its runner and a change of runner can come with almost
* the same size.
*
* WHAT IS FETCHED, and nothing else:
* - `GET /v2/library/<name>/manifests/<tag>`, the tag's manifest or manifest list.
* - For a list: `GET /v2/library/<name>/blobs/<digest>` for each child the list names. A child
* manifest is stored as a blob (Ollama fetches it the same way, `pullSelectedManifest`). It
* is metadata: about 220 KB for an MLX build of 1,055 tensors.
* - `GET .../blobs/<digest>` for the config of the child a Mac selects, a few hundred bytes,
* for its `requires` (the oldest Ollama that may pull it).
* - The GitHub API: the newest release of ollama/ollama, and the release of the pinned version.
* A digest from a `layers` entry is never requested: no model weights are ever downloaded.
* Every request has its own timeout and a byte cap, nothing is retried, every blob is checked
* against its digest, and `GITHUB_TOKEN` is sent to api.github.com only, when it is set.
*
* THE DIGEST CONTRACT. A row may pin the build it describes as `digest`: the digest of the
* manifest a Mac pulls, which is the value `/api/tags` reports after the pull (`sha256:<hex>` or
* the bare hex). The catalogue schema does not have that field yet: `LocalModelEntrySchema` is
* strict, so it must gain an optional `digest` before a row can carry one, under exactly that
* name, or this check never compares it. The catalogue keeps no digest today on purpose
* (test/local-model-catalog.test.ts, "holds no download data", refuses a `sha256` in it), so
* pinning digests changes that rule too: a decision for the catalogue, not for this script.
* Until then the report lists the digests to pin, and a rebuild of about the same size goes
* unnoticed (a change of build, mlx against gguf, never does: every pair seen so far differs
* by 2% or more, far past the size tolerance, and a row that names its runner is held to it).
*
* Everything that reaches the report from the network is validated first (digests, sizes,
* runner and format names, version tags), because the workflow puts the report in an issue.
*
* NOT CHECKED: the memory figures, the speeds and whether a coding agent works with a model.
* No manifest says anything about those; they change only when somebody measures on a Mac.
*/
import { createHash } from 'node:crypto';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
// ---------- Constants ----------
export const REGISTRY_BASE = 'https://registry.ollama.ai';
export const GITHUB_API_BASE = 'https://api.github.com';
/** The repository whose releases the runtime pin follows. */
export const RUNTIME_REPO = 'ollama/ollama';
export const MEDIA_TYPE_MANIFEST = 'application/vnd.docker.distribution.manifest.v2+json';
export const MEDIA_TYPE_MANIFEST_LIST = 'application/vnd.ollama.manifest.list.v2+json';
/** The Accept header of Ollama 0.40.15's own manifest request: the list first. */
export const MANIFEST_ACCEPT = `${MEDIA_TYPE_MANIFEST_LIST}, ${MEDIA_TYPE_MANIFEST}`;
/** The runner order of Ollama 0.40.15 on darwin/arm64 (`runnerPreferences()`). */
export const MAC_RUNNER_PREFERENCE = Object.freeze(['mlx', 'llamacpp', 'ggml']);
/** Format names the report prints as they are. Anything else is printed as `unknown`. */
export const KNOWN_FORMATS = Object.freeze(['safetensors', 'gguf']);
/**
* How far the size a Mac downloads may sit from the row's `bytes` before it is drift: 0.1%.
* The rows that were sized from layers alone sit a few hundred bytes (the config) below the
* pull total; any rebuilt model moves by far more than this.
*/
export const SIZE_TOLERANCE_FRACTION = 0.001;
/** One request may take this long, connection and body together. */
export const REQUEST_TIMEOUT_MS = 20_000;
/** A manifest or child manifest larger than this is refused. The MLX build of the 35B is 220 KB. */
export const MAX_MANIFEST_BYTES = 4 * 1024 * 1024;
/** A config blob larger than this is refused. They are a few hundred bytes. */
export const MAX_CONFIG_BYTES = 64 * 1024;
/** A GitHub release answer larger than this is refused. */
export const MAX_GITHUB_BYTES = 4 * 1024 * 1024;
/** A manifest list with more children than this is refused. */
export const MAX_LIST_CHILDREN = 8;
/** A manifest with more layers than this is refused. */
export const MAX_LAYERS = 20_000;
const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/;
const VERSION_PATTERN = /^v?(\d{1,6})\.(\d{1,6})\.(\d{1,6})$/;
const RELEASE_TAG_PATTERN = /^v\d{1,6}\.\d{1,6}\.\d{1,6}$/;
const ISO_DAY_PATTERN = /^(\d{4}-\d{2}-\d{2})T/;
const USER_AGENT = 'codeman-local-model-catalog-check';
/** The check could not finish: the run ends with exit code 2. */
export class CheckFailure extends Error {
constructor(message) {
super(message);
this.name = 'CheckFailure';
}
}
// ---------- Pure helpers ----------
/** `name:tag` into its two parts. The catalogue's pattern already rules out a slash. */
export function splitModelId(id) {
const colon = id.indexOf(':');
if (colon <= 0 || colon === id.length - 1 || id.includes('/')) throw new CheckFailure(`not a name:tag model id: ${id}`);
return { name: id.slice(0, colon), tag: id.slice(colon + 1) };
}
/** `sha256:<hex>` of a body, the way the registry names it. */
export function sha256Digest(bytes) {
return `sha256:${createHash('sha256').update(bytes).digest('hex')}`;
}
/** A digest as `sha256:<64 hex>`, accepting the bare hex `/api/tags` prints. Anything else: undefined. */
export function normalizeDigest(value) {
if (typeof value !== 'string') return undefined;
const text = value.trim().toLowerCase();
const full = text.startsWith('sha256:') ? text : `sha256:${text}`;
return DIGEST_PATTERN.test(full) ? full : undefined;
}
/** A runner name as Ollama compares it (lowercase, trimmed). */
export function canonicalRunner(runner) {
return typeof runner === 'string' ? runner.trim().toLowerCase() : '';
}
/** A runner or format name for the report: one of the known names, else `unknown`. */
export function displayName(value, known) {
const name = canonicalRunner(value);
return known.includes(name) ? name : 'unknown';
}
function isSize(value) {
return Number.isSafeInteger(value) && value >= 0;
}
/**
* What a pull of this manifest downloads, in bytes: every layer plus the config. Throws on a
* manifest that is not shaped like one, so a malformed answer never reads as a size.
*/
export function pullTotalBytes(manifest) {
if (!manifest || typeof manifest !== 'object') throw new CheckFailure('the manifest is not an object');
const { config, layers } = manifest;
if (!Array.isArray(layers) || layers.length === 0) throw new CheckFailure('the manifest has no layers');
if (layers.length > MAX_LAYERS) throw new CheckFailure(`the manifest has more than ${MAX_LAYERS} layers`);
if (!config || !isSize(config.size)) throw new CheckFailure('the manifest has no config size');
let total = config.size;
for (const layer of layers) {
if (!layer || !isSize(layer.size)) throw new CheckFailure('a layer of the manifest has no size');
total += layer.size;
}
if (!Number.isSafeInteger(total)) throw new CheckFailure('the manifest total is not a safe integer');
return total;
}
/**
* The child of a manifest list that Ollama 0.40.15 on Apple silicon pulls: for each runner in
* `MAC_RUNNER_PREFERENCE`, the first child (in list order) with that runner. A child whose
* mediaType is set to anything but a plain manifest is skipped, as Ollama skips it. Undefined
* when no child has a runner a Mac runs.
*/
export function selectMacChild(children) {
if (!Array.isArray(children)) return undefined;
for (const runner of MAC_RUNNER_PREFERENCE) {
const child = children.find(
(entry) =>
entry &&
(entry.mediaType === undefined || entry.mediaType === '' || entry.mediaType === MEDIA_TYPE_MANIFEST) &&
canonicalRunner(entry.runner) === runner
);
if (child) return child;
}
return undefined;
}
/**
* The runner and format Ollama assumes for a manifest that names neither, from its config's
* `model_format` (`MetadataForConfig()` in manifest/manifest.go): safetensors runs on mlx,
* gguf on ggml.
*/
export function metadataForConfig(config) {
const format = canonicalRunner(config?.model_format);
if (format === 'safetensors') return { runner: 'mlx', format: 'safetensors' };
if (format === 'gguf' || format === 'ggml') return { runner: 'ggml', format: 'gguf' };
return { runner: '', format };
}
/**
* A runner name in the catalogue's words (`LocalModelEntry.runner`): `mlx`, or `llamacpp` for a
* GGUF build, which Ollama 0.40.15 serves through llama-server whether the manifest names it
* `llamacpp` or names nothing (`ggml`, see `metadataForConfig()`). Undefined for anything else.
*/
export function catalogueRunner(runner) {
const name = canonicalRunner(runner);
if (name === 'mlx') return 'mlx';
if (name === 'llamacpp' || name === 'ggml') return 'llamacpp';
return undefined;
}
/** `[major, minor, patch]` of a version or `v`-tag, or undefined when it is not one. */
export function parseVersion(text) {
const match = typeof text === 'string' ? VERSION_PATTERN.exec(text.trim()) : null;
return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : undefined;
}
/** Negative when `a` is older than `b`, 0 when equal, positive when newer. Both must parse. */
export function compareVersions(a, b) {
const left = parseVersion(a);
const right = parseVersion(b);
if (!left || !right) throw new CheckFailure(`cannot compare versions ${a} and ${b}`);
for (let i = 0; i < 3; i++) if (left[i] !== right[i]) return left[i] - right[i];
return 0;
}
/** The signed difference of `actual` from `expected`, as a fraction of `expected`. */
export function sizeDifference(expected, actual) {
return (actual - expected) / expected;
}
/** Whether `actual` lies within `SIZE_TOLERANCE_FRACTION` of `expected`. */
export function withinTolerance(expected, actual, tolerance = SIZE_TOLERANCE_FRACTION) {
return Math.abs(actual - expected) <= expected * tolerance;
}
/**
* The findings for one catalogue row. `row` is `{ id, bytes, runner?, digest? }` (`runner` once
* the catalogue names it, `digest` only once the catalogue stores one); `observed` is what
* `inspectModel()` returned for it. A row may have several findings; none means it is clean.
*
* - `gone`: the tag is not in the registry, so a pull fails.
* - `runner`: a Mac now pulls a build of another runner than the row names (a tag that became
* a list with an mlx child, or the other way round). The two runners spend memory
* differently, so the row's memory figures no longer describe what runs, whatever the size.
* - `size`: what a Mac downloads is outside the tolerance of the row's `bytes`.
* - `digest`: the row pins a digest and a Mac would now pull another one.
* - `requires`: the build a Mac pulls declares a newer Ollama than the pinned runtime, so the
* pinned runtime refuses the pull.
*/
export function classifyModel(row, observed, pinnedVersion) {
if (observed.gone) return [{ kind: 'gone' }];
const findings = [];
const mac = observed.mac;
if (row.runner !== undefined) {
const actual = catalogueRunner(mac.runner) ?? 'unknown';
if (actual !== row.runner) findings.push({ kind: 'runner', expected: String(row.runner), actual });
}
if (!withinTolerance(row.bytes, mac.bytes)) {
findings.push({ kind: 'size', expected: row.bytes, actual: mac.bytes, fraction: sizeDifference(row.bytes, mac.bytes) });
}
const pinned = normalizeDigest(row.digest);
if (row.digest !== undefined && pinned !== mac.digest) {
findings.push({ kind: 'digest', expected: pinned ?? 'not a sha256 digest', actual: mac.digest });
}
if (mac.requires && compareVersions(mac.requires, pinnedVersion) > 0) {
findings.push({ kind: 'requires', requires: mac.requires, pinned: pinnedVersion });
}
return findings;
}
/**
* The findings for the runtime pin. `observed` is what `inspectRuntime()` returned.
*
* - `behind`: a newer Ollama release exists than the pinned one.
* - `pinned-release-gone`, `pinned-asset-gone`: the pinned download no longer exists, so every
* new install fails at the runtime step.
* - `pinned-asset-changed`: the pinned asset's size or sha256 no longer matches the pin. Every
* install already refuses it at the hash compare; this is also what a replaced release
* asset looks like, so read it as a security signal first.
*/
export function classifyRuntime(pin, observed) {
const findings = [];
if (compareVersions(observed.latest.version, pin.version) > 0) {
findings.push({ kind: 'behind', pinned: pin.version, latest: observed.latest.version });
}
if (observed.pinned.gone) findings.push({ kind: 'pinned-release-gone' });
else if (!observed.pinned.asset) findings.push({ kind: 'pinned-asset-gone' });
else {
const { size, digest } = observed.pinned.asset;
if (size !== pin.bytes || (digest !== undefined && digest !== `sha256:${pin.sha256}`)) {
findings.push({ kind: 'pinned-asset-changed', size, digest });
}
}
return findings;
}
/** 0 clean, 1 drift, 2 the check failed. A failure anywhere wins over drift anywhere. */
export function exitCodeFor(result) {
const failed =
result.errors.length > 0 || result.models.some((model) => model.error) || Boolean(result.runtime?.error);
if (failed) return 2;
const drift =
result.models.some((model) => model.findings.length > 0) || (result.runtime?.findings.length ?? 0) > 0;
return drift ? 1 : 0;
}
// ---------- Report ----------
/** 1234567 as `1,234,567`, independent of the locale. */
export function groupDigits(value) {
return String(value).replace(/\B(?=(\d{3})+(?!\d))/g, ',');
}
/** A fraction as a signed percentage with two decimals: `+16.61%`. */
export function formatPercent(fraction) {
const text = (fraction * 100).toFixed(2);
return fraction > 0 ? `+${text}%` : `${text}%`;
}
/** Error text for the report: one line, backticks made quotes, at most 200 characters. */
function oneLine(text) {
const flat = String(text)
.replace(/[\u0000-\u001f\u007f]+/g, ' ')
.replace(/`/g, "'")
.replace(/ {2,}/g, ' ')
.trim();
return flat.length > 200 ? `${flat.slice(0, 200)}...` : flat;
}
function buildName(variant) {
return `${displayName(variant.runner, MAC_RUNNER_PREFERENCE)}, ${displayName(variant.format, KNOWN_FORMATS)}`;
}
function modelResult(model) {
if (model.error) return 'check failed';
if (model.findings.length === 0) return 'ok';
const names = {
gone: 'tag gone',
runner: 'runner changed',
size: 'size drift',
digest: 'digest changed',
requires: 'needs a newer runtime',
};
return model.findings.map((finding) => names[finding.kind]).join(', ');
}
function describeModelFinding(model, finding) {
const id = `\`${model.id}\``;
switch (finding.kind) {
case 'gone':
return `${id}: the registry no longer has this tag, so the download fails. Replace or remove the row.`;
case 'runner':
return (
`${id}: a Mac now pulls the ${buildName(model.observed.mac)} build, which runs on ` +
`\`${finding.actual}\`, while the row's \`runner\` says \`${displayName(finding.expected, ['mlx', 'llamacpp'])}\`. ` +
'The two runners spend memory differently, so the row\'s memory figures no longer describe what runs: ' +
'recompute them for the new runner, or pin the row to a single-build tag.'
);
case 'size':
return (
`${id}: a Mac downloads the ${buildName(model.observed.mac)} build, ${groupDigits(finding.actual)} bytes, ` +
`${formatPercent(finding.fraction)} against the row's \`bytes\` of ${groupDigits(finding.expected)}. ` +
'The memory fit reads `bytes`, so correct it (and re-measure if the build changed).'
);
case 'digest':
return `${id}: the row pins \`${finding.expected}\`, a Mac now pulls \`${finding.actual}\`. The tag was pushed again.`;
case 'requires':
return (
`${id}: the build a Mac pulls declares \`requires: ${finding.requires}\`, newer than the pinned runtime ` +
`${finding.pinned}, which refuses to pull it.`
);
default:
return `${id}: ${finding.kind}`;
}
}
function describeRuntimeFinding(finding) {
switch (finding.kind) {
case 'behind':
return (
`Ollama ${finding.latest} is out; the pin is ${finding.pinned}. Read its release notes, then move ` +
'`OLLAMA_RUNTIME_PIN` (version, URL, size, sha256) and test an install on a Mac.'
);
case 'pinned-release-gone':
return 'The release of the pinned version is gone from GitHub: every new install fails at the runtime step.';
case 'pinned-asset-gone':
return 'The pinned release no longer carries the pinned archive: every new install fails at the runtime step.';
case 'pinned-asset-changed':
return (
`The pinned archive changed on GitHub (now ${finding.size === undefined ? 'no size' : `${groupDigits(finding.size)} bytes`}` +
`${finding.digest ? `, \`${finding.digest}\`` : ''}). Installs refuse it at the hash compare. Find out why before moving the pin.`
);
default:
return finding.kind;
}
}
function assetCell(asset) {
if (!asset) return 'none';
return `${groupDigits(asset.size)} bytes${asset.digest ? `, \`${asset.digest}\`` : ', no digest in the API'}`;
}
/** The markdown report of one run. Pure: everything in it was validated by the IO side. */
export function renderReport(result) {
const lines = [];
const code = exitCodeFor(result);
const driftModels = result.models.filter((model) => !model.error && model.findings.length > 0).length;
const runtimeDrift = result.runtime && !result.runtime.error ? result.runtime.findings.length > 0 : false;
lines.push('# Local model catalogue: weekly check', '');
lines.push(
`Run ${result.checkedAt}. Catalogue of ${result.catalogue.updated} (${result.models.length} models), ` +
`runtime pin Ollama ${result.pin.version}.`,
''
);
if (code === 2) {
lines.push('**The check failed**: the results below are incomplete. Nothing should act on them.', '');
} else if (code === 1) {
const parts = [];
if (driftModels > 0) parts.push(`${driftModels} of ${result.models.length} models`);
if (runtimeDrift) parts.push('the runtime pin');
lines.push(`**Drift found**: ${parts.join(' and ')}.`, '');
} else {
lines.push('**Clean**: every row matches what a Mac downloads today, and the runtime pin is current.', '');
}
for (const error of result.errors) lines.push(`- Check failed: ${oneLine(error)}`);
if (result.errors.length > 0) lines.push('');
lines.push('## Models', '');
lines.push(
'What a Mac downloads is the build Ollama on Apple silicon selects: the `mlx` build when the tag offers ' +
'one, else `llamacpp`, else `ggml`. Its size is the pull total, every layer plus the config. ' +
`Tolerance: ${(SIZE_TOLERANCE_FRACTION * 100).toFixed(1)}% of the row's \`bytes\`.`,
''
);
lines.push('| Model | Row `bytes` | A Mac downloads | Difference | Build on a Mac | Requires | Result |');
lines.push('| --- | ---: | ---: | ---: | --- | --- | --- |');
for (const model of result.models) {
const mac = model.observed?.mac;
const cells = [
`\`${model.id}\``,
groupDigits(model.bytes),
mac ? groupDigits(mac.bytes) : '',
mac ? formatPercent(sizeDifference(model.bytes, mac.bytes)) : '',
mac ? buildName(mac) : model.observed?.gone ? 'tag gone' : '',
mac?.requires ? `Ollama ${mac.requires}` : '',
modelResult(model),
];
lines.push(`| ${cells.join(' | ')} |`);
}
lines.push('');
const details = [];
for (const model of result.models) {
if (model.error) details.push(`- \`${model.id}\`: check failed: ${oneLine(model.error)}`);
for (const finding of model.findings) details.push(`- ${describeModelFinding(model, finding)}`);
const variants = model.observed?.variants ?? [];
if (model.findings.some((finding) => finding.kind === 'size' || finding.kind === 'runner') && variants.length > 1) {
const offered = variants
.map((variant) => `${buildName(variant)} ${groupDigits(variant.bytes)}${variant === model.observed.mac ? ' (a Mac takes this one)' : ''}`)
.join('; ');
details.push(` The tag offers: ${offered}.`);
}
}
if (details.length > 0) lines.push('### What changed', '', ...details, '');
lines.push('## Runtime', '');
if (!result.runtime || result.runtime.error) {
lines.push(`Check failed: ${oneLine(result.runtime?.error ?? 'not run')}`, '');
} else {
const { latest, pinned } = result.runtime.observed;
const archive = result.runtime.assetName;
lines.push(`| | Version | Published | \`${archive}\` |`);
lines.push('| --- | --- | --- | --- |');
lines.push(
`| Pinned | ${result.pin.version} | ${pinned.publishedOn ?? ''} | ` +
`${pinned.gone ? 'release gone' : assetCell(pinned.asset)} |`
);
lines.push(`| Newest release | ${latest.version} | ${latest.publishedOn ?? ''} | ${assetCell(latest.asset)} |`);
lines.push('');
if (result.runtime.findings.length === 0) {
lines.push('The pin is the newest release, and its archive still matches the pin.', '');
} else {
for (const finding of result.runtime.findings) lines.push(`- ${describeRuntimeFinding(finding)}`);
lines.push('');
}
lines.push(
`Release notes: https://github.com/${RUNTIME_REPO}/releases/tag/v${latest.version}. ` +
'A digest from the API is a lead, not a pin: check it against the release\'s `sha256sum.txt` first.',
''
);
}
const unpinned = result.models.filter((model) => model.digest === undefined && model.observed?.mac);
if (unpinned.length > 0) {
lines.push('## Digests to pin', '');
lines.push(
'The catalogue pins no digest for these rows, so a build pushed again at about the same size goes unnoticed. ' +
'Today\'s values: the tag\'s own manifest, and the manifest a Mac pulls. The one to pin is the second, the ' +
'digest `/api/tags` reports after the pull, as the row field `digest` (which the catalogue schema must ' +
'gain first).',
''
);
const measured = unpinned.filter((model) => model.figures === 'measured');
if (measured.length > 0) {
const ids = measured.map((model) => `\`${model.id}\``).join(', ');
lines.push(
`Pin first: ${ids}. Its memory figures were measured on one build, and without a digest nothing here ` +
'tells that build from a rebuild of about the same size.',
''
);
}
lines.push('| Model | Build a Mac pulls | Tag manifest | Manifest a Mac pulls |');
lines.push('| --- | --- | --- | --- |');
for (const model of unpinned) {
lines.push(
`| \`${model.id}\` | ${buildName(model.observed.mac)} | \`${model.observed.tagDigest}\` | ` +
`\`${model.observed.mac.digest}\` |`
);
}
lines.push('');
}
lines.push('## Not checked', '');
lines.push(
'The memory figures, the speeds and whether a coding agent works with a model: no manifest says anything ' +
'about those. They change only when somebody measures on a Mac.',
''
);
return `${lines.join('\n')}\n`;
}
// ---------- IO, with an injected fetch ----------
/**
* One GET, bounded: `REQUEST_TIMEOUT_MS` for the whole exchange and `maxBytes` for the body.
* Answers `{ status, body }` (body only for a 2xx). Never retries.
*/
export async function boundedGet(fetchFn, url, { headers = {}, maxBytes }) {
const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
let res;
try {
res = await fetchFn(url, { headers: { 'User-Agent': USER_AGENT, ...headers }, redirect: 'follow', signal });
} catch (err) {
throw new CheckFailure(`GET ${url} failed: ${describeError(err)}`);
}
if (res.status < 200 || res.status >= 300) {
await res.body?.cancel().catch(() => {});
return { status: res.status };
}
const declared = Number(res.headers.get('content-length'));
if (Number.isFinite(declared) && declared > maxBytes) {
await res.body?.cancel().catch(() => {});
throw new CheckFailure(`GET ${url}: ${declared} bytes is over the cap of ${maxBytes}`);
}
const chunks = [];
let total = 0;
try {
if (res.body) {
const reader = res.body.getReader();
for (;;) {
const { done, value } = await reader.read();
if (done) break;
total += value.byteLength;
if (total > maxBytes) {
await reader.cancel().catch(() => {});
throw new CheckFailure(`GET ${url}: the body is over the cap of ${maxBytes} bytes`);
}
chunks.push(value);
}
}
} catch (err) {
if (err instanceof CheckFailure) throw err;
throw new CheckFailure(`GET ${url}: reading the body failed: ${describeError(err)}`);
}
return { status: res.status, body: Buffer.concat(chunks) };
}
function describeError(err) {
if (err instanceof Error) {
const cause = err.cause instanceof Error && err.cause.message ? `: ${err.cause.message}` : '';
return `${err.name === 'TimeoutError' ? 'timed out' : err.message}${cause}`;
}
return String(err);
}
function parseJson(body, what) {
try {
return JSON.parse(body.toString('utf8'));
} catch {
throw new CheckFailure(`${what} is not JSON`);
}
}
/** A blob of the repository, checked against its digest. */
async function getBlob(fetchFn, name, digest, maxBytes, what) {
const url = `${REGISTRY_BASE}/v2/library/${encodeURIComponent(name)}/blobs/${digest}`;
const answer = await boundedGet(fetchFn, url, { maxBytes });
if (!answer.body) throw new CheckFailure(`${what} ${digest}: the registry answered ${answer.status}`);
if (sha256Digest(answer.body) !== digest) throw new CheckFailure(`${what} ${digest}: the body does not match its digest`);
return answer.body;
}
/** One build of a tag: its runner, format, digest and pull total, all validated. */
function variantOf(manifest, digest, fallback = {}) {
return {
runner: canonicalRunner(manifest.runner) || canonicalRunner(fallback.runner),
format: canonicalRunner(manifest.format) || canonicalRunner(fallback.format),
digest,
bytes: pullTotalBytes(manifest),
configDigest: normalizeDigest(manifest.config?.digest),
};
}
/**
* What the registry serves for `id` today, read the way Ollama on a Mac reads it:
* `{ gone: true }`, or `{ tagDigest, variants, mac }` where `mac` is the variant a Mac pulls
* (with its config's `requires`). Throws `CheckFailure` when the answer is not usable.
*/
export async function inspectModel(fetchFn, id) {
const { name, tag } = splitModelId(id);
const url = `${REGISTRY_BASE}/v2/library/${encodeURIComponent(name)}/manifests/${encodeURIComponent(tag)}`;
const answer = await boundedGet(fetchFn, url, { headers: { Accept: MANIFEST_ACCEPT }, maxBytes: MAX_MANIFEST_BYTES });
if (answer.status === 404) return { gone: true };
if (!answer.body) throw new CheckFailure(`the registry answered ${answer.status} for the manifest`);
const tagDigest = sha256Digest(answer.body);
const manifest = parseJson(answer.body, 'the manifest');
let variants;
let mac;
if (manifest.mediaType === MEDIA_TYPE_MANIFEST_LIST) {
const children = Array.isArray(manifest.manifests) ? manifest.manifests : [];
if (children.length === 0 || children.length > MAX_LIST_CHILDREN) {
throw new CheckFailure(`the manifest list has ${children.length} children`);
}
const selected = selectMacChild(children);
if (!selected) throw new CheckFailure('the manifest list has no build a Mac runs (mlx, llamacpp or ggml)');
variants = [];
for (const child of children) {
const digest = normalizeDigest(child?.digest);
if (!digest || !child.digest.startsWith('sha256:')) throw new CheckFailure('a child of the manifest list has no sha256 digest');
const body = await getBlob(fetchFn, name, digest, MAX_MANIFEST_BYTES, 'child manifest');
const variant = variantOf(parseJson(body, `child manifest ${digest}`), digest, child);
variants.push(variant);
if (child === selected) mac = variant;
}
} else if (manifest.mediaType === MEDIA_TYPE_MANIFEST || manifest.mediaType === undefined) {
mac = variantOf(manifest, tagDigest);
variants = [mac];
} else {
throw new CheckFailure('the manifest has a media type this check does not know');
}
if (!mac.configDigest) throw new CheckFailure('the build a Mac pulls has no config digest');
const config = parseJson(await getBlob(fetchFn, name, mac.configDigest, MAX_CONFIG_BYTES, 'config'), 'the config');
// A manifest that names no runner or format gets them from its config, as Ollama fills them.
const assumed = metadataForConfig(config);
mac.runner ||= assumed.runner;
mac.format ||= assumed.format;
// A malformed `requires` is ignored, as Ollama ignores it (`checkPullRequires`).
const requires = parseVersion(config?.requires);
mac.requires = requires ? requires.join('.') : undefined;
return { tagDigest, variants, mac };
}
function githubHeaders(token) {
return {
Accept: 'application/vnd.github+json',
'X-GitHub-Api-Version': '2022-11-28',
...(token ? { Authorization: `Bearer ${token}` } : {}),
};
}
/** The validated parts of a GitHub release: version, day it was published, the named asset. */
function releaseFacts(release, assetName) {
if (!release || typeof release.tag_name !== 'string' || !RELEASE_TAG_PATTERN.test(release.tag_name)) {
throw new CheckFailure('the release has no vX.Y.Z tag');
}
const day = typeof release.published_at === 'string' ? ISO_DAY_PATTERN.exec(release.published_at) : null;
const raw = Array.isArray(release.assets) ? release.assets.find((asset) => asset?.name === assetName) : undefined;
let asset;
if (raw) {
if (!isSize(raw.size)) throw new CheckFailure(`the release asset ${assetName} has no size`);
asset = { size: raw.size, digest: normalizeDigest(raw.digest) };
}
return { version: release.tag_name.slice(1), publishedOn: day ? day[1] : undefined, asset };
}
/** The newest release and the pinned one, from the GitHub API. Throws `CheckFailure`. */
export async function inspectRuntime(fetchFn, pin, token) {
const assetName = new URL(pin.url).pathname.split('/').pop();
const headers = githubHeaders(token);
const latestAnswer = await boundedGet(fetchFn, `${GITHUB_API_BASE}/repos/${RUNTIME_REPO}/releases/latest`, {
headers,
maxBytes: MAX_GITHUB_BYTES,
});
if (!latestAnswer.body) throw new CheckFailure(`GitHub answered ${latestAnswer.status} for the newest release`);
const latest = releaseFacts(parseJson(latestAnswer.body, 'the newest release'), assetName);
const pinnedAnswer = await boundedGet(
fetchFn,
`${GITHUB_API_BASE}/repos/${RUNTIME_REPO}/releases/tags/v${encodeURIComponent(pin.version)}`,
{ headers, maxBytes: MAX_GITHUB_BYTES }
);
let pinned;
if (pinnedAnswer.status === 404) pinned = { gone: true };
else if (!pinnedAnswer.body) throw new CheckFailure(`GitHub answered ${pinnedAnswer.status} for the pinned release`);
else pinned = releaseFacts(parseJson(pinnedAnswer.body, 'the pinned release'), assetName);
return { assetName, observed: { latest, pinned } };
}
/**
* The whole check. `catalog` is the validated catalogue, `pin` the runtime pin, `fetchFn` the
* fetch to use. Never throws for a network problem: each failure lands in the result, and
* `exitCodeFor()` turns it into 2.
*/
export async function runCheck({ catalog, pin, fetchFn, now, githubToken }) {
const result = {
checkedAt: `${now.toISOString().slice(0, 16).replace('T', ' ')} UTC`,
catalogue: { updated: catalog.updated },
pin: { version: pin.version },
models: [],
runtime: undefined,
errors: [],
};
// Rows in parallel, each row's requests one after another: a handful of rows, a few requests each.
result.models = await Promise.all(
catalog.models.map(async (row) => {
// `digest` is not in the catalogue schema yet: see the file header. `runner` is in the
// schema from the catalogue that names each row's runner on; before that it is absent.
const digest = /** @type {{ digest?: unknown }} */ (row).digest;
const runner = /** @type {{ runner?: unknown }} */ (row).runner;
const entry = { id: row.id, bytes: row.bytes, figures: row.figures, digest, findings: [] };
try {
entry.observed = await inspectModel(fetchFn, row.id);
entry.findings = classifyModel({ id: row.id, bytes: row.bytes, runner, digest }, entry.observed, pin.version);
} catch (err) {
entry.error = err instanceof CheckFailure ? err.message : describeError(err);
}
return entry;
})
);
try {
const runtime = await inspectRuntime(fetchFn, pin, githubToken);
result.runtime = { ...runtime, findings: classifyRuntime(pin, runtime.observed) };
} catch (err) {
result.runtime = { error: err instanceof CheckFailure ? err.message : describeError(err), findings: [] };
}
return result;
}
// ---------- Main ----------
async function main() {
let catalog;
let pin;
try {
const { getLocalModelCatalog } = await import('../src/config/local-model-catalog.js');
const { OLLAMA_RUNTIME_PIN } = await import('../src/local-model/runtime.js');
catalog = getLocalModelCatalog();
pin = OLLAMA_RUNTIME_PIN;
} catch (err) {
console.error(`The catalogue or the runtime pin could not be loaded: ${describeError(err)}`);
process.exitCode = 2;
return;
}
const result = await runCheck({
catalog,
pin,
fetchFn: globalThis.fetch,
now: new Date(),
githubToken: process.env.GITHUB_TOKEN || undefined,
});
process.stdout.write(renderReport(result));
process.exitCode = exitCodeFor(result);
}
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main().catch((err) => {
console.error(`The check failed: ${describeError(err)}`);
process.exitCode = 2;
});
}
src/config/local-model-catalog.ts
+278 -94
View File
@@ -8,6 +8,11 @@
* is downloaded is a compiled constant of its own (src/local-model/runtime.ts).
* - A model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`), so it can
* only ever name a model of the default registry and namespace.
* - A model row names the `runner` a Mac uses for its tag, and its memory figures follow that
* runner's rule (MEMORY below). A row's figures are its OWN: measured on that model, or
* computed from that model's architecture. test/local-model-catalog.test.ts recomputes every
* computed figure from a table of architectures and refuses a row that is not in it, so a
* new row needs its model's config.json read, not a figure borrowed from a neighbour.
* - A harness is a ROW (`{ cli, minContext, rank, gated?, slowFirstTurn? }`). The `cli` is a CLI
* registry id as data: no code may compare it with a literal. Whether that CLI exists and can
* take a custom model is checked against the registry at call time, in
@@ -19,48 +24,131 @@
* throws on a bad row. test/local-model-catalog.test.ts pins that the stock constant parses,
* so a bad row fails the gate and never reaches the route.
*
* Where each figure of the stock rows comes from, and what was checked (2026-11-06):
* Where each figure of the stock rows comes from, and what was checked (2026-11-08). [V] read
* at the source that day, [M] measured on a Mac, [C] computed from [V] inputs:
*
* | Figure | Source | Checked |
* |----------------------|-----------------------------------------------------------|--------------------------------------|
* | `id` | the registry's tag | yes, by the facts review |
* | `bytes` | `qwen3.6:35b-a3b`: the total of a real pull on a Mac | that row: measured. It is an MLX |
* | | (23,105,798,956). Every other row: the sum of the layer | build on a Mac, 2% above its |
* | | sizes of the tag's registry manifest | manifest. The others: not on a Mac |
* | `heldBytesPerToken`, | measured for `qwen3.6:35b-a3b` on an Apple M5 Pro with | that row: measured. The others carry |
* | `overheadBytes`, | 48 GB: 36,000 to 38,000 bytes held per token of context, | the SAME three figures, not their |
* | `peakExtraBytes` | 2.5 to 3 GB beside the weights, and about 5 GB more for a | own: nobody ran them. `figures` says |
* | | moment while a long prompt is processed | which row is which |
* | `license` | the model card (all Apache-2.0) | read by the research, not re-checked |
* | `paramsB` | 27.8 from the model card; the others are the count in the | no. Only the 12 billion line reads |
* | | model's name | it, and no row is near that line |
* | `contexts`, `rank` | a judgement of the design | no |
* | `speedPer100Gbps` | one published run on an M1 Max (400 GB/s): generation | the run was read. It stands for less |
* | | only, short prompt. The two q8_0 rows are SCALED by file | than a session: no long prompt, no |
* | | size, not measured | thinking tokens |
* | `id` | the registry's tag. The GGUF tags in lowercase: the | [V] every row: the registry answers |
* | | registry spells them `-q4_K_M`, and an id here is | the lowercase id with the same |
* | | lowercase (`LOCAL_MODEL_ID_PATTERN`) | manifest. NOT checked: that the |
* | | | runtime lists the pulled model under |
* | | | that name (Codeman compares exactly) |
* | `runner` | the build the runtime 0.40.15 pulls on Apple silicon: the | [V] every row |
* | | MLX child of a manifest list, else the one manifest | |
* | | (`runnerPreferences()`: mlx, then llamacpp) | |
* | `bytes` | every layer plus the config of that build: the `total` | [V] every row. [M] for |
* | | a pull reports | `qwen3.6:35b-a3b`, byte for byte |
* | `heldBytesPerToken`, | `qwen3.6:35b-a3b`: [M] on an Apple M5 Pro with 48 GB. | the others: [C], never run on a Mac. |
* | `overheadBytes`, | Every other row: [C] from its model's config.json for its | `figures` says which row is which. |
* | `peakExtraBytes` | runner, see MEMORY below. No row borrows another's | test/local-model-catalog.test.ts |
* | | figures: two rows share figures only when they share the | recomputes every one |
* | | architecture and the runner | |
* | `license`, `paramsB` | the model card (`paramsB` is the TOTAL, not the active | [V] |
* | | parameters) | |
* | `rank` | a judgement on the agentic coding results of the vendors' | no: vendor tables disagree with each |
* | | cards (SWE-bench Verified, SWE-bench Pro, Terminal-Bench) | other by several points |
* | `contexts` | up to the model's own window, and at least the 65536 a | no |
* | | harness needs | |
* | `speedPer100Gbps` | one published run per model family on an M1 Max | the run was read. It stands for less |
* | | (400 GB/s): generation only, short prompt. A row that is | than a session: no long prompt, no |
* | | not that run is SCALED from it, and says how | thinking tokens. An MTP build is |
* | | | faster than the run without it |
* | `tested` | our own run of the model with a harness on a real Mac | one entry, with its measured speed |
* | `minMacosMajor` | the runtime's documentation (macOS 14) | yes, by the facts review |
* | harness `minContext` | 65536: the runtime's documentation for both harnesses. | 65536 yes. 32768 no, which is why |
* | | 32768: recalled | that row is gated |
* | harness `minContext` | 65536: the runtime's documentation for OpenCode ("64k or | 65536 yes. 32768 no, which is why |
* | | higher"), and OpenCode v1.19.5's own fixed prompt (about | that row is gated |
* | | 8,000 to 10,000 tokens) and compaction rule: at 32768 it | |
* | | leaves 14,000 to 16,000 tokens, less than one default | |
* | | 2,000-line file read | |
* | `chips` | the bandwidth column of | yes, read at the source for this |
* | | https://github.com/ggml-org/llama.cpp/discussions/4167 | file |
*
* The memory figures decide a lot and rest on ONE measured model, so for every other row they
* can be wrong in either direction:
* - For the 9B rows they may be too HIGH (the 5 GB for a moment was seen with a prompt of
* 89,000 tokens on the 35B model). That errs on the careful side: a small Mac is told
* "none" before it is told to download something that may not fit.
* - For the two dense `qwen3.8:27b` rows the per-token figure may be too LOW: computed from
* its config alone that model holds 64 KiB per token, more than the 37,000 bytes measured
* on the 35B (whose own computed figure was 20 KiB). Their need at a large context is
* therefore the least certain figure of the catalogue.
* A row gets its own figures when somebody measures it. Until then the check after the
* download (the runtime must report the whole model on the GPU) is what catches a row that
* was judged too kindly.
* MEMORY. The need is `bytes + heldBytesPerToken x context + overheadBytes + peakExtraBytes`
* (src/local-model/recommend.ts). The two runners spend memory differently, so a row's three
* figures are filled by the rule of its runner:
*
* - `llamacpp` (llama-server on Metal; runtime v0.40.15 source, llm/llama_server.go, and the
* llama.cpp it bundles, b12111). It allocates the K/V cache for the WHOLE context when the
* model loads, in the cache type the runtime sets (`RUNTIME_KV_CACHE_TYPE` in
* src/local-model/runtime.ts, q8_0: 34 bytes per 32 values), and does not grow after that.
* "MTP" below is a GGUF with a multi-token-prediction layer (`nextn_predict_layers` 1 in
* its header, read on 2026-11-08 from the first bytes of each model layer: the two `-mtp`
* tags and `qwen3.8:27b-q8_0` have it, the two 9B tags and North Mini Code do not). For
* such a model the runtime always passes `--spec-type draft-mtp --spec-draft-n-max N`, N
* being the tag's own `draft_num_predict` (its params layer: 4 for
* `qwen3.8:27b-mtp-q4_k_m`, 2 for `qwen3.6:35b-a3b-mtp-q4_k_m`) or the runtime's default
* of 4 (`qwen3.8:27b-q8_0` sets none).
* - held = attention layers x K/V heads x head dim x 2 (K and V) x 34/32. An MTP model
* adds its prediction layer, whose cache llama-server keeps in a draft context of the
* same length in the draft's own cache type, f16 by default (2 bytes per value).
* - overhead = the fixed caches (the f32 recurrent state of the linear-attention layers,
* or the sliding-window cache of `n_swa + 512` cells), held 1 + N times for an MTP
* model (llama.cpp keeps one recurrent state per drafted token for rollback,
* `need_n_rs_seq()`), + the context checkpoints: `LLAMA_ARG_CTX_CHECKPOINTS` host copies
* (`RUNTIME_CTX_CHECKPOINTS`, 8) of the state that cannot be rolled back (the recurrent
* state, or the sliding-window cache), + 1.0 GB for the runner's own buffers (calibrated
* on a third-party table of a 27B model on llama.cpp: 8.4 GB at 4,096 tokens with
* 7.15 GB of weights), + the 1024 MiB host prompt cache that `LLAMA_ARG_CACHE_RAM`
* allows (`RUNTIME_HOST_PROMPT_CACHE_MIB`). The checkpoints and the prompt cache are
* host memory, not Metal buffers; on a Mac both come out of the same memory, so they
* are counted like the rest.
* - peak = 0: its compute buffers are allocated at load and are inside the 1.0 GB.
* The one outside check of this rule: a third party measured 65,600 bytes per token of
* growth at f16 for a model of the Qwen3.8-27B architecture, where its config gives 65,536.
* The checkpoints and the rollback copies are read in the llama.cpp source
* (tools/server/server-context.cpp `create_checkpoint`, src/llama-memory-recurrent.cpp),
* not measured.
* - `mlx` (runtime v0.40.15 source, mlxrunner/). It grows the K/V cache in bf16 with the
* tokens used, keeps the recurrent state in f32 and a snapshot of it every 8,192 tokens,
* and keeps up to 8 GiB of paged-out prefix snapshots (prefix_cache.go `maxPagedOutBytes`,
* not configurable). Third-party OpenCode sessions show that growth (runtime issues 17924
* and 18131), and the runtime books an MLX model at its weights alone (issue 19417).
* - `qwen3.6:35b-a3b`: MEASURED. 36,000 to 38,000 bytes held per token after prompts of
* 89,000 and 167,000 tokens, 2.5 to 3 GB beside the weights, about 5 GB more for a
* moment while a long prompt was processed. NOT covered by that measurement: a session
* of many requests, in which the prefix snapshots may add up to 8 GiB. Its figures stay
* the measured ones (with 8 GiB on top it would not be offered on the Mac it ran on),
* and a 30-request session on it is the first thing to measure next.
* - every other `mlx` row: held = the K/V cache in bf16 + one recurrent snapshot per
* 8,192 tokens, computed from the config, x 37,000 / 28,340: the ratio of what the
* measured row really held to the same computation for it (same runner, same
* architecture family). Overhead = the measured row's 3 GB + the whole 8 GiB prefix
* budget. Peak = the measured row's 5 GB, NOT scaled: what makes it up was not
* measured.
*
* After the download the install checks that the runtime holds the whole model on the GPU.
* For a `llamacpp` row both of the runtime's figures are the buffers llama-server reports
* allocating (weights, K/V cache, compute; `MemorySize()` in llm/llama_server.go), so the
* check proves the whole load is on the GPU, whatever the cache type; what grows after the
* load in host memory (the context checkpoints and the prompt cache) is counted above, not
* checked. For an `mlx` row it proves only the weights (issue 19417): the figures above are
* all that stands between such a row and swap.
*
* Models set aside on purpose (a weekly review should look at them again):
* - `bonsai-2:27b`, `bonsai-2:9b` (ternary builds of Qwen3.8-27B and Qwen3.5-9B; the 27B
* would make a 24 GB Mac `ok`): the vendor reports no SWE-bench and no agent loop, and
* whether 2-bit weights keep tool calls reliable was never measured.
* - `ornith-1.5:9b`, `ornith-1.5:35b`, `ornith-1.5:397b` (GGUF only): strong cards, but their
* manifests name no renderer and no tool parser, so whether tool calls work through the
* runtime is unknown (runtime pull request 17965 is open). `ornith-1.5:122b` has both.
* - `gpt-oss:20b`: weak in agentic coding (SWE-bench Verified 52.4, Terminal-Bench 2.1 15.2
* in NVIDIA's table).
* - `flint:40b`: runtime issue 19191 breaks integer tool arguments after the first call.
* - `lfm2.5:24b-a2b`: its licence allows commercial use only below USD 10M of revenue.
* - `laguna-xs-2.1`: the runtime's own page names an open problem with it on macOS.
* `laguna-s-2.1`: open runtime issues 17602 and 17632.
* - `qwen3.8-flash-next:125b`: the Qwen Community License, not an open licence.
* - The plain tags `qwen3.5:9b`, `mimo-v2.6:9b` and `qwen3.8:27b`: a Mac pulls their MLX
* builds, whose 8 GiB of prefix snapshots do not fit the Macs a 9B or a 27B is for. Their
* GGUF builds are listed instead.
* - `qwen3.5:122b-a10b`: replaced by `ornith-1.5:122b`, a fine-tune of it (SWE-bench
* Verified 84.5 against 72.0) whose MLX build carries a renderer and a tool parser.
*
* `tested` has one entry: `qwen3.6:35b-a3b` with OpenCode on an Apple M5 Pro with 48 GB,
* 2026-11-06, 67.7 tokens per second. Only a Mac of exactly that chip and memory size is told
* "Recommended" and shown that speed in place of the estimate.
* "Recommended" and shown that speed in place of the estimate. Every other row was never run
* on a Mac by us, and its row says `estimated` and "not run on a Mac like this yet".
*/
import { z } from 'zod';
@@ -114,6 +202,7 @@ const LocalModelEntrySchema = z
label: z.string().min(1).max(80),
license: z.string().min(1).max(40),
paramsB: z.number().positive(),
runner: z.enum(['mlx', 'llamacpp']),
bytes: positiveInt,
heldBytesPerToken: positiveInt,
overheadBytes: z.number().int().nonnegative(),
@@ -218,8 +307,8 @@ export const LocalModelCatalogSchema: z.ZodType<LocalModelCatalog> = z
/**
* The memory figures measured for `qwen3.6:35b-a3b` on an Apple M5 Pro with 48 GB (2026-11-06,
* the MLX runner), in decimal bytes. That row carries them as `measured`; every other row
* carries the same three as `estimated`, see the file header.
* the MLX runner), in decimal bytes. Only that row carries them; see the file header (MEMORY)
* for what they do not cover.
*/
const MEASURED_MEMORY = {
/** 36,000 to 38,000 bytes stayed held per token of context after a prompt of 89,000 and one of 167,000 tokens. */
@@ -230,117 +319,212 @@ const MEASURED_MEMORY = {
peakExtraBytes: 5_000_000_000,
} as const;
/** The llama.cpp runner's own buffers, compute buffers included (file header, MEMORY). */
const LLAMACPP_RUNNER_BYTES = 1_000_000_000;
/** The host prompt cache llama-server may keep: `LLAMA_ARG_CACHE_RAM`, 1024 MiB (`RUNTIME_HOST_PROMPT_CACHE_MIB`). */
const LLAMACPP_PROMPT_CACHE_BYTES = 1024 * 1024 * 1024;
/** The context checkpoints llama-server may keep: `LLAMA_ARG_CTX_CHECKPOINTS`, 8 (`RUNTIME_CTX_CHECKPOINTS`). */
const LLAMACPP_CTX_CHECKPOINTS = 8;
/** The tokens the runtime drafts per step for an MTP model whose tag sets no `draft_num_predict`, 4: so many rollback states. */
const LLAMACPP_DEFAULT_DRAFT_TOKENS = 4;
/** The prefix snapshots the MLX runner may keep: `maxPagedOutBytes`, 8 GiB. */
const MLX_PREFIX_CACHE_BYTES = 8 * 1024 * 1024 * 1024;
/**
* The catalogue that ships with Codeman. Never read directly outside this file and its test:
* callers use `getLocalModelCatalog()`, which validates it first.
*/
export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = {
schema: 1,
updated: '2026-11-06',
updated: '2026-11-08',
// The oldest macOS the model runtime supports.
minMacosMajor: 14,
// Ordered by rank. The higher rank wins among models that fit equally well and are equally
// fast. `flint:40b` is left out on purpose: its plain tag is the slow build on a Mac and it
// has an open bug with tool arguments.
// Ordered by rank. The higher rank wins among models that fit and whose speed is graded
// alike (recommend.ts ranks the speed grade first). See the file header for the models
// that are left out and why.
models: [
{
id: 'qwen3.5:9b',
// Small Macs (16 to 32 GB, `limited`): the base model of the row below.
id: 'qwen3.5:9b-q4_k_m',
label: 'Qwen3.5 9B',
license: 'Apache-2.0',
paramsB: 9,
bytes: 6_550_825_373,
...MEASURED_MEMORY,
runner: 'llamacpp',
bytes: 6_550_825_550,
// 8 attention layers x 4 K/V heads x 256 x 2 x 34/32.
heldBytesPerToken: 17_408,
// 24 linear-attention layers hold 52,690,944 bytes of f32 state: once live, and once in
// each context checkpoint.
overheadBytes: 52_690_944 * (1 + LLAMACPP_CTX_CHECKPOINTS) + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES,
peakExtraBytes: 0,
figures: 'estimated',
contexts: [65536, 32768],
contexts: [131072, 65536],
rank: 1,
// 37.1 tokens per second measured on 400 GB/s.
// Measured: 37.1 tokens per second on 400 GB/s (this Q4_K_M build, llama.cpp).
speedPer100Gbps: 9.3,
tested: [],
},
{
id: 'qwen3.5:9b-q8_0',
label: 'Qwen3.5 9B (Q8)',
license: 'Apache-2.0',
// Small Macs (16 to 32 GB, `limited`). MiMo-V2.6-Distill-Qwen-9B, a fine-tune of
// Qwen3.5-9B for code and agents: SWE-bench Verified 61.1 and SWE-bench Pro 44.6,
// against 60.0 and 32.0 for its base (Xiaomi's card). Requires exactly the pinned
// runtime, 0.40.15.
id: 'mimo-v2.6:9b-q4_k_m',
label: 'MiMo V2.6 9B',
license: 'MIT',
paramsB: 9,
bytes: 10_449_217_949,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [65536, 32768],
rank: 2,
// Not measured: the 9.3 of the row above scaled by file size.
speedPer100Gbps: 5.8,
tested: [],
},
{
id: 'gpt-oss:20b',
label: 'GPT-OSS 20B',
license: 'Apache-2.0',
paramsB: 20,
bytes: 13_793_441_427,
...MEASURED_MEMORY,
runner: 'llamacpp',
bytes: 6_550_811_033,
// The architecture and the quantization of the row above, so its figures.
heldBytesPerToken: 17_408,
overheadBytes: 52_690_944 * (1 + LLAMACPP_CTX_CHECKPOINTS) + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES,
peakExtraBytes: 0,
figures: 'estimated',
contexts: [131072, 65536],
rank: 3,
// No speed figure: its speed is unknown on every chip.
rank: 2,
// Not measured: the 9.3 of the row above, the same architecture and file size.
speedPer100Gbps: 9.3,
tested: [],
},
{
// Cohere's model for agentic software engineering, 3 billion parameters active:
// SWE-bench Verified 67.6 (vendor). Its card asks OpenCode for interleaved reasoning,
// which Codeman's OpenCode settings do not set.
id: 'north-mini-code-1.0:q4_k_m',
label: 'North Mini Code 1.0',
license: 'Apache-2.0',
paramsB: 30,
runner: 'llamacpp',
bytes: 18_593_967_008,
// 13 full-attention layers x 4 K/V heads x 128 x 2 x 34/32.
heldBytesPerToken: 14_144,
// 36 sliding-window layers: (4,096 + 512) cells x 4 x 128 x 2 x 34/32 = 180,486,144, live
// and in each context checkpoint.
overheadBytes: 180_486_144 * (1 + LLAMACPP_CTX_CHECKPOINTS) + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES,
peakExtraBytes: 0,
figures: 'estimated',
contexts: [262144, 131072, 65536],
rank: 3,
// Not measured: the 13.6 measured for Qwen3.6 35B-A3B on llama.cpp, which has the same
// 3 billion active parameters.
speedPer100Gbps: 13.6,
tested: [],
},
{
// Macs of 36 GB on a slower chip: the model of the tested pair, as a GGUF build whose
// memory does not grow after it loaded. With its multi-token-prediction layer.
id: 'qwen3.6:35b-a3b-mtp-q4_k_m',
label: 'Qwen3.6 35B-A3B (GGUF)',
license: 'Apache-2.0',
paramsB: 35,
runner: 'llamacpp',
bytes: 22_621_314_381,
// 10 attention layers x 2 K/V heads x 256 x 2 x 34/32 = 10,880, + the prediction layer in
// f16: 2 x 256 x 2 x 2 = 2,048.
heldBytesPerToken: 12_928,
// 30 linear-attention layers hold 65,863,680 bytes of f32 state: live, in the 2 rollback
// copies of prediction (this tag drafts 2 tokens, `draft_num_predict` 2), and in each
// context checkpoint.
overheadBytes:
65_863_680 * (1 + 2 + LLAMACPP_CTX_CHECKPOINTS) + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES,
peakExtraBytes: 0,
figures: 'estimated',
contexts: [262144, 131072, 65536],
rank: 4,
// Measured: 54.5 tokens per second on 400 GB/s (llama.cpp, without prediction).
speedPer100Gbps: 13.6,
tested: [],
},
{
// 48 GB: the tested pair. SWE-bench Verified 73.4, SWE-bench Pro 49.5 (Qwen's card).
id: 'qwen3.6:35b-a3b',
label: 'Qwen3.6 35B-A3B',
license: 'Apache-2.0',
paramsB: 35,
// What a Mac really downloads: an MLX build, 2% above the 22,621,314,161 of the manifest.
// The plain tag is a manifest list; a Mac takes its MLX child.
runner: 'mlx',
// What a Mac really downloads: the MLX build, 2% above the GGUF build of the row above.
bytes: 23_105_798_956,
...MEASURED_MEMORY,
figures: 'measured',
contexts: [131072, 65536],
rank: 4,
// 54.5 tokens per second measured on 400 GB/s (3 billion parameters are active per token).
// The MLX build ran faster than this figure predicts on the one Mac we tested (67.7
// against 41.8), so on other chips the estimate is on the low side.
speedPer100Gbps: 13.6,
rank: 5,
// Measured: 67.8 tokens per second on 400 GB/s with MLX. The one Mac we tested ran it at
// 67.7 on 307 GB/s, so on other chips the estimate is on the low side.
speedPer100Gbps: 16.9,
// Run by us with OpenCode on the MacBook Pro (Mac17,13): a four-step task with tool calls.
tested: [{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 }],
},
{
id: 'qwen3.8:27b',
label: 'Qwen3.8 27B',
license: 'Apache-2.0',
paramsB: 27.8,
bytes: 17_741_871_939,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [131072, 65536, 32768],
rank: 5,
// 11.8 tokens per second measured on 400 GB/s.
speedPer100Gbps: 2.95,
tested: [],
},
{
// Large Macs that want 8-bit weights: the row below in Q8_0, half as fast.
id: 'qwen3.8:27b-q8_0',
label: 'Qwen3.8 27B (Q8)',
license: 'Apache-2.0',
paramsB: 27.8,
bytes: 29_978_241_837,
...MEASURED_MEMORY,
runner: 'llamacpp',
bytes: 29_978_242_050,
// The architecture of the row below, prediction layer included (its GGUF has one), and the
// runtime's 4 drafted tokens (the tag sets none), so its figures.
heldBytesPerToken: 38_912,
overheadBytes:
156_893_184 * (1 + LLAMACPP_DEFAULT_DRAFT_TOKENS + LLAMACPP_CTX_CHECKPOINTS) +
LLAMACPP_RUNNER_BYTES +
LLAMACPP_PROMPT_CACHE_BYTES,
peakExtraBytes: 0,
figures: 'estimated',
contexts: [262144, 131072],
contexts: [262144, 131072, 65536],
rank: 6,
// Not measured: the 2.95 of the row above scaled by file size.
// Not measured: the 2.95 of the row below scaled by file size.
speedPer100Gbps: 1.7,
tested: [],
},
{
id: 'qwen3.5:122b-a10b',
label: 'Qwen3.5 122B-A10B',
// 36 to 96 GB on a fast chip: the strongest model of this size. Terminal-Bench 2.1 73.0,
// SWE-bench Pro 61.7 (Qwen's card). Dense, so it is slow on a Pro chip.
id: 'qwen3.8:27b-mtp-q4_k_m',
label: 'Qwen3.8 27B',
license: 'Apache-2.0',
paramsB: 122,
bytes: 81_370_035_877,
...MEASURED_MEMORY,
paramsB: 27.8,
runner: 'llamacpp',
bytes: 17_741_872_154,
// 16 attention layers x 4 K/V heads x 256 x 2 x 34/32 = 34,816, + the prediction layer in
// f16: 4 x 256 x 2 x 2 = 4,096.
heldBytesPerToken: 38_912,
// 48 linear-attention layers hold 156,893,184 bytes of f32 state: live, in the 4 rollback
// copies of prediction (`draft_num_predict` 4), and in each context checkpoint.
overheadBytes:
156_893_184 * (1 + 4 + LLAMACPP_CTX_CHECKPOINTS) + LLAMACPP_RUNNER_BYTES + LLAMACPP_PROMPT_CACHE_BYTES,
peakExtraBytes: 0,
figures: 'estimated',
contexts: [131072],
contexts: [262144, 131072, 65536],
rank: 7,
// No speed figure: its speed is unknown on every chip.
// Measured: 11.8 tokens per second on 400 GB/s (llama.cpp, Q4_K_M, without prediction).
// This build predicts several tokens at a time, so it is faster than this figure says.
speedPer100Gbps: 2.95,
tested: [],
},
{
// 128 GB and more. A fine-tune of Qwen3.5-122B-A10B, 10 billion parameters active:
// Terminal-Bench 2.1 81.2, SWE-bench Verified 84.5, SWE-bench Pro 76.9 (Ornith's card).
id: 'ornith-1.5:122b',
label: 'Ornith 1.5 122B-A10B',
license: 'MIT',
paramsB: 122,
// The plain tag is a manifest list; a Mac takes its MLX child (with a renderer and a
// tool parser, which the GGUF builds of this family lack).
runner: 'mlx',
bytes: 73_388_069_552,
// 12 attention layers x 2 K/V heads x 256 x 2 x 2 bytes = 24,576, + a 153,649,152-byte
// recurrent snapshot per 8,192 tokens = 18,756: 43,332, x 37,000 / 28,340.
heldBytesPerToken: 56_574,
overheadBytes: MEASURED_MEMORY.overheadBytes + MLX_PREFIX_CACHE_BYTES,
peakExtraBytes: MEASURED_MEMORY.peakExtraBytes,
figures: 'estimated',
contexts: [262144, 131072],
rank: 8,
// Not measured: the 16.9 of Qwen3.6 35B-A3B on MLX x 3/10, its active parameters
// against these, rounded down.
speedPer100Gbps: 5,
tested: [],
},
],
src/local-model/recommend.ts
+3 -2
View File
@@ -12,8 +12,9 @@
* - `needBytes()`: the PEAK, `bytes + heldBytesPerToken * ctx + overheadBytes +
* peakExtraBytes` of the catalogue row: the weights, what stays held per token of context,
* what the runner holds beside both, and what it takes for a moment while a long prompt is
* processed. Measured for one model on one Mac (see the catalogue header); every other row
* carries the same figures and says `estimated`.
* processed. Measured for one model on one Mac; every other row's figures are computed from
* its own model's architecture for its runner and say `estimated` (see the catalogue
* header, MEMORY).
* - Fit: need over budget up to 0.85 is good, up to 0.98 is tight, above is out. The grade
* decides whether a model is offered at all and what its row says. It does NOT sort: the
* need is the worst moment at the full context, so a tight fit is a model that runs, and a
src/local-model/runtime.ts
+44 -2
View File
@@ -16,8 +16,39 @@
* either case) are copied from the server's; HOME is the private one; and exactly these are
* set: `OLLAMA_HOST=127.0.0.1:<port>`, `OLLAMA_MODELS`, `OLLAMA_CONTEXT_LENGTH`,
* `OLLAMA_NO_CLOUD=1`, `OLLAMA_KEEP_ALIVE=10m`, `OLLAMA_NUM_PARALLEL=1`,
* `OLLAMA_MAX_LOADED_MODELS=1`. Nothing else gets through: no `CODEMAN_*` and none of the
* user's own `OLLAMA_*`.
* `OLLAMA_MAX_LOADED_MODELS=1`, and the four memory settings of the llama.cpp runner (see
* MEMORY below). Nothing else gets through: no `CODEMAN_*` and none of the user's own
* `OLLAMA_*` or `LLAMA_ARG_*`.
* - MEMORY (what the catalogue's figures for its `llamacpp` rows assume, so change both
* together: test/local-model-catalog.test.ts holds them to each other). Read in the
* runtime's v0.40.15 source, not yet measured on a Mac:
* - `OLLAMA_KV_CACHE_TYPE=q8_0` becomes `--cache-type-k q8_0 --cache-type-v q8_0` of
* llama-server (llm/llama_server.go): the K/V cache takes 34 bytes per 32 values, 17/32
* of f16. The runtime's FAQ calls the quality loss "very small". Never q4_0: its loss
* grows with the context, and a coding agent works at long contexts.
* - `OLLAMA_FLASH_ATTENTION=1` becomes `--flash-attn on`. A quantized V cache needs flash
* attention, and left unset the runtime passes `auto`, or `off` where it thinks the GPU
* cannot, which would make every load with a q8_0 cache fail.
* - `LLAMA_ARG_CACHE_RAM=1024` bounds llama-server's host prompt cache (the copies of
* earlier prompts it keeps in memory) to 1024 MiB. The runtime passes no `--cache-ram`
* for a coding model, and llama-server reads the variable from the environment it
* inherits (`cmd.Env = os.Environ()`). Unbounded, that cache can grow by gigabytes in a
* session; 1024 MiB is counted in every `llamacpp` row's `overheadBytes`.
* - `LLAMA_ARG_CTX_CHECKPOINTS=8` bounds the context checkpoints llama-server keeps per
* slot (`--ctx-checkpoints`, read from the environment the same way). The llama.cpp the
* runtime bundles (b12111, the runtime's LLAMA_CPP_VERSION) makes them for every model
* of the catalogue's `llamacpp` rows: a checkpoint is a host copy of the state that
* cannot be rolled back (the recurrent state of the linear-attention layers, or the
* sliding-window cache), two or three per request, and llama-server keeps up to 32 by
* default. 32 copies of the 27B's 157 MB state are 5 GB, most of which an agent never
* uses: its next request restores the newest checkpoint of the previous one. 8
* (llama.cpp's own default from b6800 to b8200; it became 32 at b8250, together with a
* checkpoint every 8,192 tokens of a long prompt) keeps the last few requests. The cost:
* a prompt that differs from the last checkpoints further back (a compaction, a cleared
* old tool result) is processed again from the newest checkpoint before that point, or
* from its start. 8 checkpoints are counted in every `llamacpp` row's `overheadBytes`.
* The MLX runner reads none of the four (nothing under mlxrunner/ does): an `mlx` row's
* figures do not depend on them.
* - PORT (`chooseRuntimePort()`). Chosen ONCE by binding, from `RUNTIME_PORT_FIRST` across
* `RUNTIME_PORT_SPAN`, then stored by the caller. Never searched again on a later start.
* - SUPERVISOR (`createRuntimeSupervisor()`). The runtime is a NON-detached child of the
@@ -83,6 +114,12 @@ export const RUNTIME_MAX_RESTARTS = 3;
export const RUNTIME_DOWNLOAD_RETRIES = 3;
/** `OLLAMA_KEEP_ALIVE`: how long the model stays loaded after the last request. */
export const RUNTIME_KEEP_ALIVE = '10m';
/** `OLLAMA_KV_CACHE_TYPE`: the K/V cache type of the llama.cpp runner. The catalogue's `llamacpp` rows are computed for it. */
export const RUNTIME_KV_CACHE_TYPE = 'q8_0';
/** `LLAMA_ARG_CACHE_RAM`, in MiB: the bound of llama-server's host prompt cache. Counted in every `llamacpp` row's overhead. */
export const RUNTIME_HOST_PROMPT_CACHE_MIB = 1024;
/** `LLAMA_ARG_CTX_CHECKPOINTS`: the context checkpoints llama-server keeps per slot. Counted in every `llamacpp` row's overhead. */
export const RUNTIME_CTX_CHECKPOINTS = 8;
/** The name of the runtime's binary inside its folder and inside the archive. */
const RUNTIME_BINARY_NAME = 'ollama';
@@ -253,6 +290,11 @@ export function runtimeEnv(input: RuntimeEnvInput): NodeJS.ProcessEnv {
env.OLLAMA_KEEP_ALIVE = RUNTIME_KEEP_ALIVE;
env.OLLAMA_NUM_PARALLEL = '1';
env.OLLAMA_MAX_LOADED_MODELS = '1';
// The memory settings of the llama.cpp runner (file header, MEMORY). Inert for an MLX model.
env.OLLAMA_KV_CACHE_TYPE = RUNTIME_KV_CACHE_TYPE;
env.OLLAMA_FLASH_ATTENTION = '1';
env.LLAMA_ARG_CACHE_RAM = String(RUNTIME_HOST_PROMPT_CACHE_MIB);
env.LLAMA_ARG_CTX_CHECKPOINTS = String(RUNTIME_CTX_CHECKPOINTS);
return env;
}
src/types/local-model.ts
+37 -13
View File
@@ -120,6 +120,9 @@ export interface TestedOn {
tokensPerSecond?: number;
}
/** The runner of the model runtime that runs a model on a Mac, by the registry's name for it. See `LocalModelEntry.runner`. */
export type LocalModelRunner = 'mlx' | 'llamacpp';
/** One model of the catalogue. */
export interface LocalModelEntry {
/** `name:tag` exactly as the model registry lists it. Lowercase, one colon, no slash (`LOCAL_MODEL_ID_PATTERN`). */
@@ -130,35 +133,56 @@ export interface LocalModelEntry {
license: string;
/** Parameter count in billions. When no model that fits has 12 or more, the verdict is `limited`, never `ok`. */
paramsB: number;
/**
* The runner the runtime uses for this tag ON A MAC, as the registry names it: `mlx` (a
* safetensors build, or a manifest list with an MLX child, which a Mac always takes) or
* `llamacpp` (a GGUF build, run by llama-server on Metal). The two spend memory in different
* ways, so a row's memory figures are only meaningful together with its runner: llama.cpp
* allocates the whole K/V cache for the context when the model loads (in the runtime's
* `OLLAMA_KV_CACHE_TYPE`), MLX grows it with the tokens actually used and keeps snapshots of
* it besides.
*/
runner: LocalModelRunner;
/**
* Size of the weights in bytes: both the download and what they take in memory.
*
* It is the size of the build the runtime really downloads ON A MAC. That is not always
* the sum of the registry manifest: `qwen3.6:35b-a3b` is an MLX build of 23,105,798,956 bytes
* on Apple silicon, 2% more than the manifest's 22,621,314,161. A row whose Mac size nobody
* measured keeps the manifest sum and says so in `figures`.
* It is the size of the build the runtime really downloads ON A MAC: the build of `runner`,
* every layer of its manifest plus its config, which is the `total` a real pull reports.
* That is not always the first manifest the registry answers with: `qwen3.6:35b-a3b` is an
* MLX build of 23,105,798,956 bytes on Apple silicon, 2% more than its GGUF build of
* 22,621,314,381.
*/
bytes: number;
/**
* Bytes that stay HELD in memory per token of context once the token was processed (measured
* on the MLX runner: 36,000 to 38,000, so about 37,000). A figure computed from a model's
* config alone came out too low by almost half, so a row that was not measured carries the
* measured row's figure and says so in `figures`.
* Bytes that stay HELD in memory per token of context. For a `llamacpp` row: the K/V cache
* of its attention layers in the runtime's cache type, which llama.cpp allocates for the
* whole context at load (a multi-token-prediction layer's in f16). For an `mlx` row: the K/V cache in bf16 plus the snapshots of the
* recurrent state the runner keeps every 8,192 tokens, which it grows with use. Measured
* on one row (36,000 to 38,000 on the MLX runner, so about 37,000); computed from the
* model's config for the others, see `figures` and the catalogue header.
*/
heldBytesPerToken: number;
/** What the runner holds beside the weights and the context (measured: about 3 GB). */
/**
* What the runner holds beside the weights and the context, whatever the context: its own
* buffers, the live recurrent or sliding-window caches, and the caches it keeps of earlier
* prompts (for `llamacpp` the 1024 MiB host prompt cache `LLAMA_ARG_CACHE_RAM` allows and
* the copies of the recurrent or sliding-window state in its `LLAMA_ARG_CTX_CHECKPOINTS`
* context checkpoints, for an estimated `mlx` row the 8 GiB of prefix snapshots the runner
* may keep). Measured on one row (about 3 GB); see the catalogue header for the others.
*/
overheadBytes: number;
/**
* The TRANSIENT extra the runner takes while it processes a long prompt, on top of
* what stays held (measured: about 5 GB). The fit is judged on the peak:
* what stays held (measured on the MLX runner: about 5 GB; 0 for `llamacpp`, which
* allocates its buffers at load). The fit is judged on the peak:
* `bytes + heldBytesPerToken * context + overheadBytes + peakExtraBytes`. For
* `qwen3.6:35b-a3b` at 131072 tokens that is about 36 GB, which is tight on a 48 GB Mac.
*/
peakExtraBytes: number;
/**
* Where `bytes` and the three memory figures above come from: `measured` on a Mac by
* us, or `estimated` (the manifest size, and the per-token, overhead and peak figures of the
* measured row applied to this one).
* Where the three memory figures above come from: `measured` on a Mac by us, or
* `estimated` (computed from the model's architecture for its runner, see the catalogue
* header). `bytes` is the registry's size of the build a Mac pulls in both cases.
*/
figures: 'measured' | 'estimated';
/** Context lengths (tokens) the model may be run at, largest first. */
test/check-local-model-catalog.test.ts
+751
View File
@@ -0,0 +1,751 @@
/**
* @fileoverview scripts/check-local-model-catalog.mjs, the weekly check of the local model
* catalogue: which build a Mac pulls from a manifest list, the pull total, how a row's drift is
* classified, the exit code, the report text, and that the IO side only ever asks for
* manifests, child manifests and configs (never a layer). The network is never used: the IO
* tests drive a fake registry and a fake GitHub API.
*
* The fixtures under test/fixtures/local-model-catalog/ are real bodies the registry served on
* 2026-11-08 for `qwen3.6:35b-a3b`, byte for byte, so their digests are the real ones.
*/
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { describe, expect, it } from 'vitest';
import {
CheckFailure,
MANIFEST_ACCEPT,
MAX_MANIFEST_BYTES,
MEDIA_TYPE_MANIFEST,
MEDIA_TYPE_MANIFEST_LIST,
SIZE_TOLERANCE_FRACTION,
boundedGet,
catalogueRunner,
classifyModel,
classifyRuntime,
compareVersions,
exitCodeFor,
formatPercent,
groupDigits,
inspectModel,
metadataForConfig,
normalizeDigest,
parseVersion,
pullTotalBytes,
renderReport,
runCheck,
selectMacChild,
sha256Digest,
splitModelId,
withinTolerance,
} from '../scripts/check-local-model-catalog.mjs';
const fixtureDir = resolve(import.meta.dirname, 'fixtures/local-model-catalog');
const fixture = (name: string): Buffer => readFileSync(resolve(fixtureDir, name));
const REAL_LIST = fixture('qwen3.6-35b-a3b.list.json');
const REAL_GGUF_MANIFEST = fixture('qwen3.6-35b-a3b.gguf-manifest.json');
const REAL_MLX_CONFIG = fixture('qwen3.6-35b-a3b.mlx-config.json');
/** The digest of the MLX child the real list names, which a Mac pulled on 2026-11-06. */
const REAL_MLX_CHILD = 'sha256:0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba';
const REAL_GGUF_CHILD = 'sha256:a7eb95c53bcf96b4bdd008d0fab4a5dac88047d9c1a7a9ab88ed453423fbd87c';
/** The pull total a Mac reported for `qwen3.6:35b-a3b`, which the catalogue row stores. */
const MAC_PULL_TOTAL_35B = 23_105_798_956;
const PIN = {
version: '0.40.15',
url: 'https://github.com/ollama/ollama/releases/download/v0.40.15/ollama-darwin.tgz',
bytes: 185_971_671,
sha256: '607c79d5d6315ef9d896db83adde736a4c0057196329843544e79b95afb260a8',
};
const hex = (char: string): string => `sha256:${char.repeat(64)}`;
describe('the real registry bodies', () => {
it('carry the digests the registry and a Mac reported for them', () => {
expect(sha256Digest(REAL_LIST)).toBe('sha256:86b2d171ce90bbb2dcb081933565ec867ce523cdd274bd24c8cf1ef9185feb4f');
expect(sha256Digest(REAL_GGUF_MANIFEST)).toBe(REAL_GGUF_CHILD);
expect(sha256Digest(REAL_MLX_CONFIG)).toBe(
'sha256:577d7d4694896a245c8b00f62bf53a8f4720ded4eba83a8e2da30bb87583bd6b'
);
});
it('a Mac selects the mlx child of the real list, the one whose digest /api/tags reported', () => {
const list = JSON.parse(REAL_LIST.toString('utf8'));
expect(list.mediaType).toBe(MEDIA_TYPE_MANIFEST_LIST);
expect(selectMacChild(list.manifests).digest).toBe(REAL_MLX_CHILD);
});
it('the plain GGUF manifest totals its layers plus its config, 2% below what a Mac pulls', () => {
const manifest = JSON.parse(REAL_GGUF_MANIFEST.toString('utf8'));
expect(pullTotalBytes(manifest)).toBe(22_621_314_161 + 220);
expect(classifyModel(
{ id: 'qwen3.6:35b-a3b', bytes: MAC_PULL_TOTAL_35B },
{ mac: { bytes: pullTotalBytes(manifest), digest: REAL_GGUF_CHILD } },
'0.40.15'
)).toEqual([expect.objectContaining({ kind: 'size' })]);
});
});
describe('splitModelId', () => {
it('splits name and tag', () => {
expect(splitModelId('qwen3.6:35b-a3b')).toEqual({ name: 'qwen3.6', tag: '35b-a3b' });
});
it.each(['qwen3.6', ':35b', 'qwen3.6:', 'evil.example/x:y'])('refuses %s', (id) => {
expect(() => splitModelId(id)).toThrow(CheckFailure);
});
});
describe('pullTotalBytes', () => {
const manifest = (layers: unknown, config: unknown = { size: 250 }) => ({ config, layers });
it('adds every layer and the config', () => {
expect(pullTotalBytes(manifest([{ size: 1000 }, { size: 24 }]))).toBe(1274);
});
it.each([
['no layers', manifest([])],
['layers that are not an array', manifest('x')],
['a layer without a size', manifest([{ digest: 'x' }])],
['a negative size', manifest([{ size: -1 }])],
['a size given as text', manifest([{ size: '1000' }])],
['a fractional size', manifest([{ size: 1.5 }])],
['no config size', manifest([{ size: 1 }], {})],
['a total past the safe integers', manifest([{ size: Number.MAX_SAFE_INTEGER }, { size: 10 }])],
])('refuses %s', (_label, value) => {
expect(() => pullTotalBytes(value)).toThrow(CheckFailure);
});
});
describe('selectMacChild', () => {
const child = (runner: string, digest: string, mediaType?: string) => ({
...(mediaType === undefined ? { mediaType: MEDIA_TYPE_MANIFEST } : { mediaType }),
digest,
runner,
});
it('prefers mlx, then llamacpp, then ggml, whatever the list order', () => {
expect(selectMacChild([child('ggml', 'g'), child('llamacpp', 'l'), child('mlx', 'm')]).digest).toBe('m');
expect(selectMacChild([child('ggml', 'g'), child('llamacpp', 'l')]).digest).toBe('l');
expect(selectMacChild([child('ggml', 'g')]).digest).toBe('g');
});
it('takes the first child of a runner and compares runners as Ollama does', () => {
expect(selectMacChild([child(' MLX ', 'first'), child('mlx', 'second')]).digest).toBe('first');
});
it('skips a nested list and a runner a Mac does not run', () => {
expect(selectMacChild([child('mlx', 'nested', MEDIA_TYPE_MANIFEST_LIST), child('llamacpp', 'l')]).digest).toBe('l');
expect(selectMacChild([child('vllm', 'v')])).toBeUndefined();
expect(selectMacChild('not a list')).toBeUndefined();
});
it('accepts a child without a mediaType', () => {
expect(selectMacChild([{ digest: 'm', runner: 'mlx' }]).digest).toBe('m');
});
});
describe('small helpers', () => {
it('metadataForConfig mirrors Ollama: safetensors runs on mlx, gguf on ggml', () => {
expect(metadataForConfig({ model_format: 'safetensors' })).toEqual({ runner: 'mlx', format: 'safetensors' });
expect(metadataForConfig({ model_format: 'gguf' })).toEqual({ runner: 'ggml', format: 'gguf' });
expect(metadataForConfig({})).toEqual({ runner: '', format: '' });
});
it('catalogueRunner names a build in the catalogue words: mlx, or llamacpp for any GGUF build', () => {
expect(catalogueRunner('mlx')).toBe('mlx');
expect(catalogueRunner(' MLX ')).toBe('mlx');
expect(catalogueRunner('llamacpp')).toBe('llamacpp');
// A GGUF manifest that names no runner is `ggml` to Ollama, and llama-server runs it all the same.
expect(catalogueRunner('ggml')).toBe('llamacpp');
expect(catalogueRunner(metadataForConfig({ model_format: 'gguf' }).runner)).toBe('llamacpp');
for (const other of ['', 'vllm', '<img>', undefined, 3]) expect(catalogueRunner(other)).toBeUndefined();
});
it('normalizeDigest accepts the prefixed and the bare form and nothing else', () => {
const bare = '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba';
expect(normalizeDigest(bare)).toBe(REAL_MLX_CHILD);
expect(normalizeDigest(REAL_MLX_CHILD.toUpperCase().replace('SHA256', 'sha256'))).toBe(REAL_MLX_CHILD);
expect(normalizeDigest('sha256:abc')).toBeUndefined();
expect(normalizeDigest('sha512:' + 'a'.repeat(64))).toBeUndefined();
expect(normalizeDigest(42)).toBeUndefined();
});
it('parses and compares versions', () => {
expect(parseVersion('v0.40.15')).toEqual([0, 40, 15]);
expect(parseVersion('0.32.12')).toEqual([0, 32, 12]);
expect(parseVersion('0.40.15-rc1')).toBeUndefined();
expect(compareVersions('0.40.15', 'v0.40.15')).toBe(0);
expect(compareVersions('0.41.0', '0.40.15')).toBeGreaterThan(0);
expect(compareVersions('0.9.0', '0.40.15')).toBeLessThan(0);
expect(() => compareVersions('latest', '0.40.15')).toThrow(CheckFailure);
});
it('formats numbers without the locale', () => {
expect(groupDigits(23_105_798_956)).toBe('23,105,798,956');
expect(groupDigits(250)).toBe('250');
expect(formatPercent(0.16614)).toBe('+16.61%');
expect(formatPercent(-0.02097)).toBe('-2.10%');
});
it('the tolerance is a small fraction of the row', () => {
expect(SIZE_TOLERANCE_FRACTION).toBeLessThanOrEqual(0.01);
expect(withinTolerance(1_000_000, 1_001_000)).toBe(true);
expect(withinTolerance(1_000_000, 1_001_001)).toBe(false);
});
});
describe('classifyModel', () => {
const observed = (bytes: number, extra: Record<string, unknown> = {}) => ({
mac: { bytes, digest: REAL_MLX_CHILD, runner: 'mlx', format: 'safetensors', ...extra },
});
it('a row that holds the pull total itself is clean', () => {
expect(classifyModel({ id: 'qwen3.6:35b-a3b', bytes: MAC_PULL_TOTAL_35B }, observed(MAC_PULL_TOTAL_35B), '0.40.15')).toEqual([]);
});
it('a row sized from its layers alone is clean: the config is a few hundred bytes', () => {
expect(classifyModel({ id: 'gpt-oss:20b', bytes: 13_793_441_427 }, observed(13_793_441_566), '0.40.15')).toEqual([]);
});
it('flags the two real drifts of 2026-11-08: the Mac gets the mlx build', () => {
const nine = classifyModel({ id: 'qwen3.5:9b', bytes: 6_550_825_373 }, observed(7_639_229_431), '0.40.15');
expect(nine).toEqual([expect.objectContaining({ kind: 'size', expected: 6_550_825_373, actual: 7_639_229_431 })]);
expect(nine[0].fraction).toBeCloseTo(0.1661, 3);
const dense = classifyModel({ id: 'qwen3.8:27b', bytes: 17_741_871_939 }, observed(18_174_721_847), '0.40.15');
expect(dense[0].kind).toBe('size');
expect(dense[0].fraction).toBeCloseTo(0.0244, 3);
});
it('a tag that is gone is only that', () => {
expect(classifyModel({ id: 'x:y', bytes: 1 }, { gone: true }, '0.40.15')).toEqual([{ kind: 'gone' }]);
});
it('compares a pinned digest in either form', () => {
const bare = REAL_MLX_CHILD.slice('sha256:'.length);
expect(classifyModel({ id: 'a:b', bytes: 100, digest: bare }, observed(100), '0.40.15')).toEqual([]);
expect(classifyModel({ id: 'a:b', bytes: 100, digest: hex('1') }, observed(100), '0.40.15')).toEqual([
{ kind: 'digest', expected: hex('1'), actual: REAL_MLX_CHILD },
]);
expect(classifyModel({ id: 'a:b', bytes: 100, digest: 'garbage' }, observed(100), '0.40.15')[0]).toMatchObject({
kind: 'digest',
expected: 'not a sha256 digest',
});
});
it('holds a row that names its runner to the runner of the build a Mac pulls', () => {
// The same size, another runner: the row's memory figures no longer describe what runs.
expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'llamacpp' }, observed(100), '0.40.15')).toEqual([
{ kind: 'runner', expected: 'llamacpp', actual: 'mlx' },
]);
expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'mlx' }, observed(100), '0.40.15')).toEqual([]);
// A GGUF build is a llamacpp row's, whether its manifest says llamacpp or nothing (ggml).
for (const runner of ['llamacpp', 'ggml']) {
const gguf = observed(100, { runner, format: 'gguf' });
expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'llamacpp' }, gguf, '0.40.15'), runner).toEqual([]);
expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'mlx' }, gguf, '0.40.15'), runner).toEqual([
{ kind: 'runner', expected: 'mlx', actual: 'llamacpp' },
]);
}
// A runner Ollama would not run on a Mac is never taken for the row's.
expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'mlx' }, observed(100, { runner: '' }), '0.40.15')).toEqual([
{ kind: 'runner', expected: 'mlx', actual: 'unknown' },
]);
// A catalogue without the field (before rows named their runner) is never held to one.
expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { runner: 'llamacpp' }), '0.40.15')).toEqual([]);
});
it('flags a build that needs a newer runtime than the pin, and only that', () => {
expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { requires: '0.41.0' }), '0.40.15')).toEqual([
{ kind: 'requires', requires: '0.41.0', pinned: '0.40.15' },
]);
expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { requires: '0.40.15' }), '0.40.15')).toEqual([]);
expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100), '0.40.15')).toEqual([]);
});
});
describe('classifyRuntime', () => {
const release = (version: string, asset?: { size: number; digest?: string }) => ({ version, publishedOn: '2026-11-03', asset });
const matching = { size: PIN.bytes, digest: `sha256:${PIN.sha256}` };
it('is clean when the pin is the newest release and its archive is unchanged', () => {
expect(classifyRuntime(PIN, { latest: release('0.40.15', matching), pinned: release('0.40.15', matching) })).toEqual([]);
});
it('a size that matches and no digest in the API is not drift', () => {
expect(
classifyRuntime(PIN, { latest: release('0.40.15'), pinned: release('0.40.15', { size: PIN.bytes }) })
).toEqual([]);
});
it('reports a newer release', () => {
expect(classifyRuntime(PIN, { latest: release('0.41.2'), pinned: release('0.40.15', matching) })).toEqual([
{ kind: 'behind', pinned: '0.40.15', latest: '0.41.2' },
]);
});
it('reports a pinned release or archive that is gone or changed', () => {
const latest = release('0.40.15');
expect(classifyRuntime(PIN, { latest, pinned: { gone: true } })).toEqual([{ kind: 'pinned-release-gone' }]);
expect(classifyRuntime(PIN, { latest, pinned: release('0.40.15') })).toEqual([{ kind: 'pinned-asset-gone' }]);
expect(
classifyRuntime(PIN, { latest, pinned: release('0.40.15', { size: PIN.bytes + 1, digest: matching.digest }) })[0].kind
).toBe('pinned-asset-changed');
expect(
classifyRuntime(PIN, { latest, pinned: release('0.40.15', { size: PIN.bytes, digest: hex('2') }) })[0].kind
).toBe('pinned-asset-changed');
});
});
// ---------- Exit code and report ----------
function baseResult(overrides: Record<string, unknown> = {}) {
const mac = { bytes: 100, digest: hex('a'), runner: 'mlx', format: 'safetensors', requires: '0.32.4' };
return {
checkedAt: '2026-11-09 06:00 UTC',
catalogue: { updated: '2026-11-06' },
pin: { version: '0.40.15' },
errors: [] as string[],
models: [{ id: 'a:b', bytes: 100, findings: [] as unknown[], observed: { tagDigest: hex('b'), variants: [mac], mac } }],
runtime: {
assetName: 'ollama-darwin.tgz',
findings: [] as unknown[],
observed: {
latest: { version: '0.40.15', publishedOn: '2026-11-03', asset: { size: PIN.bytes, digest: `sha256:${PIN.sha256}` } },
pinned: { version: '0.40.15', publishedOn: '2026-11-03', asset: { size: PIN.bytes, digest: `sha256:${PIN.sha256}` } },
},
} as Record<string, unknown>,
...overrides,
};
}
describe('exitCodeFor', () => {
it('is 0 when nothing drifted', () => {
expect(exitCodeFor(baseResult())).toBe(0);
});
it('is 1 for drift in a model or in the runtime', () => {
const result = baseResult();
result.models[0].findings = [{ kind: 'gone' }];
expect(exitCodeFor(result)).toBe(1);
const runtime = baseResult();
(runtime.runtime.findings as unknown[]).push({ kind: 'behind', pinned: '0.40.15', latest: '0.41.0' });
expect(exitCodeFor(runtime)).toBe(1);
});
it('is 2 when any part failed, even with drift elsewhere', () => {
const result = baseResult();
result.models[0].findings = [{ kind: 'gone' }];
result.models.push({ id: 'c:d', bytes: 1, findings: [], error: 'timed out' } as never);
expect(exitCodeFor(result)).toBe(2);
expect(exitCodeFor(baseResult({ runtime: { error: 'GitHub answered 403', findings: [] } }))).toBe(2);
expect(exitCodeFor(baseResult({ errors: ['the catalogue did not load'] }))).toBe(2);
});
});
describe('renderReport', () => {
it('says clean, and lists the digests to pin in full with the build a Mac pulls', () => {
const report = renderReport(baseResult());
expect(report).toMatch(/^# Local model catalogue: weekly check\n/);
expect(report).toContain('**Clean**');
expect(report).toContain(`| \`a:b\` | mlx, safetensors | \`${hex('b')}\` | \`${hex('a')}\` |`);
expect(report).toContain('as the row field `digest`');
expect(report).not.toContain('Pin first');
expect(report).toContain('## Not checked');
});
it('asks to pin a row with measured figures first', () => {
const result = baseResult();
(result.models[0] as Record<string, unknown>).figures = 'measured';
expect(renderReport(result)).toContain('Pin first: `a:b`.');
});
it('names a size drift with both figures and every build the tag offers', () => {
const gguf = { bytes: 90, digest: hex('c'), runner: 'llamacpp', format: 'gguf' };
const mac = { bytes: 7_639_229_431, digest: hex('a'), runner: 'mlx', format: 'safetensors' };
const result = baseResult({
models: [
{
id: 'qwen3.5:9b',
bytes: 6_550_825_373,
findings: classifyModel({ id: 'qwen3.5:9b', bytes: 6_550_825_373 }, { mac }, '0.40.15'),
observed: { tagDigest: hex('b'), variants: [gguf, mac], mac },
},
],
});
const report = renderReport(result);
expect(report).toContain('**Drift found**: 1 of 1 models.');
expect(report).toContain('| `qwen3.5:9b` | 6,550,825,373 | 7,639,229,431 | +16.61% | mlx, safetensors |');
expect(report).toContain('mlx, safetensors 7,639,229,431 (a Mac takes this one)');
expect(report).toContain('llamacpp, gguf 90');
});
it('names a runner change, what it means for the figures, and every build the tag offers', () => {
const gguf = { bytes: 100, digest: hex('c'), runner: 'llamacpp', format: 'gguf' };
const mac = { bytes: 100, digest: hex('a'), runner: 'mlx', format: 'safetensors' };
const result = baseResult({
models: [
{
id: 'qwen3.6:35b-a3b-mtp-q4_k_m',
bytes: 100,
findings: classifyModel({ id: 'x:y', bytes: 100, runner: 'llamacpp' }, { mac }, '0.40.15'),
observed: { tagDigest: hex('b'), variants: [gguf, mac], mac },
},
],
});
const report = renderReport(result);
expect(report).toContain('**Drift found**: 1 of 1 models.');
expect(report).toContain('| runner changed |');
expect(report).toContain(
"`qwen3.6:35b-a3b-mtp-q4_k_m`: a Mac now pulls the mlx, safetensors build, which runs on `mlx`, while the row's `runner` says `llamacpp`."
);
expect(report).toContain('recompute them for the new runner');
expect(report).toContain('mlx, safetensors 100 (a Mac takes this one)');
});
it('says the check failed and prints no verdict of clean', () => {
const result = baseResult({ runtime: { error: 'GitHub answered 403 for the newest release', findings: [] } });
const report = renderReport(result);
expect(report).toContain('**The check failed**');
expect(report).not.toContain('**Clean**');
expect(report).toContain('GitHub answered 403');
});
it('never prints a runner or format outside the known names', () => {
const mac = { bytes: 100, digest: hex('a'), runner: '<img src=x>', format: '@someone' };
const report = renderReport(baseResult({ models: [{ id: 'a:b', bytes: 100, findings: [], observed: { tagDigest: hex('b'), variants: [mac], mac } }] }));
expect(report).toContain('unknown, unknown');
expect(report).not.toContain('<img');
expect(report).not.toContain('@someone');
});
it('keeps an error on one line and without backticks', () => {
const result = baseResult({ errors: ['line one\nline `two`\u0007'] });
expect(renderReport(result)).toContain("- Check failed: line one line 'two'\n");
});
it('contains no em-dash', () => {
expect(renderReport(baseResult())).not.toContain(String.fromCharCode(0x2014));
});
});
// ---------- IO against a fake registry ----------
interface Seen {
url: string;
headers: Record<string, string>;
signal: unknown;
}
/**
* A fake registry and GitHub API. `routes` maps a URL to a body (string or Buffer) or a status
* number. Every request is recorded.
*/
function fakeFetch(routes: Record<string, string | Buffer | number | (() => Response)>) {
const seen: Seen[] = [];
const fetchFn = async (url: string, init: { headers: Record<string, string>; signal: unknown }) => {
seen.push({ url, headers: init.headers, signal: init.signal });
const route = routes[url];
if (route === undefined) return new Response('not found', { status: 404 });
if (typeof route === 'number') return new Response('', { status: route });
if (typeof route === 'function') return route();
return new Response(route, { status: 200 });
};
return { fetchFn, seen };
}
const REG = 'https://registry.ollama.ai/v2/library';
const GH = 'https://api.github.com/repos/ollama/ollama/releases';
/** A small registry: one tag that is a list (mlx + llamacpp), one plain tag. Digests computed. */
function buildRegistry() {
const json = (value: unknown): Buffer => Buffer.from(JSON.stringify(value));
const mlxConfig = json({ model_format: 'safetensors', requires: '0.32.4' });
const ggufConfig = json({ model_format: 'gguf', requires: '0.30.0' });
const layers = (sizes: number[], tag: string) =>
sizes.map((size, i) => ({ mediaType: 'application/vnd.ollama.image.tensor', digest: hex(String((i + tag.length) % 10)), size }));
const mlxChild = json({
schemaVersion: 2,
mediaType: MEDIA_TYPE_MANIFEST,
config: { digest: sha256Digest(mlxConfig), size: mlxConfig.length },
layers: layers([700, 300], 'mlx'),
});
const ggufChild = json({
schemaVersion: 2,
mediaType: MEDIA_TYPE_MANIFEST,
config: { digest: sha256Digest(ggufConfig), size: ggufConfig.length },
layers: layers([600], 'gguf'),
runner: 'llamacpp',
format: 'gguf',
});
const list = json({
schemaVersion: 2,
mediaType: MEDIA_TYPE_MANIFEST_LIST,
manifests: [
{ mediaType: MEDIA_TYPE_MANIFEST, digest: sha256Digest(mlxChild), runner: 'mlx', format: 'safetensors' },
{ mediaType: MEDIA_TYPE_MANIFEST, digest: sha256Digest(ggufChild), runner: 'llamacpp', format: 'gguf' },
],
});
const plain = json({
schemaVersion: 2,
mediaType: MEDIA_TYPE_MANIFEST,
config: { digest: sha256Digest(ggufConfig), size: ggufConfig.length },
layers: layers([500], 'plain'),
});
const release = (tag: string) =>
json({
tag_name: tag,
published_at: '2026-11-03T22:52:31Z',
assets: [{ name: 'ollama-darwin.tgz', size: PIN.bytes, digest: `sha256:${PIN.sha256}` }],
});
const routes: Record<string, string | Buffer | number | (() => Response)> = {
[`${REG}/listed/manifests/1b`]: list,
[`${REG}/listed/blobs/${sha256Digest(mlxChild)}`]: mlxChild,
[`${REG}/listed/blobs/${sha256Digest(ggufChild)}`]: ggufChild,
[`${REG}/listed/blobs/${sha256Digest(mlxConfig)}`]: mlxConfig,
[`${REG}/plain/manifests/2b`]: plain,
[`${REG}/plain/blobs/${sha256Digest(ggufConfig)}`]: ggufConfig,
[`${GH}/latest`]: release('v0.40.15'),
[`${GH}/tags/v0.40.15`]: release('v0.40.15'),
};
const layerDigests = new Set(
[mlxChild, ggufChild, plain].flatMap((body) => JSON.parse(body.toString()).layers.map((l: { digest: string }) => l.digest))
);
return {
routes,
layerDigests,
sizes: { mlx: 1000 + mlxConfig.length, gguf: 600 + ggufConfig.length, plain: 500 + ggufConfig.length },
digests: { list: sha256Digest(list), mlxChild: sha256Digest(mlxChild), plain: sha256Digest(plain) },
};
}
const catalogOf = (models: { id: string; bytes: number; digest?: string; figures?: string; runner?: string }[]) => ({
updated: '2026-11-06',
models,
});
const NOW = new Date('2026-11-09T06:00:00Z');
describe('inspectModel', () => {
it('reads a list the way a Mac does: the mlx child, its total, its requires', async () => {
const registry = buildRegistry();
const { fetchFn, seen } = fakeFetch(registry.routes);
const observed = await inspectModel(fetchFn, 'listed:1b');
expect(observed.tagDigest).toBe(registry.digests.list);
expect(observed.mac).toMatchObject({
runner: 'mlx',
format: 'safetensors',
digest: registry.digests.mlxChild,
bytes: registry.sizes.mlx,
requires: '0.32.4',
});
expect(observed.variants.map((variant: { runner: string }) => variant.runner)).toEqual(['mlx', 'llamacpp']);
expect(seen[0].headers.Accept).toBe(MANIFEST_ACCEPT);
expect(seen[0].headers.Accept).toContain(MEDIA_TYPE_MANIFEST_LIST);
expect(seen.every((request) => request.signal instanceof AbortSignal)).toBe(true);
});
it('fills a plain manifest without runner metadata from its config', async () => {
const registry = buildRegistry();
const { fetchFn } = fakeFetch(registry.routes);
const observed = await inspectModel(fetchFn, 'plain:2b');
expect(observed.mac).toMatchObject({ runner: 'ggml', format: 'gguf', digest: registry.digests.plain, bytes: registry.sizes.plain });
expect(observed.tagDigest).toBe(observed.mac.digest);
});
it('a 404 for the manifest is a tag that is gone', async () => {
const { fetchFn } = fakeFetch({});
expect(await inspectModel(fetchFn, 'missing:tag')).toEqual({ gone: true });
});
it('refuses a blob whose body does not match its digest', async () => {
const registry = buildRegistry();
const mlxUrl = `${REG}/listed/blobs/${registry.digests.mlxChild}`;
const { fetchFn } = fakeFetch({ ...registry.routes, [mlxUrl]: '{"tampered":true}' });
await expect(inspectModel(fetchFn, 'listed:1b')).rejects.toThrow(/does not match its digest/);
});
it('refuses a list with no build a Mac runs', async () => {
const list = JSON.stringify({ mediaType: MEDIA_TYPE_MANIFEST_LIST, manifests: [{ digest: hex('1'), runner: 'vllm' }] });
const { fetchFn } = fakeFetch({ [`${REG}/odd/manifests/x`]: list });
await expect(inspectModel(fetchFn, 'odd:x')).rejects.toThrow(/no build a Mac runs/);
});
it('a server error is a failed check, not a tag that is gone', async () => {
const { fetchFn } = fakeFetch({ [`${REG}/flaky/manifests/x`]: 503 });
await expect(inspectModel(fetchFn, 'flaky:x')).rejects.toThrow(/answered 503/);
});
});
describe('boundedGet', () => {
it('refuses a declared body over the cap before reading it', async () => {
const { fetchFn } = fakeFetch({
'https://x.test/big': () => new Response('{}', { status: 200, headers: { 'content-length': String(MAX_MANIFEST_BYTES + 1) } }),
});
await expect(boundedGet(fetchFn, 'https://x.test/big', { maxBytes: MAX_MANIFEST_BYTES })).rejects.toThrow(/over the cap/);
});
it('refuses a streamed body that grows past the cap', async () => {
const stream = new ReadableStream({
start(controller) {
controller.enqueue(new Uint8Array(600));
controller.enqueue(new Uint8Array(600));
controller.close();
},
});
const { fetchFn } = fakeFetch({ 'https://x.test/stream': () => new Response(stream, { status: 200 }) });
await expect(boundedGet(fetchFn, 'https://x.test/stream', { maxBytes: 1000 })).rejects.toThrow(/over the cap/);
});
it('turns a network error into a CheckFailure', async () => {
const fetchFn = async () => {
throw new TypeError('fetch failed', { cause: new Error('getaddrinfo ENOTFOUND registry.ollama.ai') });
};
await expect(boundedGet(fetchFn, 'https://registry.ollama.ai/v2/', { maxBytes: 10 })).rejects.toThrow(/ENOTFOUND/);
});
});
describe('runCheck', () => {
it('reports drift, and fetches only manifests, child manifests, configs and the two releases', async () => {
const registry = buildRegistry();
const { fetchFn, seen } = fakeFetch(registry.routes);
const result = await runCheck({
catalog: catalogOf([
// Sized from the llamacpp child: a Mac pulls the mlx one, so this is drift.
{ id: 'listed:1b', bytes: registry.sizes.gguf },
{ id: 'plain:2b', bytes: registry.sizes.plain },
{ id: 'missing:tag', bytes: 1000 },
]),
pin: PIN,
fetchFn,
now: NOW,
githubToken: 'token-value',
});
expect(result.models.map((model: { findings: { kind: string }[] }) => model.findings.map((f) => f.kind))).toEqual([
['size'],
[],
['gone'],
]);
expect(result.runtime.findings).toEqual([]);
expect(exitCodeFor(result)).toBe(1);
for (const request of seen) {
const blob = /\/blobs\/(sha256:[0-9a-f]{64})$/.exec(request.url);
if (blob) expect(registry.layerDigests.has(blob[1])).toBe(false);
const isGitHub = request.url.startsWith('https://api.github.com/');
expect(request.headers.Authorization === 'Bearer token-value').toBe(isGitHub);
}
expect(seen.some((request) => request.url === `${GH}/latest`)).toBe(true);
expect(seen.some((request) => request.url === `${GH}/tags/v0.40.15`)).toBe(true);
const report = renderReport(result);
expect(report).toContain('**Drift found**: 2 of 3 models.');
expect(report).toContain('`missing:tag`: the registry no longer has this tag');
});
it('holds rows that name their runner to the build a Mac pulls, a plain GGUF manifest included', async () => {
const registry = buildRegistry();
const { fetchFn } = fakeFetch(registry.routes);
const result = await runCheck({
catalog: catalogOf([
// The right size, the wrong runner: a Mac pulls the mlx child of this list.
{ id: 'listed:1b', bytes: registry.sizes.mlx, runner: 'llamacpp' },
// A manifest that names no runner is a GGUF build: a llamacpp row, clean.
{ id: 'plain:2b', bytes: registry.sizes.plain, runner: 'llamacpp' },
]),
pin: PIN,
fetchFn,
now: NOW,
});
expect(result.models.map((model: { findings: unknown[] }) => model.findings)).toEqual([
[{ kind: 'runner', expected: 'llamacpp', actual: 'mlx' }],
[],
]);
expect(exitCodeFor(result)).toBe(1);
expect(renderReport(result)).toContain('| runner changed |');
});
it('sends no Authorization header at all without a token', async () => {
const registry = buildRegistry();
const { fetchFn, seen } = fakeFetch(registry.routes);
await runCheck({ catalog: catalogOf([{ id: 'plain:2b', bytes: registry.sizes.plain }]), pin: PIN, fetchFn, now: NOW });
expect(seen.some((request) => 'Authorization' in request.headers)).toBe(false);
});
it('is clean when every row matches what a Mac pulls', async () => {
const registry = buildRegistry();
const { fetchFn } = fakeFetch(registry.routes);
const result = await runCheck({
catalog: catalogOf([
{ id: 'listed:1b', bytes: registry.sizes.mlx, digest: registry.digests.mlxChild, figures: 'measured' },
{ id: 'plain:2b', bytes: registry.sizes.plain, figures: 'measured' },
]),
pin: PIN,
fetchFn,
now: NOW,
});
expect(exitCodeFor(result)).toBe(0);
const report = renderReport(result);
expect(report).toContain('**Clean**');
// The pinned row is not offered for pinning again; the unpinned one is.
expect(report).not.toContain('| `listed:1b` | mlx');
expect(report).toContain(`| \`plain:2b\` | ggml, gguf | \`${registry.digests.plain}\``);
// Only the measured row that has no digest yet is asked for first.
expect(report).toContain('Pin first: `plain:2b`.');
});
it('a newer runtime release is drift', async () => {
const registry = buildRegistry();
registry.routes[`${GH}/latest`] = JSON.stringify({ tag_name: 'v0.41.0', published_at: '2026-11-10T00:00:00Z', assets: [] });
const { fetchFn } = fakeFetch(registry.routes);
const result = await runCheck({ catalog: catalogOf([{ id: 'plain:2b', bytes: registry.sizes.plain }]), pin: PIN, fetchFn, now: NOW });
expect(result.runtime.findings).toEqual([{ kind: 'behind', pinned: '0.40.15', latest: '0.41.0' }]);
expect(exitCodeFor(result)).toBe(1);
expect(renderReport(result)).toContain('Ollama 0.41.0 is out; the pin is 0.40.15.');
});
it('a rate-limited GitHub API fails the check (2) and keeps what the models showed', async () => {
const registry = buildRegistry();
registry.routes[`${GH}/latest`] = 403;
const { fetchFn } = fakeFetch(registry.routes);
const result = await runCheck({
catalog: catalogOf([{ id: 'listed:1b', bytes: registry.sizes.gguf }]),
pin: PIN,
fetchFn,
now: NOW,
});
expect(exitCodeFor(result)).toBe(2);
expect(result.models[0].findings[0].kind).toBe('size');
expect(renderReport(result)).toContain('**The check failed**');
});
it('a release tag that is not vX.Y.Z fails the check instead of reaching the report', async () => {
const registry = buildRegistry();
registry.routes[`${GH}/latest`] = JSON.stringify({ tag_name: 'v1.0.0 @everyone', assets: [] });
const { fetchFn } = fakeFetch(registry.routes);
const result = await runCheck({ catalog: catalogOf([]), pin: PIN, fetchFn, now: NOW });
expect(exitCodeFor(result)).toBe(2);
expect(renderReport(result)).not.toContain('@everyone');
});
it('a network failure for one row fails the whole check', async () => {
const registry = buildRegistry();
const fetchFn = async (url: string, init: { headers: Record<string, string>; signal: unknown }) => {
if (url.includes('/plain/')) throw new TypeError('fetch failed');
return fakeFetch(registry.routes).fetchFn(url, init);
};
const result = await runCheck({
catalog: catalogOf([
{ id: 'listed:1b', bytes: registry.sizes.mlx },
{ id: 'plain:2b', bytes: registry.sizes.plain },
]),
pin: PIN,
fetchFn,
now: NOW,
});
expect(result.models[1].error).toMatch(/fetch failed/);
expect(exitCodeFor(result)).toBe(2);
});
});
test/fixtures/local-model-catalog/qwen3.6-35b-a3b.gguf-manifest.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:99afa5bbf1e7038bda7d028e100ee76b7337c943468d52590de3282db7b52adb","size":220},"layers":[{"mediaType":"application/vnd.ollama.image.model","digest":"sha256:d372de8e934898a59e6ccfabc3368474711384d8f1fd4d22d87a3f0a45400cdc","size":21718480960,"from":"qwen3.6-source:35b-a3b-mtp-q4_K_M-20260824"},{"mediaType":"application/vnd.ollama.image.projector","digest":"sha256:a62390d25b4b4a2d8afd7cc3c90021c11935c1fdd48d7cb0723ab71cd02a598e","size":902821728,"from":"qwen3.6-source:35b-a3b-mtp-q4_K_M-20260824"},{"mediaType":"application/vnd.ollama.image.license","digest":"sha256:5f3a3c817e78f5b8a4ad2d2c458a3e4b2cce470d6c12642c4eeb12cb8a9bf51d","size":11357,"from":"qwen3.6-source:35b-a3b-mtp-q4_K_M-20260824"},{"mediaType":"application/vnd.ollama.image.params","digest":"sha256:ce44c0f3ca1f7a126724e5285be7e1afdab202186c6379515ab0c1ffd40d3ecc","size":116}],"runner":"llamacpp","format":"gguf"}
test/fixtures/local-model-catalog/qwen3.6-35b-a3b.list.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"schemaVersion":2,"mediaType":"application/vnd.ollama.manifest.list.v2+json","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba","runner":"mlx","format":"safetensors"},{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:a7eb95c53bcf96b4bdd008d0fab4a5dac88047d9c1a7a9ab88ed453423fbd87c","runner":"llamacpp","format":"gguf"}]}
test/fixtures/local-model-catalog/qwen3.6-35b-a3b.mlx-config.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"architecture":"","capabilities":["completion","vision","tools","thinking"],"file_type":"nvfp4","model_families":null,"model_family":"","model_format":"safetensors","model_type":"","os":"","parser":"qwen3.5","renderer":"qwen3.5","requires":"0.32.4"}
test/fixtures/local-model-rules-catalog.ts
Vendored
+182
View File
@@ -0,0 +1,182 @@
/**
* @fileoverview A frozen model catalogue for the RULE tests of src/local-model/recommend.ts:
* the stock catalogue exactly as it shipped on 2026-11-06 (seven rows that all carry the
* figures measured on one model), with the `runner` field the schema gained since.
*
* Why a frozen copy and not the stock catalogue: the stock catalogue is meant to change every
* week or so (new models, new measurements). The tests of a RULE (the ranking, the disk
* filter, the tags, the `limited` line, the harness fallback) need realistic rows whose
* answers do not move with it, or every catalogue update would rewrite a hundred rule tests
* and hide the few answers that really changed. What the STOCK catalogue answers for each Mac
* is pinned separately, in test/local-model-recommend.test.ts ("every memory size") and the
* two real-Mac blocks, and that is the table a catalogue change must update on purpose.
*
* Its figures are NOT a statement about any model: the rows borrow the measured row's figures,
* which is exactly why the stock catalogue replaced them. Validated by the real schema.
*/
import type { LocalModelCatalog } from '../../src/types.js';
const MEASURED_MEMORY = {
/** 36,000 to 38,000 bytes stayed held per token of context after a prompt of 89,000 and one of 167,000 tokens. */
heldBytesPerToken: 37_000,
/** The runner held 2.5 to 3 GB beside the weights. */
overheadBytes: 3_000_000_000,
/** While a long prompt was processed the peak stood about 5 GB above what stayed held. */
peakExtraBytes: 5_000_000_000,
} as const;
/**
* The catalogue as it shipped on 2026-11-06, with the `runner` field added. Frozen: a fixture,
* not the stock catalogue.
*/
export const RULES_LOCAL_MODEL_CATALOG: LocalModelCatalog = {
schema: 1,
updated: '2026-11-06',
// The oldest macOS the model runtime supports.
minMacosMajor: 14,
// Ordered by rank. The higher rank wins among models that fit equally well and are equally
// fast. `flint:40b` is left out on purpose: its plain tag is the slow build on a Mac and it
// has an open bug with tool arguments.
models: [
{
id: 'qwen3.5:9b',
label: 'Qwen3.5 9B',
license: 'Apache-2.0',
paramsB: 9,
runner: 'mlx',
bytes: 6_550_825_373,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [65536, 32768],
rank: 1,
// 37.1 tokens per second measured on 400 GB/s.
speedPer100Gbps: 9.3,
tested: [],
},
{
id: 'qwen3.5:9b-q8_0',
label: 'Qwen3.5 9B (Q8)',
license: 'Apache-2.0',
paramsB: 9,
runner: 'llamacpp',
bytes: 10_449_217_949,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [65536, 32768],
rank: 2,
// Not measured: the 9.3 of the row above scaled by file size.
speedPer100Gbps: 5.8,
tested: [],
},
{
id: 'gpt-oss:20b',
label: 'GPT-OSS 20B',
license: 'Apache-2.0',
paramsB: 20,
runner: 'llamacpp',
bytes: 13_793_441_427,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [131072, 65536],
rank: 3,
// No speed figure: its speed is unknown on every chip.
tested: [],
},
{
id: 'qwen3.6:35b-a3b',
label: 'Qwen3.6 35B-A3B',
license: 'Apache-2.0',
paramsB: 35,
runner: 'mlx',
// What a Mac really downloads: an MLX build, 2% above the 22,621,314,161 of the manifest.
bytes: 23_105_798_956,
...MEASURED_MEMORY,
figures: 'measured',
contexts: [131072, 65536],
rank: 4,
// 54.5 tokens per second measured on 400 GB/s (3 billion parameters are active per token).
// The MLX build ran faster than this figure predicts on the one Mac we tested (67.7
// against 41.8), so on other chips the estimate is on the low side.
speedPer100Gbps: 13.6,
// Run by us with OpenCode on the MacBook Pro (Mac17,13): a four-step task with tool calls.
tested: [{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 }],
},
{
id: 'qwen3.8:27b',
label: 'Qwen3.8 27B',
license: 'Apache-2.0',
paramsB: 27.8,
runner: 'mlx',
bytes: 17_741_871_939,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [131072, 65536, 32768],
rank: 5,
// 11.8 tokens per second measured on 400 GB/s.
speedPer100Gbps: 2.95,
tested: [],
},
{
id: 'qwen3.8:27b-q8_0',
label: 'Qwen3.8 27B (Q8)',
license: 'Apache-2.0',
paramsB: 27.8,
runner: 'llamacpp',
bytes: 29_978_241_837,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [262144, 131072],
rank: 6,
// Not measured: the 2.95 of the row above scaled by file size.
speedPer100Gbps: 1.7,
tested: [],
},
{
id: 'qwen3.5:122b-a10b',
label: 'Qwen3.5 122B-A10B',
license: 'Apache-2.0',
paramsB: 122,
runner: 'llamacpp',
bytes: 81_370_035_877,
...MEASURED_MEMORY,
figures: 'estimated',
contexts: [131072],
rank: 7,
// No speed figure: its speed is unknown on every chip.
tested: [],
},
],
// The higher rank wins when a model can run with more than one harness.
harnesses: [
{ cli: 'opencode', minContext: 65536, rank: 3 },
// Sends a fixed prompt of about 36,000 tokens, so the first answer takes minutes.
{ cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true },
// Gated: neither its 32768 minimum nor its handling of a `name:tag` model id is verified.
{ cli: 'pi', minContext: 32768, rank: 1, gated: true },
],
// Memory bandwidth in GB/s, most specific first. Only chips that shipped and are in the source
// table are listed. Where one name covers two bandwidths (M3 Max 300 or 400, M4 Max 410 or 546,
// M5 Max 460 or 614) the LOWER one stands, so a speed computed from it is a floor.
chips: [
{ match: 'M5 Ultra', gbps: 1228 },
{ match: 'M3 Ultra', gbps: 800 },
{ match: 'M2 Ultra', gbps: 800 },
{ match: 'M1 Ultra', gbps: 800 },
{ match: 'M5 Max', gbps: 460 },
{ match: 'M4 Max', gbps: 410 },
{ match: 'M3 Max', gbps: 300 },
{ match: 'M2 Max', gbps: 400 },
{ match: 'M1 Max', gbps: 400 },
{ match: 'M5 Pro', gbps: 307 },
{ match: 'M4 Pro', gbps: 273 },
{ match: 'M3 Pro', gbps: 150 },
{ match: 'M2 Pro', gbps: 200 },
{ match: 'M1 Pro', gbps: 200 },
{ match: 'M5', gbps: 154 },
{ match: 'M4', gbps: 120 },
{ match: 'M3', gbps: 100 },
{ match: 'M2', gbps: 100 },
{ match: 'M1', gbps: 68 },
],
};
test/local-model-catalog.test.ts
+305 -63
View File
@@ -12,9 +12,12 @@
* hides a more specific one).
* 3. The chip lookup never guesses: a chip that no row names has no bandwidth, including one
* whose name merely contains a known name ('Apple M10', 'Apple M6 Pro').
* 4. Which figures were MEASURED on a Mac and which are carried over from the measured row:
* one row says `measured` (its size is what a Mac really downloads, and it alone has a
* tested entry), every other row says `estimated`.
* 4. Which figures were MEASURED on a Mac and which were computed: one row says `measured`
* (it alone has a tested entry), every other row says `estimated`, and every estimated
* row's three memory figures are recomputed here from its OWN model's architecture (a
* table read from each model's config.json) for its runner and the runtime's settings. Two
* rows may share figures only when they share the architecture and the runner, so a row
* can never again carry another model's measurement without saying so.
*
* Port: N/A (pure data and functions).
*/
@@ -28,6 +31,12 @@ import {
LocalModelCatalogSchema,
STOCK_LOCAL_MODEL_CATALOG,
} from '../src/config/local-model-catalog.js';
import {
RUNTIME_CTX_CHECKPOINTS,
RUNTIME_HOST_PROMPT_CACHE_MIB,
RUNTIME_KV_CACHE_TYPE,
runtimeEnv,
} from '../src/local-model/runtime.js';
import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js';
/** A deep copy of the stock catalogue that a test may break. */
@@ -48,7 +57,121 @@ function withFirstModel(change: Partial<Record<keyof LocalModelEntry | 'url' | '
return { ...catalog, models: [{ ...catalog.models[0], ...change }, ...catalog.models.slice(1)] };
}
/** One MiB and one GiB, in bytes. */
const MIB = 1024 * 1024;
const GIB = 1024 * MIB;
/**
* The architecture of every model the stock catalogue lists, read from its config.json on
* Hugging Face (2026-11-08): full-attention layers, their K/V heads and head size, and the
* layers whose cache does not grow with the context (Gated DeltaNet layers with their value
* and key heads, or sliding-window layers with their window). Fine-tunes share their base's.
*/
const ARCHITECTURES = {
'Qwen3.5-9B': { attentionLayers: 8, kvHeads: 4, headDim: 256, linear: { layers: 24, vHeads: 32, kHeads: 16 } },
'Qwen3.8-27B': { attentionLayers: 16, kvHeads: 4, headDim: 256, linear: { layers: 48, vHeads: 48, kHeads: 16 } },
'Qwen3.6-35B-A3B': { attentionLayers: 10, kvHeads: 2, headDim: 256, linear: { layers: 30, vHeads: 32, kHeads: 16 } },
'Qwen3.5-122B-A10B': {
attentionLayers: 12,
kvHeads: 2,
headDim: 256,
linear: { layers: 36, vHeads: 64, kHeads: 16 },
},
'North-Mini-Code-1.0': { attentionLayers: 13, kvHeads: 4, headDim: 128, sliding: { layers: 36, window: 4096 } },
} as const;
type ArchitectureName = keyof typeof ARCHITECTURES;
type Architecture = (typeof ARCHITECTURES)[ArchitectureName];
/**
* Which architecture each stock row runs, and how many tokens the runtime drafts per step with
* its multi-token-prediction layer: 0 when the GGUF has none (`nextn_predict_layers` in the GGUF
* header, read on 2026-11-08 from the first bytes of each model layer), else the tag's own
* `draft_num_predict` (its params layer) or the runtime's default of 4 when the tag sets none.
* Prediction costs a cache for that layer and one rollback copy of the recurrent state per
* drafted token (see the catalogue header). A row the catalogue gains must be added here, or the
* test below fails: its figures cannot be checked without it.
*/
const ROW_ARCHITECTURE: Record<string, { architecture: ArchitectureName; draftTokens: number }> = {
'qwen3.5:9b-q4_k_m': { architecture: 'Qwen3.5-9B', draftTokens: 0 },
'mimo-v2.6:9b-q4_k_m': { architecture: 'Qwen3.5-9B', draftTokens: 0 },
'north-mini-code-1.0:q4_k_m': { architecture: 'North-Mini-Code-1.0', draftTokens: 0 },
// Its params layer sets `draft_num_predict` 2.
'qwen3.6:35b-a3b-mtp-q4_k_m': { architecture: 'Qwen3.6-35B-A3B', draftTokens: 2 },
'qwen3.6:35b-a3b': { architecture: 'Qwen3.6-35B-A3B', draftTokens: 0 },
// A prediction layer, and no `draft_num_predict`: the runtime's 4.
'qwen3.8:27b-q8_0': { architecture: 'Qwen3.8-27B', draftTokens: 4 },
// Its params layer sets `draft_num_predict` 4.
'qwen3.8:27b-mtp-q4_k_m': { architecture: 'Qwen3.8-27B', draftTokens: 4 },
'ornith-1.5:122b': { architecture: 'Qwen3.5-122B-A10B', draftTokens: 0 },
};
/** Bytes per cached value in each K/V cache type of llama.cpp: q8_0 keeps 32 values in 34 bytes. */
const KV_BYTES_PER_VALUE: Record<string, number> = { f16: 2, q8_0: 34 / 32 };
/**
* The cache type of llama-server's draft context, which holds the prediction layer's cache: its
* default, because the runtime sets only the main context's (`--cache-type-k`).
*/
const DRAFT_KV_CACHE_TYPE = 'f16';
/** The sliding-window cache holds the window plus one batch of 512 tokens (`n_swa + n_ubatch`). */
const SLIDING_EXTRA_CELLS = 512;
/** The head size of a Gated DeltaNet layer's keys and values, and its convolution kernel, in every model above. */
const LINEAR_HEAD_DIM = 128;
const LINEAR_CONV_KERNEL = 4;
/** The recurrent state of the linear-attention layers in bytes: the convolution tail and the delta state. */
function recurrentStateBytes(arch: Architecture, convBytes: number): number {
if (!('linear' in arch)) return 0;
const { layers, vHeads, kHeads } = arch.linear;
const convDim = vHeads * LINEAR_HEAD_DIM + 2 * kHeads * LINEAR_HEAD_DIM;
const conv = (LINEAR_CONV_KERNEL - 1) * convDim * convBytes;
const delta = vHeads * LINEAR_HEAD_DIM * LINEAR_HEAD_DIM * 4;
return layers * (conv + delta);
}
/** What a `llamacpp` row's three figures must be, from its architecture and the runtime's settings. */
function llamaCppFigures(arch: Architecture, draftTokens: number): [number, number, number] {
const predictionLayer = draftTokens > 0;
const perValue = KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE];
const perLayer = arch.kvHeads * arch.headDim * 2;
// The prediction layer's cache sits in the draft context, in the draft's cache type.
const held =
arch.attentionLayers * perLayer * perValue +
(predictionLayer ? perLayer * KV_BYTES_PER_VALUE[DRAFT_KV_CACHE_TYPE] : 0);
// llama.cpp keeps the recurrent state in f32 (both parts), the sliding window in the cache type.
const recurrent = recurrentStateBytes(arch, 4);
const sliding =
'sliding' in arch ? arch.sliding.layers * (arch.sliding.window + SLIDING_EXTRA_CELLS) * perLayer * perValue : 0;
// Live: the sliding window, and the recurrent state with one rollback copy per drafted token
// (`--spec-draft-n-max`, which `need_n_rs_seq()` turns into that many copies).
const live = sliding + recurrent * (1 + draftTokens);
// A context checkpoint is a copy of what cannot be rolled back: the recurrent state or the window.
const checkpoints = RUNTIME_CTX_CHECKPOINTS * (recurrent + sliding);
return [held, live + checkpoints + 1_000_000_000 + RUNTIME_HOST_PROMPT_CACHE_MIB * MIB, 0];
}
/** The per-token figure of the MLX runner computed from an architecture: K/V in bf16, a recurrent snapshot per 8,192 tokens. */
function mlxComputedPerToken(arch: Architecture): number {
return arch.attentionLayers * arch.kvHeads * arch.headDim * 2 * 2 + recurrentStateBytes(arch, 2) / 8192;
}
/** The figures measured for `qwen3.6:35b-a3b` on the MacBook Pro. */
const MEASURED: [number, number, number] = [37_000, 3_000_000_000, 5_000_000_000];
/** What an estimated `mlx` row's three figures must be: computed, scaled by what the measured row showed. */
function mlxFigures(arch: Architecture): [number, number, number] {
const ratio = MEASURED[0] / mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B']);
return [Math.ceil(mlxComputedPerToken(arch) * ratio), MEASURED[1] + 8 * GIB, MEASURED[2]];
}
/** A row's three memory figures. */
function figuresOf(model: LocalModelEntry): [number, number, number] {
return [model.heldBytesPerToken, model.overheadBytes, model.peakExtraBytes];
}
describe('the stock catalogue', () => {
const models = STOCK_LOCAL_MODEL_CATALOG.models;
it('passes its own schema', () => {
expect(issuesOf(STOCK_LOCAL_MODEL_CATALOG)).toEqual([]);
});
@@ -60,94 +183,114 @@ describe('the stock catalogue', () => {
expect(getLocalModelCatalog()).toBe(first);
});
it('lists the seven models of the design, without flint:40b', () => {
expect(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => m.id)).toEqual([
'qwen3.5:9b',
'qwen3.5:9b-q8_0',
'gpt-oss:20b',
it('lists eight models, by rank, and none of those set aside in the header', () => {
expect(models.map((m) => m.id)).toEqual([
'qwen3.5:9b-q4_k_m',
'mimo-v2.6:9b-q4_k_m',
'north-mini-code-1.0:q4_k_m',
'qwen3.6:35b-a3b-mtp-q4_k_m',
'qwen3.6:35b-a3b',
'qwen3.8:27b',
'qwen3.8:27b-q8_0',
'qwen3.5:122b-a10b',
'qwen3.8:27b-mtp-q4_k_m',
'ornith-1.5:122b',
]);
expect(STOCK_LOCAL_MODEL_CATALOG.models.some((m) => m.id.startsWith('flint'))).toBe(false);
expect(models.map((m) => m.rank)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
const setAside =
/^(bonsai-2|gpt-oss|flint|lfm2\.5|laguna|qwen3\.8-flash-next|qwen3\.5:122b)|^ornith-1\.5:(9b|35b|397b)/;
expect(models.filter((m) => setAside.test(m.id)).map((m) => m.id)).toEqual([]);
// The plain 9B and 27B tags are MLX builds on a Mac: their GGUF builds are listed instead.
for (const plain of ['qwen3.5:9b', 'mimo-v2.6:9b', 'qwen3.8:27b']) {
expect(models.some((m) => m.id === plain)).toBe(false);
}
});
it('carries the size a Mac really downloads where that was measured, else the registry manifest', () => {
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.bytes]))).toEqual({
// The sum of the layer sizes of each tag's manifest, as fetched on 2026-11-06.
'qwen3.5:9b': 6_550_825_373,
'qwen3.5:9b-q8_0': 10_449_217_949,
'gpt-oss:20b': 13_793_441_427,
// The total of the real pull on a Mac (an MLX build). Its manifest sums to 22,621,314,161.
it('names the runner a Mac uses for each tag', () => {
expect(Object.fromEntries(models.map((m) => [m.id, m.runner]))).toEqual({
'qwen3.5:9b-q4_k_m': 'llamacpp',
'mimo-v2.6:9b-q4_k_m': 'llamacpp',
'north-mini-code-1.0:q4_k_m': 'llamacpp',
'qwen3.6:35b-a3b-mtp-q4_k_m': 'llamacpp',
// Manifest lists: a Mac takes the MLX child.
'qwen3.6:35b-a3b': 'mlx',
'qwen3.8:27b-q8_0': 'llamacpp',
'qwen3.8:27b-mtp-q4_k_m': 'llamacpp',
'ornith-1.5:122b': 'mlx',
});
});
it('carries the size of the build a Mac pulls: every layer plus the config', () => {
expect(Object.fromEntries(models.map((m) => [m.id, m.bytes]))).toEqual({
// The registry's manifests as fetched on 2026-11-08, in lowercase.
'qwen3.5:9b-q4_k_m': 6_550_825_550,
'mimo-v2.6:9b-q4_k_m': 6_550_811_033,
'north-mini-code-1.0:q4_k_m': 18_593_967_008,
'qwen3.6:35b-a3b-mtp-q4_k_m': 22_621_314_381,
// The MLX child of the list, and the total of a real pull on a Mac, byte for byte.
'qwen3.6:35b-a3b': 23_105_798_956,
'qwen3.8:27b': 17_741_871_939,
'qwen3.8:27b-q8_0': 29_978_241_837,
'qwen3.5:122b-a10b': 81_370_035_877,
'qwen3.8:27b-q8_0': 29_978_242_050,
'qwen3.8:27b-mtp-q4_k_m': 17_741_872_154,
// The MLX child of the list (its GGUF child is 76,756,174,997).
'ornith-1.5:122b': 73_388_069_552,
});
});
it('says which row was measured on a Mac: one, and every other row is an estimate', () => {
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.figures]))).toEqual({
'qwen3.5:9b': 'estimated',
'qwen3.5:9b-q8_0': 'estimated',
'gpt-oss:20b': 'estimated',
'qwen3.6:35b-a3b': 'measured',
'qwen3.8:27b': 'estimated',
'qwen3.8:27b-q8_0': 'estimated',
'qwen3.5:122b-a10b': 'estimated',
});
});
it('gives every row the memory figures measured on the one row: 37 KB per token, 3 GB, 5 GB', () => {
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
expect(
[model.heldBytesPerToken, model.overheadBytes, model.peakExtraBytes],
`${model.id}: an unmeasured row carries the measured row's figures, not invented ones`
).toEqual([37_000, 3_000_000_000, 5_000_000_000]);
}
// No row keeps the figure that was computed from the model's config.
expect(JSON.stringify(STOCK_LOCAL_MODEL_CATALOG)).not.toContain('kvBytesPerToken');
expect(models.filter((m) => m.figures === 'measured').map((m) => m.id)).toEqual(['qwen3.6:35b-a3b']);
expect(figuresOf(models.find((m) => m.id === 'qwen3.6:35b-a3b') as LocalModelEntry)).toEqual(MEASURED);
});
it('has one tested entry: the 35B-A3B with OpenCode on the MacBook Pro, with its measured speed', () => {
expect(
Object.fromEntries(
STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.tested.length > 0).map((m) => [m.id, m.tested])
)
).toEqual({
expect(Object.fromEntries(models.filter((m) => m.tested.length > 0).map((m) => [m.id, m.tested]))).toEqual({
'qwen3.6:35b-a3b': [
{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 },
],
});
// A row may claim a test only when its figures were measured.
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
for (const model of models) {
if (model.tested.length > 0) expect(model.figures, model.id).toBe('measured');
}
});
it('gives every model a parameter count, an id of the default registry and a license', () => {
for (const model of STOCK_LOCAL_MODEL_CATALOG.models) {
expect(model.paramsB, model.id).toBeGreaterThan(0);
expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN);
expect(model.license).toBe('Apache-2.0');
}
expect(Object.fromEntries(models.map((m) => [m.id, [m.paramsB, m.license]]))).toEqual({
'qwen3.5:9b-q4_k_m': [9, 'Apache-2.0'],
'mimo-v2.6:9b-q4_k_m': [9, 'MIT'],
'north-mini-code-1.0:q4_k_m': [30, 'Apache-2.0'],
'qwen3.6:35b-a3b-mtp-q4_k_m': [35, 'Apache-2.0'],
'qwen3.6:35b-a3b': [35, 'Apache-2.0'],
'qwen3.8:27b-q8_0': [27.8, 'Apache-2.0'],
'qwen3.8:27b-mtp-q4_k_m': [27.8, 'Apache-2.0'],
'ornith-1.5:122b': [122, 'MIT'],
});
for (const model of models) expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN);
// The two 9B rows are the ones under the 12 billion line of the `limited` verdict.
expect(STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual([
'qwen3.5:9b',
'qwen3.5:9b-q8_0',
]);
expect(models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual(['qwen3.5:9b-q4_k_m', 'mimo-v2.6:9b-q4_k_m']);
});
it('offers no model below the 65536 tokens a harness needs, nor beyond its own window', () => {
for (const model of models) {
expect(Math.min(...model.contexts), model.id).toBeGreaterThanOrEqual(65536);
expect(model.contexts[0], model.id).toBeLessThanOrEqual(262144);
}
});
it('has a speed figure only where one was measured or scaled from a measurement', () => {
expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({
'qwen3.5:9b': 9.3,
'qwen3.5:9b-q8_0': 5.8,
'gpt-oss:20b': null,
'qwen3.6:35b-a3b': 13.6,
'qwen3.8:27b': 2.95,
expect(Object.fromEntries(models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({
// 37.1 tokens per second on 400 GB/s, and the same for the fine-tune of the same size.
'qwen3.5:9b-q4_k_m': 9.3,
'mimo-v2.6:9b-q4_k_m': 9.3,
// The 35B-A3B's llama.cpp figure: both have 3 billion active parameters.
'north-mini-code-1.0:q4_k_m': 13.6,
// 54.5 on 400 GB/s with llama.cpp.
'qwen3.6:35b-a3b-mtp-q4_k_m': 13.6,
// 67.8 on 400 GB/s with MLX.
'qwen3.6:35b-a3b': 16.9,
// 2.95 scaled by file size.
'qwen3.8:27b-q8_0': 1.7,
'qwen3.5:122b-a10b': null,
// 11.8 on 400 GB/s with llama.cpp, without prediction.
'qwen3.8:27b-mtp-q4_k_m': 2.95,
// 16.9 x 3/10 active parameters, rounded down.
'ornith-1.5:122b': 5,
});
});
@@ -174,6 +317,92 @@ describe('the stock catalogue', () => {
});
});
describe('the memory figures of the stock rows: each row has its own', () => {
const models = STOCK_LOCAL_MODEL_CATALOG.models;
it('knows the architecture of every row, and every row of that table is in the catalogue', () => {
expect(models.map((m) => m.id).sort()).toEqual(Object.keys(ROW_ARCHITECTURE).sort());
});
it('computes the llamacpp rows for the K/V cache type and the prompt cache the runtime really sets', () => {
// The figures below assume these; runtimeEnv() is what the runtime gets.
const env = runtimeEnv({ source: {}, home: '/h', modelsDir: '/m', port: 11435, contextLength: 65536 });
expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE);
expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB));
expect(env.LLAMA_ARG_CTX_CHECKPOINTS).toBe(String(RUNTIME_CTX_CHECKPOINTS));
expect(KV_BYTES_PER_VALUE[RUNTIME_KV_CACHE_TYPE]).toBeDefined();
// The runtime sets no cache type for the draft context, so the prediction layer's stays f16.
expect(Object.keys(env).filter((name) => /DRAFT|SPEC/.test(name))).toEqual([]);
});
it("recomputes every estimated row's figures from its own model's architecture and runner", () => {
for (const model of models.filter((m) => m.figures === 'estimated')) {
const { architecture, draftTokens } = ROW_ARCHITECTURE[model.id];
const arch = ARCHITECTURES[architecture];
const expected = model.runner === 'llamacpp' ? llamaCppFigures(arch, draftTokens) : mlxFigures(arch);
expect(figuresOf(model), `${model.id} (${architecture}, ${model.runner})`).toEqual(expected);
}
});
it('states the numbers the header and the row comments name', () => {
expect(Object.fromEntries(models.map((m) => [m.id, figuresOf(m)]))).toEqual({
// State x (1 live + 8 checkpoints) + 1.0 GB + 1 GiB.
'qwen3.5:9b-q4_k_m': [17_408, 2_547_960_320, 0],
'mimo-v2.6:9b-q4_k_m': [17_408, 2_547_960_320, 0],
// The sliding window x (1 live + 8 checkpoints) + 1.0 GB + 1 GiB.
'north-mini-code-1.0:q4_k_m': [14_144, 3_698_117_120, 0],
// State x (1 live + 2 rollback copies + 8 checkpoints) + 1.0 GB + 1 GiB; the prediction
// layer's cache in f16 per token. The 27B rows below draft 4 tokens: 4 rollback copies.
'qwen3.6:35b-a3b-mtp-q4_k_m': [12_928, 2_798_242_304, 0],
'qwen3.6:35b-a3b': [37_000, 3_000_000_000, 5_000_000_000],
'qwen3.8:27b-q8_0': [38_912, 4_113_353_216, 0],
'qwen3.8:27b-mtp-q4_k_m': [38_912, 4_113_353_216, 0],
'ornith-1.5:122b': [56_574, 11_589_934_592, 5_000_000_000],
});
// The two parts of a llamacpp overhead that are not computed from the architecture.
expect(RUNTIME_CTX_CHECKPOINTS).toBe(8);
expect(RUNTIME_HOST_PROMPT_CACHE_MIB).toBe(1024);
// The MLX calibration: 37,000 held against 28,340 computed for the measured row.
expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.6-35B-A3B'])).toBe(28_340);
expect(mlxComputedPerToken(ARCHITECTURES['Qwen3.5-122B-A10B'])).toBe(43_332);
});
it('lets two rows share figures only when they share the architecture, the drafted tokens and the runner', () => {
const byFigures = new Map<string, LocalModelEntry[]>();
for (const model of models) {
const key = figuresOf(model).join('/');
byFigures.set(key, [...(byFigures.get(key) ?? []), model]);
}
const shared = [...byFigures.values()].filter((group) => group.length > 1);
// The two 9B rows (a fine-tune and its base) and the two 27B rows (two quantizations of one model).
expect(shared.map((group) => group.map((m) => m.id))).toEqual([
['qwen3.5:9b-q4_k_m', 'mimo-v2.6:9b-q4_k_m'],
['qwen3.8:27b-q8_0', 'qwen3.8:27b-mtp-q4_k_m'],
]);
for (const group of shared) {
const identity = (m: LocalModelEntry): string => JSON.stringify([ROW_ARCHITECTURE[m.id], m.runner, m.figures]);
expect(new Set(group.map(identity)).size, group.map((m) => m.id).join(', ')).toBe(1);
}
});
it("gives no estimated row the measured row's per-token or overhead figure", () => {
for (const model of models.filter((m) => m.figures === 'estimated')) {
expect(model.heldBytesPerToken, model.id).not.toBe(MEASURED[0]);
expect(model.overheadBytes, model.id).not.toBe(MEASURED[1]);
expect(figuresOf(model), model.id).not.toEqual(MEASURED);
}
});
it('gives a llamacpp row no peak (its buffers are allocated at load) and an estimated mlx row the whole prefix budget', () => {
for (const model of models) {
if (model.runner === 'llamacpp') expect(model.peakExtraBytes, model.id).toBe(0);
if (model.runner === 'mlx' && model.figures === 'estimated') {
expect(model.overheadBytes - MEASURED[1], model.id).toBe(8 * GIB);
}
}
});
});
describe('the model id rule', () => {
it('accepts name:tag in lowercase', () => {
for (const id of ['qwen3.5:9b', 'gpt-oss:20b', 'qwen3.8:27b-q8_0', 'a:b', 'x_y.z-1:2.3_4']) {
@@ -240,6 +469,19 @@ describe('the schema, field by field', () => {
expect(issuesOf(withFirstModel({ peakExtraBytes: 1.5 }))).not.toEqual([]);
});
it('requires the runner on every model row, and knows two', () => {
const row: Record<string, unknown> = { ...copy().models[0] };
delete row.runner;
const issues = issuesOf({ ...copy(), models: [row] });
expect(issues).toHaveLength(1);
expect(issues[0]).toContain('models.0.runner');
expect(issuesOf(withFirstModel({ runner: 'mlx' }))).toEqual([]);
expect(issuesOf(withFirstModel({ runner: 'llamacpp' }))).toEqual([]);
for (const runner of ['ggml', 'MLX', 'llama.cpp', '']) {
expect(issuesOf(withFirstModel({ runner })), runner).not.toEqual([]);
}
});
it('requires paramsB and tested on every model row', () => {
for (const field of ['paramsB', 'tested'] as const) {
const row: Record<string, unknown> = { ...copy().models[0] };
test/local-model-fits-budget.test.ts
+11 -9
View File
@@ -51,7 +51,7 @@ describe('modelFitsBudget', () => {
const catalog = getLocalModelCatalog();
it('agrees with the recommendation on every model and context of the catalogue', () => {
for (const gb of [8, 16, 24, 32, 48, 64, 128]) {
for (const gb of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192, 256, 512]) {
const hw = mac(gb);
const offered = recommend(hw, catalog, harnessFacts());
const rows = [offered.pick, ...offered.alternatives].filter((row) => row !== undefined);
@@ -85,7 +85,7 @@ describe('modelFitsBudget', () => {
});
it('rules out a model that is too large, and one the catalogue does not have', () => {
expect(modelFitsBudget(mac(48), catalog, 'qwen3.5:122b-a10b', 131072)).toBe(false);
expect(modelFitsBudget(mac(48), catalog, 'ornith-1.5:122b', 131072)).toBe(false);
expect(modelFitsBudget(mac(48), catalog, 'no-such:1b', 65536)).toBe(false);
expect(modelFitsBudget(mac(48), catalog, 'qwen3.6:35b-a3b', 131072)).toBe(true);
});
@@ -96,14 +96,16 @@ describe('modelFitsBudget', () => {
expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 40_157_944_217 })).toBe(true);
// A runtime that reports less than the estimate rules out what the estimate allowed.
expect(modelFitsBudget(mac(48), catalog, id, 131072, { measuredTotalBytes: 33 * 1024 ** 3 })).toBe(false);
// And one that reports more admits what the estimate ruled out: the 27B on a 36 GB Mac.
expect(modelFitsBudget(mac(36), catalog, 'qwen3.8:27b', 65536)).toBe(false);
expect(modelFitsBudget(mac(36), catalog, 'qwen3.8:27b', 65536, { measuredTotalBytes: 27 * 1024 ** 3 })).toBe(true);
// And one that reports more admits what the estimate ruled out: on a 36 GB Mac the GGUF
// 35B-A3B at its full window needs 28.8 GB, 1.032 of the 27.9 GB estimate and 0.958 of 28 GiB.
const moeGguf = 'qwen3.6:35b-a3b-mtp-q4_k_m';
expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144)).toBe(false);
expect(modelFitsBudget(mac(36), catalog, moeGguf, 262144, { measuredTotalBytes: 28 * 1024 ** 3 })).toBe(true);
// The reserve still bounds it: 100 GiB "measured" on a 48 GB Mac is 40 GiB (42.9 GB) of
// budget, so the Q8 27B with its 42.8 GB stays out (0.997) although 100 GiB would hold it.
// budget, so the 122B with its 97.4 GB stays out although 100 GiB would hold it.
const huge = { measuredTotalBytes: 100 * 1024 ** 3 };
expect(modelFitsBudget(mac(48), catalog, 'qwen3.8:27b-q8_0', 131072, huge)).toBe(false);
expect(modelFitsBudget(mac(128), catalog, 'qwen3.8:27b-q8_0', 131072, huge)).toBe(true);
expect(modelFitsBudget(mac(48), catalog, 'ornith-1.5:122b', 131072, huge)).toBe(false);
expect(modelFitsBudget(mac(128), catalog, 'ornith-1.5:122b', 131072, huge)).toBe(true);
});
it('agrees with the recommendation under a measured budget too', () => {
@@ -130,7 +132,7 @@ describe('modelFitsBudget', () => {
it('is false for a context that is no token count', () => {
for (const context of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
expect(modelFitsBudget(mac(128), catalog, 'qwen3.5:9b', context), String(context)).toBe(false);
expect(modelFitsBudget(mac(128), catalog, 'qwen3.5:9b-q4_k_m', context), String(context)).toBe(false);
}
});
});
test/local-model-harness-facts.test.ts
+10
View File
@@ -29,6 +29,7 @@ import {
} from '../src/local-model/harness-facts.js';
import { stockInstallCommand } from '../src/cli-install-runner.js';
import { getCli, registryFilePath, reloadCliRegistry } from '../src/config/cli-registry/registry.js';
import { getLocalModelCatalog } from '../src/config/local-model-catalog.js';
import type { CliEntry } from '../src/config/cli-registry/types.js';
import type { LocalModelCatalog } from '../src/types.js';
@@ -106,6 +107,15 @@ describe('cliAcceptsCustomModelId (the launch path’s model id test, on the sto
}
});
it('accepts every model id of the stock catalogue for every harness the catalogue offers', () => {
const catalog = getLocalModelCatalog();
for (const harness of catalog.harnesses.filter((row) => !row.gated)) {
for (const model of catalog.models) {
expect(cliAcceptsCustomModelId(stockEntry(harness.cli), model.id), `${harness.cli} ${model.id}`).toBe(true);
}
}
});
it('refuses a colon for a CLI whose `--model` pattern has none, and accepts the same CLI without it', () => {
const omp = stockEntry('omp');
expect(cliAcceptsCustomModelId(omp, 'qwen3.5:9b')).toBe(false);
test/local-model-recommend.test.ts
+446 -284
View File
File diff suppressed because it is too large. Load diff
test/local-model-runtime.test.ts
+37 -2
View File
@@ -38,8 +38,11 @@ import {
parseInferenceComputeTotal,
removeLocalModelFiles,
removePartialModelBlobs,
RUNTIME_CTX_CHECKPOINTS,
RUNTIME_DOWNLOAD_RETRIES,
RUNTIME_HOST_PROMPT_CACHE_MIB,
RUNTIME_KEEP_ALIVE,
RUNTIME_KV_CACHE_TYPE,
RUNTIME_MAX_RESTARTS,
RUNTIME_PORT_FIRST,
RUNTIME_PORT_SPAN,
@@ -260,6 +263,7 @@ describe('runtimeEnv', () => {
OLLAMA_REMOTES: 'evil.example',
OLLAMA_AUTH: '1',
LLAMA_ARG_CACHE_RAM: '0',
LLAMA_ARG_CTX_CHECKPOINTS: '64',
// Other secrets a server environment carries.
ANTHROPIC_API_KEY: 'sk-ant-secret',
OPENAI_API_KEY: 'sk-secret',
@@ -273,7 +277,7 @@ describe('runtimeEnv', () => {
DYLD_INSERT_LIBRARIES: '/tmp/evil.dylib',
};
it('holds exactly the allowlist: what is copied, the private HOME, and seven settings', () => {
it('holds exactly the allowlist: what is copied, the private HOME, and eleven settings', () => {
expect(runtimeEnv(input(SERVER_ENV))).toEqual({
PATH: '/usr/local/bin:/usr/bin:/bin',
TMPDIR: '/var/folders/xy/T/',
@@ -292,9 +296,33 @@ describe('runtimeEnv', () => {
OLLAMA_KEEP_ALIVE: '10m',
OLLAMA_NUM_PARALLEL: '1',
OLLAMA_MAX_LOADED_MODELS: '1',
// The memory settings the catalogue's llamacpp rows are computed for.
OLLAMA_KV_CACHE_TYPE: 'q8_0',
OLLAMA_FLASH_ATTENTION: '1',
LLAMA_ARG_CACHE_RAM: '1024',
LLAMA_ARG_CTX_CHECKPOINTS: '8',
});
});
it('sets the llama.cpp memory settings from the exported constants, never from the server', () => {
expect(RUNTIME_KV_CACHE_TYPE).toBe('q8_0');
expect(RUNTIME_HOST_PROMPT_CACHE_MIB).toBe(1024);
expect(RUNTIME_CTX_CHECKPOINTS).toBe(8);
const env = runtimeEnv(input(SERVER_ENV));
expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE);
expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB));
expect(env.LLAMA_ARG_CTX_CHECKPOINTS).toBe(String(RUNTIME_CTX_CHECKPOINTS));
// The server's own `LLAMA_ARG_CACHE_RAM=0` and `LLAMA_ARG_CTX_CHECKPOINTS=64`, and a user's
// q4_0 cache, lose against ours.
const env2 = runtimeEnv(input({ ...SERVER_ENV, OLLAMA_KV_CACHE_TYPE: 'q4_0', OLLAMA_FLASH_ATTENTION: '0' }));
expect([
env2.OLLAMA_KV_CACHE_TYPE,
env2.OLLAMA_FLASH_ATTENTION,
env2.LLAMA_ARG_CACHE_RAM,
env2.LLAMA_ARG_CTX_CHECKPOINTS,
]).toEqual(['q8_0', '1', '1024', '8']);
});
it('lets no CODEMAN_ variable through, the password least of all', () => {
const env = runtimeEnv(input(SERVER_ENV));
expect(Object.keys(env).filter((name) => name.startsWith('CODEMAN_'))).toEqual([]);
@@ -309,8 +337,10 @@ describe('runtimeEnv', () => {
.sort()
).toEqual([
'OLLAMA_CONTEXT_LENGTH',
'OLLAMA_FLASH_ATTENTION',
'OLLAMA_HOST',
'OLLAMA_KEEP_ALIVE',
'OLLAMA_KV_CACHE_TYPE',
'OLLAMA_MAX_LOADED_MODELS',
'OLLAMA_MODELS',
'OLLAMA_NO_CLOUD',
@@ -329,7 +359,8 @@ describe('runtimeEnv', () => {
]) {
expect(env, name).not.toHaveProperty(name);
}
expect(env).not.toHaveProperty('LLAMA_ARG_CACHE_RAM');
// The server's `LLAMA_ARG_CACHE_RAM=0` is not copied: the value is ours.
expect(env.LLAMA_ARG_CACHE_RAM).toBe('1024');
});
it('carries no secret and no value of the server that is not on the list', () => {
@@ -365,9 +396,13 @@ describe('runtimeEnv', () => {
it('leaves out what the server does not have, and sets the rest all the same', () => {
expect(Object.keys(runtimeEnv(input({}))).sort()).toEqual([
'HOME',
'LLAMA_ARG_CACHE_RAM',
'LLAMA_ARG_CTX_CHECKPOINTS',
'OLLAMA_CONTEXT_LENGTH',
'OLLAMA_FLASH_ATTENTION',
'OLLAMA_HOST',
'OLLAMA_KEEP_ALIVE',
'OLLAMA_KV_CACHE_TYPE',
'OLLAMA_MAX_LOADED_MODELS',
'OLLAMA_MODELS',
'OLLAMA_NO_CLOUD',
test/local-model-supervisor.test.ts
+4
View File
@@ -206,6 +206,10 @@ describe('start', () => {
OLLAMA_KEEP_ALIVE: '10m',
OLLAMA_NUM_PARALLEL: '1',
OLLAMA_MAX_LOADED_MODELS: '1',
OLLAMA_KV_CACHE_TYPE: 'q8_0',
OLLAMA_FLASH_ATTENTION: '1',
LLAMA_ARG_CACHE_RAM: '1024',
LLAMA_ARG_CTX_CHECKPOINTS: '8',
});
});