mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-14 18:39:33 +01:00
ci(local-model): check the model catalogue and the runtime pin every week
scripts/check-local-model-catalog.mjs reads every catalogue tag the way Ollama 0.40.15 on Apple silicon reads it: it asks for the manifest list, takes the mlx build when the tag offers one (else llamacpp, else ggml), and compares that build's pull total (layers plus config) with the row's `bytes`. It also reports a tag that is gone, a build whose `requires` is newer than the pinned runtime, a pinned digest that moved, a newer Ollama release, and a pinned ollama-darwin.tgz that is gone or changed. It never requests a layer, bounds every request, retries nothing and validates everything that reaches the report. Exit 0 clean, 1 drift, 2 failed. The plain Docker manifest of a list tag is the llamacpp build, not what a Mac pulls. Read the right way, the first run finds two rows sized from the wrong build: qwen3.5:9b (a Mac downloads 7,639,229,431 bytes, +16.6%) and qwen3.8:27b (18,174,721,847, +2.4%). .github/workflows/local-model-catalog.yml runs it Mondays at 06:00 UTC and on dispatch. The check job installs with --ignore-scripts and holds a read-only token; a separate job with issues: write and no checkout opens or updates the one "Local model catalogue: weekly check" issue on drift, comments on it on a clean week when it is open, and is skipped when the check failed. Actions are pinned by commit. docs/local-models.md gains "Keeping the catalogue current". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codeman maintainer committed
1 parent
be8fc2980a
commit
d12bef0833
8 files changed
+1577
No files matched your search
@@ -0,0 +1,128 @@
|
||||
name: Local model catalogue
|
||||
|
||||
# Checks every Monday that the local model catalogue (src/config/local-model-catalog.ts) and
|
||||
# the pinned model runtime (OLLAMA_RUNTIME_PIN in src/local-model/runtime.ts) still match what
|
||||
# the registry and Ollama's releases serve: a tag can be pushed again or removed, the build a
|
||||
# Mac downloads can change size, and Ollama ships new versions. The catalogue is compiled into
|
||||
# Codeman on purpose, so this job only reports; a change ships in a release like any other.
|
||||
# What it checks, and what a maintainer does with its issue: docs/local-models.md, "Keeping
|
||||
# the catalogue current".
|
||||
#
|
||||
# By hand: gh workflow run local-model-catalog.yml (the file must be on master for that,
|
||||
# and the schedule only ever runs master's copy). Locally: npm run check:local-model-catalog
|
||||
#
|
||||
# Two jobs, so the token that can write issues never sits next to `npm ci`:
|
||||
# - check installs the dependencies and runs scripts/check-local-model-catalog.mjs with a
|
||||
# read-only token. Exit 0 clean, 1 drift, 2 the check failed. On 2 the job fails and the
|
||||
# issue is not touched: an incomplete report must not open or update it.
|
||||
# - issue runs no code from the repository, only gh. On drift it opens the one issue titled
|
||||
# "Local model catalogue: weekly check", or puts this week's report into it when it is
|
||||
# already open. On a clean week it comments on that issue only when one is open.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 1'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# A run by hand while the scheduled one is going must not open a second issue.
|
||||
concurrency: ${{ github.workflow }}
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check the catalogue and the runtime pin
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
status: ${{ steps.check.outputs.status }}
|
||||
report: ${{ steps.check.outputs.report }}
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: 'npm'
|
||||
|
||||
# The check needs tsx and zod, both plain JavaScript, so no install script runs here
|
||||
# (node-pty's build and the repository's postinstall included).
|
||||
- name: Install dependencies
|
||||
run: npm ci --ignore-scripts
|
||||
|
||||
- name: Run the check
|
||||
id: check
|
||||
env:
|
||||
# Read-only here. Sent to api.github.com only, so the release lookups are not rate limited.
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set +e
|
||||
npx tsx scripts/check-local-model-catalog.mjs > "$RUNNER_TEMP/report.md"
|
||||
code=$?
|
||||
set -e
|
||||
cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
case "$code" in
|
||||
0) echo "status=clean" >> "$GITHUB_OUTPUT" ;;
|
||||
1) echo "status=drift" >> "$GITHUB_OUTPUT" ;;
|
||||
*)
|
||||
echo "::error::The check itself failed (exit $code), so the issue is left alone. The summary holds what it found."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
# One line of base64, so nothing in the report can end the output early. An issue
|
||||
# body holds at most 65,536 characters; the report is a few KB.
|
||||
echo "report=$(head -c 60000 "$RUNNER_TEMP/report.md" | base64 -w0)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
issue:
|
||||
name: Keep the weekly issue current
|
||||
needs: check
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
ISSUE_TITLE: 'Local model catalogue: weekly check'
|
||||
STATUS: ${{ needs.check.outputs.status }}
|
||||
REPORT_B64: ${{ needs.check.outputs.report }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
steps:
|
||||
- name: Open, update or comment on the issue
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The issues API, not the search index, so an issue opened a minute ago is found too,
|
||||
# and an exact title match, so a similar title never counts. The oldest open one wins.
|
||||
number=$(gh issue list --state open --limit 1000 --json number,title \
|
||||
| jq -r --arg title "$ISSUE_TITLE" '[.[] | select(.title == $title)] | sort_by(.number) | first | .number // empty')
|
||||
today=$(date -u +%Y-%m-%d)
|
||||
case "$STATUS" in
|
||||
drift)
|
||||
printf '%s' "$REPORT_B64" | base64 -d > "$RUNNER_TEMP/report.md"
|
||||
printf '\n---\n\nWritten by the [weekly check](%s). Fix the catalogue or the pin in a pull request and close this issue once that is merged; the next drift opens it again.\n' \
|
||||
"$RUN_URL" >> "$RUNNER_TEMP/report.md"
|
||||
if [ -n "$number" ]; then
|
||||
gh issue edit "$number" --body-file "$RUNNER_TEMP/report.md"
|
||||
gh issue comment "$number" --body "Week of $today: drift found again. The description now holds this week's report ([run]($RUN_URL))."
|
||||
else
|
||||
gh issue create --title "$ISSUE_TITLE" --body-file "$RUNNER_TEMP/report.md"
|
||||
fi
|
||||
;;
|
||||
clean)
|
||||
if [ -n "$number" ]; then
|
||||
gh issue comment "$number" --body "Week of $today: the check was clean ([run]($RUN_URL))."
|
||||
else
|
||||
echo "Clean, and no issue is open: nothing to do."
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "::error::The check job reported no status ('$STATUS')."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -250,6 +250,62 @@ or `python3`.
|
||||
- **The model stays in memory for ten minutes** after the last request, then the runtime
|
||||
unloads it.
|
||||
|
||||
## Keeping the catalogue current
|
||||
|
||||
The catalogue (`src/config/local-model-catalog.ts`) and the runtime pin are compiled into
|
||||
Codeman on purpose: no file or download decides what is recommended or run. What the
|
||||
registry serves can still move away from them: a tag is pushed again or removed, the build
|
||||
a Mac downloads changes size, Ollama ships a new version. The workflow
|
||||
`.github/workflows/local-model-catalog.yml` looks for that every Monday at 06:00 UTC. It
|
||||
runs `scripts/check-local-model-catalog.mjs`, which also runs locally:
|
||||
`npm run check:local-model-catalog` prints the report and exits 0 (clean), 1 (drift) or 2
|
||||
(the check itself failed).
|
||||
|
||||
For every model row it checks:
|
||||
|
||||
- **The tag still exists** in the registry.
|
||||
- **The size of what a Mac downloads** is within 0.1% of the row's `bytes`. A tag can be a
|
||||
manifest list with one build per runner, and Ollama on Apple silicon takes the `mlx` build
|
||||
when there is one, else `llamacpp`, else `ggml`. So the check asks for the tag the way
|
||||
Ollama asks, picks the build a Mac picks and sizes it the way a pull counts it (every layer
|
||||
plus the config). The plain manifest other tools get for the same tag is the `llamacpp`
|
||||
build, which is not what a Mac downloads.
|
||||
- **The build a Mac pulls runs on the pinned runtime**: its config declares no newer
|
||||
Ollama (`requires`) than `OLLAMA_RUNTIME_PIN`.
|
||||
- **The pinned digest**, once a row carries one: a Mac would still pull that manifest. The
|
||||
catalogue pins none yet, so the report lists today's digests, the one to pin being what
|
||||
`/api/tags` reports after the pull.
|
||||
|
||||
For the runtime pin it checks whether a newer Ollama release exists, and whether the pinned
|
||||
`ollama-darwin.tgz` is still on GitHub with the pinned size and sha256.
|
||||
|
||||
It reads manifests and configs only, never a model layer. It does not check the memory
|
||||
figures, the speeds or whether a coding agent works with a model: no manifest says anything
|
||||
about those, and they change only when somebody measures on a Mac.
|
||||
|
||||
**The issue.** Drift opens one issue, "Local model catalogue: weekly check", or, while that
|
||||
issue is open, replaces its description with the new report and adds a comment. A clean week
|
||||
comments on the open issue and does nothing when there is none. A check that failed (the
|
||||
network, a rate limit, an answer that did not parse) fails the run and leaves the issue
|
||||
alone; what it found is in the run's summary. The schedule runs the copy of the workflow on
|
||||
`master`, and so does `gh workflow run local-model-catalog.yml`.
|
||||
|
||||
**What a maintainer does with it**, in a pull request each time:
|
||||
|
||||
- **Size drift or a changed digest**: correct the row. When the build itself changed (another
|
||||
runner or quantization), its memory figures and speed no longer describe it: mark the row
|
||||
`estimated` until somebody measures it again.
|
||||
- **A tag that is gone, or a build that needs a newer runtime**: replace or drop the row, or
|
||||
move the runtime pin first.
|
||||
- **A newer Ollama**: read its release notes, then move `OLLAMA_RUNTIME_PIN` (version, URL,
|
||||
size and sha256, checked against the release's `sha256sum.txt`) and run an install on a
|
||||
Mac before merging.
|
||||
- **A pinned archive that changed**: a security question first. Every install refuses it at
|
||||
the hash compare, so nobody gets it, but find out why before moving the pin.
|
||||
|
||||
Close the issue once the fix is merged. A catalogue or pin change reaches users only in a
|
||||
release: an installed Codeman keeps the catalogue it was built with until it is updated.
|
||||
|
||||
## Source
|
||||
|
||||
| What | Where |
|
||||
@@ -265,3 +321,4 @@ or `python3`.
|
||||
| Routes | `src/web/routes/local-model-routes.ts` |
|
||||
| Shared CLI install runner | `src/cli-install-runner.ts` |
|
||||
| Serialized writer of the endpoint store | `src/custom-model-hosts-writer.ts` |
|
||||
| Weekly catalogue check | `scripts/check-local-model-catalog.mjs`, `.github/workflows/local-model-catalog.yml` |
|
||||
@@ -30,6 +30,7 @@
|
||||
"test:mobile": "vitest run --config test/mobile/vitest.config.ts",
|
||||
"check:frontend-syntax": "node scripts/check-frontend-syntax.mjs",
|
||||
"check:browser-excludes": "node scripts/check-browser-test-excludes.mjs",
|
||||
"check:local-model-catalog": "tsx scripts/check-local-model-catalog.mjs",
|
||||
"fix:node-pty": "node scripts/fix-node-pty.mjs",
|
||||
"typecheck": "tsc --noEmit && tsc -p config/tsconfig.scripts.json",
|
||||
"lint": "eslint --config config/eslint.config.js 'src/**/*.ts'",
|
||||
|
||||
@@ -0,0 +1,728 @@
|
||||
#!/usr/bin/env -S npx tsx
|
||||
/**
|
||||
* The weekly check of the local model catalogue (src/config/local-model-catalog.ts) and of the
|
||||
* pinned model runtime (OLLAMA_RUNTIME_PIN in src/local-model/runtime.ts) against what the
|
||||
* registry and Ollama's releases serve today.
|
||||
*
|
||||
* npx tsx scripts/check-local-model-catalog.mjs # markdown report on stdout
|
||||
* npm run check:local-model-catalog # the same
|
||||
*
|
||||
* Exit code: 0 nothing drifted, 1 drift found, 2 the check itself failed (network, an answer
|
||||
* that does not parse, a catalogue that does not validate). On 2 the report is still printed
|
||||
* with everything that was found, but it is incomplete, so nothing should act on it.
|
||||
* `.github/workflows/local-model-catalog.yml` runs it every Monday and keeps one issue current.
|
||||
*
|
||||
* Runs under tsx because it imports the TypeScript catalogue and pin, and imports them only in
|
||||
* main(), so the test imports the pure parts without them.
|
||||
*
|
||||
* WHAT A MAC DOWNLOADS. A tag is not one build. Asked with the Accept header Ollama 0.40.15
|
||||
* sends (`pullModelManifest` in server/images.go), the registry answers a manifest LIST for
|
||||
* some tags, one child per runner, and Ollama on Apple silicon takes the first child whose
|
||||
* runner is `mlx`, else `llamacpp`, else `ggml` (`runnerPreferences()` in manifest/manifest.go).
|
||||
* Asked with the plain Docker Accept header the registry answers the llamacpp child instead,
|
||||
* which is NOT what a Mac pulls: that is how `qwen3.6:35b-a3b` came to be "2% above its
|
||||
* manifest" in the catalogue header. So this check asks the way Ollama asks, picks the child
|
||||
* the way a Mac picks it, and compares the row's `bytes` with that child's pull total (every
|
||||
* layer plus the config, which is exactly the `total` a real pull reported on a Mac).
|
||||
*
|
||||
* WHAT IS FETCHED, and nothing else:
|
||||
* - `GET /v2/library/<name>/manifests/<tag>`, the tag's manifest or manifest list.
|
||||
* - For a list: `GET /v2/library/<name>/blobs/<digest>` for each child the list names. A child
|
||||
* manifest is stored as a blob (Ollama fetches it the same way, `pullSelectedManifest`). It
|
||||
* is metadata: about 220 KB for an MLX build of 1,055 tensors.
|
||||
* - `GET .../blobs/<digest>` for the config of the child a Mac selects, a few hundred bytes,
|
||||
* for its `requires` (the oldest Ollama that may pull it).
|
||||
* - The GitHub API: the newest release of ollama/ollama, and the release of the pinned version.
|
||||
* A digest from a `layers` entry is never requested: no model weights are ever downloaded.
|
||||
* Every request has its own timeout and a byte cap, nothing is retried, every blob is checked
|
||||
* against its digest, and `GITHUB_TOKEN` is sent to api.github.com only, when it is set.
|
||||
*
|
||||
* Everything that reaches the report from the network is validated first (digests, sizes,
|
||||
* runner and format names, version tags), because the workflow puts the report in an issue.
|
||||
*
|
||||
* NOT CHECKED: the memory figures, the speeds and whether a coding agent works with a model.
|
||||
* No manifest says anything about those; they change only when somebody measures on a Mac.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
// ---------- Constants ----------
|
||||
|
||||
export const REGISTRY_BASE = 'https://registry.ollama.ai';
|
||||
export const GITHUB_API_BASE = 'https://api.github.com';
|
||||
/** The repository whose releases the runtime pin follows. */
|
||||
export const RUNTIME_REPO = 'ollama/ollama';
|
||||
|
||||
export const MEDIA_TYPE_MANIFEST = 'application/vnd.docker.distribution.manifest.v2+json';
|
||||
export const MEDIA_TYPE_MANIFEST_LIST = 'application/vnd.ollama.manifest.list.v2+json';
|
||||
/** The Accept header of Ollama 0.40.15's own manifest request: the list first. */
|
||||
export const MANIFEST_ACCEPT = `${MEDIA_TYPE_MANIFEST_LIST}, ${MEDIA_TYPE_MANIFEST}`;
|
||||
|
||||
/** The runner order of Ollama 0.40.15 on darwin/arm64 (`runnerPreferences()`). */
|
||||
export const MAC_RUNNER_PREFERENCE = Object.freeze(['mlx', 'llamacpp', 'ggml']);
|
||||
/** Format names the report prints as they are. Anything else is printed as `unknown`. */
|
||||
export const KNOWN_FORMATS = Object.freeze(['safetensors', 'gguf']);
|
||||
|
||||
/**
|
||||
* How far the size a Mac downloads may sit from the row's `bytes` before it is drift: 0.1%.
|
||||
* The rows that were sized from layers alone sit a few hundred bytes (the config) below the
|
||||
* pull total; any rebuilt model moves by far more than this.
|
||||
*/
|
||||
export const SIZE_TOLERANCE_FRACTION = 0.001;
|
||||
|
||||
/** One request may take this long, connection and body together. */
|
||||
export const REQUEST_TIMEOUT_MS = 20_000;
|
||||
/** A manifest or child manifest larger than this is refused. The MLX build of the 35B is 220 KB. */
|
||||
export const MAX_MANIFEST_BYTES = 4 * 1024 * 1024;
|
||||
/** A config blob larger than this is refused. They are a few hundred bytes. */
|
||||
export const MAX_CONFIG_BYTES = 64 * 1024;
|
||||
/** A GitHub release answer larger than this is refused. */
|
||||
export const MAX_GITHUB_BYTES = 4 * 1024 * 1024;
|
||||
/** A manifest list with more children than this is refused. */
|
||||
export const MAX_LIST_CHILDREN = 8;
|
||||
/** A manifest with more layers than this is refused. */
|
||||
export const MAX_LAYERS = 20_000;
|
||||
|
||||
const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
||||
const VERSION_PATTERN = /^v?(\d{1,6})\.(\d{1,6})\.(\d{1,6})$/;
|
||||
const RELEASE_TAG_PATTERN = /^v\d{1,6}\.\d{1,6}\.\d{1,6}$/;
|
||||
const ISO_DAY_PATTERN = /^(\d{4}-\d{2}-\d{2})T/;
|
||||
const USER_AGENT = 'codeman-local-model-catalog-check';
|
||||
|
||||
/** The check could not finish: the run ends with exit code 2. */
|
||||
export class CheckFailure extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = 'CheckFailure';
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- Pure helpers ----------
|
||||
|
||||
/** `name:tag` into its two parts. The catalogue's pattern already rules out a slash. */
|
||||
export function splitModelId(id) {
|
||||
const colon = id.indexOf(':');
|
||||
if (colon <= 0 || colon === id.length - 1 || id.includes('/')) throw new CheckFailure(`not a name:tag model id: ${id}`);
|
||||
return { name: id.slice(0, colon), tag: id.slice(colon + 1) };
|
||||
}
|
||||
|
||||
/** `sha256:<hex>` of a body, the way the registry names it. */
|
||||
export function sha256Digest(bytes) {
|
||||
return `sha256:${createHash('sha256').update(bytes).digest('hex')}`;
|
||||
}
|
||||
|
||||
/** A digest as `sha256:<64 hex>`, accepting the bare hex `/api/tags` prints. Anything else: undefined. */
|
||||
export function normalizeDigest(value) {
|
||||
if (typeof value !== 'string') return undefined;
|
||||
const text = value.trim().toLowerCase();
|
||||
const full = text.startsWith('sha256:') ? text : `sha256:${text}`;
|
||||
return DIGEST_PATTERN.test(full) ? full : undefined;
|
||||
}
|
||||
|
||||
/** A runner name as Ollama compares it (lowercase, trimmed). */
|
||||
export function canonicalRunner(runner) {
|
||||
return typeof runner === 'string' ? runner.trim().toLowerCase() : '';
|
||||
}
|
||||
|
||||
/** A runner or format name for the report: one of the known names, else `unknown`. */
|
||||
export function displayName(value, known) {
|
||||
const name = canonicalRunner(value);
|
||||
return known.includes(name) ? name : 'unknown';
|
||||
}
|
||||
|
||||
function isSize(value) {
|
||||
return Number.isSafeInteger(value) && value >= 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* What a pull of this manifest downloads, in bytes: every layer plus the config. Throws on a
|
||||
* manifest that is not shaped like one, so a malformed answer never reads as a size.
|
||||
*/
|
||||
export function pullTotalBytes(manifest) {
|
||||
if (!manifest || typeof manifest !== 'object') throw new CheckFailure('the manifest is not an object');
|
||||
const { config, layers } = manifest;
|
||||
if (!Array.isArray(layers) || layers.length === 0) throw new CheckFailure('the manifest has no layers');
|
||||
if (layers.length > MAX_LAYERS) throw new CheckFailure(`the manifest has more than ${MAX_LAYERS} layers`);
|
||||
if (!config || !isSize(config.size)) throw new CheckFailure('the manifest has no config size');
|
||||
let total = config.size;
|
||||
for (const layer of layers) {
|
||||
if (!layer || !isSize(layer.size)) throw new CheckFailure('a layer of the manifest has no size');
|
||||
total += layer.size;
|
||||
}
|
||||
if (!Number.isSafeInteger(total)) throw new CheckFailure('the manifest total is not a safe integer');
|
||||
return total;
|
||||
}
|
||||
|
||||
/**
|
||||
* The child of a manifest list that Ollama 0.40.15 on Apple silicon pulls: for each runner in
|
||||
* `MAC_RUNNER_PREFERENCE`, the first child (in list order) with that runner. A child whose
|
||||
* mediaType is set to anything but a plain manifest is skipped, as Ollama skips it. Undefined
|
||||
* when no child has a runner a Mac runs.
|
||||
*/
|
||||
export function selectMacChild(children) {
|
||||
if (!Array.isArray(children)) return undefined;
|
||||
for (const runner of MAC_RUNNER_PREFERENCE) {
|
||||
const child = children.find(
|
||||
(entry) =>
|
||||
entry &&
|
||||
(entry.mediaType === undefined || entry.mediaType === '' || entry.mediaType === MEDIA_TYPE_MANIFEST) &&
|
||||
canonicalRunner(entry.runner) === runner
|
||||
);
|
||||
if (child) return child;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The runner and format Ollama assumes for a manifest that names neither, from its config's
|
||||
* `model_format` (`MetadataForConfig()` in manifest/manifest.go): safetensors runs on mlx,
|
||||
* gguf on ggml.
|
||||
*/
|
||||
export function metadataForConfig(config) {
|
||||
const format = canonicalRunner(config?.model_format);
|
||||
if (format === 'safetensors') return { runner: 'mlx', format: 'safetensors' };
|
||||
if (format === 'gguf' || format === 'ggml') return { runner: 'ggml', format: 'gguf' };
|
||||
return { runner: '', format };
|
||||
}
|
||||
|
||||
/** `[major, minor, patch]` of a version or `v`-tag, or undefined when it is not one. */
|
||||
export function parseVersion(text) {
|
||||
const match = typeof text === 'string' ? VERSION_PATTERN.exec(text.trim()) : null;
|
||||
return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : undefined;
|
||||
}
|
||||
|
||||
/** Negative when `a` is older than `b`, 0 when equal, positive when newer. Both must parse. */
|
||||
export function compareVersions(a, b) {
|
||||
const left = parseVersion(a);
|
||||
const right = parseVersion(b);
|
||||
if (!left || !right) throw new CheckFailure(`cannot compare versions ${a} and ${b}`);
|
||||
for (let i = 0; i < 3; i++) if (left[i] !== right[i]) return left[i] - right[i];
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** The signed difference of `actual` from `expected`, as a fraction of `expected`. */
|
||||
export function sizeDifference(expected, actual) {
|
||||
return (actual - expected) / expected;
|
||||
}
|
||||
|
||||
/** Whether `actual` lies within `SIZE_TOLERANCE_FRACTION` of `expected`. */
|
||||
export function withinTolerance(expected, actual, tolerance = SIZE_TOLERANCE_FRACTION) {
|
||||
return Math.abs(actual - expected) <= expected * tolerance;
|
||||
}
|
||||
|
||||
/**
|
||||
* The findings for one catalogue row. `row` is `{ id, bytes, digest? }` (`digest` only once the
|
||||
* catalogue stores one); `observed` is what `inspectModel()` returned for it. A row may have
|
||||
* several findings; none means it is clean.
|
||||
*
|
||||
* - `gone`: the tag is not in the registry, so a pull fails.
|
||||
* - `size`: what a Mac downloads is outside the tolerance of the row's `bytes`.
|
||||
* - `digest`: the row pins a digest and a Mac would now pull another one.
|
||||
* - `requires`: the build a Mac pulls declares a newer Ollama than the pinned runtime, so the
|
||||
* pinned runtime refuses the pull.
|
||||
*/
|
||||
export function classifyModel(row, observed, pinnedVersion) {
|
||||
if (observed.gone) return [{ kind: 'gone' }];
|
||||
const findings = [];
|
||||
const mac = observed.mac;
|
||||
if (!withinTolerance(row.bytes, mac.bytes)) {
|
||||
findings.push({ kind: 'size', expected: row.bytes, actual: mac.bytes, fraction: sizeDifference(row.bytes, mac.bytes) });
|
||||
}
|
||||
const pinned = normalizeDigest(row.digest);
|
||||
if (row.digest !== undefined && pinned !== mac.digest) {
|
||||
findings.push({ kind: 'digest', expected: pinned ?? 'not a sha256 digest', actual: mac.digest });
|
||||
}
|
||||
if (mac.requires && compareVersions(mac.requires, pinnedVersion) > 0) {
|
||||
findings.push({ kind: 'requires', requires: mac.requires, pinned: pinnedVersion });
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
/**
|
||||
* The findings for the runtime pin. `observed` is what `inspectRuntime()` returned.
|
||||
*
|
||||
* - `behind`: a newer Ollama release exists than the pinned one.
|
||||
* - `pinned-release-gone`, `pinned-asset-gone`: the pinned download no longer exists, so every
|
||||
* new install fails at the runtime step.
|
||||
* - `pinned-asset-changed`: the pinned asset's size or sha256 no longer matches the pin. Every
|
||||
* install already refuses it at the hash compare; this is also what a replaced release
|
||||
* asset looks like, so read it as a security signal first.
|
||||
*/
|
||||
export function classifyRuntime(pin, observed) {
|
||||
const findings = [];
|
||||
if (compareVersions(observed.latest.version, pin.version) > 0) {
|
||||
findings.push({ kind: 'behind', pinned: pin.version, latest: observed.latest.version });
|
||||
}
|
||||
if (observed.pinned.gone) findings.push({ kind: 'pinned-release-gone' });
|
||||
else if (!observed.pinned.asset) findings.push({ kind: 'pinned-asset-gone' });
|
||||
else {
|
||||
const { size, digest } = observed.pinned.asset;
|
||||
if (size !== pin.bytes || (digest !== undefined && digest !== `sha256:${pin.sha256}`)) {
|
||||
findings.push({ kind: 'pinned-asset-changed', size, digest });
|
||||
}
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
/** 0 clean, 1 drift, 2 the check failed. A failure anywhere wins over drift anywhere. */
|
||||
export function exitCodeFor(result) {
|
||||
const failed =
|
||||
result.errors.length > 0 || result.models.some((model) => model.error) || Boolean(result.runtime?.error);
|
||||
if (failed) return 2;
|
||||
const drift =
|
||||
result.models.some((model) => model.findings.length > 0) || (result.runtime?.findings.length ?? 0) > 0;
|
||||
return drift ? 1 : 0;
|
||||
}
|
||||
|
||||
// ---------- Report ----------
|
||||
|
||||
/** 1234567 as `1,234,567`, independent of the locale. */
|
||||
export function groupDigits(value) {
|
||||
return String(value).replace(/\B(?=(\d{3})+(?!\d))/g, ',');
|
||||
}
|
||||
|
||||
/** A fraction as a signed percentage with two decimals: `+16.61%`. */
|
||||
export function formatPercent(fraction) {
|
||||
const text = (fraction * 100).toFixed(2);
|
||||
return fraction > 0 ? `+${text}%` : `${text}%`;
|
||||
}
|
||||
|
||||
/** Error text for the report: one line, backticks made quotes, at most 200 characters. */
|
||||
function oneLine(text) {
|
||||
const flat = String(text)
|
||||
.replace(/[\u0000-\u001f\u007f]+/g, ' ')
|
||||
.replace(/`/g, "'")
|
||||
.replace(/ {2,}/g, ' ')
|
||||
.trim();
|
||||
return flat.length > 200 ? `${flat.slice(0, 200)}...` : flat;
|
||||
}
|
||||
|
||||
function buildName(variant) {
|
||||
return `${displayName(variant.runner, MAC_RUNNER_PREFERENCE)}, ${displayName(variant.format, KNOWN_FORMATS)}`;
|
||||
}
|
||||
|
||||
function modelResult(model) {
|
||||
if (model.error) return 'check failed';
|
||||
if (model.findings.length === 0) return 'ok';
|
||||
const names = { gone: 'tag gone', size: 'size drift', digest: 'digest changed', requires: 'needs a newer runtime' };
|
||||
return model.findings.map((finding) => names[finding.kind]).join(', ');
|
||||
}
|
||||
|
||||
function describeModelFinding(model, finding) {
|
||||
const id = `\`${model.id}\``;
|
||||
switch (finding.kind) {
|
||||
case 'gone':
|
||||
return `${id}: the registry no longer has this tag, so the download fails. Replace or remove the row.`;
|
||||
case 'size':
|
||||
return (
|
||||
`${id}: a Mac downloads the ${buildName(model.observed.mac)} build, ${groupDigits(finding.actual)} bytes, ` +
|
||||
`${formatPercent(finding.fraction)} against the row's \`bytes\` of ${groupDigits(finding.expected)}. ` +
|
||||
'The memory fit reads `bytes`, so correct it (and re-measure if the build changed).'
|
||||
);
|
||||
case 'digest':
|
||||
return `${id}: the row pins \`${finding.expected}\`, a Mac now pulls \`${finding.actual}\`. The tag was pushed again.`;
|
||||
case 'requires':
|
||||
return (
|
||||
`${id}: the build a Mac pulls declares \`requires: ${finding.requires}\`, newer than the pinned runtime ` +
|
||||
`${finding.pinned}, which refuses to pull it.`
|
||||
);
|
||||
default:
|
||||
return `${id}: ${finding.kind}`;
|
||||
}
|
||||
}
|
||||
|
||||
function describeRuntimeFinding(finding) {
|
||||
switch (finding.kind) {
|
||||
case 'behind':
|
||||
return (
|
||||
`Ollama ${finding.latest} is out; the pin is ${finding.pinned}. Read its release notes, then move ` +
|
||||
'`OLLAMA_RUNTIME_PIN` (version, URL, size, sha256) and test an install on a Mac.'
|
||||
);
|
||||
case 'pinned-release-gone':
|
||||
return 'The release of the pinned version is gone from GitHub: every new install fails at the runtime step.';
|
||||
case 'pinned-asset-gone':
|
||||
return 'The pinned release no longer carries the pinned archive: every new install fails at the runtime step.';
|
||||
case 'pinned-asset-changed':
|
||||
return (
|
||||
`The pinned archive changed on GitHub (now ${finding.size === undefined ? 'no size' : `${groupDigits(finding.size)} bytes`}` +
|
||||
`${finding.digest ? `, \`${finding.digest}\`` : ''}). Installs refuse it at the hash compare. Find out why before moving the pin.`
|
||||
);
|
||||
default:
|
||||
return finding.kind;
|
||||
}
|
||||
}
|
||||
|
||||
function assetCell(asset) {
|
||||
if (!asset) return 'none';
|
||||
return `${groupDigits(asset.size)} bytes${asset.digest ? `, \`${asset.digest}\`` : ', no digest in the API'}`;
|
||||
}
|
||||
|
||||
/** The markdown report of one run. Pure: everything in it was validated by the IO side. */
|
||||
export function renderReport(result) {
|
||||
const lines = [];
|
||||
const code = exitCodeFor(result);
|
||||
const driftModels = result.models.filter((model) => !model.error && model.findings.length > 0).length;
|
||||
const runtimeDrift = result.runtime && !result.runtime.error ? result.runtime.findings.length > 0 : false;
|
||||
|
||||
lines.push('# Local model catalogue: weekly check', '');
|
||||
lines.push(
|
||||
`Run ${result.checkedAt}. Catalogue of ${result.catalogue.updated} (${result.models.length} models), ` +
|
||||
`runtime pin Ollama ${result.pin.version}.`,
|
||||
''
|
||||
);
|
||||
if (code === 2) {
|
||||
lines.push('**The check failed**: the results below are incomplete. Nothing should act on them.', '');
|
||||
} else if (code === 1) {
|
||||
const parts = [];
|
||||
if (driftModels > 0) parts.push(`${driftModels} of ${result.models.length} models`);
|
||||
if (runtimeDrift) parts.push('the runtime pin');
|
||||
lines.push(`**Drift found**: ${parts.join(' and ')}.`, '');
|
||||
} else {
|
||||
lines.push('**Clean**: every row matches what a Mac downloads today, and the runtime pin is current.', '');
|
||||
}
|
||||
|
||||
for (const error of result.errors) lines.push(`- Check failed: ${oneLine(error)}`);
|
||||
if (result.errors.length > 0) lines.push('');
|
||||
|
||||
lines.push('## Models', '');
|
||||
lines.push(
|
||||
'What a Mac downloads is the build Ollama on Apple silicon selects: the `mlx` build when the tag offers ' +
|
||||
'one, else `llamacpp`, else `ggml`. Its size is the pull total, every layer plus the config. ' +
|
||||
`Tolerance: ${(SIZE_TOLERANCE_FRACTION * 100).toFixed(1)}% of the row's \`bytes\`.`,
|
||||
''
|
||||
);
|
||||
lines.push('| Model | Row `bytes` | A Mac downloads | Difference | Build on a Mac | Requires | Result |');
|
||||
lines.push('| --- | ---: | ---: | ---: | --- | --- | --- |');
|
||||
for (const model of result.models) {
|
||||
const mac = model.observed?.mac;
|
||||
const cells = [
|
||||
`\`${model.id}\``,
|
||||
groupDigits(model.bytes),
|
||||
mac ? groupDigits(mac.bytes) : '',
|
||||
mac ? formatPercent(sizeDifference(model.bytes, mac.bytes)) : '',
|
||||
mac ? buildName(mac) : model.observed?.gone ? 'tag gone' : '',
|
||||
mac?.requires ? `Ollama ${mac.requires}` : '',
|
||||
modelResult(model),
|
||||
];
|
||||
lines.push(`| ${cells.join(' | ')} |`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
const details = [];
|
||||
for (const model of result.models) {
|
||||
if (model.error) details.push(`- \`${model.id}\`: check failed: ${oneLine(model.error)}`);
|
||||
for (const finding of model.findings) details.push(`- ${describeModelFinding(model, finding)}`);
|
||||
const variants = model.observed?.variants ?? [];
|
||||
if (model.findings.some((finding) => finding.kind === 'size') && variants.length > 1) {
|
||||
const offered = variants
|
||||
.map((variant) => `${buildName(variant)} ${groupDigits(variant.bytes)}${variant === model.observed.mac ? ' (a Mac takes this one)' : ''}`)
|
||||
.join('; ');
|
||||
details.push(` The tag offers: ${offered}.`);
|
||||
}
|
||||
}
|
||||
if (details.length > 0) lines.push('### What changed', '', ...details, '');
|
||||
|
||||
lines.push('## Runtime', '');
|
||||
if (!result.runtime || result.runtime.error) {
|
||||
lines.push(`Check failed: ${oneLine(result.runtime?.error ?? 'not run')}`, '');
|
||||
} else {
|
||||
const { latest, pinned } = result.runtime.observed;
|
||||
const archive = result.runtime.assetName;
|
||||
lines.push(`| | Version | Published | \`${archive}\` |`);
|
||||
lines.push('| --- | --- | --- | --- |');
|
||||
lines.push(
|
||||
`| Pinned | ${result.pin.version} | ${pinned.publishedOn ?? ''} | ` +
|
||||
`${pinned.gone ? 'release gone' : assetCell(pinned.asset)} |`
|
||||
);
|
||||
lines.push(`| Newest release | ${latest.version} | ${latest.publishedOn ?? ''} | ${assetCell(latest.asset)} |`);
|
||||
lines.push('');
|
||||
if (result.runtime.findings.length === 0) {
|
||||
lines.push('The pin is the newest release, and its archive still matches the pin.', '');
|
||||
} else {
|
||||
for (const finding of result.runtime.findings) lines.push(`- ${describeRuntimeFinding(finding)}`);
|
||||
lines.push('');
|
||||
}
|
||||
lines.push(
|
||||
`Release notes: https://github.com/${RUNTIME_REPO}/releases/tag/v${latest.version}. ` +
|
||||
'A digest from the API is a lead, not a pin: check it against the release\'s `sha256sum.txt` first.',
|
||||
''
|
||||
);
|
||||
}
|
||||
|
||||
const unpinned = result.models.filter((model) => model.digest === undefined && model.observed?.mac);
|
||||
if (unpinned.length > 0) {
|
||||
lines.push('## Digests to pin', '');
|
||||
lines.push(
|
||||
'The catalogue pins no digest for these rows. Today\'s values: the tag\'s own manifest, and the manifest a ' +
|
||||
'Mac pulls, which is the digest `/api/tags` reports after the pull.',
|
||||
''
|
||||
);
|
||||
lines.push('| Model | Tag manifest | Manifest a Mac pulls |');
|
||||
lines.push('| --- | --- | --- |');
|
||||
for (const model of unpinned) {
|
||||
lines.push(`| \`${model.id}\` | \`${model.observed.tagDigest}\` | \`${model.observed.mac.digest}\` |`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
lines.push('## Not checked', '');
|
||||
lines.push(
|
||||
'The memory figures, the speeds and whether a coding agent works with a model: no manifest says anything ' +
|
||||
'about those. They change only when somebody measures on a Mac.',
|
||||
''
|
||||
);
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
// ---------- IO, with an injected fetch ----------
|
||||
|
||||
/**
|
||||
* One GET, bounded: `REQUEST_TIMEOUT_MS` for the whole exchange and `maxBytes` for the body.
|
||||
* Answers `{ status, body }` (body only for a 2xx). Never retries.
|
||||
*/
|
||||
export async function boundedGet(fetchFn, url, { headers = {}, maxBytes }) {
|
||||
const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
|
||||
let res;
|
||||
try {
|
||||
res = await fetchFn(url, { headers: { 'User-Agent': USER_AGENT, ...headers }, redirect: 'follow', signal });
|
||||
} catch (err) {
|
||||
throw new CheckFailure(`GET ${url} failed: ${describeError(err)}`);
|
||||
}
|
||||
if (res.status < 200 || res.status >= 300) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
return { status: res.status };
|
||||
}
|
||||
const declared = Number(res.headers.get('content-length'));
|
||||
if (Number.isFinite(declared) && declared > maxBytes) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new CheckFailure(`GET ${url}: ${declared} bytes is over the cap of ${maxBytes}`);
|
||||
}
|
||||
const chunks = [];
|
||||
let total = 0;
|
||||
try {
|
||||
if (res.body) {
|
||||
const reader = res.body.getReader();
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
total += value.byteLength;
|
||||
if (total > maxBytes) {
|
||||
await reader.cancel().catch(() => {});
|
||||
throw new CheckFailure(`GET ${url}: the body is over the cap of ${maxBytes} bytes`);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof CheckFailure) throw err;
|
||||
throw new CheckFailure(`GET ${url}: reading the body failed: ${describeError(err)}`);
|
||||
}
|
||||
return { status: res.status, body: Buffer.concat(chunks) };
|
||||
}
|
||||
|
||||
function describeError(err) {
|
||||
if (err instanceof Error) {
|
||||
const cause = err.cause instanceof Error && err.cause.message ? `: ${err.cause.message}` : '';
|
||||
return `${err.name === 'TimeoutError' ? 'timed out' : err.message}${cause}`;
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
|
||||
function parseJson(body, what) {
|
||||
try {
|
||||
return JSON.parse(body.toString('utf8'));
|
||||
} catch {
|
||||
throw new CheckFailure(`${what} is not JSON`);
|
||||
}
|
||||
}
|
||||
|
||||
/** A blob of the repository, checked against its digest. */
|
||||
async function getBlob(fetchFn, name, digest, maxBytes, what) {
|
||||
const url = `${REGISTRY_BASE}/v2/library/${encodeURIComponent(name)}/blobs/${digest}`;
|
||||
const answer = await boundedGet(fetchFn, url, { maxBytes });
|
||||
if (!answer.body) throw new CheckFailure(`${what} ${digest}: the registry answered ${answer.status}`);
|
||||
if (sha256Digest(answer.body) !== digest) throw new CheckFailure(`${what} ${digest}: the body does not match its digest`);
|
||||
return answer.body;
|
||||
}
|
||||
|
||||
/** One build of a tag: its runner, format, digest and pull total, all validated. */
|
||||
function variantOf(manifest, digest, fallback = {}) {
|
||||
return {
|
||||
runner: canonicalRunner(manifest.runner) || canonicalRunner(fallback.runner),
|
||||
format: canonicalRunner(manifest.format) || canonicalRunner(fallback.format),
|
||||
digest,
|
||||
bytes: pullTotalBytes(manifest),
|
||||
configDigest: normalizeDigest(manifest.config?.digest),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* What the registry serves for `id` today, read the way Ollama on a Mac reads it:
|
||||
* `{ gone: true }`, or `{ tagDigest, variants, mac }` where `mac` is the variant a Mac pulls
|
||||
* (with its config's `requires`). Throws `CheckFailure` when the answer is not usable.
|
||||
*/
|
||||
export async function inspectModel(fetchFn, id) {
|
||||
const { name, tag } = splitModelId(id);
|
||||
const url = `${REGISTRY_BASE}/v2/library/${encodeURIComponent(name)}/manifests/${encodeURIComponent(tag)}`;
|
||||
const answer = await boundedGet(fetchFn, url, { headers: { Accept: MANIFEST_ACCEPT }, maxBytes: MAX_MANIFEST_BYTES });
|
||||
if (answer.status === 404) return { gone: true };
|
||||
if (!answer.body) throw new CheckFailure(`the registry answered ${answer.status} for the manifest`);
|
||||
const tagDigest = sha256Digest(answer.body);
|
||||
const manifest = parseJson(answer.body, 'the manifest');
|
||||
|
||||
let variants;
|
||||
let mac;
|
||||
if (manifest.mediaType === MEDIA_TYPE_MANIFEST_LIST) {
|
||||
const children = Array.isArray(manifest.manifests) ? manifest.manifests : [];
|
||||
if (children.length === 0 || children.length > MAX_LIST_CHILDREN) {
|
||||
throw new CheckFailure(`the manifest list has ${children.length} children`);
|
||||
}
|
||||
const selected = selectMacChild(children);
|
||||
if (!selected) throw new CheckFailure('the manifest list has no build a Mac runs (mlx, llamacpp or ggml)');
|
||||
variants = [];
|
||||
for (const child of children) {
|
||||
const digest = normalizeDigest(child?.digest);
|
||||
if (!digest || !child.digest.startsWith('sha256:')) throw new CheckFailure('a child of the manifest list has no sha256 digest');
|
||||
const body = await getBlob(fetchFn, name, digest, MAX_MANIFEST_BYTES, 'child manifest');
|
||||
const variant = variantOf(parseJson(body, `child manifest ${digest}`), digest, child);
|
||||
variants.push(variant);
|
||||
if (child === selected) mac = variant;
|
||||
}
|
||||
} else if (manifest.mediaType === MEDIA_TYPE_MANIFEST || manifest.mediaType === undefined) {
|
||||
mac = variantOf(manifest, tagDigest);
|
||||
variants = [mac];
|
||||
} else {
|
||||
throw new CheckFailure('the manifest has a media type this check does not know');
|
||||
}
|
||||
|
||||
if (!mac.configDigest) throw new CheckFailure('the build a Mac pulls has no config digest');
|
||||
const config = parseJson(await getBlob(fetchFn, name, mac.configDigest, MAX_CONFIG_BYTES, 'config'), 'the config');
|
||||
// A manifest that names no runner or format gets them from its config, as Ollama fills them.
|
||||
const assumed = metadataForConfig(config);
|
||||
mac.runner ||= assumed.runner;
|
||||
mac.format ||= assumed.format;
|
||||
// A malformed `requires` is ignored, as Ollama ignores it (`checkPullRequires`).
|
||||
const requires = parseVersion(config?.requires);
|
||||
mac.requires = requires ? requires.join('.') : undefined;
|
||||
return { tagDigest, variants, mac };
|
||||
}
|
||||
|
||||
function githubHeaders(token) {
|
||||
return {
|
||||
Accept: 'application/vnd.github+json',
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** The validated parts of a GitHub release: version, day it was published, the named asset. */
|
||||
function releaseFacts(release, assetName) {
|
||||
if (!release || typeof release.tag_name !== 'string' || !RELEASE_TAG_PATTERN.test(release.tag_name)) {
|
||||
throw new CheckFailure('the release has no vX.Y.Z tag');
|
||||
}
|
||||
const day = typeof release.published_at === 'string' ? ISO_DAY_PATTERN.exec(release.published_at) : null;
|
||||
const raw = Array.isArray(release.assets) ? release.assets.find((asset) => asset?.name === assetName) : undefined;
|
||||
let asset;
|
||||
if (raw) {
|
||||
if (!isSize(raw.size)) throw new CheckFailure(`the release asset ${assetName} has no size`);
|
||||
asset = { size: raw.size, digest: normalizeDigest(raw.digest) };
|
||||
}
|
||||
return { version: release.tag_name.slice(1), publishedOn: day ? day[1] : undefined, asset };
|
||||
}
|
||||
|
||||
/** The newest release and the pinned one, from the GitHub API. Throws `CheckFailure`. */
|
||||
export async function inspectRuntime(fetchFn, pin, token) {
|
||||
const assetName = new URL(pin.url).pathname.split('/').pop();
|
||||
const headers = githubHeaders(token);
|
||||
const latestAnswer = await boundedGet(fetchFn, `${GITHUB_API_BASE}/repos/${RUNTIME_REPO}/releases/latest`, {
|
||||
headers,
|
||||
maxBytes: MAX_GITHUB_BYTES,
|
||||
});
|
||||
if (!latestAnswer.body) throw new CheckFailure(`GitHub answered ${latestAnswer.status} for the newest release`);
|
||||
const latest = releaseFacts(parseJson(latestAnswer.body, 'the newest release'), assetName);
|
||||
|
||||
const pinnedAnswer = await boundedGet(
|
||||
fetchFn,
|
||||
`${GITHUB_API_BASE}/repos/${RUNTIME_REPO}/releases/tags/v${encodeURIComponent(pin.version)}`,
|
||||
{ headers, maxBytes: MAX_GITHUB_BYTES }
|
||||
);
|
||||
let pinned;
|
||||
if (pinnedAnswer.status === 404) pinned = { gone: true };
|
||||
else if (!pinnedAnswer.body) throw new CheckFailure(`GitHub answered ${pinnedAnswer.status} for the pinned release`);
|
||||
else pinned = releaseFacts(parseJson(pinnedAnswer.body, 'the pinned release'), assetName);
|
||||
return { assetName, observed: { latest, pinned } };
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole check. `catalog` is the validated catalogue, `pin` the runtime pin, `fetchFn` the
|
||||
* fetch to use. Never throws for a network problem: each failure lands in the result, and
|
||||
* `exitCodeFor()` turns it into 2.
|
||||
*/
|
||||
export async function runCheck({ catalog, pin, fetchFn, now, githubToken }) {
|
||||
const result = {
|
||||
checkedAt: `${now.toISOString().slice(0, 16).replace('T', ' ')} UTC`,
|
||||
catalogue: { updated: catalog.updated },
|
||||
pin: { version: pin.version },
|
||||
models: [],
|
||||
runtime: undefined,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
// Rows in parallel, each row's requests one after another: a handful of rows, a few requests each.
|
||||
result.models = await Promise.all(
|
||||
catalog.models.map(async (row) => {
|
||||
const digest = /** @type {{ digest?: unknown }} */ (row).digest;
|
||||
const entry = { id: row.id, bytes: row.bytes, digest, findings: [] };
|
||||
try {
|
||||
entry.observed = await inspectModel(fetchFn, row.id);
|
||||
entry.findings = classifyModel({ id: row.id, bytes: row.bytes, digest }, entry.observed, pin.version);
|
||||
} catch (err) {
|
||||
entry.error = err instanceof CheckFailure ? err.message : describeError(err);
|
||||
}
|
||||
return entry;
|
||||
})
|
||||
);
|
||||
|
||||
try {
|
||||
const runtime = await inspectRuntime(fetchFn, pin, githubToken);
|
||||
result.runtime = { ...runtime, findings: classifyRuntime(pin, runtime.observed) };
|
||||
} catch (err) {
|
||||
result.runtime = { error: err instanceof CheckFailure ? err.message : describeError(err), findings: [] };
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---------- Main ----------
|
||||
|
||||
async function main() {
|
||||
let catalog;
|
||||
let pin;
|
||||
try {
|
||||
const { getLocalModelCatalog } = await import('../src/config/local-model-catalog.js');
|
||||
const { OLLAMA_RUNTIME_PIN } = await import('../src/local-model/runtime.js');
|
||||
catalog = getLocalModelCatalog();
|
||||
pin = OLLAMA_RUNTIME_PIN;
|
||||
} catch (err) {
|
||||
console.error(`The catalogue or the runtime pin could not be loaded: ${describeError(err)}`);
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
const result = await runCheck({
|
||||
catalog,
|
||||
pin,
|
||||
fetchFn: globalThis.fetch,
|
||||
now: new Date(),
|
||||
githubToken: process.env.GITHUB_TOKEN || undefined,
|
||||
});
|
||||
process.stdout.write(renderReport(result));
|
||||
process.exitCode = exitCodeFor(result);
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
main().catch((err) => {
|
||||
console.error(`The check failed: ${describeError(err)}`);
|
||||
process.exitCode = 2;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,660 @@
|
||||
/**
|
||||
* @fileoverview scripts/check-local-model-catalog.mjs, the weekly check of the local model
|
||||
* catalogue: which build a Mac pulls from a manifest list, the pull total, how a row's drift is
|
||||
* classified, the exit code, the report text, and that the IO side only ever asks for
|
||||
* manifests, child manifests and configs (never a layer). The network is never used: the IO
|
||||
* tests drive a fake registry and a fake GitHub API.
|
||||
*
|
||||
* The fixtures under test/fixtures/local-model-catalog/ are real bodies the registry served on
|
||||
* 2026-11-08 for `qwen3.6:35b-a3b`, byte for byte, so their digests are the real ones.
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
CheckFailure,
|
||||
MANIFEST_ACCEPT,
|
||||
MAX_MANIFEST_BYTES,
|
||||
MEDIA_TYPE_MANIFEST,
|
||||
MEDIA_TYPE_MANIFEST_LIST,
|
||||
SIZE_TOLERANCE_FRACTION,
|
||||
boundedGet,
|
||||
classifyModel,
|
||||
classifyRuntime,
|
||||
compareVersions,
|
||||
exitCodeFor,
|
||||
formatPercent,
|
||||
groupDigits,
|
||||
inspectModel,
|
||||
metadataForConfig,
|
||||
normalizeDigest,
|
||||
parseVersion,
|
||||
pullTotalBytes,
|
||||
renderReport,
|
||||
runCheck,
|
||||
selectMacChild,
|
||||
sha256Digest,
|
||||
splitModelId,
|
||||
withinTolerance,
|
||||
} from '../scripts/check-local-model-catalog.mjs';
|
||||
|
||||
const fixtureDir = resolve(import.meta.dirname, 'fixtures/local-model-catalog');
|
||||
const fixture = (name: string): Buffer => readFileSync(resolve(fixtureDir, name));
|
||||
|
||||
const REAL_LIST = fixture('qwen3.6-35b-a3b.list.json');
|
||||
const REAL_GGUF_MANIFEST = fixture('qwen3.6-35b-a3b.gguf-manifest.json');
|
||||
const REAL_MLX_CONFIG = fixture('qwen3.6-35b-a3b.mlx-config.json');
|
||||
|
||||
/** The digest of the MLX child the real list names, which a Mac pulled on 2026-11-06. */
|
||||
const REAL_MLX_CHILD = 'sha256:0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba';
|
||||
const REAL_GGUF_CHILD = 'sha256:a7eb95c53bcf96b4bdd008d0fab4a5dac88047d9c1a7a9ab88ed453423fbd87c';
|
||||
/** The pull total a Mac reported for `qwen3.6:35b-a3b`, which the catalogue row stores. */
|
||||
const MAC_PULL_TOTAL_35B = 23_105_798_956;
|
||||
|
||||
const PIN = {
|
||||
version: '0.40.15',
|
||||
url: 'https://github.com/ollama/ollama/releases/download/v0.40.15/ollama-darwin.tgz',
|
||||
bytes: 185_971_671,
|
||||
sha256: '607c79d5d6315ef9d896db83adde736a4c0057196329843544e79b95afb260a8',
|
||||
};
|
||||
|
||||
const hex = (char: string): string => `sha256:${char.repeat(64)}`;
|
||||
|
||||
describe('the real registry bodies', () => {
|
||||
it('carry the digests the registry and a Mac reported for them', () => {
|
||||
expect(sha256Digest(REAL_LIST)).toBe('sha256:86b2d171ce90bbb2dcb081933565ec867ce523cdd274bd24c8cf1ef9185feb4f');
|
||||
expect(sha256Digest(REAL_GGUF_MANIFEST)).toBe(REAL_GGUF_CHILD);
|
||||
expect(sha256Digest(REAL_MLX_CONFIG)).toBe(
|
||||
'sha256:577d7d4694896a245c8b00f62bf53a8f4720ded4eba83a8e2da30bb87583bd6b'
|
||||
);
|
||||
});
|
||||
|
||||
it('a Mac selects the mlx child of the real list, the one whose digest /api/tags reported', () => {
|
||||
const list = JSON.parse(REAL_LIST.toString('utf8'));
|
||||
expect(list.mediaType).toBe(MEDIA_TYPE_MANIFEST_LIST);
|
||||
expect(selectMacChild(list.manifests).digest).toBe(REAL_MLX_CHILD);
|
||||
});
|
||||
|
||||
it('the plain GGUF manifest totals its layers plus its config, 2% below what a Mac pulls', () => {
|
||||
const manifest = JSON.parse(REAL_GGUF_MANIFEST.toString('utf8'));
|
||||
expect(pullTotalBytes(manifest)).toBe(22_621_314_161 + 220);
|
||||
expect(classifyModel(
|
||||
{ id: 'qwen3.6:35b-a3b', bytes: MAC_PULL_TOTAL_35B },
|
||||
{ mac: { bytes: pullTotalBytes(manifest), digest: REAL_GGUF_CHILD } },
|
||||
'0.40.15'
|
||||
)).toEqual([expect.objectContaining({ kind: 'size' })]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('splitModelId', () => {
|
||||
it('splits name and tag', () => {
|
||||
expect(splitModelId('qwen3.6:35b-a3b')).toEqual({ name: 'qwen3.6', tag: '35b-a3b' });
|
||||
});
|
||||
|
||||
it.each(['qwen3.6', ':35b', 'qwen3.6:', 'evil.example/x:y'])('refuses %s', (id) => {
|
||||
expect(() => splitModelId(id)).toThrow(CheckFailure);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pullTotalBytes', () => {
|
||||
const manifest = (layers: unknown, config: unknown = { size: 250 }) => ({ config, layers });
|
||||
|
||||
it('adds every layer and the config', () => {
|
||||
expect(pullTotalBytes(manifest([{ size: 1000 }, { size: 24 }]))).toBe(1274);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['no layers', manifest([])],
|
||||
['layers that are not an array', manifest('x')],
|
||||
['a layer without a size', manifest([{ digest: 'x' }])],
|
||||
['a negative size', manifest([{ size: -1 }])],
|
||||
['a size given as text', manifest([{ size: '1000' }])],
|
||||
['a fractional size', manifest([{ size: 1.5 }])],
|
||||
['no config size', manifest([{ size: 1 }], {})],
|
||||
['a total past the safe integers', manifest([{ size: Number.MAX_SAFE_INTEGER }, { size: 10 }])],
|
||||
])('refuses %s', (_label, value) => {
|
||||
expect(() => pullTotalBytes(value)).toThrow(CheckFailure);
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectMacChild', () => {
|
||||
const child = (runner: string, digest: string, mediaType?: string) => ({
|
||||
...(mediaType === undefined ? { mediaType: MEDIA_TYPE_MANIFEST } : { mediaType }),
|
||||
digest,
|
||||
runner,
|
||||
});
|
||||
|
||||
it('prefers mlx, then llamacpp, then ggml, whatever the list order', () => {
|
||||
expect(selectMacChild([child('ggml', 'g'), child('llamacpp', 'l'), child('mlx', 'm')]).digest).toBe('m');
|
||||
expect(selectMacChild([child('ggml', 'g'), child('llamacpp', 'l')]).digest).toBe('l');
|
||||
expect(selectMacChild([child('ggml', 'g')]).digest).toBe('g');
|
||||
});
|
||||
|
||||
it('takes the first child of a runner and compares runners as Ollama does', () => {
|
||||
expect(selectMacChild([child(' MLX ', 'first'), child('mlx', 'second')]).digest).toBe('first');
|
||||
});
|
||||
|
||||
it('skips a nested list and a runner a Mac does not run', () => {
|
||||
expect(selectMacChild([child('mlx', 'nested', MEDIA_TYPE_MANIFEST_LIST), child('llamacpp', 'l')]).digest).toBe('l');
|
||||
expect(selectMacChild([child('vllm', 'v')])).toBeUndefined();
|
||||
expect(selectMacChild('not a list')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('accepts a child without a mediaType', () => {
|
||||
expect(selectMacChild([{ digest: 'm', runner: 'mlx' }]).digest).toBe('m');
|
||||
});
|
||||
});
|
||||
|
||||
describe('small helpers', () => {
|
||||
it('metadataForConfig mirrors Ollama: safetensors runs on mlx, gguf on ggml', () => {
|
||||
expect(metadataForConfig({ model_format: 'safetensors' })).toEqual({ runner: 'mlx', format: 'safetensors' });
|
||||
expect(metadataForConfig({ model_format: 'gguf' })).toEqual({ runner: 'ggml', format: 'gguf' });
|
||||
expect(metadataForConfig({})).toEqual({ runner: '', format: '' });
|
||||
});
|
||||
|
||||
it('normalizeDigest accepts the prefixed and the bare form and nothing else', () => {
|
||||
const bare = '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba';
|
||||
expect(normalizeDigest(bare)).toBe(REAL_MLX_CHILD);
|
||||
expect(normalizeDigest(REAL_MLX_CHILD.toUpperCase().replace('SHA256', 'sha256'))).toBe(REAL_MLX_CHILD);
|
||||
expect(normalizeDigest('sha256:abc')).toBeUndefined();
|
||||
expect(normalizeDigest('sha512:' + 'a'.repeat(64))).toBeUndefined();
|
||||
expect(normalizeDigest(42)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('parses and compares versions', () => {
|
||||
expect(parseVersion('v0.40.15')).toEqual([0, 40, 15]);
|
||||
expect(parseVersion('0.32.12')).toEqual([0, 32, 12]);
|
||||
expect(parseVersion('0.40.15-rc1')).toBeUndefined();
|
||||
expect(compareVersions('0.40.15', 'v0.40.15')).toBe(0);
|
||||
expect(compareVersions('0.41.0', '0.40.15')).toBeGreaterThan(0);
|
||||
expect(compareVersions('0.9.0', '0.40.15')).toBeLessThan(0);
|
||||
expect(() => compareVersions('latest', '0.40.15')).toThrow(CheckFailure);
|
||||
});
|
||||
|
||||
it('formats numbers without the locale', () => {
|
||||
expect(groupDigits(23_105_798_956)).toBe('23,105,798,956');
|
||||
expect(groupDigits(250)).toBe('250');
|
||||
expect(formatPercent(0.16614)).toBe('+16.61%');
|
||||
expect(formatPercent(-0.02097)).toBe('-2.10%');
|
||||
});
|
||||
|
||||
it('the tolerance is a small fraction of the row', () => {
|
||||
expect(SIZE_TOLERANCE_FRACTION).toBeLessThanOrEqual(0.01);
|
||||
expect(withinTolerance(1_000_000, 1_001_000)).toBe(true);
|
||||
expect(withinTolerance(1_000_000, 1_001_001)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyModel', () => {
|
||||
const observed = (bytes: number, extra: Record<string, unknown> = {}) => ({
|
||||
mac: { bytes, digest: REAL_MLX_CHILD, runner: 'mlx', format: 'safetensors', ...extra },
|
||||
});
|
||||
|
||||
it('a row that holds the pull total itself is clean', () => {
|
||||
expect(classifyModel({ id: 'qwen3.6:35b-a3b', bytes: MAC_PULL_TOTAL_35B }, observed(MAC_PULL_TOTAL_35B), '0.40.15')).toEqual([]);
|
||||
});
|
||||
|
||||
it('a row sized from its layers alone is clean: the config is a few hundred bytes', () => {
|
||||
expect(classifyModel({ id: 'gpt-oss:20b', bytes: 13_793_441_427 }, observed(13_793_441_566), '0.40.15')).toEqual([]);
|
||||
});
|
||||
|
||||
it('flags the two real drifts of 2026-11-08: the Mac gets the mlx build', () => {
|
||||
const nine = classifyModel({ id: 'qwen3.5:9b', bytes: 6_550_825_373 }, observed(7_639_229_431), '0.40.15');
|
||||
expect(nine).toEqual([expect.objectContaining({ kind: 'size', expected: 6_550_825_373, actual: 7_639_229_431 })]);
|
||||
expect(nine[0].fraction).toBeCloseTo(0.1661, 3);
|
||||
const dense = classifyModel({ id: 'qwen3.8:27b', bytes: 17_741_871_939 }, observed(18_174_721_847), '0.40.15');
|
||||
expect(dense[0].kind).toBe('size');
|
||||
expect(dense[0].fraction).toBeCloseTo(0.0244, 3);
|
||||
});
|
||||
|
||||
it('a tag that is gone is only that', () => {
|
||||
expect(classifyModel({ id: 'x:y', bytes: 1 }, { gone: true }, '0.40.15')).toEqual([{ kind: 'gone' }]);
|
||||
});
|
||||
|
||||
it('compares a pinned digest in either form', () => {
|
||||
const bare = REAL_MLX_CHILD.slice('sha256:'.length);
|
||||
expect(classifyModel({ id: 'a:b', bytes: 100, digest: bare }, observed(100), '0.40.15')).toEqual([]);
|
||||
expect(classifyModel({ id: 'a:b', bytes: 100, digest: hex('1') }, observed(100), '0.40.15')).toEqual([
|
||||
{ kind: 'digest', expected: hex('1'), actual: REAL_MLX_CHILD },
|
||||
]);
|
||||
expect(classifyModel({ id: 'a:b', bytes: 100, digest: 'garbage' }, observed(100), '0.40.15')[0]).toMatchObject({
|
||||
kind: 'digest',
|
||||
expected: 'not a sha256 digest',
|
||||
});
|
||||
});
|
||||
|
||||
it('flags a build that needs a newer runtime than the pin, and only that', () => {
|
||||
expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { requires: '0.41.0' }), '0.40.15')).toEqual([
|
||||
{ kind: 'requires', requires: '0.41.0', pinned: '0.40.15' },
|
||||
]);
|
||||
expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { requires: '0.40.15' }), '0.40.15')).toEqual([]);
|
||||
expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100), '0.40.15')).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyRuntime', () => {
|
||||
const release = (version: string, asset?: { size: number; digest?: string }) => ({ version, publishedOn: '2026-11-03', asset });
|
||||
const matching = { size: PIN.bytes, digest: `sha256:${PIN.sha256}` };
|
||||
|
||||
it('is clean when the pin is the newest release and its archive is unchanged', () => {
|
||||
expect(classifyRuntime(PIN, { latest: release('0.40.15', matching), pinned: release('0.40.15', matching) })).toEqual([]);
|
||||
});
|
||||
|
||||
it('a size that matches and no digest in the API is not drift', () => {
|
||||
expect(
|
||||
classifyRuntime(PIN, { latest: release('0.40.15'), pinned: release('0.40.15', { size: PIN.bytes }) })
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('reports a newer release', () => {
|
||||
expect(classifyRuntime(PIN, { latest: release('0.41.2'), pinned: release('0.40.15', matching) })).toEqual([
|
||||
{ kind: 'behind', pinned: '0.40.15', latest: '0.41.2' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('reports a pinned release or archive that is gone or changed', () => {
|
||||
const latest = release('0.40.15');
|
||||
expect(classifyRuntime(PIN, { latest, pinned: { gone: true } })).toEqual([{ kind: 'pinned-release-gone' }]);
|
||||
expect(classifyRuntime(PIN, { latest, pinned: release('0.40.15') })).toEqual([{ kind: 'pinned-asset-gone' }]);
|
||||
expect(
|
||||
classifyRuntime(PIN, { latest, pinned: release('0.40.15', { size: PIN.bytes + 1, digest: matching.digest }) })[0].kind
|
||||
).toBe('pinned-asset-changed');
|
||||
expect(
|
||||
classifyRuntime(PIN, { latest, pinned: release('0.40.15', { size: PIN.bytes, digest: hex('2') }) })[0].kind
|
||||
).toBe('pinned-asset-changed');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- Exit code and report ----------
|
||||
|
||||
function baseResult(overrides: Record<string, unknown> = {}) {
|
||||
const mac = { bytes: 100, digest: hex('a'), runner: 'mlx', format: 'safetensors', requires: '0.32.4' };
|
||||
return {
|
||||
checkedAt: '2026-11-09 06:00 UTC',
|
||||
catalogue: { updated: '2026-11-06' },
|
||||
pin: { version: '0.40.15' },
|
||||
errors: [] as string[],
|
||||
models: [{ id: 'a:b', bytes: 100, findings: [] as unknown[], observed: { tagDigest: hex('b'), variants: [mac], mac } }],
|
||||
runtime: {
|
||||
assetName: 'ollama-darwin.tgz',
|
||||
findings: [] as unknown[],
|
||||
observed: {
|
||||
latest: { version: '0.40.15', publishedOn: '2026-11-03', asset: { size: PIN.bytes, digest: `sha256:${PIN.sha256}` } },
|
||||
pinned: { version: '0.40.15', publishedOn: '2026-11-03', asset: { size: PIN.bytes, digest: `sha256:${PIN.sha256}` } },
|
||||
},
|
||||
} as Record<string, unknown>,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('exitCodeFor', () => {
|
||||
it('is 0 when nothing drifted', () => {
|
||||
expect(exitCodeFor(baseResult())).toBe(0);
|
||||
});
|
||||
|
||||
it('is 1 for drift in a model or in the runtime', () => {
|
||||
const result = baseResult();
|
||||
result.models[0].findings = [{ kind: 'gone' }];
|
||||
expect(exitCodeFor(result)).toBe(1);
|
||||
const runtime = baseResult();
|
||||
(runtime.runtime.findings as unknown[]).push({ kind: 'behind', pinned: '0.40.15', latest: '0.41.0' });
|
||||
expect(exitCodeFor(runtime)).toBe(1);
|
||||
});
|
||||
|
||||
it('is 2 when any part failed, even with drift elsewhere', () => {
|
||||
const result = baseResult();
|
||||
result.models[0].findings = [{ kind: 'gone' }];
|
||||
result.models.push({ id: 'c:d', bytes: 1, findings: [], error: 'timed out' } as never);
|
||||
expect(exitCodeFor(result)).toBe(2);
|
||||
expect(exitCodeFor(baseResult({ runtime: { error: 'GitHub answered 403', findings: [] } }))).toBe(2);
|
||||
expect(exitCodeFor(baseResult({ errors: ['the catalogue did not load'] }))).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderReport', () => {
|
||||
it('says clean, and lists the digests to pin in full', () => {
|
||||
const report = renderReport(baseResult());
|
||||
expect(report).toMatch(/^# Local model catalogue: weekly check\n/);
|
||||
expect(report).toContain('**Clean**');
|
||||
expect(report).toContain(`| \`a:b\` | \`${hex('b')}\` | \`${hex('a')}\` |`);
|
||||
expect(report).toContain('## Not checked');
|
||||
});
|
||||
|
||||
it('names a size drift with both figures and every build the tag offers', () => {
|
||||
const gguf = { bytes: 90, digest: hex('c'), runner: 'llamacpp', format: 'gguf' };
|
||||
const mac = { bytes: 7_639_229_431, digest: hex('a'), runner: 'mlx', format: 'safetensors' };
|
||||
const result = baseResult({
|
||||
models: [
|
||||
{
|
||||
id: 'qwen3.5:9b',
|
||||
bytes: 6_550_825_373,
|
||||
findings: classifyModel({ id: 'qwen3.5:9b', bytes: 6_550_825_373 }, { mac }, '0.40.15'),
|
||||
observed: { tagDigest: hex('b'), variants: [gguf, mac], mac },
|
||||
},
|
||||
],
|
||||
});
|
||||
const report = renderReport(result);
|
||||
expect(report).toContain('**Drift found**: 1 of 1 models.');
|
||||
expect(report).toContain('| `qwen3.5:9b` | 6,550,825,373 | 7,639,229,431 | +16.61% | mlx, safetensors |');
|
||||
expect(report).toContain('mlx, safetensors 7,639,229,431 (a Mac takes this one)');
|
||||
expect(report).toContain('llamacpp, gguf 90');
|
||||
});
|
||||
|
||||
it('says the check failed and prints no verdict of clean', () => {
|
||||
const result = baseResult({ runtime: { error: 'GitHub answered 403 for the newest release', findings: [] } });
|
||||
const report = renderReport(result);
|
||||
expect(report).toContain('**The check failed**');
|
||||
expect(report).not.toContain('**Clean**');
|
||||
expect(report).toContain('GitHub answered 403');
|
||||
});
|
||||
|
||||
it('never prints a runner or format outside the known names', () => {
|
||||
const mac = { bytes: 100, digest: hex('a'), runner: '<img src=x>', format: '@someone' };
|
||||
const report = renderReport(baseResult({ models: [{ id: 'a:b', bytes: 100, findings: [], observed: { tagDigest: hex('b'), variants: [mac], mac } }] }));
|
||||
expect(report).toContain('unknown, unknown');
|
||||
expect(report).not.toContain('<img');
|
||||
expect(report).not.toContain('@someone');
|
||||
});
|
||||
|
||||
it('keeps an error on one line and without backticks', () => {
|
||||
const result = baseResult({ errors: ['line one\nline `two`\u0007'] });
|
||||
expect(renderReport(result)).toContain("- Check failed: line one line 'two'\n");
|
||||
});
|
||||
|
||||
it('contains no em-dash', () => {
|
||||
expect(renderReport(baseResult())).not.toContain(String.fromCharCode(0x2014));
|
||||
});
|
||||
});
|
||||
|
||||
// ---------- IO against a fake registry ----------
|
||||
|
||||
interface Seen {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
signal: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* A fake registry and GitHub API. `routes` maps a URL to a body (string or Buffer) or a status
|
||||
* number. Every request is recorded.
|
||||
*/
|
||||
function fakeFetch(routes: Record<string, string | Buffer | number | (() => Response)>) {
|
||||
const seen: Seen[] = [];
|
||||
const fetchFn = async (url: string, init: { headers: Record<string, string>; signal: unknown }) => {
|
||||
seen.push({ url, headers: init.headers, signal: init.signal });
|
||||
const route = routes[url];
|
||||
if (route === undefined) return new Response('not found', { status: 404 });
|
||||
if (typeof route === 'number') return new Response('', { status: route });
|
||||
if (typeof route === 'function') return route();
|
||||
return new Response(route, { status: 200 });
|
||||
};
|
||||
return { fetchFn, seen };
|
||||
}
|
||||
|
||||
const REG = 'https://registry.ollama.ai/v2/library';
|
||||
const GH = 'https://api.github.com/repos/ollama/ollama/releases';
|
||||
|
||||
/** A small registry: one tag that is a list (mlx + llamacpp), one plain tag. Digests computed. */
|
||||
function buildRegistry() {
|
||||
const json = (value: unknown): Buffer => Buffer.from(JSON.stringify(value));
|
||||
const mlxConfig = json({ model_format: 'safetensors', requires: '0.32.4' });
|
||||
const ggufConfig = json({ model_format: 'gguf', requires: '0.30.0' });
|
||||
const layers = (sizes: number[], tag: string) =>
|
||||
sizes.map((size, i) => ({ mediaType: 'application/vnd.ollama.image.tensor', digest: hex(String((i + tag.length) % 10)), size }));
|
||||
const mlxChild = json({
|
||||
schemaVersion: 2,
|
||||
mediaType: MEDIA_TYPE_MANIFEST,
|
||||
config: { digest: sha256Digest(mlxConfig), size: mlxConfig.length },
|
||||
layers: layers([700, 300], 'mlx'),
|
||||
});
|
||||
const ggufChild = json({
|
||||
schemaVersion: 2,
|
||||
mediaType: MEDIA_TYPE_MANIFEST,
|
||||
config: { digest: sha256Digest(ggufConfig), size: ggufConfig.length },
|
||||
layers: layers([600], 'gguf'),
|
||||
runner: 'llamacpp',
|
||||
format: 'gguf',
|
||||
});
|
||||
const list = json({
|
||||
schemaVersion: 2,
|
||||
mediaType: MEDIA_TYPE_MANIFEST_LIST,
|
||||
manifests: [
|
||||
{ mediaType: MEDIA_TYPE_MANIFEST, digest: sha256Digest(mlxChild), runner: 'mlx', format: 'safetensors' },
|
||||
{ mediaType: MEDIA_TYPE_MANIFEST, digest: sha256Digest(ggufChild), runner: 'llamacpp', format: 'gguf' },
|
||||
],
|
||||
});
|
||||
const plain = json({
|
||||
schemaVersion: 2,
|
||||
mediaType: MEDIA_TYPE_MANIFEST,
|
||||
config: { digest: sha256Digest(ggufConfig), size: ggufConfig.length },
|
||||
layers: layers([500], 'plain'),
|
||||
});
|
||||
const release = (tag: string) =>
|
||||
json({
|
||||
tag_name: tag,
|
||||
published_at: '2026-11-03T22:52:31Z',
|
||||
assets: [{ name: 'ollama-darwin.tgz', size: PIN.bytes, digest: `sha256:${PIN.sha256}` }],
|
||||
});
|
||||
const routes: Record<string, string | Buffer | number | (() => Response)> = {
|
||||
[`${REG}/listed/manifests/1b`]: list,
|
||||
[`${REG}/listed/blobs/${sha256Digest(mlxChild)}`]: mlxChild,
|
||||
[`${REG}/listed/blobs/${sha256Digest(ggufChild)}`]: ggufChild,
|
||||
[`${REG}/listed/blobs/${sha256Digest(mlxConfig)}`]: mlxConfig,
|
||||
[`${REG}/plain/manifests/2b`]: plain,
|
||||
[`${REG}/plain/blobs/${sha256Digest(ggufConfig)}`]: ggufConfig,
|
||||
[`${GH}/latest`]: release('v0.40.15'),
|
||||
[`${GH}/tags/v0.40.15`]: release('v0.40.15'),
|
||||
};
|
||||
const layerDigests = new Set(
|
||||
[mlxChild, ggufChild, plain].flatMap((body) => JSON.parse(body.toString()).layers.map((l: { digest: string }) => l.digest))
|
||||
);
|
||||
return {
|
||||
routes,
|
||||
layerDigests,
|
||||
sizes: { mlx: 1000 + mlxConfig.length, gguf: 600 + ggufConfig.length, plain: 500 + ggufConfig.length },
|
||||
digests: { list: sha256Digest(list), mlxChild: sha256Digest(mlxChild), plain: sha256Digest(plain) },
|
||||
};
|
||||
}
|
||||
|
||||
const catalogOf = (models: { id: string; bytes: number; digest?: string }[]) => ({ updated: '2026-11-06', models });
|
||||
const NOW = new Date('2026-11-09T06:00:00Z');
|
||||
|
||||
describe('inspectModel', () => {
|
||||
it('reads a list the way a Mac does: the mlx child, its total, its requires', async () => {
|
||||
const registry = buildRegistry();
|
||||
const { fetchFn, seen } = fakeFetch(registry.routes);
|
||||
const observed = await inspectModel(fetchFn, 'listed:1b');
|
||||
expect(observed.tagDigest).toBe(registry.digests.list);
|
||||
expect(observed.mac).toMatchObject({
|
||||
runner: 'mlx',
|
||||
format: 'safetensors',
|
||||
digest: registry.digests.mlxChild,
|
||||
bytes: registry.sizes.mlx,
|
||||
requires: '0.32.4',
|
||||
});
|
||||
expect(observed.variants.map((variant: { runner: string }) => variant.runner)).toEqual(['mlx', 'llamacpp']);
|
||||
expect(seen[0].headers.Accept).toBe(MANIFEST_ACCEPT);
|
||||
expect(seen[0].headers.Accept).toContain(MEDIA_TYPE_MANIFEST_LIST);
|
||||
expect(seen.every((request) => request.signal instanceof AbortSignal)).toBe(true);
|
||||
});
|
||||
|
||||
it('fills a plain manifest without runner metadata from its config', async () => {
|
||||
const registry = buildRegistry();
|
||||
const { fetchFn } = fakeFetch(registry.routes);
|
||||
const observed = await inspectModel(fetchFn, 'plain:2b');
|
||||
expect(observed.mac).toMatchObject({ runner: 'ggml', format: 'gguf', digest: registry.digests.plain, bytes: registry.sizes.plain });
|
||||
expect(observed.tagDigest).toBe(observed.mac.digest);
|
||||
});
|
||||
|
||||
it('a 404 for the manifest is a tag that is gone', async () => {
|
||||
const { fetchFn } = fakeFetch({});
|
||||
expect(await inspectModel(fetchFn, 'missing:tag')).toEqual({ gone: true });
|
||||
});
|
||||
|
||||
it('refuses a blob whose body does not match its digest', async () => {
|
||||
const registry = buildRegistry();
|
||||
const mlxUrl = `${REG}/listed/blobs/${registry.digests.mlxChild}`;
|
||||
const { fetchFn } = fakeFetch({ ...registry.routes, [mlxUrl]: '{"tampered":true}' });
|
||||
await expect(inspectModel(fetchFn, 'listed:1b')).rejects.toThrow(/does not match its digest/);
|
||||
});
|
||||
|
||||
it('refuses a list with no build a Mac runs', async () => {
|
||||
const list = JSON.stringify({ mediaType: MEDIA_TYPE_MANIFEST_LIST, manifests: [{ digest: hex('1'), runner: 'vllm' }] });
|
||||
const { fetchFn } = fakeFetch({ [`${REG}/odd/manifests/x`]: list });
|
||||
await expect(inspectModel(fetchFn, 'odd:x')).rejects.toThrow(/no build a Mac runs/);
|
||||
});
|
||||
|
||||
it('a server error is a failed check, not a tag that is gone', async () => {
|
||||
const { fetchFn } = fakeFetch({ [`${REG}/flaky/manifests/x`]: 503 });
|
||||
await expect(inspectModel(fetchFn, 'flaky:x')).rejects.toThrow(/answered 503/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('boundedGet', () => {
|
||||
it('refuses a declared body over the cap before reading it', async () => {
|
||||
const { fetchFn } = fakeFetch({
|
||||
'https://x.test/big': () => new Response('{}', { status: 200, headers: { 'content-length': String(MAX_MANIFEST_BYTES + 1) } }),
|
||||
});
|
||||
await expect(boundedGet(fetchFn, 'https://x.test/big', { maxBytes: MAX_MANIFEST_BYTES })).rejects.toThrow(/over the cap/);
|
||||
});
|
||||
|
||||
it('refuses a streamed body that grows past the cap', async () => {
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(600));
|
||||
controller.enqueue(new Uint8Array(600));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
const { fetchFn } = fakeFetch({ 'https://x.test/stream': () => new Response(stream, { status: 200 }) });
|
||||
await expect(boundedGet(fetchFn, 'https://x.test/stream', { maxBytes: 1000 })).rejects.toThrow(/over the cap/);
|
||||
});
|
||||
|
||||
it('turns a network error into a CheckFailure', async () => {
|
||||
const fetchFn = async () => {
|
||||
throw new TypeError('fetch failed', { cause: new Error('getaddrinfo ENOTFOUND registry.ollama.ai') });
|
||||
};
|
||||
await expect(boundedGet(fetchFn, 'https://registry.ollama.ai/v2/', { maxBytes: 10 })).rejects.toThrow(/ENOTFOUND/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runCheck', () => {
|
||||
it('reports drift, and fetches only manifests, child manifests, configs and the two releases', async () => {
|
||||
const registry = buildRegistry();
|
||||
const { fetchFn, seen } = fakeFetch(registry.routes);
|
||||
const result = await runCheck({
|
||||
catalog: catalogOf([
|
||||
// Sized from the llamacpp child: a Mac pulls the mlx one, so this is drift.
|
||||
{ id: 'listed:1b', bytes: registry.sizes.gguf },
|
||||
{ id: 'plain:2b', bytes: registry.sizes.plain },
|
||||
{ id: 'missing:tag', bytes: 1000 },
|
||||
]),
|
||||
pin: PIN,
|
||||
fetchFn,
|
||||
now: NOW,
|
||||
githubToken: 'token-value',
|
||||
});
|
||||
|
||||
expect(result.models.map((model: { findings: { kind: string }[] }) => model.findings.map((f) => f.kind))).toEqual([
|
||||
['size'],
|
||||
[],
|
||||
['gone'],
|
||||
]);
|
||||
expect(result.runtime.findings).toEqual([]);
|
||||
expect(exitCodeFor(result)).toBe(1);
|
||||
|
||||
for (const request of seen) {
|
||||
const blob = /\/blobs\/(sha256:[0-9a-f]{64})$/.exec(request.url);
|
||||
if (blob) expect(registry.layerDigests.has(blob[1])).toBe(false);
|
||||
const isGitHub = request.url.startsWith('https://api.github.com/');
|
||||
expect(request.headers.Authorization === 'Bearer token-value').toBe(isGitHub);
|
||||
}
|
||||
expect(seen.some((request) => request.url === `${GH}/latest`)).toBe(true);
|
||||
expect(seen.some((request) => request.url === `${GH}/tags/v0.40.15`)).toBe(true);
|
||||
|
||||
const report = renderReport(result);
|
||||
expect(report).toContain('**Drift found**: 2 of 3 models.');
|
||||
expect(report).toContain('`missing:tag`: the registry no longer has this tag');
|
||||
});
|
||||
|
||||
it('sends no Authorization header at all without a token', async () => {
|
||||
const registry = buildRegistry();
|
||||
const { fetchFn, seen } = fakeFetch(registry.routes);
|
||||
await runCheck({ catalog: catalogOf([{ id: 'plain:2b', bytes: registry.sizes.plain }]), pin: PIN, fetchFn, now: NOW });
|
||||
expect(seen.some((request) => 'Authorization' in request.headers)).toBe(false);
|
||||
});
|
||||
|
||||
it('is clean when every row matches what a Mac pulls', async () => {
|
||||
const registry = buildRegistry();
|
||||
const { fetchFn } = fakeFetch(registry.routes);
|
||||
const result = await runCheck({
|
||||
catalog: catalogOf([
|
||||
{ id: 'listed:1b', bytes: registry.sizes.mlx, digest: registry.digests.mlxChild },
|
||||
{ id: 'plain:2b', bytes: registry.sizes.plain },
|
||||
]),
|
||||
pin: PIN,
|
||||
fetchFn,
|
||||
now: NOW,
|
||||
});
|
||||
expect(exitCodeFor(result)).toBe(0);
|
||||
const report = renderReport(result);
|
||||
expect(report).toContain('**Clean**');
|
||||
// The pinned row is not offered for pinning again; the unpinned one is.
|
||||
expect(report).not.toContain(`| \`listed:1b\` | \``);
|
||||
expect(report).toContain(`| \`plain:2b\` | \`${registry.digests.plain}\``);
|
||||
});
|
||||
|
||||
it('a newer runtime release is drift', async () => {
|
||||
const registry = buildRegistry();
|
||||
registry.routes[`${GH}/latest`] = JSON.stringify({ tag_name: 'v0.41.0', published_at: '2026-11-10T00:00:00Z', assets: [] });
|
||||
const { fetchFn } = fakeFetch(registry.routes);
|
||||
const result = await runCheck({ catalog: catalogOf([{ id: 'plain:2b', bytes: registry.sizes.plain }]), pin: PIN, fetchFn, now: NOW });
|
||||
expect(result.runtime.findings).toEqual([{ kind: 'behind', pinned: '0.40.15', latest: '0.41.0' }]);
|
||||
expect(exitCodeFor(result)).toBe(1);
|
||||
expect(renderReport(result)).toContain('Ollama 0.41.0 is out; the pin is 0.40.15.');
|
||||
});
|
||||
|
||||
it('a rate-limited GitHub API fails the check (2) and keeps what the models showed', async () => {
|
||||
const registry = buildRegistry();
|
||||
registry.routes[`${GH}/latest`] = 403;
|
||||
const { fetchFn } = fakeFetch(registry.routes);
|
||||
const result = await runCheck({
|
||||
catalog: catalogOf([{ id: 'listed:1b', bytes: registry.sizes.gguf }]),
|
||||
pin: PIN,
|
||||
fetchFn,
|
||||
now: NOW,
|
||||
});
|
||||
expect(exitCodeFor(result)).toBe(2);
|
||||
expect(result.models[0].findings[0].kind).toBe('size');
|
||||
expect(renderReport(result)).toContain('**The check failed**');
|
||||
});
|
||||
|
||||
it('a release tag that is not vX.Y.Z fails the check instead of reaching the report', async () => {
|
||||
const registry = buildRegistry();
|
||||
registry.routes[`${GH}/latest`] = JSON.stringify({ tag_name: 'v1.0.0 @everyone', assets: [] });
|
||||
const { fetchFn } = fakeFetch(registry.routes);
|
||||
const result = await runCheck({ catalog: catalogOf([]), pin: PIN, fetchFn, now: NOW });
|
||||
expect(exitCodeFor(result)).toBe(2);
|
||||
expect(renderReport(result)).not.toContain('@everyone');
|
||||
});
|
||||
|
||||
it('a network failure for one row fails the whole check', async () => {
|
||||
const registry = buildRegistry();
|
||||
const fetchFn = async (url: string, init: { headers: Record<string, string>; signal: unknown }) => {
|
||||
if (url.includes('/plain/')) throw new TypeError('fetch failed');
|
||||
return fakeFetch(registry.routes).fetchFn(url, init);
|
||||
};
|
||||
const result = await runCheck({
|
||||
catalog: catalogOf([
|
||||
{ id: 'listed:1b', bytes: registry.sizes.mlx },
|
||||
{ id: 'plain:2b', bytes: registry.sizes.plain },
|
||||
]),
|
||||
pin: PIN,
|
||||
fetchFn,
|
||||
now: NOW,
|
||||
});
|
||||
expect(result.models[1].error).toMatch(/fetch failed/);
|
||||
expect(exitCodeFor(result)).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:99afa5bbf1e7038bda7d028e100ee76b7337c943468d52590de3282db7b52adb","size":220},"layers":[{"mediaType":"application/vnd.ollama.image.model","digest":"sha256:d372de8e934898a59e6ccfabc3368474711384d8f1fd4d22d87a3f0a45400cdc","size":21718480960,"from":"qwen3.6-source:35b-a3b-mtp-q4_K_M-20260824"},{"mediaType":"application/vnd.ollama.image.projector","digest":"sha256:a62390d25b4b4a2d8afd7cc3c90021c11935c1fdd48d7cb0723ab71cd02a598e","size":902821728,"from":"qwen3.6-source:35b-a3b-mtp-q4_K_M-20260824"},{"mediaType":"application/vnd.ollama.image.license","digest":"sha256:5f3a3c817e78f5b8a4ad2d2c458a3e4b2cce470d6c12642c4eeb12cb8a9bf51d","size":11357,"from":"qwen3.6-source:35b-a3b-mtp-q4_K_M-20260824"},{"mediaType":"application/vnd.ollama.image.params","digest":"sha256:ce44c0f3ca1f7a126724e5285be7e1afdab202186c6379515ab0c1ffd40d3ecc","size":116}],"runner":"llamacpp","format":"gguf"}
|
||||
@@ -0,0 +1 @@
|
||||
{"schemaVersion":2,"mediaType":"application/vnd.ollama.manifest.list.v2+json","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba","runner":"mlx","format":"safetensors"},{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:a7eb95c53bcf96b4bdd008d0fab4a5dac88047d9c1a7a9ab88ed453423fbd87c","runner":"llamacpp","format":"gguf"}]}
|
||||
@@ -0,0 +1 @@
|
||||
{"architecture":"","capabilities":["completion","vision","tools","thinking"],"file_type":"nvfp4","model_families":null,"model_family":"","model_format":"safetensors","model_type":"","os":"","parser":"qwen3.5","renderer":"qwen3.5","requires":"0.32.4"}
|
||||
Reference in new issue
Block a user