diff --git a/.github/workflows/local-model-catalog.yml b/.github/workflows/local-model-catalog.yml new file mode 100644 index 00000000..5534a628 --- /dev/null +++ b/.github/workflows/local-model-catalog.yml @@ -0,0 +1,128 @@ +name: Local model catalogue + +# Checks every Monday that the local model catalogue (src/config/local-model-catalog.ts) and +# the pinned model runtime (OLLAMA_RUNTIME_PIN in src/local-model/runtime.ts) still match what +# the registry and Ollama's releases serve: a tag can be pushed again or removed, the build a +# Mac downloads can change size, and Ollama ships new versions. The catalogue is compiled into +# Codeman on purpose, so this job only reports; a change ships in a release like any other. +# What it checks, and what a maintainer does with its issue: docs/local-models.md, "Keeping +# the catalogue current". +# +# By hand: gh workflow run local-model-catalog.yml (the file must be on master for that, +# and the schedule only ever runs master's copy). Locally: npm run check:local-model-catalog +# +# Two jobs, so the token that can write issues never sits next to `npm ci`: +# - check installs the dependencies and runs scripts/check-local-model-catalog.mjs with a +# read-only token. Exit 0 clean, 1 drift, 2 the check failed. On 2 the job fails and the +# issue is not touched: an incomplete report must not open or update it. +# - issue runs no code from the repository, only gh. On drift it opens the one issue titled +# "Local model catalogue: weekly check", or puts this week's report into it when it is +# already open. On a clean week it comments on that issue only when one is open. + +on: + schedule: + - cron: '0 6 * * 1' + workflow_dispatch: + +permissions: + contents: read + +# A run by hand while the scheduled one is going must not open a second issue. +concurrency: ${{ github.workflow }} + +jobs: + check: + name: Check the catalogue and the runtime pin + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + status: ${{ steps.check.outputs.status }} + report: ${{ steps.check.outputs.report }} + steps: + - name: Checkout repo + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + cache: 'npm' + + # The check needs tsx and zod, both plain JavaScript, so no install script runs here + # (node-pty's build and the repository's postinstall included). + - name: Install dependencies + run: npm ci --ignore-scripts + + - name: Run the check + id: check + env: + # Read-only here. Sent to api.github.com only, so the release lookups are not rate limited. + GITHUB_TOKEN: ${{ github.token }} + run: | + set +e + npx tsx scripts/check-local-model-catalog.mjs > "$RUNNER_TEMP/report.md" + code=$? + set -e + cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY" + case "$code" in + 0) echo "status=clean" >> "$GITHUB_OUTPUT" ;; + 1) echo "status=drift" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The check itself failed (exit $code), so the issue is left alone. The summary holds what it found." + exit 1 + ;; + esac + # One line of base64, so nothing in the report can end the output early. An issue + # body holds at most 65,536 characters; the report is a few KB. + echo "report=$(head -c 60000 "$RUNNER_TEMP/report.md" | base64 -w0)" >> "$GITHUB_OUTPUT" + + issue: + name: Keep the weekly issue current + needs: check + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + ISSUE_TITLE: 'Local model catalogue: weekly check' + STATUS: ${{ needs.check.outputs.status }} + REPORT_B64: ${{ needs.check.outputs.report }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - name: Open, update or comment on the issue + run: | + set -euo pipefail + # The issues API, not the search index, so an issue opened a minute ago is found too, + # and an exact title match, so a similar title never counts. The oldest open one wins. + number=$(gh issue list --state open --limit 1000 --json number,title \ + | jq -r --arg title "$ISSUE_TITLE" '[.[] | select(.title == $title)] | sort_by(.number) | first | .number // empty') + today=$(date -u +%Y-%m-%d) + case "$STATUS" in + drift) + printf '%s' "$REPORT_B64" | base64 -d > "$RUNNER_TEMP/report.md" + printf '\n---\n\nWritten by the [weekly check](%s). Fix the catalogue or the pin in a pull request and close this issue once that is merged; the next drift opens it again.\n' \ + "$RUN_URL" >> "$RUNNER_TEMP/report.md" + if [ -n "$number" ]; then + gh issue edit "$number" --body-file "$RUNNER_TEMP/report.md" + gh issue comment "$number" --body "Week of $today: drift found again. The description now holds this week's report ([run]($RUN_URL))." + else + gh issue create --title "$ISSUE_TITLE" --body-file "$RUNNER_TEMP/report.md" + fi + ;; + clean) + if [ -n "$number" ]; then + gh issue comment "$number" --body "Week of $today: the check was clean ([run]($RUN_URL))." + else + echo "Clean, and no issue is open: nothing to do." + fi + ;; + *) + echo "::error::The check job reported no status ('$STATUS')." + exit 1 + ;; + esac diff --git a/docs/local-models.md b/docs/local-models.md index 86c9f98a..04627c46 100644 --- a/docs/local-models.md +++ b/docs/local-models.md @@ -250,6 +250,62 @@ or `python3`. - **The model stays in memory for ten minutes** after the last request, then the runtime unloads it. +## Keeping the catalogue current + +The catalogue (`src/config/local-model-catalog.ts`) and the runtime pin are compiled into +Codeman on purpose: no file or download decides what is recommended or run. What the +registry serves can still move away from them: a tag is pushed again or removed, the build +a Mac downloads changes size, Ollama ships a new version. The workflow +`.github/workflows/local-model-catalog.yml` looks for that every Monday at 06:00 UTC. It +runs `scripts/check-local-model-catalog.mjs`, which also runs locally: +`npm run check:local-model-catalog` prints the report and exits 0 (clean), 1 (drift) or 2 +(the check itself failed). + +For every model row it checks: + +- **The tag still exists** in the registry. +- **The size of what a Mac downloads** is within 0.1% of the row's `bytes`. A tag can be a + manifest list with one build per runner, and Ollama on Apple silicon takes the `mlx` build + when there is one, else `llamacpp`, else `ggml`. So the check asks for the tag the way + Ollama asks, picks the build a Mac picks and sizes it the way a pull counts it (every layer + plus the config). The plain manifest other tools get for the same tag is the `llamacpp` + build, which is not what a Mac downloads. +- **The build a Mac pulls runs on the pinned runtime**: its config declares no newer + Ollama (`requires`) than `OLLAMA_RUNTIME_PIN`. +- **The pinned digest**, once a row carries one: a Mac would still pull that manifest. The + catalogue pins none yet, so the report lists today's digests, the one to pin being what + `/api/tags` reports after the pull. + +For the runtime pin it checks whether a newer Ollama release exists, and whether the pinned +`ollama-darwin.tgz` is still on GitHub with the pinned size and sha256. + +It reads manifests and configs only, never a model layer. It does not check the memory +figures, the speeds or whether a coding agent works with a model: no manifest says anything +about those, and they change only when somebody measures on a Mac. + +**The issue.** Drift opens one issue, "Local model catalogue: weekly check", or, while that +issue is open, replaces its description with the new report and adds a comment. A clean week +comments on the open issue and does nothing when there is none. A check that failed (the +network, a rate limit, an answer that did not parse) fails the run and leaves the issue +alone; what it found is in the run's summary. The schedule runs the copy of the workflow on +`master`, and so does `gh workflow run local-model-catalog.yml`. + +**What a maintainer does with it**, in a pull request each time: + +- **Size drift or a changed digest**: correct the row. When the build itself changed (another + runner or quantization), its memory figures and speed no longer describe it: mark the row + `estimated` until somebody measures it again. +- **A tag that is gone, or a build that needs a newer runtime**: replace or drop the row, or + move the runtime pin first. +- **A newer Ollama**: read its release notes, then move `OLLAMA_RUNTIME_PIN` (version, URL, + size and sha256, checked against the release's `sha256sum.txt`) and run an install on a + Mac before merging. +- **A pinned archive that changed**: a security question first. Every install refuses it at + the hash compare, so nobody gets it, but find out why before moving the pin. + +Close the issue once the fix is merged. A catalogue or pin change reaches users only in a +release: an installed Codeman keeps the catalogue it was built with until it is updated. + ## Source | What | Where | @@ -265,3 +321,4 @@ or `python3`. | Routes | `src/web/routes/local-model-routes.ts` | | Shared CLI install runner | `src/cli-install-runner.ts` | | Serialized writer of the endpoint store | `src/custom-model-hosts-writer.ts` | +| Weekly catalogue check | `scripts/check-local-model-catalog.mjs`, `.github/workflows/local-model-catalog.yml` | diff --git a/package.json b/package.json index 12b31a58..684265db 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "test:mobile": "vitest run --config test/mobile/vitest.config.ts", "check:frontend-syntax": "node scripts/check-frontend-syntax.mjs", "check:browser-excludes": "node scripts/check-browser-test-excludes.mjs", + "check:local-model-catalog": "tsx scripts/check-local-model-catalog.mjs", "fix:node-pty": "node scripts/fix-node-pty.mjs", "typecheck": "tsc --noEmit && tsc -p config/tsconfig.scripts.json", "lint": "eslint --config config/eslint.config.js 'src/**/*.ts'", diff --git a/scripts/check-local-model-catalog.mjs b/scripts/check-local-model-catalog.mjs new file mode 100644 index 00000000..8e96970a --- /dev/null +++ b/scripts/check-local-model-catalog.mjs @@ -0,0 +1,728 @@ +#!/usr/bin/env -S npx tsx +/** + * The weekly check of the local model catalogue (src/config/local-model-catalog.ts) and of the + * pinned model runtime (OLLAMA_RUNTIME_PIN in src/local-model/runtime.ts) against what the + * registry and Ollama's releases serve today. + * + * npx tsx scripts/check-local-model-catalog.mjs # markdown report on stdout + * npm run check:local-model-catalog # the same + * + * Exit code: 0 nothing drifted, 1 drift found, 2 the check itself failed (network, an answer + * that does not parse, a catalogue that does not validate). On 2 the report is still printed + * with everything that was found, but it is incomplete, so nothing should act on it. + * `.github/workflows/local-model-catalog.yml` runs it every Monday and keeps one issue current. + * + * Runs under tsx because it imports the TypeScript catalogue and pin, and imports them only in + * main(), so the test imports the pure parts without them. + * + * WHAT A MAC DOWNLOADS. A tag is not one build. Asked with the Accept header Ollama 0.40.15 + * sends (`pullModelManifest` in server/images.go), the registry answers a manifest LIST for + * some tags, one child per runner, and Ollama on Apple silicon takes the first child whose + * runner is `mlx`, else `llamacpp`, else `ggml` (`runnerPreferences()` in manifest/manifest.go). + * Asked with the plain Docker Accept header the registry answers the llamacpp child instead, + * which is NOT what a Mac pulls: that is how `qwen3.6:35b-a3b` came to be "2% above its + * manifest" in the catalogue header. So this check asks the way Ollama asks, picks the child + * the way a Mac picks it, and compares the row's `bytes` with that child's pull total (every + * layer plus the config, which is exactly the `total` a real pull reported on a Mac). + * + * WHAT IS FETCHED, and nothing else: + * - `GET /v2/library//manifests/`, the tag's manifest or manifest list. + * - For a list: `GET /v2/library//blobs/` for each child the list names. A child + * manifest is stored as a blob (Ollama fetches it the same way, `pullSelectedManifest`). It + * is metadata: about 220 KB for an MLX build of 1,055 tensors. + * - `GET .../blobs/` for the config of the child a Mac selects, a few hundred bytes, + * for its `requires` (the oldest Ollama that may pull it). + * - The GitHub API: the newest release of ollama/ollama, and the release of the pinned version. + * A digest from a `layers` entry is never requested: no model weights are ever downloaded. + * Every request has its own timeout and a byte cap, nothing is retried, every blob is checked + * against its digest, and `GITHUB_TOKEN` is sent to api.github.com only, when it is set. + * + * Everything that reaches the report from the network is validated first (digests, sizes, + * runner and format names, version tags), because the workflow puts the report in an issue. + * + * NOT CHECKED: the memory figures, the speeds and whether a coding agent works with a model. + * No manifest says anything about those; they change only when somebody measures on a Mac. + */ + +import { createHash } from 'node:crypto'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +// ---------- Constants ---------- + +export const REGISTRY_BASE = 'https://registry.ollama.ai'; +export const GITHUB_API_BASE = 'https://api.github.com'; +/** The repository whose releases the runtime pin follows. */ +export const RUNTIME_REPO = 'ollama/ollama'; + +export const MEDIA_TYPE_MANIFEST = 'application/vnd.docker.distribution.manifest.v2+json'; +export const MEDIA_TYPE_MANIFEST_LIST = 'application/vnd.ollama.manifest.list.v2+json'; +/** The Accept header of Ollama 0.40.15's own manifest request: the list first. */ +export const MANIFEST_ACCEPT = `${MEDIA_TYPE_MANIFEST_LIST}, ${MEDIA_TYPE_MANIFEST}`; + +/** The runner order of Ollama 0.40.15 on darwin/arm64 (`runnerPreferences()`). */ +export const MAC_RUNNER_PREFERENCE = Object.freeze(['mlx', 'llamacpp', 'ggml']); +/** Format names the report prints as they are. Anything else is printed as `unknown`. */ +export const KNOWN_FORMATS = Object.freeze(['safetensors', 'gguf']); + +/** + * How far the size a Mac downloads may sit from the row's `bytes` before it is drift: 0.1%. + * The rows that were sized from layers alone sit a few hundred bytes (the config) below the + * pull total; any rebuilt model moves by far more than this. + */ +export const SIZE_TOLERANCE_FRACTION = 0.001; + +/** One request may take this long, connection and body together. */ +export const REQUEST_TIMEOUT_MS = 20_000; +/** A manifest or child manifest larger than this is refused. The MLX build of the 35B is 220 KB. */ +export const MAX_MANIFEST_BYTES = 4 * 1024 * 1024; +/** A config blob larger than this is refused. They are a few hundred bytes. */ +export const MAX_CONFIG_BYTES = 64 * 1024; +/** A GitHub release answer larger than this is refused. */ +export const MAX_GITHUB_BYTES = 4 * 1024 * 1024; +/** A manifest list with more children than this is refused. */ +export const MAX_LIST_CHILDREN = 8; +/** A manifest with more layers than this is refused. */ +export const MAX_LAYERS = 20_000; + +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; +const VERSION_PATTERN = /^v?(\d{1,6})\.(\d{1,6})\.(\d{1,6})$/; +const RELEASE_TAG_PATTERN = /^v\d{1,6}\.\d{1,6}\.\d{1,6}$/; +const ISO_DAY_PATTERN = /^(\d{4}-\d{2}-\d{2})T/; +const USER_AGENT = 'codeman-local-model-catalog-check'; + +/** The check could not finish: the run ends with exit code 2. */ +export class CheckFailure extends Error { + constructor(message) { + super(message); + this.name = 'CheckFailure'; + } +} + +// ---------- Pure helpers ---------- + +/** `name:tag` into its two parts. The catalogue's pattern already rules out a slash. */ +export function splitModelId(id) { + const colon = id.indexOf(':'); + if (colon <= 0 || colon === id.length - 1 || id.includes('/')) throw new CheckFailure(`not a name:tag model id: ${id}`); + return { name: id.slice(0, colon), tag: id.slice(colon + 1) }; +} + +/** `sha256:` of a body, the way the registry names it. */ +export function sha256Digest(bytes) { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}`; +} + +/** A digest as `sha256:<64 hex>`, accepting the bare hex `/api/tags` prints. Anything else: undefined. */ +export function normalizeDigest(value) { + if (typeof value !== 'string') return undefined; + const text = value.trim().toLowerCase(); + const full = text.startsWith('sha256:') ? text : `sha256:${text}`; + return DIGEST_PATTERN.test(full) ? full : undefined; +} + +/** A runner name as Ollama compares it (lowercase, trimmed). */ +export function canonicalRunner(runner) { + return typeof runner === 'string' ? runner.trim().toLowerCase() : ''; +} + +/** A runner or format name for the report: one of the known names, else `unknown`. */ +export function displayName(value, known) { + const name = canonicalRunner(value); + return known.includes(name) ? name : 'unknown'; +} + +function isSize(value) { + return Number.isSafeInteger(value) && value >= 0; +} + +/** + * What a pull of this manifest downloads, in bytes: every layer plus the config. Throws on a + * manifest that is not shaped like one, so a malformed answer never reads as a size. + */ +export function pullTotalBytes(manifest) { + if (!manifest || typeof manifest !== 'object') throw new CheckFailure('the manifest is not an object'); + const { config, layers } = manifest; + if (!Array.isArray(layers) || layers.length === 0) throw new CheckFailure('the manifest has no layers'); + if (layers.length > MAX_LAYERS) throw new CheckFailure(`the manifest has more than ${MAX_LAYERS} layers`); + if (!config || !isSize(config.size)) throw new CheckFailure('the manifest has no config size'); + let total = config.size; + for (const layer of layers) { + if (!layer || !isSize(layer.size)) throw new CheckFailure('a layer of the manifest has no size'); + total += layer.size; + } + if (!Number.isSafeInteger(total)) throw new CheckFailure('the manifest total is not a safe integer'); + return total; +} + +/** + * The child of a manifest list that Ollama 0.40.15 on Apple silicon pulls: for each runner in + * `MAC_RUNNER_PREFERENCE`, the first child (in list order) with that runner. A child whose + * mediaType is set to anything but a plain manifest is skipped, as Ollama skips it. Undefined + * when no child has a runner a Mac runs. + */ +export function selectMacChild(children) { + if (!Array.isArray(children)) return undefined; + for (const runner of MAC_RUNNER_PREFERENCE) { + const child = children.find( + (entry) => + entry && + (entry.mediaType === undefined || entry.mediaType === '' || entry.mediaType === MEDIA_TYPE_MANIFEST) && + canonicalRunner(entry.runner) === runner + ); + if (child) return child; + } + return undefined; +} + +/** + * The runner and format Ollama assumes for a manifest that names neither, from its config's + * `model_format` (`MetadataForConfig()` in manifest/manifest.go): safetensors runs on mlx, + * gguf on ggml. + */ +export function metadataForConfig(config) { + const format = canonicalRunner(config?.model_format); + if (format === 'safetensors') return { runner: 'mlx', format: 'safetensors' }; + if (format === 'gguf' || format === 'ggml') return { runner: 'ggml', format: 'gguf' }; + return { runner: '', format }; +} + +/** `[major, minor, patch]` of a version or `v`-tag, or undefined when it is not one. */ +export function parseVersion(text) { + const match = typeof text === 'string' ? VERSION_PATTERN.exec(text.trim()) : null; + return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : undefined; +} + +/** Negative when `a` is older than `b`, 0 when equal, positive when newer. Both must parse. */ +export function compareVersions(a, b) { + const left = parseVersion(a); + const right = parseVersion(b); + if (!left || !right) throw new CheckFailure(`cannot compare versions ${a} and ${b}`); + for (let i = 0; i < 3; i++) if (left[i] !== right[i]) return left[i] - right[i]; + return 0; +} + +/** The signed difference of `actual` from `expected`, as a fraction of `expected`. */ +export function sizeDifference(expected, actual) { + return (actual - expected) / expected; +} + +/** Whether `actual` lies within `SIZE_TOLERANCE_FRACTION` of `expected`. */ +export function withinTolerance(expected, actual, tolerance = SIZE_TOLERANCE_FRACTION) { + return Math.abs(actual - expected) <= expected * tolerance; +} + +/** + * The findings for one catalogue row. `row` is `{ id, bytes, digest? }` (`digest` only once the + * catalogue stores one); `observed` is what `inspectModel()` returned for it. A row may have + * several findings; none means it is clean. + * + * - `gone`: the tag is not in the registry, so a pull fails. + * - `size`: what a Mac downloads is outside the tolerance of the row's `bytes`. + * - `digest`: the row pins a digest and a Mac would now pull another one. + * - `requires`: the build a Mac pulls declares a newer Ollama than the pinned runtime, so the + * pinned runtime refuses the pull. + */ +export function classifyModel(row, observed, pinnedVersion) { + if (observed.gone) return [{ kind: 'gone' }]; + const findings = []; + const mac = observed.mac; + if (!withinTolerance(row.bytes, mac.bytes)) { + findings.push({ kind: 'size', expected: row.bytes, actual: mac.bytes, fraction: sizeDifference(row.bytes, mac.bytes) }); + } + const pinned = normalizeDigest(row.digest); + if (row.digest !== undefined && pinned !== mac.digest) { + findings.push({ kind: 'digest', expected: pinned ?? 'not a sha256 digest', actual: mac.digest }); + } + if (mac.requires && compareVersions(mac.requires, pinnedVersion) > 0) { + findings.push({ kind: 'requires', requires: mac.requires, pinned: pinnedVersion }); + } + return findings; +} + +/** + * The findings for the runtime pin. `observed` is what `inspectRuntime()` returned. + * + * - `behind`: a newer Ollama release exists than the pinned one. + * - `pinned-release-gone`, `pinned-asset-gone`: the pinned download no longer exists, so every + * new install fails at the runtime step. + * - `pinned-asset-changed`: the pinned asset's size or sha256 no longer matches the pin. Every + * install already refuses it at the hash compare; this is also what a replaced release + * asset looks like, so read it as a security signal first. + */ +export function classifyRuntime(pin, observed) { + const findings = []; + if (compareVersions(observed.latest.version, pin.version) > 0) { + findings.push({ kind: 'behind', pinned: pin.version, latest: observed.latest.version }); + } + if (observed.pinned.gone) findings.push({ kind: 'pinned-release-gone' }); + else if (!observed.pinned.asset) findings.push({ kind: 'pinned-asset-gone' }); + else { + const { size, digest } = observed.pinned.asset; + if (size !== pin.bytes || (digest !== undefined && digest !== `sha256:${pin.sha256}`)) { + findings.push({ kind: 'pinned-asset-changed', size, digest }); + } + } + return findings; +} + +/** 0 clean, 1 drift, 2 the check failed. A failure anywhere wins over drift anywhere. */ +export function exitCodeFor(result) { + const failed = + result.errors.length > 0 || result.models.some((model) => model.error) || Boolean(result.runtime?.error); + if (failed) return 2; + const drift = + result.models.some((model) => model.findings.length > 0) || (result.runtime?.findings.length ?? 0) > 0; + return drift ? 1 : 0; +} + +// ---------- Report ---------- + +/** 1234567 as `1,234,567`, independent of the locale. */ +export function groupDigits(value) { + return String(value).replace(/\B(?=(\d{3})+(?!\d))/g, ','); +} + +/** A fraction as a signed percentage with two decimals: `+16.61%`. */ +export function formatPercent(fraction) { + const text = (fraction * 100).toFixed(2); + return fraction > 0 ? `+${text}%` : `${text}%`; +} + +/** Error text for the report: one line, backticks made quotes, at most 200 characters. */ +function oneLine(text) { + const flat = String(text) + .replace(/[\u0000-\u001f\u007f]+/g, ' ') + .replace(/`/g, "'") + .replace(/ {2,}/g, ' ') + .trim(); + return flat.length > 200 ? `${flat.slice(0, 200)}...` : flat; +} + +function buildName(variant) { + return `${displayName(variant.runner, MAC_RUNNER_PREFERENCE)}, ${displayName(variant.format, KNOWN_FORMATS)}`; +} + +function modelResult(model) { + if (model.error) return 'check failed'; + if (model.findings.length === 0) return 'ok'; + const names = { gone: 'tag gone', size: 'size drift', digest: 'digest changed', requires: 'needs a newer runtime' }; + return model.findings.map((finding) => names[finding.kind]).join(', '); +} + +function describeModelFinding(model, finding) { + const id = `\`${model.id}\``; + switch (finding.kind) { + case 'gone': + return `${id}: the registry no longer has this tag, so the download fails. Replace or remove the row.`; + case 'size': + return ( + `${id}: a Mac downloads the ${buildName(model.observed.mac)} build, ${groupDigits(finding.actual)} bytes, ` + + `${formatPercent(finding.fraction)} against the row's \`bytes\` of ${groupDigits(finding.expected)}. ` + + 'The memory fit reads `bytes`, so correct it (and re-measure if the build changed).' + ); + case 'digest': + return `${id}: the row pins \`${finding.expected}\`, a Mac now pulls \`${finding.actual}\`. The tag was pushed again.`; + case 'requires': + return ( + `${id}: the build a Mac pulls declares \`requires: ${finding.requires}\`, newer than the pinned runtime ` + + `${finding.pinned}, which refuses to pull it.` + ); + default: + return `${id}: ${finding.kind}`; + } +} + +function describeRuntimeFinding(finding) { + switch (finding.kind) { + case 'behind': + return ( + `Ollama ${finding.latest} is out; the pin is ${finding.pinned}. Read its release notes, then move ` + + '`OLLAMA_RUNTIME_PIN` (version, URL, size, sha256) and test an install on a Mac.' + ); + case 'pinned-release-gone': + return 'The release of the pinned version is gone from GitHub: every new install fails at the runtime step.'; + case 'pinned-asset-gone': + return 'The pinned release no longer carries the pinned archive: every new install fails at the runtime step.'; + case 'pinned-asset-changed': + return ( + `The pinned archive changed on GitHub (now ${finding.size === undefined ? 'no size' : `${groupDigits(finding.size)} bytes`}` + + `${finding.digest ? `, \`${finding.digest}\`` : ''}). Installs refuse it at the hash compare. Find out why before moving the pin.` + ); + default: + return finding.kind; + } +} + +function assetCell(asset) { + if (!asset) return 'none'; + return `${groupDigits(asset.size)} bytes${asset.digest ? `, \`${asset.digest}\`` : ', no digest in the API'}`; +} + +/** The markdown report of one run. Pure: everything in it was validated by the IO side. */ +export function renderReport(result) { + const lines = []; + const code = exitCodeFor(result); + const driftModels = result.models.filter((model) => !model.error && model.findings.length > 0).length; + const runtimeDrift = result.runtime && !result.runtime.error ? result.runtime.findings.length > 0 : false; + + lines.push('# Local model catalogue: weekly check', ''); + lines.push( + `Run ${result.checkedAt}. Catalogue of ${result.catalogue.updated} (${result.models.length} models), ` + + `runtime pin Ollama ${result.pin.version}.`, + '' + ); + if (code === 2) { + lines.push('**The check failed**: the results below are incomplete. Nothing should act on them.', ''); + } else if (code === 1) { + const parts = []; + if (driftModels > 0) parts.push(`${driftModels} of ${result.models.length} models`); + if (runtimeDrift) parts.push('the runtime pin'); + lines.push(`**Drift found**: ${parts.join(' and ')}.`, ''); + } else { + lines.push('**Clean**: every row matches what a Mac downloads today, and the runtime pin is current.', ''); + } + + for (const error of result.errors) lines.push(`- Check failed: ${oneLine(error)}`); + if (result.errors.length > 0) lines.push(''); + + lines.push('## Models', ''); + lines.push( + 'What a Mac downloads is the build Ollama on Apple silicon selects: the `mlx` build when the tag offers ' + + 'one, else `llamacpp`, else `ggml`. Its size is the pull total, every layer plus the config. ' + + `Tolerance: ${(SIZE_TOLERANCE_FRACTION * 100).toFixed(1)}% of the row's \`bytes\`.`, + '' + ); + lines.push('| Model | Row `bytes` | A Mac downloads | Difference | Build on a Mac | Requires | Result |'); + lines.push('| --- | ---: | ---: | ---: | --- | --- | --- |'); + for (const model of result.models) { + const mac = model.observed?.mac; + const cells = [ + `\`${model.id}\``, + groupDigits(model.bytes), + mac ? groupDigits(mac.bytes) : '', + mac ? formatPercent(sizeDifference(model.bytes, mac.bytes)) : '', + mac ? buildName(mac) : model.observed?.gone ? 'tag gone' : '', + mac?.requires ? `Ollama ${mac.requires}` : '', + modelResult(model), + ]; + lines.push(`| ${cells.join(' | ')} |`); + } + lines.push(''); + + const details = []; + for (const model of result.models) { + if (model.error) details.push(`- \`${model.id}\`: check failed: ${oneLine(model.error)}`); + for (const finding of model.findings) details.push(`- ${describeModelFinding(model, finding)}`); + const variants = model.observed?.variants ?? []; + if (model.findings.some((finding) => finding.kind === 'size') && variants.length > 1) { + const offered = variants + .map((variant) => `${buildName(variant)} ${groupDigits(variant.bytes)}${variant === model.observed.mac ? ' (a Mac takes this one)' : ''}`) + .join('; '); + details.push(` The tag offers: ${offered}.`); + } + } + if (details.length > 0) lines.push('### What changed', '', ...details, ''); + + lines.push('## Runtime', ''); + if (!result.runtime || result.runtime.error) { + lines.push(`Check failed: ${oneLine(result.runtime?.error ?? 'not run')}`, ''); + } else { + const { latest, pinned } = result.runtime.observed; + const archive = result.runtime.assetName; + lines.push(`| | Version | Published | \`${archive}\` |`); + lines.push('| --- | --- | --- | --- |'); + lines.push( + `| Pinned | ${result.pin.version} | ${pinned.publishedOn ?? ''} | ` + + `${pinned.gone ? 'release gone' : assetCell(pinned.asset)} |` + ); + lines.push(`| Newest release | ${latest.version} | ${latest.publishedOn ?? ''} | ${assetCell(latest.asset)} |`); + lines.push(''); + if (result.runtime.findings.length === 0) { + lines.push('The pin is the newest release, and its archive still matches the pin.', ''); + } else { + for (const finding of result.runtime.findings) lines.push(`- ${describeRuntimeFinding(finding)}`); + lines.push(''); + } + lines.push( + `Release notes: https://github.com/${RUNTIME_REPO}/releases/tag/v${latest.version}. ` + + 'A digest from the API is a lead, not a pin: check it against the release\'s `sha256sum.txt` first.', + '' + ); + } + + const unpinned = result.models.filter((model) => model.digest === undefined && model.observed?.mac); + if (unpinned.length > 0) { + lines.push('## Digests to pin', ''); + lines.push( + 'The catalogue pins no digest for these rows. Today\'s values: the tag\'s own manifest, and the manifest a ' + + 'Mac pulls, which is the digest `/api/tags` reports after the pull.', + '' + ); + lines.push('| Model | Tag manifest | Manifest a Mac pulls |'); + lines.push('| --- | --- | --- |'); + for (const model of unpinned) { + lines.push(`| \`${model.id}\` | \`${model.observed.tagDigest}\` | \`${model.observed.mac.digest}\` |`); + } + lines.push(''); + } + + lines.push('## Not checked', ''); + lines.push( + 'The memory figures, the speeds and whether a coding agent works with a model: no manifest says anything ' + + 'about those. They change only when somebody measures on a Mac.', + '' + ); + return `${lines.join('\n')}\n`; +} + +// ---------- IO, with an injected fetch ---------- + +/** + * One GET, bounded: `REQUEST_TIMEOUT_MS` for the whole exchange and `maxBytes` for the body. + * Answers `{ status, body }` (body only for a 2xx). Never retries. + */ +export async function boundedGet(fetchFn, url, { headers = {}, maxBytes }) { + const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS); + let res; + try { + res = await fetchFn(url, { headers: { 'User-Agent': USER_AGENT, ...headers }, redirect: 'follow', signal }); + } catch (err) { + throw new CheckFailure(`GET ${url} failed: ${describeError(err)}`); + } + if (res.status < 200 || res.status >= 300) { + await res.body?.cancel().catch(() => {}); + return { status: res.status }; + } + const declared = Number(res.headers.get('content-length')); + if (Number.isFinite(declared) && declared > maxBytes) { + await res.body?.cancel().catch(() => {}); + throw new CheckFailure(`GET ${url}: ${declared} bytes is over the cap of ${maxBytes}`); + } + const chunks = []; + let total = 0; + try { + if (res.body) { + const reader = res.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) { + await reader.cancel().catch(() => {}); + throw new CheckFailure(`GET ${url}: the body is over the cap of ${maxBytes} bytes`); + } + chunks.push(value); + } + } + } catch (err) { + if (err instanceof CheckFailure) throw err; + throw new CheckFailure(`GET ${url}: reading the body failed: ${describeError(err)}`); + } + return { status: res.status, body: Buffer.concat(chunks) }; +} + +function describeError(err) { + if (err instanceof Error) { + const cause = err.cause instanceof Error && err.cause.message ? `: ${err.cause.message}` : ''; + return `${err.name === 'TimeoutError' ? 'timed out' : err.message}${cause}`; + } + return String(err); +} + +function parseJson(body, what) { + try { + return JSON.parse(body.toString('utf8')); + } catch { + throw new CheckFailure(`${what} is not JSON`); + } +} + +/** A blob of the repository, checked against its digest. */ +async function getBlob(fetchFn, name, digest, maxBytes, what) { + const url = `${REGISTRY_BASE}/v2/library/${encodeURIComponent(name)}/blobs/${digest}`; + const answer = await boundedGet(fetchFn, url, { maxBytes }); + if (!answer.body) throw new CheckFailure(`${what} ${digest}: the registry answered ${answer.status}`); + if (sha256Digest(answer.body) !== digest) throw new CheckFailure(`${what} ${digest}: the body does not match its digest`); + return answer.body; +} + +/** One build of a tag: its runner, format, digest and pull total, all validated. */ +function variantOf(manifest, digest, fallback = {}) { + return { + runner: canonicalRunner(manifest.runner) || canonicalRunner(fallback.runner), + format: canonicalRunner(manifest.format) || canonicalRunner(fallback.format), + digest, + bytes: pullTotalBytes(manifest), + configDigest: normalizeDigest(manifest.config?.digest), + }; +} + +/** + * What the registry serves for `id` today, read the way Ollama on a Mac reads it: + * `{ gone: true }`, or `{ tagDigest, variants, mac }` where `mac` is the variant a Mac pulls + * (with its config's `requires`). Throws `CheckFailure` when the answer is not usable. + */ +export async function inspectModel(fetchFn, id) { + const { name, tag } = splitModelId(id); + const url = `${REGISTRY_BASE}/v2/library/${encodeURIComponent(name)}/manifests/${encodeURIComponent(tag)}`; + const answer = await boundedGet(fetchFn, url, { headers: { Accept: MANIFEST_ACCEPT }, maxBytes: MAX_MANIFEST_BYTES }); + if (answer.status === 404) return { gone: true }; + if (!answer.body) throw new CheckFailure(`the registry answered ${answer.status} for the manifest`); + const tagDigest = sha256Digest(answer.body); + const manifest = parseJson(answer.body, 'the manifest'); + + let variants; + let mac; + if (manifest.mediaType === MEDIA_TYPE_MANIFEST_LIST) { + const children = Array.isArray(manifest.manifests) ? manifest.manifests : []; + if (children.length === 0 || children.length > MAX_LIST_CHILDREN) { + throw new CheckFailure(`the manifest list has ${children.length} children`); + } + const selected = selectMacChild(children); + if (!selected) throw new CheckFailure('the manifest list has no build a Mac runs (mlx, llamacpp or ggml)'); + variants = []; + for (const child of children) { + const digest = normalizeDigest(child?.digest); + if (!digest || !child.digest.startsWith('sha256:')) throw new CheckFailure('a child of the manifest list has no sha256 digest'); + const body = await getBlob(fetchFn, name, digest, MAX_MANIFEST_BYTES, 'child manifest'); + const variant = variantOf(parseJson(body, `child manifest ${digest}`), digest, child); + variants.push(variant); + if (child === selected) mac = variant; + } + } else if (manifest.mediaType === MEDIA_TYPE_MANIFEST || manifest.mediaType === undefined) { + mac = variantOf(manifest, tagDigest); + variants = [mac]; + } else { + throw new CheckFailure('the manifest has a media type this check does not know'); + } + + if (!mac.configDigest) throw new CheckFailure('the build a Mac pulls has no config digest'); + const config = parseJson(await getBlob(fetchFn, name, mac.configDigest, MAX_CONFIG_BYTES, 'config'), 'the config'); + // A manifest that names no runner or format gets them from its config, as Ollama fills them. + const assumed = metadataForConfig(config); + mac.runner ||= assumed.runner; + mac.format ||= assumed.format; + // A malformed `requires` is ignored, as Ollama ignores it (`checkPullRequires`). + const requires = parseVersion(config?.requires); + mac.requires = requires ? requires.join('.') : undefined; + return { tagDigest, variants, mac }; +} + +function githubHeaders(token) { + return { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }; +} + +/** The validated parts of a GitHub release: version, day it was published, the named asset. */ +function releaseFacts(release, assetName) { + if (!release || typeof release.tag_name !== 'string' || !RELEASE_TAG_PATTERN.test(release.tag_name)) { + throw new CheckFailure('the release has no vX.Y.Z tag'); + } + const day = typeof release.published_at === 'string' ? ISO_DAY_PATTERN.exec(release.published_at) : null; + const raw = Array.isArray(release.assets) ? release.assets.find((asset) => asset?.name === assetName) : undefined; + let asset; + if (raw) { + if (!isSize(raw.size)) throw new CheckFailure(`the release asset ${assetName} has no size`); + asset = { size: raw.size, digest: normalizeDigest(raw.digest) }; + } + return { version: release.tag_name.slice(1), publishedOn: day ? day[1] : undefined, asset }; +} + +/** The newest release and the pinned one, from the GitHub API. Throws `CheckFailure`. */ +export async function inspectRuntime(fetchFn, pin, token) { + const assetName = new URL(pin.url).pathname.split('/').pop(); + const headers = githubHeaders(token); + const latestAnswer = await boundedGet(fetchFn, `${GITHUB_API_BASE}/repos/${RUNTIME_REPO}/releases/latest`, { + headers, + maxBytes: MAX_GITHUB_BYTES, + }); + if (!latestAnswer.body) throw new CheckFailure(`GitHub answered ${latestAnswer.status} for the newest release`); + const latest = releaseFacts(parseJson(latestAnswer.body, 'the newest release'), assetName); + + const pinnedAnswer = await boundedGet( + fetchFn, + `${GITHUB_API_BASE}/repos/${RUNTIME_REPO}/releases/tags/v${encodeURIComponent(pin.version)}`, + { headers, maxBytes: MAX_GITHUB_BYTES } + ); + let pinned; + if (pinnedAnswer.status === 404) pinned = { gone: true }; + else if (!pinnedAnswer.body) throw new CheckFailure(`GitHub answered ${pinnedAnswer.status} for the pinned release`); + else pinned = releaseFacts(parseJson(pinnedAnswer.body, 'the pinned release'), assetName); + return { assetName, observed: { latest, pinned } }; +} + +/** + * The whole check. `catalog` is the validated catalogue, `pin` the runtime pin, `fetchFn` the + * fetch to use. Never throws for a network problem: each failure lands in the result, and + * `exitCodeFor()` turns it into 2. + */ +export async function runCheck({ catalog, pin, fetchFn, now, githubToken }) { + const result = { + checkedAt: `${now.toISOString().slice(0, 16).replace('T', ' ')} UTC`, + catalogue: { updated: catalog.updated }, + pin: { version: pin.version }, + models: [], + runtime: undefined, + errors: [], + }; + + // Rows in parallel, each row's requests one after another: a handful of rows, a few requests each. + result.models = await Promise.all( + catalog.models.map(async (row) => { + const digest = /** @type {{ digest?: unknown }} */ (row).digest; + const entry = { id: row.id, bytes: row.bytes, digest, findings: [] }; + try { + entry.observed = await inspectModel(fetchFn, row.id); + entry.findings = classifyModel({ id: row.id, bytes: row.bytes, digest }, entry.observed, pin.version); + } catch (err) { + entry.error = err instanceof CheckFailure ? err.message : describeError(err); + } + return entry; + }) + ); + + try { + const runtime = await inspectRuntime(fetchFn, pin, githubToken); + result.runtime = { ...runtime, findings: classifyRuntime(pin, runtime.observed) }; + } catch (err) { + result.runtime = { error: err instanceof CheckFailure ? err.message : describeError(err), findings: [] }; + } + return result; +} + +// ---------- Main ---------- + +async function main() { + let catalog; + let pin; + try { + const { getLocalModelCatalog } = await import('../src/config/local-model-catalog.js'); + const { OLLAMA_RUNTIME_PIN } = await import('../src/local-model/runtime.js'); + catalog = getLocalModelCatalog(); + pin = OLLAMA_RUNTIME_PIN; + } catch (err) { + console.error(`The catalogue or the runtime pin could not be loaded: ${describeError(err)}`); + process.exitCode = 2; + return; + } + const result = await runCheck({ + catalog, + pin, + fetchFn: globalThis.fetch, + now: new Date(), + githubToken: process.env.GITHUB_TOKEN || undefined, + }); + process.stdout.write(renderReport(result)); + process.exitCode = exitCodeFor(result); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(`The check failed: ${describeError(err)}`); + process.exitCode = 2; + }); +} diff --git a/test/check-local-model-catalog.test.ts b/test/check-local-model-catalog.test.ts new file mode 100644 index 00000000..1e6ad7bd --- /dev/null +++ b/test/check-local-model-catalog.test.ts @@ -0,0 +1,660 @@ +/** + * @fileoverview scripts/check-local-model-catalog.mjs, the weekly check of the local model + * catalogue: which build a Mac pulls from a manifest list, the pull total, how a row's drift is + * classified, the exit code, the report text, and that the IO side only ever asks for + * manifests, child manifests and configs (never a layer). The network is never used: the IO + * tests drive a fake registry and a fake GitHub API. + * + * The fixtures under test/fixtures/local-model-catalog/ are real bodies the registry served on + * 2026-11-08 for `qwen3.6:35b-a3b`, byte for byte, so their digests are the real ones. + */ + +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + CheckFailure, + MANIFEST_ACCEPT, + MAX_MANIFEST_BYTES, + MEDIA_TYPE_MANIFEST, + MEDIA_TYPE_MANIFEST_LIST, + SIZE_TOLERANCE_FRACTION, + boundedGet, + classifyModel, + classifyRuntime, + compareVersions, + exitCodeFor, + formatPercent, + groupDigits, + inspectModel, + metadataForConfig, + normalizeDigest, + parseVersion, + pullTotalBytes, + renderReport, + runCheck, + selectMacChild, + sha256Digest, + splitModelId, + withinTolerance, +} from '../scripts/check-local-model-catalog.mjs'; + +const fixtureDir = resolve(import.meta.dirname, 'fixtures/local-model-catalog'); +const fixture = (name: string): Buffer => readFileSync(resolve(fixtureDir, name)); + +const REAL_LIST = fixture('qwen3.6-35b-a3b.list.json'); +const REAL_GGUF_MANIFEST = fixture('qwen3.6-35b-a3b.gguf-manifest.json'); +const REAL_MLX_CONFIG = fixture('qwen3.6-35b-a3b.mlx-config.json'); + +/** The digest of the MLX child the real list names, which a Mac pulled on 2026-11-06. */ +const REAL_MLX_CHILD = 'sha256:0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba'; +const REAL_GGUF_CHILD = 'sha256:a7eb95c53bcf96b4bdd008d0fab4a5dac88047d9c1a7a9ab88ed453423fbd87c'; +/** The pull total a Mac reported for `qwen3.6:35b-a3b`, which the catalogue row stores. */ +const MAC_PULL_TOTAL_35B = 23_105_798_956; + +const PIN = { + version: '0.40.15', + url: 'https://github.com/ollama/ollama/releases/download/v0.40.15/ollama-darwin.tgz', + bytes: 185_971_671, + sha256: '607c79d5d6315ef9d896db83adde736a4c0057196329843544e79b95afb260a8', +}; + +const hex = (char: string): string => `sha256:${char.repeat(64)}`; + +describe('the real registry bodies', () => { + it('carry the digests the registry and a Mac reported for them', () => { + expect(sha256Digest(REAL_LIST)).toBe('sha256:86b2d171ce90bbb2dcb081933565ec867ce523cdd274bd24c8cf1ef9185feb4f'); + expect(sha256Digest(REAL_GGUF_MANIFEST)).toBe(REAL_GGUF_CHILD); + expect(sha256Digest(REAL_MLX_CONFIG)).toBe( + 'sha256:577d7d4694896a245c8b00f62bf53a8f4720ded4eba83a8e2da30bb87583bd6b' + ); + }); + + it('a Mac selects the mlx child of the real list, the one whose digest /api/tags reported', () => { + const list = JSON.parse(REAL_LIST.toString('utf8')); + expect(list.mediaType).toBe(MEDIA_TYPE_MANIFEST_LIST); + expect(selectMacChild(list.manifests).digest).toBe(REAL_MLX_CHILD); + }); + + it('the plain GGUF manifest totals its layers plus its config, 2% below what a Mac pulls', () => { + const manifest = JSON.parse(REAL_GGUF_MANIFEST.toString('utf8')); + expect(pullTotalBytes(manifest)).toBe(22_621_314_161 + 220); + expect(classifyModel( + { id: 'qwen3.6:35b-a3b', bytes: MAC_PULL_TOTAL_35B }, + { mac: { bytes: pullTotalBytes(manifest), digest: REAL_GGUF_CHILD } }, + '0.40.15' + )).toEqual([expect.objectContaining({ kind: 'size' })]); + }); +}); + +describe('splitModelId', () => { + it('splits name and tag', () => { + expect(splitModelId('qwen3.6:35b-a3b')).toEqual({ name: 'qwen3.6', tag: '35b-a3b' }); + }); + + it.each(['qwen3.6', ':35b', 'qwen3.6:', 'evil.example/x:y'])('refuses %s', (id) => { + expect(() => splitModelId(id)).toThrow(CheckFailure); + }); +}); + +describe('pullTotalBytes', () => { + const manifest = (layers: unknown, config: unknown = { size: 250 }) => ({ config, layers }); + + it('adds every layer and the config', () => { + expect(pullTotalBytes(manifest([{ size: 1000 }, { size: 24 }]))).toBe(1274); + }); + + it.each([ + ['no layers', manifest([])], + ['layers that are not an array', manifest('x')], + ['a layer without a size', manifest([{ digest: 'x' }])], + ['a negative size', manifest([{ size: -1 }])], + ['a size given as text', manifest([{ size: '1000' }])], + ['a fractional size', manifest([{ size: 1.5 }])], + ['no config size', manifest([{ size: 1 }], {})], + ['a total past the safe integers', manifest([{ size: Number.MAX_SAFE_INTEGER }, { size: 10 }])], + ])('refuses %s', (_label, value) => { + expect(() => pullTotalBytes(value)).toThrow(CheckFailure); + }); +}); + +describe('selectMacChild', () => { + const child = (runner: string, digest: string, mediaType?: string) => ({ + ...(mediaType === undefined ? { mediaType: MEDIA_TYPE_MANIFEST } : { mediaType }), + digest, + runner, + }); + + it('prefers mlx, then llamacpp, then ggml, whatever the list order', () => { + expect(selectMacChild([child('ggml', 'g'), child('llamacpp', 'l'), child('mlx', 'm')]).digest).toBe('m'); + expect(selectMacChild([child('ggml', 'g'), child('llamacpp', 'l')]).digest).toBe('l'); + expect(selectMacChild([child('ggml', 'g')]).digest).toBe('g'); + }); + + it('takes the first child of a runner and compares runners as Ollama does', () => { + expect(selectMacChild([child(' MLX ', 'first'), child('mlx', 'second')]).digest).toBe('first'); + }); + + it('skips a nested list and a runner a Mac does not run', () => { + expect(selectMacChild([child('mlx', 'nested', MEDIA_TYPE_MANIFEST_LIST), child('llamacpp', 'l')]).digest).toBe('l'); + expect(selectMacChild([child('vllm', 'v')])).toBeUndefined(); + expect(selectMacChild('not a list')).toBeUndefined(); + }); + + it('accepts a child without a mediaType', () => { + expect(selectMacChild([{ digest: 'm', runner: 'mlx' }]).digest).toBe('m'); + }); +}); + +describe('small helpers', () => { + it('metadataForConfig mirrors Ollama: safetensors runs on mlx, gguf on ggml', () => { + expect(metadataForConfig({ model_format: 'safetensors' })).toEqual({ runner: 'mlx', format: 'safetensors' }); + expect(metadataForConfig({ model_format: 'gguf' })).toEqual({ runner: 'ggml', format: 'gguf' }); + expect(metadataForConfig({})).toEqual({ runner: '', format: '' }); + }); + + it('normalizeDigest accepts the prefixed and the bare form and nothing else', () => { + const bare = '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba'; + expect(normalizeDigest(bare)).toBe(REAL_MLX_CHILD); + expect(normalizeDigest(REAL_MLX_CHILD.toUpperCase().replace('SHA256', 'sha256'))).toBe(REAL_MLX_CHILD); + expect(normalizeDigest('sha256:abc')).toBeUndefined(); + expect(normalizeDigest('sha512:' + 'a'.repeat(64))).toBeUndefined(); + expect(normalizeDigest(42)).toBeUndefined(); + }); + + it('parses and compares versions', () => { + expect(parseVersion('v0.40.15')).toEqual([0, 40, 15]); + expect(parseVersion('0.32.12')).toEqual([0, 32, 12]); + expect(parseVersion('0.40.15-rc1')).toBeUndefined(); + expect(compareVersions('0.40.15', 'v0.40.15')).toBe(0); + expect(compareVersions('0.41.0', '0.40.15')).toBeGreaterThan(0); + expect(compareVersions('0.9.0', '0.40.15')).toBeLessThan(0); + expect(() => compareVersions('latest', '0.40.15')).toThrow(CheckFailure); + }); + + it('formats numbers without the locale', () => { + expect(groupDigits(23_105_798_956)).toBe('23,105,798,956'); + expect(groupDigits(250)).toBe('250'); + expect(formatPercent(0.16614)).toBe('+16.61%'); + expect(formatPercent(-0.02097)).toBe('-2.10%'); + }); + + it('the tolerance is a small fraction of the row', () => { + expect(SIZE_TOLERANCE_FRACTION).toBeLessThanOrEqual(0.01); + expect(withinTolerance(1_000_000, 1_001_000)).toBe(true); + expect(withinTolerance(1_000_000, 1_001_001)).toBe(false); + }); +}); + +describe('classifyModel', () => { + const observed = (bytes: number, extra: Record = {}) => ({ + mac: { bytes, digest: REAL_MLX_CHILD, runner: 'mlx', format: 'safetensors', ...extra }, + }); + + it('a row that holds the pull total itself is clean', () => { + expect(classifyModel({ id: 'qwen3.6:35b-a3b', bytes: MAC_PULL_TOTAL_35B }, observed(MAC_PULL_TOTAL_35B), '0.40.15')).toEqual([]); + }); + + it('a row sized from its layers alone is clean: the config is a few hundred bytes', () => { + expect(classifyModel({ id: 'gpt-oss:20b', bytes: 13_793_441_427 }, observed(13_793_441_566), '0.40.15')).toEqual([]); + }); + + it('flags the two real drifts of 2026-11-08: the Mac gets the mlx build', () => { + const nine = classifyModel({ id: 'qwen3.5:9b', bytes: 6_550_825_373 }, observed(7_639_229_431), '0.40.15'); + expect(nine).toEqual([expect.objectContaining({ kind: 'size', expected: 6_550_825_373, actual: 7_639_229_431 })]); + expect(nine[0].fraction).toBeCloseTo(0.1661, 3); + const dense = classifyModel({ id: 'qwen3.8:27b', bytes: 17_741_871_939 }, observed(18_174_721_847), '0.40.15'); + expect(dense[0].kind).toBe('size'); + expect(dense[0].fraction).toBeCloseTo(0.0244, 3); + }); + + it('a tag that is gone is only that', () => { + expect(classifyModel({ id: 'x:y', bytes: 1 }, { gone: true }, '0.40.15')).toEqual([{ kind: 'gone' }]); + }); + + it('compares a pinned digest in either form', () => { + const bare = REAL_MLX_CHILD.slice('sha256:'.length); + expect(classifyModel({ id: 'a:b', bytes: 100, digest: bare }, observed(100), '0.40.15')).toEqual([]); + expect(classifyModel({ id: 'a:b', bytes: 100, digest: hex('1') }, observed(100), '0.40.15')).toEqual([ + { kind: 'digest', expected: hex('1'), actual: REAL_MLX_CHILD }, + ]); + expect(classifyModel({ id: 'a:b', bytes: 100, digest: 'garbage' }, observed(100), '0.40.15')[0]).toMatchObject({ + kind: 'digest', + expected: 'not a sha256 digest', + }); + }); + + it('flags a build that needs a newer runtime than the pin, and only that', () => { + expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { requires: '0.41.0' }), '0.40.15')).toEqual([ + { kind: 'requires', requires: '0.41.0', pinned: '0.40.15' }, + ]); + expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { requires: '0.40.15' }), '0.40.15')).toEqual([]); + expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100), '0.40.15')).toEqual([]); + }); +}); + +describe('classifyRuntime', () => { + const release = (version: string, asset?: { size: number; digest?: string }) => ({ version, publishedOn: '2026-11-03', asset }); + const matching = { size: PIN.bytes, digest: `sha256:${PIN.sha256}` }; + + it('is clean when the pin is the newest release and its archive is unchanged', () => { + expect(classifyRuntime(PIN, { latest: release('0.40.15', matching), pinned: release('0.40.15', matching) })).toEqual([]); + }); + + it('a size that matches and no digest in the API is not drift', () => { + expect( + classifyRuntime(PIN, { latest: release('0.40.15'), pinned: release('0.40.15', { size: PIN.bytes }) }) + ).toEqual([]); + }); + + it('reports a newer release', () => { + expect(classifyRuntime(PIN, { latest: release('0.41.2'), pinned: release('0.40.15', matching) })).toEqual([ + { kind: 'behind', pinned: '0.40.15', latest: '0.41.2' }, + ]); + }); + + it('reports a pinned release or archive that is gone or changed', () => { + const latest = release('0.40.15'); + expect(classifyRuntime(PIN, { latest, pinned: { gone: true } })).toEqual([{ kind: 'pinned-release-gone' }]); + expect(classifyRuntime(PIN, { latest, pinned: release('0.40.15') })).toEqual([{ kind: 'pinned-asset-gone' }]); + expect( + classifyRuntime(PIN, { latest, pinned: release('0.40.15', { size: PIN.bytes + 1, digest: matching.digest }) })[0].kind + ).toBe('pinned-asset-changed'); + expect( + classifyRuntime(PIN, { latest, pinned: release('0.40.15', { size: PIN.bytes, digest: hex('2') }) })[0].kind + ).toBe('pinned-asset-changed'); + }); +}); + +// ---------- Exit code and report ---------- + +function baseResult(overrides: Record = {}) { + const mac = { bytes: 100, digest: hex('a'), runner: 'mlx', format: 'safetensors', requires: '0.32.4' }; + return { + checkedAt: '2026-11-09 06:00 UTC', + catalogue: { updated: '2026-11-06' }, + pin: { version: '0.40.15' }, + errors: [] as string[], + models: [{ id: 'a:b', bytes: 100, findings: [] as unknown[], observed: { tagDigest: hex('b'), variants: [mac], mac } }], + runtime: { + assetName: 'ollama-darwin.tgz', + findings: [] as unknown[], + observed: { + latest: { version: '0.40.15', publishedOn: '2026-11-03', asset: { size: PIN.bytes, digest: `sha256:${PIN.sha256}` } }, + pinned: { version: '0.40.15', publishedOn: '2026-11-03', asset: { size: PIN.bytes, digest: `sha256:${PIN.sha256}` } }, + }, + } as Record, + ...overrides, + }; +} + +describe('exitCodeFor', () => { + it('is 0 when nothing drifted', () => { + expect(exitCodeFor(baseResult())).toBe(0); + }); + + it('is 1 for drift in a model or in the runtime', () => { + const result = baseResult(); + result.models[0].findings = [{ kind: 'gone' }]; + expect(exitCodeFor(result)).toBe(1); + const runtime = baseResult(); + (runtime.runtime.findings as unknown[]).push({ kind: 'behind', pinned: '0.40.15', latest: '0.41.0' }); + expect(exitCodeFor(runtime)).toBe(1); + }); + + it('is 2 when any part failed, even with drift elsewhere', () => { + const result = baseResult(); + result.models[0].findings = [{ kind: 'gone' }]; + result.models.push({ id: 'c:d', bytes: 1, findings: [], error: 'timed out' } as never); + expect(exitCodeFor(result)).toBe(2); + expect(exitCodeFor(baseResult({ runtime: { error: 'GitHub answered 403', findings: [] } }))).toBe(2); + expect(exitCodeFor(baseResult({ errors: ['the catalogue did not load'] }))).toBe(2); + }); +}); + +describe('renderReport', () => { + it('says clean, and lists the digests to pin in full', () => { + const report = renderReport(baseResult()); + expect(report).toMatch(/^# Local model catalogue: weekly check\n/); + expect(report).toContain('**Clean**'); + expect(report).toContain(`| \`a:b\` | \`${hex('b')}\` | \`${hex('a')}\` |`); + expect(report).toContain('## Not checked'); + }); + + it('names a size drift with both figures and every build the tag offers', () => { + const gguf = { bytes: 90, digest: hex('c'), runner: 'llamacpp', format: 'gguf' }; + const mac = { bytes: 7_639_229_431, digest: hex('a'), runner: 'mlx', format: 'safetensors' }; + const result = baseResult({ + models: [ + { + id: 'qwen3.5:9b', + bytes: 6_550_825_373, + findings: classifyModel({ id: 'qwen3.5:9b', bytes: 6_550_825_373 }, { mac }, '0.40.15'), + observed: { tagDigest: hex('b'), variants: [gguf, mac], mac }, + }, + ], + }); + const report = renderReport(result); + expect(report).toContain('**Drift found**: 1 of 1 models.'); + expect(report).toContain('| `qwen3.5:9b` | 6,550,825,373 | 7,639,229,431 | +16.61% | mlx, safetensors |'); + expect(report).toContain('mlx, safetensors 7,639,229,431 (a Mac takes this one)'); + expect(report).toContain('llamacpp, gguf 90'); + }); + + it('says the check failed and prints no verdict of clean', () => { + const result = baseResult({ runtime: { error: 'GitHub answered 403 for the newest release', findings: [] } }); + const report = renderReport(result); + expect(report).toContain('**The check failed**'); + expect(report).not.toContain('**Clean**'); + expect(report).toContain('GitHub answered 403'); + }); + + it('never prints a runner or format outside the known names', () => { + const mac = { bytes: 100, digest: hex('a'), runner: '', format: '@someone' }; + const report = renderReport(baseResult({ models: [{ id: 'a:b', bytes: 100, findings: [], observed: { tagDigest: hex('b'), variants: [mac], mac } }] })); + expect(report).toContain('unknown, unknown'); + expect(report).not.toContain(' { + const result = baseResult({ errors: ['line one\nline `two`\u0007'] }); + expect(renderReport(result)).toContain("- Check failed: line one line 'two'\n"); + }); + + it('contains no em-dash', () => { + expect(renderReport(baseResult())).not.toContain(String.fromCharCode(0x2014)); + }); +}); + +// ---------- IO against a fake registry ---------- + +interface Seen { + url: string; + headers: Record; + signal: unknown; +} + +/** + * A fake registry and GitHub API. `routes` maps a URL to a body (string or Buffer) or a status + * number. Every request is recorded. + */ +function fakeFetch(routes: Record Response)>) { + const seen: Seen[] = []; + const fetchFn = async (url: string, init: { headers: Record; signal: unknown }) => { + seen.push({ url, headers: init.headers, signal: init.signal }); + const route = routes[url]; + if (route === undefined) return new Response('not found', { status: 404 }); + if (typeof route === 'number') return new Response('', { status: route }); + if (typeof route === 'function') return route(); + return new Response(route, { status: 200 }); + }; + return { fetchFn, seen }; +} + +const REG = 'https://registry.ollama.ai/v2/library'; +const GH = 'https://api.github.com/repos/ollama/ollama/releases'; + +/** A small registry: one tag that is a list (mlx + llamacpp), one plain tag. Digests computed. */ +function buildRegistry() { + const json = (value: unknown): Buffer => Buffer.from(JSON.stringify(value)); + const mlxConfig = json({ model_format: 'safetensors', requires: '0.32.4' }); + const ggufConfig = json({ model_format: 'gguf', requires: '0.30.0' }); + const layers = (sizes: number[], tag: string) => + sizes.map((size, i) => ({ mediaType: 'application/vnd.ollama.image.tensor', digest: hex(String((i + tag.length) % 10)), size })); + const mlxChild = json({ + schemaVersion: 2, + mediaType: MEDIA_TYPE_MANIFEST, + config: { digest: sha256Digest(mlxConfig), size: mlxConfig.length }, + layers: layers([700, 300], 'mlx'), + }); + const ggufChild = json({ + schemaVersion: 2, + mediaType: MEDIA_TYPE_MANIFEST, + config: { digest: sha256Digest(ggufConfig), size: ggufConfig.length }, + layers: layers([600], 'gguf'), + runner: 'llamacpp', + format: 'gguf', + }); + const list = json({ + schemaVersion: 2, + mediaType: MEDIA_TYPE_MANIFEST_LIST, + manifests: [ + { mediaType: MEDIA_TYPE_MANIFEST, digest: sha256Digest(mlxChild), runner: 'mlx', format: 'safetensors' }, + { mediaType: MEDIA_TYPE_MANIFEST, digest: sha256Digest(ggufChild), runner: 'llamacpp', format: 'gguf' }, + ], + }); + const plain = json({ + schemaVersion: 2, + mediaType: MEDIA_TYPE_MANIFEST, + config: { digest: sha256Digest(ggufConfig), size: ggufConfig.length }, + layers: layers([500], 'plain'), + }); + const release = (tag: string) => + json({ + tag_name: tag, + published_at: '2026-11-03T22:52:31Z', + assets: [{ name: 'ollama-darwin.tgz', size: PIN.bytes, digest: `sha256:${PIN.sha256}` }], + }); + const routes: Record Response)> = { + [`${REG}/listed/manifests/1b`]: list, + [`${REG}/listed/blobs/${sha256Digest(mlxChild)}`]: mlxChild, + [`${REG}/listed/blobs/${sha256Digest(ggufChild)}`]: ggufChild, + [`${REG}/listed/blobs/${sha256Digest(mlxConfig)}`]: mlxConfig, + [`${REG}/plain/manifests/2b`]: plain, + [`${REG}/plain/blobs/${sha256Digest(ggufConfig)}`]: ggufConfig, + [`${GH}/latest`]: release('v0.40.15'), + [`${GH}/tags/v0.40.15`]: release('v0.40.15'), + }; + const layerDigests = new Set( + [mlxChild, ggufChild, plain].flatMap((body) => JSON.parse(body.toString()).layers.map((l: { digest: string }) => l.digest)) + ); + return { + routes, + layerDigests, + sizes: { mlx: 1000 + mlxConfig.length, gguf: 600 + ggufConfig.length, plain: 500 + ggufConfig.length }, + digests: { list: sha256Digest(list), mlxChild: sha256Digest(mlxChild), plain: sha256Digest(plain) }, + }; +} + +const catalogOf = (models: { id: string; bytes: number; digest?: string }[]) => ({ updated: '2026-11-06', models }); +const NOW = new Date('2026-11-09T06:00:00Z'); + +describe('inspectModel', () => { + it('reads a list the way a Mac does: the mlx child, its total, its requires', async () => { + const registry = buildRegistry(); + const { fetchFn, seen } = fakeFetch(registry.routes); + const observed = await inspectModel(fetchFn, 'listed:1b'); + expect(observed.tagDigest).toBe(registry.digests.list); + expect(observed.mac).toMatchObject({ + runner: 'mlx', + format: 'safetensors', + digest: registry.digests.mlxChild, + bytes: registry.sizes.mlx, + requires: '0.32.4', + }); + expect(observed.variants.map((variant: { runner: string }) => variant.runner)).toEqual(['mlx', 'llamacpp']); + expect(seen[0].headers.Accept).toBe(MANIFEST_ACCEPT); + expect(seen[0].headers.Accept).toContain(MEDIA_TYPE_MANIFEST_LIST); + expect(seen.every((request) => request.signal instanceof AbortSignal)).toBe(true); + }); + + it('fills a plain manifest without runner metadata from its config', async () => { + const registry = buildRegistry(); + const { fetchFn } = fakeFetch(registry.routes); + const observed = await inspectModel(fetchFn, 'plain:2b'); + expect(observed.mac).toMatchObject({ runner: 'ggml', format: 'gguf', digest: registry.digests.plain, bytes: registry.sizes.plain }); + expect(observed.tagDigest).toBe(observed.mac.digest); + }); + + it('a 404 for the manifest is a tag that is gone', async () => { + const { fetchFn } = fakeFetch({}); + expect(await inspectModel(fetchFn, 'missing:tag')).toEqual({ gone: true }); + }); + + it('refuses a blob whose body does not match its digest', async () => { + const registry = buildRegistry(); + const mlxUrl = `${REG}/listed/blobs/${registry.digests.mlxChild}`; + const { fetchFn } = fakeFetch({ ...registry.routes, [mlxUrl]: '{"tampered":true}' }); + await expect(inspectModel(fetchFn, 'listed:1b')).rejects.toThrow(/does not match its digest/); + }); + + it('refuses a list with no build a Mac runs', async () => { + const list = JSON.stringify({ mediaType: MEDIA_TYPE_MANIFEST_LIST, manifests: [{ digest: hex('1'), runner: 'vllm' }] }); + const { fetchFn } = fakeFetch({ [`${REG}/odd/manifests/x`]: list }); + await expect(inspectModel(fetchFn, 'odd:x')).rejects.toThrow(/no build a Mac runs/); + }); + + it('a server error is a failed check, not a tag that is gone', async () => { + const { fetchFn } = fakeFetch({ [`${REG}/flaky/manifests/x`]: 503 }); + await expect(inspectModel(fetchFn, 'flaky:x')).rejects.toThrow(/answered 503/); + }); +}); + +describe('boundedGet', () => { + it('refuses a declared body over the cap before reading it', async () => { + const { fetchFn } = fakeFetch({ + 'https://x.test/big': () => new Response('{}', { status: 200, headers: { 'content-length': String(MAX_MANIFEST_BYTES + 1) } }), + }); + await expect(boundedGet(fetchFn, 'https://x.test/big', { maxBytes: MAX_MANIFEST_BYTES })).rejects.toThrow(/over the cap/); + }); + + it('refuses a streamed body that grows past the cap', async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(600)); + controller.enqueue(new Uint8Array(600)); + controller.close(); + }, + }); + const { fetchFn } = fakeFetch({ 'https://x.test/stream': () => new Response(stream, { status: 200 }) }); + await expect(boundedGet(fetchFn, 'https://x.test/stream', { maxBytes: 1000 })).rejects.toThrow(/over the cap/); + }); + + it('turns a network error into a CheckFailure', async () => { + const fetchFn = async () => { + throw new TypeError('fetch failed', { cause: new Error('getaddrinfo ENOTFOUND registry.ollama.ai') }); + }; + await expect(boundedGet(fetchFn, 'https://registry.ollama.ai/v2/', { maxBytes: 10 })).rejects.toThrow(/ENOTFOUND/); + }); +}); + +describe('runCheck', () => { + it('reports drift, and fetches only manifests, child manifests, configs and the two releases', async () => { + const registry = buildRegistry(); + const { fetchFn, seen } = fakeFetch(registry.routes); + const result = await runCheck({ + catalog: catalogOf([ + // Sized from the llamacpp child: a Mac pulls the mlx one, so this is drift. + { id: 'listed:1b', bytes: registry.sizes.gguf }, + { id: 'plain:2b', bytes: registry.sizes.plain }, + { id: 'missing:tag', bytes: 1000 }, + ]), + pin: PIN, + fetchFn, + now: NOW, + githubToken: 'token-value', + }); + + expect(result.models.map((model: { findings: { kind: string }[] }) => model.findings.map((f) => f.kind))).toEqual([ + ['size'], + [], + ['gone'], + ]); + expect(result.runtime.findings).toEqual([]); + expect(exitCodeFor(result)).toBe(1); + + for (const request of seen) { + const blob = /\/blobs\/(sha256:[0-9a-f]{64})$/.exec(request.url); + if (blob) expect(registry.layerDigests.has(blob[1])).toBe(false); + const isGitHub = request.url.startsWith('https://api.github.com/'); + expect(request.headers.Authorization === 'Bearer token-value').toBe(isGitHub); + } + expect(seen.some((request) => request.url === `${GH}/latest`)).toBe(true); + expect(seen.some((request) => request.url === `${GH}/tags/v0.40.15`)).toBe(true); + + const report = renderReport(result); + expect(report).toContain('**Drift found**: 2 of 3 models.'); + expect(report).toContain('`missing:tag`: the registry no longer has this tag'); + }); + + it('sends no Authorization header at all without a token', async () => { + const registry = buildRegistry(); + const { fetchFn, seen } = fakeFetch(registry.routes); + await runCheck({ catalog: catalogOf([{ id: 'plain:2b', bytes: registry.sizes.plain }]), pin: PIN, fetchFn, now: NOW }); + expect(seen.some((request) => 'Authorization' in request.headers)).toBe(false); + }); + + it('is clean when every row matches what a Mac pulls', async () => { + const registry = buildRegistry(); + const { fetchFn } = fakeFetch(registry.routes); + const result = await runCheck({ + catalog: catalogOf([ + { id: 'listed:1b', bytes: registry.sizes.mlx, digest: registry.digests.mlxChild }, + { id: 'plain:2b', bytes: registry.sizes.plain }, + ]), + pin: PIN, + fetchFn, + now: NOW, + }); + expect(exitCodeFor(result)).toBe(0); + const report = renderReport(result); + expect(report).toContain('**Clean**'); + // The pinned row is not offered for pinning again; the unpinned one is. + expect(report).not.toContain(`| \`listed:1b\` | \``); + expect(report).toContain(`| \`plain:2b\` | \`${registry.digests.plain}\``); + }); + + it('a newer runtime release is drift', async () => { + const registry = buildRegistry(); + registry.routes[`${GH}/latest`] = JSON.stringify({ tag_name: 'v0.41.0', published_at: '2026-11-10T00:00:00Z', assets: [] }); + const { fetchFn } = fakeFetch(registry.routes); + const result = await runCheck({ catalog: catalogOf([{ id: 'plain:2b', bytes: registry.sizes.plain }]), pin: PIN, fetchFn, now: NOW }); + expect(result.runtime.findings).toEqual([{ kind: 'behind', pinned: '0.40.15', latest: '0.41.0' }]); + expect(exitCodeFor(result)).toBe(1); + expect(renderReport(result)).toContain('Ollama 0.41.0 is out; the pin is 0.40.15.'); + }); + + it('a rate-limited GitHub API fails the check (2) and keeps what the models showed', async () => { + const registry = buildRegistry(); + registry.routes[`${GH}/latest`] = 403; + const { fetchFn } = fakeFetch(registry.routes); + const result = await runCheck({ + catalog: catalogOf([{ id: 'listed:1b', bytes: registry.sizes.gguf }]), + pin: PIN, + fetchFn, + now: NOW, + }); + expect(exitCodeFor(result)).toBe(2); + expect(result.models[0].findings[0].kind).toBe('size'); + expect(renderReport(result)).toContain('**The check failed**'); + }); + + it('a release tag that is not vX.Y.Z fails the check instead of reaching the report', async () => { + const registry = buildRegistry(); + registry.routes[`${GH}/latest`] = JSON.stringify({ tag_name: 'v1.0.0 @everyone', assets: [] }); + const { fetchFn } = fakeFetch(registry.routes); + const result = await runCheck({ catalog: catalogOf([]), pin: PIN, fetchFn, now: NOW }); + expect(exitCodeFor(result)).toBe(2); + expect(renderReport(result)).not.toContain('@everyone'); + }); + + it('a network failure for one row fails the whole check', async () => { + const registry = buildRegistry(); + const fetchFn = async (url: string, init: { headers: Record; signal: unknown }) => { + if (url.includes('/plain/')) throw new TypeError('fetch failed'); + return fakeFetch(registry.routes).fetchFn(url, init); + }; + const result = await runCheck({ + catalog: catalogOf([ + { id: 'listed:1b', bytes: registry.sizes.mlx }, + { id: 'plain:2b', bytes: registry.sizes.plain }, + ]), + pin: PIN, + fetchFn, + now: NOW, + }); + expect(result.models[1].error).toMatch(/fetch failed/); + expect(exitCodeFor(result)).toBe(2); + }); +}); diff --git a/test/fixtures/local-model-catalog/qwen3.6-35b-a3b.gguf-manifest.json b/test/fixtures/local-model-catalog/qwen3.6-35b-a3b.gguf-manifest.json new file mode 100644 index 00000000..8fede658 --- /dev/null +++ b/test/fixtures/local-model-catalog/qwen3.6-35b-a3b.gguf-manifest.json @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:99afa5bbf1e7038bda7d028e100ee76b7337c943468d52590de3282db7b52adb","size":220},"layers":[{"mediaType":"application/vnd.ollama.image.model","digest":"sha256:d372de8e934898a59e6ccfabc3368474711384d8f1fd4d22d87a3f0a45400cdc","size":21718480960,"from":"qwen3.6-source:35b-a3b-mtp-q4_K_M-20260824"},{"mediaType":"application/vnd.ollama.image.projector","digest":"sha256:a62390d25b4b4a2d8afd7cc3c90021c11935c1fdd48d7cb0723ab71cd02a598e","size":902821728,"from":"qwen3.6-source:35b-a3b-mtp-q4_K_M-20260824"},{"mediaType":"application/vnd.ollama.image.license","digest":"sha256:5f3a3c817e78f5b8a4ad2d2c458a3e4b2cce470d6c12642c4eeb12cb8a9bf51d","size":11357,"from":"qwen3.6-source:35b-a3b-mtp-q4_K_M-20260824"},{"mediaType":"application/vnd.ollama.image.params","digest":"sha256:ce44c0f3ca1f7a126724e5285be7e1afdab202186c6379515ab0c1ffd40d3ecc","size":116}],"runner":"llamacpp","format":"gguf"} \ No newline at end of file diff --git a/test/fixtures/local-model-catalog/qwen3.6-35b-a3b.list.json b/test/fixtures/local-model-catalog/qwen3.6-35b-a3b.list.json new file mode 100644 index 00000000..c11859f4 --- /dev/null +++ b/test/fixtures/local-model-catalog/qwen3.6-35b-a3b.list.json @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.ollama.manifest.list.v2+json","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba","runner":"mlx","format":"safetensors"},{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:a7eb95c53bcf96b4bdd008d0fab4a5dac88047d9c1a7a9ab88ed453423fbd87c","runner":"llamacpp","format":"gguf"}]} \ No newline at end of file diff --git a/test/fixtures/local-model-catalog/qwen3.6-35b-a3b.mlx-config.json b/test/fixtures/local-model-catalog/qwen3.6-35b-a3b.mlx-config.json new file mode 100644 index 00000000..7f4cfa80 --- /dev/null +++ b/test/fixtures/local-model-catalog/qwen3.6-35b-a3b.mlx-config.json @@ -0,0 +1 @@ +{"architecture":"","capabilities":["completion","vision","tools","thinking"],"file_type":"nvfp4","model_families":null,"model_family":"","model_format":"safetensors","model_type":"","os":"","parser":"qwen3.5","renderer":"qwen3.5","requires":"0.32.4"} \ No newline at end of file