From e6ed86771eef931b217786b8e7bc5f7bae79d4e6 Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Sun, 8 Nov 2026 10:30:06 +0100 Subject: [PATCH] ci(local-model): hold a row that names its runner to the build a Mac pulls The catalogue that names each row's runner (feat/local-model-catalog-v2: `runner` is `mlx` or `llamacpp`, and a row's memory figures follow that runner's rule) needs the weekly check to notice when a tag starts serving a Mac another runner: a tag that gains an mlx child, or a list that loses it. The size check usually catches that, but not when the two builds come out about the same size, and the figures are wrong either way. classifyModel() now raises a `runner` finding when the row names a runner and the build a Mac selects runs on another one. catalogueRunner() maps the registry's names into the catalogue's: `mlx` stays `mlx`; `llamacpp`, and `ggml` (what Ollama calls a GGUF manifest that names no runner), are `llamacpp`, because Ollama 0.40.15 serves every GGUF build through llama-server. A catalogue without the field (the one on this branch today) is never held to a runner. Run live on 2026-11-08 against the v2 catalogue's eight rows: exit 0, every runner matches. Tests: the mapping, the finding (both directions, ggml, an unknown runner, a row without the field), the report text, and runCheck against the fake registry with a plain GGUF manifest. Co-Authored-By: Claude Opus 5.5 --- docs/local-models.md | 4 ++ scripts/check-local-model-catalog.mjs | 53 +++++++++++++--- test/check-local-model-catalog.test.ts | 83 +++++++++++++++++++++++++- 3 files changed, 131 insertions(+), 9 deletions(-) diff --git a/docs/local-models.md b/docs/local-models.md index 39887565..ef8b226f 100644 --- a/docs/local-models.md +++ b/docs/local-models.md @@ -270,6 +270,10 @@ For every model row it checks: Ollama asks, picks the build a Mac picks and sizes it the way a pull counts it (every layer plus the config). The plain manifest other tools get for the same tag is the `llamacpp` build, which is not what a Mac downloads. +- **The runner of that build**, once a row names its `runner`: a Mac still pulls an `mlx` + build for an `mlx` row and a GGUF build (`llamacpp`, or a manifest that names no runner) + for a `llamacpp` row. A row's memory figures follow its runner, so a tag that gains or + loses an mlx build is drift even at about the same size. - **The build a Mac pulls runs on the pinned runtime**: its config declares no newer Ollama (`requires`) than `OLLAMA_RUNTIME_PIN`. - **The pinned digest**, once a row carries one: a Mac would still pull that manifest. The diff --git a/scripts/check-local-model-catalog.mjs b/scripts/check-local-model-catalog.mjs index 7d45552a..ffe7e2af 100644 --- a/scripts/check-local-model-catalog.mjs +++ b/scripts/check-local-model-catalog.mjs @@ -23,7 +23,10 @@ * which is NOT what a Mac pulls: that is how `qwen3.6:35b-a3b` came to be "2% above its * manifest" in the catalogue header. So this check asks the way Ollama asks, picks the child * the way a Mac picks it, and compares the row's `bytes` with that child's pull total (every - * layer plus the config, which is exactly the `total` a real pull reported on a Mac). + * layer plus the config, which is exactly the `total` a real pull reported on a Mac). A row + * that names its `runner` is also held to that child's runner (`catalogueRunner()`), because + * the memory figures of a row follow its runner and a change of runner can come with almost + * the same size. * * WHAT IS FETCHED, and nothing else: * - `GET /v2/library//manifests/`, the tag's manifest or manifest list. @@ -195,6 +198,18 @@ export function metadataForConfig(config) { return { runner: '', format }; } +/** + * A runner name in the catalogue's words (`LocalModelEntry.runner`): `mlx`, or `llamacpp` for a + * GGUF build, which Ollama 0.40.15 serves through llama-server whether the manifest names it + * `llamacpp` or names nothing (`ggml`, see `metadataForConfig()`). Undefined for anything else. + */ +export function catalogueRunner(runner) { + const name = canonicalRunner(runner); + if (name === 'mlx') return 'mlx'; + if (name === 'llamacpp' || name === 'ggml') return 'llamacpp'; + return undefined; +} + /** `[major, minor, patch]` of a version or `v`-tag, or undefined when it is not one. */ export function parseVersion(text) { const match = typeof text === 'string' ? VERSION_PATTERN.exec(text.trim()) : null; @@ -221,11 +236,14 @@ export function withinTolerance(expected, actual, tolerance = SIZE_TOLERANCE_FRA } /** - * The findings for one catalogue row. `row` is `{ id, bytes, digest? }` (`digest` only once the - * catalogue stores one); `observed` is what `inspectModel()` returned for it. A row may have - * several findings; none means it is clean. + * The findings for one catalogue row. `row` is `{ id, bytes, runner?, digest? }` (`runner` once + * the catalogue names it, `digest` only once the catalogue stores one); `observed` is what + * `inspectModel()` returned for it. A row may have several findings; none means it is clean. * * - `gone`: the tag is not in the registry, so a pull fails. + * - `runner`: a Mac now pulls a build of another runner than the row names (a tag that became + * a list with an mlx child, or the other way round). The two runners spend memory + * differently, so the row's memory figures no longer describe what runs, whatever the size. * - `size`: what a Mac downloads is outside the tolerance of the row's `bytes`. * - `digest`: the row pins a digest and a Mac would now pull another one. * - `requires`: the build a Mac pulls declares a newer Ollama than the pinned runtime, so the @@ -235,6 +253,10 @@ export function classifyModel(row, observed, pinnedVersion) { if (observed.gone) return [{ kind: 'gone' }]; const findings = []; const mac = observed.mac; + if (row.runner !== undefined) { + const actual = catalogueRunner(mac.runner) ?? 'unknown'; + if (actual !== row.runner) findings.push({ kind: 'runner', expected: String(row.runner), actual }); + } if (!withinTolerance(row.bytes, mac.bytes)) { findings.push({ kind: 'size', expected: row.bytes, actual: mac.bytes, fraction: sizeDifference(row.bytes, mac.bytes) }); } @@ -314,7 +336,13 @@ function buildName(variant) { function modelResult(model) { if (model.error) return 'check failed'; if (model.findings.length === 0) return 'ok'; - const names = { gone: 'tag gone', size: 'size drift', digest: 'digest changed', requires: 'needs a newer runtime' }; + const names = { + gone: 'tag gone', + runner: 'runner changed', + size: 'size drift', + digest: 'digest changed', + requires: 'needs a newer runtime', + }; return model.findings.map((finding) => names[finding.kind]).join(', '); } @@ -323,6 +351,13 @@ function describeModelFinding(model, finding) { switch (finding.kind) { case 'gone': return `${id}: the registry no longer has this tag, so the download fails. Replace or remove the row.`; + case 'runner': + return ( + `${id}: a Mac now pulls the ${buildName(model.observed.mac)} build, which runs on ` + + `\`${finding.actual}\`, while the row's \`runner\` says \`${displayName(finding.expected, ['mlx', 'llamacpp'])}\`. ` + + 'The two runners spend memory differently, so the row\'s memory figures no longer describe what runs: ' + + 'recompute them for the new runner, or pin the row to a single-build tag.' + ); case 'size': return ( `${id}: a Mac downloads the ${buildName(model.observed.mac)} build, ${groupDigits(finding.actual)} bytes, ` + @@ -423,7 +458,7 @@ export function renderReport(result) { if (model.error) details.push(`- \`${model.id}\`: check failed: ${oneLine(model.error)}`); for (const finding of model.findings) details.push(`- ${describeModelFinding(model, finding)}`); const variants = model.observed?.variants ?? []; - if (model.findings.some((finding) => finding.kind === 'size') && variants.length > 1) { + if (model.findings.some((finding) => finding.kind === 'size' || finding.kind === 'runner') && variants.length > 1) { const offered = variants .map((variant) => `${buildName(variant)} ${groupDigits(variant.bytes)}${variant === model.observed.mac ? ' (a Mac takes this one)' : ''}`) .join('; '); @@ -695,12 +730,14 @@ export async function runCheck({ catalog, pin, fetchFn, now, githubToken }) { // Rows in parallel, each row's requests one after another: a handful of rows, a few requests each. result.models = await Promise.all( catalog.models.map(async (row) => { - // `digest` is not in the catalogue schema yet: see the file header. + // `digest` is not in the catalogue schema yet: see the file header. `runner` is in the + // schema from the catalogue that names each row's runner on; before that it is absent. const digest = /** @type {{ digest?: unknown }} */ (row).digest; + const runner = /** @type {{ runner?: unknown }} */ (row).runner; const entry = { id: row.id, bytes: row.bytes, figures: row.figures, digest, findings: [] }; try { entry.observed = await inspectModel(fetchFn, row.id); - entry.findings = classifyModel({ id: row.id, bytes: row.bytes, digest }, entry.observed, pin.version); + entry.findings = classifyModel({ id: row.id, bytes: row.bytes, runner, digest }, entry.observed, pin.version); } catch (err) { entry.error = err instanceof CheckFailure ? err.message : describeError(err); } diff --git a/test/check-local-model-catalog.test.ts b/test/check-local-model-catalog.test.ts index 176e865c..f0706506 100644 --- a/test/check-local-model-catalog.test.ts +++ b/test/check-local-model-catalog.test.ts @@ -20,6 +20,7 @@ import { MEDIA_TYPE_MANIFEST_LIST, SIZE_TOLERANCE_FRACTION, boundedGet, + catalogueRunner, classifyModel, classifyRuntime, compareVersions, @@ -153,6 +154,16 @@ describe('small helpers', () => { expect(metadataForConfig({})).toEqual({ runner: '', format: '' }); }); + it('catalogueRunner names a build in the catalogue words: mlx, or llamacpp for any GGUF build', () => { + expect(catalogueRunner('mlx')).toBe('mlx'); + expect(catalogueRunner(' MLX ')).toBe('mlx'); + expect(catalogueRunner('llamacpp')).toBe('llamacpp'); + // A GGUF manifest that names no runner is `ggml` to Ollama, and llama-server runs it all the same. + expect(catalogueRunner('ggml')).toBe('llamacpp'); + expect(catalogueRunner(metadataForConfig({ model_format: 'gguf' }).runner)).toBe('llamacpp'); + for (const other of ['', 'vllm', '', undefined, 3]) expect(catalogueRunner(other)).toBeUndefined(); + }); + it('normalizeDigest accepts the prefixed and the bare form and nothing else', () => { const bare = '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba'; expect(normalizeDigest(bare)).toBe(REAL_MLX_CHILD); @@ -224,6 +235,28 @@ describe('classifyModel', () => { }); }); + it('holds a row that names its runner to the runner of the build a Mac pulls', () => { + // The same size, another runner: the row's memory figures no longer describe what runs. + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'llamacpp' }, observed(100), '0.40.15')).toEqual([ + { kind: 'runner', expected: 'llamacpp', actual: 'mlx' }, + ]); + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'mlx' }, observed(100), '0.40.15')).toEqual([]); + // A GGUF build is a llamacpp row's, whether its manifest says llamacpp or nothing (ggml). + for (const runner of ['llamacpp', 'ggml']) { + const gguf = observed(100, { runner, format: 'gguf' }); + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'llamacpp' }, gguf, '0.40.15'), runner).toEqual([]); + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'mlx' }, gguf, '0.40.15'), runner).toEqual([ + { kind: 'runner', expected: 'mlx', actual: 'llamacpp' }, + ]); + } + // A runner Ollama would not run on a Mac is never taken for the row's. + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'mlx' }, observed(100, { runner: '' }), '0.40.15')).toEqual([ + { kind: 'runner', expected: 'mlx', actual: 'unknown' }, + ]); + // A catalogue without the field (before rows named their runner) is never held to one. + expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { runner: 'llamacpp' }), '0.40.15')).toEqual([]); + }); + it('flags a build that needs a newer runtime than the pin, and only that', () => { expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { requires: '0.41.0' }), '0.40.15')).toEqual([ { kind: 'requires', requires: '0.41.0', pinned: '0.40.15' }, @@ -349,6 +382,29 @@ describe('renderReport', () => { expect(report).toContain('llamacpp, gguf 90'); }); + it('names a runner change, what it means for the figures, and every build the tag offers', () => { + const gguf = { bytes: 100, digest: hex('c'), runner: 'llamacpp', format: 'gguf' }; + const mac = { bytes: 100, digest: hex('a'), runner: 'mlx', format: 'safetensors' }; + const result = baseResult({ + models: [ + { + id: 'qwen3.6:35b-a3b-mtp-q4_k_m', + bytes: 100, + findings: classifyModel({ id: 'x:y', bytes: 100, runner: 'llamacpp' }, { mac }, '0.40.15'), + observed: { tagDigest: hex('b'), variants: [gguf, mac], mac }, + }, + ], + }); + const report = renderReport(result); + expect(report).toContain('**Drift found**: 1 of 1 models.'); + expect(report).toContain('| runner changed |'); + expect(report).toContain( + "`qwen3.6:35b-a3b-mtp-q4_k_m`: a Mac now pulls the mlx, safetensors build, which runs on `mlx`, while the row's `runner` says `llamacpp`." + ); + expect(report).toContain('recompute them for the new runner'); + expect(report).toContain('mlx, safetensors 100 (a Mac takes this one)'); + }); + it('says the check failed and prints no verdict of clean', () => { const result = baseResult({ runtime: { error: 'GitHub answered 403 for the newest release', findings: [] } }); const report = renderReport(result); @@ -465,7 +521,10 @@ function buildRegistry() { }; } -const catalogOf = (models: { id: string; bytes: number; digest?: string; figures?: string }[]) => ({ updated: '2026-11-06', models }); +const catalogOf = (models: { id: string; bytes: number; digest?: string; figures?: string; runner?: string }[]) => ({ + updated: '2026-11-06', + models, +}); const NOW = new Date('2026-11-09T06:00:00Z'); describe('inspectModel', () => { @@ -586,6 +645,28 @@ describe('runCheck', () => { expect(report).toContain('`missing:tag`: the registry no longer has this tag'); }); + it('holds rows that name their runner to the build a Mac pulls, a plain GGUF manifest included', async () => { + const registry = buildRegistry(); + const { fetchFn } = fakeFetch(registry.routes); + const result = await runCheck({ + catalog: catalogOf([ + // The right size, the wrong runner: a Mac pulls the mlx child of this list. + { id: 'listed:1b', bytes: registry.sizes.mlx, runner: 'llamacpp' }, + // A manifest that names no runner is a GGUF build: a llamacpp row, clean. + { id: 'plain:2b', bytes: registry.sizes.plain, runner: 'llamacpp' }, + ]), + pin: PIN, + fetchFn, + now: NOW, + }); + expect(result.models.map((model: { findings: unknown[] }) => model.findings)).toEqual([ + [{ kind: 'runner', expected: 'llamacpp', actual: 'mlx' }], + [], + ]); + expect(exitCodeFor(result)).toBe(1); + expect(renderReport(result)).toContain('| runner changed |'); + }); + it('sends no Authorization header at all without a token', async () => { const registry = buildRegistry(); const { fetchFn, seen } = fakeFetch(registry.routes);