diff --git a/docs/local-models.md b/docs/local-models.md index 39887565..ef8b226f 100644 --- a/docs/local-models.md +++ b/docs/local-models.md @@ -270,6 +270,10 @@ For every model row it checks: Ollama asks, picks the build a Mac picks and sizes it the way a pull counts it (every layer plus the config). The plain manifest other tools get for the same tag is the `llamacpp` build, which is not what a Mac downloads. +- **The runner of that build**, once a row names its `runner`: a Mac still pulls an `mlx` + build for an `mlx` row and a GGUF build (`llamacpp`, or a manifest that names no runner) + for a `llamacpp` row. A row's memory figures follow its runner, so a tag that gains or + loses an mlx build is drift even at about the same size. - **The build a Mac pulls runs on the pinned runtime**: its config declares no newer Ollama (`requires`) than `OLLAMA_RUNTIME_PIN`. - **The pinned digest**, once a row carries one: a Mac would still pull that manifest. The diff --git a/scripts/check-local-model-catalog.mjs b/scripts/check-local-model-catalog.mjs index 7d45552a..ffe7e2af 100644 --- a/scripts/check-local-model-catalog.mjs +++ b/scripts/check-local-model-catalog.mjs @@ -23,7 +23,10 @@ * which is NOT what a Mac pulls: that is how `qwen3.6:35b-a3b` came to be "2% above its * manifest" in the catalogue header. So this check asks the way Ollama asks, picks the child * the way a Mac picks it, and compares the row's `bytes` with that child's pull total (every - * layer plus the config, which is exactly the `total` a real pull reported on a Mac). + * layer plus the config, which is exactly the `total` a real pull reported on a Mac). A row + * that names its `runner` is also held to that child's runner (`catalogueRunner()`), because + * the memory figures of a row follow its runner and a change of runner can come with almost + * the same size. * * WHAT IS FETCHED, and nothing else: * - `GET /v2/library//manifests/`, the tag's manifest or manifest list. @@ -195,6 +198,18 @@ export function metadataForConfig(config) { return { runner: '', format }; } +/** + * A runner name in the catalogue's words (`LocalModelEntry.runner`): `mlx`, or `llamacpp` for a + * GGUF build, which Ollama 0.40.15 serves through llama-server whether the manifest names it + * `llamacpp` or names nothing (`ggml`, see `metadataForConfig()`). Undefined for anything else. + */ +export function catalogueRunner(runner) { + const name = canonicalRunner(runner); + if (name === 'mlx') return 'mlx'; + if (name === 'llamacpp' || name === 'ggml') return 'llamacpp'; + return undefined; +} + /** `[major, minor, patch]` of a version or `v`-tag, or undefined when it is not one. */ export function parseVersion(text) { const match = typeof text === 'string' ? VERSION_PATTERN.exec(text.trim()) : null; @@ -221,11 +236,14 @@ export function withinTolerance(expected, actual, tolerance = SIZE_TOLERANCE_FRA } /** - * The findings for one catalogue row. `row` is `{ id, bytes, digest? }` (`digest` only once the - * catalogue stores one); `observed` is what `inspectModel()` returned for it. A row may have - * several findings; none means it is clean. + * The findings for one catalogue row. `row` is `{ id, bytes, runner?, digest? }` (`runner` once + * the catalogue names it, `digest` only once the catalogue stores one); `observed` is what + * `inspectModel()` returned for it. A row may have several findings; none means it is clean. * * - `gone`: the tag is not in the registry, so a pull fails. + * - `runner`: a Mac now pulls a build of another runner than the row names (a tag that became + * a list with an mlx child, or the other way round). The two runners spend memory + * differently, so the row's memory figures no longer describe what runs, whatever the size. * - `size`: what a Mac downloads is outside the tolerance of the row's `bytes`. * - `digest`: the row pins a digest and a Mac would now pull another one. * - `requires`: the build a Mac pulls declares a newer Ollama than the pinned runtime, so the @@ -235,6 +253,10 @@ export function classifyModel(row, observed, pinnedVersion) { if (observed.gone) return [{ kind: 'gone' }]; const findings = []; const mac = observed.mac; + if (row.runner !== undefined) { + const actual = catalogueRunner(mac.runner) ?? 'unknown'; + if (actual !== row.runner) findings.push({ kind: 'runner', expected: String(row.runner), actual }); + } if (!withinTolerance(row.bytes, mac.bytes)) { findings.push({ kind: 'size', expected: row.bytes, actual: mac.bytes, fraction: sizeDifference(row.bytes, mac.bytes) }); } @@ -314,7 +336,13 @@ function buildName(variant) { function modelResult(model) { if (model.error) return 'check failed'; if (model.findings.length === 0) return 'ok'; - const names = { gone: 'tag gone', size: 'size drift', digest: 'digest changed', requires: 'needs a newer runtime' }; + const names = { + gone: 'tag gone', + runner: 'runner changed', + size: 'size drift', + digest: 'digest changed', + requires: 'needs a newer runtime', + }; return model.findings.map((finding) => names[finding.kind]).join(', '); } @@ -323,6 +351,13 @@ function describeModelFinding(model, finding) { switch (finding.kind) { case 'gone': return `${id}: the registry no longer has this tag, so the download fails. Replace or remove the row.`; + case 'runner': + return ( + `${id}: a Mac now pulls the ${buildName(model.observed.mac)} build, which runs on ` + + `\`${finding.actual}\`, while the row's \`runner\` says \`${displayName(finding.expected, ['mlx', 'llamacpp'])}\`. ` + + 'The two runners spend memory differently, so the row\'s memory figures no longer describe what runs: ' + + 'recompute them for the new runner, or pin the row to a single-build tag.' + ); case 'size': return ( `${id}: a Mac downloads the ${buildName(model.observed.mac)} build, ${groupDigits(finding.actual)} bytes, ` + @@ -423,7 +458,7 @@ export function renderReport(result) { if (model.error) details.push(`- \`${model.id}\`: check failed: ${oneLine(model.error)}`); for (const finding of model.findings) details.push(`- ${describeModelFinding(model, finding)}`); const variants = model.observed?.variants ?? []; - if (model.findings.some((finding) => finding.kind === 'size') && variants.length > 1) { + if (model.findings.some((finding) => finding.kind === 'size' || finding.kind === 'runner') && variants.length > 1) { const offered = variants .map((variant) => `${buildName(variant)} ${groupDigits(variant.bytes)}${variant === model.observed.mac ? ' (a Mac takes this one)' : ''}`) .join('; '); @@ -695,12 +730,14 @@ export async function runCheck({ catalog, pin, fetchFn, now, githubToken }) { // Rows in parallel, each row's requests one after another: a handful of rows, a few requests each. result.models = await Promise.all( catalog.models.map(async (row) => { - // `digest` is not in the catalogue schema yet: see the file header. + // `digest` is not in the catalogue schema yet: see the file header. `runner` is in the + // schema from the catalogue that names each row's runner on; before that it is absent. const digest = /** @type {{ digest?: unknown }} */ (row).digest; + const runner = /** @type {{ runner?: unknown }} */ (row).runner; const entry = { id: row.id, bytes: row.bytes, figures: row.figures, digest, findings: [] }; try { entry.observed = await inspectModel(fetchFn, row.id); - entry.findings = classifyModel({ id: row.id, bytes: row.bytes, digest }, entry.observed, pin.version); + entry.findings = classifyModel({ id: row.id, bytes: row.bytes, runner, digest }, entry.observed, pin.version); } catch (err) { entry.error = err instanceof CheckFailure ? err.message : describeError(err); } diff --git a/test/check-local-model-catalog.test.ts b/test/check-local-model-catalog.test.ts index 176e865c..f0706506 100644 --- a/test/check-local-model-catalog.test.ts +++ b/test/check-local-model-catalog.test.ts @@ -20,6 +20,7 @@ import { MEDIA_TYPE_MANIFEST_LIST, SIZE_TOLERANCE_FRACTION, boundedGet, + catalogueRunner, classifyModel, classifyRuntime, compareVersions, @@ -153,6 +154,16 @@ describe('small helpers', () => { expect(metadataForConfig({})).toEqual({ runner: '', format: '' }); }); + it('catalogueRunner names a build in the catalogue words: mlx, or llamacpp for any GGUF build', () => { + expect(catalogueRunner('mlx')).toBe('mlx'); + expect(catalogueRunner(' MLX ')).toBe('mlx'); + expect(catalogueRunner('llamacpp')).toBe('llamacpp'); + // A GGUF manifest that names no runner is `ggml` to Ollama, and llama-server runs it all the same. + expect(catalogueRunner('ggml')).toBe('llamacpp'); + expect(catalogueRunner(metadataForConfig({ model_format: 'gguf' }).runner)).toBe('llamacpp'); + for (const other of ['', 'vllm', '', undefined, 3]) expect(catalogueRunner(other)).toBeUndefined(); + }); + it('normalizeDigest accepts the prefixed and the bare form and nothing else', () => { const bare = '0ce5ff81f9ffdf89dd8192ca7b5d34cefd8996a875b3aaaaafaf5c4813569eba'; expect(normalizeDigest(bare)).toBe(REAL_MLX_CHILD); @@ -224,6 +235,28 @@ describe('classifyModel', () => { }); }); + it('holds a row that names its runner to the runner of the build a Mac pulls', () => { + // The same size, another runner: the row's memory figures no longer describe what runs. + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'llamacpp' }, observed(100), '0.40.15')).toEqual([ + { kind: 'runner', expected: 'llamacpp', actual: 'mlx' }, + ]); + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'mlx' }, observed(100), '0.40.15')).toEqual([]); + // A GGUF build is a llamacpp row's, whether its manifest says llamacpp or nothing (ggml). + for (const runner of ['llamacpp', 'ggml']) { + const gguf = observed(100, { runner, format: 'gguf' }); + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'llamacpp' }, gguf, '0.40.15'), runner).toEqual([]); + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'mlx' }, gguf, '0.40.15'), runner).toEqual([ + { kind: 'runner', expected: 'mlx', actual: 'llamacpp' }, + ]); + } + // A runner Ollama would not run on a Mac is never taken for the row's. + expect(classifyModel({ id: 'a:b', bytes: 100, runner: 'mlx' }, observed(100, { runner: '' }), '0.40.15')).toEqual([ + { kind: 'runner', expected: 'mlx', actual: 'unknown' }, + ]); + // A catalogue without the field (before rows named their runner) is never held to one. + expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { runner: 'llamacpp' }), '0.40.15')).toEqual([]); + }); + it('flags a build that needs a newer runtime than the pin, and only that', () => { expect(classifyModel({ id: 'a:b', bytes: 100 }, observed(100, { requires: '0.41.0' }), '0.40.15')).toEqual([ { kind: 'requires', requires: '0.41.0', pinned: '0.40.15' }, @@ -349,6 +382,29 @@ describe('renderReport', () => { expect(report).toContain('llamacpp, gguf 90'); }); + it('names a runner change, what it means for the figures, and every build the tag offers', () => { + const gguf = { bytes: 100, digest: hex('c'), runner: 'llamacpp', format: 'gguf' }; + const mac = { bytes: 100, digest: hex('a'), runner: 'mlx', format: 'safetensors' }; + const result = baseResult({ + models: [ + { + id: 'qwen3.6:35b-a3b-mtp-q4_k_m', + bytes: 100, + findings: classifyModel({ id: 'x:y', bytes: 100, runner: 'llamacpp' }, { mac }, '0.40.15'), + observed: { tagDigest: hex('b'), variants: [gguf, mac], mac }, + }, + ], + }); + const report = renderReport(result); + expect(report).toContain('**Drift found**: 1 of 1 models.'); + expect(report).toContain('| runner changed |'); + expect(report).toContain( + "`qwen3.6:35b-a3b-mtp-q4_k_m`: a Mac now pulls the mlx, safetensors build, which runs on `mlx`, while the row's `runner` says `llamacpp`." + ); + expect(report).toContain('recompute them for the new runner'); + expect(report).toContain('mlx, safetensors 100 (a Mac takes this one)'); + }); + it('says the check failed and prints no verdict of clean', () => { const result = baseResult({ runtime: { error: 'GitHub answered 403 for the newest release', findings: [] } }); const report = renderReport(result); @@ -465,7 +521,10 @@ function buildRegistry() { }; } -const catalogOf = (models: { id: string; bytes: number; digest?: string; figures?: string }[]) => ({ updated: '2026-11-06', models }); +const catalogOf = (models: { id: string; bytes: number; digest?: string; figures?: string; runner?: string }[]) => ({ + updated: '2026-11-06', + models, +}); const NOW = new Date('2026-11-09T06:00:00Z'); describe('inspectModel', () => { @@ -586,6 +645,28 @@ describe('runCheck', () => { expect(report).toContain('`missing:tag`: the registry no longer has this tag'); }); + it('holds rows that name their runner to the build a Mac pulls, a plain GGUF manifest included', async () => { + const registry = buildRegistry(); + const { fetchFn } = fakeFetch(registry.routes); + const result = await runCheck({ + catalog: catalogOf([ + // The right size, the wrong runner: a Mac pulls the mlx child of this list. + { id: 'listed:1b', bytes: registry.sizes.mlx, runner: 'llamacpp' }, + // A manifest that names no runner is a GGUF build: a llamacpp row, clean. + { id: 'plain:2b', bytes: registry.sizes.plain, runner: 'llamacpp' }, + ]), + pin: PIN, + fetchFn, + now: NOW, + }); + expect(result.models.map((model: { findings: unknown[] }) => model.findings)).toEqual([ + [{ kind: 'runner', expected: 'llamacpp', actual: 'mlx' }], + [], + ]); + expect(exitCodeFor(result)).toBe(1); + expect(renderReport(result)).toContain('| runner changed |'); + }); + it('sends no Authorization header at all without a token', async () => { const registry = buildRegistry(); const { fetchFn, seen } = fakeFetch(registry.routes);