diff --git a/docs/local-models.md b/docs/local-models.md index 04627c46..39887565 100644 --- a/docs/local-models.md +++ b/docs/local-models.md @@ -273,8 +273,12 @@ For every model row it checks: - **The build a Mac pulls runs on the pinned runtime**: its config declares no newer Ollama (`requires`) than `OLLAMA_RUNTIME_PIN`. - **The pinned digest**, once a row carries one: a Mac would still pull that manifest. The - catalogue pins none yet, so the report lists today's digests, the one to pin being what - `/api/tags` reports after the pull. + field is `digest` and its value is what `/api/tags` reports after the pull. The catalogue + schema does not have it yet: `LocalModelEntrySchema` must gain an optional `digest` under + exactly that name, or the check never compares it. Until then the report lists today's + digests, rows with measured figures first, because without a digest a rebuild of about the + same size goes unnoticed (a switch between the mlx and the gguf build does not: their sizes + differ by 2% or more). For the runtime pin it checks whether a newer Ollama release exists, and whether the pinned `ollama-darwin.tgz` is still on GitHub with the pinned size and sha256. diff --git a/scripts/check-local-model-catalog.mjs b/scripts/check-local-model-catalog.mjs index 8e96970a..7d45552a 100644 --- a/scripts/check-local-model-catalog.mjs +++ b/scripts/check-local-model-catalog.mjs @@ -37,6 +37,14 @@ * Every request has its own timeout and a byte cap, nothing is retried, every blob is checked * against its digest, and `GITHUB_TOKEN` is sent to api.github.com only, when it is set. * + * THE DIGEST CONTRACT. A row may pin the build it describes as `digest`: the digest of the + * manifest a Mac pulls, which is the value `/api/tags` reports after the pull (`sha256:` or + * the bare hex). The catalogue schema does not have that field yet: `LocalModelEntrySchema` is + * strict, so it must gain an optional `digest` before a row can carry one, under exactly that + * name, or this check never compares it. Until then the report lists the digests to pin, and + * a rebuild of about the same size goes unnoticed (a change of build, mlx against gguf, never + * does: every pair seen so far differs by 2% or more, far past the size tolerance). + * * Everything that reaches the report from the network is validated first (digests, sizes, * runner and format names, version tags), because the workflow puts the report in an issue. * @@ -455,14 +463,28 @@ export function renderReport(result) { if (unpinned.length > 0) { lines.push('## Digests to pin', ''); lines.push( - 'The catalogue pins no digest for these rows. Today\'s values: the tag\'s own manifest, and the manifest a ' + - 'Mac pulls, which is the digest `/api/tags` reports after the pull.', + 'The catalogue pins no digest for these rows, so a build pushed again at about the same size goes unnoticed. ' + + 'Today\'s values: the tag\'s own manifest, and the manifest a Mac pulls. The one to pin is the second, the ' + + 'digest `/api/tags` reports after the pull, as the row field `digest` (which the catalogue schema must ' + + 'gain first).', '' ); - lines.push('| Model | Tag manifest | Manifest a Mac pulls |'); - lines.push('| --- | --- | --- |'); + const measured = unpinned.filter((model) => model.figures === 'measured'); + if (measured.length > 0) { + const ids = measured.map((model) => `\`${model.id}\``).join(', '); + lines.push( + `Pin first: ${ids}. Its memory figures were measured on one build, and without a digest nothing here ` + + 'tells that build from a rebuild of about the same size.', + '' + ); + } + lines.push('| Model | Build a Mac pulls | Tag manifest | Manifest a Mac pulls |'); + lines.push('| --- | --- | --- | --- |'); for (const model of unpinned) { - lines.push(`| \`${model.id}\` | \`${model.observed.tagDigest}\` | \`${model.observed.mac.digest}\` |`); + lines.push( + `| \`${model.id}\` | ${buildName(model.observed.mac)} | \`${model.observed.tagDigest}\` | ` + + `\`${model.observed.mac.digest}\` |` + ); } lines.push(''); } @@ -673,8 +695,9 @@ export async function runCheck({ catalog, pin, fetchFn, now, githubToken }) { // Rows in parallel, each row's requests one after another: a handful of rows, a few requests each. result.models = await Promise.all( catalog.models.map(async (row) => { + // `digest` is not in the catalogue schema yet: see the file header. const digest = /** @type {{ digest?: unknown }} */ (row).digest; - const entry = { id: row.id, bytes: row.bytes, digest, findings: [] }; + const entry = { id: row.id, bytes: row.bytes, figures: row.figures, digest, findings: [] }; try { entry.observed = await inspectModel(fetchFn, row.id); entry.findings = classifyModel({ id: row.id, bytes: row.bytes, digest }, entry.observed, pin.version); diff --git a/test/check-local-model-catalog.test.ts b/test/check-local-model-catalog.test.ts index 1e6ad7bd..176e865c 100644 --- a/test/check-local-model-catalog.test.ts +++ b/test/check-local-model-catalog.test.ts @@ -313,14 +313,22 @@ describe('exitCodeFor', () => { }); describe('renderReport', () => { - it('says clean, and lists the digests to pin in full', () => { + it('says clean, and lists the digests to pin in full with the build a Mac pulls', () => { const report = renderReport(baseResult()); expect(report).toMatch(/^# Local model catalogue: weekly check\n/); expect(report).toContain('**Clean**'); - expect(report).toContain(`| \`a:b\` | \`${hex('b')}\` | \`${hex('a')}\` |`); + expect(report).toContain(`| \`a:b\` | mlx, safetensors | \`${hex('b')}\` | \`${hex('a')}\` |`); + expect(report).toContain('as the row field `digest`'); + expect(report).not.toContain('Pin first'); expect(report).toContain('## Not checked'); }); + it('asks to pin a row with measured figures first', () => { + const result = baseResult(); + (result.models[0] as Record).figures = 'measured'; + expect(renderReport(result)).toContain('Pin first: `a:b`.'); + }); + it('names a size drift with both figures and every build the tag offers', () => { const gguf = { bytes: 90, digest: hex('c'), runner: 'llamacpp', format: 'gguf' }; const mac = { bytes: 7_639_229_431, digest: hex('a'), runner: 'mlx', format: 'safetensors' }; @@ -457,7 +465,7 @@ function buildRegistry() { }; } -const catalogOf = (models: { id: string; bytes: number; digest?: string }[]) => ({ updated: '2026-11-06', models }); +const catalogOf = (models: { id: string; bytes: number; digest?: string; figures?: string }[]) => ({ updated: '2026-11-06', models }); const NOW = new Date('2026-11-09T06:00:00Z'); describe('inspectModel', () => { @@ -590,8 +598,8 @@ describe('runCheck', () => { const { fetchFn } = fakeFetch(registry.routes); const result = await runCheck({ catalog: catalogOf([ - { id: 'listed:1b', bytes: registry.sizes.mlx, digest: registry.digests.mlxChild }, - { id: 'plain:2b', bytes: registry.sizes.plain }, + { id: 'listed:1b', bytes: registry.sizes.mlx, digest: registry.digests.mlxChild, figures: 'measured' }, + { id: 'plain:2b', bytes: registry.sizes.plain, figures: 'measured' }, ]), pin: PIN, fetchFn, @@ -601,8 +609,10 @@ describe('runCheck', () => { const report = renderReport(result); expect(report).toContain('**Clean**'); // The pinned row is not offered for pinning again; the unpinned one is. - expect(report).not.toContain(`| \`listed:1b\` | \``); - expect(report).toContain(`| \`plain:2b\` | \`${registry.digests.plain}\``); + expect(report).not.toContain('| `listed:1b` | mlx'); + expect(report).toContain(`| \`plain:2b\` | ggml, gguf | \`${registry.digests.plain}\``); + // Only the measured row that has no digest yet is asked for first. + expect(report).toContain('Pin first: `plain:2b`.'); }); it('a newer runtime release is drift', async () => {