Files
Codeman/test/local-model-fits-budget.test.ts
T
Codeman maintainer 771072c36b feat(local-model): contract for the install stage (types, stubs, fake runtime)
The types, the module skeletons and the test fake that let the runtime, the
install job and the window be built at the same time.

Types:
- src/types/local-model-install.ts (new): the runtime pin, the install
  states, steps and error codes with one sentence and one next step each,
  the progress, the `install` object of GET /api/local-model, the bodies and
  409 answers of the install verbs, the state file. `LocalModelStatus` moves
  here; `install` is optional until the route emits it.
- src/types/local-model.ts: the catalogue row's Mac memory figures, the
  measured speed of a tested entry, the `measured` budget source,
  `RecommendOptions`, and the row tag, the coding agents and the install
  total of a candidate. All optional until the catalogue and `recommend()`
  carry them, so the read-only stage keeps compiling and its tests stay
  green.

Modules with final signatures and bodies that throw:
runtime.ts (the compiled pin is final), ollama-client.ts, developer-tools.ts,
state-store.ts, install-job.ts (the state, step and transition tables are
final), connect.ts, custom-model-hosts-writer.ts, cli-install-runner.ts.
Nothing imports them yet, so the server is unchanged.

Implemented: paths.ts (every folder of the feature, from dataPath()), and
`modelFitsBudget()` in recommend.ts, the judgement the job makes with the
measured budget before it downloads a model. `recommend()` takes the options
parameter and does not read it yet.

Test fake: test/fixtures/fake-ollama.ts, a runtime on a port the system
picks, in the shapes measured on a Mac: version, tags, ps, the pull stream
(MLX and GGUF lines, an error line, a cut stream, a resumed pull), chat with
and without a tool call, the model list, a runtime archive with Range
support, and a fake child process that logs the device total.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-06 20:05:15 +01:00

91 lines
3.7 KiB
TypeScript

/**
* @fileoverview Tests for `modelFitsBudget()` of src/local-model/recommend.ts: the judgement
* the install job makes before it downloads a model.
*
* Pinned: it agrees with what `recommend()` offers (a model that is the pick or listed fits,
* a model that is in neither and is not left out for its rank does not), it does not depend
* on the ranking, and a model the catalogue does not have never fits.
*
* Port: N/A (pure functions).
*/
import { describe, expect, it } from 'vitest';
import { getLocalModelCatalog } from '../src/config/local-model-catalog.js';
import { formatGb, memClassGb } from '../src/local-model/format.js';
import { modelFitsBudget, recommend } from '../src/local-model/recommend.js';
import type { HardwareFacts, HarnessFact } from '../src/types.js';
/** A Mac with Apple silicon and `gb` GB of memory as sold. */
function mac(gb: number, chip = 'Apple M5 Pro'): HardwareFacts {
const memBytes = gb * 1024 ** 3;
return {
platform: 'darwin',
arch: 'arm64',
arm64: true,
translated: false,
memBytes,
memGb: formatGb(memBytes),
memClassGb: memClassGb(memBytes),
chip,
macosMajor: 27,
macosVersion: '27.2',
wiredLimitMb: 0,
source: 'sysctl',
};
}
/** Every harness of the catalogue, installed and accepting every model id. */
function harnessFacts(): HarnessFact[] {
return getLocalModelCatalog().harnesses.map((row) => ({
cli: row.cli,
label: row.cli,
installed: true,
installable: false,
acceptsModelId: () => true,
}));
}
describe('modelFitsBudget', () => {
const catalog = getLocalModelCatalog();
it('agrees with the recommendation on every model and context of the catalogue', () => {
for (const gb of [8, 16, 24, 32, 48, 64, 128]) {
const hw = mac(gb);
const offered = recommend(hw, catalog, harnessFacts());
const rows = [offered.pick, ...offered.alternatives].filter((row) => row !== undefined);
// Whatever the recommendation offers fits at the context it offers it at.
for (const row of rows) {
expect(modelFitsBudget(hw, catalog, row.modelId, row.context), `${gb} GB ${row.modelId}`).toBe(true);
}
// A Mac that gets no recommendation for lack of memory has no model that fits at a
// context a harness works at.
if (offered.verdict === 'none') {
const smallestHarnessContext = Math.min(...catalog.harnesses.filter((h) => !h.gated).map((h) => h.minContext));
for (const model of catalog.models) {
for (const context of model.contexts.filter((ctx) => ctx >= smallestHarnessContext)) {
expect(modelFitsBudget(hw, catalog, model.id, context), `${gb} GB ${model.id}`).toBe(false);
}
}
}
}
});
it('does not depend on the ranking: a model that fits and is listed nowhere still fits', () => {
// On a 128 GB Mac more models fit than the recommendation lists.
const hw = mac(128, 'Apple M5 Max');
const offered = recommend(hw, catalog, harnessFacts());
const listed = new Set([offered.pick, ...offered.alternatives].map((row) => row?.modelId));
const unlisted = catalog.models.filter((model) => !listed.has(model.id));
expect(unlisted.length).toBeGreaterThan(0);
for (const model of unlisted) {
expect(modelFitsBudget(hw, catalog, model.id, model.contexts.at(-1) as number), model.id).toBe(true);
}
});
it('rules out a model that is too large, and one the catalogue does not have', () => {
expect(modelFitsBudget(mac(48), catalog, 'qwen3.5:122b-a10b', 131072)).toBe(false);
expect(modelFitsBudget(mac(48), catalog, 'no-such:1b', 65536)).toBe(false);
expect(modelFitsBudget(mac(48), catalog, 'qwen3.6:35b-a3b', 131072)).toBe(true);
});
});