diff --git a/src/local-model/format.ts b/src/local-model/format.ts index cc9aa974..4f8db32d 100644 --- a/src/local-model/format.ts +++ b/src/local-model/format.ts @@ -8,7 +8,11 @@ * size for display, the browser included: it prints the text it was sent. * * GiB and MiB exist only for the budget math (reserves, the wired limit, the prompt cache) and - * never reach a person. + * never reach a person, with one exception: `memClassGb()`, the memory size a Mac is sold as, + * is how the card names the installed memory ("48 GB memory", what About This Mac shows). It + * is a name of the machine, not a computed size: `formatGb()` of the same bytes reads 51.5, + * which no owner recognises. Every computed size (need, budget, download, free disk) is + * `formatGb()`. * * Pure: no IO, no state. */ @@ -33,8 +37,10 @@ export function formatGb(bytes: number): string { /** * The memory size a Mac is sold as: whole binary gigabytes (17179869184 bytes is 16). This is - * `HardwareFacts.memClassGb`: the key of the tier rules ("below 36 GB") and of `TestedOn.memGb`. - * It names a class of Mac. A size for display is `formatGb()`. + * `HardwareFacts.memClassGb`: the key of the tier rules ("below 36 GB") and of `TestedOn.memGb`, + * and what the card and the `noneMemory` sentence call the installed memory. It names a class + * of Mac; 0 means the size is unknown and is never shown. Any other size for display is + * `formatGb()`. */ export function memClassGb(memBytes: number): number { if (!Number.isFinite(memBytes) || memBytes <= 0) return 0; diff --git a/src/local-model/recommend.ts b/src/local-model/recommend.ts index a0513093..884cc71f 100644 --- a/src/local-model/recommend.ts +++ b/src/local-model/recommend.ts @@ -10,33 +10,48 @@ * catalogue for this Mac's memory. * - Fit: F/B up to 0.85 is good, up to 0.98 is tight, above is out. * - Speed: tokens per second = the model's `speedPer100Gbps` x the chip's bandwidth / 100. - * Below 10 is slow. A chip that is in no chips row, or a model without a speed figure, is - * UNKNOWN: a bandwidth is never guessed. + * Below 10 is slow, decided on the unrounded product (9.96 is slow although it prints as + * 10). A chip that is in no chips row, or a model without a speed figure, is UNKNOWN: a + * bandwidth is never guessed. * - Candidates: (model, context, harness) with `ctx >= harness.minContext`, the harness not * gated, installed or installable, and accepting the model id. * - Ranking, lexicographic: good before tight; ok before unknown before slow; the higher model * rank; the higher harness rank; the larger context. The pick is the first candidate, the * alternatives are the next four MODELS, each at its own best context and harness. + * - Disk: a model whose download would leave less than 10 GiB free is not offered. It is not + * dropped in silence: the best-ranked model left out that way is named in `noteDiskDropped` + * with what is missing. * * Verdict `none`, in this order (the first that applies is the reason): * 1. `not-macos`: the server does not run on macOS. * 2. `intel`: a Mac without Apple silicon. * 3. `macos`: macOS is older than the catalogue's minimum (an unknown version does not block). - * 4. `memory`: no model fits the budget at any of its contexts, whatever the harness. + * 4. `memory`: no model fits the budget at any of its contexts, whatever the harness + * (`noneMemory`, or `noneMemoryUnknown` when the memory size could not be read). * 5. `no-harness`: no harness row is usable (not gated, known to the caller, installed or * installable). - * 6. `memory`: models fit, but none at a context a usable harness needs. + * 6. `memory`: models fit, but none at a context a usable harness needs. Its own sentence + * (`noneMemoryContext`) with the budget and the smallest need at such a context, because + * this Mac can miss by a hair (the 16 GB Mac mini: 10.3 of 10.4 GB) and "too little + * memory" would hide that the answer rests on an estimate. * 7. `no-harness`: models fit at such a context, but no usable harness accepts their id. * 8. `disk`: for every remaining model, free disk minus the download is below 10 GiB (an * unknown free disk does not block). * * Rules: - * - Verdict `limited` when the pick's model has fewer than 12 billion parameters. + * - Verdict `limited` when NO model with 12 billion parameters or more fits this Mac's memory + * at a context a usable harness works at. It is a statement about the Mac ("This Mac can run + * a small model"), so it is judged on everything that fits, BEFORE the disk filter and + * whatever the pick is: a 36 GB Mac whose pick is a 9B model while a 20B, a 27B and a 35B + * are listed under it is not a small-model Mac, and a 48 GB Mac does not become one because + * its disk is full. A small pick under verdict `ok` carries `noteSmallPick` instead. * - The word "Recommended" (`LOCAL_MODEL_LABELS.headlineTested`) only for verdict `ok` with a * pick whose model has a `tested` entry for exactly this chip, this memory size and this * harness. Otherwise `headlineOk` (or `headlineLimited`) plus `noteUntested`. - * - Every label in the result comes from `LOCAL_MODEL_LABELS`, every size text from - * `formatGb()`. No number is spliced into a sentence here. + * - Every label in the result comes from `LOCAL_MODEL_LABELS`, every computed size text from + * `formatGb()`. The one size that is not decimal GB is the installed memory in `noneMemory`: + * `HardwareFacts.memClassGb`, the size the Mac is sold as. No number is spliced into a + * sentence here. * - No CLI id is ever compared with a literal: a harness is a catalogue row plus a * `HarnessFact`. */ @@ -52,6 +67,8 @@ import { type LocalHarnessEntry, type LocalModelCatalog, type LocalModelEntry, + type LocalModelLabel, + type LocalModelMessage, type NoneReason, type Recommendation, type SpeedGrade, @@ -65,7 +82,7 @@ export const FIT_GOOD_MAX_SHARE = 0.85; export const FIT_TIGHT_MAX_SHARE = 0.98; /** An estimated speed below this many tokens per second is `slow`. */ export const SLOW_BELOW_TOKENS_PER_SECOND = 10; -/** A pick whose model has fewer than this many billion parameters makes the verdict `limited`. */ +/** When no model that fits has at least this many billion parameters, the verdict is `limited`. A pick below it is a "small model". */ export const LIMITED_BELOW_PARAMS_B = 12; /** What must stay free on the disk after the model download. */ export const DISK_HEADROOM_BYTES = 10 * GIB; @@ -134,9 +151,10 @@ function round(value: number, decimals: number): number { return Math.round(value * factor) / factor; } -/** A candidate plus the two ranks the ordering needs, which are not part of the payload. */ +/** A candidate plus its catalogue row and the two ranks the ordering needs, none of which is part of the payload. */ interface Ranked { candidate: Candidate; + model: LocalModelEntry; modelRank: number; harnessRank: number; } @@ -181,12 +199,12 @@ function buildCandidate( budgetBytes: number ): Candidate { const gbps = chipBandwidthGbps(catalog.chips, hw.chip); - const tokensPerSecond = - gbps !== undefined && model.speedPer100Gbps !== undefined - ? round((model.speedPer100Gbps * gbps) / 100, 1) - : undefined; + const rawSpeed = + gbps !== undefined && model.speedPer100Gbps !== undefined ? (model.speedPer100Gbps * gbps) / 100 : undefined; + // Graded on the unrounded figure: 9.96 rounds to 10.0 for the payload and is still slow. const speed: SpeedGrade = - tokensPerSecond === undefined ? 'unknown' : tokensPerSecond < SLOW_BELOW_TOKENS_PER_SECOND ? 'slow' : 'ok'; + rawSpeed === undefined ? 'unknown' : rawSpeed < SLOW_BELOW_TOKENS_PER_SECOND ? 'slow' : 'ok'; + const tokensPerSecond = rawSpeed === undefined ? undefined : round(rawSpeed, 1); // "A Mac like this" is exact: the same chip string, the same memory size as sold, the same harness. const tested = model.tested.some( (entry) => entry.chip === hw.chip && entry.memGb === hw.memClassGb && entry.cli === harness.row.cli @@ -216,11 +234,17 @@ function buildCandidate( }; } -function none(reason: NoneReason, budget: DeviceBudget, vars?: Record): Recommendation { +/** A `none` answer. `text` replaces the reason's default sentence (the `memory` reason has three). */ +function none( + reason: NoneReason, + budget: DeviceBudget, + vars?: Record, + text: LocalModelLabel = LOCAL_MODEL_NONE_LABELS[reason] +): Recommendation { return { verdict: 'none', reason, - headline: { text: LOCAL_MODEL_NONE_LABELS[reason], ...(vars ? { vars } : {}) }, + headline: { text, ...(vars ? { vars } : {}) }, notes: [], alternatives: [], budgetBytes: budget.budgetBytes, @@ -260,7 +284,12 @@ export function recommend( if (fit !== 'out') fitting.push({ model, ctx, need, fit }); } } - const tooLittleMemory = (): Recommendation => none('memory', budget, { memory: hw.memGb }); + // Nothing fits at any context. The installed memory is named by the size the Mac is sold + // as; a size that could not be read is not stated at all (never "0 GB of memory"). + const tooLittleMemory = (): Recommendation => + Number.isFinite(hw.memClassGb) && hw.memClassGb > 0 + ? none('memory', budget, { memory: String(hw.memClassGb) }) + : none('memory', budget, undefined, LOCAL_MODEL_LABELS.noneMemoryUnknown); if (fitting.length === 0) return tooLittleMemory(); // A gated row is never offered, whatever the caller knows about its CLI. @@ -275,25 +304,46 @@ export function recommend( const atHarnessContext = fitting.flatMap((entry) => usable.filter((harness) => entry.ctx >= harness.row.minContext).map((harness) => ({ ...entry, harness })) ); - if (atHarnessContext.length === 0) return tooLittleMemory(); + if (atHarnessContext.length === 0) { + // Models fit, but only below the context a harness needs. Say both numbers: the smallest + // need at such a context can sit a hair over the line, and the budget is an estimate. + const smallestContext = Math.min(...usable.map((harness) => harness.row.minContext)); + let smallestNeed: number | undefined; + for (const model of catalog.models) { + for (const ctx of model.contexts) { + if (ctx < smallestContext) continue; + const need = needBytes(model, ctx, catalog, hw); + if (smallestNeed === undefined || need < smallestNeed) smallestNeed = need; + } + } + if (smallestNeed === undefined) return tooLittleMemory(); + return none( + 'memory', + budget, + { budget: formatGb(budget.budgetBytes), need: formatGb(smallestNeed) }, + LOCAL_MODEL_LABELS.noneMemoryContext + ); + } const accepted = atHarnessContext.filter((entry) => accepts(entry.harness.fact, entry.model.id)); if (accepted.length === 0) return none('no-harness', budget); // The download must leave headroom on the disk. An unknown free disk does not block. const free = hw.freeDiskBytes; - const downloadable = - free === undefined ? accepted : accepted.filter((entry) => free - entry.model.bytes >= DISK_HEADROOM_BYTES); - if (free !== undefined && downloadable.length === 0) { + const hasRoom = (model: LocalModelEntry): boolean => free === undefined || free - model.bytes >= DISK_HEADROOM_BYTES; + if (free !== undefined && !accepted.some((entry) => hasRoom(entry.model))) { const smallest = Math.min(...accepted.map((entry) => entry.model.bytes)); return none('disk', budget, { download: formatGb(smallest), + reserve: formatGb(DISK_HEADROOM_BYTES), free: hw.freeDiskGb ?? formatGb(free), missing: formatGb(smallest + DISK_HEADROOM_BYTES - free), }); } - const ranked: Ranked[] = downloadable + // Everything that fits is ranked, the models without room on the disk included: the best of + // those is named in a note, so a full disk never removes a model without a word. + const ranked: Ranked[] = accepted .map((entry) => ({ candidate: buildCandidate( entry.model, @@ -305,28 +355,47 @@ export function recommend( catalog, budget.budgetBytes ), + model: entry.model, modelRank: entry.model.rank, harnessRank: entry.harness.row.rank, })) .sort(compareRanked); // The first candidate of each model is that model at its best context and harness. - const bestPerModel: Candidate[] = []; - for (const { candidate } of ranked) { - if (!bestPerModel.some((kept) => kept.modelId === candidate.modelId)) bestPerModel.push(candidate); + const bestPerModel: Ranked[] = []; + for (const entry of ranked) { + if (!bestPerModel.some((kept) => kept.model.id === entry.model.id)) bestPerModel.push(entry); } - const [pick, ...others] = bestPerModel; + const offered = bestPerModel.filter((entry) => hasRoom(entry.model)).map((entry) => entry.candidate); + const leftOutForDisk = bestPerModel.find((entry) => !hasRoom(entry.model)); + const [pick, ...others] = offered; - const limited = pick.paramsB < LIMITED_BELOW_PARAMS_B; + // About the Mac, not the pick, and judged before the disk filter (see the file header). + const limited = !accepted.some((entry) => entry.model.paramsB >= LIMITED_BELOW_PARAMS_B); const headline = limited ? LOCAL_MODEL_LABELS.headlineLimited : pick.tested ? LOCAL_MODEL_LABELS.headlineTested : LOCAL_MODEL_LABELS.headlineOk; + + const notes: LocalModelMessage[] = []; + // The limited headline already says it. Under `ok` a small pick must still be called small. + if (!limited && pick.paramsB < LIMITED_BELOW_PARAMS_B) notes.push({ text: LOCAL_MODEL_LABELS.noteSmallPick }); + if (leftOutForDisk && free !== undefined) { + notes.push({ + text: LOCAL_MODEL_LABELS.noteDiskDropped, + vars: { + model: leftOutForDisk.model.label, + missing: formatGb(leftOutForDisk.model.bytes + DISK_HEADROOM_BYTES - free), + }, + }); + } + if (!pick.tested) notes.push({ text: LOCAL_MODEL_LABELS.noteUntested }); + return { verdict: limited ? 'limited' : 'ok', headline: { text: headline }, - notes: pick.tested ? [] : [{ text: LOCAL_MODEL_LABELS.noteUntested }], + notes, pick, alternatives: others.slice(0, MAX_ALTERNATIVES), budgetBytes: budget.budgetBytes, diff --git a/src/types/local-model.ts b/src/types/local-model.ts index faf42777..8b8cfc75 100644 --- a/src/types/local-model.ts +++ b/src/types/local-model.ts @@ -15,10 +15,12 @@ * * Three rules every producer and consumer follows: * - * 1. ONE unit for a size a person reads: decimal GB, computed from bytes by `formatGb()` in - * src/local-model/format.ts. Every such size travels twice, as bytes (`needBytes`) and as the - * text to show (`needGb`, a number without the unit, such as '25.5'). The browser prints the - * text and never divides bytes itself. GiB appears only inside the budget math. + * 1. ONE unit for a computed size a person reads: decimal GB, computed from bytes by + * `formatGb()` in src/local-model/format.ts. Every such size travels twice, as bytes + * (`needBytes`) and as the text to show (`needGb`, a number without the unit, such as + * '25.5'). The browser prints the text and never divides bytes itself. GiB appears only + * inside the budget math. The one exception is the NAME of the installed memory: the card + * says `HardwareFacts.memClassGb` (48 for a Mac sold with 48 GB), see that field. * 2. Every sentence or label the SERVER chooses comes from `LOCAL_MODEL_LABELS` below, so the * browser can translate it (`codemanT(text, vars)`) and its i18n test can list every string. * Numbers are never spliced into a sentence here: they travel in `vars` and fill the @@ -33,7 +35,8 @@ * How `HardwareFacts` was obtained. * - `sysctl`: the spawn answered and `hw.memsize` parsed. * - `os-fallback`: on macOS, but the spawn failed or timed out; memory, architecture and the chip - * string come from `node:os`, the rest is absent. + * string come from `node:os` and the free disk from `statfs`. What only sysctl knows (`arm64`, + * `translated`, `model`, `macosVersion`, `macosMajor`, `wiredLimitMb`) is absent. * - `not-probed`: not macOS, so `sysctl` was never spawned. */ export type HardwareProbeSource = 'sysctl' | 'os-fallback' | 'not-probed'; @@ -54,9 +57,19 @@ export interface HardwareFacts { translated?: boolean; /** Installed memory in bytes (`hw.memsize`, else `os.totalmem()`). The ONLY memory input of the budget math. */ memBytes: number; - /** `memBytes` as a person reads it: decimal GB from `formatGb()`, without the unit ('17.2' for a Mac sold as 16 GB). */ + /** + * `memBytes` in the unit of every computed size: decimal GB from `formatGb()`, without the unit + * ('17.2' for a Mac sold as 16 GB). NOT what the card calls the installed memory: that is + * `memClassGb`, see there. + */ memGb: string; - /** The memory size the Mac is sold as: whole binary gigabytes (16, 32, 48). Tier rules and `TestedOn.memGb` use it. */ + /** + * The memory size the Mac is sold as: whole binary gigabytes (16, 32, 48), 0 when the size is + * unknown. Tier rules and `TestedOn.memGb` use it, and it is the ONE size a person reads that + * is not decimal GB: the hardware line and the `memory` slot of `noneMemory` print it, because + * it is the number on the box and in About This Mac ("48 GB"), and '51.5 GB memory' reads as a + * different machine. Every computed size (need, budget, download, free disk) stays decimal GB. + */ memClassGb: number; /** `machdep.cpu.brand_string` ('Apple M4 Pro'), else `os.cpus()[0].model`. */ chip?: string; @@ -97,7 +110,7 @@ export interface LocalModelEntry { label: string; /** SPDX id of the license ('Apache-2.0'). */ license: string; - /** Parameter count in billions. Under 12 the verdict is `limited`, never `ok`. */ + /** Parameter count in billions. When no model that fits has 12 or more, the verdict is `limited`, never `ok`. */ paramsB: number; /** Size of the weights in bytes: both the download and what they take in memory. */ bytes: number; @@ -200,7 +213,10 @@ export type SpeedGrade = 'ok' | 'slow' | 'unknown'; * Why the verdict is `none`. * - `not-macos`: the server does not run on macOS (the card is hidden). * - `intel`: a Mac without Apple silicon. - * - `memory`: no model fits the memory budget with any harness. + * - `memory`: no model fits the memory budget at a context a usable harness needs. Three + * headlines share this reason: `noneMemory` (nothing fits at any context), `noneMemoryContext` + * (a model fits, but only below the context a harness needs: the sentence carries the budget + * and the need) and `noneMemoryUnknown` (the memory size could not be read). * - `disk`: models fit the memory, but for every one of them free disk minus the download is below 10 GiB. * - `macos`: macOS is older than `LocalModelCatalog.minMacosMajor`. * - `no-harness`: no catalogue harness is installed or installable and accepts a model. @@ -228,12 +244,17 @@ export interface DeviceBudget { * `{placeholder}` slots are filled by the browser from `LocalModelMessage.vars`. `{name}` is * taken (the translator fills it with the display name), so no slot here is called `name`. * - * | Key | Slots (all text; sizes are decimal GB without the unit) | - * |---------------|---------------------------------------------------------------------------------| - * | `noneMemory` | `memory`: `HardwareFacts.memGb` | - * | `noneDisk` | `download`: smallest download among the models that fit the memory; `free`: | - * | | `HardwareFacts.freeDiskGb`; `missing`: download + 10 GiB - free | - * | `noneMacos` | `version`: `HardwareFacts.macosVersion`; `minimum`: the catalogue's minimum | + * | Key | Slots (all text; sizes are decimal GB without the unit) | + * |---------------------|---------------------------------------------------------------------------| + * | `noteDiskDropped` | `model`: label of the best model left out for disk space; `missing`: its | + * | | download + the disk reserve - free | + * | `noneMemory` | `memory`: `HardwareFacts.memClassGb`, the size the Mac is sold as | + * | `noneMemoryContext` | `budget`: `Recommendation.budgetGb`; `need`: the smallest need of any | + * | | model at a context a usable harness works at | + * | `noneDisk` | `download`: smallest download among the models that fit the memory; | + * | | `reserve`: what must stay free after it (10 GiB, as decimal GB); `free`: | + * | | `HardwareFacts.freeDiskGb`; `missing`: download + reserve - free | + * | `noneMacos` | `version`: `HardwareFacts.macosVersion`; `minimum`: the catalogue minimum | * * The word "Recommended" exists only in `headlineTested`, which may be used only when the * pick's `tested` is true. No catalogue row has a tested entry today, so it is never sent. @@ -243,21 +264,33 @@ export const LOCAL_MODEL_LABELS = { headlineOk: 'Best fit for this Mac', /** Verdict `ok`, pick tested by us on a Mac like this one. */ headlineTested: 'Recommended for this Mac', - /** Verdict `limited`. */ + /** Verdict `limited`: no model with 12 billion parameters or more fits this Mac. */ headlineLimited: 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', /** A note under an `ok` or `limited` pick whose `tested` is false. */ noteUntested: 'Not yet tested by us on a Mac like this.', + /** A note under an `ok` pick that is itself a small model: larger ones fit this Mac, but rank lower or have no room on the disk. */ + noteSmallPick: + 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.', + /** A note when a model that fits the memory was left out because its download would not leave the disk reserve. */ + noteDiskDropped: + '{model} fits the memory of this Mac too, but its download needs about {missing} GB more free disk space.', fitGood: 'Good', fitTight: 'Tight', speedSlow: 'Slow', speedUnknown: 'Speed unknown', noneNotMacos: 'Codeman recommends a local model only on a Mac with Apple silicon.', noneIntel: 'This Mac has an Intel processor. Local coding models need Apple silicon.', + /** Reason `memory`: nothing fits at any context. */ noneMemory: 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', + /** Reason `memory`: a model fits, but only below the context a harness needs. Says both numbers, and that they are an estimate. */ + noneMemoryContext: + 'Codeman estimates that this Mac can give a model about {budget} GB of memory. The smallest model a coding agent can work with needs about {need} GB. That leaves no safe margin, so Codeman recommends none here.', + /** Reason `memory`: the memory size could not be read, so no size is stated. */ + noneMemoryUnknown: 'Codeman could not read how much memory this Mac has, so it recommends no model here.', noneDisk: - 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.', + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download, and {reserve} GB must stay free after it. {free} GB is free now. Free up about {missing} GB.', noneMacos: 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.', noneNoHarness: 'No coding agent that Codeman can connect to a local model is available on this Mac.', } as const; @@ -265,7 +298,7 @@ export const LOCAL_MODEL_LABELS = { /** One of the texts in `LOCAL_MODEL_LABELS`. The payload's label fields are typed with it, so an off-list string does not compile. */ export type LocalModelLabel = (typeof LOCAL_MODEL_LABELS)[keyof typeof LOCAL_MODEL_LABELS]; -/** The headline of each `none` reason. */ +/** The headline of each `none` reason. `memory` has two more (`noneMemoryContext`, `noneMemoryUnknown`), see `NoneReason`. */ export const LOCAL_MODEL_NONE_LABELS: Record = { 'not-macos': LOCAL_MODEL_LABELS.noneNotMacos, intel: LOCAL_MODEL_LABELS.noneIntel, @@ -317,7 +350,7 @@ export interface Candidate { speed: SpeedGrade; /** `speedSlow` or `speedUnknown`. Absent when `speed` is `ok`. */ speedLabel?: LocalModelLabel; - /** Estimated generation speed. Present exactly when `speed` is not `unknown`. */ + /** Estimated generation speed, rounded to one decimal. Present exactly when `speed` is not `unknown`. `speed` is decided on the unrounded figure. */ tokensPerSecond?: number; /** A `TestedOn` entry of the model matches this Mac (same chip, same `memClassGb`) and this harness. */ tested: boolean; @@ -325,22 +358,25 @@ export interface Candidate { /** * The answer of `recommend()`. - * - `ok`: `pick` is the best candidate and its model has at least 12 billion parameters. - * - `limited`: `pick` is the best candidate, but its model has fewer than 12 billion parameters. - * The card says so and never says "Recommended". + * - `ok`: `pick` is the best candidate, and at least one model with 12 billion parameters or + * more fits this Mac's memory at a context a usable harness works at. The pick itself can be + * smaller (a larger model may fit only tightly, have no speed figure, or have no room on the + * disk): `notes` then carries `noteSmallPick`. + * - `limited`: `pick` is the best candidate, and NO model with 12 billion parameters or more + * fits. The card says so and never says "Recommended". * - `none`: no pick; `reason` says why. */ export interface Recommendation { verdict: 'ok' | 'limited' | 'none'; /** Present exactly when `verdict` is `none`. */ reason?: NoneReason; - /** What the card says first: `headlineOk`, `headlineTested`, `headlineLimited` or the `none` reason's sentence. */ + /** What the card says first: `headlineOk`, `headlineTested`, `headlineLimited` or a `none` sentence. */ headline: LocalModelMessage; - /** Further sentences under the headline (`noteUntested` today). May be empty. */ + /** Further sentences under the pick, in this order: `noteSmallPick`, `noteDiskDropped`, `noteUntested`. May be empty. */ notes: LocalModelMessage[]; /** Present exactly when `verdict` is `ok` or `limited`. */ pick?: Candidate; - /** The next models that fit (at most four, each at its best context and harness), best first. Empty when `verdict` is `none`. */ + /** The next models that fit and have room on the disk (at most four, each at its best context and harness), best first. Empty when `verdict` is `none`. */ alternatives: Candidate[]; /** `DeviceBudget.budgetBytes`. 0 when no budget is computed: not macOS, or a Mac without Apple silicon. */ budgetBytes: number; diff --git a/src/web/public/i18n.js b/src/web/public/i18n.js index c9ab61a5..87c18651 100644 --- a/src/web/public/i18n.js +++ b/src/web/public/i18n.js @@ -1412,10 +1412,11 @@ 'Model on this Mac': '此 Mac 上的模型', 'What the Mac that Codeman runs on can run as a local coding model. Read-only: nothing is downloaded or installed from here.': '运行 {name} 的这台 Mac 可以运行哪种本地编程模型。只读:此处不会下载或安装任何内容。', + 'The Mac that Codeman runs on, which is not always the device you are reading on.': + '指运行 {name} 的那台 Mac,不一定是你正在使用的这台设备。', Hide: '隐藏', 'Hide this card on this device. It stays in App Settings, under Models.': '在此设备上隐藏此卡片。它仍保留在“应用设置”的“模型”中。', - 'Checking this Mac…': '正在检查此 Mac…', '{chip}, {memory} GB memory, {free} GB free': '{chip},内存 {memory} GB,可用磁盘空间 {free} GB', '{chip}, {memory} GB memory': '{chip},内存 {memory} GB', '{memory} GB memory, {free} GB free': '内存 {memory} GB,可用磁盘空间 {free} GB', @@ -1431,9 +1432,14 @@ 'About {speed} tokens per second (estimated).': '每秒约 {speed} 个 Token(估算)。', '{harness} is not installed on this Mac yet.': '此 Mac 上尚未安装 {harness}。', 'With {harness}, the first answer can take minutes.': '使用 {harness} 时,第一次回答可能需要几分钟。', - 'Other models that fit': '其他适合的模型', - 'Setting this up from Codeman is the next step. It is not built yet.': - '下一步是直接在 {name} 中完成安装和设置,该功能尚未实现。', + 'about {speed} tokens per second': '每秒约 {speed} 个 Token', + // What the fit and speed labels measure (their tooltip and accessible name). + 'Memory fit: needs {share}% of what this Mac can give a model': + '内存占用:需要此 Mac 可分配给模型的内存的 {share}%', + 'No speed estimate for this model on this chip': '此芯片上没有该模型的速度估算', + 'Other models that fit': '其他装得下的模型', + 'Codeman cannot download or install this for you yet. That is the next step, and it is not built yet.': + '{name} 目前还不能为你下载或安装它。这是下一步的功能,尚未实现。', // The sentences and labels the SERVER picks (LOCAL_MODEL_LABELS in src/types/local-model.ts), keyed // by their exact English text: changing a text there means changing its key here. 'Best fit for this Mac': '最适合此 Mac 的选择', @@ -1441,8 +1447,12 @@ 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.': '此 Mac 可以运行小型模型。它能处理小幅修改和问答,但不能替代云端模型。', 'Not yet tested by us on a Mac like this.': '我们尚未在类似的 Mac 上测试过。', + 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.': + '最适合的是一个小型模型。它能处理小幅修改和问答,但不能替代云端模型。', + '{model} fits the memory of this Mac too, but its download needs about {missing} GB more free disk space.': + '{model} 也能装入此 Mac 的内存,但下载它还需要再腾出约 {missing} GB 的可用磁盘空间。', Good: '良好', - Tight: '紧张', + Tight: '偏紧', Slow: '较慢', 'Speed unknown': '速度未知', 'Codeman recommends a local model only on a Mac with Apple silicon.': @@ -1451,8 +1461,12 @@ '此 Mac 使用 Intel 处理器。本地编程模型需要 Apple 芯片。', 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.': '此 Mac 有 {memory} GB 内存,不足以运行能够驱动编程智能体的模型,因此 {name} 在此不推荐任何模型。', - 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.': - '可用磁盘空间不足。适合此 Mac 的最小模型需要下载 {download} GB,而当前可用空间为 {free} GB。请腾出约 {missing} GB。', + 'Codeman estimates that this Mac can give a model about {budget} GB of memory. The smallest model a coding agent can work with needs about {need} GB. That leaves no safe margin, so Codeman recommends none here.': + '据 {name} 估算,此 Mac 可分配给模型的内存约为 {budget} GB。编程智能体能够使用的最小模型约需 {need} GB。这样没有留出安全余量,因此 {name} 在此不推荐任何模型。', + 'Codeman could not read how much memory this Mac has, so it recommends no model here.': + '{name} 无法读取此 Mac 的内存大小,因此在此不推荐任何模型。', + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download, and {reserve} GB must stay free after it. {free} GB is free now. Free up about {missing} GB.': + '可用磁盘空间不足。适合此 Mac 的最小模型需要下载 {download} GB,下载后还必须保留 {reserve} GB 的可用空间。当前可用空间为 {free} GB。请腾出约 {missing} GB。', 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.': '此 Mac 运行的是 macOS {version}。本地模型需要 macOS {minimum} 或更高版本。', 'No coding agent that Codeman can connect to a local model is available on this Mac.': diff --git a/src/web/public/index.html b/src/web/public/index.html index 78211715..8026b7ce 100644 --- a/src/web/public/index.html +++ b/src/web/public/index.html @@ -2879,9 +2879,11 @@ + on the row's text). The module shows the group only once the server has + answered for a Mac: it hides it for a server on another system (the title + and hint are about "this Mac"), while the first request is in flight, and + for a non-admin in multi-user mode (the route is admin-only). Not gated on + the endpoints toggle above: the answer does not depend on it. -->

Model on this Mac

What the Mac that Codeman runs on can run as a local coding model. Read-only: nothing is downloaded or installed from here.

diff --git a/src/web/public/local-model-ui.js b/src/web/public/local-model-ui.js index ab4a353b..92c02af0 100644 --- a/src/web/public/local-model-ui.js +++ b/src/web/public/local-model-ui.js @@ -4,11 +4,13 @@ * Reads `GET /api/local-model` (hardware of the machine the server runs on, the recommendation, * the catalogue date) and shows, read-only: * - the hardware line (chip, memory, free disk); - * - the verdict: the headline and notes the server sent, then the pick with its numbers - * (model, harness, context, memory need against the budget, download size, speed); - * - the other models that fit, each with its fit and speed label; - * - one line saying that setting the model up from Codeman is the next step and is not built - * yet. + * - the verdict: the headline the server sent, then the pick with its numbers (model, harness, + * context, memory need against the budget, download size, speed) and the notes; + * - one line saying that Codeman cannot download or install the model yet: that is the next + * step and is not built. It sits right under the pick, where a reader looks for a button; + * - the other models that fit, each with its fit and speed label. A label says what it + * measures in its tooltip and accessible name ("Good" is the memory fit, not a rating of + * the model), and an alternative with a speed estimate prints the figure. * * ## One body, two containers * @@ -23,9 +25,18 @@ * hidden while the first request is in flight and after a failed one (nothing is known about the * machine then, and an error card would be the only thing this feature ever showed on a server * that is not a Mac), when the answer is `not-macos`, when the request is refused (a non-admin in - * multi-user mode gets a 403, which is an answer: stop asking) and after the user hid it on this - * device. The Settings row is where the loading and failure sentences show, and it is the way - * back to the numbers after a Hide. + * multi-user mode gets a `403 FORBIDDEN`, which is an answer: stop asking) and after the user hid + * it on this device. + * + * The Settings group ("Model on this Mac") follows the same knowledge: it is hidden for + * `not-macos` (most servers are not a Mac, and a group about "this Mac" there is false), for a + * refused request and while the first request is in flight (nothing says yet that the server is + * a Mac). It shows the answer for a Mac, and one failure sentence when the first request + * failed. It is the way back to the numbers after a Hide. + * + * "Refused" means the envelope's `errorCode` is `FORBIDDEN`. The auth layer has a second 403 for + * every /api route (`PASSWORD_CHANGE_REQUIRED`, an admin who must change the password first): + * that one is a failed request, asked again on the next show and on `codeman:me`. * * Rules: * - Read-only: no control here downloads, installs or writes anything but the per-device @@ -36,8 +47,10 @@ * (`` is written by every i18n `configure()`). * - Names and labels come from the payload: this file holds no list of models or harnesses and * never compares a CLI id with a literal. - * - Sizes are printed exactly as sent (`needGb`, `downloadGb`, `budgetGb`, `memGb`, - * `freeDiskGb`: decimal GB as text). Bytes are never divided here. + * - Sizes are printed exactly as sent (`needGb`, `downloadGb`, `budgetGb`, `freeDiskGb`: + * decimal GB as text). Bytes are never divided here. The installed memory is the one size + * that is not decimal GB: the hardware line prints `memClassGb`, the size the Mac is sold as + * (48, not 51.5), which is what About This Mac shows. * - A sentence the server chose arrives as `{ text, vars }` and goes through * `localModelT(text, vars)`; this file's own strings are literals in that same call, in * `{placeholder}` form, so the i18n test can harvest them. Both kinds need a zh-CN entry in @@ -52,8 +65,12 @@ * @loadorder 12.59 of 16, after edit-history-ui.js, before entrance-animations.js */ -/** Per-device dismissal of the welcome card. */ -const LOCAL_MODEL_DISMISS_KEY = 'codeman:local-model-dismissed'; +/** + * Per-device dismissal of the welcome card, named for THIS stage (the read-only card). The stage + * that can set a model up takes a key of its own, so hiding a card that offers nothing yet does + * not hide the offer later. + */ +const LOCAL_MODEL_DISMISS_KEY = 'codeman:local-model-dismissed:readonly'; /** The server caches its hardware probe for a minute, so asking again sooner cannot change the answer. */ const LOCAL_MODEL_MAX_AGE_MS = 60000; @@ -100,18 +117,41 @@ function localModelNotAdmin() { return !!(me.multiUser && me.role !== 'admin'); } -/** Chip, memory and free disk in one line. Every part but the memory may be missing. */ +/** + * Chip, memory and free disk in one line. The memory is the size the Mac is sold as + * (`memClassGb`: 48 for a Mac whose 48 x 2^30 bytes are 51.5 decimal GB), a number or nothing: + * an unknown size (0) is left out, never printed. + */ function localModelHardwareLine(hw) { const chip = String(hw.chip || hw.model || ''); - const sizes = { chip, memory: hw.memGb, free: hw.freeDiskGb }; - if (!hw.memGb) return chip; + const memory = Number.isFinite(hw.memClassGb) && hw.memClassGb > 0 ? String(hw.memClassGb) : ''; + const sizes = { chip, memory, free: hw.freeDiskGb }; + if (!memory) return chip; if (chip && hw.freeDiskGb) return localModelT('{chip}, {memory} GB memory, {free} GB free', sizes); if (chip) return localModelT('{chip}, {memory} GB memory', sizes); if (hw.freeDiskGb) return localModelT('{memory} GB memory, {free} GB free', sizes); return localModelT('{memory} GB memory', sizes); } -/** "Qwen3.8 27B (qwen3.8:27b) with OpenCode", then the fit and speed labels the server chose. */ +/** What the fit label measures: the share of the memory budget the model needs. */ +function localModelFitHint(candidate) { + if (!Number.isFinite(candidate.budgetShare)) return ''; + const share = String(Math.round(candidate.budgetShare * 100)); + return localModelT('Memory fit: needs {share}% of what this Mac can give a model', { share }); +} + +/** What the speed label stands for: the estimate behind "Slow", or that there is none. */ +function localModelSpeedHint(candidate) { + const speed = localModelSpeed(candidate.tokensPerSecond); + if (speed) return localModelT('About {speed} tokens per second (estimated).', { speed }); + return localModelT('No speed estimate for this model on this chip'); +} + +/** + * "Qwen3.8 27B (qwen3.8:27b) with OpenCode", then the fit and speed labels the server chose. + * A bare "Good" next to a model name reads as a rating of the model, so each label carries what + * it measures as its tooltip and its accessible name. + */ function localModelCandidateHead(candidate) { const head = localModelEl('div', 'local-model-name-row'); const name = localModelT('{model} ({id}) with {harness}', { @@ -120,19 +160,36 @@ function localModelCandidateHead(candidate) { harness: candidate.cliLabel || candidate.cli, }); head.appendChild(localModelEl('span', 'local-model-name', name)); - for (const label of [candidate.fitLabel, candidate.speedLabel]) { + const pills = [ + [candidate.fitLabel, localModelFitHint], + [candidate.speedLabel, localModelSpeedHint], + ]; + for (const [label, hint] of pills) { if (typeof label !== 'string' || !label) continue; - head.appendChild(localModelEl('span', 'local-model-pill', localModelT(label))); + const pill = localModelEl('span', 'local-model-pill', localModelT(label)); + const meaning = hint(candidate); + if (meaning) { + pill.title = meaning; + pill.setAttribute('role', 'note'); + pill.setAttribute('aria-label', `${pill.textContent}. ${meaning}`); + } + head.appendChild(pill); } return head; } -/** The dim line under a candidate's name: context, memory need (alternatives), download, license (the pick). */ -function localModelMeta(candidate, { need = false, license = false } = {}) { +/** + * The dim line under a candidate's name: context, then for an alternative its memory need and, + * when there is an estimate, its speed (so "Slow" comes with a number on a screen without + * hover), the download, and for the pick the license. + */ +function localModelMeta(candidate, { alternative = false } = {}) { const bits = [localModelT('Context {context} tokens', { context: localModelCount(candidate.context) })]; - if (need && candidate.needGb) bits.push(localModelT('{need} GB of memory', { need: candidate.needGb })); + if (alternative && candidate.needGb) bits.push(localModelT('{need} GB of memory', { need: candidate.needGb })); if (candidate.downloadGb) bits.push(localModelT('{download} GB download', { download: candidate.downloadGb })); - if (license && candidate.license) bits.push(String(candidate.license)); + const speed = alternative ? localModelSpeed(candidate.tokensPerSecond) : ''; + if (speed) bits.push(localModelT('about {speed} tokens per second', { speed })); + if (!alternative && candidate.license) bits.push(String(candidate.license)); return bits.join(' · '); } @@ -140,7 +197,7 @@ function localModelMeta(candidate, { need = false, license = false } = {}) { function localModelPick(pick, recommendation) { const box = localModelEl('div', 'local-model-pick'); box.appendChild(localModelCandidateHead(pick)); - box.appendChild(localModelEl('div', 'local-model-line', localModelMeta(pick, { license: true }))); + box.appendChild(localModelEl('div', 'local-model-line', localModelMeta(pick))); const sizes = { need: pick.needGb, budget: recommendation.budgetGb }; // A wired limit is a setting of this Mac; anything else is the formula's estimate, and says so. const need = @@ -165,17 +222,19 @@ function localModelPick(pick, recommendation) { return box; } -/** The other models that fit. Folded behind a native disclosure on the welcome card, open in Settings. */ +/** + * The other models that fit. Folded behind a native disclosure on the welcome card, open in + * Settings. No count beside the title: the server lists four at most, so the number of rows is + * not the number of models that fit. + */ function localModelAlternatives(alternatives, fold) { const wrap = localModelEl(fold ? 'details' : 'div', 'local-model-alts'); - const title = localModelEl(fold ? 'summary' : 'div', 'local-model-alts-title'); - title.appendChild(localModelEl('span', '', localModelT('Other models that fit'))); - title.appendChild(localModelEl('span', 'local-model-count', String(alternatives.length))); + const title = localModelEl(fold ? 'summary' : 'div', 'local-model-alts-title', localModelT('Other models that fit')); wrap.appendChild(title); for (const candidate of alternatives) { const row = localModelEl('div', 'local-model-alt'); row.appendChild(localModelCandidateHead(candidate)); - row.appendChild(localModelEl('div', 'local-model-line', localModelMeta(candidate, { need: true }))); + row.appendChild(localModelEl('div', 'local-model-line', localModelMeta(candidate, { alternative: true }))); wrap.appendChild(row); } return wrap; @@ -200,12 +259,16 @@ function localModelBody(status, { fold }) { const text = localModelMessage(note); if (text) body.appendChild(localModelEl('div', 'local-model-note', text)); } - const alternatives = pick && Array.isArray(recommendation.alternatives) ? recommendation.alternatives : []; - if (alternatives.length) body.appendChild(localModelAlternatives(alternatives, fold)); + // Right under the pick and its notes, above the list: the pick names a download and maybe a + // harness that is not installed, so this is where a reader looks for a button. if (pick) { - const next = localModelT('Setting this up from Codeman is the next step. It is not built yet.'); + const next = localModelT( + 'Codeman cannot download or install this for you yet. That is the next step, and it is not built yet.' + ); body.appendChild(localModelEl('div', 'local-model-next', next)); } + const alternatives = pick && Array.isArray(recommendation.alternatives) ? recommendation.alternatives : []; + if (alternatives.length) body.appendChild(localModelAlternatives(alternatives, fold)); return body; } @@ -249,24 +312,27 @@ Object.assign(CodemanApp.prototype, { }, async _fetchLocalModel() { - // `_api`, not `_apiJson`: a 403 (admin-only in multi-user mode) is an answer and must be told - // apart from a request that failed. + // `_api`, not `_apiJson`: a refusal (admin-only in multi-user mode) is an answer and must be + // told apart from a request that failed. const res = await this._api(localModelUrl('/api/local-model')); - if (res?.status === 403) { - this._localModelState = 'refused'; - this._localModelStatus = null; - this.renderLocalModel(); - return; - } let body = null; - if (res?.ok) { + if (res?.ok || res?.status === 403) { try { body = await res.json(); } catch { body = null; } } - const data = localModelPayload(body); + // Only the route's own refusal is final. The auth layer answers 403 too, with + // PASSWORD_CHANGE_REQUIRED, for an admin who must change the password first: that is a + // failed request, asked again once the password is changed. + if (res?.status === 403 && body?.errorCode === 'FORBIDDEN') { + this._localModelState = 'refused'; + this._localModelStatus = null; + this.renderLocalModel(); + return; + } + const data = res?.ok ? localModelPayload(body) : null; if (!data) { // A failed refresh must not blank an answer the user is reading: keep the last good one. if (!this._localModelStatus) this._localModelState = 'failed'; @@ -336,6 +402,8 @@ Object.assign(CodemanApp.prototype, { const head = localModelEl('div', 'local-model-head'); const title = localModelEl('h3', 'local-model-title', localModelT('Model on this Mac')); title.id = 'localModelCardTitle'; + // The page can be open on another laptop or a tablet: "this Mac" is the server's. + title.title = localModelT('The Mac that Codeman runs on, which is not always the device you are reading on.'); head.appendChild(title); const hide = localModelEl('button', 'local-model-hide', localModelT('Hide')); hide.type = 'button'; @@ -353,21 +421,20 @@ Object.assign(CodemanApp.prototype, { _renderLocalModelSettings(row) { row.textContent = ''; - const refused = this._localModelRefused(); - // The whole group goes for a non-admin, the way the other admin-only groups of this modal do. - const group = document.getElementById('localModelGroup'); - if (group) group.style.display = refused ? 'none' : ''; - if (refused) return; const status = this._localModelStatus; + const failed = !status && this._localModelState === 'failed'; + // The whole group goes for a non-admin, the way the other admin-only groups of this modal + // do, and for a server that is not a Mac: its title and hint are about "this Mac". Until + // the first answer nothing says the server is one, so the group waits for it. + const shown = !this._localModelRefused() && (failed || (!!status && status.recommendation.reason !== 'not-macos')); + const group = document.getElementById('localModelGroup'); + if (group) group.style.display = shown ? '' : 'none'; + if (!shown) return; if (status) { row.appendChild(localModelBody(status, { fold: false })); return; } - const text = - this._localModelState === 'failed' - ? localModelT('Could not reach the server.') - : localModelT('Checking this Mac…'); - row.appendChild(localModelEl('div', 'local-model-headline', text)); + row.appendChild(localModelEl('div', 'local-model-headline', localModelT('Could not reach the server.'))); }, /** Listeners that outlive a render, wired once. */ diff --git a/src/web/public/styles.css b/src/web/public/styles.css index f34cd700..ba9ac133 100644 --- a/src/web/public/styles.css +++ b/src/web/public/styles.css @@ -22376,9 +22376,13 @@ html[data-tile-anim="soft"] .tile-grid-ghosts.tile-grid-ghosts--release:not(.til min-width: 0; } +/* The line that says nothing can be set up from here yet (.local-model-next) is + the one a reader must not miss, so it reads like every other line: same size, + same --text-dim. --text-muted on this surface was 3.9:1. */ .local-model-hw, .local-model-line, -.local-model-note { +.local-model-note, +.local-model-next { font-size: 0.68rem; line-height: 1.45; color: var(--text-dim); @@ -22418,8 +22422,8 @@ html[data-tile-anim="soft"] .tile-grid-ghosts.tile-grid-ghosts--release:not(.til min-width: 0; } -.local-model-pill, -.local-model-count { +/* Each pill carries what it measures as its tooltip (a title), hence the cursor. */ +.local-model-pill { flex: 0 0 auto; font-size: 0.62rem; line-height: 1.5; @@ -22428,12 +22432,7 @@ html[data-tile-anim="soft"] .tile-grid-ghosts.tile-grid-ghosts--release:not(.til border-radius: 999px; padding: 0 0.42rem; white-space: nowrap; -} - -.local-model-count { - margin-inline-start: 0.4rem; - border-color: transparent; - background: var(--control-bg-hover); + cursor: help; } .local-model-alts-title { @@ -22466,12 +22465,6 @@ html[data-tile-anim="soft"] .tile-grid-ghosts.tile-grid-ghosts--release:not(.til font-weight: 500; } -.local-model-next { - font-size: 0.66rem; - line-height: 1.45; - color: var(--text-muted); -} - /* In App Settings the body sits inside an ordinary .set-row, so it takes that surface's type scale (label 0.76rem over a muted description) and the row supplies the border. */ diff --git a/test/local-model-i18n.test.ts b/test/local-model-i18n.test.ts index cbcb514f..691bff17 100644 --- a/test/local-model-i18n.test.ts +++ b/test/local-model-i18n.test.ts @@ -89,9 +89,14 @@ describe('the local model strings in zh-CN', () => { expect(FROM_MODULE).toContain('Model on this Mac'); expect(FROM_MODULE).toContain('{model} ({id}) with {harness}'); expect(FROM_MODULE).toContain('Needs about {need} GB of the {budget} GB this Mac can give a model (estimated).'); - expect(FROM_MODULE).toContain('Setting this up from Codeman is the next step. It is not built yet.'); - expect(FROM_MODULE).toContain('Checking this Mac…'); - expect(FROM_MODULE.length).toBeGreaterThanOrEqual(18); + expect(FROM_MODULE).toContain( + 'Codeman cannot download or install this for you yet. That is the next step, and it is not built yet.' + ); + // What the fit and speed labels measure, and the speed of an alternative. + expect(FROM_MODULE).toContain('Memory fit: needs {share}% of what this Mac can give a model'); + expect(FROM_MODULE).toContain('No speed estimate for this model on this chip'); + expect(FROM_MODULE).toContain('about {speed} tokens per second'); + expect(FROM_MODULE.length).toBeGreaterThanOrEqual(20); // The only other way the module calls its translator is with payload text (a server // sentence or label), so no string of its own can reach the page unharvested. const code = MODULE.split('\n') @@ -107,10 +112,16 @@ describe('the local model strings in zh-CN', () => { it('lists every server label, the none reasons included', () => { // A label added to the contract later fails below by its text, not here by a count. - expect(FROM_SERVER.length).toBeGreaterThanOrEqual(14); + expect(FROM_SERVER.length).toBeGreaterThanOrEqual(18); for (const text of Object.values(LOCAL_MODEL_NONE_LABELS)) expect(FROM_SERVER).toContain(text); expect(FROM_SERVER).toContain('Best fit for this Mac'); expect(FROM_SERVER).toContain('Speed unknown'); + // The two other sentences of the `memory` reason and the two notes are not in the + // reason map, so they are named here. + expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noneMemoryContext); + expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noneMemoryUnknown); + expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noteSmallPick); + expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noteDiskDropped); }); it('never uses the translator slot {name} for a value of its own', () => { diff --git a/test/local-model-recommend.test.ts b/test/local-model-recommend.test.ts index 86a2c327..bf22dc8b 100644 --- a/test/local-model-recommend.test.ts +++ b/test/local-model-recommend.test.ts @@ -10,9 +10,11 @@ * parser: the MacBook Pro (M5 Pro, 48 GB) and the Mac mini (M4, 16 GB). Each test states the * expected answer and why. * 3. Every reason for verdict `none`, one fixture per reason, and their order. - * 4. The `limited` verdict, the three-state speed in the ranking, harness rows (a gated one is - * never chosen), the `tested` flag that alone may turn on the word "Recommended", and the - * shape of the answer (labels from the fixed list, sizes in one unit). + * 4. The `limited` verdict (a statement about the Mac, not about the pick), the notes that say + * what the headline cannot (a small pick, a model left out for disk space), the three-state + * speed in the ranking, harness rows (a gated one is never chosen), the `tested` flag that + * alone may turn on the word "Recommended", and the shape of the answer (labels from the + * fixed list, sizes in one unit). * * The stock catalogue is used wherever the answer for a real Mac is at stake. Small synthetic * catalogues (validated by the real schema) isolate one rule of the ranking at a time. @@ -463,13 +465,17 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { /** - * Expected: no recommendation, reason `memory`. + * Expected: no recommendation, reason `memory`, in the sentence that names both numbers. * * Why. The budget is 9.67 GiB (2/3 of 16 GiB minus 1). The only model near it is the 9B. Both * harnesses that may be offered need 65,536 tokens of context, and there the 9B needs * 6.10 GiB of weights + 2 GiB of KV cache + 1 GiB + 0.5 GiB of prompt cache = 9.60 GiB, which * is 0.993 of the budget: above the 0.98 where "tight" ends. At 32,768 tokens it would fit * (0.890, tight), but the one harness that works at that context is gated. + * + * The miss is 1.3 points on an ESTIMATED budget, so the sentence is not "too little memory" + * (what an 8 GB Mac is told): it says the budget (10.4 GB), the need (10.3 GB) and that + * Codeman estimates. */ it('gets no recommendation: nothing fits at the context a harness needs', async () => { const hw = await realMac(MACMINI_SYSCTL, 41_752_848); @@ -482,8 +488,9 @@ describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { verdict: 'none', reason: 'memory', headline: { - text: 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', - vars: { memory: '17.2' }, + text: 'Codeman estimates that this Mac can give a model about {budget} GB of memory. The smallest model a coding agent can work with needs about {need} GB. That leaves no safe margin, so Codeman recommends none here.', + // floor(17,179,869,184 x 2/3) - 1 GiB, and the 9B at 65536 tokens: 10,308,921,757 bytes. + vars: { budget: '10.4', need: '10.3' }, }, notes: [], alternatives: [], @@ -556,12 +563,13 @@ describe('every memory size, with the stock catalogue', () => { // gpt-oss:20b at 65536 tokens needs 15.8 GiB of the 15 GiB budget: out. [24, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], // gpt-oss:20b fits only as Tight here (0.853 at 65536 tokens, 0.927 at 131072), and Good - // ranks before Tight. The Q8 9B is Good but slow on a base M4 (7 tokens per second). - [32, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], - [32, 'Apple M1 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + // ranks before Tight. The Q8 9B is Good but slow on a base M4 (7 tokens per second). The + // pick is a 9B, the verdict is `ok` all the same: a 20B fits this Mac. + [32, 'Apple M4', 'ok qwen3.5:9b@65536 good ok'], + [32, 'Apple M1 Max', 'ok qwen3.5:9b-q8_0@65536 good ok'], // The 27B (0.905) and the 35B-A3B (0.974) are Tight; gpt-oss:20b is Good but has no speed // figure, and a known speed ranks before an unknown one. - [36, 'Apple M4 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + [36, 'Apple M4 Max', 'ok qwen3.5:9b-q8_0@65536 good ok'], // On a Pro chip the dense 27B is slow (8.1 tokens per second), the 35B-A3B is not. [48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'], [48, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], @@ -712,18 +720,60 @@ describe('verdict none', () => { describe('memory', () => { it('answers so at 8 GB, where nothing fits at any context', () => { const result = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); - expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemory); - // The size is the decimal figure of the hardware line, without the unit. - expect(result.headline.vars).toEqual({ memory: '8.6' }); + expectNone( + result, + 'memory', + 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.' + ); + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.noneMemory); + // The size the Mac is sold as (what About This Mac shows), not 8 x 2^30 bytes as 8.6 decimal GB. + expect(result.headline.vars).toEqual({ memory: '8' }); expect(result.budgetBytes).toBe(2 * GIB); + // Every computed size stays decimal GB. expect(result.budgetGb).toBe('2.1'); }); - it('answers so when models fit, but only below the context a harness needs', () => { + it('names the installed memory as the Mac is sold, for every size', () => { + // An empty catalogue: nothing fits whatever the size, so the sentence is the plain one. + for (const soldGb of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128]) { + const result = recommend(mac(soldGb, 'Apple M4'), catalogOf([]), [fact('alpha')]); + expect(result.headline, `${soldGb} GB`).toEqual({ + text: LOCAL_MODEL_LABELS.noneMemory, + vars: { memory: String(soldGb) }, + }); + } + }); + + it('says both numbers when models fit, but only below the context a harness needs', () => { // The 16 GB case of the Mac mini, with harnesses that are all present. const result = recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED); - expect(result.reason).toBe('memory'); - expect(result.headline.vars).toEqual({ memory: '17.2' }); + expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemoryContext); + // The need is the smallest one at a context a usable harness works at (the 9B at 65536), + // not the 9B at 32768, which fits and which only the gated harness could use. + expect(result.headline.vars).toEqual({ budget: '10.4', need: '10.3' }); + expect(result.headline.text).toMatch(/^Codeman estimates/); + expect(result.headline.text).not.toContain('{memory}'); + }); + + it('says the same two numbers when the need is far above the budget', () => { + // One model: 3 GiB at 1024 tokens (fits 10 GiB), 30 GiB at the context the harness needs. + const catalog = catalogOf( + [model('probe:1', { bytes: 2 * GIB, kvBytesPerToken: 1024, contexts: [28 * 1024 * 1024, 1024] })], + [{ cli: 'alpha', minContext: 4096, rank: 1 }] + ); + const hw = mac(16, 'Apple M4', { wiredLimitMb: 14000 }); + const result = recommend(hw, catalog, [fact('alpha')]); + expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemoryContext); + // 2 GiB + 28 GiB of KV cache + 1 GiB = 31 GiB; the budget is 10 GiB. + expect(result.headline.vars).toEqual({ budget: formatGb(10 * GIB), need: formatGb(31 * GIB) }); + expect(result.budgetSource).toBe('wired-limit'); + }); + + it('falls back to the plain sentence when no model has a context a harness could use', () => { + const catalog = catalogOf([model('probe:1')], [{ cli: 'alpha', minContext: 4096, rank: 1 }]); + const result = recommend(bigMac, catalog, [fact('alpha')]); + expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemory); + expect(result.headline.vars).toEqual({ memory: '48' }); }); it('comes before no-harness when nothing fits at all', () => { @@ -734,10 +784,17 @@ describe('verdict none', () => { expect(recommend(bigMac, catalogOf([]), [fact('alpha')]).reason).toBe('memory'); }); - it('answers so when the memory size is unknown', () => { + it('answers so when the memory size is unknown, without stating a size', () => { const hw = { ...mac(48, 'Apple M4 Max'), memBytes: 0, memGb: '0.0', memClassGb: 0 }; const result = recommend(hw, STOCK, ALL_INSTALLED); - expect(result.reason).toBe('memory'); + expectNone( + result, + 'memory', + 'Codeman could not read how much memory this Mac has, so it recommends no model here.' + ); + // No "0 GB of memory": a size that was never read is not a size. + expect(result.headline.vars).toBeUndefined(); + expect(stringsIn(result.headline).join(' ')).not.toMatch(/\d/); expect(result.budgetBytes).toBe(0); }); }); @@ -787,13 +844,23 @@ describe('verdict none', () => { expectNone( result, 'disk', - 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.' + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download, and {reserve} GB must stay free after it. {free} GB is free now. Free up about {missing} GB.' ); // download + 10 GiB - free = 6,550,825,373 + 10,737,418,240 - 12,884,901,888 bytes. - expect(result.headline.vars).toEqual({ download: '6.6', free: '12.9', missing: '4.4' }); + expect(result.headline.vars).toEqual({ download: '6.6', reserve: '10.7', free: '12.9', missing: '4.4' }); expect(result.budgetBytes).toBe(35 * GIB); }); + it('gives the reader every number of the sum: download + reserve - free = missing', () => { + // 12 GB free is more than the 6.6 GB download, so without the reserve "not enough" reads false. + const hw = mac(24, 'Apple M4', { freeDiskBytes: 12 * GB, freeDiskGb: formatGb(12 * GB) }); + const vars = recommend(hw, STOCK, ALL_INSTALLED).headline.vars as Record; + expect(vars).toEqual({ download: '6.6', reserve: '10.7', free: '12.0', missing: '5.3' }); + expect(vars.reserve).toBe(formatGb(DISK_HEADROOM_BYTES)); + // Within the rounding of four one-decimal figures. + expect(Number(vars.download) + Number(vars.reserve) - Number(vars.free)).toBeCloseTo(Number(vars.missing), 0); + }); + it('names the smallest model that fits the MEMORY, not the smallest of the catalogue', () => { // On a 48 GB Mac whose harness refuses both 9B ids, the smallest candidate is gpt-oss:20b. const facts = ALL_INSTALLED.map((entry) => ({ @@ -806,12 +873,76 @@ describe('verdict none', () => { expect(result.headline.vars?.download).toBe('13.8'); }); - it('drops only the models that do not leave 10 GiB, and picks among the rest', () => { + it('drops only the models that do not leave 10 GiB, picks among the rest, and says what was left out', () => { // 20 GiB free: the 9B downloads (6.1 and 9.7 GiB) leave 10 GiB or more, the others do not. const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 20 * GIB, freeDiskGb: formatGb(20 * GIB) }); const result = recommend(hw, STOCK, ALL_INSTALLED); - expect(result.verdict).toBe('limited'); expect(order(result)).toEqual(['qwen3.5:9b-q8_0@65536', 'qwen3.5:9b@65536']); + // A 48 GB Mac is not a small-model Mac because its disk is full: the verdict stays `ok`, + // the pick is called small in a note, and the model it would have been is named with + // what is missing: 17,741,871,939 + 10 GiB - 20 GiB = 7.0 GB. + expect(result.verdict).toBe('ok'); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + expect(result.notes).toEqual([ + { + text: 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.', + }, + { + text: '{model} fits the memory of this Mac too, but its download needs about {missing} GB more free disk space.', + vars: { model: 'Qwen3.8 27B', missing: '7.0' }, + }, + { text: 'Not yet tested by us on a Mac like this.' }, + ]); + }); + + it('turns on 0.4 GB of disk on the real MacBook without calling it a small-model Mac', async () => { + // The cut for the 35B-A3B: 22,621,314,161 + 10 GiB = 33,358,732,401 bytes free. + const macbook = await realMac(MACBOOK_SYSCTL, 890_926_084); + const withFree = (bytes: number): HardwareFacts => ({ + ...macbook, + freeDiskBytes: bytes, + freeDiskGb: formatGb(bytes), + }); + + const enough = recommend(withFree(34 * GB), STOCK, ALL_INSTALLED); + expect(enough.verdict).toBe('ok'); + expect(enough.pick?.modelId).toBe('qwen3.6:35b-a3b'); + expect(enough.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + + const short = recommend(withFree(33 * GB), STOCK, ALL_INSTALLED); + expect(short.verdict).toBe('ok'); + expect(short.headline.text).toBe(LOCAL_MODEL_LABELS.headlineOk); + expect(short.pick?.modelId).toBe('qwen3.5:9b-q8_0'); + expect(order(short)).not.toContain('qwen3.6:35b-a3b@131072'); + expect(short.notes).toEqual([ + { text: LOCAL_MODEL_LABELS.noteSmallPick }, + { text: LOCAL_MODEL_LABELS.noteDiskDropped, vars: { model: 'Qwen3.6 35B-A3B', missing: '0.4' } }, + { text: LOCAL_MODEL_LABELS.noteUntested }, + ]); + // The exact cut: one byte decides, and the note appears with the first missing byte. + const cut = 22_621_314_161 + DISK_HEADROOM_BYTES; + expect(recommend(withFree(cut), STOCK, ALL_INSTALLED).pick?.modelId).toBe('qwen3.6:35b-a3b'); + expect(recommend(withFree(cut - 1), STOCK, ALL_INSTALLED).pick?.modelId).toBe('qwen3.5:9b-q8_0'); + }); + + it('names the best-ranked model left out, also when the pick is not affected', () => { + // 64 GB, M4 Max, 35 GB free: only the Q8 27B (30 GB) has no room. The pick stays the 27B. + const hw = mac(64, 'Apple M4 Max', { freeDiskBytes: 35 * GB, freeDiskGb: '35.0' }); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.pick?.modelId).toBe('qwen3.8:27b'); + expect(order(result).some((entry) => entry.startsWith('qwen3.8:27b-q8_0'))).toBe(false); + // 29,978,241,837 + 10 GiB - 35 GB. + expect(result.notes).toEqual([ + { text: LOCAL_MODEL_LABELS.noteDiskDropped, vars: { model: 'Qwen3.8 27B (Q8)', missing: '5.7' } }, + { text: LOCAL_MODEL_LABELS.noteUntested }, + ]); + }); + + it('adds no disk note when every model that fits has room, or the free disk is unknown', () => { + for (const hw of [mac(64, 'Apple M4 Max'), without(mac(64, 'Apple M4 Max'), 'freeDiskBytes', 'freeDiskGb')]) { + const texts = recommend(hw, STOCK, ALL_INSTALLED).notes.map((note) => note.text); + expect(texts).not.toContain(LOCAL_MODEL_LABELS.noteDiskDropped); + } }); it('allows a download that leaves exactly 10 GiB', () => { @@ -845,7 +976,7 @@ describe('verdict none', () => { // ---------- 4. Verdicts, ranking, harnesses, tested ---------- describe('verdict limited', () => { - it('is the answer when the best model that fits has fewer than 12 billion parameters', () => { + it('is the answer when no model that fits has 12 billion parameters or more', () => { expect(LIMITED_BELOW_PARAMS_B).toBe(12); const result = recommend(mac(24, 'Apple M4'), STOCK, ALL_INSTALLED); expect(result.verdict).toBe('limited'); @@ -873,11 +1004,77 @@ describe('verdict limited', () => { expect(verdictFor(27.8)).toBe('ok'); }); - it('is about the PICK: a larger model further down the list does not lift it', () => { - // 32 GB: the 20B fits only as Tight, so a 9B is the pick and the answer is limited. + it('is about the MAC, not the pick: a larger model further down the list lifts it', () => { + // 32 GB: the 20B fits only as Tight, so a 9B is the pick. The Mac can still run a 20B, so + // "This Mac can run a small model" would be false of it. const result = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED); - expect(result.verdict).toBe('limited'); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', paramsB: 9 }); expect(result.alternatives.map((c) => [c.modelId, c.fit])).toContainEqual(['gpt-oss:20b', 'tight']); + expect(result.verdict).toBe('ok'); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + // The pick is still called what it is. + expect(result.notes).toEqual([ + { + text: 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.', + }, + { text: 'Not yet tested by us on a Mac like this.' }, + ]); + }); + + it('does not tell a 36 GB Mac it is a small-model Mac above a list of 20B, 27B and 35B models', () => { + const result = recommend(mac(36, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b-q8_0', fit: 'good', speed: 'ok' }); + expect(result.alternatives.map((c) => [c.modelId, c.fit, c.speed])).toEqual([ + ['qwen3.5:9b', 'good', 'ok'], + ['gpt-oss:20b', 'good', 'unknown'], + // Both Tight and fast enough: the higher model rank first. + ['qwen3.8:27b', 'tight', 'ok'], + ['qwen3.6:35b-a3b', 'tight', 'ok'], + ]); + expect(result.verdict).toBe('ok'); + expect(result.headline.text).not.toBe(LOCAL_MODEL_LABELS.headlineLimited); + expect(result.notes[0]).toEqual({ text: LOCAL_MODEL_LABELS.noteSmallPick }); + }); + + it('does not depend on whether the chip is in the chips table', () => { + // Knowing the chip must not lower the verdict: before, a known chip made the 9B win on + // speed and the answer limited, while an unknown chip got `ok` with the 20B. + for (const soldGb of [32, 36]) { + const known = recommend(mac(soldGb, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const unknown = recommend(mac(soldGb, 'Apple M6'), STOCK, ALL_INSTALLED); + expect([soldGb, known.verdict]).toEqual([soldGb, unknown.verdict]); + expect(known.verdict).toBe('ok'); + } + }); + + it('is limited exactly when nothing with 12 billion parameters fits, on every Mac of the sweep', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + if (result.verdict === 'none') continue; + const budget = deviceBudget(hw).budgetBytes; + const largeFits = STOCK.models.some( + (m) => + m.paramsB >= LIMITED_BELOW_PARAMS_B && + m.contexts.some((ctx) => ctx >= 65536 && needBytes(m, ctx, STOCK, hw) / budget <= FIT_TIGHT_MAX_SHARE) + ); + expect([hw.memClassGb, hw.chip, result.verdict]).toEqual([hw.memClassGb, hw.chip, largeFits ? 'ok' : 'limited']); + // The small-pick note: under `ok` only, and exactly when the pick is small. + const small = result.notes.some((note) => note.text === LOCAL_MODEL_LABELS.noteSmallPick); + expect(small).toBe(result.verdict === 'ok' && (result.pick as Candidate).paramsB < LIMITED_BELOW_PARAMS_B); + } + }); + + it('counts only a large model a harness can really run', () => { + // 48 GB, but the harnesses refuse every id but the two 9B ones: nothing large can be offered. + const facts = ALL_INSTALLED.map((entry) => ({ + ...entry, + acceptsModelId: (id: string) => id.startsWith('qwen3.5:9b'), + })); + const result = recommend(mac(48, 'Apple M4 Max'), STOCK, facts); + expect(result.verdict).toBe('limited'); + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); + // The headline says it, so the note does not repeat it. + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); }); it('never says "Recommended", for any Mac', () => { @@ -909,6 +1106,17 @@ describe('speed', () => { expect(speedOf(0.5)).toMatchObject({ speed: 'slow', tokensPerSecond: 2, speedLabel: 'Slow' }); }); + it('grades the unrounded figure: 9.95 to 9.9996 tokens per second print as 10 and are still slow', () => { + const speedOf = (speedPer100Gbps: number): Candidate | undefined => + recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { speedPer100Gbps })]), [fact('alpha')]).pick; + // 2.4875 x 4 = 9.95, 2.49 x 4 = 9.96, 2.4999 x 4 = 9.9996: all under 10, all rounding to 10.0. + for (const figure of [2.4875, 2.49, 2.4999]) { + expect(speedOf(figure), String(figure)).toMatchObject({ speed: 'slow', speedLabel: 'Slow', tokensPerSecond: 10 }); + } + // And the first figure that is not under 10. + expect(speedOf(2.5)).not.toHaveProperty('speedLabel'); + }); + it('is unknown for a model without a figure, on any chip', () => { // 48 GB: five models fit, so all of them are listed and gpt-oss:20b is among them. for (const chip of ['Apple M1', 'Apple M4 Max', 'Apple M5 Ultra']) { @@ -1145,7 +1353,8 @@ describe('the tested flag and the word "Recommended"', () => { } expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); if (result.verdict !== 'none') { - expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + // Always the last note; a small pick under `ok` has one more before it. + expect(result.notes[result.notes.length - 1]).toEqual({ text: LOCAL_MODEL_LABELS.noteUntested }); } } }); @@ -1239,12 +1448,30 @@ describe('the shape of the answer', () => { 'no-harness', 'ok', ]); + // The two other sentences of the `memory` reason, and an answer with every kind of note. + answers.push( + recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED), + recommend({ ...mac(48, 'Apple M4'), memBytes: 0, memGb: '0.0', memClassGb: 0 }, STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { freeDiskBytes: 20 * GIB, freeDiskGb: '21.5' }), STOCK, ALL_INSTALLED) + ); + expect(answers.slice(5).map((answer) => answer.headline.text)).toEqual([ + LOCAL_MODEL_LABELS.noneMemoryContext, + LOCAL_MODEL_LABELS.noneMemoryUnknown, + LOCAL_MODEL_LABELS.headlineOk, + ]); + expect(answers[7].notes.map((note) => note.text)).toEqual([ + LOCAL_MODEL_LABELS.noteSmallPick, + LOCAL_MODEL_LABELS.noteDiskDropped, + LOCAL_MODEL_LABELS.noteUntested, + ]); for (const answer of answers) { - const slots = [...answer.headline.text.matchAll(/\{(\w+)\}/g)].map((match) => match[1]).sort(); - expect(Object.keys(answer.headline.vars ?? {}).sort()).toEqual(slots); - // `{name}` belongs to the browser's translator. - expect(slots).not.toContain('name'); - for (const value of Object.values(answer.headline.vars ?? {})) expect(typeof value).toBe('string'); + for (const message of [answer.headline, ...answer.notes]) { + const slots = [...message.text.matchAll(/\{(\w+)\}/g)].map((match) => match[1]).sort(); + expect(Object.keys(message.vars ?? {}).sort()).toEqual(slots); + // `{name}` belongs to the browser's translator. + expect(slots).not.toContain('name'); + for (const value of Object.values(message.vars ?? {})) expect(typeof value).toBe('string'); + } } }); diff --git a/test/local-model-ui.test.ts b/test/local-model-ui.test.ts index 00daa696..07d80879 100644 --- a/test/local-model-ui.test.ts +++ b/test/local-model-ui.test.ts @@ -4,11 +4,15 @@ * module and a mocked `_api`. * * What is pinned: - * - every state the route can produce: loading, refused (403), failed, not macOS, each `none` - * reason, `limited`, `ok`; - * - the card stays hidden until the server answered for a Mac, and the Settings row carries - * the loading and failure sentences; + * - every state the route can produce: loading, refused (403 FORBIDDEN), a 403 that is not a + * refusal, failed, not macOS, each `none` sentence, `limited`, `ok`; + * - the card stays hidden until the server answered for a Mac, and so does the Settings group + * (never "Model on this Mac" on a server that is not one); the group carries the failure + * sentence; * - the card and the row show the SAME pick (one builder), the card folds the alternatives; + * - the installed memory is the size the Mac is sold as, every other size is printed as sent; + * - each fit and speed label says what it measures, and the line about the missing setup + * sits right under the pick; * - every payload string lands as text: an `` in a model, harness or chip name, * or in a slot value of a server sentence, creates no element and runs nothing; * - sizes are printed as sent (decimal GB text), and the word "Recommended" is the server's @@ -42,8 +46,10 @@ const INDEX = readFileSync(resolve(PUBLIC, 'index.html'), 'utf8'); const MODULE = readFileSync(resolve(PUBLIC, 'local-model-ui.js'), 'utf8'); const I18N = readFileSync(resolve(PUBLIC, 'i18n.js'), 'utf8'); -const DISMISS_KEY = 'codeman:local-model-dismissed'; -const NEXT_STEP = 'Setting this up from Codeman is the next step. It is not built yet.'; +/** Named for the read-only stage, so the stage that can set a model up takes a key of its own. */ +const DISMISS_KEY = 'codeman:local-model-dismissed:readonly'; +const NEXT_STEP = + 'Codeman cannot download or install this for you yet. That is the next step, and it is not built yet.'; type ApiResponse = { ok: boolean; status: number; json(): Promise } | null; @@ -253,6 +259,18 @@ function none(reason: NoneReason, vars?: Record, hardware: Parti return status({ verdict: 'none', reason, headline, budgetBytes: 0, budgetGb: '0.0' }, hardware); } +/** A `none` answer whose headline is one of the other sentences of its reason. */ +function noneWith( + reason: NoneReason, + text: LocalModelMessage['text'], + vars?: Record, + hardware: Partial = {} +) { + const answer = none(reason, undefined, hardware); + answer.recommendation.headline = { text, ...(vars ? { vars } : {}) }; + return answer; +} + const NONE_CASES: Array<{ reason: Exclude; answer: LocalModelStatus; sentence: string }> = [ { reason: 'intel', @@ -261,15 +279,15 @@ const NONE_CASES: Array<{ reason: Exclude; answer: Loca }, { reason: 'memory', - answer: none('memory', { memory: '8.6' }, { memGb: '8.6', memClassGb: 8 }), + answer: none('memory', { memory: '8' }, { memGb: '8.6', memClassGb: 8 }), sentence: - 'This Mac has 8.6 GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', + 'This Mac has 8 GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', }, { reason: 'disk', - answer: none('disk', { download: '17.7', free: '12.0', missing: '16.4' }, { freeDiskGb: '12.0' }), + answer: none('disk', { download: '17.7', reserve: '10.7', free: '12.0', missing: '16.4' }, { freeDiskGb: '12.0' }), sentence: - 'Not enough free disk space. The smallest model that fits this Mac is a 17.7 GB download and 12.0 GB is free. Free up about 16.4 GB.', + 'Not enough free disk space. The smallest model that fits this Mac is a 17.7 GB download, and 10.7 GB must stay free after it. 12.0 GB is free now. Free up about 16.4 GB.', }, { reason: 'macos', @@ -328,18 +346,41 @@ describe('the markup the module fills', () => { }); describe('before there is an answer', () => { - it('keeps the card hidden while the request is in flight, and says so in Settings', async () => { - const { newApp, card, row } = mount(); + it('keeps the card and the Settings group hidden while the first request is in flight', async () => { + const { newApp, card, row, group } = mount(); let answer!: (response: ApiResponse) => void; const app = newApp(() => new Promise((done) => (answer = done))); const loading = app.loadLocalModel(); expect(card.hidden).toBe(true); expect(card.textContent).toBe(''); - expect(row.textContent).toBe('Checking this Mac…'); + // Nothing says yet that the server is a Mac, so no group titled "Model on this Mac" and no + // sentence about "this Mac" either: on most servers both would be false. + expect(group.style.display).toBe('none'); + expect(row.textContent).toBe(''); answer(ok(OK_STATUS)); await loading; expect(card.hidden).toBe(false); - expect(row.textContent).not.toContain('Checking this Mac…'); + expect(group.style.display).toBe(''); + expect(texts(row, '.local-model-headline')).toEqual(['Best fit for this Mac']); + }); + + it('keeps an answer on screen, group included, while it is refreshed', async () => { + const { dom, newApp, row, group } = mount(); + let now = 1_000_000; + (dom.window as unknown as { Date: DateConstructor }).Date.now = () => now; + let answer!: (response: ApiResponse) => void; + const answers: Array<() => Promise> = [ + async () => ok(OK_STATUS), + () => new Promise((done) => (answer = done)), + ]; + const app = newApp(() => (answers.shift() as () => Promise)()); + await app.loadLocalModel(); + now += 61_000; + const refreshing = app.loadLocalModel(); + expect(group.style.display).toBe(''); + expect(texts(row, '.local-model-headline')).toEqual(['Best fit for this Mac']); + answer(ok(OK_STATUS)); + await refreshing; }); it.each([ @@ -383,6 +424,41 @@ describe('a refused request (a non-admin in multi-user mode)', () => { expect(app._api).toHaveBeenCalledTimes(1); }); + it('does not take another 403 for a refusal: an admin who must change the password is asked again', async () => { + // The auth layer answers 403 PASSWORD_CHANGE_REQUIRED for every /api route until the + // password is changed, and the dialog that changes it closes without a reload. + const { dom, document, newApp, card, row, group } = mount(); + const win = dom.window as unknown as { __codemanUser?: { multiUser: boolean; role: string } }; + win.__codemanUser = { multiUser: true, role: 'admin' }; + const answers: ApiResponse[] = [refusal(403, 'PASSWORD_CHANGE_REQUIRED'), ok(OK_STATUS)]; + const app = newApp(async () => answers.shift() ?? null); + await app.loadLocalModel(); + expect(app._api).toHaveBeenCalledTimes(1); + expect(card.hidden).toBe(true); + // A failed request, not a refusal: the group stays, with the failure sentence. + expect(group.style.display).toBe(''); + expect(row.textContent).toBe('Could not reach the server.'); + + // The password was changed: admin-ui.js announces the user again. + document.dispatchEvent(new dom.window.CustomEvent('codeman:me')); + await vi.waitFor(() => expect(card.hidden).toBe(false)); + expect(app._api).toHaveBeenCalledTimes(2); + expect(group.style.display).toBe(''); + expect(texts(row, '.local-model-headline')).toEqual(['Best fit for this Mac']); + }); + + it.each([ + ['a 403 without a body', { ok: false, status: 403, json: async () => Promise.reject(new Error('no body')) }], + ['a 403 with another error code', refusal(403, 'SOMETHING_ELSE')], + ] as Array<[string, ApiResponse]>)('keeps asking after %s', async (_name, response) => { + const { newApp, group } = mount(); + const app = newApp(response); + await app.loadLocalModel(); + await app.loadLocalModel(); + expect(app._api).toHaveBeenCalledTimes(2); + expect(group.style.display).toBe(''); + }); + it('does not ask at all once the page knows the role, and asks when the role turns out to be admin', async () => { const { dom, document, newApp, card, group } = mount(); const win = dom.window as unknown as { __codemanUser?: { multiUser: boolean; role: string } }; @@ -402,22 +478,45 @@ describe('a refused request (a non-admin in multi-user mode)', () => { }); describe('a server that is not a Mac', () => { - it('hides the card and leaves one plain sentence in Settings', async () => { + const LINUX = none('not-macos', undefined, { + platform: 'linux', + arch: 'x64', + arm64: undefined, + chip: 'AMD Ryzen 9 7950X', + source: 'not-probed', + }); + + it('shows neither the card nor the Settings group: nothing there is about this server', async () => { const { newApp, card, row, group } = mount(); - const answer = none('not-macos', undefined, { - platform: 'linux', - arch: 'x64', - arm64: undefined, - chip: 'AMD Ryzen 9 7950X', - source: 'not-probed', - }); - await newApp(ok(answer)).loadLocalModel(); + await newApp(ok(LINUX)).loadLocalModel(); expect(card.hidden).toBe(true); expect(card.textContent).toBe(''); - expect(group.style.display).toBe(''); - expect(row.textContent).toBe('Codeman recommends a local model only on a Mac with Apple silicon.'); - // No line listing another machine's parts under a heading that says "this Mac". - expect(row.querySelector('.local-model-hw')).toBeNull(); + // A group titled "Model on this Mac", about "the Mac that Codeman runs on", on a Linux + // server: hidden, the same lever as a refused request. + expect(group.style.display).toBe('none'); + expect(row.textContent).toBe(''); + }); + + it('never shows the group at any point of the load, and keeps it away on a refresh', async () => { + const { dom, newApp, group, row } = mount(); + let now = 1_000_000; + (dom.window as unknown as { Date: DateConstructor }).Date.now = () => now; + const seen: string[] = []; + const app = newApp(async () => { + seen.push(group.style.display); + return ok(LINUX); + }); + const first = app.loadLocalModel(); + seen.push(group.style.display); + await first; + now += 61_000; + const second = app.loadLocalModel(); + seen.push(group.style.display); + await second; + seen.push(group.style.display); + expect(app._api).toHaveBeenCalledTimes(2); + expect(seen).toEqual(['none', 'none', 'none', 'none', 'none']); + expect(row.textContent).toBe(''); }); }); @@ -441,6 +540,41 @@ describe('verdict none on a Mac', () => { const covered = [...NONE_CASES.map((c) => c.reason), 'not-macos'].sort(); expect(covered).toEqual(Object.keys(LOCAL_MODEL_NONE_LABELS).sort()); }); + + it('says both numbers when the Mac misses by a hair (the 16 GB Mac mini)', async () => { + const { newApp, card, row } = mount(); + const answer = noneWith( + 'memory', + LOCAL_MODEL_LABELS.noneMemoryContext, + { budget: '10.4', need: '10.3' }, + { memGb: '17.2', memClassGb: 16, chip: 'Apple M4', freeDiskGb: '42.8' } + ); + await newApp(ok(answer)).loadLocalModel(); + for (const root of [card, row]) { + expect(texts(root, '.local-model-hw')).toEqual(['Apple M4, 16 GB memory, 42.8 GB free']); + expect(texts(root, '.local-model-headline')).toEqual([ + 'Codeman estimates that this Mac can give a model about 10.4 GB of memory. The smallest model a coding agent can work with needs about 10.3 GB. That leaves no safe margin, so Codeman recommends none here.', + ]); + } + }); + + it('states no memory size when the server could not read one', async () => { + const { newApp, card, row } = mount(); + const answer = noneWith('memory', LOCAL_MODEL_LABELS.noneMemoryUnknown, undefined, { + memBytes: 0, + memGb: '0.0', + memClassGb: 0, + }); + await newApp(ok(answer)).loadLocalModel(); + for (const root of [card, row]) { + // The chip and nothing else: no "0 GB memory" in the hardware line, no size in the sentence. + expect(texts(root, '.local-model-hw')).toEqual(['Apple M5 Pro']); + expect(texts(root, '.local-model-headline')).toEqual([ + 'Codeman could not read how much memory this Mac has, so it recommends no model here.', + ]); + expect(root.textContent).not.toMatch(/\b0(\.0)? GB/); + } + }); }); describe('verdict ok', () => { @@ -449,7 +583,12 @@ describe('verdict ok', () => { await newApp(ok(OK_STATUS)).loadLocalModel(); expect(card.hidden).toBe(false); expect(texts(card, '.local-model-title')).toEqual(['Model on this Mac']); - expect(texts(card, '.local-model-hw')).toEqual(['Apple M5 Pro, 51.5 GB memory, 912 GB free']); + // The page can be open on another device: the title says whose Mac is meant. + expect((card.querySelector('.local-model-title') as HTMLElement).title).toBe( + 'The Mac that Codeman runs on, which is not always the device you are reading on.' + ); + // 48, the size this Mac is sold as and what About This Mac shows: not 51.5 (its bytes in decimal GB). + expect(texts(card, '.local-model-hw')).toEqual(['Apple M5 Pro, 48 GB memory, 912 GB free']); expect(texts(card, '.local-model-headline')).toEqual(['Best fit for this Mac']); expect(texts(card, '.local-model-note')).toEqual(['Not yet tested by us on a Mac like this.']); @@ -486,8 +625,10 @@ describe('verdict ok', () => { const { newApp, card } = mount(); await newApp(ok(OK_STATUS)).loadLocalModel(); const alts = card.querySelector('.local-model-alts')!; - expect(texts(alts, '.local-model-alts-title')).toEqual(['Other models that fit2']); - expect(texts(alts, '.local-model-count')).toEqual(['2']); + // No count beside the title: the server lists four at most, so a number here would be + // the number of rows, not of models that fit. + expect(texts(alts, '.local-model-alts-title')).toEqual(['Other models that fit']); + expect(alts.querySelector('.local-model-count')).toBeNull(); const rows = [...alts.querySelectorAll('.local-model-alt')]; expect(rows.map((row) => texts(row, '.local-model-name')[0])).toEqual([ 'Qwen3.6 35B-A3B (qwen3.6:35b-a3b) with OpenCode', @@ -495,23 +636,91 @@ describe('verdict ok', () => { ]); // A candidate whose speed is fine carries a fit label only. expect(rows.map((row) => texts(row, '.local-model-pill'))).toEqual([['Good'], ['Tight', 'Slow']]); + // An alternative with an estimate prints it: "Slow" alone does not say how slow. expect(rows.map((row) => texts(row, '.local-model-line')[0])).toEqual([ - 'Context 131,072 tokens · 26.4 GB of memory · 22.6 GB download', - 'Context 65,536 tokens · 36.4 GB of memory · 30.0 GB download', + 'Context 131,072 tokens · 26.4 GB of memory · 22.6 GB download · about 42 tokens per second', + 'Context 65,536 tokens · 36.4 GB of memory · 30.0 GB download · about 5.2 tokens per second', ]); }); + it('says what each label measures, as its tooltip and its accessible name', async () => { + const { newApp, card, row } = mount(); + await newApp(ok(OK_STATUS)).loadLocalModel(); + for (const root of [card, row]) { + const pills = [...root.querySelectorAll('.local-model-pill')] as HTMLElement[]; + expect(pills.map((pill) => [pill.textContent, pill.title])).toEqual([ + // The pick: 0.79 of the budget, a chip in no chips row. + ['Good', 'Memory fit: needs 79% of what this Mac can give a model'], + ['Speed unknown', 'No speed estimate for this model on this chip'], + ['Good', 'Memory fit: needs 70% of what this Mac can give a model'], + ['Tight', 'Memory fit: needs 97% of what this Mac can give a model'], + ['Slow', 'About 5.2 tokens per second (estimated).'], + ]); + for (const pill of pills) { + expect(pill.getAttribute('role')).toBe('note'); + expect(pill.getAttribute('aria-label')).toBe(`${pill.textContent}. ${pill.title}`); + } + } + }); + + it('leaves a fit label without a share bare instead of inventing a number', async () => { + const { newApp, card } = mount(); + const bare = candidate({ budgetShare: undefined as unknown as number, speedLabel: undefined, speed: 'ok' }); + await newApp(ok(status({ pick: bare }))).loadLocalModel(); + const pill = card.querySelector('.local-model-pill') as HTMLElement; + expect(pill.textContent).toBe('Good'); + expect(pill.title).toBe(''); + expect(pill.hasAttribute('aria-label')).toBe(false); + }); + + it('puts the line about the missing setup right under the pick and its notes, above the list', async () => { + const { newApp, card, row } = mount(); + const withNotes = status({ + notes: [ + { text: LOCAL_MODEL_LABELS.noteSmallPick }, + { text: LOCAL_MODEL_LABELS.noteDiskDropped, vars: { model: 'Qwen3.6 35B-A3B', missing: '0.4' } }, + { text: LOCAL_MODEL_LABELS.noteUntested }, + ], + pick: candidate(), + alternatives: [ALT_FAST, ALT_SLOW], + }); + await newApp(ok(withNotes)).loadLocalModel(); + for (const root of [card, row]) { + const body = root.querySelector('.local-model-body')!; + expect([...body.children].map((el) => el.className)).toEqual([ + 'local-model-hw', + 'local-model-headline', + 'local-model-pick', + 'local-model-note', + 'local-model-note', + 'local-model-note', + 'local-model-next', + 'local-model-alts', + ]); + // A note with slots is filled like a headline. + expect(texts(root, '.local-model-note')).toEqual([ + 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.', + 'Qwen3.6 35B-A3B fits the memory of this Mac too, but its download needs about 0.4 GB more free disk space.', + 'Not yet tested by us on a Mac like this.', + ]); + expect(texts(root, '.local-model-next')).toEqual([NEXT_STEP]); + } + }); + it('prints every size exactly as sent, in one unit', async () => { const { newApp, card, row } = mount(); await newApp(ok(OK_STATUS)).loadLocalModel(); for (const root of [card, row]) { const text = root.textContent ?? ''; expect(text).not.toMatch(/GiB|MiB|\bMB\b|bytes/); - // Every number in front of the unit is one the payload carries as text. + // Every number in front of the unit is one the payload carries: as text for the computed + // sizes, and `memClassGb` (48) for the installed memory. const printed = [...text.matchAll(/([\d.,]+) GB/g)].map((m) => m[1]); - const sent = ['51.5', '912', '37.6', '29.6', '17.7', '26.4', '22.6', '36.4', '30.0']; + const sent = ['48', '912', '37.6', '29.6', '17.7', '26.4', '22.6', '36.4', '30.0']; expect(printed.length).toBeGreaterThanOrEqual(sent.length); expect(printed.filter((value) => !sent.includes(value))).toEqual([]); + // The decimal figure of the installed memory (`memGb`) is on neither surface. + expect(text).not.toContain('51.5'); } }); @@ -546,10 +755,18 @@ describe('verdict ok', () => { it('describes the hardware with whatever parts the probe got', async () => { const { newApp, card } = mount(); const cases: Array<[Partial, string]> = [ - [{ freeDiskBytes: undefined, freeDiskGb: undefined }, 'Apple M5 Pro, 51.5 GB memory'], - [{ chip: undefined }, 'Mac17,13, 51.5 GB memory, 912 GB free'], - [{ chip: undefined, model: undefined }, '51.5 GB memory, 912 GB free'], - [{ chip: undefined, model: undefined, freeDiskBytes: undefined, freeDiskGb: undefined }, '51.5 GB memory'], + [{ freeDiskBytes: undefined, freeDiskGb: undefined }, 'Apple M5 Pro, 48 GB memory'], + [{ chip: undefined }, 'Mac17,13, 48 GB memory, 912 GB free'], + [{ chip: undefined, model: undefined }, '48 GB memory, 912 GB free'], + [{ chip: undefined, model: undefined, freeDiskBytes: undefined, freeDiskGb: undefined }, '48 GB memory'], + // The sizes Macs are sold in, as sold. + [{ memGb: '17.2', memClassGb: 16 }, 'Apple M5 Pro, 16 GB memory, 912 GB free'], + [{ memGb: '34.4', memClassGb: 32 }, 'Apple M5 Pro, 32 GB memory, 912 GB free'], + [{ memGb: '137', memClassGb: 128 }, 'Apple M5 Pro, 128 GB memory, 912 GB free'], + // An unknown size is left out, whatever else the payload carries for it. + [{ memGb: '0.0', memClassGb: 0 }, 'Apple M5 Pro'], + [{ memClassGb: undefined as unknown as number }, 'Apple M5 Pro'], + [{ memClassGb: '48' as unknown as number }, 'Apple M5 Pro'], ]; for (const [hardware, line] of cases) { await newApp(ok(status({ pick: candidate() }, hardware))).loadLocalModel(); @@ -567,7 +784,7 @@ describe('verdict limited', () => { expect(texts(root, '.local-model-headline')).toEqual([ 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', ]); - expect(texts(root, '.local-model-hw')).toEqual(['Apple M4, 19.3 GB memory, 912 GB free']); + expect(texts(root, '.local-model-hw')).toEqual(['Apple M4, 18 GB memory, 912 GB free']); expect(texts(root, '.local-model-name')).toEqual(['Qwen3.5 9B (qwen3.5:9b) with OpenCode']); expect(texts(root, '.local-model-pill')).toEqual(['Tight']); expect(texts(root, '.local-model-pick .local-model-line')).toEqual([ @@ -633,7 +850,8 @@ describe('payload text is text', () => { for (const root of [card, row]) { expect(root.querySelector('img')).toBeNull(); expect(root.querySelector('script, iframe, a, [onerror]')).toBeNull(); - expect(texts(root, '.local-model-hw')[0]).toBe(`${EVIL}, ${EVIL} GB memory, ${EVIL} GB free`); + // The memory is a number or nothing: text in its place (`memGb` here) is never printed. + expect(texts(root, '.local-model-hw')[0]).toBe(`${EVIL}, 48 GB memory, ${EVIL} GB free`); expect(texts(root, '.local-model-headline')[0]).toBe( `This Mac runs macOS ${EVIL}. Local models need macOS ${EVIL} or newer.` ); @@ -768,10 +986,13 @@ describe('with the real translator', () => { const app = newApp(ok(OK_STATUS)); await app.loadLocalModel(); expect(texts(card, '.local-model-headline')).toEqual(['最适合此 Mac 的选择']); - expect(texts(card, '.local-model-pill')).toEqual(['良好', '速度未知', '良好', '紧张', '较慢']); + expect(texts(card, '.local-model-pill')).toEqual(['良好', '速度未知', '良好', '偏紧', '较慢']); + expect((card.querySelector('.local-model-pill') as HTMLElement).title).toBe( + '内存占用:需要此 Mac 可分配给模型的内存的 79%' + ); // Names and numbers ride the slots untouched. expect(texts(card, '.local-model-name')[0]).toBe('Qwen3.8 27B(qwen3.8:27b),搭配 OpenCode'); - expect(texts(row, '.local-model-hw')).toEqual(['Apple M5 Pro,内存 51.5 GB,可用磁盘空间 912 GB']); + expect(texts(row, '.local-model-hw')).toEqual(['Apple M5 Pro,内存 48 GB,可用磁盘空间 912 GB']); // Both containers are data-i18n-skip, so the module itself must repaint. i18n.configure({ language: 'en' }); @@ -782,7 +1003,7 @@ describe('with the real translator', () => { i18n.configure({ displayName: 'Acme' }); await settle(); expect(texts(card, '.local-model-next')).toEqual([ - 'Setting this up from Acme is the next step. It is not built yet.', + 'Acme cannot download or install this for you yet. That is the next step, and it is not built yet.', ]); });