From 73ce6a1819b50a086be3ea37375e6489d961a3ef Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 14:56:13 +0100 Subject: [PATCH 1/8] feat(local-model): types and module skeleton for the hardware-aware recommendation The contract three builders code against in parallel. Nothing is recommended yet: the route answers 501 and the catalogue is empty. - src/types/local-model.ts: HardwareFacts, the catalogue rows, HarnessFact, Candidate, Recommendation, LocalModelStatus, and LOCAL_MODEL_LABELS, the one list of sentences the server may put into the payload (the zh-CN dictionary is keyed by them). - src/local-model/format.ts: formatGb, the one place a byte count becomes a size a person reads (decimal GB), and memClassGb. - src/config/local-model-catalog.ts: the Zod schema (model ids are name:tag, lowercase, no slash) and an empty stock catalogue. No override file. - Skeletons with their final signatures: hardware.ts, recommend.ts, harness-facts.ts, routes/local-model-routes.ts (injectable dependencies), public/local-model-ui.js with its script tag. Co-Authored-By: Claude Fable 5.1 --- src/config/local-model-catalog.ts | 122 +++++++++ src/local-model/format.ts | 42 ++++ src/local-model/hardware.ts | 106 ++++++++ src/local-model/harness-facts.ts | 30 +++ src/local-model/recommend.ts | 68 +++++ src/types/index.ts | 2 + src/types/local-model.ts | 357 +++++++++++++++++++++++++++ src/web/public/index.html | 1 + src/web/public/local-model-ui.js | 33 +++ src/web/routes/index.ts | 1 + src/web/routes/local-model-routes.ts | 60 +++++ src/web/server.ts | 2 + 12 files changed, 824 insertions(+) create mode 100644 src/config/local-model-catalog.ts create mode 100644 src/local-model/format.ts create mode 100644 src/local-model/hardware.ts create mode 100644 src/local-model/harness-facts.ts create mode 100644 src/local-model/recommend.ts create mode 100644 src/types/local-model.ts create mode 100644 src/web/public/local-model-ui.js create mode 100644 src/web/routes/local-model-routes.ts diff --git a/src/config/local-model-catalog.ts b/src/config/local-model-catalog.ts new file mode 100644 index 00000000..661ebd8b --- /dev/null +++ b/src/config/local-model-catalog.ts @@ -0,0 +1,122 @@ +/** + * @fileoverview The local model catalogue: which models, harnesses and chips the recommendation + * knows, as a compiled constant validated by a Zod schema. + * + * SKELETON: the schema below is the shape of the contract (it mirrors `LocalModelCatalog` in + * src/types/local-model.ts) and the stock constant is EMPTY. Still to come here: + * - the model rows (ids and byte sizes from the registry manifests, `paramsB`, an empty + * `tested` array each; `flint:40b` stays out), the harness rows and the chips table; + * - the cross-row rules the schema cannot say field by field: unique model ids, unique harness + * `cli`s, `contexts` largest first, `chips` most specific first, `promptCacheTiers` ascending + * and starting at 0; + * - `getLocalModelCatalog()` parsing the constant once, on first use, never at import. + * + * Rules that hold from the start: + * - There is NO override file and no runtime download data in this stage: nothing here reads + * the data dir, and a data file can never decide what gets recommended. + * - A model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`), so it can + * only ever name a model of the default registry and namespace. + * - A harness is a ROW (`{ cli, minContext, rank, gated?, slowFirstTurn? }`). The `cli` is a CLI + * registry id as data: no code may compare it with a literal. Whether that CLI exists and can + * take a custom model is checked against the registry at call time, in + * src/local-model/harness-facts.ts, never here. + */ + +import { z } from 'zod'; +import type { LocalModelCatalog } from '../types.js'; + +/** A model id: lowercase name, one colon, a tag. No slash, so no registry host or namespace can ride along. */ +export const LOCAL_MODEL_ID_PATTERN = /^[a-z0-9][a-z0-9._-]*:[a-z0-9][a-z0-9._-]*$/; + +const isoDate = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'must be an ISO date (YYYY-MM-DD)'); +const positiveInt = z.number().int().positive(); + +const TestedOnSchema = z + .object({ + chip: z.string().min(1).max(80), + memGb: positiveInt, + cli: z.string().min(1).max(24), + date: isoDate, + }) + .strict(); + +const LocalModelEntrySchema = z + .object({ + id: z.string().max(100).regex(LOCAL_MODEL_ID_PATTERN, 'must be name:tag in lowercase, without a slash'), + label: z.string().min(1).max(80), + license: z.string().min(1).max(40), + paramsB: z.number().positive(), + bytes: positiveInt, + kvBytesPerToken: positiveInt, + contexts: z.array(positiveInt).min(1), + rank: z.number().int(), + speedPer100Gbps: z.number().positive().optional(), + tested: z.array(TestedOnSchema), + }) + .strict(); + +const LocalHarnessEntrySchema = z + .object({ + cli: z.string().min(1).max(24), + minContext: positiveInt, + rank: z.number().int(), + gated: z.boolean().optional(), + slowFirstTurn: z.boolean().optional(), + }) + .strict(); + +const ChipEntrySchema = z + .object({ + match: z.string().min(1).max(40), + gbps: z.number().positive(), + }) + .strict(); + +const PromptCacheTierSchema = z + .object({ + minMemGb: z.number().int().nonnegative(), + bytes: z.number().int().nonnegative(), + }) + .strict(); + +/** Validates a whole catalogue. Typed against `LocalModelCatalog`, so the two cannot drift apart. */ +export const LocalModelCatalogSchema: z.ZodType = z + .object({ + schema: z.literal(1), + updated: isoDate, + minMacosMajor: positiveInt, + promptCacheTiers: z.array(PromptCacheTierSchema).min(1), + models: z.array(LocalModelEntrySchema), + harnesses: z.array(LocalHarnessEntrySchema), + chips: z.array(ChipEntrySchema), + }) + .strict(); + +/** + * The catalogue that ships with Codeman. EMPTY for now (no model, harness or chip rows), but + * valid: with it `recommend()` can only answer verdict `none`. + * + * The two constants that are already decided are filled in: macOS 14 is the oldest version the + * runtime supports, and the prompt cache is 512 MiB on a Mac below 32 GB and 2 GiB from 32 GB. + */ +export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { + schema: 1, + updated: '2026-11-06', + minMacosMajor: 14, + promptCacheTiers: [ + { minMemGb: 0, bytes: 512 * 1024 ** 2 }, + { minMemGb: 32, bytes: 2 * 1024 ** 3 }, + ], + models: [], + harnesses: [], + chips: [], +}; + +/** + * The catalogue in force. There is no override in this stage, so this is the stock constant; + * callers go through this function anyway, so a later stage can change where it comes from + * without touching them. + */ +export function getLocalModelCatalog(): LocalModelCatalog { + return STOCK_LOCAL_MODEL_CATALOG; +} diff --git a/src/local-model/format.ts b/src/local-model/format.ts new file mode 100644 index 00000000..cc9aa974 --- /dev/null +++ b/src/local-model/format.ts @@ -0,0 +1,42 @@ +/** + * @fileoverview The formatters of the local model recommendation, and its byte units. + * + * `formatGb()` is the ONE place a byte count becomes a size a person reads: decimal GB (the + * unit the Finder and the model registry use). `hardware.ts` and `recommend.ts` call it to fill + * the `...Gb` text fields of the payload (`memGb`, `freeDiskGb`, `needGb`, `downloadGb`, + * `budgetGb`, and the size slots of a `LocalModelMessage`). Nothing else may turn bytes into a + * size for display, the browser included: it prints the text it was sent. + * + * GiB and MiB exist only for the budget math (reserves, the wired limit, the prompt cache) and + * never reach a person. + * + * Pure: no IO, no state. + */ + +/** One decimal gigabyte, the unit a person reads. */ +export const GB = 1_000_000_000; +/** One binary gigabyte. Budget math only. */ +export const GIB = 1024 ** 3; +/** One binary megabyte. Budget math only. */ +export const MIB = 1024 ** 2; + +/** + * A byte count as decimal GB text, WITHOUT the unit: one decimal below 100 ('6.6', '17.7', + * '25.5'), a whole number from 100 ('212'). The caller's sentence supplies ' GB'. + * Anything that is not a positive finite number reads '0.0'. + */ +export function formatGb(bytes: number): string { + if (!Number.isFinite(bytes) || bytes <= 0) return '0.0'; + const tenths = Math.round((bytes / GB) * 10) / 10; + return tenths >= 100 ? String(Math.round(bytes / GB)) : tenths.toFixed(1); +} + +/** + * The memory size a Mac is sold as: whole binary gigabytes (17179869184 bytes is 16). This is + * `HardwareFacts.memClassGb`: the key of the tier rules ("below 36 GB") and of `TestedOn.memGb`. + * It names a class of Mac. A size for display is `formatGb()`. + */ +export function memClassGb(memBytes: number): number { + if (!Number.isFinite(memBytes) || memBytes <= 0) return 0; + return Math.round(memBytes / GIB); +} diff --git a/src/local-model/hardware.ts b/src/local-model/hardware.ts new file mode 100644 index 00000000..687d981a --- /dev/null +++ b/src/local-model/hardware.ts @@ -0,0 +1,106 @@ +/** + * @fileoverview The hardware probe of the local model recommendation: what this Mac is. + * + * SKELETON: the exported signatures are final, the bodies are not written yet. This module + * will hold: + * - `parseSysctl()`: a pure parser of `key: value` lines. The command runs WITHOUT `-n`, so a + * key that is missing on a machine (`sysctl.proc_translated` on Intel) cannot shift the + * others, and a line it does not know is ignored. + * - `readHardware()`: ONE bounded async spawn of `SYSCTL_PATH` with `SYSCTL_KEYS`, the free disk + * of the data dir through `fs.promises.statfs`, fallbacks from `node:os` when the spawn + * fails, and a 60 s cache. + * + * Rules: + * - Never throws and never rejects: a failed probe is an answer (`source: 'os-fallback'`), and + * off macOS nothing is spawned (`source: 'not-probed'`). + * - Never `system_profiler` (tens of milliseconds and a parse of free text), never a second + * spawn, never a synchronous exec on the server's event loop. + * - A non-zero exit is tolerated once `hw.memsize` parsed (a missing key makes sysctl exit 1). + * - `iogpu.wired_limit_mb` of 0 means "not set": it is reported as 0 and only a value above 0 + * is ever used as a limit. + * - `memGb`, `memClassGb` and `freeDiskGb` are filled here through src/local-model/format.ts, + * the one place a size is formatted. + */ + +import type { HardwareFacts } from '../types.js'; + +/** Absolute path, so the server's PATH (minimal under a service manager) cannot change what runs. */ +export const SYSCTL_PATH = '/usr/sbin/sysctl'; + +/** The keys of the one spawn, in the order they are passed. */ +export const SYSCTL_KEYS = [ + 'hw.memsize', + 'machdep.cpu.brand_string', + 'hw.optional.arm64', + 'sysctl.proc_translated', + 'iogpu.wired_limit_mb', + 'kern.osproductversion', + 'hw.model', +] as const; + +/** Upper bound of the spawn (it answers in about 2 ms on a real Mac). */ +export const SYSCTL_TIMEOUT_MS = 2000; + +/** How long one probe result is reused. */ +export const HARDWARE_CACHE_TTL_MS = 60_000; + +/** What `parseSysctl()` read. Every field is absent when its key was missing or its value did not parse. */ +export interface SysctlFacts { + /** `hw.memsize`. */ + memBytes?: number; + /** `machdep.cpu.brand_string`. */ + chip?: string; + /** `hw.optional.arm64` is 1. */ + arm64?: boolean; + /** `sysctl.proc_translated` is 1. */ + translated?: boolean; + /** `iogpu.wired_limit_mb`, as reported (0 stays 0). */ + wiredLimitMb?: number; + /** `kern.osproductversion`. */ + macosVersion?: string; + /** The leading number of `macosVersion`. */ + macosMajor?: number; + /** `hw.model`. */ + model?: string; +} + +/** The IO of `readHardware()`, injectable so a test spawns nothing and reads no real disk. Each member defaults to the real thing. */ +export interface HardwareProbeDeps { + /** Default `process.platform`. */ + platform?: string; + /** Default `os.arch()`. */ + arch?: string; + /** Runs the one spawn. Resolves its stdout, also after a non-zero exit, or `undefined` when it could not run or timed out. Never rejects. */ + runSysctl?: () => Promise; + /** Free bytes on the data dir's filesystem, or `undefined` when unknown. Never rejects. */ + freeDiskBytes?: () => Promise; + /** Default `os.totalmem()`. */ + totalMemBytes?: () => number; + /** Default `os.cpus()[0]?.model`. */ + cpuModel?: () => string | undefined; + /** Default `Date.now`. The clock of the cache. */ + now?: () => number; +} + +/** + * Parse the output of `sysctl ` (no `-n`): one `key: value` line per key it knows. + * Pure. Unknown lines, blank lines and error lines are skipped. + */ +export function parseSysctl(text: string): SysctlFacts { + void text; + throw new Error('not implemented: parseSysctl'); +} + +/** + * What this machine is. Cached for `HARDWARE_CACHE_TTL_MS`; concurrent callers share one probe. + * Never rejects. + */ +export async function readHardware(deps?: HardwareProbeDeps): Promise { + void deps; + throw new Error('not implemented: readHardware'); +} + +/** Drop the cached probe result (tests, and nothing else). */ +export function resetHardwareCache(): void { + throw new Error('not implemented: resetHardwareCache'); +} diff --git a/src/local-model/harness-facts.ts b/src/local-model/harness-facts.ts new file mode 100644 index 00000000..7a27d130 --- /dev/null +++ b/src/local-model/harness-facts.ts @@ -0,0 +1,30 @@ +/** + * @fileoverview What the server knows about each catalogue harness on this machine, for the + * local model recommendation. Server side: this is the one module of src/local-model/ that + * reads the CLI registry, so `recommend()` can stay pure and name no CLI. + * + * SKELETON: the exported signature is final, the body is not written yet. `buildHarnessFacts()` + * will, for every `LocalHarnessEntry` of the catalogue: + * - resolve its `cli` in the CLI registry AT CALL TIME (never in a module-level constant), and + * leave out a row whose CLI is unknown, disabled, or has no custom model mechanism + * (`capabilities.customModelInjection.kind` is `unsupported`); + * - report `label` from the registry entry and `installed` from the same probe the other + * routes use; + * - report `installable` only for a CLI that is not installed and whose install command for + * this platform needs no administrator rights (not one that starts with `brew` or `sudo`); + * - supply `acceptsModelId`, the same test a custom model launch applies to the model id + * before anything is written (the launch spec's model pattern). + * + * Rules: + * - No CLI id is compared with a literal here or anywhere: the id comes from the catalogue row + * and everything else from the registry entry it resolves to. + * - Read-only: nothing is installed, written or spawned beyond the existing availability probe. + */ + +import type { HarnessFact, LocalModelCatalog } from '../types.js'; + +/** One fact per catalogue harness that resolves in the CLI registry, in catalogue order. */ +export async function buildHarnessFacts(catalog: LocalModelCatalog): Promise { + void catalog; + throw new Error('not implemented: buildHarnessFacts'); +} diff --git a/src/local-model/recommend.ts b/src/local-model/recommend.ts new file mode 100644 index 00000000..2536e63c --- /dev/null +++ b/src/local-model/recommend.ts @@ -0,0 +1,68 @@ +/** + * @fileoverview The recommendation itself: which model, at which context, with which harness + * fits this Mac. Pure: no IO, no clock, no CLI registry, no data dir. + * + * SKELETON: the exported signatures are final, the bodies are not written yet. This module + * will hold: + * - `deviceBudget()`: L is the wired limit when it is above 0, else 2/3 of memory below 36 GB + * and 3/4 from 36 GB. B = min(L - 1 GiB, memory - reserve), with a reserve of 6 GiB up to + * 18 GB, 8 GiB up to 48 GB and 12 GiB from 64 GB. + * - `needBytes()`: F = weights + kvBytesPerToken * ctx + 1 GiB + the prompt cache tier of the + * catalogue for this Mac's memory. + * - `recommend()`: the candidates (model, context, harness) with `ctx >= harness.minContext`, + * the harness installed or installable, accepting the model id and not gated; the fit (F/B up + * to 0.85 good, up to 0.98 tight, above out); the speed (ok, slow or unknown: a chip that is + * in no chips row is unknown, a bandwidth is never guessed); the ranking (good before tight, + * then ok before unknown before slow, then model rank, harness rank, larger context); and the + * verdict. + * + * Rules: + * - Verdict `limited` when the best candidate's model has fewer than 12 billion parameters. + * - The word "Recommended" (`LOCAL_MODEL_LABELS.headlineTested`) only for a pick whose model + * has a `tested` entry matching this Mac. Otherwise `headlineOk` plus `noteUntested`. + * - Reasons for `none`: not-macos, intel, memory, disk (free disk minus the download is below + * 10 GiB for every fitting model), macos, no-harness. + * - Every label in the result comes from `LOCAL_MODEL_LABELS`, every size text from + * `formatGb()`. No number is spliced into a sentence here. + * - No CLI id is ever compared with a literal: a harness is a catalogue row plus a + * `HarnessFact`. + */ + +import type { + DeviceBudget, + HardwareFacts, + HarnessFact, + LocalModelCatalog, + LocalModelEntry, + Recommendation, +} from '../types.js'; + +/** The memory a model may use on this Mac: the device limit L, the budget B and where L came from. */ +export function deviceBudget(hw: HardwareFacts): DeviceBudget { + void hw; + throw new Error('not implemented: deviceBudget'); +} + +/** F: the bytes `model` needs in memory at a context of `ctx` tokens on this Mac. */ +export function needBytes(model: LocalModelEntry, ctx: number, catalog: LocalModelCatalog, hw: HardwareFacts): number { + void model; + void ctx; + void catalog; + void hw; + throw new Error('not implemented: needBytes'); +} + +/** + * The recommendation for this Mac. `harnessFacts` holds one fact per catalogue harness the + * caller could resolve; a catalogue harness without a fact is not offered. + */ +export function recommend( + hw: HardwareFacts, + catalog: LocalModelCatalog, + harnessFacts: readonly HarnessFact[] +): Recommendation { + void hw; + void catalog; + void harnessFacts; + throw new Error('not implemented: recommend'); +} diff --git a/src/types/index.ts b/src/types/index.ts index 4b74da44..592b28e2 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -29,6 +29,7 @@ * | plan | PlanItem, PlanTaskStatus, TddPhase | In-memory → `GET /api/sessions/:id/plan/tasks` | * | orchestrator | OrchestratorState, OrchestratorPlan, OrchestratorConfig, OrchestratorPersistState | `~/.codeman/state.json` → `GET /api/orchestrator/status` | * | mcp-sync | McpSyncResult, McpSyncTargetResult | Other CLIs' own config files → `GET`/`POST /api/mcp-sync` | + * | local-model | HardwareFacts, LocalModelCatalog, Candidate, Recommendation, LocalModelStatus, LOCAL_MODEL_LABELS | In-memory only (nothing persisted) → `GET /api/local-model` | * * ## Cross-domain relationship map * @@ -77,3 +78,4 @@ export * from './webview.js'; export * from './intent.js'; export * from './foreign-tmux.js'; export * from './mcp-sync.js'; +export * from './local-model.js'; diff --git a/src/types/local-model.ts b/src/types/local-model.ts new file mode 100644 index 00000000..5298c3bb --- /dev/null +++ b/src/types/local-model.ts @@ -0,0 +1,357 @@ +/** + * @fileoverview Types of the hardware-aware local model recommendation: what the server sees of + * this Mac, the catalogue of models and harnesses, and the answer of `GET /api/local-model`. + * + * This stage is read-only: Codeman looks at the Mac and recommends. It downloads nothing, + * installs nothing and writes no file. + * + * Who produces what: + * - `HardwareFacts`: src/local-model/hardware.ts (one bounded `sysctl` spawn, never throws). + * - `LocalModelCatalog` and its rows: src/config/local-model-catalog.ts (a compiled constant). + * - `HarnessFact`: src/local-model/harness-facts.ts (server only, read from the CLI registry at + * call time; it holds a function, so it never goes over HTTP). + * - `Recommendation`, `Candidate`, `DeviceBudget`: src/local-model/recommend.ts (pure). + * - `LocalModelStatus`: src/web/routes/local-model-routes.ts (the `data` of the GET). + * + * Three rules every producer and consumer follows: + * + * 1. ONE unit for a size a person reads: decimal GB, computed from bytes by `formatGb()` in + * src/local-model/format.ts. Every such size travels twice, as bytes (`needBytes`) and as the + * text to show (`needGb`, a number without the unit, such as '25.5'). The browser prints the + * text and never divides bytes itself. GiB appears only inside the budget math. + * 2. Every sentence or label the SERVER chooses comes from `LOCAL_MODEL_LABELS` below, so the + * browser can translate it (`codemanT(text, vars)`) and its i18n test can list every string. + * Numbers are never spliced into a sentence here: they travel in `vars` and fill the + * `{placeholder}` slots. + * 3. A CLI id is DATA (`LocalHarnessEntry.cli`, `Candidate.cli`). No code compares one with a + * literal: what differs between harnesses is a catalogue field or a `HarnessFact`. + */ + +// ---------- Hardware ---------- + +/** + * How `HardwareFacts` was obtained. + * - `sysctl`: the spawn answered and `hw.memsize` parsed. + * - `os-fallback`: on macOS, but the spawn failed or timed out; memory, architecture and the chip + * string come from `node:os`, the rest is absent. + * - `not-probed`: not macOS, so `sysctl` was never spawned. + */ +export type HardwareProbeSource = 'sysctl' | 'os-fallback' | 'not-probed'; + +/** + * What the server knows about the machine it runs on. Plain JSON (it is part of the GET payload), + * and always an answer: a failed probe yields the `os-fallback` or `not-probed` shape, never an + * exception. + */ +export interface HardwareFacts { + /** `process.platform` of the server. Anything but `darwin` ends in verdict `none`, reason `not-macos`. */ + platform: string; + /** `os.arch()` of the server process. `x64` on Apple silicon means Node runs under Rosetta. */ + arch: string; + /** `hw.optional.arm64` is 1: the MACHINE is Apple silicon whatever `arch` says. Absent when the key was not read. */ + arm64?: boolean; + /** `sysctl.proc_translated` is 1: this process runs under Rosetta. Absent when the key was not read (Intel Macs lack it). */ + translated?: boolean; + /** Installed memory in bytes (`hw.memsize`, else `os.totalmem()`). The ONLY memory input of the budget math. */ + memBytes: number; + /** `memBytes` as a person reads it: decimal GB from `formatGb()`, without the unit ('17.2' for a Mac sold as 16 GB). */ + memGb: string; + /** The memory size the Mac is sold as: whole binary gigabytes (16, 32, 48). Tier rules and `TestedOn.memGb` use it. */ + memClassGb: number; + /** `machdep.cpu.brand_string` ('Apple M4 Pro'), else `os.cpus()[0].model`. */ + chip?: string; + /** `hw.model` ('Mac16,10'). */ + model?: string; + /** `kern.osproductversion` ('27.2'). */ + macosVersion?: string; + /** The leading number of `macosVersion` (27). */ + macosMajor?: number; + /** `iogpu.wired_limit_mb` exactly as reported. 0 means "not set" (both test Macs report 0); only a value above 0 is a limit. */ + wiredLimitMb?: number; + /** Free bytes on the filesystem of Codeman's data dir (`statfs`: `bavail * bsize`). */ + freeDiskBytes?: number; + /** `freeDiskBytes` as a person reads it (decimal GB, no unit). Present exactly when `freeDiskBytes` is. */ + freeDiskGb?: string; + source: HardwareProbeSource; +} + +// ---------- Catalogue ---------- + +/** One Mac a model and harness pair was really run on by us. A model row with a matching entry may be called "Recommended". */ +export interface TestedOn { + /** The chip, as `machdep.cpu.brand_string` reports it ('Apple M4'). */ + chip: string; + /** The Mac's memory as it is sold (`HardwareFacts.memClassGb`: 16, 32, 48). */ + memGb: number; + /** The harness it was tested with: a CLI registry id, as data. */ + cli: string; + /** ISO date of the test ('2026-11-06'). */ + date: string; +} + +/** One model of the catalogue. */ +export interface LocalModelEntry { + /** `name:tag` exactly as the model registry lists it. Lowercase, one colon, no slash (`LOCAL_MODEL_ID_PATTERN`). */ + id: string; + /** The name a person reads ('Qwen3.8 27B'). */ + label: string; + /** SPDX id of the license ('Apache-2.0'). */ + license: string; + /** Parameter count in billions. Under 12 the verdict is `limited`, never `ok`. */ + paramsB: number; + /** Size of the weights in bytes: both the download and what they take in memory. */ + bytes: number; + /** Bytes of KV cache one token of context costs. */ + kvBytesPerToken: number; + /** Context lengths (tokens) the model may be run at, largest first. */ + contexts: number[]; + /** Preference among models that fit: the HIGHER rank wins. A judgement, not a measurement. */ + rank: number; + /** Measured tokens per second per 100 GB/s of memory bandwidth. Absent: the speed of this model is unknown. */ + speedPer100Gbps?: number; + /** Macs this model was really run on. Empty today for every row. */ + tested: TestedOn[]; +} + +/** One harness (coding agent CLI) that can be connected to a local model. All of it is data. */ +export interface LocalHarnessEntry { + /** A CLI registry id. Data only: never compared with a literal anywhere. */ + cli: string; + /** The smallest context (tokens) at which this harness works. */ + minContext: number; + /** Preference among harnesses: the HIGHER rank wins. */ + rank: number; + /** True: listed in the catalogue but never offered yet (its use with a local model is unverified). */ + gated?: boolean; + /** True: the first answer takes minutes, because the harness sends a very large fixed prompt. */ + slowFirstTurn?: boolean; +} + +/** + * Memory bandwidth of one chip family. `LocalModelCatalog.chips` is ordered most specific first + * ('M4 Max' before 'M4') and the first row whose `match` occurs in `HardwareFacts.chip` wins. A + * chip that matches no row has an UNKNOWN speed: a bandwidth is never guessed. + */ +export interface ChipEntry { + /** Text that must occur in the chip string ('M4 Max'). */ + match: string; + /** Memory bandwidth in GB/s. */ + gbps: number; +} + +/** + * Host memory the runtime's prompt cache takes, by the size of the Mac. Tiers are ordered by + * `minMemGb` ascending and the last tier with `minMemGb <= HardwareFacts.memClassGb` applies. + */ +export interface PromptCacheTier { + /** Smallest memory size (as sold, binary gigabytes) this tier applies to. The first tier has 0. */ + minMemGb: number; + bytes: number; +} + +/** The catalogue: a compiled constant, validated by `LocalModelCatalogSchema`. No override file, no download data. */ +export interface LocalModelCatalog { + schema: 1; + /** ISO date the rows were last checked ('2026-11-06'). Reported as `LocalModelStatus.catalogUpdated`. */ + updated: string; + /** Oldest macOS major version a local model runs on. Below it: verdict `none`, reason `macos`. */ + minMacosMajor: number; + promptCacheTiers: PromptCacheTier[]; + models: LocalModelEntry[]; + harnesses: LocalHarnessEntry[]; + chips: ChipEntry[]; +} + +// ---------- Harness facts (server only) ---------- + +/** + * What the server knows about one catalogue harness on this machine. Built from the CLI registry + * at call time, so `recommend()` names no CLI. NOT serializable (it holds a function): it never + * leaves the server. `Candidate` carries the parts a browser needs. + */ +export interface HarnessFact { + /** The CLI registry id. */ + cli: string; + /** The registry's display label ('OpenCode'). */ + label: string; + /** The CLI's binary was found on this machine. */ + installed: boolean; + /** Not installed, but the registry has an install command for this platform that needs no administrator rights. */ + installable: boolean; + /** Whether this CLI can be launched with that model id (its command-line pattern accepts it). */ + acceptsModelId: (modelId: string) => boolean; +} + +// ---------- Recommendation ---------- + +/** Need over budget: up to 0.85 is `good`, up to 0.98 is `tight`, above is `out`. */ +export type FitGrade = 'good' | 'tight' | 'out'; + +/** + * Expected generation speed. `unknown` when the chip is in no `ChipEntry` or the model has no + * `speedPer100Gbps`. In the ranking `unknown` sits between `ok` and `slow`. + */ +export type SpeedGrade = 'ok' | 'slow' | 'unknown'; + +/** + * Why the verdict is `none`. + * - `not-macos`: the server does not run on macOS (the card is hidden). + * - `intel`: a Mac without Apple silicon. + * - `memory`: no model fits the memory budget with any harness. + * - `disk`: models fit the memory, but for every one of them free disk minus the download is below 10 GiB. + * - `macos`: macOS is older than `LocalModelCatalog.minMacosMajor`. + * - `no-harness`: no catalogue harness is installed or installable and accepts a model. + */ +export type NoneReason = 'not-macos' | 'intel' | 'memory' | 'disk' | 'macos' | 'no-harness'; + +/** Where the device budget came from: a formula (`estimated`) or `iogpu.wired_limit_mb` above 0 (`wired-limit`). */ +export type BudgetSource = 'estimated' | 'wired-limit'; + +/** The memory a model may use on this Mac. */ +export interface DeviceBudget { + /** L: the wired limit when above 0, else 2/3 of memory below 36 GB and 3/4 from 36 GB. */ + limitBytes: number; + /** B = min(L - 1 GiB, memory - reserve). What `Candidate.budgetShare` is measured against. */ + budgetBytes: number; + source: BudgetSource; +} + +/** + * Every sentence and label the server may put into the payload, in English (the i18n source + * language). Frozen for the parallel build: the zh-CN dictionary in src/web/public/i18n.js has + * one entry per value, keyed by this exact text, and the browser's i18n test imports this + * constant to list them. Adding a key is fine; changing a text means changing its zh-CN key too. + * + * `{placeholder}` slots are filled by the browser from `LocalModelMessage.vars`. `{name}` is + * taken (the translator fills it with the display name), so no slot here is called `name`. + * + * | Key | Slots (all text; sizes are decimal GB without the unit) | + * |---------------|---------------------------------------------------------------------------------| + * | `noneMemory` | `memory`: `HardwareFacts.memGb` | + * | `noneDisk` | `download`: smallest download among the models that fit the memory; `free`: | + * | | `HardwareFacts.freeDiskGb`; `missing`: download + 10 GiB - free | + * | `noneMacos` | `version`: `HardwareFacts.macosVersion`; `minimum`: the catalogue's minimum | + * + * The word "Recommended" exists only in `headlineTested`, which may be used only when the + * pick's `tested` is true. No catalogue row has a tested entry today, so it is never sent. + */ +export const LOCAL_MODEL_LABELS = { + /** Verdict `ok`, pick not tested by us on a Mac like this one. */ + headlineOk: 'Best fit for this Mac', + /** Verdict `ok`, pick tested by us on a Mac like this one. */ + headlineTested: 'Recommended for this Mac', + /** Verdict `limited`. */ + headlineLimited: + 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', + /** A note under an `ok` or `limited` pick whose `tested` is false. */ + noteUntested: 'Not yet tested by us on a Mac like this.', + fitGood: 'Good', + fitTight: 'Tight', + speedSlow: 'Slow', + speedUnknown: 'Speed unknown', + noneNotMacos: 'Codeman recommends a local model only on a Mac with Apple silicon.', + noneIntel: 'This Mac has an Intel processor. Local coding models need Apple silicon.', + noneMemory: + 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', + noneDisk: + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.', + noneMacos: 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.', + noneNoHarness: 'No coding agent that Codeman can connect to a local model is available on this Mac.', +} as const; + +/** One of the texts in `LOCAL_MODEL_LABELS`. The payload's label fields are typed with it, so an off-list string does not compile. */ +export type LocalModelLabel = (typeof LOCAL_MODEL_LABELS)[keyof typeof LOCAL_MODEL_LABELS]; + +/** The headline of each `none` reason. */ +export const LOCAL_MODEL_NONE_LABELS: Record = { + 'not-macos': LOCAL_MODEL_LABELS.noneNotMacos, + intel: LOCAL_MODEL_LABELS.noneIntel, + memory: LOCAL_MODEL_LABELS.noneMemory, + disk: LOCAL_MODEL_LABELS.noneDisk, + macos: LOCAL_MODEL_LABELS.noneMacos, + 'no-harness': LOCAL_MODEL_LABELS.noneNoHarness, +}; + +/** A sentence for the card: a text from `LOCAL_MODEL_LABELS` plus the values of its `{placeholder}` slots. */ +export interface LocalModelMessage { + text: LocalModelLabel; + /** Slot values, already formatted as text. Absent when the text has no slot. */ + vars?: Record; +} + +/** One model at one context with one harness, measured against this Mac. Plain JSON. */ +export interface Candidate { + /** `LocalModelEntry.id`. */ + modelId: string; + /** `LocalModelEntry.label`. */ + modelLabel: string; + license: string; + paramsB: number; + /** Context length in tokens. */ + context: number; + /** The harness: a CLI registry id, as data. */ + cli: string; + /** The harness's display label, from the CLI registry. */ + cliLabel: string; + /** False: the harness is not on this machine yet (it is installable, or it would not be a candidate). */ + cliInstalled: boolean; + /** The harness row's `slowFirstTurn`. */ + slowFirstTurn?: boolean; + /** F = weights + kvBytesPerToken * context + 1 GiB + prompt cache. */ + needBytes: number; + /** `needBytes` as decimal GB text, no unit. */ + needGb: string; + /** The model download in bytes (`LocalModelEntry.bytes`). */ + downloadBytes: number; + /** `downloadBytes` as decimal GB text, no unit. */ + downloadGb: string; + /** `needBytes / Recommendation.budgetBytes` (0.83). */ + budgetShare: number; + /** A candidate in a `Recommendation` is never `out`. */ + fit: Exclude; + /** `fitGood` or `fitTight`. */ + fitLabel: LocalModelLabel; + speed: SpeedGrade; + /** `speedSlow` or `speedUnknown`. Absent when `speed` is `ok`. */ + speedLabel?: LocalModelLabel; + /** Estimated generation speed. Present exactly when `speed` is not `unknown`. */ + tokensPerSecond?: number; + /** A `TestedOn` entry of the model matches this Mac (same chip, same `memClassGb`) and this harness. */ + tested: boolean; +} + +/** + * The answer of `recommend()`. + * - `ok`: `pick` is the best candidate and its model has at least 12 billion parameters. + * - `limited`: `pick` is the best candidate, but its model has fewer than 12 billion parameters. + * The card says so and never says "Recommended". + * - `none`: no pick; `reason` says why. + */ +export interface Recommendation { + verdict: 'ok' | 'limited' | 'none'; + /** Present exactly when `verdict` is `none`. */ + reason?: NoneReason; + /** What the card says first: `headlineOk`, `headlineTested`, `headlineLimited` or the `none` reason's sentence. */ + headline: LocalModelMessage; + /** Further sentences under the headline (`noteUntested` today). May be empty. */ + notes: LocalModelMessage[]; + /** Present exactly when `verdict` is `ok` or `limited`. */ + pick?: Candidate; + /** The next models that fit (at most four, each at its best context and harness), best first. Empty when `verdict` is `none`. */ + alternatives: Candidate[]; + /** `DeviceBudget.budgetBytes`. 0 when the budget could not be computed (not macOS). */ + budgetBytes: number; + /** `budgetBytes` as decimal GB text, no unit. */ + budgetGb: string; + budgetSource: BudgetSource; +} + +// ---------- API ---------- + +/** The `data` of `GET /api/local-model`. Read-only; admin only in multi-user mode. */ +export interface LocalModelStatus { + hardware: HardwareFacts; + recommendation: Recommendation; + /** `LocalModelCatalog.updated`. */ + catalogUpdated: string; +} diff --git a/src/web/public/index.html b/src/web/public/index.html index c55ad232..d7b82775 100644 --- a/src/web/public/index.html +++ b/src/web/public/index.html @@ -4685,6 +4685,7 @@ + diff --git a/src/web/public/local-model-ui.js b/src/web/public/local-model-ui.js new file mode 100644 index 00000000..755527bf --- /dev/null +++ b/src/web/public/local-model-ui.js @@ -0,0 +1,33 @@ +/** + * @fileoverview The "model on this Mac" card of the welcome screen and its row in App Settings. + * + * SKELETON: this file holds no code yet. It will read `GET /api/local-model` (hardware of the + * machine the server runs on, the recommendation, the catalogue date) and show, read-only: + * - the hardware line (chip, memory, free disk); + * - the verdict: the headline and notes the server sent, then the pick with its numbers + * (model, harness, context, memory need against the budget, download size, speed); + * - the other models that fit, each with its fit and speed label; + * - one line saying that setting the model up from Codeman is the next step and is not built + * yet. + * + * Rules: + * - Read-only: no button here downloads, installs or writes anything. + * - Hidden when the server is not on macOS (`recommendation.reason` is `not-macos`), when the + * request is refused (a non-admin in multi-user mode gets a 403) and after the user dismissed + * the card on this device (a localStorage key, never a server setting). + * - Every dynamic string goes in through `textContent`. Model, harness and chip names are + * someone else's text and carry `data-i18n-skip`. + * - Names and labels come from the payload: this file holds no list of models or harnesses and + * never compares a CLI id with a literal. + * - Sizes are printed exactly as sent (`needGb`, `downloadGb`, `budgetGb`, `memGb`, + * `freeDiskGb`: decimal GB as text). Bytes are never divided here. + * - A sentence the server chose arrives as `{ text, vars }` and goes through + * `codemanT(text, vars)`; this file's own strings are literals in its translator call, so the + * i18n test can harvest them. Both kinds need a zh-CN entry in i18n.js. + * + * @mixin Extends CodemanApp.prototype via Object.assign + * @dependency app.js (CodemanApp) + * @dependency api-client.js (_api) + * @dependency i18n.js (codemanT, optional) + * @loadorder 12.59 of 16, after edit-history-ui.js, before entrance-animations.js + */ diff --git a/src/web/routes/index.ts b/src/web/routes/index.ts index 2dde7fa3..31db304e 100644 --- a/src/web/routes/index.ts +++ b/src/web/routes/index.ts @@ -49,5 +49,6 @@ export { type CustomModelSwapDisplacement, } from './custom-model-routes.js'; export { registerCliRegistryRoutes, readCliManagementEnabled, type CliListItem } from './cli-registry-routes.js'; +export { registerLocalModelRoutes } from './local-model-routes.js'; export { registerAgentSessionRoutes } from './agent-session-routes.js'; diff --git a/src/web/routes/local-model-routes.ts b/src/web/routes/local-model-routes.ts new file mode 100644 index 00000000..4c4f7563 --- /dev/null +++ b/src/web/routes/local-model-routes.ts @@ -0,0 +1,60 @@ +/** + * @fileoverview `GET /api/local-model`: what this Mac is and which local model fits it. + * + * SKELETON: the route is registered and answers 501 until its handler is written. It will + * answer `{ success: true, data: LocalModelStatus }`, built from the hardware probe, the + * catalogue, the harness facts and the pure `recommend()`: + * `{ hardware, recommendation, catalogUpdated }`. + * + * Rules: + * - Read-only. It downloads nothing, installs nothing, writes no file and reads no setting. + * - Never a 500 for a probe that failed: a failed probe is an answer (the hardware's `source` + * says so and the recommendation is verdict `none`). + * - The payload names hardware of the host, so in multi-user mode it is admin only (403), the + * same bar as `GET /api/doctor`. + * - Every dependency is injected (`LocalModelRouteDeps`), so a route test spawns nothing and + * does not need a Mac: it passes its own hardware and harness facts. + */ + +import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify'; +import { + ApiErrorCode, + createErrorResponse, + type ApiResponse, + type HardwareFacts, + type HarnessFact, + type LocalModelCatalog, + type LocalModelStatus, + type Recommendation, +} from '../../types.js'; +import { getLocalModelCatalog } from '../../config/local-model-catalog.js'; +import { readHardware } from '../../local-model/hardware.js'; +import { buildHarnessFacts } from '../../local-model/harness-facts.js'; +import { recommend } from '../../local-model/recommend.js'; + +/** What the route calls. Each member defaults to the real module; a test overrides what it needs. */ +export interface LocalModelRouteDeps { + readHardware: () => Promise; + getCatalog: () => LocalModelCatalog; + buildHarnessFacts: (catalog: LocalModelCatalog) => Promise; + recommend: (hw: HardwareFacts, catalog: LocalModelCatalog, harnessFacts: readonly HarnessFact[]) => Recommendation; +} + +export const defaultLocalModelRouteDeps: LocalModelRouteDeps = { + readHardware: () => readHardware(), + getCatalog: getLocalModelCatalog, + buildHarnessFacts, + recommend, +}; + +export function registerLocalModelRoutes(app: FastifyInstance, deps: Partial = {}): void { + const resolved: LocalModelRouteDeps = { ...defaultLocalModelRouteDeps, ...deps }; + void resolved; + app.get( + '/api/local-model', + async (_req: FastifyRequest, reply: FastifyReply): Promise> => { + reply.code(501); + return createErrorResponse(ApiErrorCode.OPERATION_FAILED, 'GET /api/local-model is not implemented yet'); + } + ); +} diff --git a/src/web/server.ts b/src/web/server.ts index f9202601..67ee93a5 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -264,6 +264,7 @@ import { pruneIdleLlamaSwapLogTails, tryWebviewRefererFallback, registerCliRegistryRoutes, + registerLocalModelRoutes, } from './routes/index.js'; import { isLostWebviewFrameNavigation } from './webview-proxy.js'; import { CronService } from '../cron/cron-service.js'; @@ -1417,6 +1418,7 @@ export class WebServer extends EventEmitter { registerDoctorRoutes(this.app); registerCustomModelRoutes(this.app); registerCliRegistryRoutes(this.app); + registerLocalModelRoutes(this.app); // Cron: build the service from the same context, recompute // due times for any persisted jobs, then expose it to its routes. From 92c20fe572ff8bbc32cd2efa39817fa78518b09c Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 16:27:33 +0100 Subject: [PATCH 2/8] feat(local-model): hardware probe, model catalogue and the pure recommendation The core of the read-only recommendation: what this Mac is, what the catalogue knows, and which model fits. Nothing is downloaded, installed or written. - hardware.ts: one bounded async spawn of /usr/sbin/sysctl with seven keys, without -n, parsed as "key: value" lines by the pure parseSysctl(). Free disk of the data dir through statfs behind the bounded path probe. Fallbacks from node:os, a 60 s cache, never throws, never system_profiler. A wired limit of 0 means "not set". - local-model-catalog.ts: seven models, three harness rows (pi gated) and the chips table as a compiled constant, validated by a Zod schema on first use. No override file. Model ids are name:tag in lowercase without a slash; every model carries paramsB and an (empty) tested list. Chip names match on the end of the chip string, so 'M4' is not the chip of 'Apple M4 Pro'. - recommend.ts: device budget (the wired limit when above 0, else 2/3 of memory below 36 GB and 3/4 from 36 GB, less the reserve), need = weights + KV cache + 1 GiB + prompt cache, fit good up to 0.85 and tight up to 0.98, speed ok, unknown or slow (a chip in no chips row is unknown), and the ranking. Verdicts ok, limited (best model under 12 billion parameters) and none with its reason. "Recommended" only for a pick with a tested entry for this Mac, which no row has today. - types/local-model.ts: comments brought in line with the code (chip matching, budgetShare rounding, a budget of 0 on an Intel Mac). - cli-registry-no-id-branching.test.ts: the guard also catches a comparison of a `cli` field with a CLI id (plus one stray blank line Prettier refused). Tests: test/local-model-format.test.ts, test/local-model-hardware.test.ts (the captured output of two real Macs), test/local-model-catalog.test.ts, test/local-model-recommend.test.ts. Co-Authored-By: Claude Fable 5.1 --- src/config/local-model-catalog.ts | 305 ++++- src/local-model/hardware.ts | 325 +++++- src/local-model/recommend.ts | 332 +++++- src/types/local-model.ts | 13 +- test/cli-registry-no-id-branching.test.ts | 11 +- test/local-model-catalog.test.ts | 383 ++++++ test/local-model-format.test.ts | 86 ++ test/local-model-hardware.test.ts | 749 ++++++++++++ test/local-model-recommend.test.ts | 1298 +++++++++++++++++++++ 9 files changed, 3423 insertions(+), 79 deletions(-) create mode 100644 test/local-model-catalog.test.ts create mode 100644 test/local-model-format.test.ts create mode 100644 test/local-model-hardware.test.ts create mode 100644 test/local-model-recommend.test.ts diff --git a/src/config/local-model-catalog.ts b/src/config/local-model-catalog.ts index 661ebd8b..a66a111e 100644 --- a/src/config/local-model-catalog.ts +++ b/src/config/local-model-catalog.ts @@ -2,16 +2,7 @@ * @fileoverview The local model catalogue: which models, harnesses and chips the recommendation * knows, as a compiled constant validated by a Zod schema. * - * SKELETON: the schema below is the shape of the contract (it mirrors `LocalModelCatalog` in - * src/types/local-model.ts) and the stock constant is EMPTY. Still to come here: - * - the model rows (ids and byte sizes from the registry manifests, `paramsB`, an empty - * `tested` array each; `flint:40b` stays out), the harness rows and the chips table; - * - the cross-row rules the schema cannot say field by field: unique model ids, unique harness - * `cli`s, `contexts` largest first, `chips` most specific first, `promptCacheTiers` ascending - * and starting at 0; - * - `getLocalModelCatalog()` parsing the constant once, on first use, never at import. - * - * Rules that hold from the start: + * Rules: * - There is NO override file and no runtime download data in this stage: nothing here reads * the data dir, and a data file can never decide what gets recommended. * - A model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`), so it can @@ -20,14 +11,71 @@ * registry id as data: no code may compare it with a literal. Whether that CLI exists and can * take a custom model is checked against the registry at call time, in * src/local-model/harness-facts.ts, never here. + * - The schema checks more than fields. Across rows it requires: unique model ids and ranks, + * unique harness `cli`s and ranks, `contexts` largest first, a `tested` entry naming a harness + * row, `chips` most specific first, and `promptCacheTiers` ascending from 0. + * - `getLocalModelCatalog()` parses the constant once, on first use and never at import, and + * throws on a bad row. test/local-model-catalog.test.ts pins that the stock constant parses, + * so a bad row fails the gate and never reaches the route. + * + * Where each figure of the stock rows comes from, and what was checked (2026-11-06): + * + * | Figure | Source | Checked | + * |----------------------|-----------------------------------------------------------|--------------------------------------| + * | `id`, `bytes` | registry manifest of the tag, sum of its layer sizes | yes, to the byte, by the facts review | + * | `license` | the model card (all Apache-2.0) | read by the research, not re-checked | + * | `paramsB` | 27.8 from the model card; the others are the count in the | no. Only the 12 billion line reads | + * | | model's name | it, and no row is near that line | + * | `kvBytesPerToken` | COMPUTED from each model's config: 2 x full-attention | no. Not measured in a running | + * | | layers x KV heads x head size x 2 bytes | runtime. The review called the | + * | | | gpt-oss figure plausible, no more | + * | `contexts`, `rank` | a judgement of the design | no | + * | `speedPer100Gbps` | one published run on an M1 Max (400 GB/s): generation | the run was read. It stands for less | + * | | only, short prompt. The two q8_0 rows are SCALED by file | than a session: no long prompt, no | + * | | size, not measured | thinking tokens | + * | `minMacosMajor` | the runtime's documentation (macOS 14) | yes, by the facts review | + * | `promptCacheTiers` | a decision of the design, not a measurement | no | + * | harness `minContext` | 65536: the runtime's documentation for both harnesses. | 65536 yes. 32768 no, which is why | + * | | 32768: recalled | that row is gated | + * | `chips` | the bandwidth column of | yes, read at the source for this | + * | | https://github.com/ggml-org/llama.cpp/discussions/4167 | file | + * + * `tested` is empty on every row: nobody has run any of these models with a harness on a Mac + * yet. Until a row has an entry, the recommendation never says "Recommended". */ import { z } from 'zod'; -import type { LocalModelCatalog } from '../types.js'; +import type { ChipEntry, LocalModelCatalog } from '../types.js'; /** A model id: lowercase name, one colon, a tag. No slash, so no registry host or namespace can ride along. */ export const LOCAL_MODEL_ID_PATTERN = /^[a-z0-9][a-z0-9._-]*:[a-z0-9][a-z0-9._-]*$/; +/** + * Whether `chip` (a `machdep.cpu.brand_string`, 'Apple M4 Pro') is the chip a `ChipEntry.match` + * ('M4 Pro') names: the string must END with the match, and the match must start at the + * beginning or after a space. Case is ignored. + * + * Stricter than "the match occurs in the string" on purpose. 'M4' is not the chip of + * 'Apple M4 Pro', 'M1' is not the chip of 'Apple M10', and a variant this table has never heard + * of ('Apple M6 Pro') matches no row instead of borrowing the bandwidth of its base chip. + */ +export function chipNameMatches(chip: string, match: string): boolean { + const name = chip.trim().toLowerCase(); + const wanted = match.trim().toLowerCase(); + if (wanted === '' || !name.endsWith(wanted)) return false; + const start = name.length - wanted.length; + return start === 0 || name[start - 1] === ' '; +} + +/** + * The memory bandwidth (GB/s) of `chip`, from the first row that names it. `undefined` when the + * chip string is absent or no row names it: the speed is then UNKNOWN, a bandwidth is never guessed. + */ +export function chipBandwidthGbps(chips: readonly ChipEntry[], chip: string | undefined): number | undefined { + if (!chip) return undefined; + return chips.find((row) => chipNameMatches(chip, row.match))?.gbps; +} + const isoDate = z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'must be an ISO date (YYYY-MM-DD)'); const positiveInt = z.number().int().positive(); @@ -79,7 +127,20 @@ const PromptCacheTierSchema = z }) .strict(); -/** Validates a whole catalogue. Typed against `LocalModelCatalog`, so the two cannot drift apart. */ +/** The index of the first value that already occurred earlier in the list, or -1. */ +function firstRepeat(values: readonly T[]): number { + const seen = new Set(); + for (let i = 0; i < values.length; i++) { + if (seen.has(values[i])) return i; + seen.add(values[i]); + } + return -1; +} + +/** + * Validates a whole catalogue: every field, then the rules that hold across rows (see the file + * header). Typed against `LocalModelCatalog`, so the two cannot drift apart. + */ export const LocalModelCatalogSchema: z.ZodType = z .object({ schema: z.literal(1), @@ -90,33 +151,223 @@ export const LocalModelCatalogSchema: z.ZodType = z harnesses: z.array(LocalHarnessEntrySchema), chips: z.array(ChipEntrySchema), }) - .strict(); + .strict() + .superRefine((catalog, ctx) => { + const issue = (path: (string | number)[], message: string): void => { + ctx.addIssue({ code: 'custom', path, message }); + }; + + // A repeated id would make "the model" ambiguous, a repeated rank would make the order of + // two models depend on where they sit in the array. + const repeatedId = firstRepeat(catalog.models.map((m) => m.id)); + if (repeatedId >= 0) issue(['models', repeatedId, 'id'], 'model ids must be unique'); + const repeatedRank = firstRepeat(catalog.models.map((m) => m.rank)); + if (repeatedRank >= 0) issue(['models', repeatedRank, 'rank'], 'model ranks must be unique'); + + const harnessClis = catalog.harnesses.map((h) => h.cli); + const repeatedCli = firstRepeat(harnessClis); + if (repeatedCli >= 0) issue(['harnesses', repeatedCli, 'cli'], 'a harness may be listed only once'); + const repeatedHarnessRank = firstRepeat(catalog.harnesses.map((h) => h.rank)); + if (repeatedHarnessRank >= 0) issue(['harnesses', repeatedHarnessRank, 'rank'], 'harness ranks must be unique'); + + catalog.models.forEach((model, m) => { + for (let i = 1; i < model.contexts.length; i++) { + if (model.contexts[i] >= model.contexts[i - 1]) { + issue(['models', m, 'contexts', i], 'contexts must be listed largest first, without repeats'); + break; + } + } + // An entry for a harness the catalogue does not list could never match a candidate: a typo. + model.tested.forEach((entry, t) => { + if (!harnessClis.includes(entry.cli)) { + issue(['models', m, 'tested', t, 'cli'], 'a tested entry must name a harness of this catalogue'); + } + }); + }); + + // Most specific first ('M4 Max' before 'M4'): a row whose text sits inside a LATER row's text + // is the less specific of the two, so it is in the wrong place. A repeated match is the same + // mistake. + for (let later = 1; later < catalog.chips.length; later++) { + const laterMatch = catalog.chips[later].match.toLowerCase(); + const shadowing = catalog.chips.slice(0, later).find((row) => laterMatch.includes(row.match.toLowerCase())); + if (shadowing) { + issue( + ['chips', later, 'match'], + `chips must be listed most specific first: '${shadowing.match}' comes before '${catalog.chips[later].match}'` + ); + break; + } + } + + // The tiers are read as "the last one whose minimum is not above this Mac's memory", so they + // must start at 0 (every Mac has a tier) and ascend. + if (catalog.promptCacheTiers[0].minMemGb !== 0) { + issue(['promptCacheTiers', 0, 'minMemGb'], 'the first prompt cache tier must start at 0'); + } + for (let i = 1; i < catalog.promptCacheTiers.length; i++) { + if (catalog.promptCacheTiers[i].minMemGb <= catalog.promptCacheTiers[i - 1].minMemGb) { + issue(['promptCacheTiers', i, 'minMemGb'], 'prompt cache tiers must ascend by minMemGb'); + break; + } + } + }); + +const KIB = 1024; +const MIB = 1024 ** 2; +const GIB = 1024 ** 3; /** - * The catalogue that ships with Codeman. EMPTY for now (no model, harness or chip rows), but - * valid: with it `recommend()` can only answer verdict `none`. - * - * The two constants that are already decided are filled in: macOS 14 is the oldest version the - * runtime supports, and the prompt cache is 512 MiB on a Mac below 32 GB and 2 GiB from 32 GB. + * The catalogue that ships with Codeman. Never read directly outside this file and its test: + * callers use `getLocalModelCatalog()`, which validates it first. */ export const STOCK_LOCAL_MODEL_CATALOG: LocalModelCatalog = { schema: 1, updated: '2026-11-06', + // The oldest macOS the model runtime supports. minMacosMajor: 14, + // Host memory the runtime's prompt cache may take beside the model: 512 MiB on a Mac below + // 32 GB, 2 GiB from 32 GB. Part of the need of every candidate. promptCacheTiers: [ - { minMemGb: 0, bytes: 512 * 1024 ** 2 }, - { minMemGb: 32, bytes: 2 * 1024 ** 3 }, + { minMemGb: 0, bytes: 512 * MIB }, + { minMemGb: 32, bytes: 2 * GIB }, + ], + // Ordered by rank. The higher rank wins among models that fit equally well and are equally + // fast. `flint:40b` is left out on purpose: its plain tag is the slow build on a Mac and it + // has an open bug with tool arguments. + models: [ + { + id: 'qwen3.5:9b', + label: 'Qwen3.5 9B', + license: 'Apache-2.0', + paramsB: 9, + bytes: 6_550_825_373, + kvBytesPerToken: 32 * KIB, + contexts: [65536, 32768], + rank: 1, + // 37.1 tokens per second measured on 400 GB/s. + speedPer100Gbps: 9.3, + tested: [], + }, + { + id: 'qwen3.5:9b-q8_0', + label: 'Qwen3.5 9B (Q8)', + license: 'Apache-2.0', + paramsB: 9, + bytes: 10_449_217_949, + kvBytesPerToken: 32 * KIB, + contexts: [65536, 32768], + rank: 2, + // Not measured: the 9.3 of the row above scaled by file size. + speedPer100Gbps: 5.8, + tested: [], + }, + { + id: 'gpt-oss:20b', + label: 'GPT-OSS 20B', + license: 'Apache-2.0', + paramsB: 20, + bytes: 13_793_441_427, + kvBytesPerToken: 24 * KIB, + contexts: [131072, 65536], + rank: 3, + // No speed figure: its speed is unknown on every chip. + tested: [], + }, + { + id: 'qwen3.6:35b-a3b', + label: 'Qwen3.6 35B-A3B', + license: 'Apache-2.0', + paramsB: 35, + bytes: 22_621_314_161, + kvBytesPerToken: 20 * KIB, + contexts: [131072, 65536], + rank: 4, + // 54.5 tokens per second measured on 400 GB/s (3 billion parameters are active per token). + speedPer100Gbps: 13.6, + tested: [], + }, + { + id: 'qwen3.8:27b', + label: 'Qwen3.8 27B', + license: 'Apache-2.0', + paramsB: 27.8, + bytes: 17_741_871_939, + kvBytesPerToken: 64 * KIB, + contexts: [131072, 65536, 32768], + rank: 5, + // 11.8 tokens per second measured on 400 GB/s. + speedPer100Gbps: 2.95, + tested: [], + }, + { + id: 'qwen3.8:27b-q8_0', + label: 'Qwen3.8 27B (Q8)', + license: 'Apache-2.0', + paramsB: 27.8, + bytes: 29_978_241_837, + kvBytesPerToken: 64 * KIB, + contexts: [262144, 131072], + rank: 6, + // Not measured: the 2.95 of the row above scaled by file size. + speedPer100Gbps: 1.7, + tested: [], + }, + { + id: 'qwen3.5:122b-a10b', + label: 'Qwen3.5 122B-A10B', + license: 'Apache-2.0', + paramsB: 122, + bytes: 81_370_035_877, + kvBytesPerToken: 24 * KIB, + contexts: [131072], + rank: 7, + // No speed figure: its speed is unknown on every chip. + tested: [], + }, + ], + // The higher rank wins when a model can run with more than one harness. + harnesses: [ + { cli: 'opencode', minContext: 65536, rank: 3 }, + // Sends a fixed prompt of about 36,000 tokens, so the first answer takes minutes. + { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true }, + // Gated: neither its 32768 minimum nor its handling of a `name:tag` model id is verified. + { cli: 'pi', minContext: 32768, rank: 1, gated: true }, + ], + // Memory bandwidth in GB/s, most specific first. Only chips that shipped and are in the source + // table are listed. Where one name covers two bandwidths (M3 Max 300 or 400, M4 Max 410 or 546, + // M5 Max 460 or 614) the LOWER one stands, so a speed computed from it is a floor. + chips: [ + { match: 'M5 Ultra', gbps: 1228 }, + { match: 'M3 Ultra', gbps: 800 }, + { match: 'M2 Ultra', gbps: 800 }, + { match: 'M1 Ultra', gbps: 800 }, + { match: 'M5 Max', gbps: 460 }, + { match: 'M4 Max', gbps: 410 }, + { match: 'M3 Max', gbps: 300 }, + { match: 'M2 Max', gbps: 400 }, + { match: 'M1 Max', gbps: 400 }, + { match: 'M5 Pro', gbps: 307 }, + { match: 'M4 Pro', gbps: 273 }, + { match: 'M3 Pro', gbps: 150 }, + { match: 'M2 Pro', gbps: 200 }, + { match: 'M1 Pro', gbps: 200 }, + { match: 'M5', gbps: 154 }, + { match: 'M4', gbps: 120 }, + { match: 'M3', gbps: 100 }, + { match: 'M2', gbps: 100 }, + { match: 'M1', gbps: 68 }, ], - models: [], - harnesses: [], - chips: [], }; +let validated: LocalModelCatalog | undefined; + /** - * The catalogue in force. There is no override in this stage, so this is the stock constant; - * callers go through this function anyway, so a later stage can change where it comes from - * without touching them. + * The catalogue in force: the stock constant, validated once on first use. Callers go through + * this function, so a later stage can change where the catalogue comes from without touching + * them. Throws when the constant breaks its own schema, which the catalogue test rules out. */ export function getLocalModelCatalog(): LocalModelCatalog { - return STOCK_LOCAL_MODEL_CATALOG; + validated ??= LocalModelCatalogSchema.parse(STOCK_LOCAL_MODEL_CATALOG); + return validated; } diff --git a/src/local-model/hardware.ts b/src/local-model/hardware.ts index 687d981a..5c49cfb7 100644 --- a/src/local-model/hardware.ts +++ b/src/local-model/hardware.ts @@ -1,8 +1,6 @@ /** * @fileoverview The hardware probe of the local model recommendation: what this Mac is. * - * SKELETON: the exported signatures are final, the bodies are not written yet. This module - * will hold: * - `parseSysctl()`: a pure parser of `key: value` lines. The command runs WITHOUT `-n`, so a * key that is missing on a machine (`sysctl.proc_translated` on Intel) cannot shift the * others, and a line it does not know is ignored. @@ -12,17 +10,30 @@ * * Rules: * - Never throws and never rejects: a failed probe is an answer (`source: 'os-fallback'`), and - * off macOS nothing is spawned (`source: 'not-probed'`). + * off macOS nothing is spawned and no disk is read (`source: 'not-probed'`). * - Never `system_profiler` (tens of milliseconds and a parse of free text), never a second * spawn, never a synchronous exec on the server's event loop. * - A non-zero exit is tolerated once `hw.memsize` parsed (a missing key makes sysctl exit 1). + * Without `hw.memsize` nothing else of that output is trusted: the answer is the fallback. + * - Only a line that ends in a newline counts. A cut-off tail is not a value: half of + * `hw.memsize` would read as a smaller, valid number. * - `iogpu.wired_limit_mb` of 0 means "not set": it is reported as 0 and only a value above 0 * is ever used as a limit. + * - The data dir can sit on a network mount that stopped answering, so the disk read is + * bounded too: the folder is probed with the bounded path probe first, the `statfs` is raced + * against a timeout, and a `statfs` that never came back is not started a second time. * - `memGb`, `memClassGb` and `freeDiskGb` are filled here through src/local-model/format.ts, * the one place a size is formatted. */ +import { execFile } from 'node:child_process'; +import { statfs } from 'node:fs/promises'; +import os from 'node:os'; +import { dirname } from 'node:path'; import type { HardwareFacts } from '../types.js'; +import { getDataDir } from '../config/instance.js'; +import { probePathKind, type PathProbeKind } from '../utils/bounded-path-probe.js'; +import { formatGb, memClassGb } from './format.js'; /** Absolute path, so the server's PATH (minimal under a service manager) cannot change what runs. */ export const SYSCTL_PATH = '/usr/sbin/sysctl'; @@ -44,6 +55,18 @@ export const SYSCTL_TIMEOUT_MS = 2000; /** How long one probe result is reused. */ export const HARDWARE_CACHE_TTL_MS = 60_000; +/** Upper bound of the wait for the free disk figure. */ +export const FREE_DISK_TIMEOUT_MS = 2000; + +/** Seven short lines are well under this. A larger output is not sysctl's. */ +const SYSCTL_MAX_OUTPUT_BYTES = 64 * 1024; + +/** Longest text value kept (a chip name, a model identifier, a version). */ +const MAX_TEXT_VALUE_LENGTH = 100; + +/** How many parent folders are tried when the data dir does not exist yet. */ +const FREE_DISK_MAX_PARENT_STEPS = 6; + /** What `parseSysctl()` read. Every field is absent when its key was missing or its value did not parse. */ export interface SysctlFacts { /** `hw.memsize`. */ @@ -82,25 +105,305 @@ export interface HardwareProbeDeps { now?: () => number; } +/** A whole number without sign, as sysctl prints one. */ +function parseCount(value: string): number | undefined { + if (!/^\d+$/.test(value)) return undefined; + const parsed = Number(value); + return Number.isSafeInteger(parsed) ? parsed : undefined; +} + +/** sysctl prints a flag as 0 or 1. Anything else is not a flag. */ +function parseFlag(value: string): boolean | undefined { + if (value === '1') return true; + if (value === '0') return false; + return undefined; +} + +function parseText(value: string): string | undefined { + return value !== '' && value.length <= MAX_TEXT_VALUE_LENGTH ? value : undefined; +} + /** * Parse the output of `sysctl ` (no `-n`): one `key: value` line per key it knows. - * Pure. Unknown lines, blank lines and error lines are skipped. + * Pure. Unknown lines, blank lines and error lines are skipped, and so is a last line that does + * not end in a newline. The first line of a key wins. */ export function parseSysctl(text: string): SysctlFacts { - void text; - throw new Error('not implemented: parseSysctl'); + const facts: SysctlFacts = {}; + const lines = text.split('\n'); + // What follows the last newline is either empty or a cut-off line. + lines.pop(); + for (const rawLine of lines) { + const line = rawLine.replace(/\r$/, ''); + const colon = line.indexOf(':'); + if (colon <= 0) continue; + const key = line.slice(0, colon).trim(); + const value = line.slice(colon + 1).trim(); + switch (key) { + case 'hw.memsize': { + const bytes = parseCount(value); + if (bytes !== undefined && bytes > 0) facts.memBytes ??= bytes; + break; + } + case 'machdep.cpu.brand_string': + facts.chip ??= parseText(value); + break; + case 'hw.optional.arm64': + facts.arm64 ??= parseFlag(value); + break; + case 'sysctl.proc_translated': + facts.translated ??= parseFlag(value); + break; + case 'iogpu.wired_limit_mb': + facts.wiredLimitMb ??= parseCount(value); + break; + case 'kern.osproductversion': { + const version = /^(\d+)(?:\.\d+)*$/.exec(value); + if (version && value.length <= MAX_TEXT_VALUE_LENGTH && facts.macosVersion === undefined) { + facts.macosVersion = value; + facts.macosMajor = Number(version[1]); + } + break; + } + case 'hw.model': + facts.model ??= parseText(value); + break; + default: + break; + } + } + return facts; } +/** + * The one spawn: `command` with `SYSCTL_KEYS` as its arguments and nothing else (no `-n`). + * Resolves stdout when the program ran, also after a non-zero exit (a key this machine lacks + * makes sysctl exit 1 while the other keys are printed). Resolves `undefined` when it could not + * be started, was killed (the timeout) or printed too much. Never rejects. + * + * The two parameters exist for the test, which runs a script of its own in place of sysctl. + */ +export function spawnSysctl( + command: string = SYSCTL_PATH, + timeoutMs: number = SYSCTL_TIMEOUT_MS +): Promise { + return new Promise((resolve) => { + try { + execFile( + command, + [...SYSCTL_KEYS], + { + timeout: timeoutMs, + maxBuffer: SYSCTL_MAX_OUTPUT_BYTES, + encoding: 'utf8', + windowsHide: true, + // sysctl needs nothing from the server's environment, which can hold a password. + env: { LC_ALL: 'C' }, + }, + (error, stdout) => { + if (!error) return resolve(stdout); + // A numeric code is an exit status: the program ran. A string code (ENOENT, EACCES, + // the buffer limit) or a signal means it did not run to its end. + const ranToEnd = typeof error.code === 'number' && !error.killed && !error.signal; + resolve(ranToEnd ? stdout : undefined); + } + ); + } catch { + resolve(undefined); + } + }); +} + +/** + * Free bytes on the filesystem that holds `path` (`bavail * bsize`: what an unprivileged + * process can use), or `undefined` when the filesystem does not say. Not bounded by itself: see + * `dataDirFreeBytes()`. + */ +export async function freeDiskBytesOf(path: string): Promise { + try { + const stat = await statfs(path); + const free = Number(stat.bavail) * Number(stat.bsize); + return Number.isFinite(free) && free >= 0 ? free : undefined; + } catch { + return undefined; + } +} + +/** The one `statfs` that may be pending. While it is, no second one is started. */ +let pendingStatfs: Promise | undefined; + +/** The IO of `dataDirFreeBytes()`. Each member defaults to the real thing; a test passes its own. */ +export interface FreeDiskDeps { + /** Default `getDataDir()`. */ + dataDir?: () => string; + /** Default the bounded path probe (`probePathKind`). */ + probeKind?: (path: string) => Promise; + /** Default `freeDiskBytesOf`. */ + freeBytesOf?: (path: string) => Promise; + /** Default `FREE_DISK_TIMEOUT_MS`. */ + timeoutMs?: number; +} + +/** + * Free bytes on the data dir's filesystem, bounded. The folder (or, when it does not exist yet, + * its nearest existing parent) is found with the bounded path probe, which never waits on a + * dead mount twice; the `statfs` itself is raced against a timeout. `undefined` when the + * filesystem does not answer in time, or is not known to be a folder. + */ +export async function dataDirFreeBytes(deps: FreeDiskDeps = {}): Promise { + // Concurrent probes share one promise in `readHardware()`, so a `statfs` that is still pending + // here was started by an EARLIER probe and has outlived its timeout: a dead mount. It holds + // one threadpool worker, and that is the most this module may ever hold. + if (pendingStatfs) return undefined; + + const probeKind = deps.probeKind ?? probePathKind; + let folder: string; + try { + folder = (deps.dataDir ?? getDataDir)(); + let kind = await probeKind(folder); + for (let step = 0; kind === 'absent' && step < FREE_DISK_MAX_PARENT_STEPS; step++) { + const parent = dirname(folder); + if (parent === folder) break; + folder = parent; + kind = await probeKind(folder); + } + // 'unknown' is a mount that does not answer (or an unreadable path): do not touch it. + if (kind !== 'directory') return undefined; + } catch { + return undefined; + } + + const started = (deps.freeBytesOf ?? freeDiskBytesOf)(folder).catch(() => undefined); + pendingStatfs = started; + const forget = (): void => { + if (pendingStatfs === started) pendingStatfs = undefined; + }; + started.then(forget, forget); + + let timer: ReturnType | undefined; + try { + return await Promise.race([ + started, + new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), deps.timeoutMs ?? FREE_DISK_TIMEOUT_MS); + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +function defaultCpuModel(): string | undefined { + return os.cpus()[0]?.model; +} + +/** A dependency that must never break the probe: its value, or `undefined` when it threw or rejected. */ +async function attempt(read: () => T | Promise): Promise { + try { + return await read(); + } catch { + return undefined; + } +} + +/** Memory and the three fields derived from it. */ +function memoryFields(memBytes: number | undefined): Pick { + const bytes = memBytes !== undefined && Number.isFinite(memBytes) && memBytes > 0 ? memBytes : 0; + return { memBytes: bytes, memGb: formatGb(bytes), memClassGb: memClassGb(bytes) }; +} + +async function probeHardware(deps: HardwareProbeDeps): Promise { + const platform = deps.platform ?? process.platform; + const arch = deps.arch ?? os.arch(); + const totalMemBytes = deps.totalMemBytes ?? os.totalmem; + const cpuModel = deps.cpuModel ?? defaultCpuModel; + + if (platform !== 'darwin') { + return { platform, arch, ...memoryFields(await attempt(totalMemBytes)), source: 'not-probed' }; + } + + const [stdout, freeDisk] = await Promise.all([ + attempt(deps.runSysctl ?? (() => spawnSysctl())), + attempt(deps.freeDiskBytes ?? (() => dataDirFreeBytes())), + ]); + const disk: Pick = + freeDisk !== undefined && Number.isFinite(freeDisk) && freeDisk >= 0 + ? { freeDiskBytes: freeDisk, freeDiskGb: formatGb(freeDisk) } + : {}; + + const facts = typeof stdout === 'string' ? parseSysctl(stdout) : {}; + if (facts.memBytes === undefined) { + const chip = await attempt(cpuModel); + return { + platform, + arch, + ...memoryFields(await attempt(totalMemBytes)), + ...(chip ? { chip } : {}), + ...disk, + source: 'os-fallback', + }; + } + + const chip = facts.chip ?? (await attempt(cpuModel)); + return { + platform, + arch, + ...(facts.arm64 !== undefined ? { arm64: facts.arm64 } : {}), + ...(facts.translated !== undefined ? { translated: facts.translated } : {}), + ...memoryFields(facts.memBytes), + ...(chip ? { chip } : {}), + ...(facts.model !== undefined ? { model: facts.model } : {}), + ...(facts.macosVersion !== undefined ? { macosVersion: facts.macosVersion } : {}), + ...(facts.macosMajor !== undefined ? { macosMajor: facts.macosMajor } : {}), + ...(facts.wiredLimitMb !== undefined ? { wiredLimitMb: facts.wiredLimitMb } : {}), + ...disk, + source: 'sysctl', + }; +} + +let cached: { at: number; facts: HardwareFacts } | undefined; +let inFlight: Promise | undefined; + /** * What this machine is. Cached for `HARDWARE_CACHE_TTL_MS`; concurrent callers share one probe. * Never rejects. + * + * The cache is one per process and does not look at `deps`: within the 60 s a second call + * returns the first call's answer whatever it injects. A test resets it first. */ -export async function readHardware(deps?: HardwareProbeDeps): Promise { - void deps; - throw new Error('not implemented: readHardware'); +export async function readHardware(deps: HardwareProbeDeps = {}): Promise { + const now = deps.now ?? Date.now; + if (cached) { + const age = now() - cached.at; + if (age >= 0 && age < HARDWARE_CACHE_TTL_MS) return cached.facts; + } + if (inFlight) return inFlight; + + const probe = probeHardware(deps) + .catch( + // Unreachable by design (every read above is guarded), and still an answer if it is reached. + (): HardwareFacts => ({ + platform: deps.platform ?? process.platform, + arch: deps.arch ?? os.arch(), + ...memoryFields(undefined), + source: (deps.platform ?? process.platform) === 'darwin' ? 'os-fallback' : 'not-probed', + }) + ) + .then((facts) => { + cached = { at: now(), facts }; + return facts; + }) + .finally(() => { + if (inFlight === probe) inFlight = undefined; + }); + inFlight = probe; + return probe; } -/** Drop the cached probe result (tests, and nothing else). */ +/** Drop the cached probe result and the memory of a pending disk read (tests, and nothing else). */ export function resetHardwareCache(): void { - throw new Error('not implemented: resetHardwareCache'); + cached = undefined; + inFlight = undefined; + pendingStatfs = undefined; } diff --git a/src/local-model/recommend.ts b/src/local-model/recommend.ts index 2536e63c..a0513093 100644 --- a/src/local-model/recommend.ts +++ b/src/local-model/recommend.ts @@ -2,54 +2,231 @@ * @fileoverview The recommendation itself: which model, at which context, with which harness * fits this Mac. Pure: no IO, no clock, no CLI registry, no data dir. * - * SKELETON: the exported signatures are final, the bodies are not written yet. This module - * will hold: + * The math, in the order it runs: * - `deviceBudget()`: L is the wired limit when it is above 0, else 2/3 of memory below 36 GB * and 3/4 from 36 GB. B = min(L - 1 GiB, memory - reserve), with a reserve of 6 GiB up to - * 18 GB, 8 GiB up to 48 GB and 12 GiB from 64 GB. + * 18 GB, 8 GiB up to 48 GB and 12 GiB above. * - `needBytes()`: F = weights + kvBytesPerToken * ctx + 1 GiB + the prompt cache tier of the * catalogue for this Mac's memory. - * - `recommend()`: the candidates (model, context, harness) with `ctx >= harness.minContext`, - * the harness installed or installable, accepting the model id and not gated; the fit (F/B up - * to 0.85 good, up to 0.98 tight, above out); the speed (ok, slow or unknown: a chip that is - * in no chips row is unknown, a bandwidth is never guessed); the ranking (good before tight, - * then ok before unknown before slow, then model rank, harness rank, larger context); and the - * verdict. + * - Fit: F/B up to 0.85 is good, up to 0.98 is tight, above is out. + * - Speed: tokens per second = the model's `speedPer100Gbps` x the chip's bandwidth / 100. + * Below 10 is slow. A chip that is in no chips row, or a model without a speed figure, is + * UNKNOWN: a bandwidth is never guessed. + * - Candidates: (model, context, harness) with `ctx >= harness.minContext`, the harness not + * gated, installed or installable, and accepting the model id. + * - Ranking, lexicographic: good before tight; ok before unknown before slow; the higher model + * rank; the higher harness rank; the larger context. The pick is the first candidate, the + * alternatives are the next four MODELS, each at its own best context and harness. + * + * Verdict `none`, in this order (the first that applies is the reason): + * 1. `not-macos`: the server does not run on macOS. + * 2. `intel`: a Mac without Apple silicon. + * 3. `macos`: macOS is older than the catalogue's minimum (an unknown version does not block). + * 4. `memory`: no model fits the budget at any of its contexts, whatever the harness. + * 5. `no-harness`: no harness row is usable (not gated, known to the caller, installed or + * installable). + * 6. `memory`: models fit, but none at a context a usable harness needs. + * 7. `no-harness`: models fit at such a context, but no usable harness accepts their id. + * 8. `disk`: for every remaining model, free disk minus the download is below 10 GiB (an + * unknown free disk does not block). * * Rules: - * - Verdict `limited` when the best candidate's model has fewer than 12 billion parameters. - * - The word "Recommended" (`LOCAL_MODEL_LABELS.headlineTested`) only for a pick whose model - * has a `tested` entry matching this Mac. Otherwise `headlineOk` plus `noteUntested`. - * - Reasons for `none`: not-macos, intel, memory, disk (free disk minus the download is below - * 10 GiB for every fitting model), macos, no-harness. + * - Verdict `limited` when the pick's model has fewer than 12 billion parameters. + * - The word "Recommended" (`LOCAL_MODEL_LABELS.headlineTested`) only for verdict `ok` with a + * pick whose model has a `tested` entry for exactly this chip, this memory size and this + * harness. Otherwise `headlineOk` (or `headlineLimited`) plus `noteUntested`. * - Every label in the result comes from `LOCAL_MODEL_LABELS`, every size text from * `formatGb()`. No number is spliced into a sentence here. * - No CLI id is ever compared with a literal: a harness is a catalogue row plus a * `HarnessFact`. */ -import type { - DeviceBudget, - HardwareFacts, - HarnessFact, - LocalModelCatalog, - LocalModelEntry, - Recommendation, +import { + LOCAL_MODEL_LABELS, + LOCAL_MODEL_NONE_LABELS, + type Candidate, + type DeviceBudget, + type FitGrade, + type HardwareFacts, + type HarnessFact, + type LocalHarnessEntry, + type LocalModelCatalog, + type LocalModelEntry, + type NoneReason, + type Recommendation, + type SpeedGrade, } from '../types.js'; +import { chipBandwidthGbps } from '../config/local-model-catalog.js'; +import { formatGb, GIB, MIB } from './format.js'; + +/** Need over budget up to this share is a `good` fit. */ +export const FIT_GOOD_MAX_SHARE = 0.85; +/** Need over budget up to this share is a `tight` fit; above it the model is out. */ +export const FIT_TIGHT_MAX_SHARE = 0.98; +/** An estimated speed below this many tokens per second is `slow`. */ +export const SLOW_BELOW_TOKENS_PER_SECOND = 10; +/** A pick whose model has fewer than this many billion parameters makes the verdict `limited`. */ +export const LIMITED_BELOW_PARAMS_B = 12; +/** What must stay free on the disk after the model download. */ +export const DISK_HEADROOM_BYTES = 10 * GIB; +/** What the runtime needs beside the weights, the KV cache and the prompt cache (compute buffers). */ +export const RUNTIME_OVERHEAD_BYTES = GIB; +/** How many other models are listed under the pick. */ +export const MAX_ALTERNATIVES = 4; + +/** A memory size of at least this many GB (as sold) gets 3/4 of its memory as device limit, below it 2/3. */ +const LARGE_LIMIT_FROM_GB = 36; + +/** The memory the system, Codeman, a browser and the agent keep for themselves, by the size the Mac is sold as. */ +function reserveBytes(memClassGb: number): number { + if (memClassGb <= 18) return 6 * GIB; + if (memClassGb <= 48) return 8 * GIB; + return 12 * GIB; +} /** The memory a model may use on this Mac: the device limit L, the budget B and where L came from. */ export function deviceBudget(hw: HardwareFacts): DeviceBudget { - void hw; - throw new Error('not implemented: deviceBudget'); + const memBytes = Number.isFinite(hw.memBytes) && hw.memBytes > 0 ? hw.memBytes : 0; + // 0 means "not set" (both test Macs report 0): only a value above 0 is a limit. + const wired = hw.wiredLimitMb !== undefined && Number.isFinite(hw.wiredLimitMb) && hw.wiredLimitMb > 0; + const limitBytes = wired + ? Math.floor((hw.wiredLimitMb as number) * MIB) + : Math.floor(memBytes * (hw.memClassGb < LARGE_LIMIT_FROM_GB ? 2 / 3 : 3 / 4)); + const budgetBytes = Math.max(0, Math.min(limitBytes - GIB, memBytes - reserveBytes(hw.memClassGb))); + return { limitBytes, budgetBytes, source: wired ? 'wired-limit' : 'estimated' }; +} + +/** The prompt cache of the last tier whose minimum is not above this Mac's memory size. */ +function promptCacheBytes(catalog: LocalModelCatalog, memClassGb: number): number { + let bytes = 0; + for (const tier of catalog.promptCacheTiers) { + if (tier.minMemGb <= memClassGb) bytes = tier.bytes; + } + return bytes; } /** F: the bytes `model` needs in memory at a context of `ctx` tokens on this Mac. */ export function needBytes(model: LocalModelEntry, ctx: number, catalog: LocalModelCatalog, hw: HardwareFacts): number { - void model; - void ctx; - void catalog; - void hw; - throw new Error('not implemented: needBytes'); + return model.bytes + model.kvBytesPerToken * ctx + RUNTIME_OVERHEAD_BYTES + promptCacheBytes(catalog, hw.memClassGb); +} + +function fitGrade(need: number, budgetBytes: number): FitGrade { + if (budgetBytes <= 0) return 'out'; + const share = need / budgetBytes; + if (share <= FIT_GOOD_MAX_SHARE) return 'good'; + if (share <= FIT_TIGHT_MAX_SHARE) return 'tight'; + return 'out'; +} + +/** + * Whether the MACHINE is Apple silicon. `hw.optional.arm64` decides when it was read. Without + * it: a process under Rosetta, an arm64 process or an Apple chip name each prove Apple silicon + * (`arch` alone cannot say no, because Node under Rosetta reports x64). + */ +function isAppleSilicon(hw: HardwareFacts): boolean { + if (hw.arm64 !== undefined) return hw.arm64; + if (hw.translated === true || hw.arch === 'arm64') return true; + return /^Apple\b/i.test(hw.chip ?? ''); +} + +function round(value: number, decimals: number): number { + const factor = 10 ** decimals; + return Math.round(value * factor) / factor; +} + +/** A candidate plus the two ranks the ordering needs, which are not part of the payload. */ +interface Ranked { + candidate: Candidate; + modelRank: number; + harnessRank: number; +} + +/** A catalogue harness row with the caller's fact about it. */ +interface UsableHarness { + row: LocalHarnessEntry; + fact: HarnessFact; +} + +const FIT_ORDER: Record = { good: 0, tight: 1 }; +const SPEED_ORDER: Record = { ok: 0, unknown: 1, slow: 2 }; + +/** Best first. */ +function compareRanked(a: Ranked, b: Ranked): number { + return ( + FIT_ORDER[a.candidate.fit] - FIT_ORDER[b.candidate.fit] || + SPEED_ORDER[a.candidate.speed] - SPEED_ORDER[b.candidate.speed] || + b.modelRank - a.modelRank || + b.harnessRank - a.harnessRank || + b.candidate.context - a.candidate.context + ); +} + +/** A function supplied by the caller must not be able to break a pure answer. */ +function accepts(fact: HarnessFact, modelId: string): boolean { + try { + return fact.acceptsModelId(modelId) === true; + } catch { + return false; + } +} + +function buildCandidate( + model: LocalModelEntry, + ctx: number, + harness: UsableHarness, + need: number, + fit: Candidate['fit'], + hw: HardwareFacts, + catalog: LocalModelCatalog, + budgetBytes: number +): Candidate { + const gbps = chipBandwidthGbps(catalog.chips, hw.chip); + const tokensPerSecond = + gbps !== undefined && model.speedPer100Gbps !== undefined + ? round((model.speedPer100Gbps * gbps) / 100, 1) + : undefined; + const speed: SpeedGrade = + tokensPerSecond === undefined ? 'unknown' : tokensPerSecond < SLOW_BELOW_TOKENS_PER_SECOND ? 'slow' : 'ok'; + // "A Mac like this" is exact: the same chip string, the same memory size as sold, the same harness. + const tested = model.tested.some( + (entry) => entry.chip === hw.chip && entry.memGb === hw.memClassGb && entry.cli === harness.row.cli + ); + return { + modelId: model.id, + modelLabel: model.label, + license: model.license, + paramsB: model.paramsB, + context: ctx, + cli: harness.row.cli, + cliLabel: harness.fact.label, + cliInstalled: harness.fact.installed, + ...(harness.row.slowFirstTurn ? { slowFirstTurn: true } : {}), + needBytes: need, + needGb: formatGb(need), + downloadBytes: model.bytes, + downloadGb: formatGb(model.bytes), + budgetShare: round(need / budgetBytes, 3), + fit, + fitLabel: fit === 'good' ? LOCAL_MODEL_LABELS.fitGood : LOCAL_MODEL_LABELS.fitTight, + speed, + ...(speed === 'slow' ? { speedLabel: LOCAL_MODEL_LABELS.speedSlow } : {}), + ...(speed === 'unknown' ? { speedLabel: LOCAL_MODEL_LABELS.speedUnknown } : {}), + ...(tokensPerSecond !== undefined ? { tokensPerSecond } : {}), + tested, + }; +} + +function none(reason: NoneReason, budget: DeviceBudget, vars?: Record): Recommendation { + return { + verdict: 'none', + reason, + headline: { text: LOCAL_MODEL_NONE_LABELS[reason], ...(vars ? { vars } : {}) }, + notes: [], + alternatives: [], + budgetBytes: budget.budgetBytes, + budgetGb: formatGb(budget.budgetBytes), + budgetSource: budget.source, + }; } /** @@ -61,8 +238,99 @@ export function recommend( catalog: LocalModelCatalog, harnessFacts: readonly HarnessFact[] ): Recommendation { - void hw; - void catalog; - void harnessFacts; - throw new Error('not implemented: recommend'); + // Off macOS and on an Intel Mac there is no budget to compute: a model has no memory to run in. + const noBudget: DeviceBudget = { limitBytes: 0, budgetBytes: 0, source: 'estimated' }; + if (hw.platform !== 'darwin') return none('not-macos', noBudget); + if (!isAppleSilicon(hw)) return none('intel', noBudget); + + const budget = deviceBudget(hw); + if (hw.macosMajor !== undefined && hw.macosMajor < catalog.minMacosMajor) { + return none('macos', budget, { + version: hw.macosVersion ?? String(hw.macosMajor), + minimum: String(catalog.minMacosMajor), + }); + } + + // Every (model, context) that fits the budget, before any harness is looked at. + const fitting: { model: LocalModelEntry; ctx: number; need: number; fit: Candidate['fit'] }[] = []; + for (const model of catalog.models) { + for (const ctx of model.contexts) { + const need = needBytes(model, ctx, catalog, hw); + const fit = fitGrade(need, budget.budgetBytes); + if (fit !== 'out') fitting.push({ model, ctx, need, fit }); + } + } + const tooLittleMemory = (): Recommendation => none('memory', budget, { memory: hw.memGb }); + if (fitting.length === 0) return tooLittleMemory(); + + // A gated row is never offered, whatever the caller knows about its CLI. + const usable: UsableHarness[] = []; + for (const row of catalog.harnesses) { + if (row.gated) continue; + const fact = harnessFacts.find((candidate) => candidate.cli === row.cli); + if (fact && (fact.installed || fact.installable)) usable.push({ row, fact }); + } + if (usable.length === 0) return none('no-harness', budget); + + const atHarnessContext = fitting.flatMap((entry) => + usable.filter((harness) => entry.ctx >= harness.row.minContext).map((harness) => ({ ...entry, harness })) + ); + if (atHarnessContext.length === 0) return tooLittleMemory(); + + const accepted = atHarnessContext.filter((entry) => accepts(entry.harness.fact, entry.model.id)); + if (accepted.length === 0) return none('no-harness', budget); + + // The download must leave headroom on the disk. An unknown free disk does not block. + const free = hw.freeDiskBytes; + const downloadable = + free === undefined ? accepted : accepted.filter((entry) => free - entry.model.bytes >= DISK_HEADROOM_BYTES); + if (free !== undefined && downloadable.length === 0) { + const smallest = Math.min(...accepted.map((entry) => entry.model.bytes)); + return none('disk', budget, { + download: formatGb(smallest), + free: hw.freeDiskGb ?? formatGb(free), + missing: formatGb(smallest + DISK_HEADROOM_BYTES - free), + }); + } + + const ranked: Ranked[] = downloadable + .map((entry) => ({ + candidate: buildCandidate( + entry.model, + entry.ctx, + entry.harness, + entry.need, + entry.fit, + hw, + catalog, + budget.budgetBytes + ), + modelRank: entry.model.rank, + harnessRank: entry.harness.row.rank, + })) + .sort(compareRanked); + + // The first candidate of each model is that model at its best context and harness. + const bestPerModel: Candidate[] = []; + for (const { candidate } of ranked) { + if (!bestPerModel.some((kept) => kept.modelId === candidate.modelId)) bestPerModel.push(candidate); + } + const [pick, ...others] = bestPerModel; + + const limited = pick.paramsB < LIMITED_BELOW_PARAMS_B; + const headline = limited + ? LOCAL_MODEL_LABELS.headlineLimited + : pick.tested + ? LOCAL_MODEL_LABELS.headlineTested + : LOCAL_MODEL_LABELS.headlineOk; + return { + verdict: limited ? 'limited' : 'ok', + headline: { text: headline }, + notes: pick.tested ? [] : [{ text: LOCAL_MODEL_LABELS.noteUntested }], + pick, + alternatives: others.slice(0, MAX_ALTERNATIVES), + budgetBytes: budget.budgetBytes, + budgetGb: formatGb(budget.budgetBytes), + budgetSource: budget.source, + }; } diff --git a/src/types/local-model.ts b/src/types/local-model.ts index 5298c3bb..77352d64 100644 --- a/src/types/local-model.ts +++ b/src/types/local-model.ts @@ -129,11 +129,14 @@ export interface LocalHarnessEntry { /** * Memory bandwidth of one chip family. `LocalModelCatalog.chips` is ordered most specific first - * ('M4 Max' before 'M4') and the first row whose `match` occurs in `HardwareFacts.chip` wins. A - * chip that matches no row has an UNKNOWN speed: a bandwidth is never guessed. + * ('M4 Max' before 'M4') and the first row that names `HardwareFacts.chip` wins. A row names a + * chip when the chip string ENDS with its `match`, starting at the beginning or after a space, + * case ignored (`chipNameMatches()` in src/config/local-model-catalog.ts): 'M4' is not the chip + * of 'Apple M4 Pro', and a variant the table has never heard of ('Apple M6 Pro') matches no row. + * A chip that matches no row has an UNKNOWN speed: a bandwidth is never guessed. */ export interface ChipEntry { - /** Text that must occur in the chip string ('M4 Max'). */ + /** The chip name the chip string must end with ('M4 Max'). */ match: string; /** Memory bandwidth in GB/s. */ gbps: number; @@ -305,7 +308,7 @@ export interface Candidate { downloadBytes: number; /** `downloadBytes` as decimal GB text, no unit. */ downloadGb: string; - /** `needBytes / Recommendation.budgetBytes` (0.83). */ + /** `needBytes / Recommendation.budgetBytes`, rounded to three decimals (0.83). The fit grade is decided on the unrounded ratio. */ budgetShare: number; /** A candidate in a `Recommendation` is never `out`. */ fit: Exclude; @@ -339,7 +342,7 @@ export interface Recommendation { pick?: Candidate; /** The next models that fit (at most four, each at its best context and harness), best first. Empty when `verdict` is `none`. */ alternatives: Candidate[]; - /** `DeviceBudget.budgetBytes`. 0 when the budget could not be computed (not macOS). */ + /** `DeviceBudget.budgetBytes`. 0 when no budget is computed: not macOS, or a Mac without Apple silicon. */ budgetBytes: number; /** `budgetBytes` as decimal GB text, no unit. */ budgetGb: string; diff --git a/test/cli-registry-no-id-branching.test.ts b/test/cli-registry-no-id-branching.test.ts index c746bdfb..cf31be83 100644 --- a/test/cli-registry-no-id-branching.test.ts +++ b/test/cli-registry-no-id-branching.test.ts @@ -167,7 +167,6 @@ const ALLOWED_BRANCHES: Record = { ), "tui/tui-render.ts::mode !== 'claude'": allow(1, 'cosmetic: suppress the mode badge for the default mode'), - // Legacy Config plumbing, same category as the `===` entries above. "web/routes/session-routes.ts::mode !== 'omp'": allow( 2, @@ -212,8 +211,10 @@ const ID_ALT = IDS.join('|'); */ const BRANCH_PATTERN = new RegExp( [ - // mode === 'codex' / mode !== 'codex' - `\\b(?:mode|id|agentType)\\s*[!=]==\\s*'(?:${ID_ALT})'`, + // mode === 'codex' / mode !== 'codex'. `cli` is the name the local model catalogue gives + // the same thing (`LocalHarnessEntry.cli`, `Candidate.cli`), so `harness.cli === 'claude'` + // is the same banned branch under another field name. + `\\b(?:mode|id|agentType|cli)\\s*[!=]==\\s*'(?:${ID_ALT})'`, // case 'codex': `\\bcase\\s+'(?:${ID_ALT})'\\s*:`, // ['codex', 'gemini'].includes(mode) — the id list IS the branch, wherever `mode` sits @@ -267,7 +268,7 @@ function scan(): { findings: Finding[]; filesScanned: number } { lines.forEach((line, i) => { BRANCH_PATTERN.lastIndex = 0; // shared /g regex — see utils/regex-patterns.ts for (const match of line.matchAll(BRANCH_PATTERN)) { - const expression = match[0].replace(/\s+/g, ' ').replace(/^(?:id|agentType)/, 'mode'); + const expression = match[0].replace(/\s+/g, ' ').replace(/^(?:id|agentType|cli)/, 'mode'); const posix = rel.split(sep).join('/'); findings.push({ file: posix, expression, line: i + 1, key: `${posix}::${expression}` }); } @@ -337,6 +338,8 @@ describe('no CLI-id branching outside the stock catalog', () => { "if (mode !== 'shell' && mode !== 'deepseek') { doSomething(); }", "switch (mode) { case 'gemini': return 1; }", "if (['codex', 'gemini'].includes(mode)) { doSomething(); }", + "if (harness.cli === 'claude') { doSomething(); }", + "const others = candidates.filter((c) => c.cli !== 'opencode');", ]; for (const sample of samples) { BRANCH_PATTERN.lastIndex = 0; diff --git a/test/local-model-catalog.test.ts b/test/local-model-catalog.test.ts new file mode 100644 index 00000000..5f72b764 --- /dev/null +++ b/test/local-model-catalog.test.ts @@ -0,0 +1,383 @@ +/** + * @fileoverview Tests for src/config/local-model-catalog.ts: the compiled catalogue of models, + * harnesses and chips, its schema, and the chip lookup. + * + * Three things are pinned: + * + * 1. The stock constant PARSES. `getLocalModelCatalog()` validates it on first use and throws on + * a bad row, so this test is what keeps a bad row from ever reaching the route. + * 2. The schema refuses what must never get in: a model id that could name another registry + * (a slash, a host), an unknown key (a `runtime` block with a download URL), and rows that + * break a rule across rows (a repeated id, contexts in the wrong order, a chip row that + * hides a more specific one). + * 3. The chip lookup never guesses: a chip that no row names has no bandwidth, including one + * whose name merely contains a known name ('Apple M10', 'Apple M6 Pro'). + * + * Port: N/A (pure data and functions). + */ + +import { describe, expect, it } from 'vitest'; +import { + chipBandwidthGbps, + chipNameMatches, + getLocalModelCatalog, + LOCAL_MODEL_ID_PATTERN, + LocalModelCatalogSchema, + STOCK_LOCAL_MODEL_CATALOG, +} from '../src/config/local-model-catalog.js'; +import type { LocalModelCatalog, LocalModelEntry } from '../src/types.js'; + +/** A deep copy of the stock catalogue that a test may break. */ +function copy(): LocalModelCatalog { + return structuredClone(STOCK_LOCAL_MODEL_CATALOG); +} + +/** The message of every issue the schema raises for `catalog`, or an empty list when it parses. */ +function issuesOf(catalog: unknown): string[] { + const result = LocalModelCatalogSchema.safeParse(catalog); + return result.success ? [] : result.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`); +} + +/** The stock catalogue with its first model changed. */ +/** `url` is no field of a model row: one test passes it to prove the schema refuses an unknown key. */ +function withFirstModel(change: Partial>): unknown { + const catalog = copy(); + return { ...catalog, models: [{ ...catalog.models[0], ...change }, ...catalog.models.slice(1)] }; +} + +describe('the stock catalogue', () => { + it('passes its own schema', () => { + expect(issuesOf(STOCK_LOCAL_MODEL_CATALOG)).toEqual([]); + }); + + it('is what the getter returns, validated once', () => { + const first = getLocalModelCatalog(); + expect(first).toEqual(STOCK_LOCAL_MODEL_CATALOG); + // Parsed on first use, then reused: the same object, not a new parse per request. + expect(getLocalModelCatalog()).toBe(first); + }); + + it('lists the seven models of the design, without flint:40b', () => { + expect(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => m.id)).toEqual([ + 'qwen3.5:9b', + 'qwen3.5:9b-q8_0', + 'gpt-oss:20b', + 'qwen3.6:35b-a3b', + 'qwen3.8:27b', + 'qwen3.8:27b-q8_0', + 'qwen3.5:122b-a10b', + ]); + expect(STOCK_LOCAL_MODEL_CATALOG.models.some((m) => m.id.startsWith('flint'))).toBe(false); + }); + + it('carries the byte sizes of the registry manifests', () => { + // The sum of the layer sizes of each tag's manifest, as fetched on 2026-11-06. + expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.bytes]))).toEqual({ + 'qwen3.5:9b': 6_550_825_373, + 'qwen3.5:9b-q8_0': 10_449_217_949, + 'gpt-oss:20b': 13_793_441_427, + 'qwen3.6:35b-a3b': 22_621_314_161, + 'qwen3.8:27b': 17_741_871_939, + 'qwen3.8:27b-q8_0': 29_978_241_837, + 'qwen3.5:122b-a10b': 81_370_035_877, + }); + }); + + it('carries the KV cost per token of each model', () => { + expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.kvBytesPerToken]))).toEqual({ + 'qwen3.5:9b': 32_768, + 'qwen3.5:9b-q8_0': 32_768, + 'gpt-oss:20b': 24_576, + 'qwen3.6:35b-a3b': 20_480, + 'qwen3.8:27b': 65_536, + 'qwen3.8:27b-q8_0': 65_536, + 'qwen3.5:122b-a10b': 24_576, + }); + }); + + it('gives every model a parameter count and an empty tested list', () => { + for (const model of STOCK_LOCAL_MODEL_CATALOG.models) { + expect(model.paramsB, model.id).toBeGreaterThan(0); + // Nobody has run any of these with a harness on a Mac yet. + expect(model.tested, model.id).toEqual([]); + expect(model.id).toMatch(LOCAL_MODEL_ID_PATTERN); + expect(model.license).toBe('Apache-2.0'); + } + // The two 9B rows are the ones under the 12 billion line of the `limited` verdict. + expect(STOCK_LOCAL_MODEL_CATALOG.models.filter((m) => m.paramsB < 12).map((m) => m.id)).toEqual([ + 'qwen3.5:9b', + 'qwen3.5:9b-q8_0', + ]); + }); + + it('has a speed figure only where one was measured or scaled from a measurement', () => { + expect(Object.fromEntries(STOCK_LOCAL_MODEL_CATALOG.models.map((m) => [m.id, m.speedPer100Gbps ?? null]))).toEqual({ + 'qwen3.5:9b': 9.3, + 'qwen3.5:9b-q8_0': 5.8, + 'gpt-oss:20b': null, + 'qwen3.6:35b-a3b': 13.6, + 'qwen3.8:27b': 2.95, + 'qwen3.8:27b-q8_0': 1.7, + 'qwen3.5:122b-a10b': null, + }); + }); + + it('lists its harnesses as data rows, one of them gated', () => { + expect(STOCK_LOCAL_MODEL_CATALOG.harnesses).toEqual([ + { cli: 'opencode', minContext: 65536, rank: 3 }, + { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true }, + { cli: 'pi', minContext: 32768, rank: 1, gated: true }, + ]); + }); + + it('counts the prompt cache as 512 MiB below 32 GB and 2 GiB from 32 GB', () => { + expect(STOCK_LOCAL_MODEL_CATALOG.promptCacheTiers).toEqual([ + { minMemGb: 0, bytes: 512 * 1024 ** 2 }, + { minMemGb: 32, bytes: 2 * 1024 ** 3 }, + ]); + expect(STOCK_LOCAL_MODEL_CATALOG.minMacosMajor).toBe(14); + expect(STOCK_LOCAL_MODEL_CATALOG.schema).toBe(1); + }); + + it('holds no download data: no URL, no digest, no runtime block', () => { + const text = JSON.stringify(STOCK_LOCAL_MODEL_CATALOG); + expect(text).not.toMatch(/https?:/); + expect(text).not.toMatch(/sha256/i); + expect(Object.keys(STOCK_LOCAL_MODEL_CATALOG).sort()).toEqual( + ['chips', 'harnesses', 'minMacosMajor', 'models', 'promptCacheTiers', 'schema', 'updated'].sort() + ); + }); +}); + +describe('the model id rule', () => { + it('accepts name:tag in lowercase', () => { + for (const id of ['qwen3.5:9b', 'gpt-oss:20b', 'qwen3.8:27b-q8_0', 'a:b', 'x_y.z-1:2.3_4']) { + expect(LOCAL_MODEL_ID_PATTERN.test(id), id).toBe(true); + } + }); + + it('refuses anything that could name another registry, namespace or no tag at all', () => { + const bad = [ + 'evil.example/x/y:z', + 'library/qwen3.5:9b', + 'registry.ollama.ai/library/qwen3.5:9b', + 'Qwen3.5:9b', + 'qwen3.5:9B', + 'qwen3.5', + 'qwen3.5:', + ':9b', + 'qwen3.5:9b:extra', + 'qwen 3.5:9b', + '-qwen:9b', + 'qwen:-9b', + 'qwen3.5:9b\n', + '', + ]; + for (const id of bad) expect(LOCAL_MODEL_ID_PATTERN.test(id), JSON.stringify(id)).toBe(false); + }); + + it('makes the schema reject a catalogue with such an id', () => { + for (const id of ['evil.example/x/y:z', 'Qwen3.5:9b', 'qwen3.5', 'qwen3.5:9b ']) { + const issues = issuesOf(withFirstModel({ id })); + expect(issues, id).toHaveLength(1); + expect(issues[0]).toContain('models.0.id'); + } + }); +}); + +describe('the schema, field by field', () => { + it('refuses a key it does not know, at the top and inside a row', () => { + // The whole point of having no `runtime` block: a data row can never carry a download URL. + expect(issuesOf({ ...copy(), runtime: { url: 'https://example.com/x.tgz', sha256: 'abc' } })).not.toEqual([]); + expect(issuesOf(withFirstModel({ url: 'https://example.com/model' }))).not.toEqual([]); + const catalog = copy(); + expect(issuesOf({ ...catalog, harnesses: [{ ...catalog.harnesses[0], dialect: 'openai' }] })).not.toEqual([]); + expect(issuesOf({ ...catalog, chips: [{ match: 'M4', gbps: 120, cores: 10 }] })).not.toEqual([]); + }); + + it('requires paramsB and tested on every model row', () => { + for (const field of ['paramsB', 'tested'] as const) { + const row: Record = { ...copy().models[0] }; + delete row[field]; + const issues = issuesOf({ ...copy(), models: [row] }); + expect(issues, field).toHaveLength(1); + expect(issues[0]).toContain(`models.0.${field}`); + } + }); + + it('refuses sizes and counts that are not positive whole numbers', () => { + expect(issuesOf(withFirstModel({ bytes: 0 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ bytes: 1.5 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ kvBytesPerToken: -1 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ contexts: [] }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ paramsB: 0 }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ speedPer100Gbps: 0 }))).not.toEqual([]); + }); + + it('accepts a tested entry and checks its shape', () => { + const good = { chip: 'Apple M4', memGb: 16, cli: 'opencode', date: '2026-11-06' }; + expect(issuesOf(withFirstModel({ tested: [good] }))).toEqual([]); + expect(issuesOf(withFirstModel({ tested: [{ ...good, date: '6 November 2026' }] }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ tested: [{ ...good, memGb: 0 }] }))).not.toEqual([]); + expect(issuesOf(withFirstModel({ tested: [{ ...good, result: 'pass' }] }))).not.toEqual([]); + }); + + it('refuses another schema version and a date that is not ISO', () => { + expect(issuesOf({ ...copy(), schema: 2 })).not.toEqual([]); + expect(issuesOf({ ...copy(), updated: 'today' })).not.toEqual([]); + }); +}); + +describe('the schema, across rows', () => { + it('refuses a repeated model id', () => { + const catalog = copy(); + catalog.models[1] = { ...catalog.models[1], id: catalog.models[0].id }; + expect(issuesOf(catalog)).toEqual(['models.1.id: model ids must be unique']); + }); + + it('refuses a repeated model rank, which would leave the order of two models to chance', () => { + const catalog = copy(); + catalog.models[1] = { ...catalog.models[1], rank: catalog.models[0].rank }; + expect(issuesOf(catalog)).toEqual(['models.1.rank: model ranks must be unique']); + }); + + it('refuses a harness listed twice and a repeated harness rank', () => { + const twice = copy(); + twice.harnesses.push({ cli: twice.harnesses[0].cli, minContext: 32768, rank: 9 }); + expect(issuesOf(twice)).toEqual(['harnesses.3.cli: a harness may be listed only once']); + + const sameRank = copy(); + sameRank.harnesses[1] = { ...sameRank.harnesses[1], rank: sameRank.harnesses[0].rank }; + expect(issuesOf(sameRank)).toEqual(['harnesses.1.rank: harness ranks must be unique']); + }); + + it('refuses contexts that are not listed largest first, or that repeat', () => { + expect(issuesOf(withFirstModel({ contexts: [32768, 65536] }))).toEqual([ + 'models.0.contexts.1: contexts must be listed largest first, without repeats', + ]); + expect(issuesOf(withFirstModel({ contexts: [65536, 65536] }))).toEqual([ + 'models.0.contexts.1: contexts must be listed largest first, without repeats', + ]); + }); + + it('refuses a tested entry for a harness the catalogue does not list', () => { + const entry = { chip: 'Apple M4', memGb: 16, cli: 'not-a-harness', date: '2026-11-06' }; + expect(issuesOf(withFirstModel({ tested: [entry] }))).toEqual([ + 'models.0.tested.0.cli: a tested entry must name a harness of this catalogue', + ]); + }); + + it('refuses chips that are not listed most specific first', () => { + const catalog = copy(); + catalog.chips = [ + { match: 'M4', gbps: 120 }, + { match: 'M4 Max', gbps: 410 }, + ]; + expect(issuesOf(catalog)).toEqual([ + "chips.1.match: chips must be listed most specific first: 'M4' comes before 'M4 Max'", + ]); + + const repeated = copy(); + repeated.chips = [ + { match: 'M4', gbps: 120 }, + { match: 'm4', gbps: 130 }, + ]; + expect(issuesOf(repeated)).toHaveLength(1); + }); + + it('refuses prompt cache tiers that do not start at 0 or do not ascend', () => { + const late = copy(); + late.promptCacheTiers = [{ minMemGb: 16, bytes: 1 }]; + expect(issuesOf(late)).toEqual(['promptCacheTiers.0.minMemGb: the first prompt cache tier must start at 0']); + + const unordered = copy(); + unordered.promptCacheTiers = [ + { minMemGb: 0, bytes: 1 }, + { minMemGb: 32, bytes: 2 }, + { minMemGb: 24, bytes: 3 }, + ]; + expect(issuesOf(unordered)).toEqual(['promptCacheTiers.2.minMemGb: prompt cache tiers must ascend by minMemGb']); + }); +}); + +describe('the chips table', () => { + const chips = STOCK_LOCAL_MODEL_CATALOG.chips; + + it('holds every chip of the source table that shipped', () => { + // Bandwidth column of https://github.com/ggml-org/llama.cpp/discussions/4167, read on + // 2026-11-06. A name that covers two bandwidths has the lower one. + expect(Object.fromEntries(chips.map((row) => [row.match, row.gbps]))).toEqual({ + 'M5 Ultra': 1228, + 'M3 Ultra': 800, + 'M2 Ultra': 800, + 'M1 Ultra': 800, + 'M5 Max': 460, + 'M4 Max': 410, + 'M3 Max': 300, + 'M2 Max': 400, + 'M1 Max': 400, + 'M5 Pro': 307, + 'M4 Pro': 273, + 'M3 Pro': 150, + 'M2 Pro': 200, + 'M1 Pro': 200, + M5: 154, + M4: 120, + M3: 100, + M2: 100, + M1: 68, + }); + }); + + it('finds each row under the name macOS reports for that chip', () => { + // No row is hidden by an earlier one: 'Apple M4 Max' is 410, not the 120 of 'M4'. + for (const row of chips) { + expect(chipBandwidthGbps(chips, `Apple ${row.match}`), row.match).toBe(row.gbps); + } + }); + + it('knows the chips of the two test Macs', () => { + expect(chipBandwidthGbps(chips, 'Apple M4')).toBe(120); + expect(chipBandwidthGbps(chips, 'Apple M5 Pro')).toBe(307); + }); + + it('never guesses: a chip that no row names has no bandwidth', () => { + const unknown = [ + // A base chip that is not in the table. + 'Apple M6', + // A variant the table has never heard of must not borrow the figure of its base chip. + 'Apple M6 Pro', + 'Apple M4 Ultra', + 'Apple M5 Extreme', + // A name that merely contains a known name. + 'Apple M10', + 'Apple M12 Pro', + 'Apple M1 Max (Virtual)', + 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz', + 'VirtualApple @ 2.50GHz processor', + 'Max', + '', + ]; + for (const chip of unknown) expect(chipBandwidthGbps(chips, chip), chip).toBeUndefined(); + expect(chipBandwidthGbps(chips, undefined)).toBeUndefined(); + expect(chipBandwidthGbps([], 'Apple M4')).toBeUndefined(); + }); +}); + +describe('chipNameMatches', () => { + it('matches the whole chip name at the end of the string, ignoring case and padding', () => { + expect(chipNameMatches('Apple M4 Pro', 'M4 Pro')).toBe(true); + expect(chipNameMatches('Apple M4', 'M4')).toBe(true); + expect(chipNameMatches('M4', 'M4')).toBe(true); + expect(chipNameMatches(' apple m4 pro ', 'M4 PRO')).toBe(true); + }); + + it('does not match a part of a longer name', () => { + expect(chipNameMatches('Apple M4 Pro', 'M4')).toBe(false); + expect(chipNameMatches('Apple M14', 'M4')).toBe(false); + expect(chipNameMatches('Apple M10', 'M1')).toBe(false); + expect(chipNameMatches('Apple XM4', 'M4')).toBe(false); + expect(chipNameMatches('Apple M4', '')).toBe(false); + expect(chipNameMatches('', 'M4')).toBe(false); + }); +}); diff --git a/test/local-model-format.test.ts b/test/local-model-format.test.ts new file mode 100644 index 00000000..2f3bd3fb --- /dev/null +++ b/test/local-model-format.test.ts @@ -0,0 +1,86 @@ +/** + * @fileoverview Tests for src/local-model/format.ts: the ONE place a byte count becomes a size a + * person reads, and the memory class a Mac is sold as. + * + * What is pinned here is a contract with the browser: it prints the text it was sent and adds + * ' GB' itself, so the text must be decimal GB, carry no unit, and round the same way everywhere. + * + * Port: N/A (pure functions). + */ + +import { describe, expect, it } from 'vitest'; +import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js'; + +describe('byte units', () => { + it('keeps decimal GB (what a person reads) apart from the binary units of the budget math', () => { + expect(GB).toBe(1_000_000_000); + expect(GIB).toBe(1_073_741_824); + expect(MIB).toBe(1_048_576); + }); +}); + +describe('formatGb', () => { + it('prints decimal GB, so a Mac sold as 16 GB reads 17.2 and one sold as 48 GB reads 51.5', () => { + expect(formatGb(17_179_869_184)).toBe('17.2'); + expect(formatGb(51_539_607_552)).toBe('51.5'); + }); + + it('prints the registry sizes of the catalogue models as the registry shows them', () => { + expect(formatGb(6_550_825_373)).toBe('6.6'); + expect(formatGb(13_793_441_427)).toBe('13.8'); + expect(formatGb(17_741_871_939)).toBe('17.7'); + expect(formatGb(22_621_314_161)).toBe('22.6'); + expect(formatGb(81_370_035_877)).toBe('81.4'); + }); + + it('keeps one decimal below 100 and none from 100', () => { + expect(formatGb(1 * GB)).toBe('1.0'); + expect(formatGb(99.94 * GB)).toBe('99.9'); + // Rounds to 100.0, which is already "from 100". + expect(formatGb(99.96 * GB)).toBe('100'); + expect(formatGb(100 * GB)).toBe('100'); + // The free disk of the two test Macs, from their `df -k` lines. + expect(formatGb(890_926_084 * 1024)).toBe('912'); + expect(formatGb(41_752_848 * 1024)).toBe('42.8'); + }); + + it('never prints a unit, a thousands separator or an exponent', () => { + for (const bytes of [1, 5 * MIB, 3 * GIB, 250 * GB, 4000 * GB, 2.5e15]) { + expect(formatGb(bytes)).toMatch(/^\d+(\.\d)?$/); + } + }); + + it("reads '0.0' for anything that is not a positive size", () => { + for (const bytes of [0, -1, -5 * GB, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) { + expect(formatGb(bytes)).toBe('0.0'); + } + }); + + it('shows a size too small for one decimal as 0.0, never as an empty text', () => { + expect(formatGb(10 * MIB)).toBe('0.0'); + expect(formatGb(512 * MIB)).toBe('0.5'); + }); +}); + +describe('memClassGb', () => { + it('names the size a Mac is sold as: whole binary gigabytes', () => { + for (const sold of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192]) { + expect(memClassGb(sold * GIB)).toBe(sold); + } + // The exact `hw.memsize` of the two test Macs. + expect(memClassGb(17_179_869_184)).toBe(16); + expect(memClassGb(51_539_607_552)).toBe(48); + }); + + it('rounds a size that is not a whole number of gigabytes', () => { + // `hw.memsize_usable` of the 16 GB test Mac is a little below 16 GiB. + expect(memClassGb(16_631_808_000)).toBe(15); + expect(memClassGb(16 * GIB - 100 * MIB)).toBe(16); + }); + + it('reads 0 for anything that is not a positive size', () => { + for (const bytes of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(memClassGb(bytes)).toBe(0); + } + }); +}); diff --git a/test/local-model-hardware.test.ts b/test/local-model-hardware.test.ts new file mode 100644 index 00000000..0bcaa61a --- /dev/null +++ b/test/local-model-hardware.test.ts @@ -0,0 +1,749 @@ +/** + * @fileoverview Tests for src/local-model/hardware.ts: the probe that tells the recommendation + * what this Mac is. + * + * Two halves, mirroring the module: + * + * 1. `parseSysctl()`, pure. Fed the REAL output of the probe command on the two test Macs (a + * MacBook Pro with an M5 Pro and 48 GB, a Mac mini with an M4 and 16 GB), copied here + * character for character with the extra lines the capture holds, plus an Intel-shaped + * output (three of the seven keys do not exist there) and outputs cut off mid-line. + * 2. `readHardware()`, with every piece of IO injected: nothing is spawned and no real disk is + * read. What is pinned: a failed spawn is an ANSWER (the `os-fallback` shape), nothing is + * spawned off macOS, the result is cached for 60 s and concurrent callers share one probe. + * + * The IO itself is tested apart, against things this suite controls: + * + * 3. `spawnSysctl()` runs a small shell script of the test in place of sysctl. That proves what + * a fake cannot: the arguments are the seven keys and nothing else, the output of a program + * that exits 1 is still read (an Intel Mac lacks three keys, so sysctl exits 1 there), a + * program that hangs is given up on, and the server's environment is not handed over. + * 4. `dataDirFreeBytes()` with injected probes: a folder that does not exist yet, a mount that + * does not answer, and a `statfs` that never comes back, which must not be started twice. + * + * One test at the end runs the real defaults once (the real spawn and the real `statfs`), to + * prove they cannot reject on whatever machine runs the suite. On Linux `sysctl` knows none of + * the keys, so that run ends in the fallback; on a Mac it ends in a real answer. Both are fine. + * + * Port: N/A (no server). + */ + +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { isAbsolute, join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + dataDirFreeBytes, + FREE_DISK_TIMEOUT_MS, + freeDiskBytesOf, + HARDWARE_CACHE_TTL_MS, + parseSysctl, + readHardware, + resetHardwareCache, + spawnSysctl, + SYSCTL_KEYS, + SYSCTL_PATH, + SYSCTL_TIMEOUT_MS, + type HardwareProbeDeps, +} from '../src/local-model/hardware.js'; + +/** + * The capture of the MacBook Pro (Mac17,13), 2026-11-06. The last four lines are not part of + * the probe: two extra keys the capture asked for, the shell's exit status and a `df -k` line. + */ +const MACBOOK_SYSCTL = `hw.memsize: 51539607552 +machdep.cpu.brand_string: Apple M5 Pro +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac17,13 +hw.perflevel0.physicalcpu: 6 +hw.perflevel1.physicalcpu: 12 +exit=0 +/dev/disk3s5 971094180 54383564 890926084 6% 542146 8909260840 0% /System/Volumes/Data +`; + +/** The capture of the Mac mini (Mac16,10), 2026-11-06, with the same four extra lines. */ +const MACMINI_SYSCTL = `hw.memsize: 17179869184 +machdep.cpu.brand_string: Apple M4 +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac16,10 +hw.perflevel0.physicalcpu: 4 +hw.perflevel1.physicalcpu: 6 +exit=0 +/dev/disk3s5 239362496 172954124 41752848 81% 2353838 417528480 1% /System/Volumes/Data +`; + +/** Free disk of the two Macs: the "available" column of their `df -k` lines, in bytes. */ +const MACBOOK_FREE_DISK = 890_926_084 * 1024; +const MACMINI_FREE_DISK = 41_752_848 * 1024; + +/** + * What an Intel Mac prints: `hw.optional.arm64`, `sysctl.proc_translated` and + * `iogpu.wired_limit_mb` do not exist there, so sysctl leaves their lines out (and exits 1). + * NOT a capture: no Intel Mac was available, the shape is recalled. + */ +const INTEL_SYSCTL = `hw.memsize: 17179869184 +machdep.cpu.brand_string: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz +kern.osproductversion: 13.6.7 +hw.model: MacBookPro15,1 +`; + +/** Deps that spawn nothing and read no disk. A test overrides what it is about. */ +function deps(overrides: HardwareProbeDeps = {}): HardwareProbeDeps { + return { + platform: 'darwin', + arch: 'arm64', + runSysctl: async () => MACMINI_SYSCTL, + freeDiskBytes: async () => MACMINI_FREE_DISK, + totalMemBytes: () => 8 * 1024 ** 3, + cpuModel: () => 'Fallback CPU', + now: () => 1_000_000, + ...overrides, + }; +} + +beforeEach(() => { + resetHardwareCache(); +}); + +afterEach(() => { + resetHardwareCache(); +}); + +describe('the probe command', () => { + it('is an absolute path and asks for exactly the seven keys, never with -n', () => { + expect(isAbsolute(SYSCTL_PATH)).toBe(true); + expect(SYSCTL_PATH).toBe('/usr/sbin/sysctl'); + expect([...SYSCTL_KEYS]).toEqual([ + 'hw.memsize', + 'machdep.cpu.brand_string', + 'hw.optional.arm64', + 'sysctl.proc_translated', + 'iogpu.wired_limit_mb', + 'kern.osproductversion', + 'hw.model', + ]); + // Every argument is a key name. `-n` would print bare values, and a key missing on a + // machine would then shift every value after it onto the wrong key. + for (const key of SYSCTL_KEYS) expect(key).toMatch(/^[a-z][a-z0-9_.]*$/); + }); + + it('is bounded, and its result is reused for 60 seconds', () => { + expect(SYSCTL_TIMEOUT_MS).toBe(2000); + expect(HARDWARE_CACHE_TTL_MS).toBe(60_000); + }); +}); + +describe('parseSysctl on the two real Macs', () => { + it('reads the MacBook Pro: M5 Pro, 48 GB, macOS 27.2, no wired limit set', () => { + expect(parseSysctl(MACBOOK_SYSCTL)).toEqual({ + memBytes: 51_539_607_552, + chip: 'Apple M5 Pro', + arm64: true, + translated: false, + wiredLimitMb: 0, + macosVersion: '27.2', + macosMajor: 27, + model: 'Mac17,13', + }); + }); + + it('reads the Mac mini: M4, 16 GB, macOS 27.2, no wired limit set', () => { + expect(parseSysctl(MACMINI_SYSCTL)).toEqual({ + memBytes: 17_179_869_184, + chip: 'Apple M4', + arm64: true, + translated: false, + wiredLimitMb: 0, + macosVersion: '27.2', + macosMajor: 27, + model: 'Mac16,10', + }); + }); + + it('skips the lines it does not know: other keys, the exit status, the df line', () => { + const facts = parseSysctl(MACMINI_SYSCTL); + expect(Object.keys(facts).sort()).toEqual( + ['arm64', 'chip', 'macosMajor', 'macosVersion', 'memBytes', 'model', 'translated', 'wiredLimitMb'].sort() + ); + }); + + it('reports a wired limit of 0 as 0 (not set), and a set one as its number', () => { + expect(parseSysctl(MACMINI_SYSCTL).wiredLimitMb).toBe(0); + expect(parseSysctl('iogpu.wired_limit_mb: 14000\n').wiredLimitMb).toBe(14000); + }); +}); + +describe('parseSysctl on other shapes', () => { + it('reads an Intel Mac, whose output lacks three of the keys, without shifting a value', () => { + expect(parseSysctl(INTEL_SYSCTL)).toEqual({ + memBytes: 17_179_869_184, + chip: 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz', + macosVersion: '13.6.7', + macosMajor: 13, + model: 'MacBookPro15,1', + }); + }); + + it('reads an Intel Mac that does print the arm64 flag, as 0', () => { + const facts = parseSysctl(`hw.optional.arm64: 0\n${INTEL_SYSCTL}`); + expect(facts.arm64).toBe(false); + expect(facts.translated).toBeUndefined(); + }); + + it('reads a process under Rosetta: the machine is arm64 and the process is translated', () => { + const facts = parseSysctl(MACMINI_SYSCTL.replace('sysctl.proc_translated: 0', 'sysctl.proc_translated: 1')); + expect(facts.arm64).toBe(true); + expect(facts.translated).toBe(true); + }); + + it('drops a line that was cut off: a truncated output never yields half a value', () => { + // Cut in the middle of the chip name. The memory line before it is whole. + const cutInChip = 'hw.memsize: 17179869184\nmachdep.cpu.brand_string: Apple M'; + expect(parseSysctl(cutInChip)).toEqual({ memBytes: 17_179_869_184 }); + + // Cut in the middle of the memory figure: '1717' would be a valid, wrong number. + expect(parseSysctl('hw.memsize: 1717')).toEqual({}); + + // Cut right after a whole line: everything up to there counts. + const cutAfterLine = MACMINI_SYSCTL.slice(0, MACMINI_SYSCTL.indexOf('hw.optional.arm64')); + expect(parseSysctl(cutAfterLine)).toEqual({ memBytes: 17_179_869_184, chip: 'Apple M4' }); + }); + + it('reads nothing from an empty output or from error text', () => { + expect(parseSysctl('')).toEqual({}); + expect(parseSysctl('\n\n')).toEqual({}); + expect( + parseSysctl("sysctl: unknown oid 'hw.memsize'\nsysctl: cannot stat /proc/sys/hw/memsize: No such file\n") + ).toEqual({}); + }); + + it('leaves out a value that does not parse instead of guessing', () => { + const facts = parseSysctl( + [ + 'hw.memsize: lots', + 'machdep.cpu.brand_string: ', + 'hw.optional.arm64: yes', + 'sysctl.proc_translated: 2', + 'iogpu.wired_limit_mb: -1', + 'kern.osproductversion: Sequoia', + `hw.model: ${'x'.repeat(200)}`, + '', + ].join('\n') + ); + expect(facts).toEqual({}); + }); + + it('refuses a memory size of 0 or one too large to be a number of bytes', () => { + expect(parseSysctl('hw.memsize: 0\n').memBytes).toBeUndefined(); + expect(parseSysctl('hw.memsize: 99999999999999999999999\n').memBytes).toBeUndefined(); + }); + + it('accepts Windows line ends and padding, and the first line of a key wins', () => { + const facts = parseSysctl('hw.memsize: 17179869184 \r\nhw.model: Mac16,10\r\nhw.model: Other9,9\r\n'); + expect(facts).toEqual({ memBytes: 17_179_869_184, model: 'Mac16,10' }); + }); + + it('reads the macOS major version from one, two or three numbers', () => { + expect(parseSysctl('kern.osproductversion: 14\n')).toMatchObject({ macosVersion: '14', macosMajor: 14 }); + expect(parseSysctl('kern.osproductversion: 15.7\n')).toMatchObject({ macosVersion: '15.7', macosMajor: 15 }); + expect(parseSysctl('kern.osproductversion: 13.6.7\n')).toMatchObject({ macosVersion: '13.6.7', macosMajor: 13 }); + }); +}); + +describe('readHardware on the two real Macs', () => { + it('describes the MacBook Pro', async () => { + const hw = await readHardware( + deps({ runSysctl: async () => MACBOOK_SYSCTL, freeDiskBytes: async () => MACBOOK_FREE_DISK }) + ); + expect(hw).toEqual({ + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 51_539_607_552, + memGb: '51.5', + memClassGb: 48, + chip: 'Apple M5 Pro', + model: 'Mac17,13', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: MACBOOK_FREE_DISK, + freeDiskGb: '912', + source: 'sysctl', + }); + }); + + it('describes the Mac mini', async () => { + const hw = await readHardware(deps()); + expect(hw).toEqual({ + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 17_179_869_184, + memGb: '17.2', + memClassGb: 16, + chip: 'Apple M4', + model: 'Mac16,10', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: MACMINI_FREE_DISK, + freeDiskGb: '42.8', + source: 'sysctl', + }); + }); + + it('is plain JSON: it survives a round trip unchanged', async () => { + const hw = await readHardware(deps()); + expect(JSON.parse(JSON.stringify(hw))).toEqual(hw); + }); + + it('describes an Intel Mac from its shorter output: no arm64 flag, an x64 process', async () => { + const hw = await readHardware(deps({ arch: 'x64', runSysctl: async () => INTEL_SYSCTL })); + expect(hw.source).toBe('sysctl'); + expect(hw.arch).toBe('x64'); + expect(hw.chip).toBe('Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'); + expect(hw.macosMajor).toBe(13); + expect(hw).not.toHaveProperty('arm64'); + expect(hw).not.toHaveProperty('translated'); + expect(hw).not.toHaveProperty('wiredLimitMb'); + }); + + it('takes the chip name from node:os when sysctl printed everything but that', async () => { + const hw = await readHardware( + deps({ runSysctl: async () => MACMINI_SYSCTL.replace('machdep.cpu.brand_string: Apple M4\n', '') }) + ); + expect(hw.source).toBe('sysctl'); + expect(hw.chip).toBe('Fallback CPU'); + }); +}); + +describe('readHardware when the probe fails', () => { + it('answers from node:os when the spawn could not run or timed out', async () => { + const hw = await readHardware( + deps({ + runSysctl: async () => undefined, + totalMemBytes: () => 34_359_738_368, + cpuModel: () => 'Apple M2 Max', + }) + ); + expect(hw).toEqual({ + platform: 'darwin', + arch: 'arm64', + memBytes: 34_359_738_368, + memGb: '34.4', + memClassGb: 32, + chip: 'Apple M2 Max', + freeDiskBytes: MACMINI_FREE_DISK, + freeDiskGb: '42.8', + source: 'os-fallback', + }); + }); + + it('never rejects when a dependency throws or rejects', async () => { + const hw = await readHardware( + deps({ + runSysctl: () => Promise.reject(new Error('spawn EACCES')), + freeDiskBytes: () => { + throw new Error('statfs failed'); + }, + cpuModel: () => { + throw new Error('no cpus'); + }, + }) + ); + expect(hw.source).toBe('os-fallback'); + expect(hw.memBytes).toBe(8 * 1024 ** 3); + expect(hw).not.toHaveProperty('chip'); + expect(hw).not.toHaveProperty('freeDiskBytes'); + expect(hw).not.toHaveProperty('freeDiskGb'); + }); + + it('still answers when even the memory size cannot be read', async () => { + const hw = await readHardware( + deps({ + runSysctl: async () => undefined, + totalMemBytes: () => { + throw new Error('no memory figure'); + }, + }) + ); + expect(hw).toMatchObject({ source: 'os-fallback', memBytes: 0, memGb: '0.0', memClassGb: 0 }); + }); + + it('trusts nothing of an output that has no memory size', async () => { + // A truncated or foreign output: the chip and version lines are whole, hw.memsize is not there. + const withoutMemory = MACMINI_SYSCTL.replace('hw.memsize: 17179869184\n', ''); + const hw = await readHardware(deps({ runSysctl: async () => withoutMemory })); + expect(hw.source).toBe('os-fallback'); + expect(hw.memBytes).toBe(8 * 1024 ** 3); + expect(hw.chip).toBe('Fallback CPU'); + expect(hw).not.toHaveProperty('macosVersion'); + expect(hw).not.toHaveProperty('macosMajor'); + expect(hw).not.toHaveProperty('model'); + expect(hw).not.toHaveProperty('wiredLimitMb'); + expect(hw).not.toHaveProperty('arm64'); + }); + + it('falls back when the output was cut off inside the memory figure', async () => { + const hw = await readHardware(deps({ runSysctl: async () => 'hw.memsize: 1717' })); + expect(hw.source).toBe('os-fallback'); + expect(hw.memBytes).toBe(8 * 1024 ** 3); + }); + + it('leaves the free disk out when it is unknown, and keeps the rest', async () => { + const hw = await readHardware(deps({ freeDiskBytes: async () => undefined })); + expect(hw.source).toBe('sysctl'); + expect(hw).not.toHaveProperty('freeDiskBytes'); + expect(hw).not.toHaveProperty('freeDiskGb'); + }); + + it('reports a full disk as 0 bytes free, which is a figure and not "unknown"', async () => { + const hw = await readHardware(deps({ freeDiskBytes: async () => 0 })); + expect(hw.freeDiskBytes).toBe(0); + expect(hw.freeDiskGb).toBe('0.0'); + }); +}); + +describe('readHardware off macOS', () => { + it('spawns nothing and reads no disk', async () => { + let spawned = 0; + let diskRead = 0; + const hw = await readHardware( + deps({ + platform: 'linux', + arch: 'x64', + runSysctl: async () => { + spawned++; + return MACMINI_SYSCTL; + }, + freeDiskBytes: async () => { + diskRead++; + return 1; + }, + totalMemBytes: () => 67_430_023_168, + }) + ); + expect(spawned).toBe(0); + expect(diskRead).toBe(0); + expect(hw).toEqual({ + platform: 'linux', + arch: 'x64', + memBytes: 67_430_023_168, + memGb: '67.4', + memClassGb: 63, + source: 'not-probed', + }); + }); +}); + +describe('the cache of readHardware', () => { + it('reuses one result for 60 seconds, then probes again', async () => { + let clock = 5_000_000; + let spawned = 0; + const counting = deps({ + now: () => clock, + runSysctl: async () => { + spawned++; + return spawned === 1 ? MACMINI_SYSCTL : MACBOOK_SYSCTL; + }, + }); + + const first = await readHardware(counting); + clock += HARDWARE_CACHE_TTL_MS - 1; + const second = await readHardware(counting); + expect(spawned).toBe(1); + expect(second).toBe(first); + + clock += 1; + const third = await readHardware(counting); + expect(spawned).toBe(2); + expect(third.chip).toBe('Apple M5 Pro'); + }); + + it('caches a failed probe too: a broken sysctl is not spawned on every request', async () => { + let spawned = 0; + const failing = deps({ + runSysctl: async () => { + spawned++; + return undefined; + }, + }); + expect((await readHardware(failing)).source).toBe('os-fallback'); + expect((await readHardware(failing)).source).toBe('os-fallback'); + expect(spawned).toBe(1); + }); + + it('lets concurrent callers share one probe', async () => { + let spawned = 0; + let release: (text: string) => void = () => {}; + const slow = deps({ + runSysctl: () => { + spawned++; + return new Promise((resolve) => { + release = resolve; + }); + }, + }); + const callers = [readHardware(slow), readHardware(slow), readHardware(slow)]; + // Let the three calls reach the spawn before it answers. + await new Promise((resolve) => setImmediate(resolve)); + release(MACMINI_SYSCTL); + const results = await Promise.all(callers); + expect(spawned).toBe(1); + expect(results[1]).toBe(results[0]); + expect(results[2]).toBe(results[0]); + }); + + it('probes again after a reset, and does not trust a clock that ran backwards', async () => { + let clock = 9_000_000; + let spawned = 0; + const counting = deps({ + now: () => clock, + runSysctl: async () => { + spawned++; + return MACMINI_SYSCTL; + }, + }); + await readHardware(counting); + resetHardwareCache(); + await readHardware(counting); + expect(spawned).toBe(2); + + clock -= 10_000; + await readHardware(counting); + expect(spawned).toBe(3); + }); +}); + +// The scripts below are POSIX shell. Codeman's server does not run on Windows. +describe.skipIf(process.platform === 'win32')('spawnSysctl, against a script of the test', () => { + let folder: string; + + beforeEach(() => { + folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-spawn-')); + }); + + afterEach(() => { + rmSync(folder, { recursive: true, force: true }); + }); + + /** An executable shell script standing in for sysctl. */ + function script(body: string, mode = 0o755): string { + const path = join(folder, 'fake-sysctl'); + writeFileSync(path, `#!/bin/sh\n${body}\n`, { mode }); + return path; + } + + it('passes the seven keys as arguments and nothing else: no -n, no shell', async () => { + // printf, not echo: echo would swallow a `-n` argument, the very thing this test looks for. + const output = await spawnSysctl(script('printf \'%s\\n\' "$@"')); + expect(output?.trimEnd().split('\n')).toEqual([...SYSCTL_KEYS]); + }); + + it('reads the output of a program that exits 1, as sysctl does on an Intel Mac', async () => { + // Three keys are unknown there: their errors go to stderr, the other four lines to stdout. + const output = await spawnSysctl( + script( + [ + "echo 'hw.memsize: 17179869184'", + "echo 'machdep.cpu.brand_string: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'", + `echo "sysctl: unknown oid 'hw.optional.arm64'" >&2`, + `echo "sysctl: unknown oid 'sysctl.proc_translated'" >&2`, + `echo "sysctl: unknown oid 'iogpu.wired_limit_mb'" >&2`, + "echo 'kern.osproductversion: 13.6.7'", + "echo 'hw.model: MacBookPro15,1'", + 'exit 1', + ].join('\n') + ) + ); + expect(output).toBeDefined(); + expect(output).not.toContain('unknown oid'); + expect(parseSysctl(output as string)).toEqual({ + memBytes: 17_179_869_184, + chip: 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz', + macosVersion: '13.6.7', + macosMajor: 13, + model: 'MacBookPro15,1', + }); + }); + + it('answers undefined for a program that does not exist or may not be run', async () => { + expect(await spawnSysctl(join(folder, 'no-such-program'))).toBeUndefined(); + expect(await spawnSysctl(script('echo "hw.memsize: 1"', 0o644))).toBeUndefined(); + }); + + it('gives up on a program that hangs, and does not hand over half an output', async () => { + const started = Date.now(); + // `exec` so the signal reaches the sleeper itself and nothing is left behind. + const output = await spawnSysctl(script("echo 'hw.memsize: 17179869184'\nexec sleep 20"), 200); + expect(output).toBeUndefined(); + expect(Date.now() - started).toBeLessThan(5000); + }); + + it('answers undefined for an output far larger than seven lines', async () => { + expect(await spawnSysctl(script("head -c 200000 /dev/zero | tr '\\0' 'x'"))).toBeUndefined(); + }); + + it("does not pass the server's environment to the program", async () => { + const name = 'CODEMAN_LOCAL_MODEL_TEST_SECRET'; + process.env[name] = 'hunter2'; + try { + const output = await spawnSysctl(script(`echo "secret=\${${name}:-unset}"`)); + expect(output?.trim()).toBe('secret=unset'); + } finally { + delete process.env[name]; + } + }); +}); + +describe('dataDirFreeBytes', () => { + let folder: string; + + beforeEach(() => { + folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-disk-')); + }); + + afterEach(() => { + rmSync(folder, { recursive: true, force: true }); + }); + + it('reads the free bytes of the data dir with the real probe and the real statfs', async () => { + const free = await dataDirFreeBytes({ dataDir: () => folder }); + expect(typeof free).toBe('number'); + expect(free).toBeGreaterThanOrEqual(0); + }); + + it('uses the nearest existing parent when the data dir does not exist yet', async () => { + const asked: string[] = []; + const free = await dataDirFreeBytes({ + dataDir: () => join(folder, 'not', 'yet', 'there'), + freeBytesOf: async (path) => { + asked.push(path); + return 123; + }, + }); + expect(free).toBe(123); + expect(asked).toEqual([folder]); + }); + + it('does not touch a folder whose mount does not answer, or a path that is a file', async () => { + for (const kind of ['unknown', 'file'] as const) { + let touched = 0; + const free = await dataDirFreeBytes({ + dataDir: () => folder, + probeKind: async () => kind, + freeBytesOf: async () => { + touched++; + return 1; + }, + }); + expect(free, kind).toBeUndefined(); + expect(touched, kind).toBe(0); + } + }); + + it('stops walking up after a few parents', async () => { + let probes = 0; + const free = await dataDirFreeBytes({ + dataDir: () => '/a/b/c/d/e/f/g/h/i/j/k/l', + probeKind: async () => { + probes++; + return 'absent'; + }, + freeBytesOf: async () => 1, + }); + expect(free).toBeUndefined(); + expect(probes).toBe(7); + }); + + it('gives up on a statfs that does not come back, and never starts a second one meanwhile', async () => { + let started = 0; + let finish: (bytes: number) => void = () => {}; + const hanging = { + dataDir: () => folder, + freeBytesOf: () => { + started++; + return new Promise((resolve) => { + finish = resolve; + }); + }, + timeoutMs: 20, + }; + + // The first read waits its timeout and answers "unknown". + expect(await dataDirFreeBytes(hanging)).toBeUndefined(); + expect(started).toBe(1); + + // While that statfs is still out, later reads answer at once and start nothing. + const before = Date.now(); + expect(await dataDirFreeBytes(hanging)).toBeUndefined(); + expect(await dataDirFreeBytes({ ...hanging, timeoutMs: 60_000 })).toBeUndefined(); + expect(Date.now() - before).toBeLessThan(1000); + expect(started).toBe(1); + + // Once it finally answers, the next read works again. + finish(5); + await new Promise((resolve) => setImmediate(resolve)); + expect(await dataDirFreeBytes({ ...hanging, freeBytesOf: async () => 42 })).toBe(42); + }); + + it('never rejects when one of its reads does', async () => { + const failing = async (): Promise => { + throw new Error('EIO'); + }; + expect(await dataDirFreeBytes({ dataDir: () => folder, freeBytesOf: failing })).toBeUndefined(); + expect(await dataDirFreeBytes({ dataDir: () => folder, probeKind: failing })).toBeUndefined(); + expect( + await dataDirFreeBytes({ + dataDir: () => { + throw new Error('no home'); + }, + }) + ).toBeUndefined(); + }); + + it('is bounded by default', () => { + expect(FREE_DISK_TIMEOUT_MS).toBe(2000); + }); +}); + +describe('the real defaults', () => { + let folder: string; + + beforeEach(() => { + folder = mkdtempSync(join(tmpdir(), 'codeman-local-model-hw-')); + }); + + afterEach(() => { + rmSync(folder, { recursive: true, force: true }); + }); + + it('reads the free bytes of a real folder, and undefined for one that does not exist', async () => { + const free = await freeDiskBytesOf(folder); + expect(typeof free).toBe('number'); + expect(free).toBeGreaterThanOrEqual(0); + expect(await freeDiskBytesOf(join(folder, 'missing', 'deeper'))).toBeUndefined(); + }); + + it('never rejects with the real spawn and the real disk read, whatever machine this is', async () => { + // Pretend to be a Mac so the spawn and the disk read really run. On Linux /usr/sbin/sysctl + // either does not exist or knows none of the keys: the answer is then the fallback. + const hw = await readHardware({ platform: 'darwin' }); + expect(['sysctl', 'os-fallback']).toContain(hw.source); + expect(hw.memBytes).toBeGreaterThan(0); + expect(hw.memGb).toMatch(/^\d+(\.\d)?$/); + if (hw.freeDiskBytes !== undefined) { + expect(hw.freeDiskBytes).toBeGreaterThanOrEqual(0); + expect(hw.freeDiskGb).toMatch(/^\d+(\.\d)?$/); + } + }); +}); diff --git a/test/local-model-recommend.test.ts b/test/local-model-recommend.test.ts new file mode 100644 index 00000000..86a2c327 --- /dev/null +++ b/test/local-model-recommend.test.ts @@ -0,0 +1,1298 @@ +/** + * @fileoverview Tests for src/local-model/recommend.ts: the pure function that turns "what this + * Mac is" into "which local model fits it". + * + * Laid out like the module: + * + * 1. The budget (`deviceBudget`) for every memory size Apple sells from 8 to 128 GB, and the + * need of a model at a context (`needBytes`). + * 2. The two REAL Macs of the project, from their captured `sysctl` output through the real + * parser: the MacBook Pro (M5 Pro, 48 GB) and the Mac mini (M4, 16 GB). Each test states the + * expected answer and why. + * 3. Every reason for verdict `none`, one fixture per reason, and their order. + * 4. The `limited` verdict, the three-state speed in the ranking, harness rows (a gated one is + * never chosen), the `tested` flag that alone may turn on the word "Recommended", and the + * shape of the answer (labels from the fixed list, sizes in one unit). + * + * The stock catalogue is used wherever the answer for a real Mac is at stake. Small synthetic + * catalogues (validated by the real schema) isolate one rule of the ranking at a time. + * + * Port: N/A (pure functions; the two hardware fixtures go through `readHardware` with every + * piece of IO injected). + */ + +import { beforeEach, describe, expect, it } from 'vitest'; +import { + getLocalModelCatalog, + LocalModelCatalogSchema, + STOCK_LOCAL_MODEL_CATALOG, +} from '../src/config/local-model-catalog.js'; +import { formatGb, GB, GIB, memClassGb, MIB } from '../src/local-model/format.js'; +import { readHardware, resetHardwareCache } from '../src/local-model/hardware.js'; +import { + deviceBudget, + DISK_HEADROOM_BYTES, + FIT_GOOD_MAX_SHARE, + FIT_TIGHT_MAX_SHARE, + LIMITED_BELOW_PARAMS_B, + MAX_ALTERNATIVES, + needBytes, + recommend, + SLOW_BELOW_TOKENS_PER_SECOND, +} from '../src/local-model/recommend.js'; +import { + LOCAL_MODEL_LABELS, + type Candidate, + type HardwareFacts, + type HarnessFact, + type LocalHarnessEntry, + type LocalModelCatalog, + type LocalModelEntry, + type Recommendation, +} from '../src/types.js'; + +// ---------- Fixtures ---------- + +/** The real probe output of the MacBook Pro (Mac17,13), captured 2026-11-06, extra lines included. */ +const MACBOOK_SYSCTL = `hw.memsize: 51539607552 +machdep.cpu.brand_string: Apple M5 Pro +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac17,13 +hw.perflevel0.physicalcpu: 6 +hw.perflevel1.physicalcpu: 12 +exit=0 +/dev/disk3s5 971094180 54383564 890926084 6% 542146 8909260840 0% /System/Volumes/Data +`; + +/** The real probe output of the Mac mini (Mac16,10), captured 2026-11-06, extra lines included. */ +const MACMINI_SYSCTL = `hw.memsize: 17179869184 +machdep.cpu.brand_string: Apple M4 +hw.optional.arm64: 1 +sysctl.proc_translated: 0 +iogpu.wired_limit_mb: 0 +kern.osproductversion: 27.2 +hw.model: Mac16,10 +hw.perflevel0.physicalcpu: 4 +hw.perflevel1.physicalcpu: 6 +exit=0 +/dev/disk3s5 239362496 172954124 41752848 81% 2353838 417528480 1% /System/Volumes/Data +`; + +/** The hardware facts of a real Mac: its captured output through the real parser, no IO. */ +async function realMac(sysctlOutput: string, freeDiskKb: number): Promise { + resetHardwareCache(); + return readHardware({ + platform: 'darwin', + arch: 'arm64', + runSysctl: async () => sysctlOutput, + freeDiskBytes: async () => freeDiskKb * 1024, + totalMemBytes: () => 0, + cpuModel: () => undefined, + now: () => 0, + }); +} + +/** An Apple silicon Mac with `soldGb` of memory, healthy in every other respect. */ +function mac(soldGb: number, chip: string, overrides: Partial = {}): HardwareFacts { + const memBytes = soldGb * GIB; + return { + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes, + memGb: formatGb(memBytes), + memClassGb: memClassGb(memBytes), + chip, + model: 'Mac0,0', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: 500 * GB, + freeDiskGb: '500', + source: 'sysctl', + ...overrides, + }; +} + +/** A copy of `facts` without the given fields (an absent field, not an `undefined` one). */ +function without(facts: HardwareFacts, ...fields: (keyof HardwareFacts)[]): HardwareFacts { + const copy: Record = { ...facts }; + for (const field of fields) delete copy[field]; + return copy as unknown as HardwareFacts; +} + +/** Display labels of the stock harnesses, as the CLI registry would supply them. Data, not a branch. */ +const HARNESS_LABELS: Record = { opencode: 'OpenCode', claude: 'Claude Code', pi: 'Pi' }; + +/** What the server knows about one harness: installed and accepting every model id, unless overridden. */ +function fact(cli: string, overrides: Partial = {}): HarnessFact { + return { + cli, + label: HARNESS_LABELS[cli] ?? cli, + installed: true, + installable: false, + acceptsModelId: () => true, + ...overrides, + }; +} + +const STOCK = getLocalModelCatalog(); + +/** Every stock harness installed, the gated one included. */ +const ALL_INSTALLED: HarnessFact[] = STOCK.harnesses.map((row) => fact(row.cli)); + +/** The stock catalogue with a change applied, validated by the real schema. */ +function stockWith(change: (catalog: LocalModelCatalog) => void): LocalModelCatalog { + const catalog = structuredClone(STOCK_LOCAL_MODEL_CATALOG); + change(catalog); + return LocalModelCatalogSchema.parse(catalog); +} + +/** A model row for a synthetic catalogue: 20 billion parameters, 10 GiB, one small context. */ +function model(id: string, overrides: Partial = {}): LocalModelEntry { + return { + id, + label: id, + license: 'Apache-2.0', + paramsB: 20, + bytes: 10 * GIB, + kvBytesPerToken: 1, + contexts: [1024], + rank: 1, + tested: [], + ...overrides, + }; +} + +/** + * A synthetic catalogue: no prompt cache, one harness `alpha` that works from 1024 tokens, and + * one chip row ('M4 Max' at 400 GB/s, so tokens per second = `speedPer100Gbps` x 4). + */ +function catalogOf(models: LocalModelEntry[], harnesses?: LocalHarnessEntry[]): LocalModelCatalog { + return LocalModelCatalogSchema.parse({ + schema: 1, + updated: '2026-11-06', + minMacosMajor: 14, + promptCacheTiers: [{ minMemGb: 0, bytes: 0 }], + models, + harnesses: harnesses ?? [{ cli: 'alpha', minContext: 1024, rank: 1 }], + chips: [{ match: 'M4 Max', gbps: 400 }], + }); +} + +/** A Mac whose budget is exactly 100 GiB: a wired limit of 101 GiB on a 128 GB machine. */ +const HUNDRED_GIB_MAC = mac(128, 'Apple M4 Max', { wiredLimitMb: 101 * 1024 }); +const HUNDRED_GIB = 100 * GIB; + +/** What a synthetic model needs beside its weights at its 1024-token context: 1 GiB plus the KV cache. */ +const SYNTHETIC_OVERHEAD = GIB + 1024; + +/** A model whose need is exactly `need` bytes in a synthetic catalogue. */ +function modelNeeding(id: string, need: number, overrides: Partial = {}): LocalModelEntry { + return model(id, { bytes: need - SYNTHETIC_OVERHEAD, ...overrides }); +} + +/** `id@context` of the pick and of each alternative, in order. */ +function order(result: Recommendation): string[] { + return [result.pick, ...result.alternatives] + .filter((candidate): candidate is Candidate => candidate !== undefined) + .map((candidate) => `${candidate.modelId}@${candidate.context}`); +} + +/** Every string anywhere in a value. */ +function stringsIn(value: unknown): string[] { + if (typeof value === 'string') return [value]; + if (Array.isArray(value)) return value.flatMap(stringsIn); + if (value && typeof value === 'object') return Object.values(value).flatMap(stringsIn); + return []; +} + +function deepFreeze(value: T): T { + if (value && typeof value === 'object') { + for (const inner of Object.values(value)) deepFreeze(inner); + Object.freeze(value); + } + return value; +} + +/** The Macs the sweeps run over: every memory size, with a slow, a fast and an unknown chip. */ +const SWEEP: HardwareFacts[] = [8, 16, 18, 24, 32, 36, 48, 64, 96, 128].flatMap((soldGb) => + ['Apple M4', 'Apple M4 Max', 'Apple M5 Ultra', 'Apple M6'].map((chip) => mac(soldGb, chip)) +); + +beforeEach(() => { + resetHardwareCache(); +}); + +// ---------- 1. Budget and need ---------- + +describe('deviceBudget', () => { + // Sold size, device limit L and budget B in GiB. L is 2/3 of memory below 36 GB and 3/4 from + // 36 GB; B = min(L - 1, memory - reserve). Only at 8 GB is the reserve the smaller bound. + const TIERS: [soldGb: number, limitGib: number, budgetGib: number][] = [ + [8, 16 / 3, 2], + [16, 32 / 3, 29 / 3], + [18, 12, 11], + [24, 16, 15], + [32, 64 / 3, 61 / 3], + [36, 27, 26], + [48, 36, 35], + [64, 48, 47], + [96, 72, 71], + [128, 96, 95], + ]; + + it.each(TIERS)('gives a %i GB Mac a limit of %f GiB and a budget of %f GiB', (soldGb, limitGib, budgetGib) => { + const budget = deviceBudget(mac(soldGb, 'Apple M4')); + // Whole bytes: a third of a byte is rounded down. + expect(Math.abs(budget.limitBytes - limitGib * GIB)).toBeLessThan(1); + expect(Math.abs(budget.budgetBytes - budgetGib * GIB)).toBeLessThan(1); + expect(Number.isInteger(budget.limitBytes)).toBe(true); + expect(Number.isInteger(budget.budgetBytes)).toBe(true); + expect(budget.source).toBe('estimated'); + }); + + it('matches the rounded budgets of the design table', () => { + const rounded = TIERS.map(([soldGb]) => [ + soldGb, + (deviceBudget(mac(soldGb, 'Apple M4')).budgetBytes / GIB).toFixed(1), + ]); + expect(rounded).toEqual([ + [8, '2.0'], + [16, '9.7'], + [18, '11.0'], + [24, '15.0'], + [32, '20.3'], + [36, '26.0'], + [48, '35.0'], + [64, '47.0'], + [96, '71.0'], + [128, '95.0'], + ]); + }); + + it('keeps a reserve of 6 GiB up to 18 GB, 8 GiB up to 48 GB and 12 GiB from 64 GB', () => { + // A wired limit far above the memory leaves "memory - reserve" as the only bound. + const reserveOf = (soldGb: number): number => { + const budget = deviceBudget(mac(soldGb, 'Apple M4', { wiredLimitMb: 1024 * 1024 })); + return (soldGb * GIB - budget.budgetBytes) / GIB; + }; + expect([8, 16, 18].map(reserveOf)).toEqual([6, 6, 6]); + expect([24, 32, 36, 48].map(reserveOf)).toEqual([8, 8, 8, 8]); + expect([64, 96, 128].map(reserveOf)).toEqual([12, 12, 12]); + }); + + it('uses the wired limit when it is above 0', () => { + // 14000 MiB on the 16 GB Mac: L - 1 GiB is 12.67 GiB, memory - reserve is 10 GiB. + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 14000 }))).toEqual({ + limitBytes: 14000 * MIB, + budgetBytes: 10 * GIB, + source: 'wired-limit', + }); + // A limit BELOW the estimate is honoured too: 8192 MiB leaves 7 GiB. + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 8192 }))).toEqual({ + limitBytes: 8 * GIB, + budgetBytes: 7 * GIB, + source: 'wired-limit', + }); + }); + + it('reads a wired limit of 0, or none at all, as "not set"', () => { + const estimated = deviceBudget(mac(16, 'Apple M4')); + expect(estimated.source).toBe('estimated'); + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 0 }))).toEqual(estimated); + expect(deviceBudget(without(mac(16, 'Apple M4'), 'wiredLimitMb'))).toEqual(estimated); + }); + + it('never goes below 0', () => { + expect(deviceBudget(mac(4, 'Apple M1')).budgetBytes).toBe(0); + expect(deviceBudget(mac(16, 'Apple M4', { wiredLimitMb: 512 })).budgetBytes).toBe(0); + expect(deviceBudget({ ...mac(16, 'Apple M4'), memBytes: 0, memClassGb: 0 })).toEqual({ + limitBytes: 0, + budgetBytes: 0, + source: 'estimated', + }); + }); +}); + +describe('needBytes', () => { + const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; + + it('adds weights, the KV cache of the context, 1 GiB and the prompt cache', () => { + // 6,550,825,373 + 32768 x 65536 + 1 GiB + 512 MiB on a Mac below 32 GB. + expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + 2 * GIB + GIB + 512 * MIB); + expect(needBytes(nineB, 65536, STOCK, mac(16, 'Apple M4'))).toBe(10_308_921_757); + // Half the context, half the KV cache. + expect(needBytes(nineB, 32768, STOCK, mac(16, 'Apple M4'))).toBe(6_550_825_373 + GIB + GIB + 512 * MIB); + }); + + it('counts a prompt cache of 512 MiB below 32 GB of memory and 2 GiB from 32 GB', () => { + const base = 6_550_825_373 + 2 * GIB + GIB; + for (const soldGb of [8, 16, 18, 24]) { + expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(512 * MIB); + } + for (const soldGb of [32, 36, 48, 64, 96, 128]) { + expect(needBytes(nineB, 65536, STOCK, mac(soldGb, 'Apple M4')) - base, `${soldGb} GB`).toBe(2 * GIB); + } + }); + + it('does not depend on the chip or the harness', () => { + expect(needBytes(nineB, 65536, STOCK, mac(48, 'Apple M1'))).toBe( + needBytes(nineB, 65536, STOCK, mac(48, 'Apple M5 Ultra')) + ); + }); +}); + +describe('the fit grade', () => { + const pickOf = (need: number): Candidate | undefined => + recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', need)]), [fact('alpha')]).pick; + + it('has the budget this group assumes', () => { + expect(deviceBudget(HUNDRED_GIB_MAC).budgetBytes).toBe(HUNDRED_GIB); + expect(FIT_GOOD_MAX_SHARE).toBe(0.85); + expect(FIT_TIGHT_MAX_SHARE).toBe(0.98); + }); + + it('is good up to 0.85 of the budget', () => { + expect(pickOf(0.5 * HUNDRED_GIB)).toMatchObject({ fit: 'good', fitLabel: 'Good', budgetShare: 0.5 }); + expect(pickOf(0.85 * HUNDRED_GIB)).toMatchObject({ fit: 'good', budgetShare: 0.85 }); + }); + + it('is tight above 0.85 and up to 0.98', () => { + expect(pickOf(0.85 * HUNDRED_GIB + 1)).toMatchObject({ fit: 'tight', fitLabel: 'Tight' }); + expect(pickOf(0.9 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.9 }); + expect(pickOf(0.98 * HUNDRED_GIB)).toMatchObject({ fit: 'tight', budgetShare: 0.98 }); + }); + + it('is out above 0.98: the model is not offered at all', () => { + const result = recommend(HUNDRED_GIB_MAC, catalogOf([modelNeeding('probe:1', 0.98 * HUNDRED_GIB + 1)]), [ + fact('alpha'), + ]); + expect(result.verdict).toBe('none'); + expect(result.reason).toBe('memory'); + expect(result.pick).toBeUndefined(); + }); +}); + +// ---------- 2. The two real Macs ---------- + +describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { + /** + * Expected: Qwen3.6 35B-A3B at 131,072 tokens with OpenCode, Good, about 42 tokens per second. + * + * Why. The budget is 35 GiB (3/4 of 48 GiB minus 1). Five models fit it as Good. The M5 Pro is + * in the chips table (307 GB/s), so each model with a speed figure gets an estimate: + * the dense 27B computes to 2.95 x 3.07 = 9.1 tokens per second, under the 10 that separate + * slow from ok, while the 35B-A3B (3 billion parameters active per token) computes to 41.8. + * Speed ranks before model rank, so the 35B-A3B wins although the 27B has the higher rank. The + * 27B stays in the list, last, marked Slow. Both harnesses can run it; OpenCode has the + * higher rank. + */ + it('gets Qwen3.6 35B-A3B at 131072 tokens with OpenCode', async () => { + const hw = await realMac(MACBOOK_SYSCTL, 890_926_084); + expect(hw.chip).toBe('Apple M5 Pro'); + expect(hw.memClassGb).toBe(48); + + const result = recommend(hw, STOCK, ALL_INSTALLED); + + expect(result.verdict).toBe('ok'); + expect(result.reason).toBeUndefined(); + expect(result.budgetBytes).toBe(35 * GIB); + expect(result.budgetGb).toBe('37.6'); + expect(result.budgetSource).toBe('estimated'); + expect(result.pick).toEqual({ + modelId: 'qwen3.6:35b-a3b', + modelLabel: 'Qwen3.6 35B-A3B', + license: 'Apache-2.0', + paramsB: 35, + context: 131072, + cli: 'opencode', + cliLabel: 'OpenCode', + cliInstalled: true, + // 22,621,314,161 + 20480 x 131072 + 1 GiB + 2 GiB. + needBytes: 28_526_894_193, + needGb: '28.5', + downloadBytes: 22_621_314_161, + downloadGb: '22.6', + budgetShare: 0.759, + fit: 'good', + fitLabel: 'Good', + speed: 'ok', + tokensPerSecond: 41.8, + tested: false, + }); + }); + + it('says "Best fit", not "Recommended", and that nobody tested it on such a Mac', async () => { + const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + }); + + it('lists the other four models that fit: the fast ones, the one of unknown speed, the slow one', async () => { + const result = recommend(await realMac(MACBOOK_SYSCTL, 890_926_084), STOCK, ALL_INSTALLED); + expect(result.alternatives.map((c) => [c.modelId, c.context, c.fit, c.speed, c.tokensPerSecond ?? null])).toEqual([ + ['qwen3.5:9b-q8_0', 65536, 'good', 'ok', 17.8], + ['qwen3.5:9b', 65536, 'good', 'ok', 28.6], + ['gpt-oss:20b', 131072, 'good', 'unknown', null], + ['qwen3.8:27b', 131072, 'good', 'slow', 9.1], + ]); + expect(result.alternatives[2].speedLabel).toBe('Speed unknown'); + expect(result.alternatives[3].speedLabel).toBe('Slow'); + // The two that do not fit 35 GiB at any context are not listed. + expect(order(result)).not.toContain('qwen3.8:27b-q8_0@131072'); + expect(order(result).some((entry) => entry.startsWith('qwen3.5:122b'))).toBe(false); + }); + + it('would get the 27B instead if its chip were in no chips row', async () => { + // The same Mac with the M5 Pro row removed: every speed is unknown, so model rank decides. + const hw = await realMac(MACBOOK_SYSCTL, 890_926_084); + const withoutChip = stockWith((catalog) => { + catalog.chips = catalog.chips.filter((row) => row.match !== 'M5 Pro'); + }); + const result = recommend(hw, withoutChip, ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', context: 131072, speed: 'unknown' }); + expect(result.pick).not.toHaveProperty('tokensPerSecond'); + }); +}); + +describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { + /** + * Expected: no recommendation, reason `memory`. + * + * Why. The budget is 9.67 GiB (2/3 of 16 GiB minus 1). The only model near it is the 9B. Both + * harnesses that may be offered need 65,536 tokens of context, and there the 9B needs + * 6.10 GiB of weights + 2 GiB of KV cache + 1 GiB + 0.5 GiB of prompt cache = 9.60 GiB, which + * is 0.993 of the budget: above the 0.98 where "tight" ends. At 32,768 tokens it would fit + * (0.890, tight), but the one harness that works at that context is gated. + */ + it('gets no recommendation: nothing fits at the context a harness needs', async () => { + const hw = await realMac(MACMINI_SYSCTL, 41_752_848); + expect(hw.chip).toBe('Apple M4'); + expect(hw.memClassGb).toBe(16); + + const result = recommend(hw, STOCK, ALL_INSTALLED); + + expect(result).toEqual({ + verdict: 'none', + reason: 'memory', + headline: { + text: 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', + vars: { memory: '17.2' }, + }, + notes: [], + alternatives: [], + // floor(17,179,869,184 x 2/3) - 1 GiB. + budgetBytes: 10_379_504_298, + budgetGb: '10.4', + budgetSource: 'estimated', + }); + }); + + it('misses by a hair at 65536 tokens and would fit at 32768', async () => { + const hw = await realMac(MACMINI_SYSCTL, 41_752_848); + const nineB = STOCK.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; + const budget = deviceBudget(hw).budgetBytes; + const shareAt64k = needBytes(nineB, 65536, STOCK, hw) / budget; + const shareAt32k = needBytes(nineB, 32768, STOCK, hw) / budget; + expect(shareAt64k).toBeCloseTo(0.9932, 4); + expect(shareAt64k).toBeGreaterThan(FIT_TIGHT_MAX_SHARE); + expect(shareAt32k).toBeCloseTo(0.8898, 4); + expect(shareAt32k).toBeLessThan(FIT_TIGHT_MAX_SHARE); + // Every other model needs more than the whole budget at its smallest context. + for (const other of STOCK.models.filter((m) => m.id !== 'qwen3.5:9b')) { + const smallest = other.contexts[other.contexts.length - 1]; + expect(needBytes(other, smallest, STOCK, hw), other.id).toBeGreaterThan(budget); + } + }); + + it('would get the 9B at 32768 tokens, as a limited answer, once the gated harness is opened', async () => { + const hw = await realMac(MACMINI_SYSCTL, 41_752_848); + const ungated = stockWith((catalog) => { + catalog.harnesses = catalog.harnesses.map((row) => ({ + cli: row.cli, + minContext: row.minContext, + rank: row.rank, + })); + }); + const result = recommend(hw, ungated, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.pick).toMatchObject({ + modelId: 'qwen3.5:9b', + context: 32768, + cli: 'pi', + fit: 'tight', + budgetShare: 0.89, + speed: 'ok', + // 9.3 x 1.2. + tokensPerSecond: 11.2, + }); + }); + + it('would get the 9B at 65536 tokens if a wired limit had been set on it', async () => { + // iogpu.wired_limit_mb = 14000 lifts the budget to 10 GiB (memory minus the 6 GiB reserve). + const hw = { ...(await realMac(MACMINI_SYSCTL, 41_752_848)), wiredLimitMb: 14000 }; + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.budgetSource).toBe('wired-limit'); + expect(result.budgetBytes).toBe(10 * GIB); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', context: 65536, cli: 'opencode', fit: 'tight' }); + }); +}); + +describe('every memory size, with the stock catalogue', () => { + // What the rules compute today. A change of a catalogue figure or of a threshold that moves + // one of these answers has to change this table on purpose. + const EXPECTED: [soldGb: number, chip: string, answer: string][] = [ + [8, 'Apple M2', 'none memory'], + [16, 'Apple M4', 'none memory'], + // The 9B at 65536 tokens is 0.87 of 11 GiB. + [18, 'Apple M3 Pro', 'limited qwen3.5:9b@65536 tight ok'], + // gpt-oss:20b at 65536 tokens needs 15.8 GiB of the 15 GiB budget: out. + [24, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], + // gpt-oss:20b fits only as Tight here (0.853 at 65536 tokens, 0.927 at 131072), and Good + // ranks before Tight. The Q8 9B is Good but slow on a base M4 (7 tokens per second). + [32, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], + [32, 'Apple M1 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + // The 27B (0.905) and the 35B-A3B (0.974) are Tight; gpt-oss:20b is Good but has no speed + // figure, and a known speed ranks before an unknown one. + [36, 'Apple M4 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + // On a Pro chip the dense 27B is slow (8.1 tokens per second), the 35B-A3B is not. + [48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'], + [48, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], + // The Q8 27B fits 47 GiB but computes to 7 tokens per second on this chip. + [64, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], + [96, 'Apple M3 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], + // The 122B fits 95 GiB only as Tight (0.861) and has no speed figure. + [128, 'Apple M5 Ultra', 'ok qwen3.8:27b-q8_0@262144 good ok'], + ]; + + it.each(EXPECTED)('%i GB, %s: %s', (soldGb, chip, answer) => { + const result = recommend(mac(soldGb, chip), STOCK, ALL_INSTALLED); + const actual = result.pick + ? `${result.verdict} ${result.pick.modelId}@${result.pick.context} ${result.pick.fit} ${result.pick.speed}` + : `${result.verdict} ${result.reason}`; + expect(actual).toBe(answer); + }); + + it('always pairs the pick with the top-ranked harness, OpenCode', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + if (result.pick) expect(result.pick.cli).toBe('opencode'); + } + }); +}); + +// ---------- 3. Verdict none ---------- + +describe('verdict none', () => { + const bigMac = mac(48, 'Apple M4 Max'); + + /** A `none` answer has no pick, no alternatives, no notes, and the reason's sentence as headline. */ + function expectNone(result: Recommendation, reason: Recommendation['reason'], headline: string): void { + expect(result.verdict).toBe('none'); + expect(result.reason).toBe(reason); + expect(result.headline.text).toBe(headline); + expect(result.pick).toBeUndefined(); + expect(result.alternatives).toEqual([]); + expect(result.notes).toEqual([]); + } + + describe('not-macos', () => { + it('answers so on Linux, with no budget at all', () => { + const linux: HardwareFacts = { + platform: 'linux', + arch: 'x64', + memBytes: 64 * GIB, + memGb: formatGb(64 * GIB), + memClassGb: 64, + source: 'not-probed', + }; + const result = recommend(linux, STOCK, ALL_INSTALLED); + expectNone(result, 'not-macos', LOCAL_MODEL_LABELS.noneNotMacos); + expect(result.headline.vars).toBeUndefined(); + expect(result.budgetBytes).toBe(0); + expect(result.budgetGb).toBe('0.0'); + expect(result.budgetSource).toBe('estimated'); + }); + + it('comes before every other reason', () => { + for (const platform of ['linux', 'win32', 'freebsd']) { + const hw = mac(8, 'Intel(R) Core(TM) i7', { platform, arch: 'x64', arm64: false, macosMajor: 12 }); + expect(recommend(hw, STOCK, []).reason, platform).toBe('not-macos'); + } + }); + }); + + describe('intel', () => { + const intelChip = 'Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz'; + + it('answers so for an Intel Mac, whose sysctl has no arm64 flag', () => { + const hw = without(mac(32, intelChip, { arch: 'x64' }), 'arm64', 'translated', 'wiredLimitMb'); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expectNone(result, 'intel', 'This Mac has an Intel processor. Local coding models need Apple silicon.'); + expect(result.budgetBytes).toBe(0); + expect(result.budgetGb).toBe('0.0'); + }); + + it('answers so when the arm64 flag reads 0', () => { + const hw = without(mac(32, intelChip, { arch: 'x64', arm64: false }), 'translated'); + expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('intel'); + }); + + it('comes before the macOS version and the memory', () => { + const hw = without(mac(8, intelChip, { arch: 'x64', macosMajor: 12, macosVersion: '12.7' }), 'arm64'); + expect(recommend(hw, STOCK, []).reason).toBe('intel'); + }); + + it('does not mistake Node under Rosetta for an Intel Mac', () => { + // An x64 process on Apple silicon: the machine flag says arm64, the process is translated. + const rosetta = mac(48, 'Apple M4 Max', { arch: 'x64', translated: true }); + expect(recommend(rosetta, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + // The same when only the translation flag was read. + expect(recommend(without(rosetta, 'arm64'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + }); + + it('decides from node:os alone when the probe failed', () => { + const fallback = (arch: string, chip: string): HardwareFacts => ({ + platform: 'darwin', + arch, + memBytes: 48 * GIB, + memGb: formatGb(48 * GIB), + memClassGb: 48, + chip, + source: 'os-fallback', + }); + // An arm64 process can only run on Apple silicon. + expect(recommend(fallback('arm64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + // An x64 process with an Apple chip name is Rosetta. + expect(recommend(fallback('x64', 'Apple M4 Max'), STOCK, ALL_INSTALLED).verdict).toBe('ok'); + // An x64 process with any other chip name is taken for an Intel Mac. + expect(recommend(fallback('x64', intelChip), STOCK, ALL_INSTALLED).reason).toBe('intel'); + expect(recommend(without(fallback('x64', intelChip), 'chip'), STOCK, ALL_INSTALLED).reason).toBe('intel'); + }); + }); + + describe('macos', () => { + it('answers so below the catalogue minimum, naming both versions', () => { + const result = recommend( + mac(48, 'Apple M1 Max', { macosVersion: '13.6.7', macosMajor: 13 }), + STOCK, + ALL_INSTALLED + ); + expectNone(result, 'macos', 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.'); + expect(result.headline.vars).toEqual({ version: '13.6.7', minimum: '14' }); + // The budget is still reported: the Mac itself is able. + expect(result.budgetBytes).toBe(35 * GIB); + }); + + it('accepts the minimum itself and anything newer', () => { + for (const macosMajor of [14, 15, 26, 27]) { + const hw = mac(48, 'Apple M4 Max', { macosMajor, macosVersion: `${macosMajor}.0` }); + expect(recommend(hw, STOCK, ALL_INSTALLED).verdict, String(macosMajor)).toBe('ok'); + } + }); + + it('does not block on a version that could not be read', () => { + const hw = without(mac(48, 'Apple M4 Max'), 'macosMajor', 'macosVersion'); + expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + }); + + it('comes before the memory', () => { + const hw = mac(8, 'Apple M1', { macosVersion: '12.7', macosMajor: 12 }); + expect(recommend(hw, STOCK, ALL_INSTALLED).reason).toBe('macos'); + }); + }); + + describe('memory', () => { + it('answers so at 8 GB, where nothing fits at any context', () => { + const result = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); + expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemory); + // The size is the decimal figure of the hardware line, without the unit. + expect(result.headline.vars).toEqual({ memory: '8.6' }); + expect(result.budgetBytes).toBe(2 * GIB); + expect(result.budgetGb).toBe('2.1'); + }); + + it('answers so when models fit, but only below the context a harness needs', () => { + // The 16 GB case of the Mac mini, with harnesses that are all present. + const result = recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED); + expect(result.reason).toBe('memory'); + expect(result.headline.vars).toEqual({ memory: '17.2' }); + }); + + it('comes before no-harness when nothing fits at all', () => { + expect(recommend(mac(8, 'Apple M2'), STOCK, []).reason).toBe('memory'); + }); + + it('answers so for an empty catalogue', () => { + expect(recommend(bigMac, catalogOf([]), [fact('alpha')]).reason).toBe('memory'); + }); + + it('answers so when the memory size is unknown', () => { + const hw = { ...mac(48, 'Apple M4 Max'), memBytes: 0, memGb: '0.0', memClassGb: 0 }; + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.reason).toBe('memory'); + expect(result.budgetBytes).toBe(0); + }); + }); + + describe('no-harness', () => { + const headline = 'No coding agent that Codeman can connect to a local model is available on this Mac.'; + + it('answers so when the caller knows no catalogue harness', () => { + const result = recommend(bigMac, STOCK, []); + expectNone(result, 'no-harness', headline); + expect(result.headline.vars).toBeUndefined(); + expect(result.budgetBytes).toBe(35 * GIB); + }); + + it('answers so when no harness is installed or installable', () => { + const facts = STOCK.harnesses.map((row) => fact(row.cli, { installed: false, installable: false })); + expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); + }); + + it('answers so when only the gated harness is on the machine', () => { + const gatedRows = STOCK.harnesses.filter((row) => row.gated); + expect(gatedRows).toHaveLength(1); + expect( + recommend( + bigMac, + STOCK, + gatedRows.map((row) => fact(row.cli)) + ).reason + ).toBe('no-harness'); + }); + + it('answers so when a harness is there but accepts no model id', () => { + const facts = ALL_INSTALLED.map((entry) => ({ ...entry, acceptsModelId: () => false })); + expect(recommend(bigMac, STOCK, facts).reason).toBe('no-harness'); + }); + + it('ignores a fact for a CLI that is not a catalogue harness', () => { + expect(recommend(bigMac, STOCK, [fact('codex'), fact('some-other-cli')]).reason).toBe('no-harness'); + }); + }); + + describe('disk', () => { + it('answers so when every fitting model would leave under 10 GiB free', () => { + // 12 GiB free: even the smallest download (6.55 GB) leaves 5.9 GiB. + const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expectNone( + result, + 'disk', + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.' + ); + // download + 10 GiB - free = 6,550,825,373 + 10,737,418,240 - 12,884,901,888 bytes. + expect(result.headline.vars).toEqual({ download: '6.6', free: '12.9', missing: '4.4' }); + expect(result.budgetBytes).toBe(35 * GIB); + }); + + it('names the smallest model that fits the MEMORY, not the smallest of the catalogue', () => { + // On a 48 GB Mac whose harness refuses both 9B ids, the smallest candidate is gpt-oss:20b. + const facts = ALL_INSTALLED.map((entry) => ({ + ...entry, + acceptsModelId: (id: string) => !id.startsWith('qwen3.5:9b'), + })); + const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 12 * GIB, freeDiskGb: formatGb(12 * GIB) }); + const result = recommend(hw, STOCK, facts); + expect(result.reason).toBe('disk'); + expect(result.headline.vars?.download).toBe('13.8'); + }); + + it('drops only the models that do not leave 10 GiB, and picks among the rest', () => { + // 20 GiB free: the 9B downloads (6.1 and 9.7 GiB) leave 10 GiB or more, the others do not. + const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 20 * GIB, freeDiskGb: formatGb(20 * GIB) }); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(order(result)).toEqual(['qwen3.5:9b-q8_0@65536', 'qwen3.5:9b@65536']); + }); + + it('allows a download that leaves exactly 10 GiB', () => { + const exact = STOCK.models[0].bytes + DISK_HEADROOM_BYTES; + expect(DISK_HEADROOM_BYTES).toBe(10 * GIB); + const fits = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact }), STOCK, ALL_INSTALLED); + expect(fits.pick?.modelId).toBe('qwen3.5:9b'); + const short = recommend(mac(24, 'Apple M4', { freeDiskBytes: exact - 1 }), STOCK, ALL_INSTALLED); + expect(short.reason).toBe('disk'); + }); + + it('does not block when the free disk is unknown', () => { + const hw = without(mac(48, 'Apple M4 Max'), 'freeDiskBytes', 'freeDiskGb'); + expect(recommend(hw, STOCK, ALL_INSTALLED).verdict).toBe('ok'); + }); + + it('formats the free size itself when the hardware carries only the bytes', () => { + const hw = without(mac(48, 'Apple M4 Max', { freeDiskBytes: 3 * GB }), 'freeDiskGb'); + expect(recommend(hw, STOCK, ALL_INSTALLED).headline.vars?.free).toBe('3.0'); + }); + + it('comes after the memory and the harness', () => { + const fullDisk = { freeDiskBytes: 0, freeDiskGb: '0.0' }; + expect(recommend(mac(16, 'Apple M4', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('memory'); + expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, []).reason).toBe('no-harness'); + expect(recommend(mac(48, 'Apple M4 Max', fullDisk), STOCK, ALL_INSTALLED).reason).toBe('disk'); + }); + }); +}); + +// ---------- 4. Verdicts, ranking, harnesses, tested ---------- + +describe('verdict limited', () => { + it('is the answer when the best model that fits has fewer than 12 billion parameters', () => { + expect(LIMITED_BELOW_PARAMS_B).toBe(12); + const result = recommend(mac(24, 'Apple M4'), STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.reason).toBeUndefined(); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', paramsB: 9, context: 65536, fit: 'good' }); + expect(result.headline).toEqual({ + text: 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', + }); + expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); + }); + + it('holds for a tight fit too (18 GB)', () => { + const result = recommend(mac(18, 'Apple M3 Pro'), STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', fit: 'tight', fitLabel: 'Tight', budgetShare: 0.873 }); + expect(result.alternatives).toEqual([]); + }); + + it('draws the line at 12 billion: 11.9 is limited, 12 is ok', () => { + const verdictFor = (paramsB: number): string => + recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { paramsB })]), [fact('alpha')]).verdict; + expect(verdictFor(11.9)).toBe('limited'); + expect(verdictFor(12)).toBe('ok'); + expect(verdictFor(9)).toBe('limited'); + expect(verdictFor(27.8)).toBe('ok'); + }); + + it('is about the PICK: a larger model further down the list does not lift it', () => { + // 32 GB: the 20B fits only as Tight, so a 9B is the pick and the answer is limited. + const result = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.alternatives.map((c) => [c.modelId, c.fit])).toContainEqual(['gpt-oss:20b', 'tight']); + }); + + it('never says "Recommended", for any Mac', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + if (result.verdict === 'limited') { + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + } + } + }); +}); + +describe('speed', () => { + it('is the model figure times the chip bandwidth, rounded to one decimal', () => { + const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + // 2.95 x 410 / 100. + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', speed: 'ok', tokensPerSecond: 12.1 }); + expect(result.pick).not.toHaveProperty('speedLabel'); + }); + + it('is slow below 10 tokens per second, and says so', () => { + expect(SLOW_BELOW_TOKENS_PER_SECOND).toBe(10); + const speedOf = (speedPer100Gbps: number): Candidate | undefined => + recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { speedPer100Gbps })]), [fact('alpha')]).pick; + // The synthetic chip has 400 GB/s: 2.5 is 10.0 tokens per second, 2.475 is 9.9. + expect(speedOf(2.5)).toMatchObject({ speed: 'ok', tokensPerSecond: 10 }); + expect(speedOf(2.475)).toMatchObject({ speed: 'slow', tokensPerSecond: 9.9, speedLabel: 'Slow' }); + expect(speedOf(0.5)).toMatchObject({ speed: 'slow', tokensPerSecond: 2, speedLabel: 'Slow' }); + }); + + it('is unknown for a model without a figure, on any chip', () => { + // 48 GB: five models fit, so all of them are listed and gpt-oss:20b is among them. + for (const chip of ['Apple M1', 'Apple M4 Max', 'Apple M5 Ultra']) { + const result = recommend(mac(48, chip), STOCK, ALL_INSTALLED); + const gptOss = [result.pick, ...result.alternatives].find((c) => c?.modelId === 'gpt-oss:20b'); + expect(gptOss, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); + expect(gptOss).not.toHaveProperty('tokensPerSecond'); + } + }); + + it('is unknown for every model on a chip that is in no chips row: no bandwidth is guessed', () => { + for (const chip of ['Apple M6', 'Apple M6 Pro', 'Apple M10', 'Some Future Chip']) { + const result = recommend(mac(64, chip), STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + expect(candidate, chip).toMatchObject({ speed: 'unknown', speedLabel: 'Speed unknown' }); + expect(candidate).not.toHaveProperty('tokensPerSecond'); + } + } + // And when the chip name itself is missing. + const result = recommend(without(mac(64, 'x'), 'chip'), STOCK, ALL_INSTALLED); + expect(result.pick?.speed).toBe('unknown'); + }); +}); + +describe('the ranking', () => { + it('puts ok before unknown before slow, ahead of the model rank', () => { + // Three models that fit equally well. The ranks run AGAINST the speed. + const catalog = catalogOf([ + model('slow-top:1', { rank: 3, speedPer100Gbps: 1 }), + model('unknown-mid:1', { rank: 2 }), + model('ok-low:1', { rank: 1, speedPer100Gbps: 5 }), + ]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(order(result)).toEqual(['ok-low:1@1024', 'unknown-mid:1@1024', 'slow-top:1@1024']); + expect([result.pick, ...result.alternatives].map((c) => c?.speed)).toEqual(['ok', 'unknown', 'slow']); + }); + + it('ranks by model rank alone when every speed is unknown', () => { + // 'Apple M6' is in no chips row. + const result = recommend(mac(48, 'Apple M6'), STOCK, ALL_INSTALLED); + expect(order(result)).toEqual([ + 'qwen3.8:27b@131072', + 'qwen3.6:35b-a3b@131072', + 'gpt-oss:20b@131072', + 'qwen3.5:9b-q8_0@65536', + 'qwen3.5:9b@65536', + ]); + }); + + it('ranks an unknown speed below a known ok one and above a slow one, on a real tier', () => { + // 48 GB, M4 Pro: three ok models, then gpt-oss:20b (unknown), then the 27B (slow). + const result = recommend(mac(48, 'Apple M4 Pro'), STOCK, ALL_INSTALLED); + expect([result.pick, ...result.alternatives].map((c) => [c?.modelId, c?.speed])).toEqual([ + ['qwen3.6:35b-a3b', 'ok'], + ['qwen3.5:9b-q8_0', 'ok'], + ['qwen3.5:9b', 'ok'], + ['gpt-oss:20b', 'unknown'], + ['qwen3.8:27b', 'slow'], + ]); + }); + + it('puts a good fit before a tight one, ahead of speed and rank', () => { + const catalog = catalogOf([ + modelNeeding('tight-fast-top:1', 0.9 * HUNDRED_GIB, { rank: 9, speedPer100Gbps: 20 }), + modelNeeding('good-slow-low:1', 0.5 * HUNDRED_GIB, { rank: 1, speedPer100Gbps: 1 }), + ]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(order(result)).toEqual(['good-slow-low:1@1024', 'tight-fast-top:1@1024']); + }); + + it('takes the higher model rank among equals', () => { + const catalog = catalogOf([model('low:1', { rank: 1 }), model('high:1', { rank: 5 }), model('mid:1', { rank: 3 })]); + expect(order(recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]))).toEqual([ + 'high:1@1024', + 'mid:1@1024', + 'low:1@1024', + ]); + }); + + it('takes the larger context of one model when both fit equally well', () => { + // 1 KiB per token: 4 GiB of KV cache at 4,194,304 tokens, still a good fit. + const catalog = catalogOf([model('probe:1', { kvBytesPerToken: 1024, contexts: [4_194_304, 1024] })]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(result.pick?.context).toBe(4_194_304); + expect(result.alternatives).toEqual([]); + }); + + it('steps a model down to the context at which it fits well', () => { + // Good at the small context, tight at the large one: the good one is that model's entry. + const catalog = catalogOf([ + model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] }), + ]); + const result = recommend(HUNDRED_GIB_MAC, catalog, [fact('alpha')]); + expect(result.pick).toMatchObject({ context: 1024, fit: 'good' }); + }); + + it('lists each model once, never the pick again, and at most four alternatives', () => { + expect(MAX_ALTERNATIVES).toBe(4); + // Six models fit a 128 GB Mac; the pick plus four are shown. + const result = recommend(mac(128, 'Apple M5 Ultra'), STOCK, ALL_INSTALLED); + expect(result.alternatives).toHaveLength(4); + const ids = [result.pick, ...result.alternatives].map((c) => c?.modelId); + expect(new Set(ids).size).toBe(ids.length); + for (const hw of SWEEP) { + const swept = recommend(hw, STOCK, ALL_INSTALLED); + const sweptIds = [swept.pick, ...swept.alternatives].filter(Boolean).map((c) => c?.modelId); + expect(new Set(sweptIds).size).toBe(sweptIds.length); + expect(swept.alternatives.length).toBeLessThanOrEqual(MAX_ALTERNATIVES); + } + }); +}); + +describe('harnesses', () => { + const hw = mac(48, 'Apple M4 Max'); + + it('never chooses a gated harness, even when it is the only one that could run a model', () => { + const gated = STOCK.harnesses.filter((row) => row.gated).map((row) => row.cli); + expect(gated).toEqual(['pi']); + for (const swept of SWEEP) { + const result = recommend(swept, STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + if (candidate) expect(gated).not.toContain(candidate.cli); + } + } + // 16 GB: the gated harness alone works at 32768 tokens, where the 9B would fit. Still none. + expect(recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED).verdict).toBe('none'); + }); + + it('never offers a context below the minimum of the harness', () => { + for (const swept of SWEEP) { + const result = recommend(swept, STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + if (!candidate) continue; + const row = STOCK.harnesses.find((harness) => harness.cli === candidate.cli); + expect(candidate.context).toBeGreaterThanOrEqual(row?.minContext ?? Number.POSITIVE_INFINITY); + } + } + }); + + it('takes the higher-ranked harness and reports its label and that it is installed', () => { + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.pick).toMatchObject({ cli: 'opencode', cliLabel: 'OpenCode', cliInstalled: true }); + expect(result.pick).not.toHaveProperty('slowFirstTurn'); + }); + + it('falls back to the next harness when the first is not on the machine, and flags its slow first turn', () => { + const facts = [fact('opencode', { installed: false, installable: false }), fact('claude'), fact('pi')]; + const result = recommend(hw, STOCK, facts); + expect(result.pick).toMatchObject({ + modelId: 'qwen3.8:27b', + cli: 'claude', + cliLabel: 'Claude Code', + cliInstalled: true, + slowFirstTurn: true, + }); + // The same when the caller has no fact for it at all. + expect(recommend(hw, STOCK, [fact('claude')]).pick?.cli).toBe('claude'); + }); + + it('offers a harness that is not installed but can be, and says it is not installed', () => { + const facts = [fact('opencode', { installed: false, installable: true }), fact('claude')]; + const result = recommend(hw, STOCK, facts); + expect(result.pick).toMatchObject({ cli: 'opencode', cliInstalled: false }); + }); + + it('pairs a model only with a harness that accepts its id', () => { + const refusing27b = (id: string): boolean => id !== 'qwen3.8:27b'; + // OpenCode refuses the 27B: it goes with the next harness. + const one = recommend(hw, STOCK, [fact('opencode', { acceptsModelId: refusing27b }), fact('claude')]); + expect(one.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'claude' }); + expect(one.alternatives[0]).toMatchObject({ modelId: 'qwen3.6:35b-a3b', cli: 'opencode' }); + // Both refuse it: the model is not offered. + const both = recommend(hw, STOCK, [ + fact('opencode', { acceptsModelId: refusing27b }), + fact('claude', { acceptsModelId: refusing27b }), + ]); + expect(both.pick?.modelId).toBe('qwen3.6:35b-a3b'); + expect(order(both)).not.toContain('qwen3.8:27b@131072'); + }); + + it('treats a harness check that throws as a refusal, and still answers', () => { + const throwing = fact('opencode', { + acceptsModelId: () => { + throw new Error('registry not loaded'); + }, + }); + const result = recommend(hw, STOCK, [throwing, fact('claude')]); + expect(result.verdict).toBe('ok'); + expect(result.pick?.cli).toBe('claude'); + expect(recommend(hw, STOCK, [throwing]).reason).toBe('no-harness'); + }); + + it('puts the fit and the model before the harness rank', () => { + // `big` needs a large context and is the higher-ranked harness; `small` works from 1024. + const harnesses: LocalHarnessEntry[] = [ + { cli: 'big', minContext: 8_388_608, rank: 2 }, + { cli: 'small', minContext: 1024, rank: 1 }, + ]; + const tightAtLarge = catalogOf( + [model('probe:1', { bytes: 80 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })], + harnesses + ); + // Tight at the context `big` needs, good at the small one: the good fit with `small` wins. + expect(recommend(HUNDRED_GIB_MAC, tightAtLarge, [fact('big'), fact('small')]).pick).toMatchObject({ + cli: 'small', + context: 1024, + fit: 'good', + }); + // Good at both: the higher-ranked harness wins, at the context it needs. + const goodAtBoth = catalogOf( + [model('probe:1', { bytes: 10 * GIB, kvBytesPerToken: 1024, contexts: [8_388_608, 1024] })], + harnesses + ); + expect(recommend(HUNDRED_GIB_MAC, goodAtBoth, [fact('big'), fact('small')]).pick).toMatchObject({ + cli: 'big', + context: 8_388_608, + }); + }); +}); + +describe('the tested flag and the word "Recommended"', () => { + const testedOn = (chip: string, memGb: number, cli: string) => ({ chip, memGb, cli, date: '2026-11-06' }); + const with27bTested = (chip: string, memGb: number, cli: string): LocalModelCatalog => + stockWith((catalog) => { + const row = catalog.models.find((m) => m.id === 'qwen3.8:27b') as LocalModelEntry; + row.tested = [testedOn(chip, memGb, cli)]; + }); + + it('is off for every model of the stock catalogue, so "Recommended" is never said today', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + for (const candidate of [result.pick, ...result.alternatives]) { + if (candidate) expect(candidate.tested).toBe(false); + } + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + if (result.verdict !== 'none') { + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + } + } + }); + + it('turns on "Recommended for this Mac" when the pick was tested on exactly such a Mac', () => { + const result = recommend(mac(48, 'Apple M4 Max'), with27bTested('Apple M4 Max', 48, 'opencode'), ALL_INSTALLED); + expect(result.verdict).toBe('ok'); + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: true }); + expect(result.headline).toEqual({ text: 'Recommended for this Mac' }); + // The "not yet tested" note goes away with it. + expect(result.notes).toEqual([]); + }); + + it('stays off when the chip, the memory size or the harness differs', () => { + const hw = mac(48, 'Apple M4 Max'); + const cases: LocalModelCatalog[] = [ + with27bTested('Apple M4 Pro', 48, 'opencode'), + with27bTested('Apple M4', 48, 'opencode'), + with27bTested('Apple M4 Max', 64, 'opencode'), + with27bTested('Apple M4 Max', 48, 'claude'), + ]; + for (const catalog of cases) { + const result = recommend(hw, catalog, ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.8:27b', cli: 'opencode', tested: false }); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + expect(result.notes).toEqual([{ text: 'Not yet tested by us on a Mac like this.' }]); + } + }); + + it('follows the harness: tested with the second harness counts when that one is the pick', () => { + const catalog = with27bTested('Apple M4 Max', 48, 'claude'); + const result = recommend(mac(48, 'Apple M4 Max'), catalog, [fact('claude')]); + expect(result.pick).toMatchObject({ cli: 'claude', tested: true }); + expect(result.headline.text).toBe('Recommended for this Mac'); + }); + + it('does not change the ranking: a tested model further down stays an alternative', () => { + // On the M4 Pro the pick is the 35B-A3B. A tested 27B is flagged in the list, no more. + const result = recommend(mac(48, 'Apple M4 Pro'), with27bTested('Apple M4 Pro', 48, 'opencode'), ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.6:35b-a3b', tested: false }); + expect(result.headline.text).toBe('Best fit for this Mac'); + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + expect(result.alternatives.find((c) => c.modelId === 'qwen3.8:27b')?.tested).toBe(true); + }); + + it('never says "Recommended" for a limited answer, tested or not', () => { + const catalog = stockWith((stock) => { + const row = stock.models.find((m) => m.id === 'qwen3.5:9b') as LocalModelEntry; + row.tested = [testedOn('Apple M4', 24, 'opencode')]; + }); + const result = recommend(mac(24, 'Apple M4'), catalog, ALL_INSTALLED); + expect(result.verdict).toBe('limited'); + expect(result.pick?.tested).toBe(true); + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); + expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); + // It WAS tested, so the "not yet tested" note would be false. + expect(result.notes).toEqual([]); + }); +}); + +describe('the shape of the answer', () => { + const labels = new Set(Object.values(LOCAL_MODEL_LABELS)); + + it('takes every sentence and label from the fixed list', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(labels.has(result.headline.text)).toBe(true); + for (const note of result.notes) expect(labels.has(note.text)).toBe(true); + for (const candidate of [result.pick, ...result.alternatives]) { + if (!candidate) continue; + expect(labels.has(candidate.fitLabel)).toBe(true); + if (candidate.speedLabel !== undefined) expect(labels.has(candidate.speedLabel)).toBe(true); + expect(candidate.speedLabel === undefined).toBe(candidate.speed === 'ok'); + expect(candidate.tokensPerSecond === undefined).toBe(candidate.speed === 'unknown'); + } + } + }); + + it('fills every placeholder of a headline and nothing else', () => { + const answers = [ + recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { macosMajor: 13, macosVersion: '13.0' }), STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { freeDiskBytes: GB, freeDiskGb: '1.0' }), STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4'), STOCK, []), + recommend(mac(48, 'Apple M4'), STOCK, ALL_INSTALLED), + ]; + expect(answers.map((answer) => answer.reason ?? answer.verdict)).toEqual([ + 'memory', + 'macos', + 'disk', + 'no-harness', + 'ok', + ]); + for (const answer of answers) { + const slots = [...answer.headline.text.matchAll(/\{(\w+)\}/g)].map((match) => match[1]).sort(); + expect(Object.keys(answer.headline.vars ?? {}).sort()).toEqual(slots); + // `{name}` belongs to the browser's translator. + expect(slots).not.toContain('name'); + for (const value of Object.values(answer.headline.vars ?? {})) expect(typeof value).toBe('string'); + } + }); + + it('prints every size in one unit: decimal GB from the one formatter, never GiB', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.budgetGb).toBe(formatGb(result.budgetBytes)); + for (const candidate of [result.pick, ...result.alternatives]) { + if (!candidate) continue; + expect(candidate.needGb).toBe(formatGb(candidate.needBytes)); + expect(candidate.downloadGb).toBe(formatGb(candidate.downloadBytes)); + expect(candidate.budgetShare).toBeCloseTo(candidate.needBytes / result.budgetBytes, 3); + } + expect(JSON.stringify(result)).not.toMatch(/GiB|MiB/); + } + }); + + it('adds up for a person: download, plus what the context and the runtime take, is the need', () => { + const result = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const pick = result.pick as Candidate; + // 17.7 GB of weights + 8 GiB of KV cache + 1 GiB + 2 GiB of prompt cache, all in decimal GB. + expect(pick.downloadGb).toBe('17.7'); + expect(pick.needGb).toBe('29.6'); + expect(formatGb(pick.needBytes - pick.downloadBytes)).toBe('11.8'); + expect(result.budgetGb).toBe('37.6'); + expect(pick.needBytes).toBeLessThan(result.budgetBytes); + }); + + it('is plain JSON and has exactly the fields of its verdict', () => { + const ok = recommend(mac(48, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + expect(JSON.parse(JSON.stringify(ok))).toEqual(ok); + expect(Object.keys(ok).sort()).toEqual( + ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'pick', 'verdict'].sort() + ); + const none = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); + expect(JSON.parse(JSON.stringify(none))).toEqual(none); + expect(Object.keys(none).sort()).toEqual( + ['alternatives', 'budgetBytes', 'budgetGb', 'budgetSource', 'headline', 'notes', 'reason', 'verdict'].sort() + ); + }); + + it('is pure: frozen inputs are enough, and the same inputs give the same answer', () => { + const hw = deepFreeze(mac(48, 'Apple M4 Pro')); + const catalog = deepFreeze(structuredClone(STOCK_LOCAL_MODEL_CATALOG)); + const facts = deepFreeze(ALL_INSTALLED.map((entry) => ({ ...entry }))); + const first = recommend(hw, catalog, facts); + const second = recommend(hw, catalog, facts); + expect(second).toEqual(first); + expect(first.verdict).toBe('ok'); + }); +}); From 34e95ceb6236293024425bfa74404f2512f744e5 Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 16:27:49 +0100 Subject: [PATCH 3/8] feat(local-model): GET /api/local-model and the harness facts The read-only route behind the recommendation, and the one module that reads the CLI registry for it. - harness-facts.ts: one fact per catalogue harness row, resolved in the CLI registry at call time: the registry's label, `installed` from the existing availability probe (asked once per build), `installable` for a stock CLI whose install command for this platform needs no administrator rights, and `acceptsModelId`, the test a custom model launch applies to the model id. A row whose CLI is unknown, disabled or has no custom model mechanism gets no fact. No CLI id is compared with a literal. - routes/local-model-routes.ts: answers { hardware, recommendation, catalogUpdated } in the usual envelope and reformats nothing. Admin only in multi-user mode (403), checked before anything is probed. A failed probe is an answer (200, the hardware's `source` says what was read); a 500 is left for a fault in the catalogue, the harness facts or recommend(). It writes no file and reads no setting. Tests: test/local-model-harness-facts.test.ts, test/routes/local-model-routes.test.ts (app.inject(), no port; the temp HOME is byte-identical before and after the requests). Co-Authored-By: Claude Fable 5.1 --- src/local-model/harness-facts.ts | 171 ++++++++-- src/web/routes/local-model-routes.ts | 72 ++++- test/local-model-harness-facts.test.ts | 306 ++++++++++++++++++ test/routes/local-model-routes.test.ts | 422 +++++++++++++++++++++++++ 4 files changed, 942 insertions(+), 29 deletions(-) create mode 100644 test/local-model-harness-facts.test.ts create mode 100644 test/routes/local-model-routes.test.ts diff --git a/src/local-model/harness-facts.ts b/src/local-model/harness-facts.ts index 7a27d130..b86403e4 100644 --- a/src/local-model/harness-facts.ts +++ b/src/local-model/harness-facts.ts @@ -3,28 +3,163 @@ * local model recommendation. Server side: this is the one module of src/local-model/ that * reads the CLI registry, so `recommend()` can stay pure and name no CLI. * - * SKELETON: the exported signature is final, the body is not written yet. `buildHarnessFacts()` - * will, for every `LocalHarnessEntry` of the catalogue: - * - resolve its `cli` in the CLI registry AT CALL TIME (never in a module-level constant), and - * leave out a row whose CLI is unknown, disabled, or has no custom model mechanism - * (`capabilities.customModelInjection.kind` is `unsupported`); - * - report `label` from the registry entry and `installed` from the same probe the other - * routes use; - * - report `installable` only for a CLI that is not installed and whose install command for - * this platform needs no administrator rights (not one that starts with `brew` or `sudo`); - * - supply `acceptsModelId`, the same test a custom model launch applies to the model id - * before anything is written (the launch spec's model pattern). + * `buildHarnessFacts()` walks the catalogue's `harnesses` rows and, for every row: + * - resolves its `cli` in the CLI registry AT CALL TIME (never in a module-level constant), + * and leaves the row out when the CLI is unknown, disabled, or has no custom model + * mechanism (`capabilities.customModelInjection.kind` is `unsupported`); + * - reports `label` from the registry entry and `installed` from the same probe the page + * render and `GET /api/clis` use (`utils/cli-installed-probes.ts`), asked once per build; + * - reports `installable` only for a STOCK CLI that is not installed and whose install + * command for this platform needs no administrator rights (`installCommandNeedsAdmin()`). + * Stock only, because the install runner never executes a custom entry's command; + * - supplies `acceptsModelId` (`cliAcceptsCustomModelId()`), the test a custom model launch + * applies to the model id before anything is written. * * Rules: - * - No CLI id is compared with a literal here or anywhere: the id comes from the catalogue row - * and everything else from the registry entry it resolves to. - * - Read-only: nothing is installed, written or spawned beyond the existing availability probe. + * - No CLI id is compared with a literal here or anywhere: the id comes from the catalogue + * row and everything else from the registry entry it resolves to. + * - Read-only: nothing is installed, written or spawned beyond the existing availability + * probe. An install command is only read as text, never run. + * - Never throws: a probe that fails reads as "not installed" (with a warning in the log), + * and the registry's install data still answers `installable`. + * + * `cliAcceptsCustomModelId()` composes the two pure pieces the launch path itself calls + * (`buildCustomModelInjection()` and `matchesPattern()` on the entry's `model` launch param). + * The launch path keeps that composition inline at two sites in + * src/web/routes/session-routes.ts (the quick-start `customModel` branch and + * `POST /api/sessions/:id/custom-model`). A change to the rule there must be made here too, + * until the three share one helper. */ -import type { HarnessFact, LocalModelCatalog } from '../types.js'; +import type { CliEntry } from '../config/cli-registry/types.js'; +import { matchesPattern } from '../config/cli-registry/patterns.js'; +import { getCli, resolveInstallCommandForPlatform } from '../config/cli-registry/registry.js'; +import { buildCustomModelInjection, type CustomModelEndpoint } from '../custom-model-injection.js'; +import { isCliEntryInstalled, probeStockCliAvailability } from '../utils/cli-installed-probes.js'; +import { getErrorMessage, type HarnessFact, type LocalModelCatalog } from '../types.js'; + +/** What `buildHarnessFacts()` reads. Each member defaults to the real module; a test overrides what it needs. */ +export interface HarnessFactsDeps { + /** One CLI of the registry, looked up when the facts are built. */ + getCli: (id: string) => CliEntry | undefined; + /** + * The availability probe, asked ONCE per build: it answers with a test for single entries. + * Under vitest the real probe is inert (it finds no CLI), so a test that cares about + * `installed` passes its own. + */ + probeInstalled: () => Promise<(entry: CliEntry) => boolean>; + /** The install command the registry holds for this platform. Read as text, never run. */ + installCommandFor: (entry: CliEntry) => string | undefined; +} + +export const defaultHarnessFactsDeps: HarnessFactsDeps = { + getCli, + probeInstalled: async () => { + const stockAvailability = await probeStockCliAvailability(); + return (entry) => isCliEntryInstalled(entry, stockAvailability); + }, + installCommandFor: resolveInstallCommandForPlatform, +}; + +/** + * A command segment that starts with `sudo` or `brew`: at the start of the text, or after a + * pipe, `&&`, `||`, `;`, a line break or an opening parenthesis. + */ +const ADMIN_COMMAND = /(?:^|[;&|(\n])\s*(?:sudo|brew)(?=\s|$)/; + +/** + * Whether an install command needs administrator rights, read off its text. + * - `sudo ...` asks for an administrator password. + * - `brew ...` needs Homebrew, and installing Homebrew needs an administrator (the same rule + * the installer's no-admin route applies to the CLI menu). + * `curl ... | bash` installers write into the home directory, and the install runner points an + * `npm install -g` at a prefix the server user can write, so neither counts. + */ +export function installCommandNeedsAdmin(command: string): boolean { + return ADMIN_COMMAND.test(command); +} + +/** Stands in for a configured endpoint: the launch id a recipe forces depends on the model id alone. */ +const PROBE_ENDPOINT: CustomModelEndpoint = { + id: 'local-model-probe', + label: 'local model probe', + baseUrl: 'http://127.0.0.1', +}; + +/** + * Whether a custom model launch of this CLI would accept `modelId`. + * + * A CLI whose config alone cannot select the model gets the id forced onto its `model` launch + * param (the recipe's `launchModel` template). The argv engine DROPS a token that fails the + * param's pattern, which would start the CLI on its own default provider, so the launch path + * refuses such an id up front. This is that test: false when the forced value does not match + * the pattern, and false for a CLI with no custom model mechanism at all. + */ +export function cliAcceptsCustomModelId(entry: Pick, modelId: string): boolean { + try { + const injection = buildCustomModelInjection(entry, PROBE_ENDPOINT, modelId); + if (injection.kind === 'unsupported') return false; + const forcedLaunchModel = injection.launchModel; + const modelSpec = entry.launch.params.model; + return !( + forcedLaunchModel !== undefined && + modelSpec?.type === 'token' && + !matchesPattern(modelSpec.pattern, forcedLaunchModel) + ); + } catch { + // A recipe that cannot be rendered cannot be launched either. + return false; + } +} /** One fact per catalogue harness that resolves in the CLI registry, in catalogue order. */ -export async function buildHarnessFacts(catalog: LocalModelCatalog): Promise { - void catalog; - throw new Error('not implemented: buildHarnessFacts'); +export async function buildHarnessFacts( + catalog: LocalModelCatalog, + deps: Partial = {} +): Promise { + const resolved: HarnessFactsDeps = { ...defaultHarnessFactsDeps, ...deps }; + + let isInstalled: (entry: CliEntry) => boolean; + try { + isInstalled = await resolved.probeInstalled(); + } catch (err) { + console.warn( + `[local-model] The CLI availability probe failed, reading every harness as not installed: ${getErrorMessage(err)}` + ); + isInstalled = () => false; + } + + const facts: HarnessFact[] = []; + const seen = new Set(); + for (const row of catalog.harnesses) { + // The catalogue lists a CLI once; a second row for it adds nothing a fact could carry. + if (seen.has(row.cli)) continue; + seen.add(row.cli); + + const entry = resolved.getCli(row.cli); + if (!entry || !entry.enabled) continue; + if (entry.capabilities.customModelInjection.kind === 'unsupported') continue; + + let installed = false; + try { + installed = isInstalled(entry); + } catch (err) { + console.warn(`[local-model] Probing ${entry.label} failed, reading it as not installed: ${getErrorMessage(err)}`); + } + + let installable = false; + if (!installed && entry.stock) { + const command = resolved.installCommandFor(entry); + installable = command !== undefined && command.trim() !== '' && !installCommandNeedsAdmin(command); + } + + facts.push({ + cli: row.cli, + label: entry.label, + installed, + installable, + acceptsModelId: (modelId: string) => cliAcceptsCustomModelId(entry, modelId), + }); + } + return facts; } diff --git a/src/web/routes/local-model-routes.ts b/src/web/routes/local-model-routes.ts index 4c4f7563..f4f0346b 100644 --- a/src/web/routes/local-model-routes.ts +++ b/src/web/routes/local-model-routes.ts @@ -1,25 +1,33 @@ /** * @fileoverview `GET /api/local-model`: what this Mac is and which local model fits it. * - * SKELETON: the route is registered and answers 501 until its handler is written. It will - * answer `{ success: true, data: LocalModelStatus }`, built from the hardware probe, the + * Answers `{ success: true, data: LocalModelStatus }`, built from the hardware probe, the * catalogue, the harness facts and the pure `recommend()`: - * `{ hardware, recommendation, catalogUpdated }`. + * `{ hardware, recommendation, catalogUpdated }`. Nothing is reformatted here: every size text + * and every sentence in the payload was chosen by the modules that produced it. * * Rules: * - Read-only. It downloads nothing, installs nothing, writes no file and reads no setting. - * - Never a 500 for a probe that failed: a failed probe is an answer (the hardware's `source` - * says so and the recommendation is verdict `none`). + * - A failed probe is an answer, never a 500. `readHardware()` reports a spawn that failed + * in the hardware's `source` (`os-fallback`), and a server that is not on macOS as + * `not-probed`; both go out as a 200 whose recommendation is verdict `none` or computed + * from what is known. Should the probe itself reject (it promises not to), the route logs + * it and answers with what `node:os` knows (`hardwareFromOs()`), under the same two + * `source` values. + * - A 500 is left for a fault that is not a probe: the catalogue, the harness facts or + * `recommend()` throwing. That is a bug, and it must not be dressed up as "no model fits". * - The payload names hardware of the host, so in multi-user mode it is admin only (403), the - * same bar as `GET /api/doctor`. + * same bar as `GET /api/doctor`. The gate runs before anything is probed. * - Every dependency is injected (`LocalModelRouteDeps`), so a route test spawns nothing and * does not need a Mac: it passes its own hardware and harness facts. */ +import { arch, totalmem } from 'node:os'; import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify'; import { ApiErrorCode, createErrorResponse, + getErrorMessage, type ApiResponse, type HardwareFacts, type HarnessFact, @@ -27,7 +35,10 @@ import { type LocalModelStatus, type Recommendation, } from '../../types.js'; +import { isAdmin } from '../route-helpers.js'; +import { isMultiUserMode } from '../../config/multiuser.js'; import { getLocalModelCatalog } from '../../config/local-model-catalog.js'; +import { formatGb, memClassGb } from '../../local-model/format.js'; import { readHardware } from '../../local-model/hardware.js'; import { buildHarnessFacts } from '../../local-model/harness-facts.js'; import { recommend } from '../../local-model/recommend.js'; @@ -43,18 +54,57 @@ export interface LocalModelRouteDeps { export const defaultLocalModelRouteDeps: LocalModelRouteDeps = { readHardware: () => readHardware(), getCatalog: getLocalModelCatalog, - buildHarnessFacts, + buildHarnessFacts: (catalog) => buildHarnessFacts(catalog), recommend, }; +/** + * The hardware answer when the probe rejected: memory and architecture from `node:os`, and the + * `source` the probe itself uses for "nothing was read from sysctl" (`os-fallback` on macOS, + * `not-probed` anywhere else). Sizes go through the one formatter, like every other size. + */ +export function hardwareFromOs(): HardwareFacts { + const memBytes = totalmem(); + return { + platform: process.platform, + arch: arch(), + memBytes, + memGb: formatGb(memBytes), + memClassGb: memClassGb(memBytes), + source: process.platform === 'darwin' ? 'os-fallback' : 'not-probed', + }; +} + export function registerLocalModelRoutes(app: FastifyInstance, deps: Partial = {}): void { const resolved: LocalModelRouteDeps = { ...defaultLocalModelRouteDeps, ...deps }; - void resolved; app.get( '/api/local-model', - async (_req: FastifyRequest, reply: FastifyReply): Promise> => { - reply.code(501); - return createErrorResponse(ApiErrorCode.OPERATION_FAILED, 'GET /api/local-model is not implemented yet'); + async (req: FastifyRequest, reply: FastifyReply): Promise> => { + if (isMultiUserMode() && !isAdmin(req)) { + reply.code(403); + return createErrorResponse(ApiErrorCode.FORBIDDEN, 'Admin only in multi-user mode'); + } + + let hardware: HardwareFacts; + try { + hardware = await resolved.readHardware(); + } catch (err) { + console.warn(`[local-model] The hardware probe failed, answering from node:os: ${getErrorMessage(err)}`); + hardware = hardwareFromOs(); + } + + try { + const catalog = resolved.getCatalog(); + const harnessFacts = await resolved.buildHarnessFacts(catalog); + const recommendation = resolved.recommend(hardware, catalog, harnessFacts); + return { success: true, data: { hardware, recommendation, catalogUpdated: catalog.updated } }; + } catch (err) { + reply.code(500); + return createErrorResponse( + ApiErrorCode.INTERNAL_ERROR, + `The local model recommendation failed: ${getErrorMessage(err)}` + ); + } } ); } diff --git a/test/local-model-harness-facts.test.ts b/test/local-model-harness-facts.test.ts new file mode 100644 index 00000000..7c0409b7 --- /dev/null +++ b/test/local-model-harness-facts.test.ts @@ -0,0 +1,306 @@ +/** + * @fileoverview `buildHarnessFacts()`: what the server knows about each catalogue harness on + * this machine (src/local-model/harness-facts.ts), read from the CLI registry at call time. + * + * The catalogue here is the test's own: the stock one is filled by another module and its rows + * are not what this file is about. The registry is the REAL stock registry unless a test says + * otherwise, so the model id anchors below are regression anchors on shipped data. + * + * Two things a reader should know before trusting a green run: + * - Under vitest the real availability probe is inert (it finds no CLI). A test that cares + * about `installed` therefore injects `probeInstalled`. + * - The registry's install command resolver reads `process.platform` and falls back to the + * Linux command, so on a Linux machine it never returns a macOS command. The macOS cases + * inject `installCommandFor` and read the entry's own `darwin` command. + * + * Port: none (no server; the on-disk case uses the per-file temp HOME from test/setup.ts). + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { + buildHarnessFacts, + cliAcceptsCustomModelId, + installCommandNeedsAdmin, +} from '../src/local-model/harness-facts.js'; +import { getCli, registryFilePath, reloadCliRegistry } from '../src/config/cli-registry/registry.js'; +import type { CliEntry } from '../src/config/cli-registry/types.js'; +import type { LocalModelCatalog } from '../src/types.js'; + +/** A catalogue whose harness rows are `clis`, in that order. Nothing else in it matters here. */ +function catalogWith(clis: string[]): LocalModelCatalog { + return { + schema: 1, + updated: '2026-11-06', + minMacosMajor: 14, + promptCacheTiers: [{ minMemGb: 0, bytes: 512 * 1024 * 1024 }], + models: [], + harnesses: clis.map((cli, index) => ({ cli, minContext: 32768, rank: clis.length - index })), + chips: [], + }; +} + +function stockEntry(id: string): CliEntry { + const entry = getCli(id); + if (!entry) throw new Error(`${id} is missing from the CLI registry`); + return entry; +} + +/** The command the registry holds for macOS, whatever machine the test runs on. */ +const darwinCommand = (entry: CliEntry): string | undefined => entry.discovery.install.command.darwin; + +/** A probe that reports exactly these CLIs as installed. */ +const installedOnly = + (...ids: string[]) => + async () => + (entry: CliEntry) => + ids.includes(entry.id as string); + +afterEach(() => { + const path = registryFilePath(); + if (existsSync(path)) rmSync(path); + reloadCliRegistry(); + vi.restoreAllMocks(); +}); + +describe('installCommandNeedsAdmin', () => { + it.each([ + ['brew install can1357/tap/omp', true], + [' brew install something', true], + ['sudo apt install something', true], + ['curl -fsSL https://example.com/install.sh | sudo bash', true], + ['npm install -g thing && sudo ln -s a b', true], + ['echo first; brew install thing', true], + ['echo first\nsudo make install', true], + ['(brew install thing)', true], + ['curl -fsSL https://opencode.ai/install | bash', false], + ['curl -fsSL https://claude.ai/install.sh | bash', false], + ['npm install -g --ignore-scripts @earendil-works/pi-coding-agent', false], + ['npm install -g @deepseek-ai/dsh && (command -v pnpm >/dev/null || npm install -g pnpm)', false], + // The words inside a URL, a package name or an argument are not a command. + ['curl -fsSL https://brew.sh/install.sh | bash', false], + ['npm install -g homebrew-sudo-helper', false], + ['echo sudo', false], + ])('%s -> %s', (command, expected) => { + expect(installCommandNeedsAdmin(command)).toBe(expected); + }); + + it('agrees with the registry on every stock macOS command: only a Homebrew one needs an administrator', () => { + expect(installCommandNeedsAdmin(darwinCommand(stockEntry('omp')) ?? '')).toBe(true); + for (const id of ['claude', 'opencode', 'codex', 'pi']) { + expect(installCommandNeedsAdmin(darwinCommand(stockEntry(id)) ?? '')).toBe(false); + } + }); +}); + +describe('cliAcceptsCustomModelId (the launch path’s model id test, on the stock registry)', () => { + it('accepts a registry style `name:tag` id for the CLIs that can carry a colon', () => { + // opencode and claude select the model in their config or environment: no launch id at all. + // pi and codex force the id onto `--model`, and both of their patterns allow a colon. + for (const id of ['opencode', 'claude', 'pi', 'codex']) { + expect(cliAcceptsCustomModelId(stockEntry(id), 'qwen3.5:9b')).toBe(true); + } + }); + + it('refuses a colon for a CLI whose `--model` pattern has none, and accepts the same CLI without it', () => { + const omp = stockEntry('omp'); + expect(cliAcceptsCustomModelId(omp, 'qwen3.5:9b')).toBe(false); + expect(cliAcceptsCustomModelId(omp, 'qwen3-coder')).toBe(true); + }); + + it('refuses the ids the launch routes’ own tests refuse', () => { + // test/routes/quick-start-custom-model.test.ts pins both pairs against the real route. + expect(cliAcceptsCustomModelId(stockEntry('pi'), 'qwen 3 with spaces')).toBe(false); + expect(cliAcceptsCustomModelId(stockEntry('codex'), 'org/model@q4_k_m')).toBe(false); + }); + + it('is false for a CLI with no custom model mechanism', () => { + expect(cliAcceptsCustomModelId(stockEntry('shell'), 'qwen3.5:9b')).toBe(false); + expect(cliAcceptsCustomModelId(stockEntry('antigravity'), 'qwen3.5:9b')).toBe(false); + }); + + it('reads the entry it is given, so a changed pattern changes the answer (no id is looked at)', () => { + const omp = stockEntry('omp'); + const widened: CliEntry = { + ...omp, + launch: { ...omp.launch, params: { ...omp.launch.params, model: { type: 'token', pattern: 'model-pi' } } }, + }; + expect(cliAcceptsCustomModelId(widened, 'qwen3.5:9b')).toBe(true); + }); +}); + +describe('buildHarnessFacts', () => { + it('answers one fact per catalogue harness, in catalogue order, with the registry’s label', async () => { + const facts = await buildHarnessFacts(catalogWith(['pi', 'opencode', 'claude']), { + probeInstalled: installedOnly('opencode'), + installCommandFor: darwinCommand, + }); + expect(facts.map((f) => f.cli)).toEqual(['pi', 'opencode', 'claude']); + expect(facts.map((f) => f.label)).toEqual(['pi', 'opencode', 'claude'].map((id) => stockEntry(id).label)); + expect(facts.map((f) => f.installed)).toEqual([false, true, false]); + }); + + it('leaves out a CLI the registry does not know, one with no custom model mechanism, and a disabled one', async () => { + const disabled = (id: string): CliEntry | undefined => { + const entry = getCli(id); + return entry && id === 'claude' ? { ...entry, enabled: false } : entry; + }; + const facts = await buildHarnessFacts(catalogWith(['no-such-cli', 'shell', 'antigravity', 'claude', 'opencode']), { + getCli: disabled, + probeInstalled: installedOnly(), + }); + expect(facts.map((f) => f.cli)).toEqual(['opencode']); + }); + + it('keeps one fact for a CLI the catalogue lists twice', async () => { + const facts = await buildHarnessFacts(catalogWith(['opencode', 'opencode']), { probeInstalled: installedOnly() }); + expect(facts.map((f) => f.cli)).toEqual(['opencode']); + }); + + it('installable: not installed, a stock CLI, and a macOS command that needs no administrator', async () => { + const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi', 'omp']), { + probeInstalled: installedOnly('claude'), + installCommandFor: darwinCommand, + }); + const byCli = new Map(facts.map((f) => [f.cli, f])); + // curl installer into the home directory. + expect(byCli.get('opencode')).toMatchObject({ installed: false, installable: true }); + // Already there: nothing to install. + expect(byCli.get('claude')).toMatchObject({ installed: true, installable: false }); + // npm: the install runner points it at a prefix the server user can write. + expect(byCli.get('pi')).toMatchObject({ installed: false, installable: true }); + // Homebrew on macOS. + expect(byCli.get('omp')).toMatchObject({ installed: false, installable: false }); + }); + + it('installable is false without an install command, for a sudo command, and for a custom entry', async () => { + const command = vi.fn<(entry: CliEntry) => string | undefined>(); + command.mockReturnValueOnce(undefined); + expect( + ( + await buildHarnessFacts(catalogWith(['opencode']), { + probeInstalled: installedOnly(), + installCommandFor: command, + }) + )[0].installable + ).toBe(false); + command.mockReturnValueOnce(' '); + expect( + ( + await buildHarnessFacts(catalogWith(['opencode']), { + probeInstalled: installedOnly(), + installCommandFor: command, + }) + )[0].installable + ).toBe(false); + command.mockReturnValueOnce('sudo installer -pkg agent.pkg -target /'); + expect( + ( + await buildHarnessFacts(catalogWith(['opencode']), { + probeInstalled: installedOnly(), + installCommandFor: command, + }) + )[0].installable + ).toBe(false); + + // A custom entry's install command is never run by the install runner, so it is never + // offered, however harmless the text looks. + const custom = (id: string): CliEntry | undefined => { + const entry = getCli(id); + return entry ? { ...entry, stock: false } : entry; + }; + const harmless = vi.fn(() => 'curl -fsSL https://example.com/install | bash'); + const facts = await buildHarnessFacts(catalogWith(['opencode']), { + getCli: custom, + probeInstalled: installedOnly(), + installCommandFor: harmless, + }); + expect(facts[0].installable).toBe(false); + expect(harmless).not.toHaveBeenCalled(); + }); + + it('never reads an install command for a CLI that is already installed', async () => { + const command = vi.fn(() => 'curl -fsSL https://example.com/install | bash'); + const facts = await buildHarnessFacts(catalogWith(['opencode']), { + probeInstalled: installedOnly('opencode'), + installCommandFor: command, + }); + expect(facts[0]).toMatchObject({ installed: true, installable: false }); + expect(command).not.toHaveBeenCalled(); + }); + + it('acceptsModelId is the launch path’s test for that harness', async () => { + const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi', 'omp']), { + probeInstalled: installedOnly(), + }); + const accepts = (cli: string, modelId: string) => facts.find((f) => f.cli === cli)?.acceptsModelId(modelId); + expect(accepts('opencode', 'qwen3.5:9b')).toBe(true); + expect(accepts('claude', 'qwen3.8:27b-q8_0')).toBe(true); + expect(accepts('pi', 'gpt-oss:20b')).toBe(true); + expect(accepts('omp', 'gpt-oss:20b')).toBe(false); + }); + + it('asks the availability probe once per build, not once per harness', async () => { + const probe = vi.fn(installedOnly('opencode')); + await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi']), { probeInstalled: probe }); + expect(probe).toHaveBeenCalledTimes(1); + }); + + it('a probe that fails is an answer: every harness reads as not installed, and install data still counts', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const facts = await buildHarnessFacts(catalogWith(['opencode', 'omp']), { + probeInstalled: async () => { + throw new Error('login shell did not answer'); + }, + installCommandFor: darwinCommand, + }); + expect(facts.map((f) => [f.cli, f.installed, f.installable])).toEqual([ + ['opencode', false, true], + ['omp', false, false], + ]); + expect(warn).toHaveBeenCalledTimes(1); + expect(String(warn.mock.calls[0][0])).toContain('login shell did not answer'); + }); + + it('a probe that throws for one harness does not take the others down', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude']), { + probeInstalled: async () => (entry) => { + if (entry.label === stockEntry('opencode').label) throw new Error('boom'); + return true; + }, + }); + expect(facts.map((f) => [f.cli, f.installed])).toEqual([ + ['opencode', false], + ['claude', true], + ]); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it('reads the registry when it is called, not when the module loaded', async () => { + const before = await buildHarnessFacts(catalogWith(['opencode', 'claude'])); + expect(before.map((f) => f.cli)).toEqual(['opencode', 'claude']); + + // The user disables a CLI in ~/.codeman/clis.json after the module was imported. + const path = registryFilePath(); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, JSON.stringify({ schemaVersion: 1, clis: { opencode: { enabled: false } } }), { mode: 0o600 }); + reloadCliRegistry(); + + const after = await buildHarnessFacts(catalogWith(['opencode', 'claude'])); + expect(after.map((f) => f.cli)).toEqual(['claude']); + }); + + it('with its default dependencies it spawns nothing under vitest and finds no CLI installed', async () => { + const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi'])); + expect(facts.map((f) => f.cli)).toEqual(['opencode', 'claude', 'pi']); + expect(facts.every((f) => f.installed === false)).toBe(true); + // opencode's install command is a curl installer on every platform the registry lists. + expect(facts[0].installable).toBe(true); + }); + + it('an empty harness list is an empty answer', async () => { + expect(await buildHarnessFacts(catalogWith([]))).toEqual([]); + }); +}); diff --git a/test/routes/local-model-routes.test.ts b/test/routes/local-model-routes.test.ts new file mode 100644 index 00000000..a3e7d31c --- /dev/null +++ b/test/routes/local-model-routes.test.ts @@ -0,0 +1,422 @@ +/** + * @fileoverview GET /api/local-model: what this Mac is and which local model fits it. + * + * The route only composes four injected functions (the hardware probe, the catalogue getter, + * the harness facts and the pure `recommend()`), so every test passes its own and no test + * depends on what those modules compute. What is pinned here is the route's own contract: + * the envelope, the admin gate in multi-user mode, that an unsupported platform and a failed + * probe are answers (200) and never a 500, what does answer 500, and that a request writes + * nothing. + * + * Port: N/A (app.inject()). + */ +import { afterEach, describe, expect, it, vi, type Mock } from 'vitest'; +import { existsSync, lstatSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { arch, homedir, totalmem } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRouteTestHarness } from './_route-test-utils.js'; +import { + defaultLocalModelRouteDeps, + hardwareFromOs, + registerLocalModelRoutes, + type LocalModelRouteDeps, +} from '../../src/web/routes/local-model-routes.js'; +import { buildHarnessFacts } from '../../src/local-model/harness-facts.js'; +import { formatGb, memClassGb } from '../../src/local-model/format.js'; +import { getDataDir } from '../../src/config/instance.js'; +import { + LOCAL_MODEL_LABELS, + type HardwareFacts, + type HarnessFact, + type LocalModelCatalog, + type Recommendation, +} from '../../src/types.js'; + +const CATALOG: LocalModelCatalog = { + schema: 1, + updated: '2026-11-06', + minMacosMajor: 14, + promptCacheTiers: [ + { minMemGb: 0, bytes: 512 * 1024 * 1024 }, + { minMemGb: 32, bytes: 2 * 1024 ** 3 }, + ], + models: [], + harnesses: [ + { cli: 'opencode', minContext: 65536, rank: 3 }, + { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true }, + ], + chips: [], +}; + +/** A 48 GB Apple silicon Mac whose chip is in no chips row, as the probe would report it. */ +const MAC: HardwareFacts = { + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 51539607552, + memGb: '51.5', + memClassGb: 48, + chip: 'Apple M5 Pro', + model: 'Mac17,13', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: 912308310016, + freeDiskGb: '912', + source: 'sysctl', +}; + +/** The same Mac after a `sysctl` spawn that failed: `node:os` supplied what little is known. */ +const MAC_FALLBACK: HardwareFacts = { + platform: 'darwin', + arch: 'arm64', + memBytes: 51539607552, + memGb: '51.5', + memClassGb: 48, + chip: 'Apple M5 Pro', + source: 'os-fallback', +}; + +const LINUX: HardwareFacts = { + platform: 'linux', + arch: 'x64', + memBytes: 67430404096, + memGb: '67.4', + memClassGb: 63, + source: 'not-probed', +}; + +const OK: Recommendation = { + verdict: 'ok', + headline: { text: LOCAL_MODEL_LABELS.headlineOk }, + notes: [{ text: LOCAL_MODEL_LABELS.noteUntested }], + pick: { + modelId: 'qwen3.8:27b', + modelLabel: 'Qwen3.8 27B', + license: 'Apache-2.0', + paramsB: 27.8, + context: 131072, + cli: 'opencode', + cliLabel: 'OpenCode', + cliInstalled: true, + needBytes: 29551160064, + needGb: '29.6', + downloadBytes: 17740000000, + downloadGb: '17.7', + budgetShare: 0.79, + fit: 'good', + fitLabel: LOCAL_MODEL_LABELS.fitGood, + speed: 'unknown', + speedLabel: LOCAL_MODEL_LABELS.speedUnknown, + tested: false, + }, + alternatives: [], + budgetBytes: 37580963840, + budgetGb: '37.6', + budgetSource: 'estimated', +}; + +const NOT_MACOS: Recommendation = { + verdict: 'none', + reason: 'not-macos', + headline: { text: LOCAL_MODEL_LABELS.noneNotMacos }, + notes: [], + alternatives: [], + budgetBytes: 0, + budgetGb: '0.0', + budgetSource: 'estimated', +}; + +const FACTS: HarnessFact[] = [ + { cli: 'opencode', label: 'OpenCode', installed: true, installable: false, acceptsModelId: () => true }, +]; + +interface Fakes { + deps: LocalModelRouteDeps; + readHardware: Mock; + getCatalog: Mock; + buildHarnessFacts: Mock; + recommend: Mock; +} + +function fakes(hardware: HardwareFacts = MAC, recommendation: Recommendation = OK): Fakes { + const readHardware = vi.fn(async () => hardware); + const getCatalog = vi.fn(() => CATALOG); + const buildFacts = vi.fn(async () => FACTS); + const recommend = vi.fn(() => recommendation); + return { + deps: { readHardware, getCatalog, buildHarnessFacts: buildFacts, recommend }, + readHardware, + getCatalog, + buildHarnessFacts: buildFacts, + recommend, + }; +} + +const harness = (deps: Partial, authUser?: { username: string; role: 'admin' | 'user' }) => + createRouteTestHarness((app) => registerLocalModelRoutes(app, deps), authUser ? { authUser } : undefined); + +const get = async (deps: Partial, authUser?: { username: string; role: 'admin' | 'user' }) => { + const { app } = await harness(deps, authUser); + return app.inject({ method: 'GET', url: '/api/local-model' }); +}; + +/** Every file and folder under `root`, with its size and modification time. */ +function snapshot(root: string): string[] { + if (!existsSync(root)) return []; + const rows: string[] = []; + const walk = (dir: string) => { + for (const name of readdirSync(dir).sort()) { + const path = join(dir, name); + const stat = lstatSync(path); + rows.push(`${path} ${stat.isDirectory() ? 'dir' : stat.size} ${stat.mtimeMs}`); + if (stat.isDirectory()) walk(path); + } + }; + walk(root); + return rows; +} + +afterEach(() => { + delete process.env.CODEMAN_MULTIUSER; + vi.restoreAllMocks(); +}); + +describe('GET /api/local-model', () => { + it('answers the success envelope with the hardware, the recommendation and the catalogue date', async () => { + const f = fakes(); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + expect(res.json()).toEqual({ + success: true, + data: { hardware: MAC, recommendation: OK, catalogUpdated: '2026-11-06' }, + }); + }); + + it('hands recommend() exactly what the probe, the catalogue and the harness facts answered', async () => { + const f = fakes(); + await get(f.deps); + expect(f.readHardware).toHaveBeenCalledTimes(1); + expect(f.buildHarnessFacts).toHaveBeenCalledTimes(1); + expect(f.buildHarnessFacts).toHaveBeenCalledWith(CATALOG); + expect(f.recommend).toHaveBeenCalledTimes(1); + const [hw, catalog, facts] = f.recommend.mock.calls[0]; + expect(hw).toBe(MAC); + expect(catalog).toBe(CATALOG); + expect(facts).toBe(FACTS); + }); + + it('sends the payload as the modules built it: no size is reformatted and no function leaks', async () => { + const res = await get(fakes().deps); + const { data } = res.json(); + expect(Object.keys(data).sort()).toEqual(['catalogUpdated', 'hardware', 'recommendation']); + expect(data.hardware.memGb).toBe('51.5'); + expect(data.hardware.freeDiskGb).toBe('912'); + expect(data.recommendation.pick.needGb).toBe('29.6'); + expect(data.recommendation.budgetGb).toBe('37.6'); + // Harness facts hold a function and stay on the server. + expect(res.body).not.toContain('acceptsModelId'); + expect(res.body).not.toContain('installable'); + }); + + it('registers a GET and nothing else: this stage has no route that changes anything', async () => { + const { app } = await harness(fakes().deps); + expect((await app.inject({ method: 'POST', url: '/api/local-model', payload: {} })).statusCode).toBe(404); + expect((await app.inject({ method: 'DELETE', url: '/api/local-model' })).statusCode).toBe(404); + }); + + describe('multi-user mode', () => { + it('refuses a non-admin with 403 before anything is probed', async () => { + process.env.CODEMAN_MULTIUSER = '1'; + const f = fakes(); + const res = await get(f.deps, { username: 'bob', role: 'user' }); + expect(res.statusCode).toBe(403); + expect(res.json()).toMatchObject({ success: false, errorCode: 'FORBIDDEN' }); + expect(res.body).not.toContain('Apple M5 Pro'); + expect(f.readHardware).not.toHaveBeenCalled(); + expect(f.getCatalog).not.toHaveBeenCalled(); + expect(f.buildHarnessFacts).not.toHaveBeenCalled(); + expect(f.recommend).not.toHaveBeenCalled(); + }); + + it('answers an admin', async () => { + process.env.CODEMAN_MULTIUSER = '1'; + const res = await get(fakes().deps, { username: 'root', role: 'admin' }); + expect(res.statusCode).toBe(200); + expect(res.json().data.hardware.chip).toBe('Apple M5 Pro'); + }); + + it('is not gated while multi-user mode is off, whoever asks', async () => { + const res = await get(fakes().deps, { username: 'bob', role: 'user' }); + expect(res.statusCode).toBe(200); + }); + }); + + describe('an unsupported platform is an answer', () => { + it('answers 200 with verdict none and reason not-macos on a server that is not a Mac', async () => { + const f = fakes(LINUX, NOT_MACOS); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + const { success, data } = res.json(); + expect(success).toBe(true); + expect(data.hardware).toEqual(LINUX); + expect(data.hardware.source).toBe('not-probed'); + expect(data.recommendation.verdict).toBe('none'); + expect(data.recommendation.reason).toBe('not-macos'); + expect(data.recommendation.pick).toBeUndefined(); + expect(data.recommendation.alternatives).toEqual([]); + expect(data.recommendation.headline.text).toBe(LOCAL_MODEL_LABELS.noneNotMacos); + expect(data.catalogUpdated).toBe('2026-11-06'); + expect(f.recommend.mock.calls[0][0]).toBe(LINUX); + }); + }); + + describe('a failed probe is an answer', () => { + it('answers 200 when the probe reports that sysctl failed (source os-fallback)', async () => { + const f = fakes(MAC_FALLBACK, OK); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + expect(res.json().data.hardware).toEqual(MAC_FALLBACK); + expect(res.json().data.recommendation.verdict).toBe('ok'); + }); + + it('answers 200 from node:os when the probe itself rejects, and still recommends', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const f = fakes(); + f.readHardware.mockRejectedValue(new Error('spawn EAGAIN')); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + const { success, data } = res.json(); + expect(success).toBe(true); + expect(data.hardware).toEqual(hardwareFromOs()); + expect(data.hardware.memBytes).toBe(totalmem()); + expect(data.recommendation).toEqual(OK); + expect(f.recommend).toHaveBeenCalledTimes(1); + expect(f.recommend.mock.calls[0][0]).toEqual(hardwareFromOs()); + expect(String(warn.mock.calls[0][0])).toContain('spawn EAGAIN'); + }); + + it('hardwareFromOs() is a complete answer of its own, sized by the one formatter', () => { + const hw = hardwareFromOs(); + expect(hw.platform).toBe(process.platform); + expect(hw.arch).toBe(arch()); + expect(hw.memBytes).toBe(totalmem()); + expect(hw.memGb).toBe(formatGb(totalmem())); + expect(hw.memClassGb).toBe(memClassGb(totalmem())); + expect(hw.source).toBe(process.platform === 'darwin' ? 'os-fallback' : 'not-probed'); + // Nothing that only sysctl or statfs can know is invented. + expect(Object.keys(hw).sort()).toEqual(['arch', 'memBytes', 'memClassGb', 'memGb', 'platform', 'source']); + }); + + it('answers 200 when no harness fact could be built (the facts are just empty)', async () => { + const none: Recommendation = { + ...NOT_MACOS, + reason: 'no-harness', + headline: { text: LOCAL_MODEL_LABELS.noneNoHarness }, + }; + const f = fakes(MAC, none); + f.buildHarnessFacts.mockResolvedValue([]); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + expect(res.json().data.recommendation.reason).toBe('no-harness'); + expect(f.recommend.mock.calls[0][2]).toEqual([]); + }); + }); + + describe('a fault that is not a probe is a 500 in the envelope, never a made-up answer', () => { + it.each([ + [ + 'recommend() throws', + (f: Fakes) => + f.recommend.mockImplementation(() => { + throw new Error('rank table is empty'); + }), + 'rank table is empty', + ], + [ + 'the catalogue getter throws', + (f: Fakes) => + f.getCatalog.mockImplementation(() => { + throw new Error('catalogue did not parse'); + }), + 'catalogue did not parse', + ], + [ + 'the harness facts reject', + (f: Fakes) => f.buildHarnessFacts.mockRejectedValue(new Error('registry unreadable')), + 'registry unreadable', + ], + ])('%s', async (_label, breakIt, message) => { + const f = fakes(); + breakIt(f); + const res = await get(f.deps); + expect(res.statusCode).toBe(500); + const body = res.json(); + expect(body.success).toBe(false); + expect(body.errorCode).toBe('INTERNAL_ERROR'); + expect(body.error).toContain(message); + expect(body.data).toBeUndefined(); + }); + }); + + describe('read-only', () => { + it('writes nothing: the home directory and the data dir are byte for byte what they were', async () => { + // The real harness facts (they read the CLI registry and run the availability probe), + // the route's own composition, and a hardware answer and a recommendation of the test's. + const f = fakes(); + const { app } = await harness({ ...f.deps, buildHarnessFacts: (catalog) => buildHarnessFacts(catalog) }); + + const home = homedir(); + const before = snapshot(home); + const dataDirExisted = existsSync(getDataDir()); + + for (let i = 0; i < 3; i++) { + const res = await app.inject({ method: 'GET', url: '/api/local-model' }); + expect(res.statusCode).toBe(200); + } + // A refused request and a failing one write nothing either. + f.recommend.mockImplementationOnce(() => { + throw new Error('boom'); + }); + expect((await app.inject({ method: 'GET', url: '/api/local-model' })).statusCode).toBe(500); + + expect(snapshot(home)).toEqual(before); + expect(existsSync(getDataDir())).toBe(dataDirExisted); + expect(existsSync(join(getDataDir(), 'settings.json'))).toBe(false); + expect(existsSync(join(getDataDir(), 'local-model.json'))).toBe(false); + expect(existsSync(join(getDataDir(), 'local-models.json'))).toBe(false); + // The facts really were built from the registry for the catalogue's two harnesses. + expect(f.recommend.mock.calls[0][2].map((fact) => fact.cli)).toEqual(['opencode', 'claude']); + + // The comparison above is not vacuous: one new file in the home directory shows up. + const probe = join(home, 'local-model-route-test-probe'); + writeFileSync(probe, 'x'); + try { + expect(snapshot(home)).not.toEqual(before); + } finally { + rmSync(probe); + } + }); + + it('is registered on the real server through the routes barrel', () => { + const root = fileURLToPath(new URL('../../', import.meta.url)); + const server = readFileSync(join(root, 'src/web/server.ts'), 'utf8'); + const barrel = readFileSync(join(root, 'src/web/routes/index.ts'), 'utf8'); + expect(barrel).toContain("export { registerLocalModelRoutes } from './local-model-routes.js';"); + // No second argument: the server runs the real probe, catalogue, facts and recommend(). + expect(server).toContain('registerLocalModelRoutes(this.app);'); + }); + + it('the default dependencies are the real modules, read at call time', () => { + expect(Object.keys(defaultLocalModelRouteDeps).sort()).toEqual([ + 'buildHarnessFacts', + 'getCatalog', + 'readHardware', + 'recommend', + ]); + for (const dep of Object.values(defaultLocalModelRouteDeps)) expect(typeof dep).toBe('function'); + }); + }); +}); From 1976431ce6dd585a4b3b73da2437b3a6ac0c5c06 Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 16:28:04 +0100 Subject: [PATCH 4/8] feat(local-model): welcome card and App Settings row for the model on this Mac The read-only face of GET /api/local-model: what the Mac the server runs on can run as a local coding model. - local-model-ui.js: one builder paints both containers from the same payload: the hardware line, the server's headline, the pick with its numbers (context, download, memory need against the budget, speed), the server's notes, the other models that fit with their Good, Tight, Slow or Speed unknown labels, and one line saying that setting it up from Codeman is the next step and is not built yet. - The welcome card ships hidden and shows only once the server answered for a Mac. It stays hidden for not-macos, for a refused request (a non-admin in multi-user mode) and after "Hide", a per-device localStorage key. The App Settings group "Model on this Mac" (Models) carries the loading and failure sentences and is the way back after a Hide. - Every dynamic string goes in through textContent, both containers are data-i18n-skip and the module translates its own sentences, repainting on a language change. Sizes are printed as sent. Labels and harness names come from the payload: the module names no CLI and the word "Recommended" is not in it. - One request a minute at most, shared while in flight; a failed refresh keeps the last answer on screen. - i18n.js: zh-CN entries for the module's strings and for every server label. - frontend-cli-no-id-branching.test.ts scans the new module. Tests: test/local-model-ui.test.ts (JSDOM over the real markup), test/local-model-i18n.test.ts. Co-Authored-By: Claude Fable 5.1 --- src/web/public/i18n.js | 50 ++ src/web/public/index.html | 24 + src/web/public/local-model-ui.js | 372 +++++++++- src/web/public/mobile.css | 21 + src/web/public/settings-ui.js | 3 + src/web/public/styles.css | 197 ++++++ src/web/public/terminal-ui.js | 3 + test/frontend-cli-no-id-branching.test.ts | 3 + test/local-model-i18n.test.ts | 172 +++++ test/local-model-ui.test.ts | 797 ++++++++++++++++++++++ 10 files changed, 1633 insertions(+), 9 deletions(-) create mode 100644 test/local-model-i18n.test.ts create mode 100644 test/local-model-ui.test.ts diff --git a/src/web/public/i18n.js b/src/web/public/i18n.js index a3730c91..c9ab61a5 100644 --- a/src/web/public/i18n.js +++ b/src/web/public/i18n.js @@ -1407,6 +1407,56 @@ '该 tmux 会话在此处打开时所在的位置。它是在 Codeman 之外启动的，关闭此标签页后仍会继续运行。', 'Git status is not available for a tmux session opened outside Codeman.': '在 Codeman 之外打开的 tmux 会话无法使用 Git 状态。', + // Model on this Mac (local-model-ui.js: the welcome card and its App Settings group). The first + // block is the module's own text; sizes and names fill the {placeholder} slots. + 'Model on this Mac': '此 Mac 上的模型', + 'What the Mac that Codeman runs on can run as a local coding model. Read-only: nothing is downloaded or installed from here.': + '运行 {name} 的这台 Mac 可以运行哪种本地编程模型。只读：此处不会下载或安装任何内容。', + Hide: '隐藏', + 'Hide this card on this device. It stays in App Settings, under Models.': + '在此设备上隐藏此卡片。它仍保留在“应用设置”的“模型”中。', + 'Checking this Mac…': '正在检查此 Mac…', + '{chip}, {memory} GB memory, {free} GB free': '{chip}，内存 {memory} GB，可用磁盘空间 {free} GB', + '{chip}, {memory} GB memory': '{chip}，内存 {memory} GB', + '{memory} GB memory, {free} GB free': '内存 {memory} GB，可用磁盘空间 {free} GB', + '{memory} GB memory': '内存 {memory} GB', + '{model} ({id}) with {harness}': '{model}（{id}），搭配 {harness}', + 'Context {context} tokens': '上下文 {context} 个 Token', + '{need} GB of memory': '占用内存 {need} GB', + '{download} GB download': '下载 {download} GB', + 'Needs about {need} GB of the {budget} GB this Mac can give a model.': + '大约需要 {need} GB，此 Mac 可分配给模型的内存为 {budget} GB。', + 'Needs about {need} GB of the {budget} GB this Mac can give a model (estimated).': + '大约需要 {need} GB，此 Mac 可分配给模型的内存约为 {budget} GB（估算）。', + 'About {speed} tokens per second (estimated).': '每秒约 {speed} 个 Token（估算）。', + '{harness} is not installed on this Mac yet.': '此 Mac 上尚未安装 {harness}。', + 'With {harness}, the first answer can take minutes.': '使用 {harness} 时，第一次回答可能需要几分钟。', + 'Other models that fit': '其他适合的模型', + 'Setting this up from Codeman is the next step. It is not built yet.': + '下一步是直接在 {name} 中完成安装和设置，该功能尚未实现。', + // The sentences and labels the SERVER picks (LOCAL_MODEL_LABELS in src/types/local-model.ts), keyed + // by their exact English text: changing a text there means changing its key here. + 'Best fit for this Mac': '最适合此 Mac 的选择', + 'Recommended for this Mac': '为此 Mac 推荐', + 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.': + '此 Mac 可以运行小型模型。它能处理小幅修改和问答，但不能替代云端模型。', + 'Not yet tested by us on a Mac like this.': '我们尚未在类似的 Mac 上测试过。', + Good: '良好', + Tight: '紧张', + Slow: '较慢', + 'Speed unknown': '速度未知', + 'Codeman recommends a local model only on a Mac with Apple silicon.': + '{name} 仅在搭载 Apple 芯片的 Mac 上推荐本地模型。', + 'This Mac has an Intel processor. Local coding models need Apple silicon.': + '此 Mac 使用 Intel 处理器。本地编程模型需要 Apple 芯片。', + 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.': + '此 Mac 有 {memory} GB 内存，不足以运行能够驱动编程智能体的模型，因此 {name} 在此不推荐任何模型。', + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.': + '可用磁盘空间不足。适合此 Mac 的最小模型需要下载 {download} GB，而当前可用空间为 {free} GB。请腾出约 {missing} GB。', + 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.': + '此 Mac 运行的是 macOS {version}。本地模型需要 macOS {minimum} 或更高版本。', + 'No coding agent that Codeman can connect to a local model is available on this Mac.': + '此 Mac 上没有 {name} 可以连接到本地模型的编程智能体。', }); const ZH_CN_LOWER = new Map(Object.entries(ZH_CN).map(([key, value]) => [key.toLocaleLowerCase('en'), value])); diff --git a/src/web/public/index.html b/src/web/public/index.html index d7b82775..78211715 100644 --- a/src/web/public/index.html +++ b/src/web/public/index.html @@ -583,6 +583,14 @@
+ + @@ -2867,6 +2875,22 @@ + + +
+

Model on this Mac

+

What the Mac that Codeman runs on can run as a local coding model. Read-only: nothing is downloaded or installed from here.

+
+
+
+
+
+
diff --git a/src/web/public/local-model-ui.js b/src/web/public/local-model-ui.js index 755527bf..ab4a353b 100644 --- a/src/web/public/local-model-ui.js +++ b/src/web/public/local-model-ui.js @@ -1,8 +1,8 @@ /** * @fileoverview The "model on this Mac" card of the welcome screen and its row in App Settings. * - * SKELETON: this file holds no code yet. It will read `GET /api/local-model` (hardware of the - * machine the server runs on, the recommendation, the catalogue date) and show, read-only: + * Reads `GET /api/local-model` (hardware of the machine the server runs on, the recommendation, + * the catalogue date) and shows, read-only: * - the hardware line (chip, memory, free disk); * - the verdict: the headline and notes the server sent, then the pick with its numbers * (model, harness, context, memory need against the budget, download size, speed); @@ -10,24 +10,378 @@ * - one line saying that setting the model up from Codeman is the next step and is not built * yet. * + * ## One body, two containers + * + * The welcome card (`#localModelCard`) and the App Settings row (`#localModelSettings`) are + * painted from the same payload by the same builder (`localModelBody`), so the two can never + * name a different pick. They differ only in chrome: the card has a title and a Hide button and + * folds the alternatives; the row lists them open. + * + * ## When the card shows + * + * The card ships `hidden` and appears only once the server has answered for a Mac. It stays + * hidden while the first request is in flight and after a failed one (nothing is known about the + * machine then, and an error card would be the only thing this feature ever showed on a server + * that is not a Mac), when the answer is `not-macos`, when the request is refused (a non-admin in + * multi-user mode gets a 403, which is an answer: stop asking) and after the user hid it on this + * device. The Settings row is where the loading and failure sentences show, and it is the way + * back to the numbers after a Hide. + * * Rules: - * - Read-only: no button here downloads, installs or writes anything. - * - Hidden when the server is not on macOS (`recommendation.reason` is `not-macos`), when the - * request is refused (a non-admin in multi-user mode gets a 403) and after the user dismissed - * the card on this device (a localStorage key, never a server setting). + * - Read-only: no control here downloads, installs or writes anything but the per-device + * dismissal (a localStorage key, never a server setting). * - Every dynamic string goes in through `textContent`. Model, harness and chip names are - * someone else's text and carry `data-i18n-skip`. + * someone else's text, so both containers carry `data-i18n-skip` and this file translates + * its own sentences at render time. A language or display-name change therefore re-renders + * (`` is written by every i18n `configure()`). * - Names and labels come from the payload: this file holds no list of models or harnesses and * never compares a CLI id with a literal. * - Sizes are printed exactly as sent (`needGb`, `downloadGb`, `budgetGb`, `memGb`, * `freeDiskGb`: decimal GB as text). Bytes are never divided here. * - A sentence the server chose arrives as `{ text, vars }` and goes through - * `codemanT(text, vars)`; this file's own strings are literals in its translator call, so the - * i18n test can harvest them. Both kinds need a zh-CN entry in i18n.js. + * `localModelT(text, vars)`; this file's own strings are literals in that same call, in + * `{placeholder}` form, so the i18n test can harvest them. Both kinds need a zh-CN entry in + * i18n.js. `{name}` is the translator's own slot (the display name) and is never used here. + * - The word "Recommended" is not in this file. Which headline a Mac gets is the server's call. * * @mixin Extends CodemanApp.prototype via Object.assign * @dependency app.js (CodemanApp) * @dependency api-client.js (_api) + * @dependency constants.js (CodemanBase, optional) * @dependency i18n.js (codemanT, optional) * @loadorder 12.59 of 16, after edit-history-ui.js, before entrance-animations.js */ + +/** Per-device dismissal of the welcome card. */ +const LOCAL_MODEL_DISMISS_KEY = 'codeman:local-model-dismissed'; + +/** The server caches its hardware probe for a minute, so asking again sooner cannot change the answer. */ +const LOCAL_MODEL_MAX_AGE_MS = 60000; + +/** i18n.js's translator with the `{placeholder}` slots filled, or the English source filled the same way. */ +function localModelT(source, vars) { + const values = vars && typeof vars === 'object' ? vars : {}; + if (typeof codemanT === 'function') return codemanT(source, values); + return String(source).replace(/\{([a-zA-Z]\w*)\}/g, (_match, key) => String(values[key] ?? '')); +} + +/** The route, under the reverse-proxy base path when there is one. */ +function localModelUrl(path) { + return typeof CodemanBase !== 'undefined' && CodemanBase ? CodemanBase.url(path) : path; +} + +function localModelEl(tag, className, text) { + const el = document.createElement(tag); + if (className) el.className = className; + if (text !== undefined) el.textContent = text; + return el; +} + +/** A sentence the server chose: `{ text, vars }` from the payload. */ +function localModelMessage(message) { + if (!message || typeof message.text !== 'string') return ''; + return localModelT(message.text, message.vars); +} + +/** A token count as people write it (131,072). */ +function localModelCount(value) { + return Number.isFinite(value) ? Number(value).toLocaleString('en-US') : String(value ?? ''); +} + +/** Tokens per second: whole from 10, one decimal below. Empty when the server sent none. */ +function localModelSpeed(value) { + if (!Number.isFinite(value) || value <= 0) return ''; + return String(value >= 10 ? Math.round(value) : Math.round(value * 10) / 10); +} + +/** Whether the page already knows the route will refuse it (a non-admin in multi-user mode). */ +function localModelNotAdmin() { + const me = (typeof window !== 'undefined' && window.__codemanUser) || {}; + return !!(me.multiUser && me.role !== 'admin'); +} + +/** Chip, memory and free disk in one line. Every part but the memory may be missing. */ +function localModelHardwareLine(hw) { + const chip = String(hw.chip || hw.model || ''); + const sizes = { chip, memory: hw.memGb, free: hw.freeDiskGb }; + if (!hw.memGb) return chip; + if (chip && hw.freeDiskGb) return localModelT('{chip}, {memory} GB memory, {free} GB free', sizes); + if (chip) return localModelT('{chip}, {memory} GB memory', sizes); + if (hw.freeDiskGb) return localModelT('{memory} GB memory, {free} GB free', sizes); + return localModelT('{memory} GB memory', sizes); +} + +/** "Qwen3.8 27B (qwen3.8:27b) with OpenCode", then the fit and speed labels the server chose. */ +function localModelCandidateHead(candidate) { + const head = localModelEl('div', 'local-model-name-row'); + const name = localModelT('{model} ({id}) with {harness}', { + model: candidate.modelLabel || candidate.modelId, + id: candidate.modelId, + harness: candidate.cliLabel || candidate.cli, + }); + head.appendChild(localModelEl('span', 'local-model-name', name)); + for (const label of [candidate.fitLabel, candidate.speedLabel]) { + if (typeof label !== 'string' || !label) continue; + head.appendChild(localModelEl('span', 'local-model-pill', localModelT(label))); + } + return head; +} + +/** The dim line under a candidate's name: context, memory need (alternatives), download, license (the pick). */ +function localModelMeta(candidate, { need = false, license = false } = {}) { + const bits = [localModelT('Context {context} tokens', { context: localModelCount(candidate.context) })]; + if (need && candidate.needGb) bits.push(localModelT('{need} GB of memory', { need: candidate.needGb })); + if (candidate.downloadGb) bits.push(localModelT('{download} GB download', { download: candidate.downloadGb })); + if (license && candidate.license) bits.push(String(candidate.license)); + return bits.join(' · '); +} + +/** The pick with its numbers. */ +function localModelPick(pick, recommendation) { + const box = localModelEl('div', 'local-model-pick'); + box.appendChild(localModelCandidateHead(pick)); + box.appendChild(localModelEl('div', 'local-model-line', localModelMeta(pick, { license: true }))); + const sizes = { need: pick.needGb, budget: recommendation.budgetGb }; + // A wired limit is a setting of this Mac; anything else is the formula's estimate, and says so. + const need = + recommendation.budgetSource === 'wired-limit' + ? localModelT('Needs about {need} GB of the {budget} GB this Mac can give a model.', sizes) + : localModelT('Needs about {need} GB of the {budget} GB this Mac can give a model (estimated).', sizes); + box.appendChild(localModelEl('div', 'local-model-line', need)); + const speed = localModelSpeed(pick.tokensPerSecond); + if (speed) { + const line = localModelT('About {speed} tokens per second (estimated).', { speed }); + box.appendChild(localModelEl('div', 'local-model-line', line)); + } + const harness = pick.cliLabel || pick.cli; + if (pick.cliInstalled === false) { + const line = localModelT('{harness} is not installed on this Mac yet.', { harness }); + box.appendChild(localModelEl('div', 'local-model-line', line)); + } + if (pick.slowFirstTurn === true) { + const line = localModelT('With {harness}, the first answer can take minutes.', { harness }); + box.appendChild(localModelEl('div', 'local-model-line', line)); + } + return box; +} + +/** The other models that fit. Folded behind a native disclosure on the welcome card, open in Settings. */ +function localModelAlternatives(alternatives, fold) { + const wrap = localModelEl(fold ? 'details' : 'div', 'local-model-alts'); + const title = localModelEl(fold ? 'summary' : 'div', 'local-model-alts-title'); + title.appendChild(localModelEl('span', '', localModelT('Other models that fit'))); + title.appendChild(localModelEl('span', 'local-model-count', String(alternatives.length))); + wrap.appendChild(title); + for (const candidate of alternatives) { + const row = localModelEl('div', 'local-model-alt'); + row.appendChild(localModelCandidateHead(candidate)); + row.appendChild(localModelEl('div', 'local-model-line', localModelMeta(candidate, { need: true }))); + wrap.appendChild(row); + } + return wrap; +} + +/** + * Everything the server said, in reading order. The ONLY builder: the card and the Settings row + * both call it, see @fileoverview. + */ +function localModelBody(status, { fold }) { + const hardware = status.hardware || {}; + const recommendation = status.recommendation || {}; + const body = localModelEl('div', 'local-model-body'); + // Not a Mac: one plain sentence, and no line calling another machine's parts "this Mac". + const hardwareLine = recommendation.reason === 'not-macos' ? '' : localModelHardwareLine(hardware); + if (hardwareLine) body.appendChild(localModelEl('div', 'local-model-hw', hardwareLine)); + const headline = localModelMessage(recommendation.headline); + if (headline) body.appendChild(localModelEl('div', 'local-model-headline', headline)); + const pick = recommendation.verdict === 'none' ? null : recommendation.pick; + if (pick) body.appendChild(localModelPick(pick, recommendation)); + for (const note of Array.isArray(recommendation.notes) ? recommendation.notes : []) { + const text = localModelMessage(note); + if (text) body.appendChild(localModelEl('div', 'local-model-note', text)); + } + const alternatives = pick && Array.isArray(recommendation.alternatives) ? recommendation.alternatives : []; + if (alternatives.length) body.appendChild(localModelAlternatives(alternatives, fold)); + if (pick) { + const next = localModelT('Setting this up from Codeman is the next step. It is not built yet.'); + body.appendChild(localModelEl('div', 'local-model-next', next)); + } + return body; +} + +/** The `data` of the GET, or null for anything that is not one. */ +function localModelPayload(body) { + const data = body && body.success === true ? body.data : null; + const recommendation = data && typeof data === 'object' ? data.recommendation : null; + if (!recommendation || typeof recommendation !== 'object' || typeof recommendation.verdict !== 'string') return null; + return data; +} + +Object.assign(CodemanApp.prototype, { + // ═══════════════════════════════════════════════════════════════ + // Data + // ═══════════════════════════════════════════════════════════════ + + /** + * Ask the server what this Mac can run, at most once a minute, and paint both containers. + * Called when the welcome screen shows and when App Settings opens. Idempotent: a request in + * flight is shared, and a fresh answer is only re-rendered. + */ + loadLocalModel() { + this._wireLocalModel(); + if (this._localModelRefused()) { + this.renderLocalModel(); + return Promise.resolve(); + } + if (this._localModelInFlight) return this._localModelInFlight; + const fresh = this._localModelStatus && Date.now() - (this._localModelLoadedAt || 0) < LOCAL_MODEL_MAX_AGE_MS; + if (fresh) { + this.renderLocalModel(); + return Promise.resolve(); + } + // An answer already on screen stays there while it is refreshed. + if (!this._localModelStatus) this._localModelState = 'loading'; + this.renderLocalModel(); + this._localModelInFlight = this._fetchLocalModel().finally(() => { + this._localModelInFlight = null; + }); + return this._localModelInFlight; + }, + + async _fetchLocalModel() { + // `_api`, not `_apiJson`: a 403 (admin-only in multi-user mode) is an answer and must be told + // apart from a request that failed. + const res = await this._api(localModelUrl('/api/local-model')); + if (res?.status === 403) { + this._localModelState = 'refused'; + this._localModelStatus = null; + this.renderLocalModel(); + return; + } + let body = null; + if (res?.ok) { + try { + body = await res.json(); + } catch { + body = null; + } + } + const data = localModelPayload(body); + if (!data) { + // A failed refresh must not blank an answer the user is reading: keep the last good one. + if (!this._localModelStatus) this._localModelState = 'failed'; + this.renderLocalModel(); + return; + } + this._localModelStatus = data; + this._localModelState = 'ready'; + this._localModelLoadedAt = Date.now(); + this.renderLocalModel(); + }, + + // ═══════════════════════════════════════════════════════════════ + // Dismissal (per device) + // ═══════════════════════════════════════════════════════════════ + + isLocalModelCardDismissed() { + if (this._localModelDismissed === true) return true; + try { + return localStorage.getItem(LOCAL_MODEL_DISMISS_KEY) === '1'; + } catch { + return false; + } + }, + + /** Hide the welcome card on this device. The Settings row keeps showing the same answer. */ + dismissLocalModelCard() { + // Remembered on the instance too: with storage blocked the card still stays away until a reload. + this._localModelDismissed = true; + try { + localStorage.setItem(LOCAL_MODEL_DISMISS_KEY, '1'); + } catch { + /* storage blocked: hidden for this page load only */ + } + this.renderLocalModel(); + }, + + // ═══════════════════════════════════════════════════════════════ + // Render + // ═══════════════════════════════════════════════════════════════ + + /** Paint every mounted container (welcome card + App Settings row). */ + renderLocalModel() { + const card = document.getElementById('localModelCard'); + if (card) this._renderLocalModelCard(card); + const row = document.getElementById('localModelSettings'); + if (row) this._renderLocalModelSettings(row); + }, + + _localModelRefused() { + return this._localModelState === 'refused' || localModelNotAdmin(); + }, + + _renderLocalModelCard(card) { + // The one piece of state the card holds itself: whether the alternatives were unfolded. + const wasOpen = card.querySelector('details')?.open === true; + card.textContent = ''; + const status = this._localModelStatus; + const hidden = + !status || + this._localModelRefused() || + status.recommendation.reason === 'not-macos' || + this.isLocalModelCardDismissed(); + card.hidden = hidden; + if (hidden) return; + + const head = localModelEl('div', 'local-model-head'); + const title = localModelEl('h3', 'local-model-title', localModelT('Model on this Mac')); + title.id = 'localModelCardTitle'; + head.appendChild(title); + const hide = localModelEl('button', 'local-model-hide', localModelT('Hide')); + hide.type = 'button'; + hide.title = localModelT('Hide this card on this device. It stays in App Settings, under Models.'); + hide.addEventListener('click', () => this.dismissLocalModelCard()); + head.appendChild(hide); + card.appendChild(head); + + const box = localModelEl('div', 'local-model-box'); + box.appendChild(localModelBody(status, { fold: true })); + card.appendChild(box); + const details = card.querySelector('details'); + if (details && wasOpen) details.open = true; + }, + + _renderLocalModelSettings(row) { + row.textContent = ''; + const refused = this._localModelRefused(); + // The whole group goes for a non-admin, the way the other admin-only groups of this modal do. + const group = document.getElementById('localModelGroup'); + if (group) group.style.display = refused ? 'none' : ''; + if (refused) return; + const status = this._localModelStatus; + if (status) { + row.appendChild(localModelBody(status, { fold: false })); + return; + } + const text = + this._localModelState === 'failed' + ? localModelT('Could not reach the server.') + : localModelT('Checking this Mac…'); + row.appendChild(localModelEl('div', 'local-model-headline', text)); + }, + + /** Listeners that outlive a render, wired once. */ + _wireLocalModel() { + if (this._localModelWired) return; + this._localModelWired = true; + // Both containers are data-i18n-skip (see @fileoverview), so the DOM translator never + // revisits them: a language or display-name change has to repaint them from here. + if (typeof MutationObserver === 'function' && document.documentElement) { + const observer = new MutationObserver(() => this.renderLocalModel()); + observer.observe(document.documentElement, { attributes: true, attributeFilter: ['lang'] }); + } + // The real role can arrive after the first render (admin-ui.js fetches /api/me): a non-admin + // loses the card and the group, and an admin the page had taken for one is asked for at last. + document.addEventListener?.('codeman:me', () => void this.loadLocalModel()); + }, +}); diff --git a/src/web/public/mobile.css b/src/web/public/mobile.css index 16e00c1c..a099bfa9 100644 --- a/src/web/public/mobile.css +++ b/src/web/public/mobile.css @@ -1857,6 +1857,17 @@ html.mobile-init .file-browser-panel { margin-top: 0.75rem; } + /* Model on this Mac (local-model-ui.js). Only reached with the phone overview + switched off. The card has no fixed width and every line in it wraps, so + the narrow column only takes some of the spacing back. */ + .local-model-card { + margin-top: 0.6rem; + } + + .local-model-box { + padding: 0.5rem 0.55rem; + } + /* Ralph wizard responsive */ .modal-wizard { display: flex; @@ -3969,6 +3980,16 @@ html:is([data-skin="paper-gray"], [data-skin="solarized-light"], [data-skin="cat max-width: none; } + /* Model on this Mac (local-model-ui.js): its row follows the label size above. */ + #localModelSettings .local-model-headline, + #localModelSettings .local-model-name { + font-size: 0.84rem; + } + + #localModelSettings .local-model-alt .local-model-name { + font-size: 0.78rem; + } + /* Fields go full width under their label instead of fighting for the row */ :is(#appSettingsModal, #sessionOptionsModal, #createCaseModal) .set-row.has-field { flex-direction: column; diff --git a/src/web/public/settings-ui.js b/src/web/public/settings-ui.js index 8b6270b2..dc988bb6 100644 --- a/src/web/public/settings-ui.js +++ b/src/web/public/settings-ui.js @@ -550,6 +550,9 @@ Object.assign(CodemanApp.prototype, { // Assigning .checked above does not fire onchange, so the body's visibility // (and its lazy load) needs an explicit sync on every open, not just a save. this.applyCustomModelEndpointsVisibility(); + // Model on this Mac (local-model-ui.js): a read-only row, not a setting. Server state, + // asked for at most once a minute; nothing here is loaded or saved by id. + this.loadLocalModel?.(); // CLI management (docs/cli-enable-disable-plan.md): synced, default OFF. document.getElementById('appSettingsCliManagement').checked = settings.cliManagementEnabled === true; // Same reasoning as applyCustomModelEndpointsVisibility above: assigning diff --git a/src/web/public/styles.css b/src/web/public/styles.css index 35f95160..f34cd700 100644 --- a/src/web/public/styles.css +++ b/src/web/public/styles.css @@ -22303,6 +22303,203 @@ html[data-tile-anim="soft"] .tile-grid-ghosts.tile-grid-ghosts--release:not(.til line-height: 1.35; } +/* ═══════════════════════════════════════════════════════════════ + Model on this Mac (local-model-ui.js): what the server's Mac can run as + a local coding model. One body (`.local-model-body`), drawn in two places: + the welcome card and a row of App Settings > Models. + + Same vocabulary as the two blocks above it on the welcome screen: a dim + heading row, then one bordered surface with a name over dim meta lines. + Theme tokens only (`--control-bg`, never a white alpha: that fill is + invisible on the light skins), and the fit and speed labels are neutral + pills: the words carry the meaning, so no skin needs a colour of its own. + Nothing here is keyed on a CLI or a model id, and nothing animates. + ═══════════════════════════════════════════════════════════════ */ + +.local-model-card { + display: flex; + flex-direction: column; + gap: 0.4rem; + width: 100%; + margin-top: 0.75rem; + text-align: left; +} + +/* A flex container, so `[hidden]` needs re-asserting or the module's only + visibility lever does nothing (same trap as .foreign-sessions). */ +.local-model-card[hidden] { + display: none; +} + +.local-model-head { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.local-model-title { + margin: 0; + font-size: 0.85rem; + color: var(--text-dim); + font-weight: 500; +} + +.local-model-hide { + margin-left: auto; + font: inherit; + font-size: 0.68rem; + color: var(--text-dim); + background: transparent; + border: 1px solid var(--border); + border-radius: 999px; + padding: 0.12rem 0.5rem; + cursor: pointer; +} + +.local-model-hide:hover { + color: var(--text); + background: var(--control-bg-hover); +} + +.local-model-box { + padding: 0.55rem 0.65rem; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--control-bg); + min-width: 0; +} + +.local-model-body { + display: flex; + flex-direction: column; + gap: 0.3rem; + min-width: 0; +} + +.local-model-hw, +.local-model-line, +.local-model-note { + font-size: 0.68rem; + line-height: 1.45; + color: var(--text-dim); + overflow-wrap: anywhere; +} + +.local-model-headline { + font-size: 0.82rem; + line-height: 1.45; + color: var(--text); + overflow-wrap: anywhere; +} + +/* The pick is the one thing in the card set apart: a rule in the accent, on the + reading side (logical, so it follows a right-to-left page). */ +.local-model-pick { + display: flex; + flex-direction: column; + gap: 0.12rem; + padding-inline-start: 0.55rem; + border-inline-start: 2px solid var(--accent); +} + +.local-model-name-row { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.2rem 0.4rem; + min-width: 0; +} + +.local-model-name { + font-size: 0.82rem; + font-weight: 600; + color: var(--text); + overflow-wrap: anywhere; + min-width: 0; +} + +.local-model-pill, +.local-model-count { + flex: 0 0 auto; + font-size: 0.62rem; + line-height: 1.5; + color: var(--text-dim); + border: 1px solid var(--border); + border-radius: 999px; + padding: 0 0.42rem; + white-space: nowrap; +} + +.local-model-count { + margin-inline-start: 0.4rem; + border-color: transparent; + background: var(--control-bg-hover); +} + +.local-model-alts-title { + font-size: 0.72rem; + color: var(--text-dim); +} + +/* The welcome card folds the list behind a native disclosure (keyboard and + screen-reader behaviour for free); the summary keeps `display: list-item` + so its marker stays. */ +.local-model-alts > summary { + cursor: pointer; +} + +.local-model-alts > summary:hover { + color: var(--text); +} + +.local-model-alt { + display: flex; + flex-direction: column; + gap: 0.1rem; + margin-top: 0.3rem; + padding-top: 0.3rem; + border-top: 1px solid var(--border); +} + +.local-model-alt .local-model-name { + font-size: 0.76rem; + font-weight: 500; +} + +.local-model-next { + font-size: 0.66rem; + line-height: 1.45; + color: var(--text-muted); +} + +/* In App Settings the body sits inside an ordinary .set-row, so it takes that + surface's type scale (label 0.76rem over a muted description) and the row + supplies the border. */ +#localModelSettings { + padding: 3px 0; +} + +#localModelSettings .local-model-headline, +#localModelSettings .local-model-name { + font-size: 0.76rem; +} + +#localModelSettings .local-model-alt .local-model-name { + font-size: 0.72rem; +} + +/* Touch: the two things in the card that take a press get a finger's height. */ +@media (pointer: coarse) { + .local-model-hide { + min-height: 32px; + padding: 0.3rem 0.8rem; + } + + .local-model-alts > summary { + padding: 0.4rem 0; + } +} + /* XLSX preview in the file-preview overlay (spreadsheet-preview.js). Read-only; cells are absolutely positioned inside a virtualized grid, so only the visible tile is in the DOM. */ .spreadsheet-preview-shell { diff --git a/src/web/public/terminal-ui.js b/src/web/public/terminal-ui.js index 75419a9f..949cbbc8 100644 --- a/src/web/public/terminal-ui.js +++ b/src/web/public/terminal-ui.js @@ -2964,6 +2964,9 @@ Object.assign(CodemanApp.prototype, { // up (stopped in hideWelcome), see foreign-sessions.js. this.wireForeignSessions?.(); this.startForeignPolling?.(); + // What this Mac can run as a local model (local-model-ui.js). One request, + // at most once a minute, no timer: nothing to stop in hideWelcome. + this.loadLocalModel?.(); } // Home screen has no input target — hide the CJK textarea (activeSessionId // is null by the time we get here). Guarded: defined on the app object. diff --git a/test/frontend-cli-no-id-branching.test.ts b/test/frontend-cli-no-id-branching.test.ts index 41a1221c..9eee6e28 100644 --- a/test/frontend-cli-no-id-branching.test.ts +++ b/test/frontend-cli-no-id-branching.test.ts @@ -38,6 +38,9 @@ const SCANNED_FILES = [ 'panels-ui.js', // A row's mode comes from the same catalog, never from a list of ids here. 'foreign-sessions.js', + // The harness a local model is paired with arrives as a label in the payload + // (`Candidate.cliLabel`); the card names no CLI and compares no id. + 'local-model-ui.js', ]; /** diff --git a/test/local-model-i18n.test.ts b/test/local-model-i18n.test.ts new file mode 100644 index 00000000..cbcb514f --- /dev/null +++ b/test/local-model-i18n.test.ts @@ -0,0 +1,172 @@ +/** + * @fileoverview The "Model on this Mac" card and its App Settings group in Chinese (zh-CN). + * + * Three sources of text, all checked against the real dictionary in i18n.js: + * - the module's own strings, HARVESTED from the real local-model-ui.js (every literal it + * passes to `localModelT`), so a new string without an entry fails here instead of showing + * up English on one surface; + * - every sentence and label the SERVER may send (`LOCAL_MODEL_LABELS`, imported from the + * contract, so a label added there fails here until it has an entry); + * - the static heading and hint of the Settings group, read from the real index.html and run + * through the real DOM translator. + * + * Both containers carry data-i18n-skip (model, harness and chip names are someone else's + * text), so nothing in them is reached by the DOM translator: these strings translate only + * through the module's own call, which is why they are checked through `t()` here. + * + * Beyond "it is Chinese now", each entry must keep its `{placeholder}` slots: a translation + * that drops one prints a sentence without its number. + * + * Port: N/A. + */ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import vm from 'node:vm'; +import { JSDOM } from 'jsdom'; +import { afterAll, describe, expect, it } from 'vitest'; +import { LOCAL_MODEL_LABELS, LOCAL_MODEL_NONE_LABELS } from '../src/types.js'; + +const PUBLIC = resolve(import.meta.dirname, '../src/web/public'); +const I18N = readFileSync(resolve(PUBLIC, 'i18n.js'), 'utf8'); +const INDEX = readFileSync(resolve(PUBLIC, 'index.html'), 'utf8'); +const MODULE = readFileSync(resolve(PUBLIC, 'local-model-ui.js'), 'utf8'); + +type Api = { + configure(o: Record): void; + start(): void; + t(source: string, vars?: Record): string; +}; + +/** Every literal the module hands to its translator call (Prettier may break after the paren). */ +const FROM_MODULE = [...new Set([...MODULE.matchAll(/localModelT\(\s*'([^']+)'/g)].map((m) => m[1]))]; +/** Every text the server may send. */ +const FROM_SERVER: string[] = Object.values(LOCAL_MODEL_LABELS); +/** Already in the dictionary for the outside-sessions block; reused here for a failed request. */ +const SHARED = ['Could not reach the server.']; +const OWN = FROM_MODULE.filter((source) => !SHARED.includes(source)); + +/** A page with only the translator in it: enough for `t()`, and cheap to re-configure. */ +const small = new JSDOM('', { + runScripts: 'outside-only', + url: 'http://localhost/', +}); +vm.runInContext(I18N, small.getInternalVMContext(), { filename: 'i18n.js' }); +const api = (small.window as unknown as { CodemanI18n: Api }).CodemanI18n; +api.configure({ language: 'zh-CN' }); + +/** The real page, translated once, for the strings that live in index.html. */ +const page = new JSDOM(INDEX, { runScripts: 'outside-only', url: 'http://localhost/' }); +vm.runInContext(I18N, page.getInternalVMContext(), { filename: 'i18n.js' }); +const pageApi = (page.window as unknown as { CodemanI18n: Api }).CodemanI18n; +pageApi.start(); +pageApi.configure({ language: 'zh-CN' }); + +afterAll(() => { + small.window.close(); + page.window.close(); +}); + +/** The Settings group's static strings, as index.html ships them (English). */ +const GROUP_MARKUP = INDEX.match( + /
\s*

([^<]+)<\/h4><\/div>\s*

([^<]+)<\/p>/ +); +const STATIC = GROUP_MARKUP ? [GROUP_MARKUP[1].trim(), GROUP_MARKUP[2].trim()] : []; + +const ALL = [...new Set([...OWN, ...FROM_SERVER, ...STATIC])]; + +/** The `{slot}` names of a source text. */ +const slots = (text: string) => [...text.matchAll(/\{([a-zA-Z]\w*)\}/g)].map((m) => m[1]); + +/** U+2013, U+2014 and U+2015, built from their code points so this file holds none of them. */ +const LONG_DASH = new RegExp(`[${String.fromCharCode(0x2013, 0x2014, 0x2015)}]`); + +/** What may stay Latin: the product, the platform and its vendors, the unit, and Token (the dictionary's own word). */ +const leftover = (text: string) => + text.replace(/\b(Codeman|macOS|Mac|GB|Apple|Intel|Token)\b/g, '').match(/[A-Za-z]+/g) ?? []; + +describe('the local model strings in zh-CN', () => { + it('harvests the module (the check is not vacuous)', () => { + expect(FROM_MODULE).toContain('Model on this Mac'); + expect(FROM_MODULE).toContain('{model} ({id}) with {harness}'); + expect(FROM_MODULE).toContain('Needs about {need} GB of the {budget} GB this Mac can give a model (estimated).'); + expect(FROM_MODULE).toContain('Setting this up from Codeman is the next step. It is not built yet.'); + expect(FROM_MODULE).toContain('Checking this Mac…'); + expect(FROM_MODULE.length).toBeGreaterThanOrEqual(18); + // The only other way the module calls its translator is with payload text (a server + // sentence or label), so no string of its own can reach the page unharvested. + const code = MODULE.split('\n') + .filter((line) => !/^\s*(\/\/|\*|\/\*)/.test(line)) + .join('\n'); + const calls = code.split('localModelT(').length - 1; + const literalCalls = [...code.matchAll(/localModelT\(\s*'/g)].length; + const payloadCalls = [...code.matchAll(/localModelT\((message\.text, message\.vars|label)\)/g)].length; + // One more: the function's own declaration. + expect(calls).toBe(literalCalls + payloadCalls + 1); + expect(payloadCalls).toBe(2); + }); + + it('lists every server label, the none reasons included', () => { + // A label added to the contract later fails below by its text, not here by a count. + expect(FROM_SERVER.length).toBeGreaterThanOrEqual(14); + for (const text of Object.values(LOCAL_MODEL_NONE_LABELS)) expect(FROM_SERVER).toContain(text); + expect(FROM_SERVER).toContain('Best fit for this Mac'); + expect(FROM_SERVER).toContain('Speed unknown'); + }); + + it('never uses the translator slot {name} for a value of its own', () => { + for (const text of ALL) expect(slots(text), text).not.toContain('name'); + }); + + it.each(ALL)('translates %s, with no English left', (text) => { + const chinese = api.t(text); + expect(chinese).not.toBe(text); + expect(leftover(chinese)).toEqual([]); + // No long dash in any form: commas, colons and parentheses only. + expect(chinese).not.toMatch(LONG_DASH); + }); + + it.each(ALL.filter((text) => slots(text).length > 0))('keeps every slot of %s', (text) => { + // Digits only, so a marker is neither translated nor counted as leftover English. + const vars = Object.fromEntries(slots(text).map((slot, i) => [slot, `90${i}1`])); + const chinese = api.t(text, vars); + for (const value of Object.values(vars)) expect(chinese, `${text} lost a slot`).toContain(value); + expect(chinese).not.toMatch(/[{}]/); + }); + + it('keeps the product name a slot, so a custom display name reaches the Chinese text', () => { + api.configure({ language: 'zh-CN', displayName: 'Acme' }); + try { + for (const text of ALL.filter((candidate) => candidate.includes('Codeman'))) { + const chinese = api.t(text); + expect(chinese, text).toContain('Acme'); + expect(chinese, text).not.toContain('Codeman'); + } + } finally { + api.configure({ language: 'zh-CN', displayName: 'Codeman' }); + } + }); + + it('adds each key to the dictionary once', () => { + // A repeated key in the object literal silently replaces the first. Prettier leaves a key + // that is one word unquoted. + for (const text of ALL) { + const bare = /^\w+$/.test(text) ? I18N.split(`\n ${text}: `).length - 1 : 0; + expect(I18N.split(`'${text}':`).length - 1 + bare, text).toBe(1); + } + }); + + it('translates the Settings group heading and hint in the page, and leaves both containers alone', () => { + expect(STATIC).toHaveLength(2); + expect(STATIC[0]).toBe('Model on this Mac'); + const translated = page.window.document.getElementById('localModelGroup')!; + const heading = translated.querySelector('.set-group-head h4')!.textContent!.trim(); + const hint = translated.querySelector('.set-group-hint')!.textContent!.trim(); + expect(heading).toBe('此 Mac 上的模型'); + expect(hint).not.toBe(STATIC[1]); + expect(hint).toBe(api.t(STATIC[1])); + expect(leftover(hint)).toEqual([]); + // Written only by the module, through its own translator call. + expect(INDEX).toMatch(/id="localModelCard"[^>]*data-i18n-skip/); + expect(INDEX).toMatch(/id="localModelSettings"[^>]*data-i18n-skip/); + }); +}); diff --git a/test/local-model-ui.test.ts b/test/local-model-ui.test.ts new file mode 100644 index 00000000..1689d6ff --- /dev/null +++ b/test/local-model-ui.test.ts @@ -0,0 +1,797 @@ +/** + * @fileoverview The "Model on this Mac" card of the welcome screen and its App Settings row + * (src/web/public/local-model-ui.js), driven in JSDOM over the REAL index.html with the real + * module and a mocked `_api`. + * + * What is pinned: + * - every state the route can produce: loading, refused (403), failed, not macOS, each `none` + * reason, `limited`, `ok`; + * - the card stays hidden until the server answered for a Mac, and the Settings row carries + * the loading and failure sentences; + * - the card and the row show the SAME pick (one builder), the card folds the alternatives; + * - every payload string lands as text: an `` in a model, harness or chip name, + * or in a slot value of a server sentence, creates no element and runs nothing; + * - sizes are printed as sent (decimal GB text), and the word "Recommended" is the server's + * to send, never this module's; + * - the dismissal is a per-device localStorage key and survives a new app instance; + * - one request a minute at most, shared while in flight, and a failed refresh keeps the answer. + * + * The route answers 501 until the server stage lands, so the payloads here are fixtures typed + * with the contract's `LocalModelStatus`, built from `LOCAL_MODEL_LABELS`. + * + * Port: none. + */ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import vm from 'node:vm'; +import { JSDOM } from 'jsdom'; +import { afterAll, afterEach, describe, expect, it, vi } from 'vitest'; +import { LOCAL_MODEL_LABELS, LOCAL_MODEL_NONE_LABELS } from '../src/types.js'; +import type { + Candidate, + HardwareFacts, + LocalModelMessage, + LocalModelStatus, + NoneReason, + Recommendation, +} from '../src/types.js'; + +const PUBLIC = resolve(import.meta.dirname, '../src/web/public'); +const INDEX = readFileSync(resolve(PUBLIC, 'index.html'), 'utf8'); +const MODULE = readFileSync(resolve(PUBLIC, 'local-model-ui.js'), 'utf8'); +const I18N = readFileSync(resolve(PUBLIC, 'i18n.js'), 'utf8'); + +const DISMISS_KEY = 'codeman:local-model-dismissed'; +const NEXT_STEP = 'Setting this up from Codeman is the next step. It is not built yet.'; + +type ApiResponse = { ok: boolean; status: number; json(): Promise } | null; + +interface App { + _api: ReturnType; + loadLocalModel(): Promise; + renderLocalModel(): void; + dismissLocalModelCard(): void; + isLocalModelCardDismissed(): boolean; +} + +interface I18nApi { + configure(options: Record): void; + start(): void; +} + +/** + * The two containers exactly as index.html ships them, lifted out of the real page once. Each + * test then runs in a small page made of that markup: parsing the whole of index.html for + * every case would cost the gate half a minute and prove nothing more. + */ +const REAL = new JSDOM(INDEX); +const REAL_DOCUMENT = REAL.window.document; +const CARD_MARKUP = REAL_DOCUMENT.getElementById('localModelCard')?.outerHTML ?? ''; +const GROUP_MARKUP = REAL_DOCUMENT.getElementById('localModelGroup')?.outerHTML ?? ''; +const PAGE = ` +

${CARD_MARKUP}
+
${GROUP_MARKUP}
+`; + +const open: JSDOM[] = []; + +afterEach(() => { + for (const dom of open.splice(0)) dom.window.close(); +}); + +afterAll(() => { + REAL.window.close(); +}); + +/** A response as `_api` hands it over, the server's envelope included. */ +function ok(data: unknown): ApiResponse { + return { ok: true, status: 200, json: async () => ({ success: true, data }) }; +} + +function refusal(status: number, errorCode: string): ApiResponse { + return { ok: false, status, json: async () => ({ success: false, error: 'no', errorCode }) }; +} + +/** + * The real markup with the real module in it. `CodemanApp` is the only stand-in: app.js defines + * it as a classic-script global, and the module mixes into its prototype. + */ +function mount(options: { i18n?: boolean; before?: string } = {}) { + const dom = new JSDOM(PAGE, { runScripts: 'outside-only', url: 'http://localhost/' }); + open.push(dom); + const context = dom.getInternalVMContext(); + if (options.i18n) vm.runInContext(I18N, context, { filename: 'i18n.js' }); + if (options.before) vm.runInContext(options.before, context); + vm.runInContext('function CodemanApp() {}', context); + vm.runInContext(MODULE, context, { filename: 'local-model-ui.js' }); + const document = dom.window.document; + const newApp = (response?: ApiResponse | (() => Promise)) => { + const app = vm.runInContext('new CodemanApp()', context) as App; + app._api = vi.fn(async () => (typeof response === 'function' ? response() : (response ?? null))); + return app; + }; + return { + dom, + document, + newApp, + card: document.getElementById('localModelCard')!, + row: document.getElementById('localModelSettings')!, + group: document.getElementById('localModelGroup')!, + i18n: (dom.window as unknown as { CodemanI18n: I18nApi }).CodemanI18n, + }; +} + +/** Text of every element with that class under a root, in document order. */ +function texts(root: Element, selector: string): string[] { + return [...root.querySelectorAll(selector)].map((el) => el.textContent ?? ''); +} + +/** Lets a queued MutationObserver callback (a microtask in JSDOM) run. */ +const settle = () => new Promise((done) => setTimeout(done, 0)); + +// ---------- Fixtures (the contract's types) ---------- + +/** The MacBook of the probe: 48 GB as sold, a chip that is in no chips row. */ +const HARDWARE: HardwareFacts = { + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 51539607552, + memGb: '51.5', + memClassGb: 48, + chip: 'Apple M5 Pro', + model: 'Mac17,13', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: 912308310016, + freeDiskGb: '912', + source: 'sysctl', +}; + +function candidate(over: Partial = {}): Candidate { + return { + modelId: 'qwen3.8:27b', + modelLabel: 'Qwen3.8 27B', + license: 'Apache-2.0', + paramsB: 27.8, + context: 131072, + cli: 'opencode', + cliLabel: 'OpenCode', + cliInstalled: true, + needBytes: 29551160064, + needGb: '29.6', + downloadBytes: 17740000000, + downloadGb: '17.7', + budgetShare: 0.79, + fit: 'good', + fitLabel: LOCAL_MODEL_LABELS.fitGood, + speed: 'unknown', + speedLabel: LOCAL_MODEL_LABELS.speedUnknown, + tested: false, + ...over, + }; +} + +const ALT_SLOW = candidate({ + modelId: 'qwen3.8:27b-q8_0', + modelLabel: 'Qwen3.8 27B Q8', + context: 65536, + needGb: '36.4', + downloadGb: '30.0', + budgetShare: 0.97, + fit: 'tight', + fitLabel: LOCAL_MODEL_LABELS.fitTight, + speed: 'slow', + speedLabel: LOCAL_MODEL_LABELS.speedSlow, + tokensPerSecond: 5.2, +}); + +const ALT_FAST = candidate({ + modelId: 'qwen3.6:35b-a3b', + modelLabel: 'Qwen3.6 35B-A3B', + needGb: '26.4', + downloadGb: '22.6', + budgetShare: 0.7, + speed: 'ok', + speedLabel: undefined, + tokensPerSecond: 41.7, +}); + +function status(recommendation: Partial, hardware: Partial = {}): LocalModelStatus { + return { + hardware: { ...HARDWARE, ...hardware }, + recommendation: { + verdict: 'ok', + headline: { text: LOCAL_MODEL_LABELS.headlineOk }, + notes: [], + alternatives: [], + budgetBytes: 37580963840, + budgetGb: '37.6', + budgetSource: 'estimated', + ...recommendation, + }, + catalogUpdated: '2026-11-06', + }; +} + +const OK_STATUS = status({ + notes: [{ text: LOCAL_MODEL_LABELS.noteUntested }], + pick: candidate(), + alternatives: [ALT_FAST, ALT_SLOW], +}); + +const LIMITED_STATUS = status( + { + verdict: 'limited', + headline: { text: LOCAL_MODEL_LABELS.headlineLimited }, + notes: [{ text: LOCAL_MODEL_LABELS.noteUntested }], + pick: candidate({ + modelId: 'qwen3.5:9b', + modelLabel: 'Qwen3.5 9B', + paramsB: 9.7, + context: 65536, + needGb: '10.9', + downloadGb: '6.6', + budgetShare: 0.87, + fit: 'tight', + fitLabel: LOCAL_MODEL_LABELS.fitTight, + speed: 'ok', + speedLabel: undefined, + tokensPerSecond: 11.16, + }), + budgetGb: '12.5', + }, + { memGb: '19.3', memClassGb: 18, chip: 'Apple M4' } +); + +/** A `none` answer: no pick, the reason's sentence as headline, its slots in `vars`. */ +function none(reason: NoneReason, vars?: Record, hardware: Partial = {}) { + const headline: LocalModelMessage = { text: LOCAL_MODEL_NONE_LABELS[reason], ...(vars ? { vars } : {}) }; + return status({ verdict: 'none', reason, headline, budgetBytes: 0, budgetGb: '0.0' }, hardware); +} + +const NONE_CASES: Array<{ reason: Exclude; answer: LocalModelStatus; sentence: string }> = [ + { + reason: 'intel', + answer: none('intel', undefined, { arch: 'x64', arm64: false, chip: 'Intel(R) Core(TM) i7-9750H' }), + sentence: 'This Mac has an Intel processor. Local coding models need Apple silicon.', + }, + { + reason: 'memory', + answer: none('memory', { memory: '8.6' }, { memGb: '8.6', memClassGb: 8 }), + sentence: + 'This Mac has 8.6 GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', + }, + { + reason: 'disk', + answer: none('disk', { download: '17.7', free: '12.0', missing: '16.4' }, { freeDiskGb: '12.0' }), + sentence: + 'Not enough free disk space. The smallest model that fits this Mac is a 17.7 GB download and 12.0 GB is free. Free up about 16.4 GB.', + }, + { + reason: 'macos', + answer: none('macos', { version: '13.6', minimum: '14' }, { macosVersion: '13.6', macosMajor: 13 }), + sentence: 'This Mac runs macOS 13.6. Local models need macOS 14 or newer.', + }, + { + reason: 'no-harness', + answer: none('no-harness'), + sentence: 'No coding agent that Codeman can connect to a local model is available on this Mac.', + }, +]; + +// ---------- The page ---------- + +describe('the markup the module fills', () => { + it('ships the card hidden on the welcome screen and one read-only group in Settings > Models', () => { + const card = REAL_DOCUMENT.getElementById('localModelCard')!; + const row = REAL_DOCUMENT.getElementById('localModelSettings')!; + const group = REAL_DOCUMENT.getElementById('localModelGroup')!; + expect(card.hidden).toBe(true); + expect(card.closest('#welcomeOverlay .welcome-content')).not.toBeNull(); + expect(card.hasAttribute('data-i18n-skip')).toBe(true); + // A direct child of the column (no wrapper that would hold a margin open while the card is + // hidden), under the Resume list and above the sessions opened outside Codeman. + const order = [...REAL_DOCUMENT.querySelector('.welcome-content')!.children].map((el) => el.id || el.className); + expect(order.indexOf('localModelCard')).toBeGreaterThan(order.indexOf('historySessions')); + expect(order.indexOf('localModelCard')).toBeLessThan(order.indexOf('welcome-foreign')); + // The phone overview is a different screen and does not carry the card. + expect(card.closest('#mobileOverview')).toBeNull(); + + expect(group.closest('#appSettingsModal #settings-models')).not.toBeNull(); + // Its own group: the endpoints toggle hides #customModelEndpointsBody, and this answer does not depend on it. + expect(row.closest('#customModelEndpointsBody')).toBeNull(); + expect(row.closest('#customModelEndpointsGroup')).toBeNull(); + expect(row.closest('.set-row')).not.toBeNull(); + expect(row.hasAttribute('data-i18n-skip')).toBe(true); + // Read-only: no control of any kind in the group. + expect(group.querySelectorAll('input, select, button, textarea').length).toBe(0); + expect(INDEX.split('').length - 1).toBe(1); + }); + + it('is what every other test here runs on', () => { + const { card, row, group } = mount(); + expect(card.outerHTML).toBe(CARD_MARKUP); + expect(group.outerHTML).toBe(GROUP_MARKUP); + expect(row.textContent).toBe(''); + }); + + it('is wired into the two places that show it', () => { + const read = (file: string) => readFileSync(resolve(PUBLIC, file), 'utf8'); + // The welcome screen (not the phone overview branch above it) and App Settings. + expect(read('terminal-ui.js').split('this.loadLocalModel?.();').length - 1).toBe(1); + expect(read('settings-ui.js').split('this.loadLocalModel?.();').length - 1).toBe(1); + }); +}); + +describe('before there is an answer', () => { + it('keeps the card hidden while the request is in flight, and says so in Settings', async () => { + const { newApp, card, row } = mount(); + let answer!: (response: ApiResponse) => void; + const app = newApp(() => new Promise((done) => (answer = done))); + const loading = app.loadLocalModel(); + expect(card.hidden).toBe(true); + expect(card.textContent).toBe(''); + expect(row.textContent).toBe('Checking this Mac…'); + answer(ok(OK_STATUS)); + await loading; + expect(card.hidden).toBe(false); + expect(row.textContent).not.toContain('Checking this Mac…'); + }); + + it.each([ + ['the route answers 501', refusal(501, 'OPERATION_FAILED')], + ['the request never reached the server', null], + ['the body is not the envelope', { ok: true, status: 200, json: async () => ({ hello: 'world' }) }], + ['the body is not JSON', { ok: true, status: 200, json: async () => Promise.reject(new Error('bad')) }], + ['the data carries no recommendation', ok({ hardware: HARDWARE })], + ] as Array<[string, ApiResponse]>)('shows a failed request only in Settings: %s', async (_name, response) => { + const { newApp, card, row, group } = mount(); + const app = newApp(response); + await app.loadLocalModel(); + // Nothing is known about the machine, so the welcome screen gets no card at all. + expect(card.hidden).toBe(true); + expect(group.style.display).toBe(''); + expect(row.textContent).toBe('Could not reach the server.'); + }); + + it('asks again the next time it is shown after a failure', async () => { + const { newApp, card, row } = mount(); + const answers: ApiResponse[] = [null, ok(OK_STATUS)]; + const app = newApp(async () => answers.shift() ?? null); + await app.loadLocalModel(); + expect(row.textContent).toBe('Could not reach the server.'); + await app.loadLocalModel(); + expect(app._api).toHaveBeenCalledTimes(2); + expect(card.hidden).toBe(false); + expect(texts(row, '.local-model-headline')).toEqual(['Best fit for this Mac']); + }); +}); + +describe('a refused request (a non-admin in multi-user mode)', () => { + it('shows neither the card nor the Settings group, and stops asking', async () => { + const { newApp, card, row, group } = mount(); + const app = newApp(refusal(403, 'FORBIDDEN')); + await app.loadLocalModel(); + expect(card.hidden).toBe(true); + expect(group.style.display).toBe('none'); + expect(row.textContent).toBe(''); + await app.loadLocalModel(); + expect(app._api).toHaveBeenCalledTimes(1); + }); + + it('does not ask at all once the page knows the role, and asks when the role turns out to be admin', async () => { + const { dom, document, newApp, card, group } = mount(); + const win = dom.window as unknown as { __codemanUser?: { multiUser: boolean; role: string } }; + win.__codemanUser = { multiUser: true, role: 'user' }; + const app = newApp(ok(OK_STATUS)); + await app.loadLocalModel(); + expect(app._api).not.toHaveBeenCalled(); + expect(group.style.display).toBe('none'); + expect(card.hidden).toBe(true); + + win.__codemanUser = { multiUser: true, role: 'admin' }; + document.dispatchEvent(new dom.window.CustomEvent('codeman:me')); + await vi.waitFor(() => expect(card.hidden).toBe(false)); + expect(app._api).toHaveBeenCalledTimes(1); + expect(group.style.display).toBe(''); + }); +}); + +describe('a server that is not a Mac', () => { + it('hides the card and leaves one plain sentence in Settings', async () => { + const { newApp, card, row, group } = mount(); + const answer = none('not-macos', undefined, { + platform: 'linux', + arch: 'x64', + arm64: undefined, + chip: 'AMD Ryzen 9 7950X', + source: 'not-probed', + }); + await newApp(ok(answer)).loadLocalModel(); + expect(card.hidden).toBe(true); + expect(card.textContent).toBe(''); + expect(group.style.display).toBe(''); + expect(row.textContent).toBe('Codeman recommends a local model only on a Mac with Apple silicon.'); + // No line listing another machine's parts under a heading that says "this Mac". + expect(row.querySelector('.local-model-hw')).toBeNull(); + }); +}); + +describe('verdict none on a Mac', () => { + it.each(NONE_CASES)('says why for $reason, with the numbers the server sent', async ({ answer, sentence }) => { + const { newApp, card, row } = mount(); + await newApp(ok(answer)).loadLocalModel(); + expect(card.hidden).toBe(false); + for (const root of [card, row]) { + expect(texts(root, '.local-model-headline')).toEqual([sentence]); + expect(texts(root, '.local-model-hw')).toHaveLength(1); + expect(root.querySelector('.local-model-pick')).toBeNull(); + expect(root.querySelector('.local-model-alts')).toBeNull(); + // Nothing to set up, so no line about setting it up. + expect(root.textContent).not.toContain(NEXT_STEP); + expect(root.textContent).not.toContain('Recommended'); + } + }); + + it('covers every reason the contract knows', () => { + const covered = [...NONE_CASES.map((c) => c.reason), 'not-macos'].sort(); + expect(covered).toEqual(Object.keys(LOCAL_MODEL_NONE_LABELS).sort()); + }); +}); + +describe('verdict ok', () => { + it('shows the hardware, the headline, the pick with its numbers, the note and the next step', async () => { + const { newApp, card } = mount(); + await newApp(ok(OK_STATUS)).loadLocalModel(); + expect(card.hidden).toBe(false); + expect(texts(card, '.local-model-title')).toEqual(['Model on this Mac']); + expect(texts(card, '.local-model-hw')).toEqual(['Apple M5 Pro, 51.5 GB memory, 912 GB free']); + expect(texts(card, '.local-model-headline')).toEqual(['Best fit for this Mac']); + expect(texts(card, '.local-model-note')).toEqual(['Not yet tested by us on a Mac like this.']); + + const pick = card.querySelector('.local-model-pick')!; + expect(texts(pick, '.local-model-name')).toEqual(['Qwen3.8 27B (qwen3.8:27b) with OpenCode']); + expect(texts(pick, '.local-model-pill')).toEqual(['Good', 'Speed unknown']); + expect(texts(pick, '.local-model-line')).toEqual([ + 'Context 131,072 tokens · 17.7 GB download · Apache-2.0', + 'Needs about 29.6 GB of the 37.6 GB this Mac can give a model (estimated).', + ]); + expect(texts(card, '.local-model-next')).toEqual([NEXT_STEP]); + }); + + it('never says "Recommended" for a pick nobody tested: that word is the server\'s to send', async () => { + const { newApp, card, row } = mount(); + await newApp(ok(OK_STATUS)).loadLocalModel(); + expect(card.textContent).not.toMatch(/recommend/i); + expect(row.textContent).not.toMatch(/recommend/i); + // The module holds no such string of its own (comments aside). + const code = MODULE.split('\n') + .filter((line) => !/^\s*(\/\/|\*|\/\*)/.test(line)) + .join('\n'); + expect(code).not.toMatch(/Recommended/); + }); + + it('shows the tested headline only because the server sent it', async () => { + const { newApp, card } = mount(); + const tested = status({ headline: { text: LOCAL_MODEL_LABELS.headlineTested }, pick: candidate({ tested: true }) }); + await newApp(ok(tested)).loadLocalModel(); + expect(texts(card, '.local-model-headline')).toEqual(['Recommended for this Mac']); + }); + + it('lists the other models that fit with Good, Tight, Slow or Speed unknown', async () => { + const { newApp, card } = mount(); + await newApp(ok(OK_STATUS)).loadLocalModel(); + const alts = card.querySelector('.local-model-alts')!; + expect(texts(alts, '.local-model-alts-title')).toEqual(['Other models that fit2']); + expect(texts(alts, '.local-model-count')).toEqual(['2']); + const rows = [...alts.querySelectorAll('.local-model-alt')]; + expect(rows.map((row) => texts(row, '.local-model-name')[0])).toEqual([ + 'Qwen3.6 35B-A3B (qwen3.6:35b-a3b) with OpenCode', + 'Qwen3.8 27B Q8 (qwen3.8:27b-q8_0) with OpenCode', + ]); + // A candidate whose speed is fine carries a fit label only. + expect(rows.map((row) => texts(row, '.local-model-pill'))).toEqual([['Good'], ['Tight', 'Slow']]); + expect(rows.map((row) => texts(row, '.local-model-line')[0])).toEqual([ + 'Context 131,072 tokens · 26.4 GB of memory · 22.6 GB download', + 'Context 65,536 tokens · 36.4 GB of memory · 30.0 GB download', + ]); + }); + + it('prints every size exactly as sent, in one unit', async () => { + const { newApp, card, row } = mount(); + await newApp(ok(OK_STATUS)).loadLocalModel(); + for (const root of [card, row]) { + const text = root.textContent ?? ''; + expect(text).not.toMatch(/GiB|MiB|\bMB\b|bytes/); + // Every number in front of the unit is one the payload carries as text. + const printed = [...text.matchAll(/([\d.,]+) GB/g)].map((m) => m[1]); + const sent = ['51.5', '912', '37.6', '29.6', '17.7', '26.4', '22.6', '36.4', '30.0']; + expect(printed.length).toBeGreaterThanOrEqual(sent.length); + expect(printed.filter((value) => !sent.includes(value))).toEqual([]); + } + }); + + it('calls a budget that is the wired limit of this Mac a fact, not an estimate', async () => { + const { newApp, card } = mount(); + const wired = status({ pick: candidate(), budgetSource: 'wired-limit', budgetGb: '40.0' }); + await newApp(ok(wired)).loadLocalModel(); + expect(texts(card, '.local-model-pick .local-model-line')[1]).toBe( + 'Needs about 29.6 GB of the 40.0 GB this Mac can give a model.' + ); + }); + + it('adds the speed, a missing harness and a slow first answer only when the payload says so', async () => { + const { newApp, card } = mount(); + const pick = candidate({ + speed: 'ok', + speedLabel: undefined, + tokensPerSecond: 12.3, + cliInstalled: false, + slowFirstTurn: true, + cliLabel: 'Some Harness', + }); + await newApp(ok(status({ pick }))).loadLocalModel(); + expect(texts(card, '.local-model-pick .local-model-pill')).toEqual(['Good']); + expect(texts(card, '.local-model-pick .local-model-line').slice(2)).toEqual([ + 'About 12 tokens per second (estimated).', + 'Some Harness is not installed on this Mac yet.', + 'With Some Harness, the first answer can take minutes.', + ]); + }); + + it('describes the hardware with whatever parts the probe got', async () => { + const { newApp, card } = mount(); + const cases: Array<[Partial, string]> = [ + [{ freeDiskBytes: undefined, freeDiskGb: undefined }, 'Apple M5 Pro, 51.5 GB memory'], + [{ chip: undefined }, 'Mac17,13, 51.5 GB memory, 912 GB free'], + [{ chip: undefined, model: undefined }, '51.5 GB memory, 912 GB free'], + [{ chip: undefined, model: undefined, freeDiskBytes: undefined, freeDiskGb: undefined }, '51.5 GB memory'], + ]; + for (const [hardware, line] of cases) { + await newApp(ok(status({ pick: candidate() }, hardware))).loadLocalModel(); + expect(texts(card, '.local-model-hw')).toEqual([line]); + } + }); +}); + +describe('verdict limited', () => { + it('says what a small model is good for, shows the pick, and never recommends', async () => { + const { newApp, card, row } = mount(); + await newApp(ok(LIMITED_STATUS)).loadLocalModel(); + expect(card.hidden).toBe(false); + for (const root of [card, row]) { + expect(texts(root, '.local-model-headline')).toEqual([ + 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', + ]); + expect(texts(root, '.local-model-hw')).toEqual(['Apple M4, 19.3 GB memory, 912 GB free']); + expect(texts(root, '.local-model-name')).toEqual(['Qwen3.5 9B (qwen3.5:9b) with OpenCode']); + expect(texts(root, '.local-model-pill')).toEqual(['Tight']); + expect(texts(root, '.local-model-pick .local-model-line')).toEqual([ + 'Context 65,536 tokens · 6.6 GB download · Apache-2.0', + 'Needs about 10.9 GB of the 12.5 GB this Mac can give a model (estimated).', + 'About 11 tokens per second (estimated).', + ]); + expect(texts(root, '.local-model-note')).toEqual(['Not yet tested by us on a Mac like this.']); + expect(texts(root, '.local-model-next')).toEqual([NEXT_STEP]); + expect(root.textContent).not.toMatch(/recommend/i); + expect(root.textContent).not.toContain('Best fit'); + } + }); +}); + +describe('the Settings row', () => { + it('shows the same answer as the card, with the alternatives open', async () => { + const { newApp, card, row } = mount(); + await newApp(ok(OK_STATUS)).loadLocalModel(); + for (const part of ['.local-model-hw', '.local-model-headline', '.local-model-pick', '.local-model-note']) { + expect(texts(row, part)).toEqual(texts(card, part)); + expect(texts(row, part).length).toBeGreaterThan(0); + } + expect(texts(row, '.local-model-alt')).toEqual(texts(card, '.local-model-alt')); + expect(texts(row, '.local-model-next')).toEqual([NEXT_STEP]); + // The card folds the list behind a native disclosure; Settings scrolls, so it lists them. + expect(card.querySelector('details.local-model-alts')).not.toBeNull(); + expect((card.querySelector('details') as HTMLDetailsElement).open).toBe(false); + expect(row.querySelector('details')).toBeNull(); + // The row has no chrome of the card: no title, no Hide. + expect(row.querySelector('button')).toBeNull(); + }); + + it('keeps the alternatives unfolded across a repaint of the card', async () => { + const { newApp, card } = mount(); + const app = newApp(ok(OK_STATUS)); + await app.loadLocalModel(); + (card.querySelector('details') as HTMLDetailsElement).open = true; + app.renderLocalModel(); + expect((card.querySelector('details') as HTMLDetailsElement).open).toBe(true); + }); +}); + +describe('payload text is text', () => { + const EVIL = ''; + + it('renders markup in a name, a license or a sentence slot as characters, and runs nothing', async () => { + const { dom, newApp, card, row } = mount(); + const evil = candidate({ modelLabel: EVIL, modelId: `m:${EVIL}`, cliLabel: EVIL, license: EVIL, needGb: EVIL }); + const answer = status( + { + // A slot value is server text too. + headline: { text: LOCAL_MODEL_LABELS.noneMacos, vars: { version: EVIL, minimum: EVIL } }, + notes: [{ text: LOCAL_MODEL_LABELS.noneMemory, vars: { memory: EVIL } }], + pick: evil, + alternatives: [evil], + budgetGb: EVIL, + }, + { chip: EVIL, memGb: EVIL, freeDiskGb: EVIL } + ); + await newApp(ok(answer)).loadLocalModel(); + expect(card.hidden).toBe(false); + for (const root of [card, row]) { + expect(root.querySelector('img')).toBeNull(); + expect(root.querySelector('script, iframe, a, [onerror]')).toBeNull(); + expect(texts(root, '.local-model-hw')[0]).toBe(`${EVIL}, ${EVIL} GB memory, ${EVIL} GB free`); + expect(texts(root, '.local-model-headline')[0]).toBe( + `This Mac runs macOS ${EVIL}. Local models need macOS ${EVIL} or newer.` + ); + expect(texts(root, '.local-model-name')[0]).toBe(`${EVIL} (m:${EVIL}) with ${EVIL}`); + expect(texts(root, '.local-model-pick .local-model-line')[0]).toContain(` · ${EVIL}`); + expect(texts(root, '.local-model-note')[0]).toContain(`This Mac has ${EVIL} GB of memory.`); + } + expect((dom.window as unknown as { __pwned?: number }).__pwned).toBeUndefined(); + }); + + it('never assigns markup: the module has no innerHTML, outerHTML or insertAdjacentHTML', () => { + expect(MODULE).not.toMatch(/innerHTML|outerHTML|insertAdjacentHTML|document\.write/); + }); + + it('shows a label the contract does not know as it came, still as text', async () => { + const { newApp, card } = mount(); + const odd = candidate({ fitLabel: EVIL as Candidate['fitLabel'], speedLabel: undefined }); + await newApp(ok(status({ pick: odd }))).loadLocalModel(); + expect(texts(card, '.local-model-pill')).toEqual([EVIL]); + expect(card.querySelector('img')).toBeNull(); + }); +}); + +describe('hiding the card', () => { + it('hides it on this device for good, and leaves the Settings row as the way back', async () => { + const { dom, newApp, card, row } = mount(); + const app = newApp(ok(OK_STATUS)); + await app.loadLocalModel(); + const hide = card.querySelector('button.local-model-hide') as HTMLButtonElement; + expect(hide.textContent).toBe('Hide'); + expect(hide.type).toBe('button'); + expect(hide.title).toBe('Hide this card on this device. It stays in App Settings, under Models.'); + hide.click(); + expect(card.hidden).toBe(true); + expect(card.textContent).toBe(''); + expect(dom.window.localStorage.getItem(DISMISS_KEY)).toBe('1'); + expect(texts(row, '.local-model-name')[0]).toBe('Qwen3.8 27B (qwen3.8:27b) with OpenCode'); + + // A reload is a new app instance over the same storage. + const reloaded = newApp(ok(OK_STATUS)); + expect(reloaded.isLocalModelCardDismissed()).toBe(true); + await reloaded.loadLocalModel(); + expect(card.hidden).toBe(true); + expect(texts(row, '.local-model-headline')).toEqual(['Best fit for this Mac']); + }); + + it('shows the card on a device that never hid it', async () => { + const { dom, newApp, card } = mount(); + expect(dom.window.localStorage.getItem(DISMISS_KEY)).toBeNull(); + await newApp(ok(OK_STATUS)).loadLocalModel(); + expect(card.hidden).toBe(false); + }); + + it('stays hidden for the page load when storage is blocked', async () => { + const { dom, newApp, card } = mount(); + const storage = dom.window.Storage.prototype; + vi.spyOn(storage, 'getItem').mockImplementation(() => { + throw new Error('blocked'); + }); + vi.spyOn(storage, 'setItem').mockImplementation(() => { + throw new Error('blocked'); + }); + const app = newApp(ok(OK_STATUS)); + await app.loadLocalModel(); + expect(card.hidden).toBe(false); + app.dismissLocalModelCard(); + expect(card.hidden).toBe(true); + app.renderLocalModel(); + expect(card.hidden).toBe(true); + vi.restoreAllMocks(); + }); + + it('writes nothing to the server: the only request the module ever makes is the GET', async () => { + const { newApp, card } = mount(); + const app = newApp(ok(OK_STATUS)); + await app.loadLocalModel(); + (card.querySelector('button.local-model-hide') as HTMLButtonElement).click(); + expect(app._api.mock.calls).toEqual([['/api/local-model']]); + }); +}); + +describe('the request', () => { + it('goes through the base path when Codeman is mounted under one', async () => { + const before = "const CodemanBase = { base: '/cm', url: (path) => '/cm' + path };"; + const { newApp } = mount({ before }); + const app = newApp(ok(OK_STATUS)); + await app.loadLocalModel(); + expect(app._api.mock.calls).toEqual([['/cm/api/local-model']]); + }); + + it('is sent once a minute at most, and shared while in flight', async () => { + const { dom, newApp, card } = mount(); + let now = 1_000_000; + (dom.window as unknown as { Date: DateConstructor }).Date.now = () => now; + const app = newApp(ok(OK_STATUS)); + const first = app.loadLocalModel(); + const second = app.loadLocalModel(); + await Promise.all([first, second]); + expect(app._api).toHaveBeenCalledTimes(1); + + now += 59_000; + await app.loadLocalModel(); + expect(app._api).toHaveBeenCalledTimes(1); + + now += 2_000; + await app.loadLocalModel(); + expect(app._api).toHaveBeenCalledTimes(2); + expect(card.hidden).toBe(false); + }); + + it('keeps the answer on screen when a later refresh fails', async () => { + const { dom, newApp, card, row } = mount(); + let now = 1_000_000; + (dom.window as unknown as { Date: DateConstructor }).Date.now = () => now; + const answers: ApiResponse[] = [ok(OK_STATUS), null]; + const app = newApp(async () => answers.shift() ?? null); + await app.loadLocalModel(); + now += 61_000; + await app.loadLocalModel(); + expect(app._api).toHaveBeenCalledTimes(2); + expect(card.hidden).toBe(false); + expect(texts(row, '.local-model-headline')).toEqual(['Best fit for this Mac']); + expect(row.textContent).not.toContain('Could not reach the server.'); + }); +}); + +describe('with the real translator', () => { + it('renders in Chinese, and follows a language or display-name change without a reload', async () => { + const { newApp, card, row, i18n } = mount({ i18n: true }); + i18n.start(); + i18n.configure({ language: 'zh-CN' }); + const app = newApp(ok(OK_STATUS)); + await app.loadLocalModel(); + expect(texts(card, '.local-model-headline')).toEqual(['最适合此 Mac 的选择']); + expect(texts(card, '.local-model-pill')).toEqual(['良好', '速度未知', '良好', '紧张', '较慢']); + // Names and numbers ride the slots untouched. + expect(texts(card, '.local-model-name')[0]).toBe('Qwen3.8 27B（qwen3.8:27b），搭配 OpenCode'); + expect(texts(row, '.local-model-hw')).toEqual(['Apple M5 Pro，内存 51.5 GB，可用磁盘空间 912 GB']); + + // Both containers are data-i18n-skip, so the module itself must repaint. + i18n.configure({ language: 'en' }); + await settle(); + expect(texts(card, '.local-model-headline')).toEqual(['Best fit for this Mac']); + expect(texts(row, '.local-model-next')).toEqual([NEXT_STEP]); + + i18n.configure({ displayName: 'Acme' }); + await settle(); + expect(texts(card, '.local-model-next')).toEqual([ + 'Setting this up from Acme is the next step. It is not built yet.', + ]); + }); + + it('leaves a slot-shaped piece of a model name alone', async () => { + const { newApp, card, i18n } = mount({ i18n: true }); + i18n.start(); + const odd = candidate({ modelLabel: 'Model {harness} {name}' }); + await newApp(ok(status({ pick: odd }))).loadLocalModel(); + await settle(); + // The DOM translator would empty `{harness}` and brand `{name}`; the skip attribute keeps it out. + expect(texts(card, '.local-model-name')[0]).toBe('Model {harness} {name} (qwen3.8:27b) with OpenCode'); + }); +}); From 58e20cf613dbb2f92fa9f9ba8529330e11697112 Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 16:28:04 +0100 Subject: [PATCH 5/8] docs(api): document GET /api/local-model A "Local model recommendation" section in the API reference: the read-only route, its admin gate in multi-user mode, why a failed probe is a 200, and every field of { hardware, recommendation, catalogUpdated }, including that a size travels as bytes and as decimal GB text. Co-Authored-By: Claude Fable 5.1 --- docs/api-reference.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/api-reference.md b/docs/api-reference.md index d8de8665..756631ff 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1676,6 +1676,39 @@ Only replies from `allowedUserIds`, in `guildId`, inside a thread the bridge cre `GET /api/doctor[?category=core|office|other]` returns the `codeman doctor --json` report (`platform`, `summary`, `tools[]` with `status` `ok` \| `missing` \| `outdated` \| `skipped` \| `error`, `version`, `path`, `installHint`). The probe engine is synchronous, so it runs in a child process of the same entry script, never on the server's event loop (30 s timeout). It names install paths and versions, so it is admin only in multi-user mode (`403`). `400` for an unknown category, `500` if the child produces no report. +## Local model recommendation + +`GET /api/local-model` looks at the machine the server runs on and says which local coding model fits it, and with which agent CLI. It covers Apple silicon macOS only; anywhere else the answer is still a `200`, with verdict `none`. **Read-only:** the route downloads nothing, installs nothing, writes no file and reads or changes no setting (setting a model up from Codeman is a later step and is not built yet). It names the host's hardware, so it is admin only in multi-user mode (`403`, answered before anything is probed). The models, their sizes and the agent CLIs it considers are a catalogue compiled into the release; there is no override file. + +| Method | Path | Body | Notes | +| ------ | ------------------ | ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GET` | `/api/local-model` | none | `200` with `{ hardware, recommendation, catalogUpdated }`. `403 FORBIDDEN` for a non-admin in multi-user mode. `500 INTERNAL_ERROR` only when the recommendation itself fails, never for a failed probe. | + +A probe that fails is an answer, not an error: a server that is not on macOS, or a Mac whose `sysctl` call failed or timed out, still gets a `200`, and `hardware.source` says what was read. + +`data`: + +- `hardware`: what the server sees of the machine it runs on. + - `platform`, `arch`: `process.platform` and `os.arch()` of the server process. `arm64: true` means the machine is Apple silicon whatever `arch` says; `translated: true` means the server process runs under Rosetta. + - `memBytes`, `memGb` (decimal GB text) and `memClassGb` (the memory size the Mac is sold as: 16, 32, 48). + - `chip`, `model`, `macosVersion`, `macosMajor`, `wiredLimitMb` (`0` means not set), `freeDiskBytes` and `freeDiskGb` (free space on the filesystem of Codeman's data directory). Each is absent when it could not be read. + - `source`: `sysctl` (the one bounded `sysctl` call answered), `os-fallback` (macOS, but that call failed, so only memory, architecture and the chip name are known) or `not-probed` (not macOS, nothing was run). +- `recommendation`: + - `verdict`: `ok`, `limited` (the best model that fits has fewer than 12 billion parameters: it handles small edits and questions and does not replace a cloud model) or `none`. + - `reason`, present only with `none`: `not-macos`, `intel`, `memory`, `disk`, `macos` (too old) or `no-harness` (no agent CLI that can be connected to a local model is installed or installable). + - `headline` and `notes[]`: `{ text, vars? }`. `text` is an English sentence that may hold `{placeholder}` slots, and `vars` carries the slot values as text. + - `pick` (absent with `none`) and `alternatives[]` (at most four, best first; empty with `none`), each one model at one context length with one agent CLI: + - `modelId` (`name:tag`), `modelLabel`, `license`, `paramsB` (parameters in billions), `context` (tokens); + - `cli`, `cliLabel`, `cliInstalled` (`false`: the CLI is not on this machine yet but can be installed without administrator rights), `slowFirstTurn?`; + - `needBytes` and `needGb` (memory the model needs at that context), `downloadBytes` and `downloadGb`, `budgetShare` (need divided by `budgetBytes`, rounded to three decimals); + - `fit`: `good` (up to 0.85 of the budget) or `tight` (up to 0.98), with its label `fitLabel`; + - `speed`: `ok`, `slow` or `unknown`, with `speedLabel` (absent for `ok`) and `tokensPerSecond` (absent for `unknown`). `unknown` means the chip is in no row of the catalogue's chip table, or the catalogue has no speed figure for the model: a speed is never guessed; + - `tested`: we ran this model with this CLI on a Mac like this one. No catalogue row has such an entry yet, so it is `false` today. + - `budgetBytes` and `budgetGb`: the memory a model may use on this Mac (`0` for `not-macos` and `intel`, where none is computed). `budgetSource` is `wired-limit` (taken from `iogpu.wired_limit_mb`) or `estimated` (computed from the installed memory). +- `catalogUpdated`: ISO date on which the catalogue's rows were last checked. + +Every size a person reads travels twice: as bytes, and as decimal GB text without the unit (`"needGb": "29.6"`). Show the text and add the unit; do not divide the bytes yourself, so one request never shows two different roundings. + ## Voice dictation Browser dictation transcribed through this server's Claude Code login, i.e. the From 4739542397fe9887b18639b25048a1945c85e759 Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 16:30:59 +0100 Subject: [PATCH 6/8] chore(local-model): drop build-time wording from two comments The label list was "frozen for the parallel build" and the card's test said the route answers 501: both were true while the pieces were written side by side and are not any more. Comments only. Co-Authored-By: Claude Fable 5.1 --- src/types/local-model.ts | 6 +++--- test/local-model-ui.test.ts | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/types/local-model.ts b/src/types/local-model.ts index 77352d64..faf42777 100644 --- a/src/types/local-model.ts +++ b/src/types/local-model.ts @@ -221,9 +221,9 @@ export interface DeviceBudget { /** * Every sentence and label the server may put into the payload, in English (the i18n source - * language). Frozen for the parallel build: the zh-CN dictionary in src/web/public/i18n.js has - * one entry per value, keyed by this exact text, and the browser's i18n test imports this - * constant to list them. Adding a key is fine; changing a text means changing its zh-CN key too. + * language). The zh-CN dictionary in src/web/public/i18n.js has one entry per value, keyed by + * this exact text, and test/local-model-i18n.test.ts imports this constant to list them. Adding + * a key is fine; changing a text means changing its zh-CN key too. * * `{placeholder}` slots are filled by the browser from `LocalModelMessage.vars`. `{name}` is * taken (the translator fills it with the display name), so no slot here is called `name`. diff --git a/test/local-model-ui.test.ts b/test/local-model-ui.test.ts index 1689d6ff..00daa696 100644 --- a/test/local-model-ui.test.ts +++ b/test/local-model-ui.test.ts @@ -16,8 +16,9 @@ * - the dismissal is a per-device localStorage key and survives a new app instance; * - one request a minute at most, shared while in flight, and a failed refresh keeps the answer. * - * The route answers 501 until the server stage lands, so the payloads here are fixtures typed - * with the contract's `LocalModelStatus`, built from `LOCAL_MODEL_LABELS`. + * The payloads here are fixtures typed with `LocalModelStatus` and built from + * `LOCAL_MODEL_LABELS`, not the output of the route: what the server computes for a given Mac is + * pinned by test/local-model-recommend.test.ts and test/routes/local-model-routes.test.ts. * * Port: none. */ @@ -131,7 +132,7 @@ const settle = () => new Promise((done) => setTimeout(done, 0)); // ---------- Fixtures (the contract's types) ---------- -/** The MacBook of the probe: 48 GB as sold, a chip that is in no chips row. */ +/** The MacBook of the probe: 48 GB as sold. The candidates below give it an unknown speed, as for a chip in no chips row. */ const HARDWARE: HardwareFacts = { platform: 'darwin', arch: 'arm64', From 8281045cf24d8ec37c38a06bc2288604ef26c379 Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 17:37:53 +0100 Subject: [PATCH 7/8] fix(local-model): say what is true of the Mac, and why a model is missing Review fixes for the read-only recommendation and its card. Recommendation (pure): - `limited` is now a statement about the Mac: no model with 12 billion parameters or more fits. It is judged on everything that fits, before the disk filter and whatever the pick is. A 32 or 36 GB Mac whose pick is a 9B model no longer reads "This Mac can run a small model" above a list of 20B, 27B and 35B models, and knowing the chip no longer lowers the verdict. A small pick under `ok` carries a note that says so. - A model that fits the memory but has no room on the disk is no longer dropped in silence: the best-ranked one is named in a note with the GB that are missing. A full disk cannot turn a 48 GB Mac into a small-model Mac. - "Models fit, but not at a context a harness needs" has its own sentence with the budget and the need, worded as an estimate. The 16 GB Mac mini misses by 1.3 points (10.3 of 10.4 GB) and was told flatly that its memory is too little, the sentence of an 8 GB Mac. - The installed memory is named by the size the Mac is sold as (48, not 51.5): the hardware line and the memory sentence. Every computed size stays decimal GB from the one formatter. An unknown memory size is no longer reported as "0.0 GB". - Slow is graded on the unrounded speed: 9.95 to 9.9996 tokens per second printed as 10 and read ok. - The disk sentence names the 10 GiB that must stay free, so its numbers add up for the reader. Card and Settings group: - The Settings group "Model on this Mac" is hidden on a server that is not a Mac, and until the first answer says it is one. - Only `403 FORBIDDEN` is a refusal. `403 PASSWORD_CHANGE_REQUIRED` (an admin who must change the password) is a failed request and is asked again, so that admin no longer loses the card until a reload. - Each fit and speed label carries what it measures as its tooltip and accessible name, an alternative with a speed estimate prints it, and the count beside "Other models that fit" is gone (it was the number of rows, capped at four, not the number of models that fit). - The line saying nothing can be set up yet sits right under the pick, in the colour of every other line (it was 3.9:1), and says whose step it is. - The dismissal key is named for this stage, so hiding the read-only card does not hide the install offer later. Co-Authored-By: Claude Fable 5.1 --- src/local-model/format.ts | 12 +- src/local-model/recommend.ts | 125 +++++++++--- src/types/local-model.ts | 86 +++++--- src/web/public/i18n.js | 28 ++- src/web/public/index.html | 8 +- src/web/public/local-model-ui.js | 169 +++++++++++----- src/web/public/styles.css | 23 +-- test/local-model-i18n.test.ts | 19 +- test/local-model-recommend.test.ts | 291 ++++++++++++++++++++++++--- test/local-model-ui.test.ts | 307 +++++++++++++++++++++++++---- 10 files changed, 857 insertions(+), 211 deletions(-) diff --git a/src/local-model/format.ts b/src/local-model/format.ts index cc9aa974..4f8db32d 100644 --- a/src/local-model/format.ts +++ b/src/local-model/format.ts @@ -8,7 +8,11 @@ * size for display, the browser included: it prints the text it was sent. * * GiB and MiB exist only for the budget math (reserves, the wired limit, the prompt cache) and - * never reach a person. + * never reach a person, with one exception: `memClassGb()`, the memory size a Mac is sold as, + * is how the card names the installed memory ("48 GB memory", what About This Mac shows). It + * is a name of the machine, not a computed size: `formatGb()` of the same bytes reads 51.5, + * which no owner recognises. Every computed size (need, budget, download, free disk) is + * `formatGb()`. * * Pure: no IO, no state. */ @@ -33,8 +37,10 @@ export function formatGb(bytes: number): string { /** * The memory size a Mac is sold as: whole binary gigabytes (17179869184 bytes is 16). This is - * `HardwareFacts.memClassGb`: the key of the tier rules ("below 36 GB") and of `TestedOn.memGb`. - * It names a class of Mac. A size for display is `formatGb()`. + * `HardwareFacts.memClassGb`: the key of the tier rules ("below 36 GB") and of `TestedOn.memGb`, + * and what the card and the `noneMemory` sentence call the installed memory. It names a class + * of Mac; 0 means the size is unknown and is never shown. Any other size for display is + * `formatGb()`. */ export function memClassGb(memBytes: number): number { if (!Number.isFinite(memBytes) || memBytes <= 0) return 0; diff --git a/src/local-model/recommend.ts b/src/local-model/recommend.ts index a0513093..884cc71f 100644 --- a/src/local-model/recommend.ts +++ b/src/local-model/recommend.ts @@ -10,33 +10,48 @@ * catalogue for this Mac's memory. * - Fit: F/B up to 0.85 is good, up to 0.98 is tight, above is out. * - Speed: tokens per second = the model's `speedPer100Gbps` x the chip's bandwidth / 100. - * Below 10 is slow. A chip that is in no chips row, or a model without a speed figure, is - * UNKNOWN: a bandwidth is never guessed. + * Below 10 is slow, decided on the unrounded product (9.96 is slow although it prints as + * 10). A chip that is in no chips row, or a model without a speed figure, is UNKNOWN: a + * bandwidth is never guessed. * - Candidates: (model, context, harness) with `ctx >= harness.minContext`, the harness not * gated, installed or installable, and accepting the model id. * - Ranking, lexicographic: good before tight; ok before unknown before slow; the higher model * rank; the higher harness rank; the larger context. The pick is the first candidate, the * alternatives are the next four MODELS, each at its own best context and harness. + * - Disk: a model whose download would leave less than 10 GiB free is not offered. It is not + * dropped in silence: the best-ranked model left out that way is named in `noteDiskDropped` + * with what is missing. * * Verdict `none`, in this order (the first that applies is the reason): * 1. `not-macos`: the server does not run on macOS. * 2. `intel`: a Mac without Apple silicon. * 3. `macos`: macOS is older than the catalogue's minimum (an unknown version does not block). - * 4. `memory`: no model fits the budget at any of its contexts, whatever the harness. + * 4. `memory`: no model fits the budget at any of its contexts, whatever the harness + * (`noneMemory`, or `noneMemoryUnknown` when the memory size could not be read). * 5. `no-harness`: no harness row is usable (not gated, known to the caller, installed or * installable). - * 6. `memory`: models fit, but none at a context a usable harness needs. + * 6. `memory`: models fit, but none at a context a usable harness needs. Its own sentence + * (`noneMemoryContext`) with the budget and the smallest need at such a context, because + * this Mac can miss by a hair (the 16 GB Mac mini: 10.3 of 10.4 GB) and "too little + * memory" would hide that the answer rests on an estimate. * 7. `no-harness`: models fit at such a context, but no usable harness accepts their id. * 8. `disk`: for every remaining model, free disk minus the download is below 10 GiB (an * unknown free disk does not block). * * Rules: - * - Verdict `limited` when the pick's model has fewer than 12 billion parameters. + * - Verdict `limited` when NO model with 12 billion parameters or more fits this Mac's memory + * at a context a usable harness works at. It is a statement about the Mac ("This Mac can run + * a small model"), so it is judged on everything that fits, BEFORE the disk filter and + * whatever the pick is: a 36 GB Mac whose pick is a 9B model while a 20B, a 27B and a 35B + * are listed under it is not a small-model Mac, and a 48 GB Mac does not become one because + * its disk is full. A small pick under verdict `ok` carries `noteSmallPick` instead. * - The word "Recommended" (`LOCAL_MODEL_LABELS.headlineTested`) only for verdict `ok` with a * pick whose model has a `tested` entry for exactly this chip, this memory size and this * harness. Otherwise `headlineOk` (or `headlineLimited`) plus `noteUntested`. - * - Every label in the result comes from `LOCAL_MODEL_LABELS`, every size text from - * `formatGb()`. No number is spliced into a sentence here. + * - Every label in the result comes from `LOCAL_MODEL_LABELS`, every computed size text from + * `formatGb()`. The one size that is not decimal GB is the installed memory in `noneMemory`: + * `HardwareFacts.memClassGb`, the size the Mac is sold as. No number is spliced into a + * sentence here. * - No CLI id is ever compared with a literal: a harness is a catalogue row plus a * `HarnessFact`. */ @@ -52,6 +67,8 @@ import { type LocalHarnessEntry, type LocalModelCatalog, type LocalModelEntry, + type LocalModelLabel, + type LocalModelMessage, type NoneReason, type Recommendation, type SpeedGrade, @@ -65,7 +82,7 @@ export const FIT_GOOD_MAX_SHARE = 0.85; export const FIT_TIGHT_MAX_SHARE = 0.98; /** An estimated speed below this many tokens per second is `slow`. */ export const SLOW_BELOW_TOKENS_PER_SECOND = 10; -/** A pick whose model has fewer than this many billion parameters makes the verdict `limited`. */ +/** When no model that fits has at least this many billion parameters, the verdict is `limited`. A pick below it is a "small model". */ export const LIMITED_BELOW_PARAMS_B = 12; /** What must stay free on the disk after the model download. */ export const DISK_HEADROOM_BYTES = 10 * GIB; @@ -134,9 +151,10 @@ function round(value: number, decimals: number): number { return Math.round(value * factor) / factor; } -/** A candidate plus the two ranks the ordering needs, which are not part of the payload. */ +/** A candidate plus its catalogue row and the two ranks the ordering needs, none of which is part of the payload. */ interface Ranked { candidate: Candidate; + model: LocalModelEntry; modelRank: number; harnessRank: number; } @@ -181,12 +199,12 @@ function buildCandidate( budgetBytes: number ): Candidate { const gbps = chipBandwidthGbps(catalog.chips, hw.chip); - const tokensPerSecond = - gbps !== undefined && model.speedPer100Gbps !== undefined - ? round((model.speedPer100Gbps * gbps) / 100, 1) - : undefined; + const rawSpeed = + gbps !== undefined && model.speedPer100Gbps !== undefined ? (model.speedPer100Gbps * gbps) / 100 : undefined; + // Graded on the unrounded figure: 9.96 rounds to 10.0 for the payload and is still slow. const speed: SpeedGrade = - tokensPerSecond === undefined ? 'unknown' : tokensPerSecond < SLOW_BELOW_TOKENS_PER_SECOND ? 'slow' : 'ok'; + rawSpeed === undefined ? 'unknown' : rawSpeed < SLOW_BELOW_TOKENS_PER_SECOND ? 'slow' : 'ok'; + const tokensPerSecond = rawSpeed === undefined ? undefined : round(rawSpeed, 1); // "A Mac like this" is exact: the same chip string, the same memory size as sold, the same harness. const tested = model.tested.some( (entry) => entry.chip === hw.chip && entry.memGb === hw.memClassGb && entry.cli === harness.row.cli @@ -216,11 +234,17 @@ function buildCandidate( }; } -function none(reason: NoneReason, budget: DeviceBudget, vars?: Record): Recommendation { +/** A `none` answer. `text` replaces the reason's default sentence (the `memory` reason has three). */ +function none( + reason: NoneReason, + budget: DeviceBudget, + vars?: Record, + text: LocalModelLabel = LOCAL_MODEL_NONE_LABELS[reason] +): Recommendation { return { verdict: 'none', reason, - headline: { text: LOCAL_MODEL_NONE_LABELS[reason], ...(vars ? { vars } : {}) }, + headline: { text, ...(vars ? { vars } : {}) }, notes: [], alternatives: [], budgetBytes: budget.budgetBytes, @@ -260,7 +284,12 @@ export function recommend( if (fit !== 'out') fitting.push({ model, ctx, need, fit }); } } - const tooLittleMemory = (): Recommendation => none('memory', budget, { memory: hw.memGb }); + // Nothing fits at any context. The installed memory is named by the size the Mac is sold + // as; a size that could not be read is not stated at all (never "0 GB of memory"). + const tooLittleMemory = (): Recommendation => + Number.isFinite(hw.memClassGb) && hw.memClassGb > 0 + ? none('memory', budget, { memory: String(hw.memClassGb) }) + : none('memory', budget, undefined, LOCAL_MODEL_LABELS.noneMemoryUnknown); if (fitting.length === 0) return tooLittleMemory(); // A gated row is never offered, whatever the caller knows about its CLI. @@ -275,25 +304,46 @@ export function recommend( const atHarnessContext = fitting.flatMap((entry) => usable.filter((harness) => entry.ctx >= harness.row.minContext).map((harness) => ({ ...entry, harness })) ); - if (atHarnessContext.length === 0) return tooLittleMemory(); + if (atHarnessContext.length === 0) { + // Models fit, but only below the context a harness needs. Say both numbers: the smallest + // need at such a context can sit a hair over the line, and the budget is an estimate. + const smallestContext = Math.min(...usable.map((harness) => harness.row.minContext)); + let smallestNeed: number | undefined; + for (const model of catalog.models) { + for (const ctx of model.contexts) { + if (ctx < smallestContext) continue; + const need = needBytes(model, ctx, catalog, hw); + if (smallestNeed === undefined || need < smallestNeed) smallestNeed = need; + } + } + if (smallestNeed === undefined) return tooLittleMemory(); + return none( + 'memory', + budget, + { budget: formatGb(budget.budgetBytes), need: formatGb(smallestNeed) }, + LOCAL_MODEL_LABELS.noneMemoryContext + ); + } const accepted = atHarnessContext.filter((entry) => accepts(entry.harness.fact, entry.model.id)); if (accepted.length === 0) return none('no-harness', budget); // The download must leave headroom on the disk. An unknown free disk does not block. const free = hw.freeDiskBytes; - const downloadable = - free === undefined ? accepted : accepted.filter((entry) => free - entry.model.bytes >= DISK_HEADROOM_BYTES); - if (free !== undefined && downloadable.length === 0) { + const hasRoom = (model: LocalModelEntry): boolean => free === undefined || free - model.bytes >= DISK_HEADROOM_BYTES; + if (free !== undefined && !accepted.some((entry) => hasRoom(entry.model))) { const smallest = Math.min(...accepted.map((entry) => entry.model.bytes)); return none('disk', budget, { download: formatGb(smallest), + reserve: formatGb(DISK_HEADROOM_BYTES), free: hw.freeDiskGb ?? formatGb(free), missing: formatGb(smallest + DISK_HEADROOM_BYTES - free), }); } - const ranked: Ranked[] = downloadable + // Everything that fits is ranked, the models without room on the disk included: the best of + // those is named in a note, so a full disk never removes a model without a word. + const ranked: Ranked[] = accepted .map((entry) => ({ candidate: buildCandidate( entry.model, @@ -305,28 +355,47 @@ export function recommend( catalog, budget.budgetBytes ), + model: entry.model, modelRank: entry.model.rank, harnessRank: entry.harness.row.rank, })) .sort(compareRanked); // The first candidate of each model is that model at its best context and harness. - const bestPerModel: Candidate[] = []; - for (const { candidate } of ranked) { - if (!bestPerModel.some((kept) => kept.modelId === candidate.modelId)) bestPerModel.push(candidate); + const bestPerModel: Ranked[] = []; + for (const entry of ranked) { + if (!bestPerModel.some((kept) => kept.model.id === entry.model.id)) bestPerModel.push(entry); } - const [pick, ...others] = bestPerModel; + const offered = bestPerModel.filter((entry) => hasRoom(entry.model)).map((entry) => entry.candidate); + const leftOutForDisk = bestPerModel.find((entry) => !hasRoom(entry.model)); + const [pick, ...others] = offered; - const limited = pick.paramsB < LIMITED_BELOW_PARAMS_B; + // About the Mac, not the pick, and judged before the disk filter (see the file header). + const limited = !accepted.some((entry) => entry.model.paramsB >= LIMITED_BELOW_PARAMS_B); const headline = limited ? LOCAL_MODEL_LABELS.headlineLimited : pick.tested ? LOCAL_MODEL_LABELS.headlineTested : LOCAL_MODEL_LABELS.headlineOk; + + const notes: LocalModelMessage[] = []; + // The limited headline already says it. Under `ok` a small pick must still be called small. + if (!limited && pick.paramsB < LIMITED_BELOW_PARAMS_B) notes.push({ text: LOCAL_MODEL_LABELS.noteSmallPick }); + if (leftOutForDisk && free !== undefined) { + notes.push({ + text: LOCAL_MODEL_LABELS.noteDiskDropped, + vars: { + model: leftOutForDisk.model.label, + missing: formatGb(leftOutForDisk.model.bytes + DISK_HEADROOM_BYTES - free), + }, + }); + } + if (!pick.tested) notes.push({ text: LOCAL_MODEL_LABELS.noteUntested }); + return { verdict: limited ? 'limited' : 'ok', headline: { text: headline }, - notes: pick.tested ? [] : [{ text: LOCAL_MODEL_LABELS.noteUntested }], + notes, pick, alternatives: others.slice(0, MAX_ALTERNATIVES), budgetBytes: budget.budgetBytes, diff --git a/src/types/local-model.ts b/src/types/local-model.ts index faf42777..8b8cfc75 100644 --- a/src/types/local-model.ts +++ b/src/types/local-model.ts @@ -15,10 +15,12 @@ * * Three rules every producer and consumer follows: * - * 1. ONE unit for a size a person reads: decimal GB, computed from bytes by `formatGb()` in - * src/local-model/format.ts. Every such size travels twice, as bytes (`needBytes`) and as the - * text to show (`needGb`, a number without the unit, such as '25.5'). The browser prints the - * text and never divides bytes itself. GiB appears only inside the budget math. + * 1. ONE unit for a computed size a person reads: decimal GB, computed from bytes by + * `formatGb()` in src/local-model/format.ts. Every such size travels twice, as bytes + * (`needBytes`) and as the text to show (`needGb`, a number without the unit, such as + * '25.5'). The browser prints the text and never divides bytes itself. GiB appears only + * inside the budget math. The one exception is the NAME of the installed memory: the card + * says `HardwareFacts.memClassGb` (48 for a Mac sold with 48 GB), see that field. * 2. Every sentence or label the SERVER chooses comes from `LOCAL_MODEL_LABELS` below, so the * browser can translate it (`codemanT(text, vars)`) and its i18n test can list every string. * Numbers are never spliced into a sentence here: they travel in `vars` and fill the @@ -33,7 +35,8 @@ * How `HardwareFacts` was obtained. * - `sysctl`: the spawn answered and `hw.memsize` parsed. * - `os-fallback`: on macOS, but the spawn failed or timed out; memory, architecture and the chip - * string come from `node:os`, the rest is absent. + * string come from `node:os` and the free disk from `statfs`. What only sysctl knows (`arm64`, + * `translated`, `model`, `macosVersion`, `macosMajor`, `wiredLimitMb`) is absent. * - `not-probed`: not macOS, so `sysctl` was never spawned. */ export type HardwareProbeSource = 'sysctl' | 'os-fallback' | 'not-probed'; @@ -54,9 +57,19 @@ export interface HardwareFacts { translated?: boolean; /** Installed memory in bytes (`hw.memsize`, else `os.totalmem()`). The ONLY memory input of the budget math. */ memBytes: number; - /** `memBytes` as a person reads it: decimal GB from `formatGb()`, without the unit ('17.2' for a Mac sold as 16 GB). */ + /** + * `memBytes` in the unit of every computed size: decimal GB from `formatGb()`, without the unit + * ('17.2' for a Mac sold as 16 GB). NOT what the card calls the installed memory: that is + * `memClassGb`, see there. + */ memGb: string; - /** The memory size the Mac is sold as: whole binary gigabytes (16, 32, 48). Tier rules and `TestedOn.memGb` use it. */ + /** + * The memory size the Mac is sold as: whole binary gigabytes (16, 32, 48), 0 when the size is + * unknown. Tier rules and `TestedOn.memGb` use it, and it is the ONE size a person reads that + * is not decimal GB: the hardware line and the `memory` slot of `noneMemory` print it, because + * it is the number on the box and in About This Mac ("48 GB"), and '51.5 GB memory' reads as a + * different machine. Every computed size (need, budget, download, free disk) stays decimal GB. + */ memClassGb: number; /** `machdep.cpu.brand_string` ('Apple M4 Pro'), else `os.cpus()[0].model`. */ chip?: string; @@ -97,7 +110,7 @@ export interface LocalModelEntry { label: string; /** SPDX id of the license ('Apache-2.0'). */ license: string; - /** Parameter count in billions. Under 12 the verdict is `limited`, never `ok`. */ + /** Parameter count in billions. When no model that fits has 12 or more, the verdict is `limited`, never `ok`. */ paramsB: number; /** Size of the weights in bytes: both the download and what they take in memory. */ bytes: number; @@ -200,7 +213,10 @@ export type SpeedGrade = 'ok' | 'slow' | 'unknown'; * Why the verdict is `none`. * - `not-macos`: the server does not run on macOS (the card is hidden). * - `intel`: a Mac without Apple silicon. - * - `memory`: no model fits the memory budget with any harness. + * - `memory`: no model fits the memory budget at a context a usable harness needs. Three + * headlines share this reason: `noneMemory` (nothing fits at any context), `noneMemoryContext` + * (a model fits, but only below the context a harness needs: the sentence carries the budget + * and the need) and `noneMemoryUnknown` (the memory size could not be read). * - `disk`: models fit the memory, but for every one of them free disk minus the download is below 10 GiB. * - `macos`: macOS is older than `LocalModelCatalog.minMacosMajor`. * - `no-harness`: no catalogue harness is installed or installable and accepts a model. @@ -228,12 +244,17 @@ export interface DeviceBudget { * `{placeholder}` slots are filled by the browser from `LocalModelMessage.vars`. `{name}` is * taken (the translator fills it with the display name), so no slot here is called `name`. * - * | Key | Slots (all text; sizes are decimal GB without the unit) | - * |---------------|---------------------------------------------------------------------------------| - * | `noneMemory` | `memory`: `HardwareFacts.memGb` | - * | `noneDisk` | `download`: smallest download among the models that fit the memory; `free`: | - * | | `HardwareFacts.freeDiskGb`; `missing`: download + 10 GiB - free | - * | `noneMacos` | `version`: `HardwareFacts.macosVersion`; `minimum`: the catalogue's minimum | + * | Key | Slots (all text; sizes are decimal GB without the unit) | + * |---------------------|---------------------------------------------------------------------------| + * | `noteDiskDropped` | `model`: label of the best model left out for disk space; `missing`: its | + * | | download + the disk reserve - free | + * | `noneMemory` | `memory`: `HardwareFacts.memClassGb`, the size the Mac is sold as | + * | `noneMemoryContext` | `budget`: `Recommendation.budgetGb`; `need`: the smallest need of any | + * | | model at a context a usable harness works at | + * | `noneDisk` | `download`: smallest download among the models that fit the memory; | + * | | `reserve`: what must stay free after it (10 GiB, as decimal GB); `free`: | + * | | `HardwareFacts.freeDiskGb`; `missing`: download + reserve - free | + * | `noneMacos` | `version`: `HardwareFacts.macosVersion`; `minimum`: the catalogue minimum | * * The word "Recommended" exists only in `headlineTested`, which may be used only when the * pick's `tested` is true. No catalogue row has a tested entry today, so it is never sent. @@ -243,21 +264,33 @@ export const LOCAL_MODEL_LABELS = { headlineOk: 'Best fit for this Mac', /** Verdict `ok`, pick tested by us on a Mac like this one. */ headlineTested: 'Recommended for this Mac', - /** Verdict `limited`. */ + /** Verdict `limited`: no model with 12 billion parameters or more fits this Mac. */ headlineLimited: 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', /** A note under an `ok` or `limited` pick whose `tested` is false. */ noteUntested: 'Not yet tested by us on a Mac like this.', + /** A note under an `ok` pick that is itself a small model: larger ones fit this Mac, but rank lower or have no room on the disk. */ + noteSmallPick: + 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.', + /** A note when a model that fits the memory was left out because its download would not leave the disk reserve. */ + noteDiskDropped: + '{model} fits the memory of this Mac too, but its download needs about {missing} GB more free disk space.', fitGood: 'Good', fitTight: 'Tight', speedSlow: 'Slow', speedUnknown: 'Speed unknown', noneNotMacos: 'Codeman recommends a local model only on a Mac with Apple silicon.', noneIntel: 'This Mac has an Intel processor. Local coding models need Apple silicon.', + /** Reason `memory`: nothing fits at any context. */ noneMemory: 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', + /** Reason `memory`: a model fits, but only below the context a harness needs. Says both numbers, and that they are an estimate. */ + noneMemoryContext: + 'Codeman estimates that this Mac can give a model about {budget} GB of memory. The smallest model a coding agent can work with needs about {need} GB. That leaves no safe margin, so Codeman recommends none here.', + /** Reason `memory`: the memory size could not be read, so no size is stated. */ + noneMemoryUnknown: 'Codeman could not read how much memory this Mac has, so it recommends no model here.', noneDisk: - 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.', + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download, and {reserve} GB must stay free after it. {free} GB is free now. Free up about {missing} GB.', noneMacos: 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.', noneNoHarness: 'No coding agent that Codeman can connect to a local model is available on this Mac.', } as const; @@ -265,7 +298,7 @@ export const LOCAL_MODEL_LABELS = { /** One of the texts in `LOCAL_MODEL_LABELS`. The payload's label fields are typed with it, so an off-list string does not compile. */ export type LocalModelLabel = (typeof LOCAL_MODEL_LABELS)[keyof typeof LOCAL_MODEL_LABELS]; -/** The headline of each `none` reason. */ +/** The headline of each `none` reason. `memory` has two more (`noneMemoryContext`, `noneMemoryUnknown`), see `NoneReason`. */ export const LOCAL_MODEL_NONE_LABELS: Record = { 'not-macos': LOCAL_MODEL_LABELS.noneNotMacos, intel: LOCAL_MODEL_LABELS.noneIntel, @@ -317,7 +350,7 @@ export interface Candidate { speed: SpeedGrade; /** `speedSlow` or `speedUnknown`. Absent when `speed` is `ok`. */ speedLabel?: LocalModelLabel; - /** Estimated generation speed. Present exactly when `speed` is not `unknown`. */ + /** Estimated generation speed, rounded to one decimal. Present exactly when `speed` is not `unknown`. `speed` is decided on the unrounded figure. */ tokensPerSecond?: number; /** A `TestedOn` entry of the model matches this Mac (same chip, same `memClassGb`) and this harness. */ tested: boolean; @@ -325,22 +358,25 @@ export interface Candidate { /** * The answer of `recommend()`. - * - `ok`: `pick` is the best candidate and its model has at least 12 billion parameters. - * - `limited`: `pick` is the best candidate, but its model has fewer than 12 billion parameters. - * The card says so and never says "Recommended". + * - `ok`: `pick` is the best candidate, and at least one model with 12 billion parameters or + * more fits this Mac's memory at a context a usable harness works at. The pick itself can be + * smaller (a larger model may fit only tightly, have no speed figure, or have no room on the + * disk): `notes` then carries `noteSmallPick`. + * - `limited`: `pick` is the best candidate, and NO model with 12 billion parameters or more + * fits. The card says so and never says "Recommended". * - `none`: no pick; `reason` says why. */ export interface Recommendation { verdict: 'ok' | 'limited' | 'none'; /** Present exactly when `verdict` is `none`. */ reason?: NoneReason; - /** What the card says first: `headlineOk`, `headlineTested`, `headlineLimited` or the `none` reason's sentence. */ + /** What the card says first: `headlineOk`, `headlineTested`, `headlineLimited` or a `none` sentence. */ headline: LocalModelMessage; - /** Further sentences under the headline (`noteUntested` today). May be empty. */ + /** Further sentences under the pick, in this order: `noteSmallPick`, `noteDiskDropped`, `noteUntested`. May be empty. */ notes: LocalModelMessage[]; /** Present exactly when `verdict` is `ok` or `limited`. */ pick?: Candidate; - /** The next models that fit (at most four, each at its best context and harness), best first. Empty when `verdict` is `none`. */ + /** The next models that fit and have room on the disk (at most four, each at its best context and harness), best first. Empty when `verdict` is `none`. */ alternatives: Candidate[]; /** `DeviceBudget.budgetBytes`. 0 when no budget is computed: not macOS, or a Mac without Apple silicon. */ budgetBytes: number; diff --git a/src/web/public/i18n.js b/src/web/public/i18n.js index c9ab61a5..87c18651 100644 --- a/src/web/public/i18n.js +++ b/src/web/public/i18n.js @@ -1412,10 +1412,11 @@ 'Model on this Mac': '此 Mac 上的模型', 'What the Mac that Codeman runs on can run as a local coding model. Read-only: nothing is downloaded or installed from here.': '运行 {name} 的这台 Mac 可以运行哪种本地编程模型。只读：此处不会下载或安装任何内容。', + 'The Mac that Codeman runs on, which is not always the device you are reading on.': + '指运行 {name} 的那台 Mac，不一定是你正在使用的这台设备。', Hide: '隐藏', 'Hide this card on this device. It stays in App Settings, under Models.': '在此设备上隐藏此卡片。它仍保留在“应用设置”的“模型”中。', - 'Checking this Mac…': '正在检查此 Mac…', '{chip}, {memory} GB memory, {free} GB free': '{chip}，内存 {memory} GB，可用磁盘空间 {free} GB', '{chip}, {memory} GB memory': '{chip}，内存 {memory} GB', '{memory} GB memory, {free} GB free': '内存 {memory} GB，可用磁盘空间 {free} GB', @@ -1431,9 +1432,14 @@ 'About {speed} tokens per second (estimated).': '每秒约 {speed} 个 Token（估算）。', '{harness} is not installed on this Mac yet.': '此 Mac 上尚未安装 {harness}。', 'With {harness}, the first answer can take minutes.': '使用 {harness} 时，第一次回答可能需要几分钟。', - 'Other models that fit': '其他适合的模型', - 'Setting this up from Codeman is the next step. It is not built yet.': - '下一步是直接在 {name} 中完成安装和设置，该功能尚未实现。', + 'about {speed} tokens per second': '每秒约 {speed} 个 Token', + // What the fit and speed labels measure (their tooltip and accessible name). + 'Memory fit: needs {share}% of what this Mac can give a model': + '内存占用：需要此 Mac 可分配给模型的内存的 {share}%', + 'No speed estimate for this model on this chip': '此芯片上没有该模型的速度估算', + 'Other models that fit': '其他装得下的模型', + 'Codeman cannot download or install this for you yet. That is the next step, and it is not built yet.': + '{name} 目前还不能为你下载或安装它。这是下一步的功能，尚未实现。', // The sentences and labels the SERVER picks (LOCAL_MODEL_LABELS in src/types/local-model.ts), keyed // by their exact English text: changing a text there means changing its key here. 'Best fit for this Mac': '最适合此 Mac 的选择', @@ -1441,8 +1447,12 @@ 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.': '此 Mac 可以运行小型模型。它能处理小幅修改和问答，但不能替代云端模型。', 'Not yet tested by us on a Mac like this.': '我们尚未在类似的 Mac 上测试过。', + 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.': + '最适合的是一个小型模型。它能处理小幅修改和问答，但不能替代云端模型。', + '{model} fits the memory of this Mac too, but its download needs about {missing} GB more free disk space.': + '{model} 也能装入此 Mac 的内存，但下载它还需要再腾出约 {missing} GB 的可用磁盘空间。', Good: '良好', - Tight: '紧张', + Tight: '偏紧', Slow: '较慢', 'Speed unknown': '速度未知', 'Codeman recommends a local model only on a Mac with Apple silicon.': @@ -1451,8 +1461,12 @@ '此 Mac 使用 Intel 处理器。本地编程模型需要 Apple 芯片。', 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.': '此 Mac 有 {memory} GB 内存，不足以运行能够驱动编程智能体的模型，因此 {name} 在此不推荐任何模型。', - 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.': - '可用磁盘空间不足。适合此 Mac 的最小模型需要下载 {download} GB，而当前可用空间为 {free} GB。请腾出约 {missing} GB。', + 'Codeman estimates that this Mac can give a model about {budget} GB of memory. The smallest model a coding agent can work with needs about {need} GB. That leaves no safe margin, so Codeman recommends none here.': + '据 {name} 估算，此 Mac 可分配给模型的内存约为 {budget} GB。编程智能体能够使用的最小模型约需 {need} GB。这样没有留出安全余量，因此 {name} 在此不推荐任何模型。', + 'Codeman could not read how much memory this Mac has, so it recommends no model here.': + '{name} 无法读取此 Mac 的内存大小，因此在此不推荐任何模型。', + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download, and {reserve} GB must stay free after it. {free} GB is free now. Free up about {missing} GB.': + '可用磁盘空间不足。适合此 Mac 的最小模型需要下载 {download} GB，下载后还必须保留 {reserve} GB 的可用空间。当前可用空间为 {free} GB。请腾出约 {missing} GB。', 'This Mac runs macOS {version}. Local models need macOS {minimum} or newer.': '此 Mac 运行的是 macOS {version}。本地模型需要 macOS {minimum} 或更高版本。', 'No coding agent that Codeman can connect to a local model is available on this Mac.': diff --git a/src/web/public/index.html b/src/web/public/index.html index 78211715..8026b7ce 100644 --- a/src/web/public/index.html +++ b/src/web/public/index.html @@ -2879,9 +2879,11 @@ + on the row's text). The module shows the group only once the server has + answered for a Mac: it hides it for a server on another system (the title + and hint are about "this Mac"), while the first request is in flight, and + for a non-admin in multi-user mode (the route is admin-only). Not gated on + the endpoints toggle above: the answer does not depend on it. -->

Model on this Mac

What the Mac that Codeman runs on can run as a local coding model. Read-only: nothing is downloaded or installed from here.

diff --git a/src/web/public/local-model-ui.js b/src/web/public/local-model-ui.js index ab4a353b..92c02af0 100644 --- a/src/web/public/local-model-ui.js +++ b/src/web/public/local-model-ui.js @@ -4,11 +4,13 @@ * Reads `GET /api/local-model` (hardware of the machine the server runs on, the recommendation, * the catalogue date) and shows, read-only: * - the hardware line (chip, memory, free disk); - * - the verdict: the headline and notes the server sent, then the pick with its numbers - * (model, harness, context, memory need against the budget, download size, speed); - * - the other models that fit, each with its fit and speed label; - * - one line saying that setting the model up from Codeman is the next step and is not built - * yet. + * - the verdict: the headline the server sent, then the pick with its numbers (model, harness, + * context, memory need against the budget, download size, speed) and the notes; + * - one line saying that Codeman cannot download or install the model yet: that is the next + * step and is not built. It sits right under the pick, where a reader looks for a button; + * - the other models that fit, each with its fit and speed label. A label says what it + * measures in its tooltip and accessible name ("Good" is the memory fit, not a rating of + * the model), and an alternative with a speed estimate prints the figure. * * ## One body, two containers * @@ -23,9 +25,18 @@ * hidden while the first request is in flight and after a failed one (nothing is known about the * machine then, and an error card would be the only thing this feature ever showed on a server * that is not a Mac), when the answer is `not-macos`, when the request is refused (a non-admin in - * multi-user mode gets a 403, which is an answer: stop asking) and after the user hid it on this - * device. The Settings row is where the loading and failure sentences show, and it is the way - * back to the numbers after a Hide. + * multi-user mode gets a `403 FORBIDDEN`, which is an answer: stop asking) and after the user hid + * it on this device. + * + * The Settings group ("Model on this Mac") follows the same knowledge: it is hidden for + * `not-macos` (most servers are not a Mac, and a group about "this Mac" there is false), for a + * refused request and while the first request is in flight (nothing says yet that the server is + * a Mac). It shows the answer for a Mac, and one failure sentence when the first request + * failed. It is the way back to the numbers after a Hide. + * + * "Refused" means the envelope's `errorCode` is `FORBIDDEN`. The auth layer has a second 403 for + * every /api route (`PASSWORD_CHANGE_REQUIRED`, an admin who must change the password first): + * that one is a failed request, asked again on the next show and on `codeman:me`. * * Rules: * - Read-only: no control here downloads, installs or writes anything but the per-device @@ -36,8 +47,10 @@ * (`` is written by every i18n `configure()`). * - Names and labels come from the payload: this file holds no list of models or harnesses and * never compares a CLI id with a literal. - * - Sizes are printed exactly as sent (`needGb`, `downloadGb`, `budgetGb`, `memGb`, - * `freeDiskGb`: decimal GB as text). Bytes are never divided here. + * - Sizes are printed exactly as sent (`needGb`, `downloadGb`, `budgetGb`, `freeDiskGb`: + * decimal GB as text). Bytes are never divided here. The installed memory is the one size + * that is not decimal GB: the hardware line prints `memClassGb`, the size the Mac is sold as + * (48, not 51.5), which is what About This Mac shows. * - A sentence the server chose arrives as `{ text, vars }` and goes through * `localModelT(text, vars)`; this file's own strings are literals in that same call, in * `{placeholder}` form, so the i18n test can harvest them. Both kinds need a zh-CN entry in @@ -52,8 +65,12 @@ * @loadorder 12.59 of 16, after edit-history-ui.js, before entrance-animations.js */ -/** Per-device dismissal of the welcome card. */ -const LOCAL_MODEL_DISMISS_KEY = 'codeman:local-model-dismissed'; +/** + * Per-device dismissal of the welcome card, named for THIS stage (the read-only card). The stage + * that can set a model up takes a key of its own, so hiding a card that offers nothing yet does + * not hide the offer later. + */ +const LOCAL_MODEL_DISMISS_KEY = 'codeman:local-model-dismissed:readonly'; /** The server caches its hardware probe for a minute, so asking again sooner cannot change the answer. */ const LOCAL_MODEL_MAX_AGE_MS = 60000; @@ -100,18 +117,41 @@ function localModelNotAdmin() { return !!(me.multiUser && me.role !== 'admin'); } -/** Chip, memory and free disk in one line. Every part but the memory may be missing. */ +/** + * Chip, memory and free disk in one line. The memory is the size the Mac is sold as + * (`memClassGb`: 48 for a Mac whose 48 x 2^30 bytes are 51.5 decimal GB), a number or nothing: + * an unknown size (0) is left out, never printed. + */ function localModelHardwareLine(hw) { const chip = String(hw.chip || hw.model || ''); - const sizes = { chip, memory: hw.memGb, free: hw.freeDiskGb }; - if (!hw.memGb) return chip; + const memory = Number.isFinite(hw.memClassGb) && hw.memClassGb > 0 ? String(hw.memClassGb) : ''; + const sizes = { chip, memory, free: hw.freeDiskGb }; + if (!memory) return chip; if (chip && hw.freeDiskGb) return localModelT('{chip}, {memory} GB memory, {free} GB free', sizes); if (chip) return localModelT('{chip}, {memory} GB memory', sizes); if (hw.freeDiskGb) return localModelT('{memory} GB memory, {free} GB free', sizes); return localModelT('{memory} GB memory', sizes); } -/** "Qwen3.8 27B (qwen3.8:27b) with OpenCode", then the fit and speed labels the server chose. */ +/** What the fit label measures: the share of the memory budget the model needs. */ +function localModelFitHint(candidate) { + if (!Number.isFinite(candidate.budgetShare)) return ''; + const share = String(Math.round(candidate.budgetShare * 100)); + return localModelT('Memory fit: needs {share}% of what this Mac can give a model', { share }); +} + +/** What the speed label stands for: the estimate behind "Slow", or that there is none. */ +function localModelSpeedHint(candidate) { + const speed = localModelSpeed(candidate.tokensPerSecond); + if (speed) return localModelT('About {speed} tokens per second (estimated).', { speed }); + return localModelT('No speed estimate for this model on this chip'); +} + +/** + * "Qwen3.8 27B (qwen3.8:27b) with OpenCode", then the fit and speed labels the server chose. + * A bare "Good" next to a model name reads as a rating of the model, so each label carries what + * it measures as its tooltip and its accessible name. + */ function localModelCandidateHead(candidate) { const head = localModelEl('div', 'local-model-name-row'); const name = localModelT('{model} ({id}) with {harness}', { @@ -120,19 +160,36 @@ function localModelCandidateHead(candidate) { harness: candidate.cliLabel || candidate.cli, }); head.appendChild(localModelEl('span', 'local-model-name', name)); - for (const label of [candidate.fitLabel, candidate.speedLabel]) { + const pills = [ + [candidate.fitLabel, localModelFitHint], + [candidate.speedLabel, localModelSpeedHint], + ]; + for (const [label, hint] of pills) { if (typeof label !== 'string' || !label) continue; - head.appendChild(localModelEl('span', 'local-model-pill', localModelT(label))); + const pill = localModelEl('span', 'local-model-pill', localModelT(label)); + const meaning = hint(candidate); + if (meaning) { + pill.title = meaning; + pill.setAttribute('role', 'note'); + pill.setAttribute('aria-label', `${pill.textContent}. ${meaning}`); + } + head.appendChild(pill); } return head; } -/** The dim line under a candidate's name: context, memory need (alternatives), download, license (the pick). */ -function localModelMeta(candidate, { need = false, license = false } = {}) { +/** + * The dim line under a candidate's name: context, then for an alternative its memory need and, + * when there is an estimate, its speed (so "Slow" comes with a number on a screen without + * hover), the download, and for the pick the license. + */ +function localModelMeta(candidate, { alternative = false } = {}) { const bits = [localModelT('Context {context} tokens', { context: localModelCount(candidate.context) })]; - if (need && candidate.needGb) bits.push(localModelT('{need} GB of memory', { need: candidate.needGb })); + if (alternative && candidate.needGb) bits.push(localModelT('{need} GB of memory', { need: candidate.needGb })); if (candidate.downloadGb) bits.push(localModelT('{download} GB download', { download: candidate.downloadGb })); - if (license && candidate.license) bits.push(String(candidate.license)); + const speed = alternative ? localModelSpeed(candidate.tokensPerSecond) : ''; + if (speed) bits.push(localModelT('about {speed} tokens per second', { speed })); + if (!alternative && candidate.license) bits.push(String(candidate.license)); return bits.join(' · '); } @@ -140,7 +197,7 @@ function localModelMeta(candidate, { need = false, license = false } = {}) { function localModelPick(pick, recommendation) { const box = localModelEl('div', 'local-model-pick'); box.appendChild(localModelCandidateHead(pick)); - box.appendChild(localModelEl('div', 'local-model-line', localModelMeta(pick, { license: true }))); + box.appendChild(localModelEl('div', 'local-model-line', localModelMeta(pick))); const sizes = { need: pick.needGb, budget: recommendation.budgetGb }; // A wired limit is a setting of this Mac; anything else is the formula's estimate, and says so. const need = @@ -165,17 +222,19 @@ function localModelPick(pick, recommendation) { return box; } -/** The other models that fit. Folded behind a native disclosure on the welcome card, open in Settings. */ +/** + * The other models that fit. Folded behind a native disclosure on the welcome card, open in + * Settings. No count beside the title: the server lists four at most, so the number of rows is + * not the number of models that fit. + */ function localModelAlternatives(alternatives, fold) { const wrap = localModelEl(fold ? 'details' : 'div', 'local-model-alts'); - const title = localModelEl(fold ? 'summary' : 'div', 'local-model-alts-title'); - title.appendChild(localModelEl('span', '', localModelT('Other models that fit'))); - title.appendChild(localModelEl('span', 'local-model-count', String(alternatives.length))); + const title = localModelEl(fold ? 'summary' : 'div', 'local-model-alts-title', localModelT('Other models that fit')); wrap.appendChild(title); for (const candidate of alternatives) { const row = localModelEl('div', 'local-model-alt'); row.appendChild(localModelCandidateHead(candidate)); - row.appendChild(localModelEl('div', 'local-model-line', localModelMeta(candidate, { need: true }))); + row.appendChild(localModelEl('div', 'local-model-line', localModelMeta(candidate, { alternative: true }))); wrap.appendChild(row); } return wrap; @@ -200,12 +259,16 @@ function localModelBody(status, { fold }) { const text = localModelMessage(note); if (text) body.appendChild(localModelEl('div', 'local-model-note', text)); } - const alternatives = pick && Array.isArray(recommendation.alternatives) ? recommendation.alternatives : []; - if (alternatives.length) body.appendChild(localModelAlternatives(alternatives, fold)); + // Right under the pick and its notes, above the list: the pick names a download and maybe a + // harness that is not installed, so this is where a reader looks for a button. if (pick) { - const next = localModelT('Setting this up from Codeman is the next step. It is not built yet.'); + const next = localModelT( + 'Codeman cannot download or install this for you yet. That is the next step, and it is not built yet.' + ); body.appendChild(localModelEl('div', 'local-model-next', next)); } + const alternatives = pick && Array.isArray(recommendation.alternatives) ? recommendation.alternatives : []; + if (alternatives.length) body.appendChild(localModelAlternatives(alternatives, fold)); return body; } @@ -249,24 +312,27 @@ Object.assign(CodemanApp.prototype, { }, async _fetchLocalModel() { - // `_api`, not `_apiJson`: a 403 (admin-only in multi-user mode) is an answer and must be told - // apart from a request that failed. + // `_api`, not `_apiJson`: a refusal (admin-only in multi-user mode) is an answer and must be + // told apart from a request that failed. const res = await this._api(localModelUrl('/api/local-model')); - if (res?.status === 403) { - this._localModelState = 'refused'; - this._localModelStatus = null; - this.renderLocalModel(); - return; - } let body = null; - if (res?.ok) { + if (res?.ok || res?.status === 403) { try { body = await res.json(); } catch { body = null; } } - const data = localModelPayload(body); + // Only the route's own refusal is final. The auth layer answers 403 too, with + // PASSWORD_CHANGE_REQUIRED, for an admin who must change the password first: that is a + // failed request, asked again once the password is changed. + if (res?.status === 403 && body?.errorCode === 'FORBIDDEN') { + this._localModelState = 'refused'; + this._localModelStatus = null; + this.renderLocalModel(); + return; + } + const data = res?.ok ? localModelPayload(body) : null; if (!data) { // A failed refresh must not blank an answer the user is reading: keep the last good one. if (!this._localModelStatus) this._localModelState = 'failed'; @@ -336,6 +402,8 @@ Object.assign(CodemanApp.prototype, { const head = localModelEl('div', 'local-model-head'); const title = localModelEl('h3', 'local-model-title', localModelT('Model on this Mac')); title.id = 'localModelCardTitle'; + // The page can be open on another laptop or a tablet: "this Mac" is the server's. + title.title = localModelT('The Mac that Codeman runs on, which is not always the device you are reading on.'); head.appendChild(title); const hide = localModelEl('button', 'local-model-hide', localModelT('Hide')); hide.type = 'button'; @@ -353,21 +421,20 @@ Object.assign(CodemanApp.prototype, { _renderLocalModelSettings(row) { row.textContent = ''; - const refused = this._localModelRefused(); - // The whole group goes for a non-admin, the way the other admin-only groups of this modal do. - const group = document.getElementById('localModelGroup'); - if (group) group.style.display = refused ? 'none' : ''; - if (refused) return; const status = this._localModelStatus; + const failed = !status && this._localModelState === 'failed'; + // The whole group goes for a non-admin, the way the other admin-only groups of this modal + // do, and for a server that is not a Mac: its title and hint are about "this Mac". Until + // the first answer nothing says the server is one, so the group waits for it. + const shown = !this._localModelRefused() && (failed || (!!status && status.recommendation.reason !== 'not-macos')); + const group = document.getElementById('localModelGroup'); + if (group) group.style.display = shown ? '' : 'none'; + if (!shown) return; if (status) { row.appendChild(localModelBody(status, { fold: false })); return; } - const text = - this._localModelState === 'failed' - ? localModelT('Could not reach the server.') - : localModelT('Checking this Mac…'); - row.appendChild(localModelEl('div', 'local-model-headline', text)); + row.appendChild(localModelEl('div', 'local-model-headline', localModelT('Could not reach the server.'))); }, /** Listeners that outlive a render, wired once. */ diff --git a/src/web/public/styles.css b/src/web/public/styles.css index f34cd700..ba9ac133 100644 --- a/src/web/public/styles.css +++ b/src/web/public/styles.css @@ -22376,9 +22376,13 @@ html[data-tile-anim="soft"] .tile-grid-ghosts.tile-grid-ghosts--release:not(.til min-width: 0; } +/* The line that says nothing can be set up from here yet (.local-model-next) is + the one a reader must not miss, so it reads like every other line: same size, + same --text-dim. --text-muted on this surface was 3.9:1. */ .local-model-hw, .local-model-line, -.local-model-note { +.local-model-note, +.local-model-next { font-size: 0.68rem; line-height: 1.45; color: var(--text-dim); @@ -22418,8 +22422,8 @@ html[data-tile-anim="soft"] .tile-grid-ghosts.tile-grid-ghosts--release:not(.til min-width: 0; } -.local-model-pill, -.local-model-count { +/* Each pill carries what it measures as its tooltip (a title), hence the cursor. */ +.local-model-pill { flex: 0 0 auto; font-size: 0.62rem; line-height: 1.5; @@ -22428,12 +22432,7 @@ html[data-tile-anim="soft"] .tile-grid-ghosts.tile-grid-ghosts--release:not(.til border-radius: 999px; padding: 0 0.42rem; white-space: nowrap; -} - -.local-model-count { - margin-inline-start: 0.4rem; - border-color: transparent; - background: var(--control-bg-hover); + cursor: help; } .local-model-alts-title { @@ -22466,12 +22465,6 @@ html[data-tile-anim="soft"] .tile-grid-ghosts.tile-grid-ghosts--release:not(.til font-weight: 500; } -.local-model-next { - font-size: 0.66rem; - line-height: 1.45; - color: var(--text-muted); -} - /* In App Settings the body sits inside an ordinary .set-row, so it takes that surface's type scale (label 0.76rem over a muted description) and the row supplies the border. */ diff --git a/test/local-model-i18n.test.ts b/test/local-model-i18n.test.ts index cbcb514f..691bff17 100644 --- a/test/local-model-i18n.test.ts +++ b/test/local-model-i18n.test.ts @@ -89,9 +89,14 @@ describe('the local model strings in zh-CN', () => { expect(FROM_MODULE).toContain('Model on this Mac'); expect(FROM_MODULE).toContain('{model} ({id}) with {harness}'); expect(FROM_MODULE).toContain('Needs about {need} GB of the {budget} GB this Mac can give a model (estimated).'); - expect(FROM_MODULE).toContain('Setting this up from Codeman is the next step. It is not built yet.'); - expect(FROM_MODULE).toContain('Checking this Mac…'); - expect(FROM_MODULE.length).toBeGreaterThanOrEqual(18); + expect(FROM_MODULE).toContain( + 'Codeman cannot download or install this for you yet. That is the next step, and it is not built yet.' + ); + // What the fit and speed labels measure, and the speed of an alternative. + expect(FROM_MODULE).toContain('Memory fit: needs {share}% of what this Mac can give a model'); + expect(FROM_MODULE).toContain('No speed estimate for this model on this chip'); + expect(FROM_MODULE).toContain('about {speed} tokens per second'); + expect(FROM_MODULE.length).toBeGreaterThanOrEqual(20); // The only other way the module calls its translator is with payload text (a server // sentence or label), so no string of its own can reach the page unharvested. const code = MODULE.split('\n') @@ -107,10 +112,16 @@ describe('the local model strings in zh-CN', () => { it('lists every server label, the none reasons included', () => { // A label added to the contract later fails below by its text, not here by a count. - expect(FROM_SERVER.length).toBeGreaterThanOrEqual(14); + expect(FROM_SERVER.length).toBeGreaterThanOrEqual(18); for (const text of Object.values(LOCAL_MODEL_NONE_LABELS)) expect(FROM_SERVER).toContain(text); expect(FROM_SERVER).toContain('Best fit for this Mac'); expect(FROM_SERVER).toContain('Speed unknown'); + // The two other sentences of the `memory` reason and the two notes are not in the + // reason map, so they are named here. + expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noneMemoryContext); + expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noneMemoryUnknown); + expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noteSmallPick); + expect(FROM_SERVER).toContain(LOCAL_MODEL_LABELS.noteDiskDropped); }); it('never uses the translator slot {name} for a value of its own', () => { diff --git a/test/local-model-recommend.test.ts b/test/local-model-recommend.test.ts index 86a2c327..bf22dc8b 100644 --- a/test/local-model-recommend.test.ts +++ b/test/local-model-recommend.test.ts @@ -10,9 +10,11 @@ * parser: the MacBook Pro (M5 Pro, 48 GB) and the Mac mini (M4, 16 GB). Each test states the * expected answer and why. * 3. Every reason for verdict `none`, one fixture per reason, and their order. - * 4. The `limited` verdict, the three-state speed in the ranking, harness rows (a gated one is - * never chosen), the `tested` flag that alone may turn on the word "Recommended", and the - * shape of the answer (labels from the fixed list, sizes in one unit). + * 4. The `limited` verdict (a statement about the Mac, not about the pick), the notes that say + * what the headline cannot (a small pick, a model left out for disk space), the three-state + * speed in the ranking, harness rows (a gated one is never chosen), the `tested` flag that + * alone may turn on the word "Recommended", and the shape of the answer (labels from the + * fixed list, sizes in one unit). * * The stock catalogue is used wherever the answer for a real Mac is at stake. Small synthetic * catalogues (validated by the real schema) isolate one rule of the ranking at a time. @@ -463,13 +465,17 @@ describe('the MacBook Pro (Mac17,13: Apple M5 Pro, 48 GB)', () => { describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { /** - * Expected: no recommendation, reason `memory`. + * Expected: no recommendation, reason `memory`, in the sentence that names both numbers. * * Why. The budget is 9.67 GiB (2/3 of 16 GiB minus 1). The only model near it is the 9B. Both * harnesses that may be offered need 65,536 tokens of context, and there the 9B needs * 6.10 GiB of weights + 2 GiB of KV cache + 1 GiB + 0.5 GiB of prompt cache = 9.60 GiB, which * is 0.993 of the budget: above the 0.98 where "tight" ends. At 32,768 tokens it would fit * (0.890, tight), but the one harness that works at that context is gated. + * + * The miss is 1.3 points on an ESTIMATED budget, so the sentence is not "too little memory" + * (what an 8 GB Mac is told): it says the budget (10.4 GB), the need (10.3 GB) and that + * Codeman estimates. */ it('gets no recommendation: nothing fits at the context a harness needs', async () => { const hw = await realMac(MACMINI_SYSCTL, 41_752_848); @@ -482,8 +488,9 @@ describe('the Mac mini (Mac16,10: Apple M4, 16 GB)', () => { verdict: 'none', reason: 'memory', headline: { - text: 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', - vars: { memory: '17.2' }, + text: 'Codeman estimates that this Mac can give a model about {budget} GB of memory. The smallest model a coding agent can work with needs about {need} GB. That leaves no safe margin, so Codeman recommends none here.', + // floor(17,179,869,184 x 2/3) - 1 GiB, and the 9B at 65536 tokens: 10,308,921,757 bytes. + vars: { budget: '10.4', need: '10.3' }, }, notes: [], alternatives: [], @@ -556,12 +563,13 @@ describe('every memory size, with the stock catalogue', () => { // gpt-oss:20b at 65536 tokens needs 15.8 GiB of the 15 GiB budget: out. [24, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], // gpt-oss:20b fits only as Tight here (0.853 at 65536 tokens, 0.927 at 131072), and Good - // ranks before Tight. The Q8 9B is Good but slow on a base M4 (7 tokens per second). - [32, 'Apple M4', 'limited qwen3.5:9b@65536 good ok'], - [32, 'Apple M1 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + // ranks before Tight. The Q8 9B is Good but slow on a base M4 (7 tokens per second). The + // pick is a 9B, the verdict is `ok` all the same: a 20B fits this Mac. + [32, 'Apple M4', 'ok qwen3.5:9b@65536 good ok'], + [32, 'Apple M1 Max', 'ok qwen3.5:9b-q8_0@65536 good ok'], // The 27B (0.905) and the 35B-A3B (0.974) are Tight; gpt-oss:20b is Good but has no speed // figure, and a known speed ranks before an unknown one. - [36, 'Apple M4 Max', 'limited qwen3.5:9b-q8_0@65536 good ok'], + [36, 'Apple M4 Max', 'ok qwen3.5:9b-q8_0@65536 good ok'], // On a Pro chip the dense 27B is slow (8.1 tokens per second), the 35B-A3B is not. [48, 'Apple M4 Pro', 'ok qwen3.6:35b-a3b@131072 good ok'], [48, 'Apple M4 Max', 'ok qwen3.8:27b@131072 good ok'], @@ -712,18 +720,60 @@ describe('verdict none', () => { describe('memory', () => { it('answers so at 8 GB, where nothing fits at any context', () => { const result = recommend(mac(8, 'Apple M2'), STOCK, ALL_INSTALLED); - expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemory); - // The size is the decimal figure of the hardware line, without the unit. - expect(result.headline.vars).toEqual({ memory: '8.6' }); + expectNone( + result, + 'memory', + 'This Mac has {memory} GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.' + ); + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.noneMemory); + // The size the Mac is sold as (what About This Mac shows), not 8 x 2^30 bytes as 8.6 decimal GB. + expect(result.headline.vars).toEqual({ memory: '8' }); expect(result.budgetBytes).toBe(2 * GIB); + // Every computed size stays decimal GB. expect(result.budgetGb).toBe('2.1'); }); - it('answers so when models fit, but only below the context a harness needs', () => { + it('names the installed memory as the Mac is sold, for every size', () => { + // An empty catalogue: nothing fits whatever the size, so the sentence is the plain one. + for (const soldGb of [8, 16, 18, 24, 32, 36, 48, 64, 96, 128]) { + const result = recommend(mac(soldGb, 'Apple M4'), catalogOf([]), [fact('alpha')]); + expect(result.headline, `${soldGb} GB`).toEqual({ + text: LOCAL_MODEL_LABELS.noneMemory, + vars: { memory: String(soldGb) }, + }); + } + }); + + it('says both numbers when models fit, but only below the context a harness needs', () => { // The 16 GB case of the Mac mini, with harnesses that are all present. const result = recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED); - expect(result.reason).toBe('memory'); - expect(result.headline.vars).toEqual({ memory: '17.2' }); + expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemoryContext); + // The need is the smallest one at a context a usable harness works at (the 9B at 65536), + // not the 9B at 32768, which fits and which only the gated harness could use. + expect(result.headline.vars).toEqual({ budget: '10.4', need: '10.3' }); + expect(result.headline.text).toMatch(/^Codeman estimates/); + expect(result.headline.text).not.toContain('{memory}'); + }); + + it('says the same two numbers when the need is far above the budget', () => { + // One model: 3 GiB at 1024 tokens (fits 10 GiB), 30 GiB at the context the harness needs. + const catalog = catalogOf( + [model('probe:1', { bytes: 2 * GIB, kvBytesPerToken: 1024, contexts: [28 * 1024 * 1024, 1024] })], + [{ cli: 'alpha', minContext: 4096, rank: 1 }] + ); + const hw = mac(16, 'Apple M4', { wiredLimitMb: 14000 }); + const result = recommend(hw, catalog, [fact('alpha')]); + expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemoryContext); + // 2 GiB + 28 GiB of KV cache + 1 GiB = 31 GiB; the budget is 10 GiB. + expect(result.headline.vars).toEqual({ budget: formatGb(10 * GIB), need: formatGb(31 * GIB) }); + expect(result.budgetSource).toBe('wired-limit'); + }); + + it('falls back to the plain sentence when no model has a context a harness could use', () => { + const catalog = catalogOf([model('probe:1')], [{ cli: 'alpha', minContext: 4096, rank: 1 }]); + const result = recommend(bigMac, catalog, [fact('alpha')]); + expectNone(result, 'memory', LOCAL_MODEL_LABELS.noneMemory); + expect(result.headline.vars).toEqual({ memory: '48' }); }); it('comes before no-harness when nothing fits at all', () => { @@ -734,10 +784,17 @@ describe('verdict none', () => { expect(recommend(bigMac, catalogOf([]), [fact('alpha')]).reason).toBe('memory'); }); - it('answers so when the memory size is unknown', () => { + it('answers so when the memory size is unknown, without stating a size', () => { const hw = { ...mac(48, 'Apple M4 Max'), memBytes: 0, memGb: '0.0', memClassGb: 0 }; const result = recommend(hw, STOCK, ALL_INSTALLED); - expect(result.reason).toBe('memory'); + expectNone( + result, + 'memory', + 'Codeman could not read how much memory this Mac has, so it recommends no model here.' + ); + // No "0 GB of memory": a size that was never read is not a size. + expect(result.headline.vars).toBeUndefined(); + expect(stringsIn(result.headline).join(' ')).not.toMatch(/\d/); expect(result.budgetBytes).toBe(0); }); }); @@ -787,13 +844,23 @@ describe('verdict none', () => { expectNone( result, 'disk', - 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download and {free} GB is free. Free up about {missing} GB.' + 'Not enough free disk space. The smallest model that fits this Mac is a {download} GB download, and {reserve} GB must stay free after it. {free} GB is free now. Free up about {missing} GB.' ); // download + 10 GiB - free = 6,550,825,373 + 10,737,418,240 - 12,884,901,888 bytes. - expect(result.headline.vars).toEqual({ download: '6.6', free: '12.9', missing: '4.4' }); + expect(result.headline.vars).toEqual({ download: '6.6', reserve: '10.7', free: '12.9', missing: '4.4' }); expect(result.budgetBytes).toBe(35 * GIB); }); + it('gives the reader every number of the sum: download + reserve - free = missing', () => { + // 12 GB free is more than the 6.6 GB download, so without the reserve "not enough" reads false. + const hw = mac(24, 'Apple M4', { freeDiskBytes: 12 * GB, freeDiskGb: formatGb(12 * GB) }); + const vars = recommend(hw, STOCK, ALL_INSTALLED).headline.vars as Record; + expect(vars).toEqual({ download: '6.6', reserve: '10.7', free: '12.0', missing: '5.3' }); + expect(vars.reserve).toBe(formatGb(DISK_HEADROOM_BYTES)); + // Within the rounding of four one-decimal figures. + expect(Number(vars.download) + Number(vars.reserve) - Number(vars.free)).toBeCloseTo(Number(vars.missing), 0); + }); + it('names the smallest model that fits the MEMORY, not the smallest of the catalogue', () => { // On a 48 GB Mac whose harness refuses both 9B ids, the smallest candidate is gpt-oss:20b. const facts = ALL_INSTALLED.map((entry) => ({ @@ -806,12 +873,76 @@ describe('verdict none', () => { expect(result.headline.vars?.download).toBe('13.8'); }); - it('drops only the models that do not leave 10 GiB, and picks among the rest', () => { + it('drops only the models that do not leave 10 GiB, picks among the rest, and says what was left out', () => { // 20 GiB free: the 9B downloads (6.1 and 9.7 GiB) leave 10 GiB or more, the others do not. const hw = mac(48, 'Apple M4 Max', { freeDiskBytes: 20 * GIB, freeDiskGb: formatGb(20 * GIB) }); const result = recommend(hw, STOCK, ALL_INSTALLED); - expect(result.verdict).toBe('limited'); expect(order(result)).toEqual(['qwen3.5:9b-q8_0@65536', 'qwen3.5:9b@65536']); + // A 48 GB Mac is not a small-model Mac because its disk is full: the verdict stays `ok`, + // the pick is called small in a note, and the model it would have been is named with + // what is missing: 17,741,871,939 + 10 GiB - 20 GiB = 7.0 GB. + expect(result.verdict).toBe('ok'); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + expect(result.notes).toEqual([ + { + text: 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.', + }, + { + text: '{model} fits the memory of this Mac too, but its download needs about {missing} GB more free disk space.', + vars: { model: 'Qwen3.8 27B', missing: '7.0' }, + }, + { text: 'Not yet tested by us on a Mac like this.' }, + ]); + }); + + it('turns on 0.4 GB of disk on the real MacBook without calling it a small-model Mac', async () => { + // The cut for the 35B-A3B: 22,621,314,161 + 10 GiB = 33,358,732,401 bytes free. + const macbook = await realMac(MACBOOK_SYSCTL, 890_926_084); + const withFree = (bytes: number): HardwareFacts => ({ + ...macbook, + freeDiskBytes: bytes, + freeDiskGb: formatGb(bytes), + }); + + const enough = recommend(withFree(34 * GB), STOCK, ALL_INSTALLED); + expect(enough.verdict).toBe('ok'); + expect(enough.pick?.modelId).toBe('qwen3.6:35b-a3b'); + expect(enough.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + + const short = recommend(withFree(33 * GB), STOCK, ALL_INSTALLED); + expect(short.verdict).toBe('ok'); + expect(short.headline.text).toBe(LOCAL_MODEL_LABELS.headlineOk); + expect(short.pick?.modelId).toBe('qwen3.5:9b-q8_0'); + expect(order(short)).not.toContain('qwen3.6:35b-a3b@131072'); + expect(short.notes).toEqual([ + { text: LOCAL_MODEL_LABELS.noteSmallPick }, + { text: LOCAL_MODEL_LABELS.noteDiskDropped, vars: { model: 'Qwen3.6 35B-A3B', missing: '0.4' } }, + { text: LOCAL_MODEL_LABELS.noteUntested }, + ]); + // The exact cut: one byte decides, and the note appears with the first missing byte. + const cut = 22_621_314_161 + DISK_HEADROOM_BYTES; + expect(recommend(withFree(cut), STOCK, ALL_INSTALLED).pick?.modelId).toBe('qwen3.6:35b-a3b'); + expect(recommend(withFree(cut - 1), STOCK, ALL_INSTALLED).pick?.modelId).toBe('qwen3.5:9b-q8_0'); + }); + + it('names the best-ranked model left out, also when the pick is not affected', () => { + // 64 GB, M4 Max, 35 GB free: only the Q8 27B (30 GB) has no room. The pick stays the 27B. + const hw = mac(64, 'Apple M4 Max', { freeDiskBytes: 35 * GB, freeDiskGb: '35.0' }); + const result = recommend(hw, STOCK, ALL_INSTALLED); + expect(result.pick?.modelId).toBe('qwen3.8:27b'); + expect(order(result).some((entry) => entry.startsWith('qwen3.8:27b-q8_0'))).toBe(false); + // 29,978,241,837 + 10 GiB - 35 GB. + expect(result.notes).toEqual([ + { text: LOCAL_MODEL_LABELS.noteDiskDropped, vars: { model: 'Qwen3.8 27B (Q8)', missing: '5.7' } }, + { text: LOCAL_MODEL_LABELS.noteUntested }, + ]); + }); + + it('adds no disk note when every model that fits has room, or the free disk is unknown', () => { + for (const hw of [mac(64, 'Apple M4 Max'), without(mac(64, 'Apple M4 Max'), 'freeDiskBytes', 'freeDiskGb')]) { + const texts = recommend(hw, STOCK, ALL_INSTALLED).notes.map((note) => note.text); + expect(texts).not.toContain(LOCAL_MODEL_LABELS.noteDiskDropped); + } }); it('allows a download that leaves exactly 10 GiB', () => { @@ -845,7 +976,7 @@ describe('verdict none', () => { // ---------- 4. Verdicts, ranking, harnesses, tested ---------- describe('verdict limited', () => { - it('is the answer when the best model that fits has fewer than 12 billion parameters', () => { + it('is the answer when no model that fits has 12 billion parameters or more', () => { expect(LIMITED_BELOW_PARAMS_B).toBe(12); const result = recommend(mac(24, 'Apple M4'), STOCK, ALL_INSTALLED); expect(result.verdict).toBe('limited'); @@ -873,11 +1004,77 @@ describe('verdict limited', () => { expect(verdictFor(27.8)).toBe('ok'); }); - it('is about the PICK: a larger model further down the list does not lift it', () => { - // 32 GB: the 20B fits only as Tight, so a 9B is the pick and the answer is limited. + it('is about the MAC, not the pick: a larger model further down the list lifts it', () => { + // 32 GB: the 20B fits only as Tight, so a 9B is the pick. The Mac can still run a 20B, so + // "This Mac can run a small model" would be false of it. const result = recommend(mac(32, 'Apple M4'), STOCK, ALL_INSTALLED); - expect(result.verdict).toBe('limited'); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b', paramsB: 9 }); expect(result.alternatives.map((c) => [c.modelId, c.fit])).toContainEqual(['gpt-oss:20b', 'tight']); + expect(result.verdict).toBe('ok'); + expect(result.headline).toEqual({ text: 'Best fit for this Mac' }); + // The pick is still called what it is. + expect(result.notes).toEqual([ + { + text: 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.', + }, + { text: 'Not yet tested by us on a Mac like this.' }, + ]); + }); + + it('does not tell a 36 GB Mac it is a small-model Mac above a list of 20B, 27B and 35B models', () => { + const result = recommend(mac(36, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + expect(result.pick).toMatchObject({ modelId: 'qwen3.5:9b-q8_0', fit: 'good', speed: 'ok' }); + expect(result.alternatives.map((c) => [c.modelId, c.fit, c.speed])).toEqual([ + ['qwen3.5:9b', 'good', 'ok'], + ['gpt-oss:20b', 'good', 'unknown'], + // Both Tight and fast enough: the higher model rank first. + ['qwen3.8:27b', 'tight', 'ok'], + ['qwen3.6:35b-a3b', 'tight', 'ok'], + ]); + expect(result.verdict).toBe('ok'); + expect(result.headline.text).not.toBe(LOCAL_MODEL_LABELS.headlineLimited); + expect(result.notes[0]).toEqual({ text: LOCAL_MODEL_LABELS.noteSmallPick }); + }); + + it('does not depend on whether the chip is in the chips table', () => { + // Knowing the chip must not lower the verdict: before, a known chip made the 9B win on + // speed and the answer limited, while an unknown chip got `ok` with the 20B. + for (const soldGb of [32, 36]) { + const known = recommend(mac(soldGb, 'Apple M4 Max'), STOCK, ALL_INSTALLED); + const unknown = recommend(mac(soldGb, 'Apple M6'), STOCK, ALL_INSTALLED); + expect([soldGb, known.verdict]).toEqual([soldGb, unknown.verdict]); + expect(known.verdict).toBe('ok'); + } + }); + + it('is limited exactly when nothing with 12 billion parameters fits, on every Mac of the sweep', () => { + for (const hw of SWEEP) { + const result = recommend(hw, STOCK, ALL_INSTALLED); + if (result.verdict === 'none') continue; + const budget = deviceBudget(hw).budgetBytes; + const largeFits = STOCK.models.some( + (m) => + m.paramsB >= LIMITED_BELOW_PARAMS_B && + m.contexts.some((ctx) => ctx >= 65536 && needBytes(m, ctx, STOCK, hw) / budget <= FIT_TIGHT_MAX_SHARE) + ); + expect([hw.memClassGb, hw.chip, result.verdict]).toEqual([hw.memClassGb, hw.chip, largeFits ? 'ok' : 'limited']); + // The small-pick note: under `ok` only, and exactly when the pick is small. + const small = result.notes.some((note) => note.text === LOCAL_MODEL_LABELS.noteSmallPick); + expect(small).toBe(result.verdict === 'ok' && (result.pick as Candidate).paramsB < LIMITED_BELOW_PARAMS_B); + } + }); + + it('counts only a large model a harness can really run', () => { + // 48 GB, but the harnesses refuse every id but the two 9B ones: nothing large can be offered. + const facts = ALL_INSTALLED.map((entry) => ({ + ...entry, + acceptsModelId: (id: string) => id.startsWith('qwen3.5:9b'), + })); + const result = recommend(mac(48, 'Apple M4 Max'), STOCK, facts); + expect(result.verdict).toBe('limited'); + expect(result.headline.text).toBe(LOCAL_MODEL_LABELS.headlineLimited); + // The headline says it, so the note does not repeat it. + expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); }); it('never says "Recommended", for any Mac', () => { @@ -909,6 +1106,17 @@ describe('speed', () => { expect(speedOf(0.5)).toMatchObject({ speed: 'slow', tokensPerSecond: 2, speedLabel: 'Slow' }); }); + it('grades the unrounded figure: 9.95 to 9.9996 tokens per second print as 10 and are still slow', () => { + const speedOf = (speedPer100Gbps: number): Candidate | undefined => + recommend(HUNDRED_GIB_MAC, catalogOf([model('probe:1', { speedPer100Gbps })]), [fact('alpha')]).pick; + // 2.4875 x 4 = 9.95, 2.49 x 4 = 9.96, 2.4999 x 4 = 9.9996: all under 10, all rounding to 10.0. + for (const figure of [2.4875, 2.49, 2.4999]) { + expect(speedOf(figure), String(figure)).toMatchObject({ speed: 'slow', speedLabel: 'Slow', tokensPerSecond: 10 }); + } + // And the first figure that is not under 10. + expect(speedOf(2.5)).not.toHaveProperty('speedLabel'); + }); + it('is unknown for a model without a figure, on any chip', () => { // 48 GB: five models fit, so all of them are listed and gpt-oss:20b is among them. for (const chip of ['Apple M1', 'Apple M4 Max', 'Apple M5 Ultra']) { @@ -1145,7 +1353,8 @@ describe('the tested flag and the word "Recommended"', () => { } expect(stringsIn(result).join(' ')).not.toMatch(/recommended/i); if (result.verdict !== 'none') { - expect(result.notes).toEqual([{ text: LOCAL_MODEL_LABELS.noteUntested }]); + // Always the last note; a small pick under `ok` has one more before it. + expect(result.notes[result.notes.length - 1]).toEqual({ text: LOCAL_MODEL_LABELS.noteUntested }); } } }); @@ -1239,12 +1448,30 @@ describe('the shape of the answer', () => { 'no-harness', 'ok', ]); + // The two other sentences of the `memory` reason, and an answer with every kind of note. + answers.push( + recommend(mac(16, 'Apple M4'), STOCK, ALL_INSTALLED), + recommend({ ...mac(48, 'Apple M4'), memBytes: 0, memGb: '0.0', memClassGb: 0 }, STOCK, ALL_INSTALLED), + recommend(mac(48, 'Apple M4', { freeDiskBytes: 20 * GIB, freeDiskGb: '21.5' }), STOCK, ALL_INSTALLED) + ); + expect(answers.slice(5).map((answer) => answer.headline.text)).toEqual([ + LOCAL_MODEL_LABELS.noneMemoryContext, + LOCAL_MODEL_LABELS.noneMemoryUnknown, + LOCAL_MODEL_LABELS.headlineOk, + ]); + expect(answers[7].notes.map((note) => note.text)).toEqual([ + LOCAL_MODEL_LABELS.noteSmallPick, + LOCAL_MODEL_LABELS.noteDiskDropped, + LOCAL_MODEL_LABELS.noteUntested, + ]); for (const answer of answers) { - const slots = [...answer.headline.text.matchAll(/\{(\w+)\}/g)].map((match) => match[1]).sort(); - expect(Object.keys(answer.headline.vars ?? {}).sort()).toEqual(slots); - // `{name}` belongs to the browser's translator. - expect(slots).not.toContain('name'); - for (const value of Object.values(answer.headline.vars ?? {})) expect(typeof value).toBe('string'); + for (const message of [answer.headline, ...answer.notes]) { + const slots = [...message.text.matchAll(/\{(\w+)\}/g)].map((match) => match[1]).sort(); + expect(Object.keys(message.vars ?? {}).sort()).toEqual(slots); + // `{name}` belongs to the browser's translator. + expect(slots).not.toContain('name'); + for (const value of Object.values(message.vars ?? {})) expect(typeof value).toBe('string'); + } } }); diff --git a/test/local-model-ui.test.ts b/test/local-model-ui.test.ts index 00daa696..07d80879 100644 --- a/test/local-model-ui.test.ts +++ b/test/local-model-ui.test.ts @@ -4,11 +4,15 @@ * module and a mocked `_api`. * * What is pinned: - * - every state the route can produce: loading, refused (403), failed, not macOS, each `none` - * reason, `limited`, `ok`; - * - the card stays hidden until the server answered for a Mac, and the Settings row carries - * the loading and failure sentences; + * - every state the route can produce: loading, refused (403 FORBIDDEN), a 403 that is not a + * refusal, failed, not macOS, each `none` sentence, `limited`, `ok`; + * - the card stays hidden until the server answered for a Mac, and so does the Settings group + * (never "Model on this Mac" on a server that is not one); the group carries the failure + * sentence; * - the card and the row show the SAME pick (one builder), the card folds the alternatives; + * - the installed memory is the size the Mac is sold as, every other size is printed as sent; + * - each fit and speed label says what it measures, and the line about the missing setup + * sits right under the pick; * - every payload string lands as text: an `` in a model, harness or chip name, * or in a slot value of a server sentence, creates no element and runs nothing; * - sizes are printed as sent (decimal GB text), and the word "Recommended" is the server's @@ -42,8 +46,10 @@ const INDEX = readFileSync(resolve(PUBLIC, 'index.html'), 'utf8'); const MODULE = readFileSync(resolve(PUBLIC, 'local-model-ui.js'), 'utf8'); const I18N = readFileSync(resolve(PUBLIC, 'i18n.js'), 'utf8'); -const DISMISS_KEY = 'codeman:local-model-dismissed'; -const NEXT_STEP = 'Setting this up from Codeman is the next step. It is not built yet.'; +/** Named for the read-only stage, so the stage that can set a model up takes a key of its own. */ +const DISMISS_KEY = 'codeman:local-model-dismissed:readonly'; +const NEXT_STEP = + 'Codeman cannot download or install this for you yet. That is the next step, and it is not built yet.'; type ApiResponse = { ok: boolean; status: number; json(): Promise } | null; @@ -253,6 +259,18 @@ function none(reason: NoneReason, vars?: Record, hardware: Parti return status({ verdict: 'none', reason, headline, budgetBytes: 0, budgetGb: '0.0' }, hardware); } +/** A `none` answer whose headline is one of the other sentences of its reason. */ +function noneWith( + reason: NoneReason, + text: LocalModelMessage['text'], + vars?: Record, + hardware: Partial = {} +) { + const answer = none(reason, undefined, hardware); + answer.recommendation.headline = { text, ...(vars ? { vars } : {}) }; + return answer; +} + const NONE_CASES: Array<{ reason: Exclude; answer: LocalModelStatus; sentence: string }> = [ { reason: 'intel', @@ -261,15 +279,15 @@ const NONE_CASES: Array<{ reason: Exclude; answer: Loca }, { reason: 'memory', - answer: none('memory', { memory: '8.6' }, { memGb: '8.6', memClassGb: 8 }), + answer: none('memory', { memory: '8' }, { memGb: '8.6', memClassGb: 8 }), sentence: - 'This Mac has 8.6 GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', + 'This Mac has 8 GB of memory. That is too little to run a model that can drive a coding agent, so Codeman recommends none here.', }, { reason: 'disk', - answer: none('disk', { download: '17.7', free: '12.0', missing: '16.4' }, { freeDiskGb: '12.0' }), + answer: none('disk', { download: '17.7', reserve: '10.7', free: '12.0', missing: '16.4' }, { freeDiskGb: '12.0' }), sentence: - 'Not enough free disk space. The smallest model that fits this Mac is a 17.7 GB download and 12.0 GB is free. Free up about 16.4 GB.', + 'Not enough free disk space. The smallest model that fits this Mac is a 17.7 GB download, and 10.7 GB must stay free after it. 12.0 GB is free now. Free up about 16.4 GB.', }, { reason: 'macos', @@ -328,18 +346,41 @@ describe('the markup the module fills', () => { }); describe('before there is an answer', () => { - it('keeps the card hidden while the request is in flight, and says so in Settings', async () => { - const { newApp, card, row } = mount(); + it('keeps the card and the Settings group hidden while the first request is in flight', async () => { + const { newApp, card, row, group } = mount(); let answer!: (response: ApiResponse) => void; const app = newApp(() => new Promise((done) => (answer = done))); const loading = app.loadLocalModel(); expect(card.hidden).toBe(true); expect(card.textContent).toBe(''); - expect(row.textContent).toBe('Checking this Mac…'); + // Nothing says yet that the server is a Mac, so no group titled "Model on this Mac" and no + // sentence about "this Mac" either: on most servers both would be false. + expect(group.style.display).toBe('none'); + expect(row.textContent).toBe(''); answer(ok(OK_STATUS)); await loading; expect(card.hidden).toBe(false); - expect(row.textContent).not.toContain('Checking this Mac…'); + expect(group.style.display).toBe(''); + expect(texts(row, '.local-model-headline')).toEqual(['Best fit for this Mac']); + }); + + it('keeps an answer on screen, group included, while it is refreshed', async () => { + const { dom, newApp, row, group } = mount(); + let now = 1_000_000; + (dom.window as unknown as { Date: DateConstructor }).Date.now = () => now; + let answer!: (response: ApiResponse) => void; + const answers: Array<() => Promise> = [ + async () => ok(OK_STATUS), + () => new Promise((done) => (answer = done)), + ]; + const app = newApp(() => (answers.shift() as () => Promise)()); + await app.loadLocalModel(); + now += 61_000; + const refreshing = app.loadLocalModel(); + expect(group.style.display).toBe(''); + expect(texts(row, '.local-model-headline')).toEqual(['Best fit for this Mac']); + answer(ok(OK_STATUS)); + await refreshing; }); it.each([ @@ -383,6 +424,41 @@ describe('a refused request (a non-admin in multi-user mode)', () => { expect(app._api).toHaveBeenCalledTimes(1); }); + it('does not take another 403 for a refusal: an admin who must change the password is asked again', async () => { + // The auth layer answers 403 PASSWORD_CHANGE_REQUIRED for every /api route until the + // password is changed, and the dialog that changes it closes without a reload. + const { dom, document, newApp, card, row, group } = mount(); + const win = dom.window as unknown as { __codemanUser?: { multiUser: boolean; role: string } }; + win.__codemanUser = { multiUser: true, role: 'admin' }; + const answers: ApiResponse[] = [refusal(403, 'PASSWORD_CHANGE_REQUIRED'), ok(OK_STATUS)]; + const app = newApp(async () => answers.shift() ?? null); + await app.loadLocalModel(); + expect(app._api).toHaveBeenCalledTimes(1); + expect(card.hidden).toBe(true); + // A failed request, not a refusal: the group stays, with the failure sentence. + expect(group.style.display).toBe(''); + expect(row.textContent).toBe('Could not reach the server.'); + + // The password was changed: admin-ui.js announces the user again. + document.dispatchEvent(new dom.window.CustomEvent('codeman:me')); + await vi.waitFor(() => expect(card.hidden).toBe(false)); + expect(app._api).toHaveBeenCalledTimes(2); + expect(group.style.display).toBe(''); + expect(texts(row, '.local-model-headline')).toEqual(['Best fit for this Mac']); + }); + + it.each([ + ['a 403 without a body', { ok: false, status: 403, json: async () => Promise.reject(new Error('no body')) }], + ['a 403 with another error code', refusal(403, 'SOMETHING_ELSE')], + ] as Array<[string, ApiResponse]>)('keeps asking after %s', async (_name, response) => { + const { newApp, group } = mount(); + const app = newApp(response); + await app.loadLocalModel(); + await app.loadLocalModel(); + expect(app._api).toHaveBeenCalledTimes(2); + expect(group.style.display).toBe(''); + }); + it('does not ask at all once the page knows the role, and asks when the role turns out to be admin', async () => { const { dom, document, newApp, card, group } = mount(); const win = dom.window as unknown as { __codemanUser?: { multiUser: boolean; role: string } }; @@ -402,22 +478,45 @@ describe('a refused request (a non-admin in multi-user mode)', () => { }); describe('a server that is not a Mac', () => { - it('hides the card and leaves one plain sentence in Settings', async () => { + const LINUX = none('not-macos', undefined, { + platform: 'linux', + arch: 'x64', + arm64: undefined, + chip: 'AMD Ryzen 9 7950X', + source: 'not-probed', + }); + + it('shows neither the card nor the Settings group: nothing there is about this server', async () => { const { newApp, card, row, group } = mount(); - const answer = none('not-macos', undefined, { - platform: 'linux', - arch: 'x64', - arm64: undefined, - chip: 'AMD Ryzen 9 7950X', - source: 'not-probed', - }); - await newApp(ok(answer)).loadLocalModel(); + await newApp(ok(LINUX)).loadLocalModel(); expect(card.hidden).toBe(true); expect(card.textContent).toBe(''); - expect(group.style.display).toBe(''); - expect(row.textContent).toBe('Codeman recommends a local model only on a Mac with Apple silicon.'); - // No line listing another machine's parts under a heading that says "this Mac". - expect(row.querySelector('.local-model-hw')).toBeNull(); + // A group titled "Model on this Mac", about "the Mac that Codeman runs on", on a Linux + // server: hidden, the same lever as a refused request. + expect(group.style.display).toBe('none'); + expect(row.textContent).toBe(''); + }); + + it('never shows the group at any point of the load, and keeps it away on a refresh', async () => { + const { dom, newApp, group, row } = mount(); + let now = 1_000_000; + (dom.window as unknown as { Date: DateConstructor }).Date.now = () => now; + const seen: string[] = []; + const app = newApp(async () => { + seen.push(group.style.display); + return ok(LINUX); + }); + const first = app.loadLocalModel(); + seen.push(group.style.display); + await first; + now += 61_000; + const second = app.loadLocalModel(); + seen.push(group.style.display); + await second; + seen.push(group.style.display); + expect(app._api).toHaveBeenCalledTimes(2); + expect(seen).toEqual(['none', 'none', 'none', 'none', 'none']); + expect(row.textContent).toBe(''); }); }); @@ -441,6 +540,41 @@ describe('verdict none on a Mac', () => { const covered = [...NONE_CASES.map((c) => c.reason), 'not-macos'].sort(); expect(covered).toEqual(Object.keys(LOCAL_MODEL_NONE_LABELS).sort()); }); + + it('says both numbers when the Mac misses by a hair (the 16 GB Mac mini)', async () => { + const { newApp, card, row } = mount(); + const answer = noneWith( + 'memory', + LOCAL_MODEL_LABELS.noneMemoryContext, + { budget: '10.4', need: '10.3' }, + { memGb: '17.2', memClassGb: 16, chip: 'Apple M4', freeDiskGb: '42.8' } + ); + await newApp(ok(answer)).loadLocalModel(); + for (const root of [card, row]) { + expect(texts(root, '.local-model-hw')).toEqual(['Apple M4, 16 GB memory, 42.8 GB free']); + expect(texts(root, '.local-model-headline')).toEqual([ + 'Codeman estimates that this Mac can give a model about 10.4 GB of memory. The smallest model a coding agent can work with needs about 10.3 GB. That leaves no safe margin, so Codeman recommends none here.', + ]); + } + }); + + it('states no memory size when the server could not read one', async () => { + const { newApp, card, row } = mount(); + const answer = noneWith('memory', LOCAL_MODEL_LABELS.noneMemoryUnknown, undefined, { + memBytes: 0, + memGb: '0.0', + memClassGb: 0, + }); + await newApp(ok(answer)).loadLocalModel(); + for (const root of [card, row]) { + // The chip and nothing else: no "0 GB memory" in the hardware line, no size in the sentence. + expect(texts(root, '.local-model-hw')).toEqual(['Apple M5 Pro']); + expect(texts(root, '.local-model-headline')).toEqual([ + 'Codeman could not read how much memory this Mac has, so it recommends no model here.', + ]); + expect(root.textContent).not.toMatch(/\b0(\.0)? GB/); + } + }); }); describe('verdict ok', () => { @@ -449,7 +583,12 @@ describe('verdict ok', () => { await newApp(ok(OK_STATUS)).loadLocalModel(); expect(card.hidden).toBe(false); expect(texts(card, '.local-model-title')).toEqual(['Model on this Mac']); - expect(texts(card, '.local-model-hw')).toEqual(['Apple M5 Pro, 51.5 GB memory, 912 GB free']); + // The page can be open on another device: the title says whose Mac is meant. + expect((card.querySelector('.local-model-title') as HTMLElement).title).toBe( + 'The Mac that Codeman runs on, which is not always the device you are reading on.' + ); + // 48, the size this Mac is sold as and what About This Mac shows: not 51.5 (its bytes in decimal GB). + expect(texts(card, '.local-model-hw')).toEqual(['Apple M5 Pro, 48 GB memory, 912 GB free']); expect(texts(card, '.local-model-headline')).toEqual(['Best fit for this Mac']); expect(texts(card, '.local-model-note')).toEqual(['Not yet tested by us on a Mac like this.']); @@ -486,8 +625,10 @@ describe('verdict ok', () => { const { newApp, card } = mount(); await newApp(ok(OK_STATUS)).loadLocalModel(); const alts = card.querySelector('.local-model-alts')!; - expect(texts(alts, '.local-model-alts-title')).toEqual(['Other models that fit2']); - expect(texts(alts, '.local-model-count')).toEqual(['2']); + // No count beside the title: the server lists four at most, so a number here would be + // the number of rows, not of models that fit. + expect(texts(alts, '.local-model-alts-title')).toEqual(['Other models that fit']); + expect(alts.querySelector('.local-model-count')).toBeNull(); const rows = [...alts.querySelectorAll('.local-model-alt')]; expect(rows.map((row) => texts(row, '.local-model-name')[0])).toEqual([ 'Qwen3.6 35B-A3B (qwen3.6:35b-a3b) with OpenCode', @@ -495,23 +636,91 @@ describe('verdict ok', () => { ]); // A candidate whose speed is fine carries a fit label only. expect(rows.map((row) => texts(row, '.local-model-pill'))).toEqual([['Good'], ['Tight', 'Slow']]); + // An alternative with an estimate prints it: "Slow" alone does not say how slow. expect(rows.map((row) => texts(row, '.local-model-line')[0])).toEqual([ - 'Context 131,072 tokens · 26.4 GB of memory · 22.6 GB download', - 'Context 65,536 tokens · 36.4 GB of memory · 30.0 GB download', + 'Context 131,072 tokens · 26.4 GB of memory · 22.6 GB download · about 42 tokens per second', + 'Context 65,536 tokens · 36.4 GB of memory · 30.0 GB download · about 5.2 tokens per second', ]); }); + it('says what each label measures, as its tooltip and its accessible name', async () => { + const { newApp, card, row } = mount(); + await newApp(ok(OK_STATUS)).loadLocalModel(); + for (const root of [card, row]) { + const pills = [...root.querySelectorAll('.local-model-pill')] as HTMLElement[]; + expect(pills.map((pill) => [pill.textContent, pill.title])).toEqual([ + // The pick: 0.79 of the budget, a chip in no chips row. + ['Good', 'Memory fit: needs 79% of what this Mac can give a model'], + ['Speed unknown', 'No speed estimate for this model on this chip'], + ['Good', 'Memory fit: needs 70% of what this Mac can give a model'], + ['Tight', 'Memory fit: needs 97% of what this Mac can give a model'], + ['Slow', 'About 5.2 tokens per second (estimated).'], + ]); + for (const pill of pills) { + expect(pill.getAttribute('role')).toBe('note'); + expect(pill.getAttribute('aria-label')).toBe(`${pill.textContent}. ${pill.title}`); + } + } + }); + + it('leaves a fit label without a share bare instead of inventing a number', async () => { + const { newApp, card } = mount(); + const bare = candidate({ budgetShare: undefined as unknown as number, speedLabel: undefined, speed: 'ok' }); + await newApp(ok(status({ pick: bare }))).loadLocalModel(); + const pill = card.querySelector('.local-model-pill') as HTMLElement; + expect(pill.textContent).toBe('Good'); + expect(pill.title).toBe(''); + expect(pill.hasAttribute('aria-label')).toBe(false); + }); + + it('puts the line about the missing setup right under the pick and its notes, above the list', async () => { + const { newApp, card, row } = mount(); + const withNotes = status({ + notes: [ + { text: LOCAL_MODEL_LABELS.noteSmallPick }, + { text: LOCAL_MODEL_LABELS.noteDiskDropped, vars: { model: 'Qwen3.6 35B-A3B', missing: '0.4' } }, + { text: LOCAL_MODEL_LABELS.noteUntested }, + ], + pick: candidate(), + alternatives: [ALT_FAST, ALT_SLOW], + }); + await newApp(ok(withNotes)).loadLocalModel(); + for (const root of [card, row]) { + const body = root.querySelector('.local-model-body')!; + expect([...body.children].map((el) => el.className)).toEqual([ + 'local-model-hw', + 'local-model-headline', + 'local-model-pick', + 'local-model-note', + 'local-model-note', + 'local-model-note', + 'local-model-next', + 'local-model-alts', + ]); + // A note with slots is filled like a headline. + expect(texts(root, '.local-model-note')).toEqual([ + 'The best fit is a small model. It handles small edits and questions. It does not replace a cloud model.', + 'Qwen3.6 35B-A3B fits the memory of this Mac too, but its download needs about 0.4 GB more free disk space.', + 'Not yet tested by us on a Mac like this.', + ]); + expect(texts(root, '.local-model-next')).toEqual([NEXT_STEP]); + } + }); + it('prints every size exactly as sent, in one unit', async () => { const { newApp, card, row } = mount(); await newApp(ok(OK_STATUS)).loadLocalModel(); for (const root of [card, row]) { const text = root.textContent ?? ''; expect(text).not.toMatch(/GiB|MiB|\bMB\b|bytes/); - // Every number in front of the unit is one the payload carries as text. + // Every number in front of the unit is one the payload carries: as text for the computed + // sizes, and `memClassGb` (48) for the installed memory. const printed = [...text.matchAll(/([\d.,]+) GB/g)].map((m) => m[1]); - const sent = ['51.5', '912', '37.6', '29.6', '17.7', '26.4', '22.6', '36.4', '30.0']; + const sent = ['48', '912', '37.6', '29.6', '17.7', '26.4', '22.6', '36.4', '30.0']; expect(printed.length).toBeGreaterThanOrEqual(sent.length); expect(printed.filter((value) => !sent.includes(value))).toEqual([]); + // The decimal figure of the installed memory (`memGb`) is on neither surface. + expect(text).not.toContain('51.5'); } }); @@ -546,10 +755,18 @@ describe('verdict ok', () => { it('describes the hardware with whatever parts the probe got', async () => { const { newApp, card } = mount(); const cases: Array<[Partial, string]> = [ - [{ freeDiskBytes: undefined, freeDiskGb: undefined }, 'Apple M5 Pro, 51.5 GB memory'], - [{ chip: undefined }, 'Mac17,13, 51.5 GB memory, 912 GB free'], - [{ chip: undefined, model: undefined }, '51.5 GB memory, 912 GB free'], - [{ chip: undefined, model: undefined, freeDiskBytes: undefined, freeDiskGb: undefined }, '51.5 GB memory'], + [{ freeDiskBytes: undefined, freeDiskGb: undefined }, 'Apple M5 Pro, 48 GB memory'], + [{ chip: undefined }, 'Mac17,13, 48 GB memory, 912 GB free'], + [{ chip: undefined, model: undefined }, '48 GB memory, 912 GB free'], + [{ chip: undefined, model: undefined, freeDiskBytes: undefined, freeDiskGb: undefined }, '48 GB memory'], + // The sizes Macs are sold in, as sold. + [{ memGb: '17.2', memClassGb: 16 }, 'Apple M5 Pro, 16 GB memory, 912 GB free'], + [{ memGb: '34.4', memClassGb: 32 }, 'Apple M5 Pro, 32 GB memory, 912 GB free'], + [{ memGb: '137', memClassGb: 128 }, 'Apple M5 Pro, 128 GB memory, 912 GB free'], + // An unknown size is left out, whatever else the payload carries for it. + [{ memGb: '0.0', memClassGb: 0 }, 'Apple M5 Pro'], + [{ memClassGb: undefined as unknown as number }, 'Apple M5 Pro'], + [{ memClassGb: '48' as unknown as number }, 'Apple M5 Pro'], ]; for (const [hardware, line] of cases) { await newApp(ok(status({ pick: candidate() }, hardware))).loadLocalModel(); @@ -567,7 +784,7 @@ describe('verdict limited', () => { expect(texts(root, '.local-model-headline')).toEqual([ 'This Mac can run a small model. It handles small edits and questions. It does not replace a cloud model.', ]); - expect(texts(root, '.local-model-hw')).toEqual(['Apple M4, 19.3 GB memory, 912 GB free']); + expect(texts(root, '.local-model-hw')).toEqual(['Apple M4, 18 GB memory, 912 GB free']); expect(texts(root, '.local-model-name')).toEqual(['Qwen3.5 9B (qwen3.5:9b) with OpenCode']); expect(texts(root, '.local-model-pill')).toEqual(['Tight']); expect(texts(root, '.local-model-pick .local-model-line')).toEqual([ @@ -633,7 +850,8 @@ describe('payload text is text', () => { for (const root of [card, row]) { expect(root.querySelector('img')).toBeNull(); expect(root.querySelector('script, iframe, a, [onerror]')).toBeNull(); - expect(texts(root, '.local-model-hw')[0]).toBe(`${EVIL}, ${EVIL} GB memory, ${EVIL} GB free`); + // The memory is a number or nothing: text in its place (`memGb` here) is never printed. + expect(texts(root, '.local-model-hw')[0]).toBe(`${EVIL}, 48 GB memory, ${EVIL} GB free`); expect(texts(root, '.local-model-headline')[0]).toBe( `This Mac runs macOS ${EVIL}. Local models need macOS ${EVIL} or newer.` ); @@ -768,10 +986,13 @@ describe('with the real translator', () => { const app = newApp(ok(OK_STATUS)); await app.loadLocalModel(); expect(texts(card, '.local-model-headline')).toEqual(['最适合此 Mac 的选择']); - expect(texts(card, '.local-model-pill')).toEqual(['良好', '速度未知', '良好', '紧张', '较慢']); + expect(texts(card, '.local-model-pill')).toEqual(['良好', '速度未知', '良好', '偏紧', '较慢']); + expect((card.querySelector('.local-model-pill') as HTMLElement).title).toBe( + '内存占用：需要此 Mac 可分配给模型的内存的 79%' + ); // Names and numbers ride the slots untouched. expect(texts(card, '.local-model-name')[0]).toBe('Qwen3.8 27B（qwen3.8:27b），搭配 OpenCode'); - expect(texts(row, '.local-model-hw')).toEqual(['Apple M5 Pro，内存 51.5 GB，可用磁盘空间 912 GB']); + expect(texts(row, '.local-model-hw')).toEqual(['Apple M5 Pro，内存 48 GB，可用磁盘空间 912 GB']); // Both containers are data-i18n-skip, so the module itself must repaint. i18n.configure({ language: 'en' }); @@ -782,7 +1003,7 @@ describe('with the real translator', () => { i18n.configure({ displayName: 'Acme' }); await settle(); expect(texts(card, '.local-model-next')).toEqual([ - 'Setting this up from Acme is the next step. It is not built yet.', + 'Acme cannot download or install this for you yet. That is the next step, and it is not built yet.', ]); }); From 66ca92f7cf37197717cbf59e0cb0a7946ff3d06f Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Fri, 6 Nov 2026 17:38:12 +0100 Subject: [PATCH 8/8] docs(local-model): label the route experimental and write down the rules - docs/api-reference.md: GET /api/local-model is labeled experimental until the step that sets a model up ships, so its payload is not a stable /api/v1 contract one stage before that step changes it. The section now describes the verdict as judged, the three sentences of the memory reason, the notes, the speed rounding and which size to show as the installed memory. The os-fallback description was wrong: the free disk space is reported there too. - docs/versioning-policy.md: the route is listed under the experimental surfaces. - docs/wiki/Settings-Reference.md: App Settings > Models > Model on this Mac, in the words of a user. - CLAUDE.md and src/web/CLAUDE.md: the invariants the code relies on (read-only, the probe, harness rows as data, the meaning of limited, the word Recommended, one unit, labels from the payload, when the card and the group show). Co-Authored-By: Claude Fable 5.1 --- CLAUDE.md | 2 ++ docs/api-reference.md | 18 ++++++++++-------- docs/versioning-policy.md | 13 +++++++++++++ docs/wiki/Settings-Reference.md | 14 ++++++++++++++ src/web/CLAUDE.md | 2 ++ src/web/routes/local-model-routes.ts | 5 +++++ 6 files changed, 46 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f729ae40..496d72b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -191,6 +191,8 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph ⚠️ **llama-swap endpoints** (one model at a time): the apply routes check `GET /running` and return `requiresConfirmation` before evicting a model another live session uses; `confirmedSwap` and `confirmedContext` are SEPARATE flags and must stay so. Claude alone gets a context floor (`CLAUDE_MIN_SAFE_CONTEXT_TOKENS`); context is parsed from `/running`'s `cmd`, never trusted from `/props`. The floor also divides that context by the model's parallel slot count (`modelSlotCounts`: `--parallel`/`-np` from `cmd`, else `/props` `total_slots`; `perSlotContext()`), and `/props` only ever RAISES that warning: it supplies the context only when `cmd` states none and the count only when `cmd` has no `--parallel`, never a larger pool. `/props` is probed once per launch command (it refreshes llama-swap's last-use time, so a `ttl` would never expire), and a `/props` context beside an explicit `--parallel N` is already per slot, so it is not divided again. Backend log lines come from llama-swap's `/api/events` `upstream` source, never `/logs`. → [architecture-invariants#custom-model-endpoint-profiles](docs/architecture-invariants.md#custom-model-endpoint-profiles) +**Local model recommendation** (`GET /api/local-model`, `src/local-model/`, `src/config/local-model-catalog.ts`, `src/types/local-model.ts`; EXPERIMENTAL, read-only stage): the server looks at the Mac it runs on and says which local coding model fits it and with which agent CLI. ⚠️ **Read-only**: it downloads nothing, installs nothing, writes no file and adds no settings key; the install flow is a later stage and nothing here may grow a write before it. Apple silicon macOS only: anywhere else the answer is a `200` with verdict `none` (reasons `not-macos`, `intel`, `memory`, `disk`, `macos`, `no-harness`), and a failed probe is an answer, never a 500. The probe (`hardware.ts`) is ONE bounded async spawn of `/usr/sbin/sysctl` WITHOUT `-n`, parsed as `key: value` lines by the pure `parseSysctl()`, plus `statfs` of the data dir (probed bounded first, #516) and `node:os` fallbacks, cached 60 s; ⚠️ never `system_profiler`, never a second spawn, and `iogpu.wired_limit_mb` of 0 means not set. The catalogue is a compiled constant validated by a Zod schema: ⚠️ no override file and no runtime data, and a model id is `name:tag` in lowercase with no slash (`LOCAL_MODEL_ID_PATTERN`). ⚠️ A harness is a catalogue ROW (`{ cli, minContext, rank, gated?, slowFirstTurn? }`) plus a `HarnessFact` built from the CLI registry at call time (`harness-facts.ts`); no code compares a CLI id (both id-branching guards scan these files). `recommend()` (`recommend.ts`) is pure. ⚠️ `limited` is a statement about the MAC (no model with 12 billion parameters or more fits), judged before the disk filter and whatever the pick is; a small pick under `ok` carries `noteSmallPick`, and a model left out for disk space is named in `noteDiskDropped`, never dropped in silence. ⚠️ The word "Recommended" exists only in `headlineTested` and is sent only when the pick's model has a `tested` entry for exactly this chip, memory size and harness; no row has one today. ⚠️ Speed is `ok`/`slow`/`unknown`, graded on the unrounded figure, and a chip in no chips row is `unknown`: never guess a bandwidth. ⚠️ ONE unit for every computed size a person reads: decimal GB text from `formatGb()` (`format.ts`), sent beside the bytes, and the browser never divides bytes; the one exception is the installed memory, shown as `memClassGb` (the size the Mac is sold as, 48 and not 51.5). ⚠️ Every sentence the server chooses is a `LOCAL_MODEL_LABELS` text with `{placeholder}` slots filled from `vars` (no apostrophe in a text: the zh-CN key is single-quoted), and each needs a zh-CN entry. Admin only in multi-user mode (403 before anything is probed). Tests: `test/local-model-*.test.ts`, `test/routes/local-model-routes.test.ts`. Card rules: `src/web/CLAUDE.md` (Model on this Mac). → `docs/api-reference.md` (Local model recommendation), `docs/versioning-policy.md` + **MCP server sync** (opt-in, `mcpSyncEnabled`, SYNCED, default OFF; `src/mcp-sync.ts`, `GET`/`POST /api/mcp-sync`): copies each installed, enabled CLI's user-level MCP servers into the others. It is the ONE subsystem that writes another CLI's REAL user config (`~/.claude.json`, `~/.codex/config.toml`, `~/.gemini/*`, opencode's), which is why it is opt-in and admin-only in multi-user mode (both verbs 403 for a non-admin, and the Settings group is hidden for them). Where each CLI keeps the file is registry data, `capabilities.mcpConfig` (`{ path, format, relocation? }`), never a branch on the id. ⚠️ Tools that sync but are not run modes (GitHub Copilot CLI, the first) are plain data in `MCP_SYNC_ONLY_TOOLS` (`src/mcp-sync-targets.ts`), take part only when installed or already configured, and their companion `settings.json` (`disabledMcpServers`) is read, never written; a test must clear their relocation vars too. ⚠️ ADDITIVE only: a name already defined, in any shape, is never edited or removed (a different same-name definition is a reported conflict), and a server switched off in its own CLI is never copied. ⚠️ Never write a file that did not parse; re-parse the NEW text and require every added server to read back before the tmp+rename (written through a symlink, previous file kept as `.codeman-bak`, one apply at a time, else 409). ⚠️ A file that receives copied `env`/`headers` (secrets) is left `0600`, and so is the backup. ⚠️ Responses carry server NAMES only, never env values, headers or file text: a parse failure is reported by line and column (`describeMcpSyncError`), never the parser's own message (smol-toml and V8 both quote source). ⚠️ `mcpConfig.relocation` names the env var the CLI reads to move its file (`CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `XDG_CONFIG_HOME`, `GEMINI_CLI_HOME`, and `COPILOT_HOME` for Copilot), resolved from the SERVER env at call time; a relative value reports the target `skipped`, never a guessed write, and a per-session `envOverrides` relocation is not followed. Tests must pass `home` (which drops the `process.env` default) or clear those vars first. → `docs/cli-registry.md` (MCP server sync), `docs/api-reference.md`, `docs/wiki/Settings-Reference.md` **Auto-named sessions** (`autoNameSessions`, SYNCED, default OFF): a placeholder tab (`w3-myapp`) takes its first real prompt as a title in the `: ` form, so the case identity and `w<n>` counter survive. Ownership is `SessionState.nameSource` (`placeholder` | `auto` | `manual`; the `name` setter / `PUT /api/sessions/:id/name` makes it `manual`, never touched again). ⚠️ `applyAutoName()` flips to `auto` even if the string is unchanged, so only the FIRST titled prompt names the tab. ⚠️ Only user input counts: `SessionWriteOptions.fromUser` is set by the browser WS path, `POST /api/sessions/:id/input`, the experimental guarded stage (`stageGuardedInput()`, `POST /api/sessions/:id/stage`, typed on an agent's behalf like the input route) and a Discord reply (`DiscordBridge.onGatewayMessage`, through the bridge's `sendInput`) ONLY; any new user-input path must set it (and the send-key Shift+Enter path must call `trackUserInput()`). ⚠️ The pure tracker (`session-auto-name.ts`) sits on the raw keystroke stream with an explicit rule per key; add a rule for any new key class. ⚠️ `nameSource` also decides `--name`: by default only a `manual` name is pinned on the claude CLI (`Session.cliPinnedName`), since `--name` is also the `/resume` title and a pinned one stops Claude generating its own; the opt-in synced `pinCliSessionName` (default OFF, re-read at every spawn and respawn) pins every tab name, after which a Codeman rename holds only until the next turn; a rename appends a `custom-title` row to a LOCAL, non-docker transcript, and a same-name PUT is a no-op (never flips to `manual`). Tests: `test/session-auto-name.test.ts`. → [architecture-invariants#auto-named-sessions-first-prompt--tab-title](docs/architecture-invariants.md#auto-named-sessions-first-prompt--tab-title) diff --git a/docs/api-reference.md b/docs/api-reference.md index 756631ff..b0464ac8 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1680,6 +1680,8 @@ Only replies from `allowedUserIds`, in `guildId`, inside a thread the bridge cre `GET /api/local-model` looks at the machine the server runs on and says which local coding model fits it, and with which agent CLI. It covers Apple silicon macOS only; anywhere else the answer is still a `200`, with verdict `none`. **Read-only:** the route downloads nothing, installs nothing, writes no file and reads or changes no setting (setting a model up from Codeman is a later step and is not built yet). It names the host's hardware, so it is admin only in multi-user mode (`403`, answered before anything is probed). The models, their sizes and the agent CLIs it considers are a catalogue compiled into the release; there is no override file. +**Experimental.** This endpoint and its payload are labeled experimental until the step that sets a model up ships: that step will add to and may change the fields below, the verdicts and the sentences, in a MINOR or PATCH release (see [versioning-policy.md](versioning-policy.md), "What SemVer does NOT cover", item 3). Do not build on its shape yet. + | Method | Path | Body | Notes | | ------ | ------------------ | ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `GET` | `/api/local-model` | none | `200` with `{ hardware, recommendation, catalogUpdated }`. `403 FORBIDDEN` for a non-admin in multi-user mode. `500 INTERNAL_ERROR` only when the recommendation itself fails, never for a failed probe. | @@ -1690,24 +1692,24 @@ A probe that fails is an answer, not an error: a server that is not on macOS, or - `hardware`: what the server sees of the machine it runs on. - `platform`, `arch`: `process.platform` and `os.arch()` of the server process. `arm64: true` means the machine is Apple silicon whatever `arch` says; `translated: true` means the server process runs under Rosetta. - - `memBytes`, `memGb` (decimal GB text) and `memClassGb` (the memory size the Mac is sold as: 16, 32, 48). + - `memBytes`, `memGb` (the bytes as decimal GB text, `"51.5"` for a 48 GB Mac) and `memClassGb` (the memory size the Mac is sold as: 16, 32, 48; `0` when the size could not be read). `memClassGb` is what to show as the installed memory: it is the number on the box and in About This Mac. It is the one size here that is not decimal GB. - `chip`, `model`, `macosVersion`, `macosMajor`, `wiredLimitMb` (`0` means not set), `freeDiskBytes` and `freeDiskGb` (free space on the filesystem of Codeman's data directory). Each is absent when it could not be read. - - `source`: `sysctl` (the one bounded `sysctl` call answered), `os-fallback` (macOS, but that call failed, so only memory, architecture and the chip name are known) or `not-probed` (not macOS, nothing was run). + - `source`: `sysctl` (the one bounded `sysctl` call answered), `os-fallback` (macOS, but that call failed: memory, architecture, the chip name and the free disk space come from Node, and nothing only `sysctl` knows, such as `arm64`, `translated`, `model`, `macosVersion`, `macosMajor` or `wiredLimitMb`, is present) or `not-probed` (not macOS, nothing was run). - `recommendation`: - - `verdict`: `ok`, `limited` (the best model that fits has fewer than 12 billion parameters: it handles small edits and questions and does not replace a cloud model) or `none`. - - `reason`, present only with `none`: `not-macos`, `intel`, `memory`, `disk`, `macos` (too old) or `no-harness` (no agent CLI that can be connected to a local model is installed or installable). - - `headline` and `notes[]`: `{ text, vars? }`. `text` is an English sentence that may hold `{placeholder}` slots, and `vars` carries the slot values as text. - - `pick` (absent with `none`) and `alternatives[]` (at most four, best first; empty with `none`), each one model at one context length with one agent CLI: + - `verdict`: `ok`, `limited` or `none`. `limited` is a statement about the Mac: no model with 12 billion parameters or more fits its memory, so what it can run handles small edits and questions and does not replace a cloud model. It is judged on every model that fits, before the free disk space is looked at and whatever the pick is. With `ok` the pick can still be a small model (a larger one may fit only tightly, have no speed figure or have no room on the disk); `notes` then says so. + - `reason`, present only with `none`: `not-macos`, `intel`, `memory`, `disk`, `macos` (too old) or `no-harness` (no agent CLI that can be connected to a local model is installed or installable). `memory` has three sentences: nothing fits at all (slot `memory`, the size the Mac is sold as), a model fits but only below the context length an agent CLI needs (slots `budget` and `need`, so a Mac that misses by a little reads as that), and the memory size could not be read (no slot). + - `headline` and `notes[]`: `{ text, vars? }`. `text` is an English sentence that may hold `{placeholder}` slots, and `vars` carries the slot values as text. Notes, in order: the pick is a small model although larger ones fit (verdict `ok` only); a model that fits the memory was left out because its download would not leave 10 GiB free on the disk (slots `model` and `missing`: the best-ranked such model and the GB to free up); nobody tested the pick on a Mac like this one. + - `pick` (absent with `none`) and `alternatives[]` (at most four, best first; empty with `none`), each one model at one context length with one agent CLI. A model whose download would leave less than 10 GiB free on the disk is not listed (an unknown free disk space excludes nothing): - `modelId` (`name:tag`), `modelLabel`, `license`, `paramsB` (parameters in billions), `context` (tokens); - `cli`, `cliLabel`, `cliInstalled` (`false`: the CLI is not on this machine yet but can be installed without administrator rights), `slowFirstTurn?`; - `needBytes` and `needGb` (memory the model needs at that context), `downloadBytes` and `downloadGb`, `budgetShare` (need divided by `budgetBytes`, rounded to three decimals); - `fit`: `good` (up to 0.85 of the budget) or `tight` (up to 0.98), with its label `fitLabel`; - - `speed`: `ok`, `slow` or `unknown`, with `speedLabel` (absent for `ok`) and `tokensPerSecond` (absent for `unknown`). `unknown` means the chip is in no row of the catalogue's chip table, or the catalogue has no speed figure for the model: a speed is never guessed; + - `speed`: `ok`, `slow` (under 10 tokens per second) or `unknown`, with `speedLabel` (absent for `ok`) and `tokensPerSecond` (rounded to one decimal, absent for `unknown`; `speed` is decided on the unrounded figure, so `10` can be `slow`). `unknown` means the chip is in no row of the catalogue's chip table, or the catalogue has no speed figure for the model: a speed is never guessed; - `tested`: we ran this model with this CLI on a Mac like this one. No catalogue row has such an entry yet, so it is `false` today. - `budgetBytes` and `budgetGb`: the memory a model may use on this Mac (`0` for `not-macos` and `intel`, where none is computed). `budgetSource` is `wired-limit` (taken from `iogpu.wired_limit_mb`) or `estimated` (computed from the installed memory). - `catalogUpdated`: ISO date on which the catalogue's rows were last checked. -Every size a person reads travels twice: as bytes, and as decimal GB text without the unit (`"needGb": "29.6"`). Show the text and add the unit; do not divide the bytes yourself, so one request never shows two different roundings. +Every computed size a person reads travels twice: as bytes, and as decimal GB text without the unit (`"needGb": "29.6"`). Show the text and add the unit; do not divide the bytes yourself, so one request never shows two different roundings. The installed memory is the exception named above: show `memClassGb`. ## Voice dictation diff --git a/docs/versioning-policy.md b/docs/versioning-policy.md index d049ae5f..ed6dd809 100644 --- a/docs/versioning-policy.md +++ b/docs/versioning-policy.md @@ -86,6 +86,19 @@ covered like any other stable behavior. Other agent commands and API routes keep their existing guarantees. Promotion to stable requires an explicit documented decision. +### Experimental local model recommendation + +`GET /api/v1/local-model` (and its `/api` alias) is labeled experimental, so item 3 +above applies to it: its path, its payload (the hardware facts, the verdicts and +reasons, the candidate fields and the English sentences in `headline` and +`notes`) and its behavior may change or be removed in a MINOR or PATCH release. +It is read-only today: it looks at the Mac the server runs on and says which +local model fits. The step that sets a model up from Codeman is not built yet +and will change what the route reports, which is why the shape is not promised. +It is not behind a setting (it writes nothing and adds no settings key); it is +admin only in multi-user mode. Promotion to stable requires an explicit +documented decision. + ## Deprecation policy When we need to change a covered surface: diff --git a/docs/wiki/Settings-Reference.md b/docs/wiki/Settings-Reference.md index 81ac6650..0ffcdf52 100644 --- a/docs/wiki/Settings-Reference.md +++ b/docs/wiki/Settings-Reference.md @@ -193,6 +193,20 @@ attached. section to the Run dropdown, for pointing a harness at your own OpenAI-compatible server instead of its native cloud backend. See [Custom Model Endpoints](Custom-Model-Endpoints). +**Model on this Mac** is not a setting: it is a read-only answer. When the Codeman server +runs on a Mac with Apple silicon, it shows that Mac's chip, memory and free disk space, the +local coding model that fits it best with the agent CLI to run it, what the model needs of +the memory the Mac can give one, its download size and an estimated speed, and the other +models that fit (Good or Tight for the memory, Slow or Speed unknown for the speed). A Mac +that can only run a model under 12 billion parameters is told so in plain words, and a Mac +with too little memory, too little free disk space or an Intel processor is told why nothing +is offered. The numbers are estimates from a catalogue that ships with each release; nothing +has been tested on your Mac. **Nothing is downloaded or installed from here yet**: setting +the model up from Codeman is the next step and is not built. The same answer appears as a +card on the welcome screen, which you can hide per device (it stays here). The group is not +shown when the server runs on Linux or Windows, and in multi-user mode only an admin sees +it. The "Mac" is always the machine the server runs on, not the device you are reading on. + ### Agents & CLIs | Setting | Notes | diff --git a/src/web/CLAUDE.md b/src/web/CLAUDE.md index 409274d5..2f29a211 100644 --- a/src/web/CLAUDE.md +++ b/src/web/CLAUDE.md @@ -48,6 +48,8 @@ Frontend JS modules have `@fileoverview` with `@dependency`/`@loadorder` tags. L **Opened outside Codeman** (`foreign-sessions.js`, both home screens): the tmux sessions a human started, listed from `GET /api/mux/foreign` (root CLAUDE.md, Adopted tmux sessions). ONE renderer, `renderForeignSessions(container)`, fills the welcome block and the phone overview's, and every row goes through `textContent`, since a session name and a command line come from someone else's process. ⚠️ It polls only while a home screen is up (`startForeignPolling`/`stopForeignPolling` from `showWelcome()`/`hideWelcome()`) and the tab is visible (`_foreignPollTick()` is the one gate, and `visibilitychange` catches up; the cache TTL sits under the poll interval, so one tab's every tick is a full socket sweep), and the poll is local only: the docker and ssh scan costs one connection per target, so it runs once on the toggle and again on Rescan, and its rows and notes are kept apart (`_foreignWideSessions`, `_foreignWideNotes`) or the next local poll would wipe them. ⚠️ A 403 is a non-admin in multi-user mode, an answer and not a failed poll: `_foreignRefused` stops the polling and shows no block. ⚠️ `closeSession()` re-reads the list after an adopted tab's DELETE, since the home screen's own load races it and the row would say `Go to tab` until the next poll. ⚠️ An adopted tab has no kill outcome to offer, so the close dialog hides the red button and always tears the wrapper down. ⚠️ `session.remote` and `session.docker` are undefined on an adopted session, so a check that hides a row for those two usually needs `session.adopt` as well. Tests: `test/foreign-sessions-ui.test.ts`, `test/foreign-sessions-i18n.test.ts` (harvests every string from the real module). +**Model on this Mac** (`local-model-ui.js`; welcome card `#localModelCard` + App Settings > Models group `#localModelGroup`; server rules in the root CLAUDE.md, Local model recommendation): a READ-ONLY view of `GET /api/local-model`. ONE builder (`localModelBody`) paints both containers, so the card and the Settings row can never name a different pick; the card folds the alternatives, Settings lists them. ⚠️ Every dynamic string goes in through `textContent` (model, harness and chip names are someone else's text), both containers carry `data-i18n-skip`, and the module translates its own sentences at render time, so a language or display-name change must repaint from the module (the `<html lang>` observer). ⚠️ Labels and sentences come from the payload (`{ text, vars }` through `localModelT`); the module holds no list of models or harnesses, never compares a CLI id, and never contains the word "Recommended" (that headline is the server's to send). ⚠️ `test/local-model-i18n.test.ts` harvests every `localModelT('...')` literal: a new string is a single-quoted literal with no apostrophe and gets a zh-CN entry, and the only non-literal calls are the two payload ones. ⚠️ Sizes are printed as sent (decimal GB text), bytes are never divided here; the installed memory alone is `memClassGb`, a number or nothing (an unknown size is left out, never `0 GB`). ⚠️ The card ships `hidden` and shows only once the server answered for a Mac; the card AND the Settings group stay hidden for `not-macos` (the server is usually Linux, and "this Mac" always means the machine the server runs on), for a refusal and while the first request is in flight; a failed first request shows one sentence in Settings only. ⚠️ Refused means `403` with `errorCode: FORBIDDEN` (a non-admin in multi-user mode: stop asking); the auth layer's `403 PASSWORD_CHANGE_REQUIRED` is a failed request that is asked again on `codeman:me`. ⚠️ Each fit and speed pill carries what it measures as `title` + `aria-label` ("Good" is the memory fit, not a rating), and the line saying Codeman cannot download or install yet sits right under the pick and its notes, in `--text-dim` like every other line. The dismissal is a per-device localStorage key named for this stage (`codeman:local-model-dismissed:readonly`), never a server setting: the stage that can set a model up takes a new key so its offer shows once more. At most one request a minute (the server caches its probe for 60 s), shared while in flight, and a failed refresh keeps the answer on screen. Tests: `test/local-model-ui.test.ts`, `test/local-model-i18n.test.ts`. + **Home-screen session order** (`CodemanSessionOrder` in constants.js, pure): BOTH home screens (phone overview, desktop rail) must order rows through this ONE comparator. Rank `needs` → `error` → `waiting` → `working` → `idle` → `done`. ⚠️ The tiebreak flips: states a session is still IN sort oldest-first, states it has STOPPED sort newest-first. ⚠️ The running group keys off `lastSubmitAt`, never `lastActivityAt` (a working pane repaints constantly). ⚠️ A 0 stamp means unknown and sorts last within its state. Final tiebreak is `orderIndex`, so the list never shuffles. The tab strip itself is NOT sorted by this. Test: `test/session-overview-order.test.ts`. → [architecture-invariants#home-screen-session-order](../../docs/architecture-invariants.md#home-screen-session-order) **Welcome "Resume Conversation" list** (terminal-ui.js): `loadHistorySessions()` fetches once and caches the corpus on `_historyAll`/`_historyCases`; every subsequent view (filter box, sort select, expand, the periodic refresh in panels-ui.js) goes through `_renderHistoryList()`, so never append rows to `#historyList` directly or re-fetch to re-sort. ⚠️ The box height is **class-driven**: expanding the list without `.history-list.expanded` leaves the collapsed `max-height` in place and just deepens a scroll well, which is the bug #260 reported (35 sessions in a ~4-row box). ⚠️ The A–Z sort keys off `_historyRowLabel()`, the SAME string the row renders (`name || firstPrompt || path`), most rows are transcript-backed and have no session name, so sorting on `name` alone silently does nothing. ⚠️ A filter implies expansion, and `_renderSearch()` hides `#historyHeader` (title + controls) as one unit while a search is active. Tests: `test/history-list-controls.test.ts`. diff --git a/src/web/routes/local-model-routes.ts b/src/web/routes/local-model-routes.ts index f4f0346b..12b3cc4a 100644 --- a/src/web/routes/local-model-routes.ts +++ b/src/web/routes/local-model-routes.ts @@ -6,6 +6,11 @@ * `{ hardware, recommendation, catalogUpdated }`. Nothing is reformatted here: every size text * and every sentence in the payload was chosen by the modules that produced it. * + * EXPERIMENTAL (docs/versioning-policy.md, "Experimental local model recommendation"): the + * route and its payload are outside the stable `/api/v1` contract until the stage that sets a + * model up ships, because that stage will change what the fields mean. docs/api-reference.md + * says so in its section; keep the two in step, and drop both labels together. + * * Rules: * - Read-only. It downloads nothing, installs nothing, writes no file and reads no setting. * - A failed probe is an answer, never a 500. `readHardware()` reports a spawn that failed