diff --git a/src/local-model/harness-facts.ts b/src/local-model/harness-facts.ts index 7a27d130..b86403e4 100644 --- a/src/local-model/harness-facts.ts +++ b/src/local-model/harness-facts.ts @@ -3,28 +3,163 @@ * local model recommendation. Server side: this is the one module of src/local-model/ that * reads the CLI registry, so `recommend()` can stay pure and name no CLI. * - * SKELETON: the exported signature is final, the body is not written yet. `buildHarnessFacts()` - * will, for every `LocalHarnessEntry` of the catalogue: - * - resolve its `cli` in the CLI registry AT CALL TIME (never in a module-level constant), and - * leave out a row whose CLI is unknown, disabled, or has no custom model mechanism - * (`capabilities.customModelInjection.kind` is `unsupported`); - * - report `label` from the registry entry and `installed` from the same probe the other - * routes use; - * - report `installable` only for a CLI that is not installed and whose install command for - * this platform needs no administrator rights (not one that starts with `brew` or `sudo`); - * - supply `acceptsModelId`, the same test a custom model launch applies to the model id - * before anything is written (the launch spec's model pattern). + * `buildHarnessFacts()` walks the catalogue's `harnesses` rows and, for every row: + * - resolves its `cli` in the CLI registry AT CALL TIME (never in a module-level constant), + * and leaves the row out when the CLI is unknown, disabled, or has no custom model + * mechanism (`capabilities.customModelInjection.kind` is `unsupported`); + * - reports `label` from the registry entry and `installed` from the same probe the page + * render and `GET /api/clis` use (`utils/cli-installed-probes.ts`), asked once per build; + * - reports `installable` only for a STOCK CLI that is not installed and whose install + * command for this platform needs no administrator rights (`installCommandNeedsAdmin()`). + * Stock only, because the install runner never executes a custom entry's command; + * - supplies `acceptsModelId` (`cliAcceptsCustomModelId()`), the test a custom model launch + * applies to the model id before anything is written. * * Rules: - * - No CLI id is compared with a literal here or anywhere: the id comes from the catalogue row - * and everything else from the registry entry it resolves to. - * - Read-only: nothing is installed, written or spawned beyond the existing availability probe. + * - No CLI id is compared with a literal here or anywhere: the id comes from the catalogue + * row and everything else from the registry entry it resolves to. + * - Read-only: nothing is installed, written or spawned beyond the existing availability + * probe. An install command is only read as text, never run. + * - Never throws: a probe that fails reads as "not installed" (with a warning in the log), + * and the registry's install data still answers `installable`. + * + * `cliAcceptsCustomModelId()` composes the two pure pieces the launch path itself calls + * (`buildCustomModelInjection()` and `matchesPattern()` on the entry's `model` launch param). + * The launch path keeps that composition inline at two sites in + * src/web/routes/session-routes.ts (the quick-start `customModel` branch and + * `POST /api/sessions/:id/custom-model`). A change to the rule there must be made here too, + * until the three share one helper. */ -import type { HarnessFact, LocalModelCatalog } from '../types.js'; +import type { CliEntry } from '../config/cli-registry/types.js'; +import { matchesPattern } from '../config/cli-registry/patterns.js'; +import { getCli, resolveInstallCommandForPlatform } from '../config/cli-registry/registry.js'; +import { buildCustomModelInjection, type CustomModelEndpoint } from '../custom-model-injection.js'; +import { isCliEntryInstalled, probeStockCliAvailability } from '../utils/cli-installed-probes.js'; +import { getErrorMessage, type HarnessFact, type LocalModelCatalog } from '../types.js'; + +/** What `buildHarnessFacts()` reads. Each member defaults to the real module; a test overrides what it needs. */ +export interface HarnessFactsDeps { + /** One CLI of the registry, looked up when the facts are built. */ + getCli: (id: string) => CliEntry | undefined; + /** + * The availability probe, asked ONCE per build: it answers with a test for single entries. + * Under vitest the real probe is inert (it finds no CLI), so a test that cares about + * `installed` passes its own. + */ + probeInstalled: () => Promise<(entry: CliEntry) => boolean>; + /** The install command the registry holds for this platform. Read as text, never run. */ + installCommandFor: (entry: CliEntry) => string | undefined; +} + +export const defaultHarnessFactsDeps: HarnessFactsDeps = { + getCli, + probeInstalled: async () => { + const stockAvailability = await probeStockCliAvailability(); + return (entry) => isCliEntryInstalled(entry, stockAvailability); + }, + installCommandFor: resolveInstallCommandForPlatform, +}; + +/** + * A command segment that starts with `sudo` or `brew`: at the start of the text, or after a + * pipe, `&&`, `||`, `;`, a line break or an opening parenthesis. + */ +const ADMIN_COMMAND = /(?:^|[;&|(\n])\s*(?:sudo|brew)(?=\s|$)/; + +/** + * Whether an install command needs administrator rights, read off its text. + * - `sudo ...` asks for an administrator password. + * - `brew ...` needs Homebrew, and installing Homebrew needs an administrator (the same rule + * the installer's no-admin route applies to the CLI menu). + * `curl ... | bash` installers write into the home directory, and the install runner points an + * `npm install -g` at a prefix the server user can write, so neither counts. + */ +export function installCommandNeedsAdmin(command: string): boolean { + return ADMIN_COMMAND.test(command); +} + +/** Stands in for a configured endpoint: the launch id a recipe forces depends on the model id alone. */ +const PROBE_ENDPOINT: CustomModelEndpoint = { + id: 'local-model-probe', + label: 'local model probe', + baseUrl: 'http://127.0.0.1', +}; + +/** + * Whether a custom model launch of this CLI would accept `modelId`. + * + * A CLI whose config alone cannot select the model gets the id forced onto its `model` launch + * param (the recipe's `launchModel` template). The argv engine DROPS a token that fails the + * param's pattern, which would start the CLI on its own default provider, so the launch path + * refuses such an id up front. This is that test: false when the forced value does not match + * the pattern, and false for a CLI with no custom model mechanism at all. + */ +export function cliAcceptsCustomModelId(entry: Pick, modelId: string): boolean { + try { + const injection = buildCustomModelInjection(entry, PROBE_ENDPOINT, modelId); + if (injection.kind === 'unsupported') return false; + const forcedLaunchModel = injection.launchModel; + const modelSpec = entry.launch.params.model; + return !( + forcedLaunchModel !== undefined && + modelSpec?.type === 'token' && + !matchesPattern(modelSpec.pattern, forcedLaunchModel) + ); + } catch { + // A recipe that cannot be rendered cannot be launched either. + return false; + } +} /** One fact per catalogue harness that resolves in the CLI registry, in catalogue order. */ -export async function buildHarnessFacts(catalog: LocalModelCatalog): Promise { - void catalog; - throw new Error('not implemented: buildHarnessFacts'); +export async function buildHarnessFacts( + catalog: LocalModelCatalog, + deps: Partial = {} +): Promise { + const resolved: HarnessFactsDeps = { ...defaultHarnessFactsDeps, ...deps }; + + let isInstalled: (entry: CliEntry) => boolean; + try { + isInstalled = await resolved.probeInstalled(); + } catch (err) { + console.warn( + `[local-model] The CLI availability probe failed, reading every harness as not installed: ${getErrorMessage(err)}` + ); + isInstalled = () => false; + } + + const facts: HarnessFact[] = []; + const seen = new Set(); + for (const row of catalog.harnesses) { + // The catalogue lists a CLI once; a second row for it adds nothing a fact could carry. + if (seen.has(row.cli)) continue; + seen.add(row.cli); + + const entry = resolved.getCli(row.cli); + if (!entry || !entry.enabled) continue; + if (entry.capabilities.customModelInjection.kind === 'unsupported') continue; + + let installed = false; + try { + installed = isInstalled(entry); + } catch (err) { + console.warn(`[local-model] Probing ${entry.label} failed, reading it as not installed: ${getErrorMessage(err)}`); + } + + let installable = false; + if (!installed && entry.stock) { + const command = resolved.installCommandFor(entry); + installable = command !== undefined && command.trim() !== '' && !installCommandNeedsAdmin(command); + } + + facts.push({ + cli: row.cli, + label: entry.label, + installed, + installable, + acceptsModelId: (modelId: string) => cliAcceptsCustomModelId(entry, modelId), + }); + } + return facts; } diff --git a/src/web/routes/local-model-routes.ts b/src/web/routes/local-model-routes.ts index 4c4f7563..f4f0346b 100644 --- a/src/web/routes/local-model-routes.ts +++ b/src/web/routes/local-model-routes.ts @@ -1,25 +1,33 @@ /** * @fileoverview `GET /api/local-model`: what this Mac is and which local model fits it. * - * SKELETON: the route is registered and answers 501 until its handler is written. It will - * answer `{ success: true, data: LocalModelStatus }`, built from the hardware probe, the + * Answers `{ success: true, data: LocalModelStatus }`, built from the hardware probe, the * catalogue, the harness facts and the pure `recommend()`: - * `{ hardware, recommendation, catalogUpdated }`. + * `{ hardware, recommendation, catalogUpdated }`. Nothing is reformatted here: every size text + * and every sentence in the payload was chosen by the modules that produced it. * * Rules: * - Read-only. It downloads nothing, installs nothing, writes no file and reads no setting. - * - Never a 500 for a probe that failed: a failed probe is an answer (the hardware's `source` - * says so and the recommendation is verdict `none`). + * - A failed probe is an answer, never a 500. `readHardware()` reports a spawn that failed + * in the hardware's `source` (`os-fallback`), and a server that is not on macOS as + * `not-probed`; both go out as a 200 whose recommendation is verdict `none` or computed + * from what is known. Should the probe itself reject (it promises not to), the route logs + * it and answers with what `node:os` knows (`hardwareFromOs()`), under the same two + * `source` values. + * - A 500 is left for a fault that is not a probe: the catalogue, the harness facts or + * `recommend()` throwing. That is a bug, and it must not be dressed up as "no model fits". * - The payload names hardware of the host, so in multi-user mode it is admin only (403), the - * same bar as `GET /api/doctor`. + * same bar as `GET /api/doctor`. The gate runs before anything is probed. * - Every dependency is injected (`LocalModelRouteDeps`), so a route test spawns nothing and * does not need a Mac: it passes its own hardware and harness facts. */ +import { arch, totalmem } from 'node:os'; import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify'; import { ApiErrorCode, createErrorResponse, + getErrorMessage, type ApiResponse, type HardwareFacts, type HarnessFact, @@ -27,7 +35,10 @@ import { type LocalModelStatus, type Recommendation, } from '../../types.js'; +import { isAdmin } from '../route-helpers.js'; +import { isMultiUserMode } from '../../config/multiuser.js'; import { getLocalModelCatalog } from '../../config/local-model-catalog.js'; +import { formatGb, memClassGb } from '../../local-model/format.js'; import { readHardware } from '../../local-model/hardware.js'; import { buildHarnessFacts } from '../../local-model/harness-facts.js'; import { recommend } from '../../local-model/recommend.js'; @@ -43,18 +54,57 @@ export interface LocalModelRouteDeps { export const defaultLocalModelRouteDeps: LocalModelRouteDeps = { readHardware: () => readHardware(), getCatalog: getLocalModelCatalog, - buildHarnessFacts, + buildHarnessFacts: (catalog) => buildHarnessFacts(catalog), recommend, }; +/** + * The hardware answer when the probe rejected: memory and architecture from `node:os`, and the + * `source` the probe itself uses for "nothing was read from sysctl" (`os-fallback` on macOS, + * `not-probed` anywhere else). Sizes go through the one formatter, like every other size. + */ +export function hardwareFromOs(): HardwareFacts { + const memBytes = totalmem(); + return { + platform: process.platform, + arch: arch(), + memBytes, + memGb: formatGb(memBytes), + memClassGb: memClassGb(memBytes), + source: process.platform === 'darwin' ? 'os-fallback' : 'not-probed', + }; +} + export function registerLocalModelRoutes(app: FastifyInstance, deps: Partial = {}): void { const resolved: LocalModelRouteDeps = { ...defaultLocalModelRouteDeps, ...deps }; - void resolved; app.get( '/api/local-model', - async (_req: FastifyRequest, reply: FastifyReply): Promise> => { - reply.code(501); - return createErrorResponse(ApiErrorCode.OPERATION_FAILED, 'GET /api/local-model is not implemented yet'); + async (req: FastifyRequest, reply: FastifyReply): Promise> => { + if (isMultiUserMode() && !isAdmin(req)) { + reply.code(403); + return createErrorResponse(ApiErrorCode.FORBIDDEN, 'Admin only in multi-user mode'); + } + + let hardware: HardwareFacts; + try { + hardware = await resolved.readHardware(); + } catch (err) { + console.warn(`[local-model] The hardware probe failed, answering from node:os: ${getErrorMessage(err)}`); + hardware = hardwareFromOs(); + } + + try { + const catalog = resolved.getCatalog(); + const harnessFacts = await resolved.buildHarnessFacts(catalog); + const recommendation = resolved.recommend(hardware, catalog, harnessFacts); + return { success: true, data: { hardware, recommendation, catalogUpdated: catalog.updated } }; + } catch (err) { + reply.code(500); + return createErrorResponse( + ApiErrorCode.INTERNAL_ERROR, + `The local model recommendation failed: ${getErrorMessage(err)}` + ); + } } ); } diff --git a/test/local-model-harness-facts.test.ts b/test/local-model-harness-facts.test.ts new file mode 100644 index 00000000..7c0409b7 --- /dev/null +++ b/test/local-model-harness-facts.test.ts @@ -0,0 +1,306 @@ +/** + * @fileoverview `buildHarnessFacts()`: what the server knows about each catalogue harness on + * this machine (src/local-model/harness-facts.ts), read from the CLI registry at call time. + * + * The catalogue here is the test's own: the stock one is filled by another module and its rows + * are not what this file is about. The registry is the REAL stock registry unless a test says + * otherwise, so the model id anchors below are regression anchors on shipped data. + * + * Two things a reader should know before trusting a green run: + * - Under vitest the real availability probe is inert (it finds no CLI). A test that cares + * about `installed` therefore injects `probeInstalled`. + * - The registry's install command resolver reads `process.platform` and falls back to the + * Linux command, so on a Linux machine it never returns a macOS command. The macOS cases + * inject `installCommandFor` and read the entry's own `darwin` command. + * + * Port: none (no server; the on-disk case uses the per-file temp HOME from test/setup.ts). + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { + buildHarnessFacts, + cliAcceptsCustomModelId, + installCommandNeedsAdmin, +} from '../src/local-model/harness-facts.js'; +import { getCli, registryFilePath, reloadCliRegistry } from '../src/config/cli-registry/registry.js'; +import type { CliEntry } from '../src/config/cli-registry/types.js'; +import type { LocalModelCatalog } from '../src/types.js'; + +/** A catalogue whose harness rows are `clis`, in that order. Nothing else in it matters here. */ +function catalogWith(clis: string[]): LocalModelCatalog { + return { + schema: 1, + updated: '2026-11-06', + minMacosMajor: 14, + promptCacheTiers: [{ minMemGb: 0, bytes: 512 * 1024 * 1024 }], + models: [], + harnesses: clis.map((cli, index) => ({ cli, minContext: 32768, rank: clis.length - index })), + chips: [], + }; +} + +function stockEntry(id: string): CliEntry { + const entry = getCli(id); + if (!entry) throw new Error(`${id} is missing from the CLI registry`); + return entry; +} + +/** The command the registry holds for macOS, whatever machine the test runs on. */ +const darwinCommand = (entry: CliEntry): string | undefined => entry.discovery.install.command.darwin; + +/** A probe that reports exactly these CLIs as installed. */ +const installedOnly = + (...ids: string[]) => + async () => + (entry: CliEntry) => + ids.includes(entry.id as string); + +afterEach(() => { + const path = registryFilePath(); + if (existsSync(path)) rmSync(path); + reloadCliRegistry(); + vi.restoreAllMocks(); +}); + +describe('installCommandNeedsAdmin', () => { + it.each([ + ['brew install can1357/tap/omp', true], + [' brew install something', true], + ['sudo apt install something', true], + ['curl -fsSL https://example.com/install.sh | sudo bash', true], + ['npm install -g thing && sudo ln -s a b', true], + ['echo first; brew install thing', true], + ['echo first\nsudo make install', true], + ['(brew install thing)', true], + ['curl -fsSL https://opencode.ai/install | bash', false], + ['curl -fsSL https://claude.ai/install.sh | bash', false], + ['npm install -g --ignore-scripts @earendil-works/pi-coding-agent', false], + ['npm install -g @deepseek-ai/dsh && (command -v pnpm >/dev/null || npm install -g pnpm)', false], + // The words inside a URL, a package name or an argument are not a command. + ['curl -fsSL https://brew.sh/install.sh | bash', false], + ['npm install -g homebrew-sudo-helper', false], + ['echo sudo', false], + ])('%s -> %s', (command, expected) => { + expect(installCommandNeedsAdmin(command)).toBe(expected); + }); + + it('agrees with the registry on every stock macOS command: only a Homebrew one needs an administrator', () => { + expect(installCommandNeedsAdmin(darwinCommand(stockEntry('omp')) ?? '')).toBe(true); + for (const id of ['claude', 'opencode', 'codex', 'pi']) { + expect(installCommandNeedsAdmin(darwinCommand(stockEntry(id)) ?? '')).toBe(false); + } + }); +}); + +describe('cliAcceptsCustomModelId (the launch path’s model id test, on the stock registry)', () => { + it('accepts a registry style `name:tag` id for the CLIs that can carry a colon', () => { + // opencode and claude select the model in their config or environment: no launch id at all. + // pi and codex force the id onto `--model`, and both of their patterns allow a colon. + for (const id of ['opencode', 'claude', 'pi', 'codex']) { + expect(cliAcceptsCustomModelId(stockEntry(id), 'qwen3.5:9b')).toBe(true); + } + }); + + it('refuses a colon for a CLI whose `--model` pattern has none, and accepts the same CLI without it', () => { + const omp = stockEntry('omp'); + expect(cliAcceptsCustomModelId(omp, 'qwen3.5:9b')).toBe(false); + expect(cliAcceptsCustomModelId(omp, 'qwen3-coder')).toBe(true); + }); + + it('refuses the ids the launch routes’ own tests refuse', () => { + // test/routes/quick-start-custom-model.test.ts pins both pairs against the real route. + expect(cliAcceptsCustomModelId(stockEntry('pi'), 'qwen 3 with spaces')).toBe(false); + expect(cliAcceptsCustomModelId(stockEntry('codex'), 'org/model@q4_k_m')).toBe(false); + }); + + it('is false for a CLI with no custom model mechanism', () => { + expect(cliAcceptsCustomModelId(stockEntry('shell'), 'qwen3.5:9b')).toBe(false); + expect(cliAcceptsCustomModelId(stockEntry('antigravity'), 'qwen3.5:9b')).toBe(false); + }); + + it('reads the entry it is given, so a changed pattern changes the answer (no id is looked at)', () => { + const omp = stockEntry('omp'); + const widened: CliEntry = { + ...omp, + launch: { ...omp.launch, params: { ...omp.launch.params, model: { type: 'token', pattern: 'model-pi' } } }, + }; + expect(cliAcceptsCustomModelId(widened, 'qwen3.5:9b')).toBe(true); + }); +}); + +describe('buildHarnessFacts', () => { + it('answers one fact per catalogue harness, in catalogue order, with the registry’s label', async () => { + const facts = await buildHarnessFacts(catalogWith(['pi', 'opencode', 'claude']), { + probeInstalled: installedOnly('opencode'), + installCommandFor: darwinCommand, + }); + expect(facts.map((f) => f.cli)).toEqual(['pi', 'opencode', 'claude']); + expect(facts.map((f) => f.label)).toEqual(['pi', 'opencode', 'claude'].map((id) => stockEntry(id).label)); + expect(facts.map((f) => f.installed)).toEqual([false, true, false]); + }); + + it('leaves out a CLI the registry does not know, one with no custom model mechanism, and a disabled one', async () => { + const disabled = (id: string): CliEntry | undefined => { + const entry = getCli(id); + return entry && id === 'claude' ? { ...entry, enabled: false } : entry; + }; + const facts = await buildHarnessFacts(catalogWith(['no-such-cli', 'shell', 'antigravity', 'claude', 'opencode']), { + getCli: disabled, + probeInstalled: installedOnly(), + }); + expect(facts.map((f) => f.cli)).toEqual(['opencode']); + }); + + it('keeps one fact for a CLI the catalogue lists twice', async () => { + const facts = await buildHarnessFacts(catalogWith(['opencode', 'opencode']), { probeInstalled: installedOnly() }); + expect(facts.map((f) => f.cli)).toEqual(['opencode']); + }); + + it('installable: not installed, a stock CLI, and a macOS command that needs no administrator', async () => { + const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi', 'omp']), { + probeInstalled: installedOnly('claude'), + installCommandFor: darwinCommand, + }); + const byCli = new Map(facts.map((f) => [f.cli, f])); + // curl installer into the home directory. + expect(byCli.get('opencode')).toMatchObject({ installed: false, installable: true }); + // Already there: nothing to install. + expect(byCli.get('claude')).toMatchObject({ installed: true, installable: false }); + // npm: the install runner points it at a prefix the server user can write. + expect(byCli.get('pi')).toMatchObject({ installed: false, installable: true }); + // Homebrew on macOS. + expect(byCli.get('omp')).toMatchObject({ installed: false, installable: false }); + }); + + it('installable is false without an install command, for a sudo command, and for a custom entry', async () => { + const command = vi.fn<(entry: CliEntry) => string | undefined>(); + command.mockReturnValueOnce(undefined); + expect( + ( + await buildHarnessFacts(catalogWith(['opencode']), { + probeInstalled: installedOnly(), + installCommandFor: command, + }) + )[0].installable + ).toBe(false); + command.mockReturnValueOnce(' '); + expect( + ( + await buildHarnessFacts(catalogWith(['opencode']), { + probeInstalled: installedOnly(), + installCommandFor: command, + }) + )[0].installable + ).toBe(false); + command.mockReturnValueOnce('sudo installer -pkg agent.pkg -target /'); + expect( + ( + await buildHarnessFacts(catalogWith(['opencode']), { + probeInstalled: installedOnly(), + installCommandFor: command, + }) + )[0].installable + ).toBe(false); + + // A custom entry's install command is never run by the install runner, so it is never + // offered, however harmless the text looks. + const custom = (id: string): CliEntry | undefined => { + const entry = getCli(id); + return entry ? { ...entry, stock: false } : entry; + }; + const harmless = vi.fn(() => 'curl -fsSL https://example.com/install | bash'); + const facts = await buildHarnessFacts(catalogWith(['opencode']), { + getCli: custom, + probeInstalled: installedOnly(), + installCommandFor: harmless, + }); + expect(facts[0].installable).toBe(false); + expect(harmless).not.toHaveBeenCalled(); + }); + + it('never reads an install command for a CLI that is already installed', async () => { + const command = vi.fn(() => 'curl -fsSL https://example.com/install | bash'); + const facts = await buildHarnessFacts(catalogWith(['opencode']), { + probeInstalled: installedOnly('opencode'), + installCommandFor: command, + }); + expect(facts[0]).toMatchObject({ installed: true, installable: false }); + expect(command).not.toHaveBeenCalled(); + }); + + it('acceptsModelId is the launch path’s test for that harness', async () => { + const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi', 'omp']), { + probeInstalled: installedOnly(), + }); + const accepts = (cli: string, modelId: string) => facts.find((f) => f.cli === cli)?.acceptsModelId(modelId); + expect(accepts('opencode', 'qwen3.5:9b')).toBe(true); + expect(accepts('claude', 'qwen3.8:27b-q8_0')).toBe(true); + expect(accepts('pi', 'gpt-oss:20b')).toBe(true); + expect(accepts('omp', 'gpt-oss:20b')).toBe(false); + }); + + it('asks the availability probe once per build, not once per harness', async () => { + const probe = vi.fn(installedOnly('opencode')); + await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi']), { probeInstalled: probe }); + expect(probe).toHaveBeenCalledTimes(1); + }); + + it('a probe that fails is an answer: every harness reads as not installed, and install data still counts', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const facts = await buildHarnessFacts(catalogWith(['opencode', 'omp']), { + probeInstalled: async () => { + throw new Error('login shell did not answer'); + }, + installCommandFor: darwinCommand, + }); + expect(facts.map((f) => [f.cli, f.installed, f.installable])).toEqual([ + ['opencode', false, true], + ['omp', false, false], + ]); + expect(warn).toHaveBeenCalledTimes(1); + expect(String(warn.mock.calls[0][0])).toContain('login shell did not answer'); + }); + + it('a probe that throws for one harness does not take the others down', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude']), { + probeInstalled: async () => (entry) => { + if (entry.label === stockEntry('opencode').label) throw new Error('boom'); + return true; + }, + }); + expect(facts.map((f) => [f.cli, f.installed])).toEqual([ + ['opencode', false], + ['claude', true], + ]); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it('reads the registry when it is called, not when the module loaded', async () => { + const before = await buildHarnessFacts(catalogWith(['opencode', 'claude'])); + expect(before.map((f) => f.cli)).toEqual(['opencode', 'claude']); + + // The user disables a CLI in ~/.codeman/clis.json after the module was imported. + const path = registryFilePath(); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, JSON.stringify({ schemaVersion: 1, clis: { opencode: { enabled: false } } }), { mode: 0o600 }); + reloadCliRegistry(); + + const after = await buildHarnessFacts(catalogWith(['opencode', 'claude'])); + expect(after.map((f) => f.cli)).toEqual(['claude']); + }); + + it('with its default dependencies it spawns nothing under vitest and finds no CLI installed', async () => { + const facts = await buildHarnessFacts(catalogWith(['opencode', 'claude', 'pi'])); + expect(facts.map((f) => f.cli)).toEqual(['opencode', 'claude', 'pi']); + expect(facts.every((f) => f.installed === false)).toBe(true); + // opencode's install command is a curl installer on every platform the registry lists. + expect(facts[0].installable).toBe(true); + }); + + it('an empty harness list is an empty answer', async () => { + expect(await buildHarnessFacts(catalogWith([]))).toEqual([]); + }); +}); diff --git a/test/routes/local-model-routes.test.ts b/test/routes/local-model-routes.test.ts new file mode 100644 index 00000000..a3e7d31c --- /dev/null +++ b/test/routes/local-model-routes.test.ts @@ -0,0 +1,422 @@ +/** + * @fileoverview GET /api/local-model: what this Mac is and which local model fits it. + * + * The route only composes four injected functions (the hardware probe, the catalogue getter, + * the harness facts and the pure `recommend()`), so every test passes its own and no test + * depends on what those modules compute. What is pinned here is the route's own contract: + * the envelope, the admin gate in multi-user mode, that an unsupported platform and a failed + * probe are answers (200) and never a 500, what does answer 500, and that a request writes + * nothing. + * + * Port: N/A (app.inject()). + */ +import { afterEach, describe, expect, it, vi, type Mock } from 'vitest'; +import { existsSync, lstatSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { arch, homedir, totalmem } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRouteTestHarness } from './_route-test-utils.js'; +import { + defaultLocalModelRouteDeps, + hardwareFromOs, + registerLocalModelRoutes, + type LocalModelRouteDeps, +} from '../../src/web/routes/local-model-routes.js'; +import { buildHarnessFacts } from '../../src/local-model/harness-facts.js'; +import { formatGb, memClassGb } from '../../src/local-model/format.js'; +import { getDataDir } from '../../src/config/instance.js'; +import { + LOCAL_MODEL_LABELS, + type HardwareFacts, + type HarnessFact, + type LocalModelCatalog, + type Recommendation, +} from '../../src/types.js'; + +const CATALOG: LocalModelCatalog = { + schema: 1, + updated: '2026-11-06', + minMacosMajor: 14, + promptCacheTiers: [ + { minMemGb: 0, bytes: 512 * 1024 * 1024 }, + { minMemGb: 32, bytes: 2 * 1024 ** 3 }, + ], + models: [], + harnesses: [ + { cli: 'opencode', minContext: 65536, rank: 3 }, + { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true }, + ], + chips: [], +}; + +/** A 48 GB Apple silicon Mac whose chip is in no chips row, as the probe would report it. */ +const MAC: HardwareFacts = { + platform: 'darwin', + arch: 'arm64', + arm64: true, + translated: false, + memBytes: 51539607552, + memGb: '51.5', + memClassGb: 48, + chip: 'Apple M5 Pro', + model: 'Mac17,13', + macosVersion: '27.2', + macosMajor: 27, + wiredLimitMb: 0, + freeDiskBytes: 912308310016, + freeDiskGb: '912', + source: 'sysctl', +}; + +/** The same Mac after a `sysctl` spawn that failed: `node:os` supplied what little is known. */ +const MAC_FALLBACK: HardwareFacts = { + platform: 'darwin', + arch: 'arm64', + memBytes: 51539607552, + memGb: '51.5', + memClassGb: 48, + chip: 'Apple M5 Pro', + source: 'os-fallback', +}; + +const LINUX: HardwareFacts = { + platform: 'linux', + arch: 'x64', + memBytes: 67430404096, + memGb: '67.4', + memClassGb: 63, + source: 'not-probed', +}; + +const OK: Recommendation = { + verdict: 'ok', + headline: { text: LOCAL_MODEL_LABELS.headlineOk }, + notes: [{ text: LOCAL_MODEL_LABELS.noteUntested }], + pick: { + modelId: 'qwen3.8:27b', + modelLabel: 'Qwen3.8 27B', + license: 'Apache-2.0', + paramsB: 27.8, + context: 131072, + cli: 'opencode', + cliLabel: 'OpenCode', + cliInstalled: true, + needBytes: 29551160064, + needGb: '29.6', + downloadBytes: 17740000000, + downloadGb: '17.7', + budgetShare: 0.79, + fit: 'good', + fitLabel: LOCAL_MODEL_LABELS.fitGood, + speed: 'unknown', + speedLabel: LOCAL_MODEL_LABELS.speedUnknown, + tested: false, + }, + alternatives: [], + budgetBytes: 37580963840, + budgetGb: '37.6', + budgetSource: 'estimated', +}; + +const NOT_MACOS: Recommendation = { + verdict: 'none', + reason: 'not-macos', + headline: { text: LOCAL_MODEL_LABELS.noneNotMacos }, + notes: [], + alternatives: [], + budgetBytes: 0, + budgetGb: '0.0', + budgetSource: 'estimated', +}; + +const FACTS: HarnessFact[] = [ + { cli: 'opencode', label: 'OpenCode', installed: true, installable: false, acceptsModelId: () => true }, +]; + +interface Fakes { + deps: LocalModelRouteDeps; + readHardware: Mock; + getCatalog: Mock; + buildHarnessFacts: Mock; + recommend: Mock; +} + +function fakes(hardware: HardwareFacts = MAC, recommendation: Recommendation = OK): Fakes { + const readHardware = vi.fn(async () => hardware); + const getCatalog = vi.fn(() => CATALOG); + const buildFacts = vi.fn(async () => FACTS); + const recommend = vi.fn(() => recommendation); + return { + deps: { readHardware, getCatalog, buildHarnessFacts: buildFacts, recommend }, + readHardware, + getCatalog, + buildHarnessFacts: buildFacts, + recommend, + }; +} + +const harness = (deps: Partial, authUser?: { username: string; role: 'admin' | 'user' }) => + createRouteTestHarness((app) => registerLocalModelRoutes(app, deps), authUser ? { authUser } : undefined); + +const get = async (deps: Partial, authUser?: { username: string; role: 'admin' | 'user' }) => { + const { app } = await harness(deps, authUser); + return app.inject({ method: 'GET', url: '/api/local-model' }); +}; + +/** Every file and folder under `root`, with its size and modification time. */ +function snapshot(root: string): string[] { + if (!existsSync(root)) return []; + const rows: string[] = []; + const walk = (dir: string) => { + for (const name of readdirSync(dir).sort()) { + const path = join(dir, name); + const stat = lstatSync(path); + rows.push(`${path} ${stat.isDirectory() ? 'dir' : stat.size} ${stat.mtimeMs}`); + if (stat.isDirectory()) walk(path); + } + }; + walk(root); + return rows; +} + +afterEach(() => { + delete process.env.CODEMAN_MULTIUSER; + vi.restoreAllMocks(); +}); + +describe('GET /api/local-model', () => { + it('answers the success envelope with the hardware, the recommendation and the catalogue date', async () => { + const f = fakes(); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + expect(res.json()).toEqual({ + success: true, + data: { hardware: MAC, recommendation: OK, catalogUpdated: '2026-11-06' }, + }); + }); + + it('hands recommend() exactly what the probe, the catalogue and the harness facts answered', async () => { + const f = fakes(); + await get(f.deps); + expect(f.readHardware).toHaveBeenCalledTimes(1); + expect(f.buildHarnessFacts).toHaveBeenCalledTimes(1); + expect(f.buildHarnessFacts).toHaveBeenCalledWith(CATALOG); + expect(f.recommend).toHaveBeenCalledTimes(1); + const [hw, catalog, facts] = f.recommend.mock.calls[0]; + expect(hw).toBe(MAC); + expect(catalog).toBe(CATALOG); + expect(facts).toBe(FACTS); + }); + + it('sends the payload as the modules built it: no size is reformatted and no function leaks', async () => { + const res = await get(fakes().deps); + const { data } = res.json(); + expect(Object.keys(data).sort()).toEqual(['catalogUpdated', 'hardware', 'recommendation']); + expect(data.hardware.memGb).toBe('51.5'); + expect(data.hardware.freeDiskGb).toBe('912'); + expect(data.recommendation.pick.needGb).toBe('29.6'); + expect(data.recommendation.budgetGb).toBe('37.6'); + // Harness facts hold a function and stay on the server. + expect(res.body).not.toContain('acceptsModelId'); + expect(res.body).not.toContain('installable'); + }); + + it('registers a GET and nothing else: this stage has no route that changes anything', async () => { + const { app } = await harness(fakes().deps); + expect((await app.inject({ method: 'POST', url: '/api/local-model', payload: {} })).statusCode).toBe(404); + expect((await app.inject({ method: 'DELETE', url: '/api/local-model' })).statusCode).toBe(404); + }); + + describe('multi-user mode', () => { + it('refuses a non-admin with 403 before anything is probed', async () => { + process.env.CODEMAN_MULTIUSER = '1'; + const f = fakes(); + const res = await get(f.deps, { username: 'bob', role: 'user' }); + expect(res.statusCode).toBe(403); + expect(res.json()).toMatchObject({ success: false, errorCode: 'FORBIDDEN' }); + expect(res.body).not.toContain('Apple M5 Pro'); + expect(f.readHardware).not.toHaveBeenCalled(); + expect(f.getCatalog).not.toHaveBeenCalled(); + expect(f.buildHarnessFacts).not.toHaveBeenCalled(); + expect(f.recommend).not.toHaveBeenCalled(); + }); + + it('answers an admin', async () => { + process.env.CODEMAN_MULTIUSER = '1'; + const res = await get(fakes().deps, { username: 'root', role: 'admin' }); + expect(res.statusCode).toBe(200); + expect(res.json().data.hardware.chip).toBe('Apple M5 Pro'); + }); + + it('is not gated while multi-user mode is off, whoever asks', async () => { + const res = await get(fakes().deps, { username: 'bob', role: 'user' }); + expect(res.statusCode).toBe(200); + }); + }); + + describe('an unsupported platform is an answer', () => { + it('answers 200 with verdict none and reason not-macos on a server that is not a Mac', async () => { + const f = fakes(LINUX, NOT_MACOS); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + const { success, data } = res.json(); + expect(success).toBe(true); + expect(data.hardware).toEqual(LINUX); + expect(data.hardware.source).toBe('not-probed'); + expect(data.recommendation.verdict).toBe('none'); + expect(data.recommendation.reason).toBe('not-macos'); + expect(data.recommendation.pick).toBeUndefined(); + expect(data.recommendation.alternatives).toEqual([]); + expect(data.recommendation.headline.text).toBe(LOCAL_MODEL_LABELS.noneNotMacos); + expect(data.catalogUpdated).toBe('2026-11-06'); + expect(f.recommend.mock.calls[0][0]).toBe(LINUX); + }); + }); + + describe('a failed probe is an answer', () => { + it('answers 200 when the probe reports that sysctl failed (source os-fallback)', async () => { + const f = fakes(MAC_FALLBACK, OK); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + expect(res.json().data.hardware).toEqual(MAC_FALLBACK); + expect(res.json().data.recommendation.verdict).toBe('ok'); + }); + + it('answers 200 from node:os when the probe itself rejects, and still recommends', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const f = fakes(); + f.readHardware.mockRejectedValue(new Error('spawn EAGAIN')); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + const { success, data } = res.json(); + expect(success).toBe(true); + expect(data.hardware).toEqual(hardwareFromOs()); + expect(data.hardware.memBytes).toBe(totalmem()); + expect(data.recommendation).toEqual(OK); + expect(f.recommend).toHaveBeenCalledTimes(1); + expect(f.recommend.mock.calls[0][0]).toEqual(hardwareFromOs()); + expect(String(warn.mock.calls[0][0])).toContain('spawn EAGAIN'); + }); + + it('hardwareFromOs() is a complete answer of its own, sized by the one formatter', () => { + const hw = hardwareFromOs(); + expect(hw.platform).toBe(process.platform); + expect(hw.arch).toBe(arch()); + expect(hw.memBytes).toBe(totalmem()); + expect(hw.memGb).toBe(formatGb(totalmem())); + expect(hw.memClassGb).toBe(memClassGb(totalmem())); + expect(hw.source).toBe(process.platform === 'darwin' ? 'os-fallback' : 'not-probed'); + // Nothing that only sysctl or statfs can know is invented. + expect(Object.keys(hw).sort()).toEqual(['arch', 'memBytes', 'memClassGb', 'memGb', 'platform', 'source']); + }); + + it('answers 200 when no harness fact could be built (the facts are just empty)', async () => { + const none: Recommendation = { + ...NOT_MACOS, + reason: 'no-harness', + headline: { text: LOCAL_MODEL_LABELS.noneNoHarness }, + }; + const f = fakes(MAC, none); + f.buildHarnessFacts.mockResolvedValue([]); + const res = await get(f.deps); + expect(res.statusCode).toBe(200); + expect(res.json().data.recommendation.reason).toBe('no-harness'); + expect(f.recommend.mock.calls[0][2]).toEqual([]); + }); + }); + + describe('a fault that is not a probe is a 500 in the envelope, never a made-up answer', () => { + it.each([ + [ + 'recommend() throws', + (f: Fakes) => + f.recommend.mockImplementation(() => { + throw new Error('rank table is empty'); + }), + 'rank table is empty', + ], + [ + 'the catalogue getter throws', + (f: Fakes) => + f.getCatalog.mockImplementation(() => { + throw new Error('catalogue did not parse'); + }), + 'catalogue did not parse', + ], + [ + 'the harness facts reject', + (f: Fakes) => f.buildHarnessFacts.mockRejectedValue(new Error('registry unreadable')), + 'registry unreadable', + ], + ])('%s', async (_label, breakIt, message) => { + const f = fakes(); + breakIt(f); + const res = await get(f.deps); + expect(res.statusCode).toBe(500); + const body = res.json(); + expect(body.success).toBe(false); + expect(body.errorCode).toBe('INTERNAL_ERROR'); + expect(body.error).toContain(message); + expect(body.data).toBeUndefined(); + }); + }); + + describe('read-only', () => { + it('writes nothing: the home directory and the data dir are byte for byte what they were', async () => { + // The real harness facts (they read the CLI registry and run the availability probe), + // the route's own composition, and a hardware answer and a recommendation of the test's. + const f = fakes(); + const { app } = await harness({ ...f.deps, buildHarnessFacts: (catalog) => buildHarnessFacts(catalog) }); + + const home = homedir(); + const before = snapshot(home); + const dataDirExisted = existsSync(getDataDir()); + + for (let i = 0; i < 3; i++) { + const res = await app.inject({ method: 'GET', url: '/api/local-model' }); + expect(res.statusCode).toBe(200); + } + // A refused request and a failing one write nothing either. + f.recommend.mockImplementationOnce(() => { + throw new Error('boom'); + }); + expect((await app.inject({ method: 'GET', url: '/api/local-model' })).statusCode).toBe(500); + + expect(snapshot(home)).toEqual(before); + expect(existsSync(getDataDir())).toBe(dataDirExisted); + expect(existsSync(join(getDataDir(), 'settings.json'))).toBe(false); + expect(existsSync(join(getDataDir(), 'local-model.json'))).toBe(false); + expect(existsSync(join(getDataDir(), 'local-models.json'))).toBe(false); + // The facts really were built from the registry for the catalogue's two harnesses. + expect(f.recommend.mock.calls[0][2].map((fact) => fact.cli)).toEqual(['opencode', 'claude']); + + // The comparison above is not vacuous: one new file in the home directory shows up. + const probe = join(home, 'local-model-route-test-probe'); + writeFileSync(probe, 'x'); + try { + expect(snapshot(home)).not.toEqual(before); + } finally { + rmSync(probe); + } + }); + + it('is registered on the real server through the routes barrel', () => { + const root = fileURLToPath(new URL('../../', import.meta.url)); + const server = readFileSync(join(root, 'src/web/server.ts'), 'utf8'); + const barrel = readFileSync(join(root, 'src/web/routes/index.ts'), 'utf8'); + expect(barrel).toContain("export { registerLocalModelRoutes } from './local-model-routes.js';"); + // No second argument: the server runs the real probe, catalogue, facts and recommend(). + expect(server).toContain('registerLocalModelRoutes(this.app);'); + }); + + it('the default dependencies are the real modules, read at call time', () => { + expect(Object.keys(defaultLocalModelRouteDeps).sort()).toEqual([ + 'buildHarnessFacts', + 'getCatalog', + 'readHardware', + 'recommend', + ]); + for (const dep of Object.values(defaultLocalModelRouteDeps)) expect(typeof dep).toBe('function'); + }); + }); +});