Codeman/test/fixtures/local-model-rules-catalog.ts

/**
 * @fileoverview A frozen model catalogue for the RULE tests of src/local-model/recommend.ts:
 * the stock catalogue exactly as it shipped on 2026-11-06 (seven rows that all carry the
 * figures measured on one model), with the `runner` field the schema gained since.
 *
 * Why a frozen copy and not the stock catalogue: the stock catalogue is meant to change every
 * week or so (new models, new measurements). The tests of a RULE (the ranking, the disk
 * filter, the tags, the `limited` line, the harness fallback) need realistic rows whose
 * answers do not move with it, or every catalogue update would rewrite a hundred rule tests
 * and hide the few answers that really changed. What the STOCK catalogue answers for each Mac
 * is pinned separately, in test/local-model-recommend.test.ts ("every memory size") and the
 * two real-Mac blocks, and that is the table a catalogue change must update on purpose.
 *
 * Its figures are NOT a statement about any model: the rows borrow the measured row's figures,
 * which is exactly why the stock catalogue replaced them. Validated by the real schema.
 */

import type { LocalModelCatalog } from '../../src/types.js';

const MEASURED_MEMORY = {
  /** 36,000 to 38,000 bytes stayed held per token of context after a prompt of 89,000 and one of 167,000 tokens. */
  heldBytesPerToken: 37_000,
  /** The runner held 2.5 to 3 GB beside the weights. */
  overheadBytes: 3_000_000_000,
  /** While a long prompt was processed the peak stood about 5 GB above what stayed held. */
  peakExtraBytes: 5_000_000_000,
} as const;

/**
 * The catalogue as it shipped on 2026-11-06, with the `runner` field added. Frozen: a fixture,
 * not the stock catalogue.
 */
export const RULES_LOCAL_MODEL_CATALOG: LocalModelCatalog = {
  schema: 1,
  updated: '2026-11-06',
  // The oldest macOS the model runtime supports.
  minMacosMajor: 14,
  // Ordered by rank. The higher rank wins among models that fit equally well and are equally
  // fast. `flint:40b` is left out on purpose: its plain tag is the slow build on a Mac and it
  // has an open bug with tool arguments.
  models: [
    {
      id: 'qwen3.5:9b',
      label: 'Qwen3.5 9B',
      license: 'Apache-2.0',
      paramsB: 9,
      runner: 'mlx',
      bytes: 6_550_825_373,
      ...MEASURED_MEMORY,
      figures: 'estimated',
      contexts: [65536, 32768],
      rank: 1,
      // 37.1 tokens per second measured on 400 GB/s.
      speedPer100Gbps: 9.3,
      tested: [],
    },
    {
      id: 'qwen3.5:9b-q8_0',
      label: 'Qwen3.5 9B (Q8)',
      license: 'Apache-2.0',
      paramsB: 9,
      runner: 'llamacpp',
      bytes: 10_449_217_949,
      ...MEASURED_MEMORY,
      figures: 'estimated',
      contexts: [65536, 32768],
      rank: 2,
      // Not measured: the 9.3 of the row above scaled by file size.
      speedPer100Gbps: 5.8,
      tested: [],
    },
    {
      id: 'gpt-oss:20b',
      label: 'GPT-OSS 20B',
      license: 'Apache-2.0',
      paramsB: 20,
      runner: 'llamacpp',
      bytes: 13_793_441_427,
      ...MEASURED_MEMORY,
      figures: 'estimated',
      contexts: [131072, 65536],
      rank: 3,
      // No speed figure: its speed is unknown on every chip.
      tested: [],
    },
    {
      id: 'qwen3.6:35b-a3b',
      label: 'Qwen3.6 35B-A3B',
      license: 'Apache-2.0',
      paramsB: 35,
      runner: 'mlx',
      // What a Mac really downloads: an MLX build, 2% above the 22,621,314,161 of the manifest.
      bytes: 23_105_798_956,
      ...MEASURED_MEMORY,
      figures: 'measured',
      contexts: [131072, 65536],
      rank: 4,
      // 54.5 tokens per second measured on 400 GB/s (3 billion parameters are active per token).
      // The MLX build ran faster than this figure predicts on the one Mac we tested (67.7
      // against 41.8), so on other chips the estimate is on the low side.
      speedPer100Gbps: 13.6,
      // Run by us with OpenCode on the MacBook Pro (Mac17,13): a four-step task with tool calls.
      tested: [{ chip: 'Apple M5 Pro', memGb: 48, cli: 'opencode', date: '2026-11-06', tokensPerSecond: 67.7 }],
    },
    {
      id: 'qwen3.8:27b',
      label: 'Qwen3.8 27B',
      license: 'Apache-2.0',
      paramsB: 27.8,
      runner: 'mlx',
      bytes: 17_741_871_939,
      ...MEASURED_MEMORY,
      figures: 'estimated',
      contexts: [131072, 65536, 32768],
      rank: 5,
      // 11.8 tokens per second measured on 400 GB/s.
      speedPer100Gbps: 2.95,
      tested: [],
    },
    {
      id: 'qwen3.8:27b-q8_0',
      label: 'Qwen3.8 27B (Q8)',
      license: 'Apache-2.0',
      paramsB: 27.8,
      runner: 'llamacpp',
      bytes: 29_978_241_837,
      ...MEASURED_MEMORY,
      figures: 'estimated',
      contexts: [262144, 131072],
      rank: 6,
      // Not measured: the 2.95 of the row above scaled by file size.
      speedPer100Gbps: 1.7,
      tested: [],
    },
    {
      id: 'qwen3.5:122b-a10b',
      label: 'Qwen3.5 122B-A10B',
      license: 'Apache-2.0',
      paramsB: 122,
      runner: 'llamacpp',
      bytes: 81_370_035_877,
      ...MEASURED_MEMORY,
      figures: 'estimated',
      contexts: [131072],
      rank: 7,
      // No speed figure: its speed is unknown on every chip.
      tested: [],
    },
  ],
  // The higher rank wins when a model can run with more than one harness.
  harnesses: [
    { cli: 'opencode', minContext: 65536, rank: 3 },
    // Sends a fixed prompt of about 36,000 tokens, so the first answer takes minutes.
    { cli: 'claude', minContext: 65536, rank: 2, slowFirstTurn: true },
    // Gated: neither its 32768 minimum nor its handling of a `name:tag` model id is verified.
    { cli: 'pi', minContext: 32768, rank: 1, gated: true },
  ],
  // Memory bandwidth in GB/s, most specific first. Only chips that shipped and are in the source
  // table are listed. Where one name covers two bandwidths (M3 Max 300 or 400, M4 Max 410 or 546,
  // M5 Max 460 or 614) the LOWER one stands, so a speed computed from it is a floor.
  chips: [
    { match: 'M5 Ultra', gbps: 1228 },
    { match: 'M3 Ultra', gbps: 800 },
    { match: 'M2 Ultra', gbps: 800 },
    { match: 'M1 Ultra', gbps: 800 },
    { match: 'M5 Max', gbps: 460 },
    { match: 'M4 Max', gbps: 410 },
    { match: 'M3 Max', gbps: 300 },
    { match: 'M2 Max', gbps: 400 },
    { match: 'M1 Max', gbps: 400 },
    { match: 'M5 Pro', gbps: 307 },
    { match: 'M4 Pro', gbps: 273 },
    { match: 'M3 Pro', gbps: 150 },
    { match: 'M2 Pro', gbps: 200 },
    { match: 'M1 Pro', gbps: 200 },
    { match: 'M5', gbps: 154 },
    { match: 'M4', gbps: 120 },
    { match: 'M3', gbps: 100 },
    { match: 'M2', gbps: 100 },
    { match: 'M1', gbps: 68 },
  ],
};