From 33d697afe3aee1e8f200b348266795b0202de01d Mon Sep 17 00:00:00 2001 From: Devvyn <22340871+opticon454@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:32:24 +0200 Subject: [PATCH] fix(custom-model): warn when parallel llama.cpp slots leave too little context per request (#658) Squash of PR #658. Discovery records each llama.cpp model's parallel slot count (from --parallel/-np, or /props total_slots when the launch command states none), and a Claude launch warns when the context split across those slots leaves less than 40K tokens per request. An explicit --parallel beside a /props context records a slot count of 1, so a later round overwrites an earlier count, and a non-2xx /props is remembered per launch command while a network error is retried. Merge-time fix by the maintainer: the architecture-invariants sentence on the explicit --parallel case now says it records a slot count of 1. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/slot-aware-context.md | 5 + CLAUDE.md | 2 +- docs/api-reference.md | 5 +- docs/architecture-invariants.md | 4 +- docs/custom-model-endpoints.md | 11 +- docs/wiki/Custom-Model-Endpoints.md | 28 +++ docs/wiki/Troubleshooting.md | 14 ++ src/custom-model-hosts.ts | 16 ++ src/web/public/session-ui.js | 42 +++- src/web/routes/custom-model-routes.ts | 146 +++++++++++-- src/web/routes/session-routes.ts | 8 +- src/web/schemas.ts | 2 + .../custom-model-endpoint-rediscovery.test.ts | 200 +++++++++++++++++- test/custom-model-run-menu-ui.test.ts | 68 +++++- test/custom-model-slot-awareness.test.ts | 80 +++++++ test/routes/quick-start-custom-model.test.ts | 41 ++++ test/routes/session-custom-model.test.ts | 72 +++++++ 17 files changed, 710 insertions(+), 34 deletions(-) create mode 100644 .changeset/slot-aware-context.md create mode 100644 test/custom-model-slot-awareness.test.ts diff --git a/.changeset/slot-aware-context.md b/.changeset/slot-aware-context.md new file mode 100644 index 00000000..4a39c89a --- /dev/null +++ b/.changeset/slot-aware-context.md @@ -0,0 +1,5 @@ +--- +'aicodeman': patch +--- + +A custom llama.cpp model whose parallel slots share a context pool too small for concurrent Claude Code requests now gets an in-app warning. Four simultaneous 40K-token requests against a 4-slot ~131K pool all failed with "Context size has been exceeded" while the old total-context check stayed silent. Discovery records each model's slot count (from `--parallel`/`-np`, or `/props` `total_slots` when the launch command states none) and the warning names the fix: `--parallel 1`, or a context large enough for every slot. diff --git a/CLAUDE.md b/CLAUDE.md index 2a061c91..30af1eae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -181,7 +181,7 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph **Custom Model Endpoint Profiles** (opt-in, `customModelEndpointsEnabled`, SYNCED, default OFF; `docs/custom-model-endpoints.md`): points a session at a user-configured OpenAI-compatible endpoint (store `custom-model-hosts.ts`, `~/.codeman/custom-model-hosts.json`, 0600). `authStyle` is `bearer` or `api-key`, never both headers (hangs the server). The per-CLI redirect is registry data, `capabilities.customModelInjection` (`env` / `configContentEnv` / `configDir` / `unsupported`), computed by the pure `custom-model-injection.ts`; ⚠️ `configDir` writes an isolated per-session config, NEVER the user's real `~/.codex`/`~/.pi`/`~/.omp`/grok config. ⚠️ Claude applies via `Session.restartCli()` (a relaunch in the existing pane, so the conversation is pinned as `resumeSessionId`), the rest one-shot at launch; retired env keys must also be `setenv -u`'d (`_pendingEnvUnsets`), since tmux env survives `respawn-pane`. ⚠️ Remote/Docker sessions are refused (400). ⚠️ Persist only KEYS (`__customModel`), never values (they carry the API key). ⚠️ Every redirectable var must be in that CLI's `privilegedEnvKeys`, and `ANTHROPIC_*` stays out of claude's `allowedPrefixes`. ⚠️ The Run-menu picker builds entries from `window.__codemanCustomModelClis` (escaped via `escapeScriptJson()`), never a hardcoded CLI id list, and launches through `run()` via a temporary `_runMode` swap, never `setRunMode()`. → [architecture-invariants#custom-model-endpoint-profiles](docs/architecture-invariants.md#custom-model-endpoint-profiles) -⚠️ **llama-swap endpoints** (one model at a time): the apply routes check `GET /running` and return `requiresConfirmation` before evicting a model another live session uses; `confirmedSwap` and `confirmedContext` are SEPARATE flags and must stay so. Claude alone gets a context floor (`CLAUDE_MIN_SAFE_CONTEXT_TOKENS`); context is parsed from `/running`'s `cmd`, never trusted from `/props`. Backend log lines come from llama-swap's `/api/events` `upstream` source, never `/logs`. → [architecture-invariants#custom-model-endpoint-profiles](docs/architecture-invariants.md#custom-model-endpoint-profiles) +⚠️ **llama-swap endpoints** (one model at a time): the apply routes check `GET /running` and return `requiresConfirmation` before evicting a model another live session uses; `confirmedSwap` and `confirmedContext` are SEPARATE flags and must stay so. Claude alone gets a context floor (`CLAUDE_MIN_SAFE_CONTEXT_TOKENS`); context is parsed from `/running`'s `cmd`, never trusted from `/props`. The floor also divides that context by the model's parallel slot count (`modelSlotCounts`: `--parallel`/`-np` from `cmd`, else `/props` `total_slots`; `perSlotContext()`), and `/props` only ever RAISES that warning: it supplies the context only when `cmd` states none and the count only when `cmd` has no `--parallel`, never a larger pool. `/props` is probed once per launch command (it refreshes llama-swap's last-use time, so a `ttl` would never expire), and a `/props` context beside an explicit `--parallel N` is already per slot, so it is not divided again. Backend log lines come from llama-swap's `/api/events` `upstream` source, never `/logs`. → [architecture-invariants#custom-model-endpoint-profiles](docs/architecture-invariants.md#custom-model-endpoint-profiles) **MCP server sync** (opt-in, `mcpSyncEnabled`, SYNCED, default OFF; `src/mcp-sync.ts`, `GET`/`POST /api/mcp-sync`): copies each installed, enabled CLI's user-level MCP servers into the others. It is the ONE subsystem that writes another CLI's REAL user config (`~/.claude.json`, `~/.codex/config.toml`, `~/.gemini/*`, opencode's), which is why it is opt-in and admin-only in multi-user mode (both verbs 403 for a non-admin, and the Settings group is hidden for them). Where each CLI keeps the file is registry data, `capabilities.mcpConfig` (`{ path, format, relocation? }`), never a branch on the id. ⚠️ Tools that sync but are not run modes (GitHub Copilot CLI, the first) are plain data in `MCP_SYNC_ONLY_TOOLS` (`src/mcp-sync-targets.ts`), take part only when installed or already configured, and their companion `settings.json` (`disabledMcpServers`) is read, never written; a test must clear their relocation vars too. ⚠️ ADDITIVE only: a name already defined, in any shape, is never edited or removed (a different same-name definition is a reported conflict), and a server switched off in its own CLI is never copied. ⚠️ Never write a file that did not parse; re-parse the NEW text and require every added server to read back before the tmp+rename (written through a symlink, previous file kept as `.codeman-bak`, one apply at a time, else 409). ⚠️ A file that receives copied `env`/`headers` (secrets) is left `0600`, and so is the backup. ⚠️ Responses carry server NAMES only, never env values, headers or file text: a parse failure is reported by line and column (`describeMcpSyncError`), never the parser's own message (smol-toml and V8 both quote source). ⚠️ `mcpConfig.relocation` names the env var the CLI reads to move its file (`CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `XDG_CONFIG_HOME`, `GEMINI_CLI_HOME`, and `COPILOT_HOME` for Copilot), resolved from the SERVER env at call time; a relative value reports the target `skipped`, never a guessed write, and a per-session `envOverrides` relocation is not followed. Tests must pass `home` (which drops the `process.env` default) or clear those vars first. → `docs/cli-registry.md` (MCP server sync), `docs/api-reference.md`, `docs/wiki/Settings-Reference.md` diff --git a/docs/api-reference.md b/docs/api-reference.md index 97f3bc22..d14cc05c 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -996,7 +996,10 @@ confirmed? } | { clear: true }` applies (or clears) the session's just because a swap is needed at all — only when it would disrupt someone else. - `{requiresContextWarning: true, modelId, contextLength, -minSafeContextTokens}` — Claude Code's own fixed per-turn overhead +minSafeContextTokens, slotCount?}` — `slotCount` is present whenever the model runs more than + one parallel llama.cpp slot, and the warning also fires when the context + divided across those slots is below the floor even though the whole context + is not. Claude Code's own fixed per-turn overhead (system prompt + tool schemas) can exceed a small model's entire discovered context on its own, before any conversation history exists to compact, guaranteeing the very first message fails regardless of diff --git a/docs/architecture-invariants.md b/docs/architecture-invariants.md index 2d2e72bf..ec0f3561 100644 --- a/docs/architecture-invariants.md +++ b/docs/architecture-invariants.md @@ -647,9 +647,9 @@ Either way the actual launch (`runCustomModelEntry`) routes through `run()` itse ⚠️ **That launch-time check cannot catch a swap caused by a DIFFERENT session's LATER, ordinary use** — confirmed live: a second Codex session picking a different model launched with no warning at all (nothing conflicted at that exact instant), yet it silently evicted the first session's model regardless, since llama-swap has no push notification of its own. `detectCustomModelSwapDisplacements()` (`custom-model-routes.ts`) is a separate periodic sweep (`server.ts`, `CUSTOM_MODEL_SWAP_CHECK_INTERVAL_MS` = 20s) that compares each live custom-model session's own `modelId` against what `/running` actually reports loaded, broadcasting a `custom-model:swapped-out` SSE event — shown as a global toast, never tied to the displaced session's own tab, since the whole point is telling the user before they type into it — the first time a mismatch appears, via a caller-owned de-dupe `Set` cleared once that session's own model is loaded and ready again so a later, genuinely new displacement notifies again rather than staying silently un-notified forever after the first one. -⚠️ **Context length is read from the REAL launch command, never `/props`** — `/props?model=`'s `default_generation_settings.n_ctx` was confirmed live to report a `--fit-ctx`-launched backend's theoretical/trained maximum rather than the real runtime-configured size (a measured 154112-vs-16384 discrepancy, caught only because the unfixed value still overflowed), so discovery parses the actual configured size straight out of `/running`'s own `cmd` field instead (`parseCtxFromCmd`: `--fit-ctx ` first, then plain llama.cpp `-c`/`--ctx-size`), falling back to `/props` only when `cmd` states no recognizable flag at all. +⚠️ **Context length is read from the REAL launch command, never `/props`** — `/props?model=`'s `default_generation_settings.n_ctx` was confirmed live to report a `--fit-ctx`-launched backend's theoretical/trained maximum rather than the real runtime-configured size (a measured 154112-vs-16384 discrepancy, caught only because the unfixed value still overflowed), so discovery parses the actual configured size straight out of `/running`'s own `cmd` field instead (`parseCtxFromCmd`: `--fit-ctx ` first, then plain llama.cpp `-c`/`--ctx-size`), falling back to `/props` only when `cmd` states no recognizable flag at all. The same rule covers the slot check: `/props` supplies `total_slots` only when `cmd` has no `--parallel`, and it NEVER widens the pool the slots divide (an earlier cut let a larger `/props` `n_ctx` become the pool and skip the warning; the 154112-vs-16384 measurement is exactly why that is wrong). `/props` refreshes llama-swap's last-use time for the model, so `discoverModels` probes it once per launch command (`propsByCmd`, in memory, keyed by host and model id and compared with `/running`'s `cmd`); probing on every 5-minute sweep would keep a model with a `ttl` loaded forever. With an explicit `--parallel N` llama.cpp reports `n_ctx` per slot, so a `/props` context beside it is stored with a slot count of 1: it is not divided a second time, and the 1 overwrites a count an earlier discovery round stored (`applyDiscoveredModels` merges onto what was learned before, so recording no count would keep the stale one). -⚠️ **Claude alone gets a context-window FLOOR check, on top of the ceiling `contextLengthVar` already fixes** — `exceedsSafeContextFloor()` (gated on the registry declaring `contextLengthVar`, so a no-op for every other CLI by construction) compares a model's discovered context against `CLAUDE_MIN_SAFE_CONTEXT_TOKENS` (40000): confirmed live, twice, that Claude Code's own system prompt and tool schemas cost roughly 36.4K tokens on the very first message, before any conversation history exists to compact, so a smaller real context fails outright regardless of what `CLAUDE_CODE_MAX_CONTEXT_TOKENS` says (that var only controls when HISTORY gets compacted, and there is none yet on message one). Below the floor, the apply returns `{requiresContextWarning, modelId, contextLength, minSafeContextTokens}` instead of launching, shown as an in-app dialog naming the actual fix: give the model an explicit larger `-c`/`--ctx-size` in llama-swap's config instead of relying on `--fit-ctx` auto-fit, which optimizes for the biggest MODEL that fits rather than the biggest CONTEXT. +⚠️ **Claude alone gets a context-window FLOOR check, on top of the ceiling `contextLengthVar` already fixes** — `exceedsSafeContextFloor()` (gated on the registry declaring `contextLengthVar`, so a no-op for every other CLI by construction) compares a model's discovered context against `CLAUDE_MIN_SAFE_CONTEXT_TOKENS` (40000): confirmed live, twice, that Claude Code's own system prompt and tool schemas cost roughly 36.4K tokens on the very first message, before any conversation history exists to compact, so a smaller real context fails outright regardless of what `CLAUDE_CODE_MAX_CONTEXT_TOKENS` says (that var only controls when HISTORY gets compacted, and there is none yet on message one). Below the floor, or when `contextLength / slotCount` (`perSlotContext()`, from `modelSlotCounts`; an unknown count is one slot) is below it, the apply returns `{requiresContextWarning, modelId, contextLength, minSafeContextTokens}` (plus `slotCount` when it is above 1) instead of launching, shown as an in-app dialog naming the actual fix: give the model an explicit larger `-c`/`--ctx-size` in llama-swap's config instead of relying on `--fit-ctx` auto-fit, which optimizes for the biggest MODEL that fits rather than the biggest CONTEXT. ⚠️ **A fresh, isolated `CLAUDE_CONFIG_DIR` looks like a brand-new Claude Code profile and replays its ENTIRE first-run sequence on every launch** — the theme picker, the security-notes screen, the per-project "trust this folder?" dialog, and (running with a bypass-permissions flag) a one-time warning about it, confirmed live, none of which a real, already-onboarded profile shows again. `skipFirstRunPrompts` (claude's entry only, requires `apiKeyTrustFile` since it reuses the same file) pre-seeds that same "already been through this" state: `hasCompletedOnboarding` and this session's own `projects[workingDir].hasTrustDialogAccepted` merge into the same `.claude.json` the API-key trust file already writes to, and `skipDangerousModePermissionPrompt` merges into `settings.json` (a different file, same corrupt-tolerant merge). diff --git a/docs/custom-model-endpoints.md b/docs/custom-model-endpoints.md index f37ddebe..f76c21a1 100644 --- a/docs/custom-model-endpoints.md +++ b/docs/custom-model-endpoints.md @@ -97,6 +97,13 @@ llama-swap's own launch command instead (`GET /running`'s `cmd` field — hand-written command might use), and only falls back to the `/props` probe when `cmd` states no recognizable flag at all. +**Parallel slots are read the same way.** `--parallel N` / `-np N` from `cmd` gives +the slot count (`modelSlotCounts`); only when `cmd` has none does discovery read +`total_slots` from `/props`, because a server started without the flag picks the +number itself. `/props` never widens the context the slot check divides, and it is +probed once per launch command rather than on every sweep, because llama-swap +treats the call as use of the model and a `ttl` would otherwise never expire. + **File size is discovered too, when the server states one.** llama-swap writes a GB figure into an auto-discovered model's own `description` (`"Auto-discovered 16.35 GB - parameters auto-fitted by llama.cpp"`), parsed @@ -438,11 +445,11 @@ construction, never a hardcoded mode check). If the model's discovered context (`modelContextLengths`, from discovery above) is below `CLAUDE_MIN_SAFE_CONTEXT_TOKENS` (40000, comfortably above the measured ~36.4K overhead), the response is `{requiresContextWarning: true, modelId, -contextLength, minSafeContextTokens}` instead of applying — nothing is +contextLength, minSafeContextTokens}` (plus `slotCount` when the model runs more than one parallel slot) instead of applying — nothing is restarted or created yet. A context length that was never discovered at all skips the check entirely (nothing to compare, so it fails open rather than warning on every model an endpoint hasn't reported a size for). -Retrying with `confirmedContext: true` launches anyway (the legacy `confirmed: true` still means both questions). +The same warning fires when the context divided across the model's parallel slots (`contextLength / slotCount`) is below the floor, because Claude Code sends several 40K-token requests at once and the slots share one pool. Retrying with `confirmedContext: true` launches anyway (the legacy `confirmed: true` still means both questions). The Run-menu picker shows this as an in-app modal (`#customModelContextWarningModal`, matching the llama-swap conflict diff --git a/docs/wiki/Custom-Model-Endpoints.md b/docs/wiki/Custom-Model-Endpoints.md index a10f6f7b..9d2b6a30 100644 --- a/docs/wiki/Custom-Model-Endpoints.md +++ b/docs/wiki/Custom-Model-Endpoints.md @@ -150,6 +150,34 @@ relying on auto-fit (`--fit-ctx`), which sizes the context around fitting the bi rather than the biggest context — for example adding `-c 65536` to that model's llama-swap entry. "Launch anyway" is still there if you want to try regardless. +**A context that is big enough overall can still be too small once it is split across +slots.** `llama-server` runs several parallel slots by default (four on the build this was measured on) and, +unless you say otherwise, they all share one pool of context. Claude Code sends more than one +request at a time: the main session plus every subagent it runs in parallel, and each of those +requests carries the roughly 40K tokens of system prompt and tools on its own. Four requests +of 40K tokens do not fit in a shared 131K pool, so the server refuses them with HTTP 500 +`Context size has been exceeded`, even though any single request would have been fine. +Discovery therefore also records how many slots each loaded model has. It reads an explicit +`--parallel` or `-np` from the launch command, and otherwise asks the server for +`total_slots`, because a server started without the flag chooses the number itself. When the +context divided by the slot count drops below what Claude Code needs, picking that model shows +the same in-app dialog, this time explaining the shared slots. The division uses the context +Codeman read from the launch command (`--fit-ctx` or `-c`), never a larger figure from the +server's `/props`, which has been measured reporting far more than the server really runs +with. So a default llama-swap model with auto slots shows the dialog, which names the fix and +lets you launch anyway. The fix is a single slot: + +```yaml +cmd: >- + llama-server -m /models/your-model.gguf --parallel 1 ... +``` + +With one slot the active conversation gets the whole context, requests from parallel +subagents queue behind it instead of failing, and Claude Code's repeated prompt prefix stays +in the cache instead of being processed again in a different slot each time. The alternative +is a context large enough that every slot can hold 40K tokens. A slot count Codeman could not +read is treated as one slot, never guessed. + ## Which harnesses actually work | Harness | Status | diff --git a/docs/wiki/Troubleshooting.md b/docs/wiki/Troubleshooting.md index cd191493..ab74e79c 100644 --- a/docs/wiki/Troubleshooting.md +++ b/docs/wiki/Troubleshooting.md @@ -161,6 +161,20 @@ CODEMAN_INSTANCE=beta CODEMAN_PORT=5000 codeman web `codeman web -d` and `codeman service install` both refuse to start a second server on one data directory for exactly this reason. +### `Context size has been exceeded` from a local llama.cpp model + +Sessions pointed at a llama.cpp or llama-swap endpoint (see [Custom Model +Endpoints](Custom-Model-Endpoints)) fail with `500 Context size has been exceeded`, usually +when Claude Code runs subagents, or an agent fails with `Prompt is too long` while the main +session carries on. The server runs several parallel slots that share one pool of context, +and every Claude Code request carries about 40K tokens of its own system prompt and tools, so +a few concurrent requests use the pool up. Start the model with a single slot by adding +`--parallel 1` to its launch command, or give it a context large enough that each slot holds +40K tokens. Check the current count with `GET /props` on the server (on llama-swap, +`GET /props?model=` for a loaded model): `total_slots` is the number of slots. `n_ctx` is the +shared context only when the slot count is automatic; with an explicit `--parallel N` it is the +size of each slot. + ## The terminal ### I cannot scroll back through history diff --git a/src/custom-model-hosts.ts b/src/custom-model-hosts.ts index bd90cef7..8ad2c17d 100644 --- a/src/custom-model-hosts.ts +++ b/src/custom-model-hosts.ts @@ -61,6 +61,22 @@ export interface CustomModelHost { * has no entry for simply gets no context-length env override applied — never a guess. */ modelContextLengths?: Record; + /** + * Discovered number of parallel server slots per model id, keyed like `modelContextLengths` + * and populated on the same loaded-models-only basis. Read from the launch command's + * `--parallel`/`-np` when it states one, otherwise from llama.cpp's own `/props` + * `total_slots` (a llama-server started without `--parallel` picks the count itself, so + * the command cannot say). llama-server splits its context across these slots (by default + * they share one unified KV pool) and Claude Code sends several requests at once (the + * main session plus each parallel subagent), so with N slots each request can only rely + * on roughly `contextLength / N` tokens. Used only by the context warning; a model this + * has no entry for is treated as a single slot, never a guess. The division always uses + * `modelContextLengths` (the launch command's figure, or `/props` only when the command states + * none): `/props` never widens the pool. An explicit `--parallel N` whose context came from + * `/props` is recorded as 1 slot, because that figure is already per slot, and that also + * overwrites any count an earlier round stored. + */ + modelSlotCounts?: Record; /** * Discovered file size (GB) per model id, keyed by the same strings as `models`. * Populated during discovery by parsing llama-swap's own `description` field for an diff --git a/src/web/public/session-ui.js b/src/web/public/session-ui.js index 5009c2d5..d3eca870 100644 --- a/src/web/public/session-ui.js +++ b/src/web/public/session-ui.js @@ -1114,11 +1114,33 @@ Object.assign(CodemanApp.prototype, { * history yet for compaction to trim. Same promise-based pattern as * `_confirmModelSwap`; `_resolveContextWarningConfirm` settles it. */ - _confirmContextWarning(modelId, contextLength, minSafeContextTokens) { + _confirmContextWarning(modelId, contextLength, minSafeContextTokens, slotCount) { const modal = document.getElementById('customModelContextWarningModal'); const messageEl = document.getElementById('customModelContextWarningMessage'); if (messageEl) { const known = typeof contextLength === 'number'; + const slots = Number.isFinite(slotCount) && slotCount > 1 ? Math.floor(slotCount) : 1; + // Enough total context, but split across parallel server slots: llama-server shares its + // context between --parallel slots, and this CLI sends concurrent requests (the main + // session plus each parallel subagent), so several 40k-token requests spend the shared + // pool and the server answers HTTP 500 "Context size has been exceeded". + if (known && slots > 1 && contextLength >= minSafeContextTokens) { + messageEl.textContent = + `${modelId}'s ${contextLength.toLocaleString()} tokens of context are split across ${slots} parallel ` + + `server slots, about ${Math.floor(contextLength / slots).toLocaleString()} each. This CLI ` + + `needs roughly ${minSafeContextTokens.toLocaleString()}+ tokens just for its own system prompt and tools on ` + + `every request, and it sends several requests at once (the main session plus each parallel subagent). ` + + `Concurrent requests can use up the shared context, and the server then fails them with ` + + `"Context size has been exceeded".\n\n` + + `To fix this, start the model with a single slot: add "--parallel 1" to its llama-swap cmd. The active ` + + `conversation then gets the whole context, other requests queue instead of failing, and the CLI's ` + + `repeated prompt prefix stays cached. Alternatively give the model a larger context (raise --fit-ctx or ` + + `--ctx-size) so every slot has at least ${minSafeContextTokens.toLocaleString()} tokens.`; + modal?.classList.add('active'); + return new Promise((resolve) => { + this._resolveContextWarningConfirmPromise = resolve; + }); + } messageEl.textContent = `${modelId} is configured with ` + (known ? `only ${contextLength.toLocaleString()} tokens of` : 'an unknown (too small)') + @@ -1129,7 +1151,11 @@ Object.assign(CodemanApp.prototype, { `instead of relying on auto-fit (--fit-ctx), which optimizes for the biggest MODEL that fits, not the ` + `biggest CONTEXT — e.g. add "-c 65536" (or as large a --ctx-size as your hardware holds) to its llama-swap ` + `config entry. A smaller model at a much larger explicit context often fits in the same VRAM a bigger ` + - `model's auto-fit context gets shrunk to make room for.`; + `model's auto-fit context gets shrunk to make room for.` + + (slots > 1 + ? `\n\nThis server also runs ${slots} parallel slots that share that context; add "--parallel 1" to ` + + `the same cmd so a single conversation gets all of it.` + : ''); } modal?.classList.add('active'); return new Promise((resolve) => { @@ -1254,8 +1280,13 @@ Object.assign(CodemanApp.prototype, { let answered = {}; let data = await post(bodyObj); if (data?.data?.requiresContextWarning) { - const { modelId, contextLength, minSafeContextTokens } = data.data; - const proceed = await this._confirmContextWarning(modelId, contextLength, minSafeContextTokens); + const { modelId, contextLength, minSafeContextTokens, slotCount } = data.data; + const proceed = await this._confirmContextWarning( + modelId, + contextLength, + minSafeContextTokens, + slotCount + ); if (!proceed) { this._lastCustomModelLaunchResult = undefined; return { success: false, error: 'Launch cancelled — context window too small' }; @@ -1371,7 +1402,8 @@ Object.assign(CodemanApp.prototype, { const proceed = await this._confirmContextWarning( payload.modelId, payload.contextLength, - payload.minSafeContextTokens + payload.minSafeContextTokens, + payload.slotCount ); if (!proceed) { switchingToast?.dismiss(); diff --git a/src/web/routes/custom-model-routes.ts b/src/web/routes/custom-model-routes.ts index 7f96a7c6..3339ae3c 100644 --- a/src/web/routes/custom-model-routes.ts +++ b/src/web/routes/custom-model-routes.ts @@ -56,11 +56,31 @@ export const CLAUDE_MIN_SAFE_CONTEXT_TOKENS = 40000; */ export function exceedsSafeContextFloor( entry: Pick, - contextLength: number | undefined + contextLength: number | undefined, + slotCount?: number ): boolean { const cap = entry.capabilities.customModelInjection; if (cap.kind !== 'env' || !cap.contextLengthVar) return false; - return typeof contextLength === 'number' && contextLength < CLAUDE_MIN_SAFE_CONTEXT_TOKENS; + if (typeof contextLength !== 'number') return false; + if (contextLength < CLAUDE_MIN_SAFE_CONTEXT_TOKENS) return true; + return perSlotContext(contextLength, slotCount) < CLAUDE_MIN_SAFE_CONTEXT_TOKENS; +} + +/** + * The context one request can rely on when the server runs `slotCount` parallel slots. + * llama-server divides its context across `--parallel` slots (by default they share ONE + * unified KV pool), and Claude Code sends concurrent requests: the main session plus every + * parallel subagent, each carrying the full `CLAUDE_MIN_SAFE_CONTEXT_TOKENS` of its own + * system prompt and tools. With N busy slots each request therefore gets roughly + * `contextLength / N`, and once the pool is spent llama.cpp answers HTTP 500 "Context size + * has been exceeded" (measured: 4 slots, ~131k pool, four simultaneous 40k-token requests + * all failed; the same requests against one slot all succeeded). An unknown or single slot + * count leaves the context whole. + */ +export function perSlotContext(contextLength: number, slotCount: number | undefined): number { + const slots = + typeof slotCount === 'number' && Number.isFinite(slotCount) && slotCount > 1 ? Math.floor(slotCount) : 1; + return Math.floor(contextLength / slots); } function adminOnly(req: FastifyRequest, reply: { code: (n: number) => unknown }): ApiResponse | null { @@ -109,7 +129,7 @@ function applyStoredApiKey(incoming: CustomModelHost, existing: CustomModelHost) } /** - * `modelContextLengths`/`modelSizesGB` are server-populated by discovery, never + * `modelContextLengths`/`modelSlotCounts`/`modelSizesGB` are server-populated by discovery, never * user-entered, and PUT replaces the whole record — so merge them back in from the * stored host rather than trust whatever the editor's body carried (or omitted). * The editor only ever sends `models`/`lastDiscoveredAt` verbatim from its cached @@ -117,7 +137,12 @@ function applyStoredApiKey(incoming: CustomModelHost, existing: CustomModelHost) * future caller forgets, same class of bug `applyStoredApiKey` exists to prevent. */ function applyDiscoveredFields(incoming: CustomModelHost, existing: CustomModelHost): CustomModelHost { - return { ...incoming, modelContextLengths: existing.modelContextLengths, modelSizesGB: existing.modelSizesGB }; + return { + ...incoming, + modelContextLengths: existing.modelContextLengths, + modelSlotCounts: existing.modelSlotCounts, + modelSizesGB: existing.modelSizesGB, + }; } function authHeaders(host: Pick): Record { @@ -135,6 +160,8 @@ export interface DiscoveryResult { models: string[]; /** See `CustomModelHost.modelContextLengths` — only ever populated for models already loaded. */ contextLengths: Record; + /** See `CustomModelHost.modelSlotCounts`; populated on the same loaded-models-only basis as `contextLengths`. */ + slotCounts: Record; /** See `CustomModelHost.modelSizesGB` — populated for every model whose own listing states one. */ sizesGB: Record; } @@ -157,7 +184,7 @@ function parseSizeGB(description: unknown): number | undefined { /** * Best-effort: fetches `GET /props?model=` (llama.cpp-native, llama-swap-proxied) for - * ONE already-loaded model and pulls its real `n_ctx` out. Never called for a model that + * ONE already-loaded model and pulls its `n_ctx` and `total_slots` out. Never called for a model that * isn't already loaded — see the caller and `CustomModelHost.modelContextLengths` for why * that's a hard safety requirement, not just a nicety: llama-swap treats this endpoint's * `?model=` as a routing hint, and asking it about an unloaded model risks triggering an @@ -173,20 +200,39 @@ function parseSizeGB(description: unknown): number | undefined { * (below), which reads the actual launch flag `/running` reports, is the primary source; * this is only used when that parse comes up empty (no recognized flag in `cmd`, or `cmd` * itself unavailable). + * + * ⚠️ `/props` only ever RAISES the slot warning, never skips it: its `n_ctx` is the context + * fallback when `cmd` states no context flag at all, and `total_slots` is the slot count when + * `cmd` has no `--parallel`. It never widens the pool the slot check divides. And it is not + * free: llama-swap routes it through the model's process handler, which refreshes the model's + * last-use time, so probing a model with a `ttl` on every 5-minute sweep would keep it loaded + * forever. `discoverModels` therefore probes once per launch command (`propsByCmd`). + * + * Returns `undefined` when the request itself failed (network error, timeout, unreadable body), + * which is retried next sweep; a non-2xx status, or a 2xx without the fields, gives an object with + * them unset, which `propsForLaunch` remembers. */ -async function fetchContextLength( +async function fetchServerProps( host: Pick, modelId: string, headers: Record -): Promise { +): Promise<{ nCtx?: number; totalSlots?: number } | undefined> { try { const url = new URL(`${host.baseUrl.replace(/\/+$/, '')}/props`); url.searchParams.set('model', modelId); const res = await webviewFetch(url, { headers, signal: AbortSignal.timeout(PROPS_TIMEOUT_MS) }); - if (!res.ok) return undefined; - const body = (await res.json()) as { n_ctx?: unknown; default_generation_settings?: { n_ctx?: unknown } }; + // A non-2xx answer (an upstream that does not serve /props) is a real answer: empty, so it is + // remembered per launch command instead of being re-probed every sweep. + if (!res.ok) return {}; + const body = (await res.json()) as { + n_ctx?: unknown; + total_slots?: unknown; + default_generation_settings?: { n_ctx?: unknown }; + }; const nCtx = body.n_ctx ?? body.default_generation_settings?.n_ctx; - return typeof nCtx === 'number' && Number.isFinite(nCtx) && nCtx > 0 ? nCtx : undefined; + const positive = (v: unknown): number | undefined => + typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : undefined; + return { nCtx: positive(nCtx), totalSlots: positive(body.total_slots) }; } catch { return undefined; } @@ -196,7 +242,7 @@ async function fetchContextLength( * Parses the REAL configured context size out of llama-swap's own launch command for a * model (`/running`'s `cmd` field, e.g. `"llama-server -m ... --fit-ctx 16384 ..."`) — * the primary source for `modelContextLengths`, preferred over `/props`'s `n_ctx` (see - * `fetchContextLength`'s own doc comment for why that field is unreliable here). Checks + * `fetchServerProps`'s own doc comment for why that field is unreliable here). Checks * `--fit-ctx` first (llama-swap's own auto-fit flag), then the plain llama.cpp * `-c`/`--ctx-size`/`--ctx_size` flags a hand-written launch command might use instead. * Returns `undefined` when `cmd` has none of these — not every launch command needs to @@ -211,6 +257,51 @@ function parseCtxFromCmd(cmd: unknown): number | undefined { return Number.isFinite(value) && value > 0 ? value : undefined; } +/** + * Parses an explicit `--parallel N` / `-np N` (or `--parallel=N`) out of a launch command. + * Returns `undefined` when the command states none, including `-np -1` (llama.cpp's "pick + * automatically"), because a llama-server started without it chooses the slot count itself + * (4 on the build this was measured on) and only `/props`'s `total_slots` can say what it chose. + */ +function parseSlotsFromCmd(cmd: unknown): number | undefined { + if (typeof cmd !== 'string') return undefined; + const match = /(?:^|\s)(?:-np|--parallel)(?:\s+|=)(\d+)(?=\s|$)/.exec(cmd); + if (!match) return undefined; + const value = Number(match[1]); + return Number.isFinite(value) && value > 0 ? value : undefined; +} + +/** The `/props` answer for the launch command a model was last probed under, by host and model id. */ +const propsByCmd = new Map(); + +/** Forget what was probed (tests only: a changed launch command already probes again). */ +export function clearPropsProbeCache(): void { + propsByCmd.clear(); +} + +/** + * `/props` for one loaded model, but only once per launch command: the probe refreshes the model's + * last-use time in llama-swap, so repeating it on every rediscovery sweep would stop a model with a + * `ttl` from ever unloading. A changed `cmd` (a reloaded or reconfigured model) probes again; a + * `cmd` that is not a string cannot be told apart, so it is probed each time; a probe that did not + * answer (network error, timeout) is not remembered; a non-2xx status is. Held in memory only, so a restart probes once more. + */ +async function propsForLaunch( + host: Pick, + modelId: string, + cmd: unknown, + headers: Record +): Promise<{ nCtx?: number; totalSlots?: number } | undefined> { + const key = `${host.baseUrl}\0${modelId}`; + if (typeof cmd === 'string') { + const cached = propsByCmd.get(key); + if (cached?.cmd === cmd) return cached.props; + } + const props = await fetchServerProps(host, modelId, headers); + if (props && typeof cmd === 'string') propsByCmd.set(key, { cmd, props }); + return props; +} + async function discoverModels( host: Pick ): Promise { @@ -239,6 +330,7 @@ async function discoverModels( // fetchContextLength must only ever run against a model this server itself calls loaded. const hasStatusField = entries.some((m) => m && typeof m === 'object' && 'status' in m); const contextLengths: Record = {}; + const slotCounts: Record = {}; if (hasStatusField) { const loadedIds = entries .filter((m) => m.status && typeof m.status === 'object' && (m.status as { value?: unknown }).value === 'loaded') @@ -252,13 +344,32 @@ async function discoverModels( const swapStatus = await getLlamaSwapStatus(host); const cmdById = new Map(swapStatus.running.map((r) => [r.model, r.cmd])); for (const id of loadedIds) { - const fromCmd = parseCtxFromCmd(cmdById.get(id)); - const ctx = fromCmd ?? (await fetchContextLength(host, id, headers)); + const cmd = cmdById.get(id); + let ctx = parseCtxFromCmd(cmd); + const cmdSlots = parseSlotsFromCmd(cmd); + let slots = cmdSlots; + // /props is only needed for whatever the launch command did not answer. The context + // still comes from the command whenever it states one (see fetchServerProps's doc + // comment for why /props's n_ctx is only a fallback); the slot count usually cannot + // come from the command at all, since an unset --parallel is chosen by the server. + if (ctx === undefined || slots === undefined) { + const props = await propsForLaunch(host, id, cmd, headers); + const ctxFromProps = ctx === undefined && props?.nCtx !== undefined; + ctx ??= props?.nCtx; + slots ??= props?.totalSlots; + // With an explicit --parallel N, llama.cpp reports /props's n_ctx PER SLOT (the unified + // KV pool is only the default when the count is auto), so it is already the figure one + // request can use, so the count is 1: recording N would divide it a second time, and + // recording nothing would leave an earlier round's count in place (applyDiscoveredModels + // merges onto what was learned before). + if (cmdSlots !== undefined && ctxFromProps) slots = 1; + } if (ctx !== undefined) contextLengths[id] = ctx; + if (slots !== undefined) slotCounts[id] = slots; } } } - return { models, contextLengths, sizesGB }; + return { models, contextLengths, slotCounts, sizesGB }; } /** @@ -573,7 +684,7 @@ export function triggerLlamaSwapLoad( } function applyDiscoveredModels(host: CustomModelHost, result: DiscoveryResult): CustomModelHost { - const { models, contextLengths, sizesGB } = result; + const { models, contextLengths, slotCounts, sizesGB } = result; const defaultModelId = host.defaultModelId && models.includes(host.defaultModelId) ? host.defaultModelId : undefined; // Merge onto what's already known rather than replacing: a model not probed this round // (not currently loaded) keeps whatever context length an earlier round already learned @@ -581,6 +692,10 @@ function applyDiscoveredModels(host: CustomModelHost, result: DiscoveryResult): const merged = { ...host.modelContextLengths, ...contextLengths }; const kept = Object.fromEntries(Object.entries(merged).filter(([id]) => models.includes(id))); const modelContextLengths = Object.keys(kept).length > 0 ? kept : undefined; + // Slot counts follow the same loaded-only, keep-what-an-earlier-round-learned rule. + const mergedSlots = { ...host.modelSlotCounts, ...slotCounts }; + const keptSlots = Object.fromEntries(Object.entries(mergedSlots).filter(([id]) => models.includes(id))); + const modelSlotCounts = Object.keys(keptSlots).length > 0 ? keptSlots : undefined; // sizesGB, unlike contextLengths, is populated for every model in the SAME pass (no // loaded-only restriction — see parseSizeGB), so this is closer to a plain replace, but // still merges onto the previous round rather than dropping a size for a model whose @@ -593,6 +708,7 @@ function applyDiscoveredModels(host: CustomModelHost, result: DiscoveryResult): models, defaultModelId, modelContextLengths, + modelSlotCounts, modelSizesGB, lastDiscoveredAt: new Date().toISOString(), }; diff --git a/src/web/routes/session-routes.ts b/src/web/routes/session-routes.ts index d8dbb40c..7d08288d 100644 --- a/src/web/routes/session-routes.ts +++ b/src/web/routes/session-routes.ts @@ -1535,6 +1535,7 @@ export function registerSessionRoutes( return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found'); } const contextLength = endpoint.modelContextLengths?.[body.modelId]; + const slotCount = endpoint.modelSlotCounts?.[body.modelId]; // Some CLIs (today: only claude) carry enough of their own fixed system-prompt/tool- // schema overhead that a small enough real context guarantees a first-message failure @@ -1545,12 +1546,13 @@ export function registerSessionRoutes( // `confirmed`, which still means both) — NOT by `confirmedSwap`: this warning is // about the caller's own session, and the swap warning below is about someone // else's, so an answer to one is not consent to the other. - if (!(body.confirmed || body.confirmedContext) && exceedsSafeContextFloor(entry, contextLength)) { + if (!(body.confirmed || body.confirmedContext) && exceedsSafeContextFloor(entry, contextLength, slotCount)) { return { requiresContextWarning: true, modelId: body.modelId, contextLength, minSafeContextTokens: CLAUDE_MIN_SAFE_CONTEXT_TOKENS, + ...(slotCount && slotCount > 1 ? { slotCount } : {}), }; } @@ -4332,6 +4334,7 @@ export function registerSessionRoutes( const cmEndpoint = cmHosts.find((h) => h.id === customModel.endpointId); if (!cmEndpoint) return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found'); const cmContextLength = cmEndpoint.modelContextLengths?.[customModel.modelId]; + const cmSlotCount = cmEndpoint.modelSlotCounts?.[customModel.modelId]; // See the dedicated route's own comment for the full reasoning: some CLIs' own fixed // overhead can exceed a small enough real context on the very first message, @@ -4341,13 +4344,14 @@ export function registerSessionRoutes( // `confirmedSwap`. if ( !(customModel.confirmed || customModel.confirmedContext) && - exceedsSafeContextFloor(cmEntry, cmContextLength) + exceedsSafeContextFloor(cmEntry, cmContextLength, cmSlotCount) ) { return { requiresContextWarning: true, modelId: customModel.modelId, contextLength: cmContextLength, minSafeContextTokens: CLAUDE_MIN_SAFE_CONTEXT_TOKENS, + ...(cmSlotCount && cmSlotCount > 1 ? { slotCount: cmSlotCount } : {}), }; } diff --git a/src/web/schemas.ts b/src/web/schemas.ts index 725760ef..623038ea 100644 --- a/src/web/schemas.ts +++ b/src/web/schemas.ts @@ -2274,6 +2274,8 @@ export const CustomModelHostSchema = z.object({ // round-tripping the GET response back through PUT (edit-save) doesn't drop it. modelContextLengths: z.record(z.string().max(200), z.number().int().positive().max(100_000_000)).optional(), // Same reasoning as modelContextLengths above. + modelSlotCounts: z.record(z.string().max(200), z.number().int().positive().max(4096)).optional(), + // Same reasoning as modelContextLengths above. modelSizesGB: z.record(z.string().max(200), z.number().positive().max(100_000)).optional(), }); diff --git a/test/custom-model-endpoint-rediscovery.test.ts b/test/custom-model-endpoint-rediscovery.test.ts index b0dedf7c..fd60f845 100644 --- a/test/custom-model-endpoint-rediscovery.test.ts +++ b/test/custom-model-endpoint-rediscovery.test.ts @@ -15,7 +15,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { getDataDir } from '../src/config/instance.js'; import { readCustomModelHosts, writeCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js'; -import { refreshAllCustomModelHosts } from '../src/web/routes/custom-model-routes.js'; +import { clearPropsProbeCache, refreshAllCustomModelHosts } from '../src/web/routes/custom-model-routes.js'; import { webviewFetch } from '../src/web/webview-egress.js'; vi.mock('../src/web/webview-egress.js', async () => { @@ -31,6 +31,7 @@ function host(overrides: Partial & Pick { fetchMock.mockReset(); + clearPropsProbeCache(); }); describe('refreshAllCustomModelHosts (the periodic re-discovery sweep)', () => { @@ -211,8 +212,9 @@ describe('refreshAllCustomModelHosts: context-length enrichment (llama.cpp/llama it('prefers the REAL configured context size parsed from /running’s launch command over /props’s unreliable n_ctx', async () => { // Confirmed live: llama-swap launched a model with --fit-ctx 16384 (the real, working // limit — the actual server then refused a request over it), but /props reported - // n_ctx: 154112 for the same model, well over what it would really accept. /props must - // never be reached at all once the /running command parse already answered it. + // n_ctx: 154112 for the same model, well over what it would really accept. The context + // must therefore keep coming from the /running command; /props is now also probed, but + // only for `total_slots`, which the command does not state. const dir = getDataDir(); await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]); fetchMock.mockImplementation(async (url: URL) => { @@ -235,7 +237,9 @@ describe('refreshAllCustomModelHosts: context-length enrichment (llama.cpp/llama { status: 200 } ); } - if (url.pathname === '/props') throw new Error('must never be reached — the cmd parse already answered it'); + if (url.pathname === '/props') { + return new Response(JSON.stringify({ n_ctx: 154112, total_slots: 4 }), { status: 200 }); + } throw new Error(`unexpected request: ${url.href}`); }); @@ -243,6 +247,8 @@ describe('refreshAllCustomModelHosts: context-length enrichment (llama.cpp/llama const [updated] = await readCustomModelHosts(dir); expect(updated.modelContextLengths).toEqual({ 'qwen3.8-27b': 16384 }); + expect(updated.modelSlotCounts).toEqual({ 'qwen3.8-27b': 4 }); + // /props's 154112 never becomes the context, and nothing else stores it }); it('falls back to /props when /running has no cmd, or the cmd states no recognizable context flag', async () => { @@ -283,13 +289,197 @@ describe('refreshAllCustomModelHosts: context-length enrichment (llama.cpp/llama { status: 200 } ); } - throw new Error(`unexpected request: ${url.href}`); // /props must never be reached + if (url.pathname === '/props') return new Response('not found', { status: 404 }); // slot probe only + throw new Error(`unexpected request: ${url.href}`); }); await refreshAllCustomModelHosts(); const [updated] = await readCustomModelHosts(dir); expect(updated.modelContextLengths).toEqual({ a: 8192 }); + expect(updated.modelSlotCounts).toBeUndefined(); + }); +}); + +describe('refreshAllCustomModelHosts: parallel-slot enrichment (llama.cpp splits its context across --parallel slots)', () => { + /** One loaded model whose launch command is `cmd`; `/props` behaviour is supplied by the test. */ + function serve(cmd: string, props: () => Response) { + fetchMock.mockImplementation(async (url: URL) => { + if (url.pathname === '/v1/models') { + return new Response(JSON.stringify({ data: [{ id: 'm', status: { value: 'loaded' } }] }), { status: 200 }); + } + if (url.pathname === '/running') { + return new Response(JSON.stringify({ running: [{ model: 'm', state: 'ready', cmd }] }), { status: 200 }); + } + if (url.pathname === '/props') return props(); + throw new Error(`unexpected request: ${url.href}`); + }); + } + const propsMustNotBeReached = () => { + throw new Error('/props must not be probed: the launch command already answered both questions'); + }; + + it.each([ + ['--parallel 1', 1], + ['-np 2', 2], + ['--parallel=3', 3], + ['--fit-ctx 65536 --parallel 1 --host 0.0.0.0', 1], + ])('reads an explicit %s from the launch command without probing /props', async (flags, slots) => { + const dir = getDataDir(); + await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]); + serve(`llama-server -m /models/m.gguf --ctx-size 65536 ${flags}`, propsMustNotBeReached); + + await refreshAllCustomModelHosts(); + + const [updated] = await readCustomModelHosts(dir); + expect(updated.modelSlotCounts).toEqual({ m: slots }); + expect(updated.modelContextLengths).toEqual({ m: 65536 }); + }); + + it('asks /props for total_slots when the command states no --parallel (llama-server picks the count itself)', async () => { + const dir = getDataDir(); + await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]); + serve( + 'llama-server -m /models/m.gguf --fit-ctx 65536', + () => new Response(JSON.stringify({ n_ctx: 133888, total_slots: 4 }), { status: 200 }) + ); + + await refreshAllCustomModelHosts(); + + const [updated] = await readCustomModelHosts(dir); + expect(updated.modelSlotCounts).toEqual({ m: 4 }); + // the context still comes from the command's floor, never from /props's n_ctx + expect(updated.modelContextLengths).toEqual({ m: 65536 }); + }); + + it('probes /props once per launch command: a repeat sweep reuses the answer, a changed command probes again', async () => { + const dir = getDataDir(); + await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]); + let probes = 0; + const answer = () => (probes++, new Response(JSON.stringify({ n_ctx: 133888, total_slots: 4 }), { status: 200 })); + serve('llama-server -m /models/m.gguf --fit-ctx 65536', answer); + await refreshAllCustomModelHosts(); + await refreshAllCustomModelHosts(); + expect(probes).toBe(1); // a /props call refreshes the model's last-use time in llama-swap, so not every sweep + expect((await readCustomModelHosts(dir))[0].modelSlotCounts).toEqual({ m: 4 }); + + serve('llama-server -m /models/m.gguf --fit-ctx 65536 --flash-attn on', answer); + await refreshAllCustomModelHosts(); + expect(probes).toBe(2); + }); + + it('does not divide a /props context a second time when --parallel is explicit and the command states no context', async () => { + const dir = getDataDir(); + await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]); + // with an explicit count llama.cpp reports n_ctx per slot: 65,536 each is already what one request gets + serve( + 'llama-server -m /models/m.gguf --parallel 4', + () => new Response(JSON.stringify({ n_ctx: 65536, total_slots: 4 }), { status: 200 }) + ); + + await refreshAllCustomModelHosts(); + + const [updated] = await readCustomModelHosts(dir); + expect(updated.modelContextLengths).toEqual({ m: 65536 }); + expect(updated.modelSlotCounts).toEqual({ m: 1 }); // already per slot, so one request gets all of it + }); + + it('a later round with an explicit --parallel replaces the slot count an earlier round learned', async () => { + const dir = getDataDir(); + await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]); + // round 1: auto slots, the server chose 4 sharing 131,072 tokens + serve( + 'llama-server -m /models/m.gguf', + () => new Response(JSON.stringify({ n_ctx: 131072, total_slots: 4 }), { status: 200 }) + ); + await refreshAllCustomModelHosts(); + expect((await readCustomModelHosts(dir))[0].modelSlotCounts).toEqual({ m: 4 }); + + // round 2: the user followed the dialog and added --parallel 1 + serve( + 'llama-server -m /models/m.gguf --parallel 1', + () => new Response(JSON.stringify({ n_ctx: 131072, total_slots: 1 }), { status: 200 }) + ); + await refreshAllCustomModelHosts(); + expect((await readCustomModelHosts(dir))[0].modelSlotCounts).toEqual({ m: 1 }); + + // round 3: --parallel 2 with a per-slot /props context must not bring a pool back + serve( + 'llama-server -m /models/m.gguf --parallel 2', + () => new Response(JSON.stringify({ n_ctx: 65536, total_slots: 2 }), { status: 200 }) + ); + await refreshAllCustomModelHosts(); + const [updated] = await readCustomModelHosts(dir); + expect(updated.modelSlotCounts).toEqual({ m: 1 }); + expect(updated.modelContextLengths).toEqual({ m: 65536 }); + }); + + it('remembers a /props that answers non-2xx per launch command, but retries a network failure', async () => { + const dir = getDataDir(); + await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]); + let probes = 0; + serve('llama-server -m /models/m.gguf --fit-ctx 65536', () => (probes++, new Response('nope', { status: 404 }))); + await refreshAllCustomModelHosts(); + await refreshAllCustomModelHosts(); + expect(probes).toBe(1); // no probe every sweep: a /props call keeps a model with a ttl loaded + + clearPropsProbeCache(); + probes = 0; + serve('llama-server -m /models/m.gguf --fit-ctx 65536', () => { + probes++; + throw new TypeError('fetch failed'); + }); + await refreshAllCustomModelHosts(); + await refreshAllCustomModelHosts(); + expect(probes).toBe(2); // a network error says nothing about the server, so it is tried again + }); + + it('treats -np -1 (llama.cpp auto) as unstated and falls back to /props', async () => { + const dir = getDataDir(); + await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]); + serve( + 'llama-server -m /models/m.gguf --fit-ctx 65536 -np -1', + () => new Response(JSON.stringify({ total_slots: 4 }), { status: 200 }) + ); + + await refreshAllCustomModelHosts(); + + const [updated] = await readCustomModelHosts(dir); + expect(updated.modelSlotCounts).toEqual({ m: 4 }); + }); + + it('a failed or slot-less /props leaves no slot count rather than failing the sweep or guessing one', async () => { + const dir = getDataDir(); + await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]); + serve( + 'llama-server -m /models/m.gguf --fit-ctx 65536', + () => new Response(JSON.stringify({ n_ctx: 8192 }), { status: 200 }) + ); + + await expect(refreshAllCustomModelHosts()).resolves.toBeUndefined(); + + const [updated] = await readCustomModelHosts(dir); + expect(updated.modelSlotCounts).toBeUndefined(); + expect(updated.modelContextLengths).toEqual({ m: 65536 }); + }); + + it('keeps a previously-learned slot count for a model that is no longer loaded, and drops it once the model disappears', async () => { + const dir = getDataDir(); + await writeCustomModelHosts(dir, [ + host({ + id: 'ep', + baseUrl: 'http://localhost:8080', + models: ['a', 'b'], + modelContextLengths: { a: 65536, b: 65536 }, + modelSlotCounts: { a: 4, b: 2 }, + }), + ]); + fetchMock.mockResolvedValue(new Response(JSON.stringify({ data: [{ id: 'a' }] }), { status: 200 })); + + await refreshAllCustomModelHosts(); + + const [updated] = await readCustomModelHosts(dir); + expect(updated.modelSlotCounts).toEqual({ a: 4 }); }); }); diff --git a/test/custom-model-run-menu-ui.test.ts b/test/custom-model-run-menu-ui.test.ts index c437822c..967e4457 100644 --- a/test/custom-model-run-menu-ui.test.ts +++ b/test/custom-model-run-menu-ui.test.ts @@ -1281,7 +1281,7 @@ describe("Custom Model Endpoint Profiles: requiresContextWarning (this CLI's own await app.runCustomModelEntry('claude', 'llama-box', 'qwen3'); - expect(confirmArgs).toEqual(['qwen3', 16384, 40000]); + expect(confirmArgs).toEqual(['qwen3', 16384, 40000, undefined]); expect(applyBodies).toEqual([ { endpointId: 'llama-box', modelId: 'qwen3' }, { endpointId: 'llama-box', modelId: 'qwen3', confirmedContext: true }, @@ -1289,6 +1289,29 @@ describe("Custom Model Endpoint Profiles: requiresContextWarning (this CLI's own expect(win.localStorage.getItem('codeman:customModelLastUsed:claude:llama-box')).toBe('qwen3'); }); + it('forwards the slot count from the apply response into the modal', async () => { + const { app, applyBodies } = launchHarness([ + { + requiresContextWarning: true, + modelId: 'qwen3', + contextLength: 131072, + minSafeContextTokens: 40000, + slotCount: 4, + }, + { customModel: { endpointId: 'llama-box' }, restarted: true, modelSwapInProgress: false }, + ]); + let confirmArgs: unknown[] | undefined; + app._confirmContextWarning = async (...args: unknown[]) => { + confirmArgs = args; + return true; + }; + + await app.runCustomModelEntry('claude', 'llama-box', 'qwen3'); + + expect(confirmArgs).toEqual(['qwen3', 131072, 40000, 4]); + expect(applyBodies).toHaveLength(2); + }); + it('declining the in-app context-warning modal keeps the native backend, never re-sends the apply, and must NEVER record this model as "last used" — it cannot work with this CLI at all', async () => { const { win, app, applyBodies } = launchHarness([ { requiresContextWarning: true, modelId: 'qwen3', contextLength: 16384, minSafeContextTokens: 40000 }, @@ -1361,6 +1384,49 @@ describe('Custom Model Endpoint Profiles: _confirmContextWarning (in-app modal, expect(win.document.getElementById('customModelContextWarningModal')!.classList.contains('active')).toBe(false); }); + it('explains shared parallel slots, and the --parallel 1 fix, when the context is enough only as a whole', async () => { + const { win, app } = bootApp({}); + void app._confirmContextWarning('qwen3.8-27b-ud-q4_k_xl', 131072, 40000, 4); + const message = win.document.getElementById('customModelContextWarningMessage')!.textContent!; + expect(message).toContain('qwen3.8-27b-ud-q4_k_xl'); + expect(message).toContain('4 parallel'); + expect(message).toContain('131,072'); + expect(message).toContain('32,768'); // per slot + expect(message).toContain('--parallel 1'); + expect(message).toContain('Context size has been exceeded'); + // this is not the "first message fails outright" case, so it must not claim that + expect(message).not.toMatch(/first message will fail/i); + expect(win.document.getElementById('customModelContextWarningModal')!.classList.contains('active')).toBe(true); + app._resolveContextWarningConfirm(false); + }); + + it('says the context is split across the slots, about N each, from the discovered context alone', async () => { + const { win, app } = bootApp({}); + void app._confirmContextWarning('qwen3', 65536, 40000, 4); + const message = win.document.getElementById('customModelContextWarningMessage')!.textContent!; + expect(message).toContain('65,536 tokens of context are split across 4 parallel server slots, about 16,384 each'); + app._resolveContextWarningConfirm(false); + }); + + it('keeps the too-small-context message for a small model and adds the slot advice when slots are known', async () => { + const { win, app } = bootApp({}); + void app._confirmContextWarning('qwen3', 16384, 40000, 4); + const message = win.document.getElementById('customModelContextWarningMessage')!.textContent!; + expect(message).toContain('16,384'); + expect(message).toMatch(/first message will fail/i); + expect(message).toContain('4 parallel slots'); + expect(message).toContain('--parallel 1'); + app._resolveContextWarningConfirm(false); + }); + + it('does not mention slots when there is one (or none known)', async () => { + const { win, app } = bootApp({}); + void app._confirmContextWarning('qwen3', 16384, 40000, 1); + const message = win.document.getElementById('customModelContextWarningMessage')!.textContent!; + expect(message).not.toMatch(/parallel/i); + app._resolveContextWarningConfirm(false); + }); + it('describes an unknown context length without printing a bogus number', async () => { const { win, app } = bootApp({}); void app._confirmContextWarning('qwen3', undefined, 40000); diff --git a/test/custom-model-slot-awareness.test.ts b/test/custom-model-slot-awareness.test.ts new file mode 100644 index 00000000..821ef91f --- /dev/null +++ b/test/custom-model-slot-awareness.test.ts @@ -0,0 +1,80 @@ +/** + * @fileoverview Unit tests for the parallel-slot awareness of the custom-model context floor + * (`perSlotContext` / `exceedsSafeContextFloor`, src/web/routes/custom-model-routes.ts). + * + * llama-server splits its context across `--parallel` slots (by default they share one + * unified KV pool) and Claude Code sends concurrent requests, each carrying its own ~40k + * tokens of system prompt and tools. Measured against a real llama-swap server: with 4 slots + * sharing ~131k tokens, four simultaneous 40k-token requests all failed with HTTP 500 + * "Context size has been exceeded"; the same requests against one slot all succeeded. + * + * Port: N/A (pure functions, no server). + */ +import { describe, it, expect } from 'vitest'; +import { + CLAUDE_MIN_SAFE_CONTEXT_TOKENS, + exceedsSafeContextFloor, + perSlotContext, +} from '../src/web/routes/custom-model-routes.js'; +import type { CliEntry } from '../src/config/cli-registry/types.js'; + +const withContextVar = { + capabilities: { customModelInjection: { kind: 'env', contextLengthVar: 'CLAUDE_CODE_MAX_CONTEXT_TOKENS' } }, +} as unknown as Pick; +const withoutContextVar = { + capabilities: { customModelInjection: { kind: 'env' } }, +} as unknown as Pick; + +describe('perSlotContext', () => { + it('divides the context across the slots', () => { + expect(perSlotContext(131072, 4)).toBe(32768); + expect(perSlotContext(262144, 4)).toBe(65536); + }); + + it('rounds down, since a fraction of a token is not usable', () => { + expect(perSlotContext(100000, 3)).toBe(33333); + }); + + it.each([undefined, 0, 1, -2, Number.NaN, Number.POSITIVE_INFINITY])( + 'leaves the context whole for an unusable or single slot count (%s)', + (slots) => { + expect(perSlotContext(65536, slots as number | undefined)).toBe(65536); + } + ); +}); + +describe('exceedsSafeContextFloor with a slot count', () => { + it('divides only the context it is given, so a larger /props figure can never skip the warning', () => { + // a --fit-ctx floor of 65,536 over 4 slots is 16,384 each, whatever /props might say about the server + expect(exceedsSafeContextFloor(withContextVar, 65536, 4)).toBe(true); + expect(exceedsSafeContextFloor(withContextVar, 65536, 1)).toBe(false); + // the measured failure: ~131k over 4 slots is ~33k each + expect(exceedsSafeContextFloor(withContextVar, 131072, 4)).toBe(true); + expect(exceedsSafeContextFloor(withContextVar, 262144, 4)).toBe(false); + }); + + it('still warns when the context itself is below the minimum', () => { + expect(exceedsSafeContextFloor(withContextVar, 16384, 4)).toBe(true); + expect(exceedsSafeContextFloor(withContextVar, 16384, 1)).toBe(true); + }); + + it('keeps its existing behaviour when no slot count is given', () => { + expect(exceedsSafeContextFloor(withContextVar, 16384)).toBe(true); + expect(exceedsSafeContextFloor(withContextVar, CLAUDE_MIN_SAFE_CONTEXT_TOKENS)).toBe(false); + expect(exceedsSafeContextFloor(withContextVar, undefined)).toBe(false); + }); + + it('flags a context that only clears the floor as a whole, not per slot', () => { + expect(exceedsSafeContextFloor(withContextVar, 131072, 4)).toBe(true); + expect(exceedsSafeContextFloor(withContextVar, 131072, 1)).toBe(false); + }); + + it('accepts exactly the floor per slot', () => { + expect(exceedsSafeContextFloor(withContextVar, CLAUDE_MIN_SAFE_CONTEXT_TOKENS * 4, 4)).toBe(false); + expect(exceedsSafeContextFloor(withContextVar, CLAUDE_MIN_SAFE_CONTEXT_TOKENS * 4 - 4, 4)).toBe(true); + }); + + it('never applies to a CLI that declares no context variable, whatever the slots', () => { + expect(exceedsSafeContextFloor(withoutContextVar, 4096, 8)).toBe(false); + }); +}); diff --git a/test/routes/quick-start-custom-model.test.ts b/test/routes/quick-start-custom-model.test.ts index 3bc0da96..17a0402f 100644 --- a/test/routes/quick-start-custom-model.test.ts +++ b/test/routes/quick-start-custom-model.test.ts @@ -316,6 +316,47 @@ describe('POST /api/quick-start: customModel (one-shot custom-model launch)', () expect(ctx.sessions.size).toBe(2); }); + describe('parallel slots share the context (the quick-start path has its own slot branch)', () => { + const slotEndpoint = (contextLength: number, slots: number): CustomModelHost => ({ + id: 'ep-slots', + label: 'multi-slot box', + baseUrl: 'http://192.168.1.52:8080', + apiKey: 'k', + modelContextLengths: { m: contextLength }, + modelSlotCounts: { m: slots }, + }); + + it('warns when the context divided across the slots is below the floor, and reports the slot count', async () => { + await writeCustomModelHosts(getDataDir(), [ENDPOINT, slotEndpoint(131072, 4)]); // ~33k per slot + + const res = await quickStart({ + caseName: 'cm-slots-warn', + mode: 'claude', + customModel: { endpointId: 'ep-slots', modelId: 'm' }, + }); + + const body = res.json(); + expect(body.requiresContextWarning).toBe(true); + expect(body.contextLength).toBe(131072); + expect(body.slotCount).toBe(4); + expect(ctx.sessions.size).toBe(1); // nothing was created + }); + + it('launches when every slot can hold the floor', async () => { + await writeCustomModelHosts(getDataDir(), [ENDPOINT, slotEndpoint(262144, 4)]); // 65,536 per slot + + const res = await quickStart({ + caseName: 'cm-slots-ok', + mode: 'claude', + customModel: { endpointId: 'ep-slots', modelId: 'm' }, + }); + + expect(res.statusCode).toBe(200); + expect(res.json().requiresContextWarning).toBeUndefined(); + expect(ctx.sessions.size).toBe(2); + }); + }); + it('does not warn when nothing about context was discovered', async () => { const res = await quickStart({ caseName: 'cm-no-ctx-data', diff --git a/test/routes/session-custom-model.test.ts b/test/routes/session-custom-model.test.ts index 1c29618d..9c8062d6 100644 --- a/test/routes/session-custom-model.test.ts +++ b/test/routes/session-custom-model.test.ts @@ -449,6 +449,78 @@ describe('POST /api/sessions/:id/custom-model', () => { expect(session.restartCli).not.toHaveBeenCalled(); }); + // llama-server splits its context across --parallel slots, and Claude Code sends several + // requests at once, so a context that clears the floor as a whole can still fail per slot. + describe('parallel slots share the context', () => { + const slotEndpoint = (contextLength: number, slots?: number): CustomModelHost => ({ + id: 'ep-slots', + label: 'multi-slot box', + baseUrl: 'http://192.168.1.52:8080', + apiKey: 'k', + modelContextLengths: { m: contextLength }, + ...(slots === undefined ? {} : { modelSlotCounts: { m: slots } }), + }); + + async function apply(endpoint: CustomModelHost) { + const { app, ctx } = await setup(); + await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, endpoint]); + const session = ctx.sessions.get('test-session-1')!; + session.mode = 'claude'; + const res = await app.inject({ + method: 'POST', + url: '/api/sessions/test-session-1/custom-model', + payload: { endpointId: endpoint.id, modelId: 'm' }, + }); + return { body: res.json(), session }; + } + + it('warns when the context divided across the slots is below the floor, and reports the slot count', async () => { + // the measured case: ~131k shared by 4 slots is ~33k each, under the 40k a request needs + const { body, session } = await apply(slotEndpoint(131072, 4)); + + expect(body.requiresContextWarning).toBe(true); + expect(body.contextLength).toBe(131072); + expect(body.slotCount).toBe(4); + expect(session.setCustomModel).not.toHaveBeenCalled(); + }); + + it('divides the discovered context across the slots even when it clears the floor as a whole', async () => { + // 65,536 over 4 slots is 16,384 each: nothing the server reports can widen this pool + const { body } = await apply(slotEndpoint(65536, 4)); + + expect(body.requiresContextWarning).toBe(true); + expect(body.contextLength).toBe(65536); + expect(body.slotCount).toBe(4); + }); + + it('does not warn once every slot can hold the floor', async () => { + const { body, session } = await apply(slotEndpoint(262144, 4)); // 65,536 each + + expect(body.requiresContextWarning).toBeUndefined(); + expect(session.setCustomModel).toHaveBeenCalledTimes(1); + }); + + it('does not warn for a single slot with the same context that failed across four', async () => { + const { body, session } = await apply(slotEndpoint(131072, 1)); + + expect(body.requiresContextWarning).toBeUndefined(); + expect(session.setCustomModel).toHaveBeenCalledTimes(1); + }); + + it('an unknown slot count is treated as one slot, never guessed', async () => { + const { body } = await apply(slotEndpoint(131072)); + + expect(body.requiresContextWarning).toBeUndefined(); + }); + + it('omits slotCount from the warning when the context is too small on its own with a single slot', async () => { + const { body } = await apply(slotEndpoint(16384, 1)); + + expect(body.requiresContextWarning).toBe(true); + expect(body.slotCount).toBeUndefined(); + }); + }); + it('applies once confirmed, skipping the context check the second time', async () => { const { app, ctx } = await setup(); await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, SMALL_CTX_ENDPOINT]);