fix(custom-model): warn when parallel llama.cpp slots leave too little context per request (#658)

Squash of PR #658. Discovery records each llama.cpp model's parallel slot
count (from --parallel/-np, or /props total_slots when the launch command
states none), and a Claude launch warns when the context split across
those slots leaves less than 40K tokens per request. An explicit
--parallel beside a /props context records a slot count of 1, so a later
round overwrites an earlier count, and a non-2xx /props is remembered per
launch command while a network error is retried.

Merge-time fix by the maintainer: the architecture-invariants sentence on
the explicit --parallel case now says it records a slot count of 1.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/web/public/session-ui.js

@@ -1114,11 +1114,33 @@ Object.assign(CodemanApp.prototype, {
    * history yet for compaction to trim. Same promise-based pattern as
    * `_confirmModelSwap`; `_resolveContextWarningConfirm` settles it.
    */
-  _confirmContextWarning(modelId, contextLength, minSafeContextTokens) {
+  _confirmContextWarning(modelId, contextLength, minSafeContextTokens, slotCount) {
     const modal = document.getElementById('customModelContextWarningModal');
     const messageEl = document.getElementById('customModelContextWarningMessage');
     if (messageEl) {
       const known = typeof contextLength === 'number';
+      const slots = Number.isFinite(slotCount) && slotCount > 1 ? Math.floor(slotCount) : 1;
+      // Enough total context, but split across parallel server slots: llama-server shares its
+      // context between --parallel slots, and this CLI sends concurrent requests (the main
+      // session plus each parallel subagent), so several 40k-token requests spend the shared
+      // pool and the server answers HTTP 500 "Context size has been exceeded".
+      if (known && slots > 1 && contextLength >= minSafeContextTokens) {
+        messageEl.textContent =
+          `${modelId}'s ${contextLength.toLocaleString()} tokens of context are split across ${slots} parallel ` +
+          `server slots, about ${Math.floor(contextLength / slots).toLocaleString()} each. This CLI ` +
+          `needs roughly ${minSafeContextTokens.toLocaleString()}+ tokens just for its own system prompt and tools on ` +
+          `every request, and it sends several requests at once (the main session plus each parallel subagent). ` +
+          `Concurrent requests can use up the shared context, and the server then fails them with ` +
+          `"Context size has been exceeded".\n\n` +
+          `To fix this, start the model with a single slot: add "--parallel 1" to its llama-swap cmd. The active ` +
+          `conversation then gets the whole context, other requests queue instead of failing, and the CLI's ` +
+          `repeated prompt prefix stays cached. Alternatively give the model a larger context (raise --fit-ctx or ` +
+          `--ctx-size) so every slot has at least ${minSafeContextTokens.toLocaleString()} tokens.`;
+        modal?.classList.add('active');
+        return new Promise((resolve) => {
+          this._resolveContextWarningConfirmPromise = resolve;
+        });
+      }
       messageEl.textContent =
         `${modelId} is configured with ` +
         (known ? `only ${contextLength.toLocaleString()} tokens of` : 'an unknown (too small)') +
@@ -1129,7 +1151,11 @@ Object.assign(CodemanApp.prototype, {
         `instead of relying on auto-fit (--fit-ctx), which optimizes for the biggest MODEL that fits, not the ` +
         `biggest CONTEXT — e.g. add "-c 65536" (or as large a --ctx-size as your hardware holds) to its llama-swap ` +
         `config entry. A smaller model at a much larger explicit context often fits in the same VRAM a bigger ` +
-        `model's auto-fit context gets shrunk to make room for.`;
+        `model's auto-fit context gets shrunk to make room for.` +
+        (slots > 1
+          ? `\n\nThis server also runs ${slots} parallel slots that share that context; add "--parallel 1" to ` +
+            `the same cmd so a single conversation gets all of it.`
+          : '');
     }
     modal?.classList.add('active');
     return new Promise((resolve) => {
@@ -1254,8 +1280,13 @@ Object.assign(CodemanApp.prototype, {
     let answered = {};
     let data = await post(bodyObj);
     if (data?.data?.requiresContextWarning) {
-      const { modelId, contextLength, minSafeContextTokens } = data.data;
-      const proceed = await this._confirmContextWarning(modelId, contextLength, minSafeContextTokens);
+      const { modelId, contextLength, minSafeContextTokens, slotCount } = data.data;
+      const proceed = await this._confirmContextWarning(
+        modelId,
+        contextLength,
+        minSafeContextTokens,
+        slotCount
+      );
       if (!proceed) {
         this._lastCustomModelLaunchResult = undefined;
         return { success: false, error: 'Launch cancelled — context window too small' };
@@ -1371,7 +1402,8 @@ Object.assign(CodemanApp.prototype, {
       const proceed = await this._confirmContextWarning(
         payload.modelId,
         payload.contextLength,
-        payload.minSafeContextTokens
+        payload.minSafeContextTokens,
+        payload.slotCount
       );
       if (!proceed) {
         switchingToast?.dismiss();

src/web/routes/custom-model-routes.ts

@@ -56,11 +56,31 @@ export const CLAUDE_MIN_SAFE_CONTEXT_TOKENS = 40000;
  */
 export function exceedsSafeContextFloor(
   entry: Pick<CliEntry, 'capabilities'>,
-  contextLength: number | undefined
+  contextLength: number | undefined,
+  slotCount?: number
 ): boolean {
   const cap = entry.capabilities.customModelInjection;
   if (cap.kind !== 'env' || !cap.contextLengthVar) return false;
-  return typeof contextLength === 'number' && contextLength < CLAUDE_MIN_SAFE_CONTEXT_TOKENS;
+  if (typeof contextLength !== 'number') return false;
+  if (contextLength < CLAUDE_MIN_SAFE_CONTEXT_TOKENS) return true;
+  return perSlotContext(contextLength, slotCount) < CLAUDE_MIN_SAFE_CONTEXT_TOKENS;
+}
+
+/**
+ * The context one request can rely on when the server runs `slotCount` parallel slots.
+ * llama-server divides its context across `--parallel` slots (by default they share ONE
+ * unified KV pool), and Claude Code sends concurrent requests: the main session plus every
+ * parallel subagent, each carrying the full `CLAUDE_MIN_SAFE_CONTEXT_TOKENS` of its own
+ * system prompt and tools. With N busy slots each request therefore gets roughly
+ * `contextLength / N`, and once the pool is spent llama.cpp answers HTTP 500 "Context size
+ * has been exceeded" (measured: 4 slots, ~131k pool, four simultaneous 40k-token requests
+ * all failed; the same requests against one slot all succeeded). An unknown or single slot
+ * count leaves the context whole.
+ */
+export function perSlotContext(contextLength: number, slotCount: number | undefined): number {
+  const slots =
+    typeof slotCount === 'number' && Number.isFinite(slotCount) && slotCount > 1 ? Math.floor(slotCount) : 1;
+  return Math.floor(contextLength / slots);
 }
 
 function adminOnly(req: FastifyRequest, reply: { code: (n: number) => unknown }): ApiResponse<never> | null {
@@ -109,7 +129,7 @@ function applyStoredApiKey(incoming: CustomModelHost, existing: CustomModelHost)
 }
 
 /**
- * `modelContextLengths`/`modelSizesGB` are server-populated by discovery, never
+ * `modelContextLengths`/`modelSlotCounts`/`modelSizesGB` are server-populated by discovery, never
  * user-entered, and PUT replaces the whole record — so merge them back in from the
  * stored host rather than trust whatever the editor's body carried (or omitted).
  * The editor only ever sends `models`/`lastDiscoveredAt` verbatim from its cached
@@ -117,7 +137,12 @@ function applyStoredApiKey(incoming: CustomModelHost, existing: CustomModelHost)
  * future caller forgets, same class of bug `applyStoredApiKey` exists to prevent.
  */
 function applyDiscoveredFields(incoming: CustomModelHost, existing: CustomModelHost): CustomModelHost {
-  return { ...incoming, modelContextLengths: existing.modelContextLengths, modelSizesGB: existing.modelSizesGB };
+  return {
+    ...incoming,
+    modelContextLengths: existing.modelContextLengths,
+    modelSlotCounts: existing.modelSlotCounts,
+    modelSizesGB: existing.modelSizesGB,
+  };
 }
 
 function authHeaders(host: Pick<CustomModelHost, 'apiKey' | 'authStyle'>): Record<string, string> {
@@ -135,6 +160,8 @@ export interface DiscoveryResult {
   models: string[];
   /** See `CustomModelHost.modelContextLengths` — only ever populated for models already loaded. */
   contextLengths: Record<string, number>;
+  /** See `CustomModelHost.modelSlotCounts`; populated on the same loaded-models-only basis as `contextLengths`. */
+  slotCounts: Record<string, number>;
   /** See `CustomModelHost.modelSizesGB` — populated for every model whose own listing states one. */
   sizesGB: Record<string, number>;
 }
@@ -157,7 +184,7 @@ function parseSizeGB(description: unknown): number | undefined {
 
 /**
  * Best-effort: fetches `GET /props?model=<id>` (llama.cpp-native, llama-swap-proxied) for
- * ONE already-loaded model and pulls its real `n_ctx` out. Never called for a model that
+ * ONE already-loaded model and pulls its `n_ctx` and `total_slots` out. Never called for a model that
  * isn't already loaded — see the caller and `CustomModelHost.modelContextLengths` for why
  * that's a hard safety requirement, not just a nicety: llama-swap treats this endpoint's
  * `?model=` as a routing hint, and asking it about an unloaded model risks triggering an
@@ -173,20 +200,39 @@ function parseSizeGB(description: unknown): number | undefined {
  * (below), which reads the actual launch flag `/running` reports, is the primary source;
  * this is only used when that parse comes up empty (no recognized flag in `cmd`, or `cmd`
  * itself unavailable).
+ *
+ * ⚠️ `/props` only ever RAISES the slot warning, never skips it: its `n_ctx` is the context
+ * fallback when `cmd` states no context flag at all, and `total_slots` is the slot count when
+ * `cmd` has no `--parallel`. It never widens the pool the slot check divides. And it is not
+ * free: llama-swap routes it through the model's process handler, which refreshes the model's
+ * last-use time, so probing a model with a `ttl` on every 5-minute sweep would keep it loaded
+ * forever. `discoverModels` therefore probes once per launch command (`propsByCmd`).
+ *
+ * Returns `undefined` when the request itself failed (network error, timeout, unreadable body),
+ * which is retried next sweep; a non-2xx status, or a 2xx without the fields, gives an object with
+ * them unset, which `propsForLaunch` remembers.
  */
-async function fetchContextLength(
+async function fetchServerProps(
   host: Pick<CustomModelHost, 'baseUrl'>,
   modelId: string,
   headers: Record<string, string>
-): Promise<number | undefined> {
+): Promise<{ nCtx?: number; totalSlots?: number } | undefined> {
   try {
     const url = new URL(`${host.baseUrl.replace(/\/+$/, '')}/props`);
     url.searchParams.set('model', modelId);
     const res = await webviewFetch(url, { headers, signal: AbortSignal.timeout(PROPS_TIMEOUT_MS) });
-    if (!res.ok) return undefined;
-    const body = (await res.json()) as { n_ctx?: unknown; default_generation_settings?: { n_ctx?: unknown } };
+    // A non-2xx answer (an upstream that does not serve /props) is a real answer: empty, so it is
+    // remembered per launch command instead of being re-probed every sweep.
+    if (!res.ok) return {};
+    const body = (await res.json()) as {
+      n_ctx?: unknown;
+      total_slots?: unknown;
+      default_generation_settings?: { n_ctx?: unknown };
+    };
     const nCtx = body.n_ctx ?? body.default_generation_settings?.n_ctx;
-    return typeof nCtx === 'number' && Number.isFinite(nCtx) && nCtx > 0 ? nCtx : undefined;
+    const positive = (v: unknown): number | undefined =>
+      typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : undefined;
+    return { nCtx: positive(nCtx), totalSlots: positive(body.total_slots) };
   } catch {
     return undefined;
   }
@@ -196,7 +242,7 @@ async function fetchContextLength(
  * Parses the REAL configured context size out of llama-swap's own launch command for a
  * model (`/running`'s `cmd` field, e.g. `"llama-server -m ... --fit-ctx 16384 ..."`) —
  * the primary source for `modelContextLengths`, preferred over `/props`'s `n_ctx` (see
- * `fetchContextLength`'s own doc comment for why that field is unreliable here). Checks
+ * `fetchServerProps`'s own doc comment for why that field is unreliable here). Checks
  * `--fit-ctx` first (llama-swap's own auto-fit flag), then the plain llama.cpp
  * `-c`/`--ctx-size`/`--ctx_size` flags a hand-written launch command might use instead.
  * Returns `undefined` when `cmd` has none of these — not every launch command needs to
@@ -211,6 +257,51 @@ function parseCtxFromCmd(cmd: unknown): number | undefined {
   return Number.isFinite(value) && value > 0 ? value : undefined;
 }
 
+/**
+ * Parses an explicit `--parallel N` / `-np N` (or `--parallel=N`) out of a launch command.
+ * Returns `undefined` when the command states none, including `-np -1` (llama.cpp's "pick
+ * automatically"), because a llama-server started without it chooses the slot count itself
+ * (4 on the build this was measured on) and only `/props`'s `total_slots` can say what it chose.
+ */
+function parseSlotsFromCmd(cmd: unknown): number | undefined {
+  if (typeof cmd !== 'string') return undefined;
+  const match = /(?:^|\s)(?:-np|--parallel)(?:\s+|=)(\d+)(?=\s|$)/.exec(cmd);
+  if (!match) return undefined;
+  const value = Number(match[1]);
+  return Number.isFinite(value) && value > 0 ? value : undefined;
+}
+
+/** The `/props` answer for the launch command a model was last probed under, by host and model id. */
+const propsByCmd = new Map<string, { cmd: string; props: { nCtx?: number; totalSlots?: number } }>();
+
+/** Forget what was probed (tests only: a changed launch command already probes again). */
+export function clearPropsProbeCache(): void {
+  propsByCmd.clear();
+}
+
+/**
+ * `/props` for one loaded model, but only once per launch command: the probe refreshes the model's
+ * last-use time in llama-swap, so repeating it on every rediscovery sweep would stop a model with a
+ * `ttl` from ever unloading. A changed `cmd` (a reloaded or reconfigured model) probes again; a
+ * `cmd` that is not a string cannot be told apart, so it is probed each time; a probe that did not
+ * answer (network error, timeout) is not remembered; a non-2xx status is. Held in memory only, so a restart probes once more.
+ */
+async function propsForLaunch(
+  host: Pick<CustomModelHost, 'baseUrl'>,
+  modelId: string,
+  cmd: unknown,
+  headers: Record<string, string>
+): Promise<{ nCtx?: number; totalSlots?: number } | undefined> {
+  const key = `${host.baseUrl}\0${modelId}`;
+  if (typeof cmd === 'string') {
+    const cached = propsByCmd.get(key);
+    if (cached?.cmd === cmd) return cached.props;
+  }
+  const props = await fetchServerProps(host, modelId, headers);
+  if (props && typeof cmd === 'string') propsByCmd.set(key, { cmd, props });
+  return props;
+}
+
 async function discoverModels(
   host: Pick<CustomModelHost, 'baseUrl' | 'apiKey' | 'authStyle'>
 ): Promise<DiscoveryResult> {
@@ -239,6 +330,7 @@ async function discoverModels(
   // fetchContextLength must only ever run against a model this server itself calls loaded.
   const hasStatusField = entries.some((m) => m && typeof m === 'object' && 'status' in m);
   const contextLengths: Record<string, number> = {};
+  const slotCounts: Record<string, number> = {};
   if (hasStatusField) {
     const loadedIds = entries
       .filter((m) => m.status && typeof m.status === 'object' && (m.status as { value?: unknown }).value === 'loaded')
@@ -252,13 +344,32 @@ async function discoverModels(
       const swapStatus = await getLlamaSwapStatus(host);
       const cmdById = new Map(swapStatus.running.map((r) => [r.model, r.cmd]));
       for (const id of loadedIds) {
-        const fromCmd = parseCtxFromCmd(cmdById.get(id));
-        const ctx = fromCmd ?? (await fetchContextLength(host, id, headers));
+        const cmd = cmdById.get(id);
+        let ctx = parseCtxFromCmd(cmd);
+        const cmdSlots = parseSlotsFromCmd(cmd);
+        let slots = cmdSlots;
+        // /props is only needed for whatever the launch command did not answer. The context
+        // still comes from the command whenever it states one (see fetchServerProps's doc
+        // comment for why /props's n_ctx is only a fallback); the slot count usually cannot
+        // come from the command at all, since an unset --parallel is chosen by the server.
+        if (ctx === undefined || slots === undefined) {
+          const props = await propsForLaunch(host, id, cmd, headers);
+          const ctxFromProps = ctx === undefined && props?.nCtx !== undefined;
+          ctx ??= props?.nCtx;
+          slots ??= props?.totalSlots;
+          // With an explicit --parallel N, llama.cpp reports /props's n_ctx PER SLOT (the unified
+          // KV pool is only the default when the count is auto), so it is already the figure one
+          // request can use, so the count is 1: recording N would divide it a second time, and
+          // recording nothing would leave an earlier round's count in place (applyDiscoveredModels
+          // merges onto what was learned before).
+          if (cmdSlots !== undefined && ctxFromProps) slots = 1;
+        }
         if (ctx !== undefined) contextLengths[id] = ctx;
+        if (slots !== undefined) slotCounts[id] = slots;
       }
     }
   }
-  return { models, contextLengths, sizesGB };
+  return { models, contextLengths, slotCounts, sizesGB };
 }
 
 /**
@@ -573,7 +684,7 @@ export function triggerLlamaSwapLoad(
 }
 
 function applyDiscoveredModels(host: CustomModelHost, result: DiscoveryResult): CustomModelHost {
-  const { models, contextLengths, sizesGB } = result;
+  const { models, contextLengths, slotCounts, sizesGB } = result;
   const defaultModelId = host.defaultModelId && models.includes(host.defaultModelId) ? host.defaultModelId : undefined;
   // Merge onto what's already known rather than replacing: a model not probed this round
   // (not currently loaded) keeps whatever context length an earlier round already learned
@@ -581,6 +692,10 @@ function applyDiscoveredModels(host: CustomModelHost, result: DiscoveryResult):
   const merged = { ...host.modelContextLengths, ...contextLengths };
   const kept = Object.fromEntries(Object.entries(merged).filter(([id]) => models.includes(id)));
   const modelContextLengths = Object.keys(kept).length > 0 ? kept : undefined;
+  // Slot counts follow the same loaded-only, keep-what-an-earlier-round-learned rule.
+  const mergedSlots = { ...host.modelSlotCounts, ...slotCounts };
+  const keptSlots = Object.fromEntries(Object.entries(mergedSlots).filter(([id]) => models.includes(id)));
+  const modelSlotCounts = Object.keys(keptSlots).length > 0 ? keptSlots : undefined;
   // sizesGB, unlike contextLengths, is populated for every model in the SAME pass (no
   // loaded-only restriction — see parseSizeGB), so this is closer to a plain replace, but
   // still merges onto the previous round rather than dropping a size for a model whose
@@ -593,6 +708,7 @@ function applyDiscoveredModels(host: CustomModelHost, result: DiscoveryResult):
     models,
     defaultModelId,
     modelContextLengths,
+    modelSlotCounts,
     modelSizesGB,
     lastDiscoveredAt: new Date().toISOString(),
   };

src/web/routes/session-routes.ts

@@ -1535,6 +1535,7 @@ export function registerSessionRoutes(
       return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found');
     }
     const contextLength = endpoint.modelContextLengths?.[body.modelId];
+    const slotCount = endpoint.modelSlotCounts?.[body.modelId];
 
     // Some CLIs (today: only claude) carry enough of their own fixed system-prompt/tool-
     // schema overhead that a small enough real context guarantees a first-message failure
@@ -1545,12 +1546,13 @@ export function registerSessionRoutes(
     // `confirmed`, which still means both) — NOT by `confirmedSwap`: this warning is
     // about the caller's own session, and the swap warning below is about someone
     // else's, so an answer to one is not consent to the other.
-    if (!(body.confirmed || body.confirmedContext) && exceedsSafeContextFloor(entry, contextLength)) {
+    if (!(body.confirmed || body.confirmedContext) && exceedsSafeContextFloor(entry, contextLength, slotCount)) {
       return {
         requiresContextWarning: true,
         modelId: body.modelId,
         contextLength,
         minSafeContextTokens: CLAUDE_MIN_SAFE_CONTEXT_TOKENS,
+        ...(slotCount && slotCount > 1 ? { slotCount } : {}),
       };
     }
 
@@ -4332,6 +4334,7 @@ export function registerSessionRoutes(
       const cmEndpoint = cmHosts.find((h) => h.id === customModel.endpointId);
       if (!cmEndpoint) return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found');
       const cmContextLength = cmEndpoint.modelContextLengths?.[customModel.modelId];
+      const cmSlotCount = cmEndpoint.modelSlotCounts?.[customModel.modelId];
 
       // See the dedicated route's own comment for the full reasoning: some CLIs' own fixed
       // overhead can exceed a small enough real context on the very first message,
@@ -4341,13 +4344,14 @@ export function registerSessionRoutes(
       // `confirmedSwap`.
       if (
         !(customModel.confirmed || customModel.confirmedContext) &&
-        exceedsSafeContextFloor(cmEntry, cmContextLength)
+        exceedsSafeContextFloor(cmEntry, cmContextLength, cmSlotCount)
       ) {
         return {
           requiresContextWarning: true,
           modelId: customModel.modelId,
           contextLength: cmContextLength,
           minSafeContextTokens: CLAUDE_MIN_SAFE_CONTEXT_TOKENS,
+          ...(cmSlotCount && cmSlotCount > 1 ? { slotCount: cmSlotCount } : {}),
         };
       }
 

src/web/schemas.ts

@@ -2274,6 +2274,8 @@ export const CustomModelHostSchema = z.object({
   // round-tripping the GET response back through PUT (edit-save) doesn't drop it.
   modelContextLengths: z.record(z.string().max(200), z.number().int().positive().max(100_000_000)).optional(),
   // Same reasoning as modelContextLengths above.
+  modelSlotCounts: z.record(z.string().max(200), z.number().int().positive().max(4096)).optional(),
+  // Same reasoning as modelContextLengths above.
   modelSizesGB: z.record(z.string().max(200), z.number().positive().max(100_000)).optional(),
 });