Merge pull request #752: fix(custom-model): context limits, Codex model override, context discovery and picker filter

src/web/routes/custom-model-routes.ts

@@ -21,7 +21,7 @@ import { isMultiUserMode } from '../../config/multiuser.js';
 import { getDataDir } from '../../config/instance.js';
 import { isBlockedWebviewUrl } from '../webview-egress-policy.js';
 import { egressBlockedReason, webviewFetch } from '../webview-egress.js';
-import { CustomModelHostSchema } from '../schemas.js';
+import { CustomModelHostSchema, MAX_DISCOVERED_CONTEXT_TOKENS } from '../schemas.js';
 import { readCustomModelHosts, writeCustomModelHosts, type CustomModelHost } from '../../custom-model-hosts.js';
 import type { CliEntry } from '../../config/cli-registry/types.js';
 
@@ -158,9 +158,14 @@ function authHeaders(host: Pick<CustomModelHost, 'apiKey' | 'authStyle'>): Recor
 
 export interface DiscoveryResult {
   models: string[];
-  /** See `CustomModelHost.modelContextLengths` — only ever populated for models already loaded. */
+  /**
+   * See `CustomModelHost.modelContextLengths`: populated from a model's own `/v1/models`
+   * entry (`context_length`, else `context_window`) whenever the server states one, and
+   * from the launch command or `/props` for loaded models, which overwrite (and so win
+   * over) any listed figure for the same model.
+   */
   contextLengths: Record<string, number>;
-  /** See `CustomModelHost.modelSlotCounts`; populated on the same loaded-models-only basis as `contextLengths`. */
+  /** See `CustomModelHost.modelSlotCounts`; populated only for models the server reports loaded (launch command or `/props`). No listing states a slot count, unlike `contextLengths`. */
   slotCounts: Record<string, number>;
   /** See `CustomModelHost.modelSizesGB` — populated for every model whose own listing states one. */
   sizesGB: Record<string, number>;
@@ -182,6 +187,20 @@ function parseSizeGB(description: unknown): number | undefined {
   return Number.isFinite(size) && size > 0 ? size : undefined;
 }
 
+/**
+ * Best-effort: reads a context length (tokens) stated by a model's OWN entry in
+ * `GET /v1/models`. OpenRouter-style servers call the field `context_length`; current
+ * llama-swap sends both `context_length` and `context_window` per model, and current
+ * Bifrost (a gateway in front of llama-swap) passes `context_window` through. Older
+ * servers send neither, and a server that states nothing gets no entry, never a guess.
+ * Non-numeric, zero, negative, fractional or over-the-ceiling values are ignored, the
+ * same rule the stored field's schema enforces: a wrong figure here is worse than none.
+ */
+function parseListedContextLength(value: unknown): number | undefined {
+  if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) return undefined;
+  return value <= MAX_DISCOVERED_CONTEXT_TOKENS ? value : undefined;
+}
+
 /**
  * Best-effort: fetches `GET /props?model=<id>` (llama.cpp-native, llama-swap-proxied) for
  * ONE already-loaded model and pulls its `n_ctx` and `total_slots` out. Never called for a model that
@@ -312,25 +331,42 @@ async function discoverModels(
   });
   if (!res.ok) throw new Error(`HTTP ${res.status}`);
   const body = (await res.json()) as {
-    data?: Array<{ id?: unknown; status?: { value?: unknown }; description?: unknown }>;
+    data?: Array<{
+      id?: unknown;
+      status?: { value?: unknown };
+      description?: unknown;
+      context_length?: unknown;
+      context_window?: unknown;
+    }>;
   };
   const entries = body.data ?? [];
   const models = entries.map((m) => m.id).filter((id): id is string => typeof id === 'string' && id.length > 0);
 
   const sizesGB: Record<string, number> = {};
+  // The lowest-precedence context source, and the only one a gateway without /running or
+  // /props of its own (e.g. Bifrost) can offer: the figure a model's own listing states
+  // (`context_length`, else `context_window`). The loaded-model pass below OVERWRITES
+  // these for whatever it reads from the launch command or /props, so precedence per
+  // model stays: `/running` cmd, then `/props`, then this listed figure.
+  const contextLengths: Record<string, number> = {};
   for (const entry of entries) {
     if (typeof entry.id !== 'string' || !entry.id) continue;
     const size = parseSizeGB(entry.description);
     if (size !== undefined) sizesGB[entry.id] = size;
+    const listed = parseListedContextLength(entry.context_length) ?? parseListedContextLength(entry.context_window);
+    if (listed !== undefined) contextLengths[entry.id] = listed;
   }
 
   // llama-swap-specific, feature-detected: a server that never mentions `status` on ANY
-  // entry gets no context-length enrichment at all, rather than treating "no status field"
-  // as "assume unloaded" — either reading is a guess, and skipping is the safe one, since
-  // fetchContextLength must only ever run against a model this server itself calls loaded.
+  // entry gets no launch-command/props context enrichment at all, rather than treating
+  // "no status field" as "assume unloaded"; either reading is a guess, and skipping is
+  // the safe one, since fetchContextLength must only ever run against a model this server
+  // itself calls loaded. Listed figures above are unaffected: reading them triggers nothing.
   const hasStatusField = entries.some((m) => m && typeof m === 'object' && 'status' in m);
-  const contextLengths: Record<string, number> = {};
   const slotCounts: Record<string, number> = {};
+  // Ids whose figure THIS round's loaded-model probe answered — the listed figure remains
+  // the source only for the ids NOT in this set (see the per-slot correction below).
+  const ctxFromProbe = new Set<string>();
   if (hasStatusField) {
     const loadedIds = entries
       .filter((m) => m.status && typeof m.status === 'object' && (m.status as { value?: unknown }).value === 'loaded')
@@ -364,11 +400,30 @@ async function discoverModels(
           // merges onto what was learned before).
           if (cmdSlots !== undefined && ctxFromProps) slots = 1;
         }
-        if (ctx !== undefined) contextLengths[id] = ctx;
+        // Assigning only when a probe answered keeps the listed figure (if any) when
+        // neither the launch command nor /props stated one, and replaces it with the
+        // stronger source whenever they did: cmd > /props > listed, never the reverse.
+        if (ctx !== undefined) {
+          contextLengths[id] = ctx;
+          ctxFromProbe.add(id);
+        }
         if (slots !== undefined) slotCounts[id] = slots;
       }
     }
   }
+  // llama-swap documents the LISTED `context_length` as the PER-SLOT window, so whenever
+  // the listed figure is the one taken this round — the model was not loaded, or the
+  // probe answered nothing — its slot count is 1 by definition: the floor check must NOT
+  // divide a per-slot figure a second time. Without recording 1 here, the count a loaded
+  // round stored survives the merge (applyDiscoveredModels keeps what a later round does
+  // not overwrite): 262144 @ 4 slots stored, model unloaded, listed 65536 → 65536 with 4
+  // slots → a false 16384-per-slot warning. Even a count this round's launch command
+  // parsed must yield — pairing a per-slot listed figure with the cmd's --parallel is the
+  // same double division. Same rule the /props branch above applies. Ids whose figure the
+  // probe DID answer keep their probe-derived count untouched.
+  for (const id of Object.keys(contextLengths)) {
+    if (!ctxFromProbe.has(id)) slotCounts[id] = 1;
+  }
   return { models, contextLengths, slotCounts, sizesGB };
 }
 

src/web/routes/session-routes.ts

@@ -70,8 +70,9 @@ import {
   SessionWaitOutputQuerySchema,
   CustomModelSelectionSchema,
 } from '../schemas.js';
-import { readCustomModelHosts } from '../../custom-model-hosts.js';
+import { readCustomModelHosts, resolveCustomModelContextLength } from '../../custom-model-hosts.js';
 import { applyCustomModelInjection, removeConfigDir } from '../../custom-model-injection-apply.js';
+import { buildCustomModelInjection } from '../../custom-model-injection.js';
 import {
   getLlamaSwapStatus,
   triggerLlamaSwapLoad,
@@ -1541,7 +1542,10 @@ export function registerSessionRoutes(
     if (!endpoint) {
       return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found');
     }
-    const contextLength = endpoint.modelContextLengths?.[body.modelId];
+    // Discovery's per-model figure wins; the endpoint's user-set default covers models
+    // whose size discovery could not determine (a gateway that states none, say). With
+    // neither, the existing "no override, CLI assumes its own default" behaviour stands.
+    const contextLength = resolveCustomModelContextLength(endpoint, body.modelId);
     const slotCount = endpoint.modelSlotCounts?.[body.modelId];
 
     // Some CLIs (today: only claude) carry enough of their own fixed system-prompt/tool-
@@ -1608,10 +1612,28 @@ export function registerSessionRoutes(
     }
 
     // A CLI whose config alone cannot select the model also gets its `model` launch param
-    // forced (pi/omp `custom/<id>`, grok's block name). The argv engine DROPS a token that
+    // forced (pi/omp `custom/<id>`, grok's block name, and codex now carries its bare
+    // selected id too). The argv engine DROPS a token that
     // fails its pattern rather than quoting it, which would silently launch the CLI on its
     // own default provider again, so refuse an id the pattern cannot carry up front.
+    // The check runs on the PURE build result, before applyCustomModelInjection below:
+    // the forced launch id only comes out of that build, and writing first would mean a
+    // refused id already replaced the config dir a live codex session uses as CODEX_HOME.
     const modelSpec = entry.launch.params.model;
+    const injection = buildCustomModelInjection(entry, endpoint, body.modelId, contextLength);
+    if (injection.kind === 'unsupported') {
+      return createErrorResponse(ApiErrorCode.OPERATION_FAILED, `${session.mode} has no known custom-model mechanism`);
+    }
+    if (
+      injection.launchModel !== undefined &&
+      modelSpec?.type === 'token' &&
+      !matchesPattern(modelSpec.pattern, injection.launchModel)
+    ) {
+      return createErrorResponse(
+        ApiErrorCode.INVALID_INPUT,
+        `Model id ${JSON.stringify(body.modelId)} cannot be passed to ${session.mode} on its command line`
+      );
+    }
     const applied = applyCustomModelInjection(
       entry,
       endpoint,
@@ -1623,17 +1645,6 @@ export function registerSessionRoutes(
     if (!applied) {
       return createErrorResponse(ApiErrorCode.OPERATION_FAILED, `${session.mode} has no known custom-model mechanism`);
     }
-    if (
-      applied.launchModel !== undefined &&
-      modelSpec?.type === 'token' &&
-      !matchesPattern(modelSpec.pattern, applied.launchModel)
-    ) {
-      removeConfigDir(applied.configDir);
-      return createErrorResponse(
-        ApiErrorCode.INVALID_INPUT,
-        `Model id ${JSON.stringify(body.modelId)} cannot be passed to ${session.mode} on its command line`
-      );
-    }
 
     const { previousConfigDir } = session.setCustomModel(
       {
@@ -4447,7 +4458,9 @@ export function registerSessionRoutes(
       const cmHosts = await readCustomModelHosts(CODEMAN_CONFIG_DIR);
       const cmEndpoint = cmHosts.find((h) => h.id === customModel.endpointId);
       if (!cmEndpoint) return createErrorResponse(ApiErrorCode.NOT_FOUND, 'Model endpoint not found');
-      const cmContextLength = cmEndpoint.modelContextLengths?.[customModel.modelId];
+      // Per-model discovery figure first, user-set endpoint default second, nothing third
+      // (see the dedicated route's own comment above).
+      const cmContextLength = resolveCustomModelContextLength(cmEndpoint, customModel.modelId);
       const cmSlotCount = cmEndpoint.modelSlotCounts?.[customModel.modelId];
 
       // See the dedicated route's own comment for the full reasoning: some CLIs' own fixed