From a5a1b6f9a267921ad76986dbe3e82371a0fe9235 Mon Sep 17 00:00:00 2001 From: Codeman maintainer Date: Wed, 11 Nov 2026 09:53:12 +0100 Subject: [PATCH] fix(custom-model): pi is told the endpoint's context window A pi session on a custom endpoint got a model list entry with only the model id. pi then takes the model for a 128,000-token one (its footer reads "/128k") and asks for 16,384 reply tokens, so on a 65,536 or 32,768 window it never compacts before the server's real window is full. The entry now carries `contextWindow` (the full window) and `maxTokens` (a quarter of it, 8,192 at most) whenever the endpoint's context length is known: the same two figures the DeepSeek Harness patch already carries, now from one helper. An unknown length writes neither, as before. Measured on pi 1.5.2: with both fields `pi --list-models` shows the window, and the request carries `max_tokens: 8192`. Co-Authored-By: Claude Fable 5.2 --- src/custom-model-injection.ts | 36 ++++++++++++++++++----------- test/custom-model-injection.test.ts | 24 ++++++++++++++++--- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/custom-model-injection.ts b/src/custom-model-injection.ts index 5bee035c..9f230ccb 100644 --- a/src/custom-model-injection.ts +++ b/src/custom-model-injection.ts @@ -194,6 +194,21 @@ function opencodeModelLimit(contextLength: number | undefined): { context: numbe return { context: total, output }; } +/** + * The two fields pi's model list and the dsh provider patch (which is built on pi's model + * layer) take for a model's window: `contextWindow`, the FULL window, and `maxTokens`, the + * reply's share of it (a quarter, capped at 8192 on a large window). Empty for an unknown or + * non-finite size: the CLI then keeps its own default, and no limit is invented here. + */ +function modelWindowFields(contextLength: number | undefined): { contextWindow?: number; maxTokens?: number } { + if (contextLength === undefined || !Number.isFinite(contextLength) || contextLength <= 0) return {}; + return { + contextWindow: Math.trunc(contextLength), + // A quarter of the window keeps the reply inside the prompt's share; 8192 caps it on a large window. + maxTokens: Math.max(1, Math.min(8192, Math.floor(contextLength / 4))), + }; +} + function renderConfigContent( template: 'opencode-json', endpoint: CustomModelEndpoint, @@ -260,19 +275,7 @@ function renderConfigFile( apiKeyEnv: CUSTOM_MODEL_API_KEY_ENV_VAR, api: 'openai-completions', baseURL: baseUrl, - models: [ - { - id: modelId, - name: modelId, - ...(contextLength !== undefined && Number.isFinite(contextLength) && contextLength > 0 - ? { - contextWindow: Math.trunc(contextLength), - // A quarter of the window keeps the reply inside the prompt's share; 8192 caps it on a large window. - maxTokens: Math.max(1, Math.min(8192, Math.floor(contextLength / 4))), - } - : {}), - }, - ], + models: [{ id: modelId, name: modelId, ...modelWindowFields(contextLength) }], }, }, }, @@ -327,6 +330,11 @@ function renderConfigFile( // true` is required too: pi does not automatically send `Authorization: Bearer // ` just because `apiKey` is set (per the same doc) — without it, a real // (non-llama.cpp) endpoint that actually checks the key would reject every request. + // The model entry carries the endpoint's context window when it is known. Without it pi + // takes an unknown model for a 128,000-token one (its footer reads `/128k`) and asks for + // 16,384 reply tokens, so on a 65,536 or 32,768 window it never compacts before the + // server's real window is full. Measured on pi 1.5.2: with both fields `pi --list-models` + // shows the window and the request carries `max_tokens: 8192`. return { content: JSON.stringify( { @@ -336,7 +344,7 @@ function renderConfigFile( apiKey, api: 'openai-completions', authHeader: true, - models: [{ id: modelId }], + models: [{ id: modelId, ...modelWindowFields(contextLength) }], }, }, }, diff --git a/test/custom-model-injection.test.ts b/test/custom-model-injection.test.ts index b88dbbab..05b85b7e 100644 --- a/test/custom-model-injection.test.ts +++ b/test/custom-model-injection.test.ts @@ -194,6 +194,24 @@ describe('buildCustomModelInjection', () => { expect(parsed.providers.custom.models).toEqual([{ id: 'qwen3' }]); // array, NOT keyed by id }); + it('pi: the model entry carries the endpoint window and a reply share of it, or pi takes the model for a 128k one', () => { + const models = (contextLength?: number) => { + const result = buildCustomModelInjection(entryOrThrow('pi'), endpoint, 'qwen3.6:35b-a3b', contextLength); + if (result.kind !== 'configDir') throw new Error('unreachable'); + return JSON.parse(result.files[0].content).providers.custom.models; + }; + // The full window, and 8192 reply tokens at most (measured on pi 1.5.2: the request then + // carries max_tokens 8192 instead of its own 16384). + expect(models(65536)).toEqual([{ id: 'qwen3.6:35b-a3b', contextWindow: 65536, maxTokens: 8192 }]); + expect(models(131072.9)).toEqual([{ id: 'qwen3.6:35b-a3b', contextWindow: 131072, maxTokens: 8192 }]); + // A small window keeps three quarters for the prompt. + expect(models(16384)).toEqual([{ id: 'qwen3.6:35b-a3b', contextWindow: 16384, maxTokens: 4096 }]); + // Unknown stays unknown: no limit is invented. + for (const contextLength of [undefined, NaN, 0, -1, Infinity]) { + expect(models(contextLength), String(contextLength)).toEqual([{ id: 'qwen3.6:35b-a3b' }]); + } + }); + it('omp: configDir writes .omp/agent/models.yml, redirected via HOME (verified live end-to-end)', () => { const result = buildCustomModelInjection(entryOrThrow('omp'), endpoint, 'qwen3'); if (result.kind !== 'configDir') throw new Error('unreachable'); @@ -239,9 +257,9 @@ describe('buildCustomModelInjection', () => { // The patch references the DEDICATED var by name, never dsh's own DEEPSEEK_API_KEY: // dsh's llm-deepseek/web-search rows read that too and would leak this key to // api.deepseek.com on an in-session /model switch back to the official route. - expect(patch.find((row: { id: string }) => row.id === 'llm-pi-ai').config.providers['codeman-custom'].apiKeyEnv).toBe( - 'CODEMAN_CUSTOM_MODEL_API_KEY' - ); + expect( + patch.find((row: { id: string }) => row.id === 'llm-pi-ai').config.providers['codeman-custom'].apiKeyEnv + ).toBe('CODEMAN_CUSTOM_MODEL_API_KEY'); expect(result.files[0].content).not.toContain('DEEPSEEK_API_KEY'); expect(patch.find((row: { id: string }) => row.id === 'dsh-tui').config.model).toBe('qwen3'); });