mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-14 10:39:32 +01:00
fix(custom-model): warn when parallel llama.cpp slots leave too little context per request (#658)
Squash of PR #658. Discovery records each llama.cpp model's parallel slot count (from --parallel/-np, or /props total_slots when the launch command states none), and a Claude launch warns when the context split across those slots leaves less than 40K tokens per request. An explicit --parallel beside a /props context records a slot count of 1, so a later round overwrites an earlier count, and a non-2xx /props is remembered per launch command while a network error is retried. Merge-time fix by the maintainer: the architecture-invariants sentence on the explicit --parallel case now says it records a slot count of 1. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
34cf798b10
commit
33d697afe3
17 files changed
+710
-34
No files matched your search
@@ -15,7 +15,7 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { getDataDir } from '../src/config/instance.js';
|
||||
import { readCustomModelHosts, writeCustomModelHosts, type CustomModelHost } from '../src/custom-model-hosts.js';
|
||||
import { refreshAllCustomModelHosts } from '../src/web/routes/custom-model-routes.js';
|
||||
import { clearPropsProbeCache, refreshAllCustomModelHosts } from '../src/web/routes/custom-model-routes.js';
|
||||
import { webviewFetch } from '../src/web/webview-egress.js';
|
||||
|
||||
vi.mock('../src/web/webview-egress.js', async () => {
|
||||
@@ -31,6 +31,7 @@ function host(overrides: Partial<CustomModelHost> & Pick<CustomModelHost, 'id' |
|
||||
|
||||
beforeEach(() => {
|
||||
fetchMock.mockReset();
|
||||
clearPropsProbeCache();
|
||||
});
|
||||
|
||||
describe('refreshAllCustomModelHosts (the periodic re-discovery sweep)', () => {
|
||||
@@ -211,8 +212,9 @@ describe('refreshAllCustomModelHosts: context-length enrichment (llama.cpp/llama
|
||||
it('prefers the REAL configured context size parsed from /running’s launch command over /props’s unreliable n_ctx', async () => {
|
||||
// Confirmed live: llama-swap launched a model with --fit-ctx 16384 (the real, working
|
||||
// limit — the actual server then refused a request over it), but /props reported
|
||||
// n_ctx: 154112 for the same model, well over what it would really accept. /props must
|
||||
// never be reached at all once the /running command parse already answered it.
|
||||
// n_ctx: 154112 for the same model, well over what it would really accept. The context
|
||||
// must therefore keep coming from the /running command; /props is now also probed, but
|
||||
// only for `total_slots`, which the command does not state.
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]);
|
||||
fetchMock.mockImplementation(async (url: URL) => {
|
||||
@@ -235,7 +237,9 @@ describe('refreshAllCustomModelHosts: context-length enrichment (llama.cpp/llama
|
||||
{ status: 200 }
|
||||
);
|
||||
}
|
||||
if (url.pathname === '/props') throw new Error('must never be reached — the cmd parse already answered it');
|
||||
if (url.pathname === '/props') {
|
||||
return new Response(JSON.stringify({ n_ctx: 154112, total_slots: 4 }), { status: 200 });
|
||||
}
|
||||
throw new Error(`unexpected request: ${url.href}`);
|
||||
});
|
||||
|
||||
@@ -243,6 +247,8 @@ describe('refreshAllCustomModelHosts: context-length enrichment (llama.cpp/llama
|
||||
|
||||
const [updated] = await readCustomModelHosts(dir);
|
||||
expect(updated.modelContextLengths).toEqual({ 'qwen3.8-27b': 16384 });
|
||||
expect(updated.modelSlotCounts).toEqual({ 'qwen3.8-27b': 4 });
|
||||
// /props's 154112 never becomes the context, and nothing else stores it
|
||||
});
|
||||
|
||||
it('falls back to /props when /running has no cmd, or the cmd states no recognizable context flag', async () => {
|
||||
@@ -283,13 +289,197 @@ describe('refreshAllCustomModelHosts: context-length enrichment (llama.cpp/llama
|
||||
{ status: 200 }
|
||||
);
|
||||
}
|
||||
throw new Error(`unexpected request: ${url.href}`); // /props must never be reached
|
||||
if (url.pathname === '/props') return new Response('not found', { status: 404 }); // slot probe only
|
||||
throw new Error(`unexpected request: ${url.href}`);
|
||||
});
|
||||
|
||||
await refreshAllCustomModelHosts();
|
||||
|
||||
const [updated] = await readCustomModelHosts(dir);
|
||||
expect(updated.modelContextLengths).toEqual({ a: 8192 });
|
||||
expect(updated.modelSlotCounts).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('refreshAllCustomModelHosts: parallel-slot enrichment (llama.cpp splits its context across --parallel slots)', () => {
|
||||
/** One loaded model whose launch command is `cmd`; `/props` behaviour is supplied by the test. */
|
||||
function serve(cmd: string, props: () => Response) {
|
||||
fetchMock.mockImplementation(async (url: URL) => {
|
||||
if (url.pathname === '/v1/models') {
|
||||
return new Response(JSON.stringify({ data: [{ id: 'm', status: { value: 'loaded' } }] }), { status: 200 });
|
||||
}
|
||||
if (url.pathname === '/running') {
|
||||
return new Response(JSON.stringify({ running: [{ model: 'm', state: 'ready', cmd }] }), { status: 200 });
|
||||
}
|
||||
if (url.pathname === '/props') return props();
|
||||
throw new Error(`unexpected request: ${url.href}`);
|
||||
});
|
||||
}
|
||||
const propsMustNotBeReached = () => {
|
||||
throw new Error('/props must not be probed: the launch command already answered both questions');
|
||||
};
|
||||
|
||||
it.each([
|
||||
['--parallel 1', 1],
|
||||
['-np 2', 2],
|
||||
['--parallel=3', 3],
|
||||
['--fit-ctx 65536 --parallel 1 --host 0.0.0.0', 1],
|
||||
])('reads an explicit %s from the launch command without probing /props', async (flags, slots) => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]);
|
||||
serve(`llama-server -m /models/m.gguf --ctx-size 65536 ${flags}`, propsMustNotBeReached);
|
||||
|
||||
await refreshAllCustomModelHosts();
|
||||
|
||||
const [updated] = await readCustomModelHosts(dir);
|
||||
expect(updated.modelSlotCounts).toEqual({ m: slots });
|
||||
expect(updated.modelContextLengths).toEqual({ m: 65536 });
|
||||
});
|
||||
|
||||
it('asks /props for total_slots when the command states no --parallel (llama-server picks the count itself)', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]);
|
||||
serve(
|
||||
'llama-server -m /models/m.gguf --fit-ctx 65536',
|
||||
() => new Response(JSON.stringify({ n_ctx: 133888, total_slots: 4 }), { status: 200 })
|
||||
);
|
||||
|
||||
await refreshAllCustomModelHosts();
|
||||
|
||||
const [updated] = await readCustomModelHosts(dir);
|
||||
expect(updated.modelSlotCounts).toEqual({ m: 4 });
|
||||
// the context still comes from the command's floor, never from /props's n_ctx
|
||||
expect(updated.modelContextLengths).toEqual({ m: 65536 });
|
||||
});
|
||||
|
||||
it('probes /props once per launch command: a repeat sweep reuses the answer, a changed command probes again', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]);
|
||||
let probes = 0;
|
||||
const answer = () => (probes++, new Response(JSON.stringify({ n_ctx: 133888, total_slots: 4 }), { status: 200 }));
|
||||
serve('llama-server -m /models/m.gguf --fit-ctx 65536', answer);
|
||||
await refreshAllCustomModelHosts();
|
||||
await refreshAllCustomModelHosts();
|
||||
expect(probes).toBe(1); // a /props call refreshes the model's last-use time in llama-swap, so not every sweep
|
||||
expect((await readCustomModelHosts(dir))[0].modelSlotCounts).toEqual({ m: 4 });
|
||||
|
||||
serve('llama-server -m /models/m.gguf --fit-ctx 65536 --flash-attn on', answer);
|
||||
await refreshAllCustomModelHosts();
|
||||
expect(probes).toBe(2);
|
||||
});
|
||||
|
||||
it('does not divide a /props context a second time when --parallel is explicit and the command states no context', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]);
|
||||
// with an explicit count llama.cpp reports n_ctx per slot: 65,536 each is already what one request gets
|
||||
serve(
|
||||
'llama-server -m /models/m.gguf --parallel 4',
|
||||
() => new Response(JSON.stringify({ n_ctx: 65536, total_slots: 4 }), { status: 200 })
|
||||
);
|
||||
|
||||
await refreshAllCustomModelHosts();
|
||||
|
||||
const [updated] = await readCustomModelHosts(dir);
|
||||
expect(updated.modelContextLengths).toEqual({ m: 65536 });
|
||||
expect(updated.modelSlotCounts).toEqual({ m: 1 }); // already per slot, so one request gets all of it
|
||||
});
|
||||
|
||||
it('a later round with an explicit --parallel replaces the slot count an earlier round learned', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]);
|
||||
// round 1: auto slots, the server chose 4 sharing 131,072 tokens
|
||||
serve(
|
||||
'llama-server -m /models/m.gguf',
|
||||
() => new Response(JSON.stringify({ n_ctx: 131072, total_slots: 4 }), { status: 200 })
|
||||
);
|
||||
await refreshAllCustomModelHosts();
|
||||
expect((await readCustomModelHosts(dir))[0].modelSlotCounts).toEqual({ m: 4 });
|
||||
|
||||
// round 2: the user followed the dialog and added --parallel 1
|
||||
serve(
|
||||
'llama-server -m /models/m.gguf --parallel 1',
|
||||
() => new Response(JSON.stringify({ n_ctx: 131072, total_slots: 1 }), { status: 200 })
|
||||
);
|
||||
await refreshAllCustomModelHosts();
|
||||
expect((await readCustomModelHosts(dir))[0].modelSlotCounts).toEqual({ m: 1 });
|
||||
|
||||
// round 3: --parallel 2 with a per-slot /props context must not bring a pool back
|
||||
serve(
|
||||
'llama-server -m /models/m.gguf --parallel 2',
|
||||
() => new Response(JSON.stringify({ n_ctx: 65536, total_slots: 2 }), { status: 200 })
|
||||
);
|
||||
await refreshAllCustomModelHosts();
|
||||
const [updated] = await readCustomModelHosts(dir);
|
||||
expect(updated.modelSlotCounts).toEqual({ m: 1 });
|
||||
expect(updated.modelContextLengths).toEqual({ m: 65536 });
|
||||
});
|
||||
|
||||
it('remembers a /props that answers non-2xx per launch command, but retries a network failure', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]);
|
||||
let probes = 0;
|
||||
serve('llama-server -m /models/m.gguf --fit-ctx 65536', () => (probes++, new Response('nope', { status: 404 })));
|
||||
await refreshAllCustomModelHosts();
|
||||
await refreshAllCustomModelHosts();
|
||||
expect(probes).toBe(1); // no probe every sweep: a /props call keeps a model with a ttl loaded
|
||||
|
||||
clearPropsProbeCache();
|
||||
probes = 0;
|
||||
serve('llama-server -m /models/m.gguf --fit-ctx 65536', () => {
|
||||
probes++;
|
||||
throw new TypeError('fetch failed');
|
||||
});
|
||||
await refreshAllCustomModelHosts();
|
||||
await refreshAllCustomModelHosts();
|
||||
expect(probes).toBe(2); // a network error says nothing about the server, so it is tried again
|
||||
});
|
||||
|
||||
it('treats -np -1 (llama.cpp auto) as unstated and falls back to /props', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]);
|
||||
serve(
|
||||
'llama-server -m /models/m.gguf --fit-ctx 65536 -np -1',
|
||||
() => new Response(JSON.stringify({ total_slots: 4 }), { status: 200 })
|
||||
);
|
||||
|
||||
await refreshAllCustomModelHosts();
|
||||
|
||||
const [updated] = await readCustomModelHosts(dir);
|
||||
expect(updated.modelSlotCounts).toEqual({ m: 4 });
|
||||
});
|
||||
|
||||
it('a failed or slot-less /props leaves no slot count rather than failing the sweep or guessing one', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [host({ id: 'ep', baseUrl: 'http://localhost:8080' })]);
|
||||
serve(
|
||||
'llama-server -m /models/m.gguf --fit-ctx 65536',
|
||||
() => new Response(JSON.stringify({ n_ctx: 8192 }), { status: 200 })
|
||||
);
|
||||
|
||||
await expect(refreshAllCustomModelHosts()).resolves.toBeUndefined();
|
||||
|
||||
const [updated] = await readCustomModelHosts(dir);
|
||||
expect(updated.modelSlotCounts).toBeUndefined();
|
||||
expect(updated.modelContextLengths).toEqual({ m: 65536 });
|
||||
});
|
||||
|
||||
it('keeps a previously-learned slot count for a model that is no longer loaded, and drops it once the model disappears', async () => {
|
||||
const dir = getDataDir();
|
||||
await writeCustomModelHosts(dir, [
|
||||
host({
|
||||
id: 'ep',
|
||||
baseUrl: 'http://localhost:8080',
|
||||
models: ['a', 'b'],
|
||||
modelContextLengths: { a: 65536, b: 65536 },
|
||||
modelSlotCounts: { a: 4, b: 2 },
|
||||
}),
|
||||
]);
|
||||
fetchMock.mockResolvedValue(new Response(JSON.stringify({ data: [{ id: 'a' }] }), { status: 200 }));
|
||||
|
||||
await refreshAllCustomModelHosts();
|
||||
|
||||
const [updated] = await readCustomModelHosts(dir);
|
||||
expect(updated.modelSlotCounts).toEqual({ a: 4 });
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1281,7 +1281,7 @@ describe("Custom Model Endpoint Profiles: requiresContextWarning (this CLI's own
|
||||
|
||||
await app.runCustomModelEntry('claude', 'llama-box', 'qwen3');
|
||||
|
||||
expect(confirmArgs).toEqual(['qwen3', 16384, 40000]);
|
||||
expect(confirmArgs).toEqual(['qwen3', 16384, 40000, undefined]);
|
||||
expect(applyBodies).toEqual([
|
||||
{ endpointId: 'llama-box', modelId: 'qwen3' },
|
||||
{ endpointId: 'llama-box', modelId: 'qwen3', confirmedContext: true },
|
||||
@@ -1289,6 +1289,29 @@ describe("Custom Model Endpoint Profiles: requiresContextWarning (this CLI's own
|
||||
expect(win.localStorage.getItem('codeman:customModelLastUsed:claude:llama-box')).toBe('qwen3');
|
||||
});
|
||||
|
||||
it('forwards the slot count from the apply response into the modal', async () => {
|
||||
const { app, applyBodies } = launchHarness([
|
||||
{
|
||||
requiresContextWarning: true,
|
||||
modelId: 'qwen3',
|
||||
contextLength: 131072,
|
||||
minSafeContextTokens: 40000,
|
||||
slotCount: 4,
|
||||
},
|
||||
{ customModel: { endpointId: 'llama-box' }, restarted: true, modelSwapInProgress: false },
|
||||
]);
|
||||
let confirmArgs: unknown[] | undefined;
|
||||
app._confirmContextWarning = async (...args: unknown[]) => {
|
||||
confirmArgs = args;
|
||||
return true;
|
||||
};
|
||||
|
||||
await app.runCustomModelEntry('claude', 'llama-box', 'qwen3');
|
||||
|
||||
expect(confirmArgs).toEqual(['qwen3', 131072, 40000, 4]);
|
||||
expect(applyBodies).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('declining the in-app context-warning modal keeps the native backend, never re-sends the apply, and must NEVER record this model as "last used" — it cannot work with this CLI at all', async () => {
|
||||
const { win, app, applyBodies } = launchHarness([
|
||||
{ requiresContextWarning: true, modelId: 'qwen3', contextLength: 16384, minSafeContextTokens: 40000 },
|
||||
@@ -1361,6 +1384,49 @@ describe('Custom Model Endpoint Profiles: _confirmContextWarning (in-app modal,
|
||||
expect(win.document.getElementById('customModelContextWarningModal')!.classList.contains('active')).toBe(false);
|
||||
});
|
||||
|
||||
it('explains shared parallel slots, and the --parallel 1 fix, when the context is enough only as a whole', async () => {
|
||||
const { win, app } = bootApp({});
|
||||
void app._confirmContextWarning('qwen3.8-27b-ud-q4_k_xl', 131072, 40000, 4);
|
||||
const message = win.document.getElementById('customModelContextWarningMessage')!.textContent!;
|
||||
expect(message).toContain('qwen3.8-27b-ud-q4_k_xl');
|
||||
expect(message).toContain('4 parallel');
|
||||
expect(message).toContain('131,072');
|
||||
expect(message).toContain('32,768'); // per slot
|
||||
expect(message).toContain('--parallel 1');
|
||||
expect(message).toContain('Context size has been exceeded');
|
||||
// this is not the "first message fails outright" case, so it must not claim that
|
||||
expect(message).not.toMatch(/first message will fail/i);
|
||||
expect(win.document.getElementById('customModelContextWarningModal')!.classList.contains('active')).toBe(true);
|
||||
app._resolveContextWarningConfirm(false);
|
||||
});
|
||||
|
||||
it('says the context is split across the slots, about N each, from the discovered context alone', async () => {
|
||||
const { win, app } = bootApp({});
|
||||
void app._confirmContextWarning('qwen3', 65536, 40000, 4);
|
||||
const message = win.document.getElementById('customModelContextWarningMessage')!.textContent!;
|
||||
expect(message).toContain('65,536 tokens of context are split across 4 parallel server slots, about 16,384 each');
|
||||
app._resolveContextWarningConfirm(false);
|
||||
});
|
||||
|
||||
it('keeps the too-small-context message for a small model and adds the slot advice when slots are known', async () => {
|
||||
const { win, app } = bootApp({});
|
||||
void app._confirmContextWarning('qwen3', 16384, 40000, 4);
|
||||
const message = win.document.getElementById('customModelContextWarningMessage')!.textContent!;
|
||||
expect(message).toContain('16,384');
|
||||
expect(message).toMatch(/first message will fail/i);
|
||||
expect(message).toContain('4 parallel slots');
|
||||
expect(message).toContain('--parallel 1');
|
||||
app._resolveContextWarningConfirm(false);
|
||||
});
|
||||
|
||||
it('does not mention slots when there is one (or none known)', async () => {
|
||||
const { win, app } = bootApp({});
|
||||
void app._confirmContextWarning('qwen3', 16384, 40000, 1);
|
||||
const message = win.document.getElementById('customModelContextWarningMessage')!.textContent!;
|
||||
expect(message).not.toMatch(/parallel/i);
|
||||
app._resolveContextWarningConfirm(false);
|
||||
});
|
||||
|
||||
it('describes an unknown context length without printing a bogus number', async () => {
|
||||
const { win, app } = bootApp({});
|
||||
void app._confirmContextWarning('qwen3', undefined, 40000);
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* @fileoverview Unit tests for the parallel-slot awareness of the custom-model context floor
|
||||
* (`perSlotContext` / `exceedsSafeContextFloor`, src/web/routes/custom-model-routes.ts).
|
||||
*
|
||||
* llama-server splits its context across `--parallel` slots (by default they share one
|
||||
* unified KV pool) and Claude Code sends concurrent requests, each carrying its own ~40k
|
||||
* tokens of system prompt and tools. Measured against a real llama-swap server: with 4 slots
|
||||
* sharing ~131k tokens, four simultaneous 40k-token requests all failed with HTTP 500
|
||||
* "Context size has been exceeded"; the same requests against one slot all succeeded.
|
||||
*
|
||||
* Port: N/A (pure functions, no server).
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
CLAUDE_MIN_SAFE_CONTEXT_TOKENS,
|
||||
exceedsSafeContextFloor,
|
||||
perSlotContext,
|
||||
} from '../src/web/routes/custom-model-routes.js';
|
||||
import type { CliEntry } from '../src/config/cli-registry/types.js';
|
||||
|
||||
const withContextVar = {
|
||||
capabilities: { customModelInjection: { kind: 'env', contextLengthVar: 'CLAUDE_CODE_MAX_CONTEXT_TOKENS' } },
|
||||
} as unknown as Pick<CliEntry, 'capabilities'>;
|
||||
const withoutContextVar = {
|
||||
capabilities: { customModelInjection: { kind: 'env' } },
|
||||
} as unknown as Pick<CliEntry, 'capabilities'>;
|
||||
|
||||
describe('perSlotContext', () => {
|
||||
it('divides the context across the slots', () => {
|
||||
expect(perSlotContext(131072, 4)).toBe(32768);
|
||||
expect(perSlotContext(262144, 4)).toBe(65536);
|
||||
});
|
||||
|
||||
it('rounds down, since a fraction of a token is not usable', () => {
|
||||
expect(perSlotContext(100000, 3)).toBe(33333);
|
||||
});
|
||||
|
||||
it.each([undefined, 0, 1, -2, Number.NaN, Number.POSITIVE_INFINITY])(
|
||||
'leaves the context whole for an unusable or single slot count (%s)',
|
||||
(slots) => {
|
||||
expect(perSlotContext(65536, slots as number | undefined)).toBe(65536);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe('exceedsSafeContextFloor with a slot count', () => {
|
||||
it('divides only the context it is given, so a larger /props figure can never skip the warning', () => {
|
||||
// a --fit-ctx floor of 65,536 over 4 slots is 16,384 each, whatever /props might say about the server
|
||||
expect(exceedsSafeContextFloor(withContextVar, 65536, 4)).toBe(true);
|
||||
expect(exceedsSafeContextFloor(withContextVar, 65536, 1)).toBe(false);
|
||||
// the measured failure: ~131k over 4 slots is ~33k each
|
||||
expect(exceedsSafeContextFloor(withContextVar, 131072, 4)).toBe(true);
|
||||
expect(exceedsSafeContextFloor(withContextVar, 262144, 4)).toBe(false);
|
||||
});
|
||||
|
||||
it('still warns when the context itself is below the minimum', () => {
|
||||
expect(exceedsSafeContextFloor(withContextVar, 16384, 4)).toBe(true);
|
||||
expect(exceedsSafeContextFloor(withContextVar, 16384, 1)).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps its existing behaviour when no slot count is given', () => {
|
||||
expect(exceedsSafeContextFloor(withContextVar, 16384)).toBe(true);
|
||||
expect(exceedsSafeContextFloor(withContextVar, CLAUDE_MIN_SAFE_CONTEXT_TOKENS)).toBe(false);
|
||||
expect(exceedsSafeContextFloor(withContextVar, undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it('flags a context that only clears the floor as a whole, not per slot', () => {
|
||||
expect(exceedsSafeContextFloor(withContextVar, 131072, 4)).toBe(true);
|
||||
expect(exceedsSafeContextFloor(withContextVar, 131072, 1)).toBe(false);
|
||||
});
|
||||
|
||||
it('accepts exactly the floor per slot', () => {
|
||||
expect(exceedsSafeContextFloor(withContextVar, CLAUDE_MIN_SAFE_CONTEXT_TOKENS * 4, 4)).toBe(false);
|
||||
expect(exceedsSafeContextFloor(withContextVar, CLAUDE_MIN_SAFE_CONTEXT_TOKENS * 4 - 4, 4)).toBe(true);
|
||||
});
|
||||
|
||||
it('never applies to a CLI that declares no context variable, whatever the slots', () => {
|
||||
expect(exceedsSafeContextFloor(withoutContextVar, 4096, 8)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -316,6 +316,47 @@ describe('POST /api/quick-start: customModel (one-shot custom-model launch)', ()
|
||||
expect(ctx.sessions.size).toBe(2);
|
||||
});
|
||||
|
||||
describe('parallel slots share the context (the quick-start path has its own slot branch)', () => {
|
||||
const slotEndpoint = (contextLength: number, slots: number): CustomModelHost => ({
|
||||
id: 'ep-slots',
|
||||
label: 'multi-slot box',
|
||||
baseUrl: 'http://192.168.1.52:8080',
|
||||
apiKey: 'k',
|
||||
modelContextLengths: { m: contextLength },
|
||||
modelSlotCounts: { m: slots },
|
||||
});
|
||||
|
||||
it('warns when the context divided across the slots is below the floor, and reports the slot count', async () => {
|
||||
await writeCustomModelHosts(getDataDir(), [ENDPOINT, slotEndpoint(131072, 4)]); // ~33k per slot
|
||||
|
||||
const res = await quickStart({
|
||||
caseName: 'cm-slots-warn',
|
||||
mode: 'claude',
|
||||
customModel: { endpointId: 'ep-slots', modelId: 'm' },
|
||||
});
|
||||
|
||||
const body = res.json();
|
||||
expect(body.requiresContextWarning).toBe(true);
|
||||
expect(body.contextLength).toBe(131072);
|
||||
expect(body.slotCount).toBe(4);
|
||||
expect(ctx.sessions.size).toBe(1); // nothing was created
|
||||
});
|
||||
|
||||
it('launches when every slot can hold the floor', async () => {
|
||||
await writeCustomModelHosts(getDataDir(), [ENDPOINT, slotEndpoint(262144, 4)]); // 65,536 per slot
|
||||
|
||||
const res = await quickStart({
|
||||
caseName: 'cm-slots-ok',
|
||||
mode: 'claude',
|
||||
customModel: { endpointId: 'ep-slots', modelId: 'm' },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.json().requiresContextWarning).toBeUndefined();
|
||||
expect(ctx.sessions.size).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
it('does not warn when nothing about context was discovered', async () => {
|
||||
const res = await quickStart({
|
||||
caseName: 'cm-no-ctx-data',
|
||||
|
||||
@@ -449,6 +449,78 @@ describe('POST /api/sessions/:id/custom-model', () => {
|
||||
expect(session.restartCli).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// llama-server splits its context across --parallel slots, and Claude Code sends several
|
||||
// requests at once, so a context that clears the floor as a whole can still fail per slot.
|
||||
describe('parallel slots share the context', () => {
|
||||
const slotEndpoint = (contextLength: number, slots?: number): CustomModelHost => ({
|
||||
id: 'ep-slots',
|
||||
label: 'multi-slot box',
|
||||
baseUrl: 'http://192.168.1.52:8080',
|
||||
apiKey: 'k',
|
||||
modelContextLengths: { m: contextLength },
|
||||
...(slots === undefined ? {} : { modelSlotCounts: { m: slots } }),
|
||||
});
|
||||
|
||||
async function apply(endpoint: CustomModelHost) {
|
||||
const { app, ctx } = await setup();
|
||||
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, endpoint]);
|
||||
const session = ctx.sessions.get('test-session-1')!;
|
||||
session.mode = 'claude';
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/sessions/test-session-1/custom-model',
|
||||
payload: { endpointId: endpoint.id, modelId: 'm' },
|
||||
});
|
||||
return { body: res.json(), session };
|
||||
}
|
||||
|
||||
it('warns when the context divided across the slots is below the floor, and reports the slot count', async () => {
|
||||
// the measured case: ~131k shared by 4 slots is ~33k each, under the 40k a request needs
|
||||
const { body, session } = await apply(slotEndpoint(131072, 4));
|
||||
|
||||
expect(body.requiresContextWarning).toBe(true);
|
||||
expect(body.contextLength).toBe(131072);
|
||||
expect(body.slotCount).toBe(4);
|
||||
expect(session.setCustomModel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('divides the discovered context across the slots even when it clears the floor as a whole', async () => {
|
||||
// 65,536 over 4 slots is 16,384 each: nothing the server reports can widen this pool
|
||||
const { body } = await apply(slotEndpoint(65536, 4));
|
||||
|
||||
expect(body.requiresContextWarning).toBe(true);
|
||||
expect(body.contextLength).toBe(65536);
|
||||
expect(body.slotCount).toBe(4);
|
||||
});
|
||||
|
||||
it('does not warn once every slot can hold the floor', async () => {
|
||||
const { body, session } = await apply(slotEndpoint(262144, 4)); // 65,536 each
|
||||
|
||||
expect(body.requiresContextWarning).toBeUndefined();
|
||||
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not warn for a single slot with the same context that failed across four', async () => {
|
||||
const { body, session } = await apply(slotEndpoint(131072, 1));
|
||||
|
||||
expect(body.requiresContextWarning).toBeUndefined();
|
||||
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('an unknown slot count is treated as one slot, never guessed', async () => {
|
||||
const { body } = await apply(slotEndpoint(131072));
|
||||
|
||||
expect(body.requiresContextWarning).toBeUndefined();
|
||||
});
|
||||
|
||||
it('omits slotCount from the warning when the context is too small on its own with a single slot', async () => {
|
||||
const { body } = await apply(slotEndpoint(16384, 1));
|
||||
|
||||
expect(body.requiresContextWarning).toBe(true);
|
||||
expect(body.slotCount).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
it('applies once confirmed, skipping the context check the second time', async () => {
|
||||
const { app, ctx } = await setup();
|
||||
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, SMALL_CTX_ENDPOINT]);
|
||||
|
||||
Reference in new issue
Block a user