fix(custom-model): warn when parallel llama.cpp slots leave too little context per request (#658)

Squash of PR #658. Discovery records each llama.cpp model's parallel slot
count (from --parallel/-np, or /props total_slots when the launch command
states none), and a Claude launch warns when the context split across
those slots leaves less than 40K tokens per request. An explicit
--parallel beside a /props context records a slot count of 1, so a later
round overwrites an earlier count, and a non-2xx /props is remembered per
launch command while a network error is retried.

Merge-time fix by the maintainer: the architecture-invariants sentence on
the explicit --parallel case now says it records a slot count of 1.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
DevvynandClaude Opus 5.5 authored and Codeman maintainer committed 2026-10-22 19:32:24 +02:00
1 parent 34cf798b10
commit 33d697afe3
17 files changed
+710 -34

No files matched your search

@@ -316,6 +316,47 @@ describe('POST /api/quick-start: customModel (one-shot custom-model launch)', ()
expect(ctx.sessions.size).toBe(2);
});
describe('parallel slots share the context (the quick-start path has its own slot branch)', () => {
const slotEndpoint = (contextLength: number, slots: number): CustomModelHost => ({
id: 'ep-slots',
label: 'multi-slot box',
baseUrl: 'http://192.168.1.52:8080',
apiKey: 'k',
modelContextLengths: { m: contextLength },
modelSlotCounts: { m: slots },
});
it('warns when the context divided across the slots is below the floor, and reports the slot count', async () => {
await writeCustomModelHosts(getDataDir(), [ENDPOINT, slotEndpoint(131072, 4)]); // ~33k per slot
const res = await quickStart({
caseName: 'cm-slots-warn',
mode: 'claude',
customModel: { endpointId: 'ep-slots', modelId: 'm' },
});
const body = res.json();
expect(body.requiresContextWarning).toBe(true);
expect(body.contextLength).toBe(131072);
expect(body.slotCount).toBe(4);
expect(ctx.sessions.size).toBe(1); // nothing was created
});
it('launches when every slot can hold the floor', async () => {
await writeCustomModelHosts(getDataDir(), [ENDPOINT, slotEndpoint(262144, 4)]); // 65,536 per slot
const res = await quickStart({
caseName: 'cm-slots-ok',
mode: 'claude',
customModel: { endpointId: 'ep-slots', modelId: 'm' },
});
expect(res.statusCode).toBe(200);
expect(res.json().requiresContextWarning).toBeUndefined();
expect(ctx.sessions.size).toBe(2);
});
});
it('does not warn when nothing about context was discovered', async () => {
const res = await quickStart({
caseName: 'cm-no-ctx-data',
+72
View File
@@ -449,6 +449,78 @@ describe('POST /api/sessions/:id/custom-model', () => {
expect(session.restartCli).not.toHaveBeenCalled();
});
// llama-server splits its context across --parallel slots, and Claude Code sends several
// requests at once, so a context that clears the floor as a whole can still fail per slot.
describe('parallel slots share the context', () => {
const slotEndpoint = (contextLength: number, slots?: number): CustomModelHost => ({
id: 'ep-slots',
label: 'multi-slot box',
baseUrl: 'http://192.168.1.52:8080',
apiKey: 'k',
modelContextLengths: { m: contextLength },
...(slots === undefined ? {} : { modelSlotCounts: { m: slots } }),
});
async function apply(endpoint: CustomModelHost) {
const { app, ctx } = await setup();
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, endpoint]);
const session = ctx.sessions.get('test-session-1')!;
session.mode = 'claude';
const res = await app.inject({
method: 'POST',
url: '/api/sessions/test-session-1/custom-model',
payload: { endpointId: endpoint.id, modelId: 'm' },
});
return { body: res.json(), session };
}
it('warns when the context divided across the slots is below the floor, and reports the slot count', async () => {
// the measured case: ~131k shared by 4 slots is ~33k each, under the 40k a request needs
const { body, session } = await apply(slotEndpoint(131072, 4));
expect(body.requiresContextWarning).toBe(true);
expect(body.contextLength).toBe(131072);
expect(body.slotCount).toBe(4);
expect(session.setCustomModel).not.toHaveBeenCalled();
});
it('divides the discovered context across the slots even when it clears the floor as a whole', async () => {
// 65,536 over 4 slots is 16,384 each: nothing the server reports can widen this pool
const { body } = await apply(slotEndpoint(65536, 4));
expect(body.requiresContextWarning).toBe(true);
expect(body.contextLength).toBe(65536);
expect(body.slotCount).toBe(4);
});
it('does not warn once every slot can hold the floor', async () => {
const { body, session } = await apply(slotEndpoint(262144, 4)); // 65,536 each
expect(body.requiresContextWarning).toBeUndefined();
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
});
it('does not warn for a single slot with the same context that failed across four', async () => {
const { body, session } = await apply(slotEndpoint(131072, 1));
expect(body.requiresContextWarning).toBeUndefined();
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
});
it('an unknown slot count is treated as one slot, never guessed', async () => {
const { body } = await apply(slotEndpoint(131072));
expect(body.requiresContextWarning).toBeUndefined();
});
it('omits slotCount from the warning when the context is too small on its own with a single slot', async () => {
const { body } = await apply(slotEndpoint(16384, 1));
expect(body.requiresContextWarning).toBe(true);
expect(body.slotCount).toBeUndefined();
});
});
it('applies once confirmed, skipping the context check the second time', async () => {
const { app, ctx } = await setup();
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, SMALL_CTX_ENDPOINT]);