mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-16 19:29:33 +01:00
fix(custom-model): warn when parallel llama.cpp slots leave too little context per request (#658)
Squash of PR #658. Discovery records each llama.cpp model's parallel slot count (from --parallel/-np, or /props total_slots when the launch command states none), and a Claude launch warns when the context split across those slots leaves less than 40K tokens per request. An explicit --parallel beside a /props context records a slot count of 1, so a later round overwrites an earlier count, and a non-2xx /props is remembered per launch command while a network error is retried. Merge-time fix by the maintainer: the architecture-invariants sentence on the explicit --parallel case now says it records a slot count of 1. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
34cf798b10
commit
33d697afe3
17 files changed
+710
-34
No files matched your search
@@ -316,6 +316,47 @@ describe('POST /api/quick-start: customModel (one-shot custom-model launch)', ()
|
||||
expect(ctx.sessions.size).toBe(2);
|
||||
});
|
||||
|
||||
describe('parallel slots share the context (the quick-start path has its own slot branch)', () => {
|
||||
const slotEndpoint = (contextLength: number, slots: number): CustomModelHost => ({
|
||||
id: 'ep-slots',
|
||||
label: 'multi-slot box',
|
||||
baseUrl: 'http://192.168.1.52:8080',
|
||||
apiKey: 'k',
|
||||
modelContextLengths: { m: contextLength },
|
||||
modelSlotCounts: { m: slots },
|
||||
});
|
||||
|
||||
it('warns when the context divided across the slots is below the floor, and reports the slot count', async () => {
|
||||
await writeCustomModelHosts(getDataDir(), [ENDPOINT, slotEndpoint(131072, 4)]); // ~33k per slot
|
||||
|
||||
const res = await quickStart({
|
||||
caseName: 'cm-slots-warn',
|
||||
mode: 'claude',
|
||||
customModel: { endpointId: 'ep-slots', modelId: 'm' },
|
||||
});
|
||||
|
||||
const body = res.json();
|
||||
expect(body.requiresContextWarning).toBe(true);
|
||||
expect(body.contextLength).toBe(131072);
|
||||
expect(body.slotCount).toBe(4);
|
||||
expect(ctx.sessions.size).toBe(1); // nothing was created
|
||||
});
|
||||
|
||||
it('launches when every slot can hold the floor', async () => {
|
||||
await writeCustomModelHosts(getDataDir(), [ENDPOINT, slotEndpoint(262144, 4)]); // 65,536 per slot
|
||||
|
||||
const res = await quickStart({
|
||||
caseName: 'cm-slots-ok',
|
||||
mode: 'claude',
|
||||
customModel: { endpointId: 'ep-slots', modelId: 'm' },
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.json().requiresContextWarning).toBeUndefined();
|
||||
expect(ctx.sessions.size).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
it('does not warn when nothing about context was discovered', async () => {
|
||||
const res = await quickStart({
|
||||
caseName: 'cm-no-ctx-data',
|
||||
|
||||
@@ -449,6 +449,78 @@ describe('POST /api/sessions/:id/custom-model', () => {
|
||||
expect(session.restartCli).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// llama-server splits its context across --parallel slots, and Claude Code sends several
|
||||
// requests at once, so a context that clears the floor as a whole can still fail per slot.
|
||||
describe('parallel slots share the context', () => {
|
||||
const slotEndpoint = (contextLength: number, slots?: number): CustomModelHost => ({
|
||||
id: 'ep-slots',
|
||||
label: 'multi-slot box',
|
||||
baseUrl: 'http://192.168.1.52:8080',
|
||||
apiKey: 'k',
|
||||
modelContextLengths: { m: contextLength },
|
||||
...(slots === undefined ? {} : { modelSlotCounts: { m: slots } }),
|
||||
});
|
||||
|
||||
async function apply(endpoint: CustomModelHost) {
|
||||
const { app, ctx } = await setup();
|
||||
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, endpoint]);
|
||||
const session = ctx.sessions.get('test-session-1')!;
|
||||
session.mode = 'claude';
|
||||
const res = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/sessions/test-session-1/custom-model',
|
||||
payload: { endpointId: endpoint.id, modelId: 'm' },
|
||||
});
|
||||
return { body: res.json(), session };
|
||||
}
|
||||
|
||||
it('warns when the context divided across the slots is below the floor, and reports the slot count', async () => {
|
||||
// the measured case: ~131k shared by 4 slots is ~33k each, under the 40k a request needs
|
||||
const { body, session } = await apply(slotEndpoint(131072, 4));
|
||||
|
||||
expect(body.requiresContextWarning).toBe(true);
|
||||
expect(body.contextLength).toBe(131072);
|
||||
expect(body.slotCount).toBe(4);
|
||||
expect(session.setCustomModel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('divides the discovered context across the slots even when it clears the floor as a whole', async () => {
|
||||
// 65,536 over 4 slots is 16,384 each: nothing the server reports can widen this pool
|
||||
const { body } = await apply(slotEndpoint(65536, 4));
|
||||
|
||||
expect(body.requiresContextWarning).toBe(true);
|
||||
expect(body.contextLength).toBe(65536);
|
||||
expect(body.slotCount).toBe(4);
|
||||
});
|
||||
|
||||
it('does not warn once every slot can hold the floor', async () => {
|
||||
const { body, session } = await apply(slotEndpoint(262144, 4)); // 65,536 each
|
||||
|
||||
expect(body.requiresContextWarning).toBeUndefined();
|
||||
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not warn for a single slot with the same context that failed across four', async () => {
|
||||
const { body, session } = await apply(slotEndpoint(131072, 1));
|
||||
|
||||
expect(body.requiresContextWarning).toBeUndefined();
|
||||
expect(session.setCustomModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('an unknown slot count is treated as one slot, never guessed', async () => {
|
||||
const { body } = await apply(slotEndpoint(131072));
|
||||
|
||||
expect(body.requiresContextWarning).toBeUndefined();
|
||||
});
|
||||
|
||||
it('omits slotCount from the warning when the context is too small on its own with a single slot', async () => {
|
||||
const { body } = await apply(slotEndpoint(16384, 1));
|
||||
|
||||
expect(body.requiresContextWarning).toBe(true);
|
||||
expect(body.slotCount).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
it('applies once confirmed, skipping the context check the second time', async () => {
|
||||
const { app, ctx } = await setup();
|
||||
await writeCustomModelHosts(getDataDir(), [CLAUDE_ENDPOINT, SMALL_CTX_ENDPOINT]);
|
||||
|
||||
Reference in new issue
Block a user