mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-16 03:09:33 +01:00
feat(local-model): q8_0 K/V cache, flash attention and a bounded prompt cache for llama.cpp
runtimeEnv() now also sets OLLAMA_KV_CACHE_TYPE=q8_0, OLLAMA_FLASH_ATTENTION=1 and LLAMA_ARG_CACHE_RAM=1024. Read in the v0.40.15 source of the pinned runtime: the first becomes --cache-type-k/-v of llama-server (17/32 of the f16 cache), the second --flash-attn on (a quantized V cache needs it, and "auto" or "off" would fail such a load), and llama-server inherits the third from ollama serve's environment, which bounds its host prompt cache to 1 GiB instead of letting it grow by gigabytes in a session. The MLX runner reads none of them. The catalogue's llama.cpp rows are computed for these values, so the two change together. Not yet measured on a Mac. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a9e08732c0
commit
f60b63c005
4 files changed
+61
-5
No files matched your search
@@ -39,7 +39,9 @@ import {
|
||||
removeLocalModelFiles,
|
||||
removePartialModelBlobs,
|
||||
RUNTIME_DOWNLOAD_RETRIES,
|
||||
RUNTIME_HOST_PROMPT_CACHE_MIB,
|
||||
RUNTIME_KEEP_ALIVE,
|
||||
RUNTIME_KV_CACHE_TYPE,
|
||||
RUNTIME_MAX_RESTARTS,
|
||||
RUNTIME_PORT_FIRST,
|
||||
RUNTIME_PORT_SPAN,
|
||||
@@ -273,7 +275,7 @@ describe('runtimeEnv', () => {
|
||||
DYLD_INSERT_LIBRARIES: '/tmp/evil.dylib',
|
||||
};
|
||||
|
||||
it('holds exactly the allowlist: what is copied, the private HOME, and seven settings', () => {
|
||||
it('holds exactly the allowlist: what is copied, the private HOME, and ten settings', () => {
|
||||
expect(runtimeEnv(input(SERVER_ENV))).toEqual({
|
||||
PATH: '/usr/local/bin:/usr/bin:/bin',
|
||||
TMPDIR: '/var/folders/xy/T/',
|
||||
@@ -292,9 +294,28 @@ describe('runtimeEnv', () => {
|
||||
OLLAMA_KEEP_ALIVE: '10m',
|
||||
OLLAMA_NUM_PARALLEL: '1',
|
||||
OLLAMA_MAX_LOADED_MODELS: '1',
|
||||
// The memory settings the catalogue's llamacpp rows are computed for.
|
||||
OLLAMA_KV_CACHE_TYPE: 'q8_0',
|
||||
OLLAMA_FLASH_ATTENTION: '1',
|
||||
LLAMA_ARG_CACHE_RAM: '1024',
|
||||
});
|
||||
});
|
||||
|
||||
it('sets the llama.cpp memory settings from the exported constants, never from the server', () => {
|
||||
expect(RUNTIME_KV_CACHE_TYPE).toBe('q8_0');
|
||||
expect(RUNTIME_HOST_PROMPT_CACHE_MIB).toBe(1024);
|
||||
const env = runtimeEnv(input(SERVER_ENV));
|
||||
expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE);
|
||||
expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB));
|
||||
// The server's own `LLAMA_ARG_CACHE_RAM=0` and a user's q4_0 cache lose against ours.
|
||||
const env2 = runtimeEnv(input({ ...SERVER_ENV, OLLAMA_KV_CACHE_TYPE: 'q4_0', OLLAMA_FLASH_ATTENTION: '0' }));
|
||||
expect([env2.OLLAMA_KV_CACHE_TYPE, env2.OLLAMA_FLASH_ATTENTION, env2.LLAMA_ARG_CACHE_RAM]).toEqual([
|
||||
'q8_0',
|
||||
'1',
|
||||
'1024',
|
||||
]);
|
||||
});
|
||||
|
||||
it('lets no CODEMAN_ variable through, the password least of all', () => {
|
||||
const env = runtimeEnv(input(SERVER_ENV));
|
||||
expect(Object.keys(env).filter((name) => name.startsWith('CODEMAN_'))).toEqual([]);
|
||||
@@ -309,8 +330,10 @@ describe('runtimeEnv', () => {
|
||||
.sort()
|
||||
).toEqual([
|
||||
'OLLAMA_CONTEXT_LENGTH',
|
||||
'OLLAMA_FLASH_ATTENTION',
|
||||
'OLLAMA_HOST',
|
||||
'OLLAMA_KEEP_ALIVE',
|
||||
'OLLAMA_KV_CACHE_TYPE',
|
||||
'OLLAMA_MAX_LOADED_MODELS',
|
||||
'OLLAMA_MODELS',
|
||||
'OLLAMA_NO_CLOUD',
|
||||
@@ -329,7 +352,8 @@ describe('runtimeEnv', () => {
|
||||
]) {
|
||||
expect(env, name).not.toHaveProperty(name);
|
||||
}
|
||||
expect(env).not.toHaveProperty('LLAMA_ARG_CACHE_RAM');
|
||||
// The server's `LLAMA_ARG_CACHE_RAM=0` is not copied: the value is ours.
|
||||
expect(env.LLAMA_ARG_CACHE_RAM).toBe('1024');
|
||||
});
|
||||
|
||||
it('carries no secret and no value of the server that is not on the list', () => {
|
||||
@@ -365,9 +389,12 @@ describe('runtimeEnv', () => {
|
||||
it('leaves out what the server does not have, and sets the rest all the same', () => {
|
||||
expect(Object.keys(runtimeEnv(input({}))).sort()).toEqual([
|
||||
'HOME',
|
||||
'LLAMA_ARG_CACHE_RAM',
|
||||
'OLLAMA_CONTEXT_LENGTH',
|
||||
'OLLAMA_FLASH_ATTENTION',
|
||||
'OLLAMA_HOST',
|
||||
'OLLAMA_KEEP_ALIVE',
|
||||
'OLLAMA_KV_CACHE_TYPE',
|
||||
'OLLAMA_MAX_LOADED_MODELS',
|
||||
'OLLAMA_MODELS',
|
||||
'OLLAMA_NO_CLOUD',
|
||||
|
||||
@@ -206,6 +206,9 @@ describe('start', () => {
|
||||
OLLAMA_KEEP_ALIVE: '10m',
|
||||
OLLAMA_NUM_PARALLEL: '1',
|
||||
OLLAMA_MAX_LOADED_MODELS: '1',
|
||||
OLLAMA_KV_CACHE_TYPE: 'q8_0',
|
||||
OLLAMA_FLASH_ATTENTION: '1',
|
||||
LLAMA_ARG_CACHE_RAM: '1024',
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user