feat(local-model): q8_0 K/V cache, flash attention and a bounded prompt cache for llama.cpp

runtimeEnv() now also sets OLLAMA_KV_CACHE_TYPE=q8_0, OLLAMA_FLASH_ATTENTION=1
and LLAMA_ARG_CACHE_RAM=1024. Read in the v0.40.15 source of the pinned
runtime: the first becomes --cache-type-k/-v of llama-server (17/32 of the
f16 cache), the second --flash-attn on (a quantized V cache needs it, and
"auto" or "off" would fail such a load), and llama-server inherits the third
from ollama serve's environment, which bounds its host prompt cache to
1 GiB instead of letting it grow by gigabytes in a session. The MLX runner
reads none of them.

The catalogue's llama.cpp rows are computed for these values, so the two
change together. Not yet measured on a Mac.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codeman maintainer committed 2026-11-08 09:18:51 +01:00
1 parent a9e08732c0
commit f60b63c005
4 files changed
+61 -5

No files matched your search

+29 -2
View File
@@ -39,7 +39,9 @@ import {
removeLocalModelFiles,
removePartialModelBlobs,
RUNTIME_DOWNLOAD_RETRIES,
RUNTIME_HOST_PROMPT_CACHE_MIB,
RUNTIME_KEEP_ALIVE,
RUNTIME_KV_CACHE_TYPE,
RUNTIME_MAX_RESTARTS,
RUNTIME_PORT_FIRST,
RUNTIME_PORT_SPAN,
@@ -273,7 +275,7 @@ describe('runtimeEnv', () => {
DYLD_INSERT_LIBRARIES: '/tmp/evil.dylib',
};
it('holds exactly the allowlist: what is copied, the private HOME, and seven settings', () => {
it('holds exactly the allowlist: what is copied, the private HOME, and ten settings', () => {
expect(runtimeEnv(input(SERVER_ENV))).toEqual({
PATH: '/usr/local/bin:/usr/bin:/bin',
TMPDIR: '/var/folders/xy/T/',
@@ -292,9 +294,28 @@ describe('runtimeEnv', () => {
OLLAMA_KEEP_ALIVE: '10m',
OLLAMA_NUM_PARALLEL: '1',
OLLAMA_MAX_LOADED_MODELS: '1',
// The memory settings the catalogue's llamacpp rows are computed for.
OLLAMA_KV_CACHE_TYPE: 'q8_0',
OLLAMA_FLASH_ATTENTION: '1',
LLAMA_ARG_CACHE_RAM: '1024',
});
});
it('sets the llama.cpp memory settings from the exported constants, never from the server', () => {
expect(RUNTIME_KV_CACHE_TYPE).toBe('q8_0');
expect(RUNTIME_HOST_PROMPT_CACHE_MIB).toBe(1024);
const env = runtimeEnv(input(SERVER_ENV));
expect(env.OLLAMA_KV_CACHE_TYPE).toBe(RUNTIME_KV_CACHE_TYPE);
expect(env.LLAMA_ARG_CACHE_RAM).toBe(String(RUNTIME_HOST_PROMPT_CACHE_MIB));
// The server's own `LLAMA_ARG_CACHE_RAM=0` and a user's q4_0 cache lose against ours.
const env2 = runtimeEnv(input({ ...SERVER_ENV, OLLAMA_KV_CACHE_TYPE: 'q4_0', OLLAMA_FLASH_ATTENTION: '0' }));
expect([env2.OLLAMA_KV_CACHE_TYPE, env2.OLLAMA_FLASH_ATTENTION, env2.LLAMA_ARG_CACHE_RAM]).toEqual([
'q8_0',
'1',
'1024',
]);
});
it('lets no CODEMAN_ variable through, the password least of all', () => {
const env = runtimeEnv(input(SERVER_ENV));
expect(Object.keys(env).filter((name) => name.startsWith('CODEMAN_'))).toEqual([]);
@@ -309,8 +330,10 @@ describe('runtimeEnv', () => {
.sort()
).toEqual([
'OLLAMA_CONTEXT_LENGTH',
'OLLAMA_FLASH_ATTENTION',
'OLLAMA_HOST',
'OLLAMA_KEEP_ALIVE',
'OLLAMA_KV_CACHE_TYPE',
'OLLAMA_MAX_LOADED_MODELS',
'OLLAMA_MODELS',
'OLLAMA_NO_CLOUD',
@@ -329,7 +352,8 @@ describe('runtimeEnv', () => {
]) {
expect(env, name).not.toHaveProperty(name);
}
expect(env).not.toHaveProperty('LLAMA_ARG_CACHE_RAM');
// The server's `LLAMA_ARG_CACHE_RAM=0` is not copied: the value is ours.
expect(env.LLAMA_ARG_CACHE_RAM).toBe('1024');
});
it('carries no secret and no value of the server that is not on the list', () => {
@@ -365,9 +389,12 @@ describe('runtimeEnv', () => {
it('leaves out what the server does not have, and sets the rest all the same', () => {
expect(Object.keys(runtimeEnv(input({}))).sort()).toEqual([
'HOME',
'LLAMA_ARG_CACHE_RAM',
'OLLAMA_CONTEXT_LENGTH',
'OLLAMA_FLASH_ATTENTION',
'OLLAMA_HOST',
'OLLAMA_KEEP_ALIVE',
'OLLAMA_KV_CACHE_TYPE',
'OLLAMA_MAX_LOADED_MODELS',
'OLLAMA_MODELS',
'OLLAMA_NO_CLOUD',
+3
View File
@@ -206,6 +206,9 @@ describe('start', () => {
OLLAMA_KEEP_ALIVE: '10m',
OLLAMA_NUM_PARALLEL: '1',
OLLAMA_MAX_LOADED_MODELS: '1',
OLLAMA_KV_CACHE_TYPE: 'q8_0',
OLLAMA_FLASH_ATTENTION: '1',
LLAMA_ARG_CACHE_RAM: '1024',
});
});