From 0cedeb2657f8bb1c3f5da1a5db073cb4957d77f6 Mon Sep 17 00:00:00 2001 From: Ludo Fourrage Date: Sat, 24 Oct 2026 15:57:07 +0000 Subject: [PATCH] fix(voice): make dictation silence timeout configurable --- .changeset/quiet-voice-pauses.md | 5 + docs/wiki/Input-And-Voice.md | 8 ++ src/web/public/index.html | 14 +++ src/web/public/settings-ui.js | 4 + src/web/public/voice-input.js | 108 +++++++++++++------- src/web/schemas.ts | 2 + test/voice-silence-timeout.test.ts | 153 +++++++++++++++++++++++++++++ 7 files changed, 260 insertions(+), 34 deletions(-) create mode 100644 .changeset/quiet-voice-pauses.md create mode 100644 test/voice-silence-timeout.test.ts diff --git a/.changeset/quiet-voice-pauses.md b/.changeset/quiet-voice-pauses.md new file mode 100644 index 00000000..aa529f8a --- /dev/null +++ b/.changeset/quiet-voice-pauses.md @@ -0,0 +1,5 @@ +--- +"aicodeman": patch +--- + +Add a configurable voice silence timeout and manual-stop dictation so thinking pauses do not cut off recording. diff --git a/docs/wiki/Input-And-Voice.md b/docs/wiki/Input-And-Voice.md index 928e511f..35c89830 100644 --- a/docs/wiki/Input-And-Voice.md +++ b/docs/wiki/Input-And-Voice.md @@ -78,6 +78,14 @@ sequences, so word jumps and history keys keep working. | **Deepgram** | A Deepgram API key. | Nova-3, with automatic silence detection. | | **Web Speech** | Nothing. | Browser-provided, quality varies. | +### Thinking pauses + +In **App Settings → Voice → Stop after silence**, choose 3, 6, 15 or 30 seconds, +or **Manual** to keep listening until you tap the mic again (`Ctrl+Shift+V`). +**Provider default** preserves the existing timeout: 6 seconds for Claude and +3 seconds for Deepgram or Web Speech. The timeout counts from the latest transcript +update. Connection failures or microphone errors can still end a recording. + ### Dictating through your Claude login Off by default; enable it in **App Settings → Voice**. diff --git a/src/web/public/index.html b/src/web/public/index.html index f840e25f..09cf82e3 100644 --- a/src/web/public/index.html +++ b/src/web/public/index.html @@ -3456,6 +3456,20 @@ +
+
+ + Allow time to think between phrases. Manual keeps listening until you tap the mic again. +
+ +
Language diff --git a/src/web/public/settings-ui.js b/src/web/public/settings-ui.js index 80360345..6e7d9221 100644 --- a/src/web/public/settings-ui.js +++ b/src/web/public/settings-ui.js @@ -810,6 +810,8 @@ Object.assign(CodemanApp.prototype, { document.getElementById('voiceKeyterms').value = voiceCfg.keyterms || DEFAULT_VOICE_KEYTERMS; document.getElementById('voiceInsertMode').value = voiceCfg.insertMode || 'direct'; document.getElementById('voiceProvider').value = voiceCfg.provider || 'auto'; + document.getElementById('voiceSilenceTimeout').value = [0, 3, 6, 15, 30].includes(voiceCfg.silenceTimeout) + ? String(voiceCfg.silenceTimeout) : 'default'; document.getElementById('appSettingsClaudeVoice').checked = settings.claudeVoiceEnabled ?? false; // Reset key visibility to hidden const keyInput = document.getElementById('voiceDeepgramKey'); @@ -3297,6 +3299,8 @@ Object.assign(CodemanApp.prototype, { language: document.getElementById('voiceLanguage').value, keyterms: document.getElementById('voiceKeyterms').value.trim(), insertMode: document.getElementById('voiceInsertMode').value, + silenceTimeout: document.getElementById('voiceSilenceTimeout').value === 'default' + ? undefined : Number(document.getElementById('voiceSilenceTimeout').value), }; VoiceInput._saveDeepgramConfig(voiceSettings); diff --git a/src/web/public/voice-input.js b/src/web/public/voice-input.js index 741296ab..27f0c2a3 100644 --- a/src/web/public/voice-input.js +++ b/src/web/public/voice-input.js @@ -14,7 +14,7 @@ * auto-detects MIME type (opus/webm/mp4), and supports custom key terms for dev vocabulary. * * - VoiceInput — High-level voice input controller. Toggle mode: tap mic to start, tap - * again to stop. Auto-stops after 3s silence. Shows floating preview overlay with recording + * again to stop. Auto-stops after configurable silence. Shows floating preview overlay with recording * indicator, level meter (AnalyserNode), and elapsed timer. Two insert modes: "direct" * (inject into local echo overlay or PTY) and "compose" (editable textarea overlay). * Includes a temporary green Send button that replaces the settings gear icon after voice input. @@ -216,9 +216,12 @@ const DeepgramProvider = { _resetSilenceTimeout() { clearTimeout(this._silenceTimeout); + this._silenceTimeout = null; + const delay = VoiceInput.getSilenceTimeoutMs(3000); + if (delay === 0) return; this._silenceTimeout = setTimeout(() => { this.stop(); - }, 3000); + }, delay); }, stop() { @@ -455,7 +458,10 @@ const ClaudeVoiceProvider = { _resetSilenceTimeout() { clearTimeout(this._silenceTimeout); - this._silenceTimeout = setTimeout(() => this.stop(), this.SILENCE_MS); + this._silenceTimeout = null; + const delay = VoiceInput.getSilenceTimeoutMs(this.SILENCE_MS); + if (delay === 0) return; + this._silenceTimeout = setTimeout(() => this.stop(), delay); }, /** @@ -545,6 +551,7 @@ const VoiceInput = { _lastTranscript: '', _stabilityTimer: null, _accumulatedFinal: '', + _pendingInterim: '', _activeProvider: null, // 'deepgram' | 'webspeech' | null _recordingStartedAt: 0, // timestamp when recording started _retryCount: 0, // auto-retry counter for premature Web Speech API ends @@ -599,6 +606,12 @@ const VoiceInput = { } }, + /** Seconds in settings; zero means stop only when the user asks. */ + getSilenceTimeoutMs(fallback) { + const seconds = this._getDeepgramConfig().silenceTimeout; + return [0, 3, 6, 15, 30].includes(seconds) ? seconds * 1000 : fallback; + }, + _saveDeepgramConfig(config) { localStorage.setItem('codeman-voice-settings', JSON.stringify(config)); }, @@ -706,6 +719,7 @@ const VoiceInput = { this.isRecording = true; this._activeProvider = 'claude'; this._accumulatedFinal = ''; + this._pendingInterim = ''; this._lastTranscript = ''; this._hasReceivedResult = false; this._recordingStartedAt = Date.now(); @@ -728,22 +742,19 @@ const VoiceInput = { // Each frame is the WHOLE running transcript, so replace rather than append. this._accumulatedFinal = text; if (isFinal) { - this._hidePreview(); - this._insertText(text); - this.stop(); + this.stop(true); } else { this._showPreview(text, 'claude'); } }, onError: (msg) => { const wasRecording = this.isRecording; - this.stop(); + this.stop(true); if (wasRecording) app.showToast(msg, 'error'); }, onEnd: () => { if (this.isRecording) { - if (this._accumulatedFinal) this._insertText(this._accumulatedFinal); - this.stop(); + this.stop(true); } } }); @@ -756,6 +767,7 @@ const VoiceInput = { this.isRecording = true; this._activeProvider = 'deepgram'; this._accumulatedFinal = ''; + this._pendingInterim = ''; this._lastTranscript = ''; this._hasReceivedResult = false; this._recordingStartedAt = Date.now(); @@ -777,12 +789,12 @@ const VoiceInput = { if (!this.isRecording) return; this._hasReceivedResult = true; if (isFinal) { - this._accumulatedFinal += text; - this._hidePreview(); - this._insertText(this._accumulatedFinal); - this.stop(); + this._accumulatedFinal += (this._accumulatedFinal ? ' ' : '') + text; + this._pendingInterim = ''; + this._showPreview(this._accumulatedFinal, 'deepgram'); } else { - const display = this._accumulatedFinal + text; + this._pendingInterim = (this._accumulatedFinal ? ' ' : '') + text; + const display = this._accumulatedFinal + this._pendingInterim; this._showPreview(display, 'deepgram'); } }, @@ -793,9 +805,6 @@ const VoiceInput = { }, onEnd: () => { if (this.isRecording) { - if (this._accumulatedFinal) { - this._insertText(this._accumulatedFinal); - } this.stop(); } } @@ -819,6 +828,7 @@ const VoiceInput = { this.isRecording = true; this._activeProvider = 'webspeech'; this._accumulatedFinal = ''; + this._pendingInterim = ''; this._lastTranscript = ''; this._hasReceivedResult = false; this._recordingStartedAt = Date.now(); @@ -849,9 +859,19 @@ const VoiceInput = { if (navigator.vibrate) navigator.vibrate(50); }, - stop() { + stop(claudeFinished = false) { if (!this.isRecording) return; + // Claude delivers its final transcript after finalize. Keep accepting it + // until that frame (or onEnd) completes this recording. + if (this._activeProvider === 'claude' && !claudeFinished && ClaudeVoiceProvider._ws?.readyState === WebSocket.OPEN) { + ClaudeVoiceProvider.stop(); + return; + } this.isRecording = false; + const text = this._accumulatedFinal + this._pendingInterim; + this._accumulatedFinal = ''; + this._pendingInterim = ''; + if (text) this._insertText(text); clearTimeout(this.silenceTimeout); clearTimeout(this._stabilityTimer); this.silenceTimeout = null; @@ -901,11 +921,14 @@ const VoiceInput = { } if (finalText) { + clearTimeout(this._stabilityTimer); + this._stabilityTimer = null; this._accumulatedFinal += finalText; - this._hidePreview(); - this._insertText(this._accumulatedFinal); - this.stop(); - } else if (interim) { + this._pendingInterim = interim; + this._showPreview(this._accumulatedFinal + interim); + } + if (interim) { + this._pendingInterim = interim; const display = this._accumulatedFinal + interim; this._showPreview(display); // iOS Safari workaround: isFinal is always false. @@ -918,6 +941,9 @@ const VoiceInput = { // During auto-retry, 'aborted' and 'no-speech' errors are expected — ignore them if (this._retryCount > 0 && (event.error === 'aborted' || event.error === 'no-speech')) return; + // Browser silence events must not override an extended/manual deadline. + if (event.error === 'no-speech' && this.getSilenceTimeoutMs(3000) !== 3000) return; + const wasRecording = this.isRecording; this.stop(); if (!wasRecording) return; @@ -953,16 +979,30 @@ const VoiceInput = { this.recognition.start(); } catch (_e) { // If restart fails, fall through to stop - if (this._accumulatedFinal) this._insertText(this._accumulatedFinal); this.stop(); } return; } - // Genuine end — finalize any accumulated text - if (this._accumulatedFinal) { - this._insertText(this._accumulatedFinal); + // Some browsers end recognition independently of our silence timer. For an + // extended/manual session, restart without resetting the original deadline. + const delay = this.getSilenceTimeoutMs(3000); + if (delay === 0 || delay > 3000) { + try { + // Each recognition run starts a fresh result list. Retain Safari's + // interim-only phrase before the next run can replace it. + if (this._pendingInterim) { + this._accumulatedFinal += this._pendingInterim + ' '; + this._pendingInterim = ''; + } + this.recognition.start(); + return; + } catch (_e) { + // A failed restart ends recording normally. + } } + + // Genuine end — finalize any accumulated text this.stop(); }, @@ -1138,26 +1178,25 @@ const VoiceInput = { _resetSilenceTimeout() { clearTimeout(this.silenceTimeout); + this.silenceTimeout = null; + const delay = this.getSilenceTimeoutMs(3000); + if (delay === 0) return; this.silenceTimeout = setTimeout(() => { if (this.isRecording) { - // Finalize any accumulated text before stopping - if (this._accumulatedFinal) { - this._insertText(this._accumulatedFinal); - } this.stop(); } - }, 3000); + }, delay); }, _iosStabilityCheck(transcript) { + // The Safari shortcut must not override the chosen thinking pause. + if (this.getSilenceTimeoutMs(3000) !== 3000) return; if (transcript !== this._lastTranscript) { this._lastTranscript = transcript; clearTimeout(this._stabilityTimer); this._stabilityTimer = setTimeout(() => { if (this.isRecording) { - const finalText = this._accumulatedFinal + transcript; - this._hidePreview(); - this._insertText(finalText); + this._pendingInterim = transcript; this.stop(); } }, 750); @@ -1340,6 +1379,7 @@ const VoiceInput = { this.silenceTimeout = null; this._stabilityTimer = null; this._accumulatedFinal = ''; + this._pendingInterim = ''; this._lastTranscript = ''; this._retryCount = 0; this._hasReceivedResult = false; diff --git a/src/web/schemas.ts b/src/web/schemas.ts index 5fb327c3..7104fef5 100644 --- a/src/web/schemas.ts +++ b/src/web/schemas.ts @@ -1783,6 +1783,8 @@ export const SettingsUpdateSchema = z language: z.string().max(20).optional(), keyterms: z.string().max(500).optional(), insertMode: z.string().max(20).optional(), + /** Silence timeout in seconds; 0 disables automatic stopping. Omitted keeps provider defaults. */ + silenceTimeout: z.union([z.literal(0), z.literal(3), z.literal(6), z.literal(15), z.literal(30)]).optional(), }) .optional(), // Run mode preference (cross-device sync) diff --git a/test/voice-silence-timeout.test.ts b/test/voice-silence-timeout.test.ts new file mode 100644 index 00000000..d903b750 --- /dev/null +++ b/test/voice-silence-timeout.test.ts @@ -0,0 +1,153 @@ +import { readFileSync } from 'node:fs'; +import vm from 'node:vm'; +import { describe, expect, it, vi } from 'vitest'; +import { SettingsUpdateSchema } from '../src/web/schemas.js'; + +const source = readFileSync(new URL('../src/web/public/voice-input.js', import.meta.url), 'utf8'); +function load(silenceTimeout?: unknown) { + vi.useRealTimers(); + vi.useFakeTimers(); + const context = vm.createContext({ + setTimeout, + clearTimeout, + console, + navigator: {}, + WebSocket: { OPEN: 1 }, + localStorage: { getItem: () => JSON.stringify({ silenceTimeout }) }, + }); + const providers = vm.runInContext(`${source}\n[DeepgramProvider, ClaudeVoiceProvider, VoiceInput]`, context); + return providers as Array>; +} + +describe('voice silence timeout', () => { + for (const [index, name, legacy] of [ + [0, 'Deepgram', 3000], + [1, 'Claude', 6000], + [2, 'browser', 3000], + ] as const) { + it(`${name} keeps its default and accepts longer thinking pauses`, () => { + for (const [setting, delay] of [ + [undefined, legacy], + [15, 15000], + [30, 30000], + ] as const) { + const provider = load(setting)[index]; + provider.isRecording = true; + provider.stop = vi.fn(); + provider._resetSilenceTimeout(); + vi.advanceTimersByTime(delay - 1); + expect(provider.stop).not.toHaveBeenCalled(); + vi.advanceTimersByTime(1); + expect(provider.stop).toHaveBeenCalledOnce(); + } + }); + it(`${name} resets its deadline and cancels it in manual mode`, () => { + const providers = load(15); + const provider = providers[index]; + provider.isRecording = true; + provider.stop = vi.fn(); + provider._resetSilenceTimeout(); + vi.advanceTimersByTime(10000); + provider._resetSilenceTimeout(); + vi.advanceTimersByTime(10000); + expect(provider.stop).not.toHaveBeenCalled(); + providers[2]._getDeepgramConfig = () => ({ silenceTimeout: 0 }); + provider._resetSilenceTimeout(); + vi.advanceTimersByTime(120000); + expect(provider.stop).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + } + it.each([null, -1, '15', 1.5, 999])('ignores invalid local values: %s', (setting) => { + const voice = load(setting)[2]; + expect(voice.getSilenceTimeoutMs(6000)).toBe(6000); + }); + it('does not let the iOS stability shortcut stop extended/manual dictation', () => { + for (const setting of [0, 15]) { + const voice = load(setting)[2]; + voice.isRecording = true; + voice._iosStabilityCheck('thinking'); + expect(vi.getTimerCount()).toBe(0); + } + }); + it('restarts browser recognition without extending the silence deadline', () => { + const voice = load(15)[2]; + voice.isRecording = true; + voice._recordingStartedAt = Date.now() - 1000; + voice.recognition = { start: vi.fn() }; + voice.stop = vi.fn(); + voice._resetSilenceTimeout(); + vi.advanceTimersByTime(10000); + voice._onWebSpeechEnd(); + expect(voice.recognition.start).toHaveBeenCalledOnce(); + vi.advanceTimersByTime(5000); + expect(voice.stop).toHaveBeenCalledOnce(); + }); + it('keeps final browser phrases and Safari interim text until an explicit stop', () => { + const voice = load(0)[2]; + voice.isRecording = true; + voice._activeProvider = 'webspeech'; + for (const method of ['_showPreview', '_stopDurationTimer', '_stopLevelMeter', '_updateButtons', '_hidePreview']) + voice[method] = vi.fn(); + voice._insertText = vi.fn(); + voice.recognition = { stop: vi.fn() }; + const result = [{ transcript: 'first phrase ' }]; + Object.assign(result, { isFinal: true }); + voice._onWebSpeechResult({ resultIndex: 0, results: [result] }); + expect(voice.isRecording).toBe(true); + expect(voice._insertText).not.toHaveBeenCalled(); + const interim = [{ transcript: 'thinking aloud' }]; + voice._onWebSpeechResult({ resultIndex: 0, results: [interim] }); + voice.stop(); + voice.stop(); + expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('first phrase thinking aloud'); + }); + it('preserves mixed final/interim results across browser recognition restarts', () => { + const voice = load(0)[2]; + voice.isRecording = true; + voice._recordingStartedAt = Date.now() - 1000; + voice._showPreview = vi.fn(); + voice.recognition = { start: vi.fn() }; + const final = Object.assign([{ transcript: 'first ' }], { isFinal: true }); + const interim = [{ transcript: 'second' }]; + voice._onWebSpeechResult({ resultIndex: 0, results: [final, interim] }); + expect(voice._accumulatedFinal + voice._pendingInterim).toBe('first second'); + voice._onWebSpeechEnd(); + voice._onWebSpeechResult({ resultIndex: 0, results: [[{ transcript: 'third' }]] }); + expect(voice._accumulatedFinal + voice._pendingInterim).toBe('first second third'); + }); + it('ignores repeated stop requests while waiting for Claude final text', () => { + const [_, provider, voice] = load(0); + voice.isRecording = true; + voice._activeProvider = 'claude'; + provider._ws = { readyState: 1 }; + provider.stop = vi.fn(() => { + provider._finalized = true; + }); + voice._insertText = vi.fn(); + voice.stop(); + voice.stop(); + expect(voice._insertText).not.toHaveBeenCalled(); + expect(voice.isRecording).toBe(true); + }); + it('cancels Safari stability text when a final result supersedes it', () => { + const voice = load(3)[2]; + voice.isRecording = true; + voice._showPreview = vi.fn(); + voice.stop = vi.fn(); + voice._onWebSpeechResult({ resultIndex: 0, results: [[{ transcript: 'hello' }]] }); + const final = Object.assign([{ transcript: 'hello' }], { isFinal: true }); + voice._onWebSpeechResult({ resultIndex: 0, results: [final] }); + vi.advanceTimersByTime(750); + expect(voice.stop).not.toHaveBeenCalled(); + expect(voice._accumulatedFinal + voice._pendingInterim).toBe('hello'); + }); + it('round-trips the setting through server validation', () => { + for (const silenceTimeout of [0, 3, 6, 15, 30]) { + expect(SettingsUpdateSchema.parse({ voiceSettings: { silenceTimeout } }).voiceSettings).toEqual({ + silenceTimeout, + }); + } + expect(SettingsUpdateSchema.safeParse({ voiceSettings: { silenceTimeout: -1 } }).success).toBe(false); + }); +});