From 2d7cd4656530c416f08b9a15fc3a2beb3a610b5f Mon Sep 17 00:00:00 2001 From: Ludo Fourrage Date: Sat, 24 Oct 2026 19:04:49 +0000 Subject: [PATCH] fix(voice): preserve Android transcripts and Claude utterances --- .changeset/quiet-voice-pauses.md | 6 +- CLAUDE.md | 2 +- config/test-suites.ts | 1 + docs/architecture-invariants.md | 2 +- docs/wiki/Input-And-Voice.md | 16 +- src/web/public/settings-ui.js | 2 +- src/web/public/voice-input.js | 297 +++++++++++-------- src/web/voice-stream.ts | 2 +- test/voice-input-results.test.ts | 443 +++++++++++++++++++++++++++++ test/voice-input.browser.test.ts | 85 ++++++ test/voice-silence-timeout.test.ts | 213 +++++++++++++- 11 files changed, 930 insertions(+), 139 deletions(-) create mode 100644 test/voice-input-results.test.ts create mode 100644 test/voice-input.browser.test.ts diff --git a/.changeset/quiet-voice-pauses.md b/.changeset/quiet-voice-pauses.md index aa529f8a..1730e86a 100644 --- a/.changeset/quiet-voice-pauses.md +++ b/.changeset/quiet-voice-pauses.md @@ -1,5 +1,9 @@ --- -"aicodeman": patch +'aicodeman': patch --- Add a configurable voice silence timeout and manual-stop dictation so thinking pauses do not cut off recording. + +Provider-default Deepgram and Web Speech final events now wait for the 3-second silence deadline instead of stopping immediately. Claude keeps recording across utterance endpoints and preserves final words after Stop, with a bounded wait on failed connections. + +Fix repeated words and lost final transcripts on Chrome for Android, animate browser speech activity without a competing microphone capture, and preserve partial text when reconnecting. Bound repeated empty browser recognition restarts. diff --git a/CLAUDE.md b/CLAUDE.md index 245016ee..7fe412b5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -216,7 +216,7 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph **Agent inbox** (discussion #558; persistence `agentInboxPersistEnabled`, SYNCED, default OFF): per-session mailbox in `web/agent-inbox.ts`, routes `routes/inbox-routes.ts`, CLI `codeman agent post|inbox|ack`. Never writes to a pane, never in `toState()` (counts via `GET /api/agent-inbox/summary`). ⚠️ Read until ack by id; seen/ack only for the receiver resolved via `resolveParentSessionId()` (ack from anyone else = 403); `?wait=` parks while nothing is UNSEEN; full = 409, never drop the oldest; dropped on delete (`killMux`) only, boot prune only after a successful restore. → [architecture-invariants#agent-inbox](docs/architecture-invariants.md#agent-inbox) -**Voice dictation via Claude** (`claudeVoiceEnabled`, SYNCED, default OFF): the mic transcribes through this machine's Claude Code login (the CLI `/voice` backend) instead of Deepgram; the browser captures, `src/web/voice-stream.ts` relays to Anthropic. ⚠️ The OAuth token never reaches the page. ⚠️ Credentials are READ-ONLY (`src/claude-credentials.ts`): never refresh them (it rotates the refresh token and can sign the user out of their CLI). ⚠️ Capture must be linear16/16 kHz/mono via an AudioWorklet; `voice-pcm-worklet.js` borrows voice-input.js's `?v=` token, so edit the two together. ⚠️ Claude transcript frames are cumulative: replace, never append. → [architecture-invariants#voice-dictation-via-claude](docs/architecture-invariants.md#voice-dictation-via-claude) +**Voice dictation via Claude** (`claudeVoiceEnabled`, SYNCED, default OFF): the mic transcribes through this machine's Claude Code login (the CLI `/voice` backend) instead of Deepgram; the browser captures, `src/web/voice-stream.ts` relays to Anthropic. ⚠️ The OAuth token never reaches the page. ⚠️ Credentials are READ-ONLY (`src/claude-credentials.ts`): never refresh them (it rotates the refresh token and can sign the user out of their CLI). ⚠️ Capture must be linear16/16 kHz/mono via an AudioWorklet; `voice-pcm-worklet.js` borrows voice-input.js's `?v=` token, so edit the two together. ⚠️ Claude transcript frames are cumulative within each utterance: replace interims, append endpoint finals once, and keep recording until Stop or the configured silence deadline. → [architecture-invariants#voice-dictation-via-claude](docs/architecture-invariants.md#voice-dictation-via-claude) **Agent Teams**: `TeamWatcher` polls `~/.claude/teams/`, matches to sessions via `leadSessionId`. Teammates are in-process threads appearing as subagents. Enable: `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`. See `docs/agent-teams/`. diff --git a/config/test-suites.ts b/config/test-suites.ts index ffcfb2f8..54d1792e 100644 --- a/config/test-suites.ts +++ b/config/test-suites.ts @@ -21,6 +21,7 @@ export const BROWSER_TEST_GLOBS = [ 'test/wrapped-file-paths.browser.test.ts', 'test/command-palette-agent-arrows.browser.test.ts', + 'test/voice-input.browser.test.ts', 'test/tmux-capture-sgr.browser.test.ts', 'test/tab-rail-resize.browser.test.ts', 'test/tab-activation.browser.test.ts', diff --git a/docs/architecture-invariants.md b/docs/architecture-invariants.md index d280b477..8d99970d 100644 --- a/docs/architecture-invariants.md +++ b/docs/architecture-invariants.md @@ -782,7 +782,7 @@ Frontend `readmymind-ui.js`: header 🧠 marker-hidden (`btn-readmymind--hidden` ⚠️ Capture MUST be linear16/16 kHz/mono, so it uses an **AudioWorklet**, not MediaRecorder (which cannot emit raw PCM); `voice-pcm-worklet.js` is fetched from JS, so it is invisible to `cacheBustAssets` and borrows voice-input.js's `?v=` token — **edit the two together**. -⚠️ Transcript frames carry the WHOLE running transcript, not deltas: the Claude path replaces where the Deepgram path appends. Provider choice is `voiceSettings.provider` (`auto` prefers Claude → Deepgram → Web Speech). → `docs/claude-voice-plan.md` +⚠️ Claude transcript frames are cumulative within the current utterance: replace interims, then append each endpoint final once. An endpoint does not stop dictation unless finalization was requested. Finalization has a 5-second client backstop; reconnect cleanup salvages accumulated/interim text and releases the microphone immediately. Provider choice is `voiceSettings.provider` (`auto` prefers Claude → Deepgram → Web Speech). → `docs/claude-voice-plan.md` ### Files panel search diff --git a/docs/wiki/Input-And-Voice.md b/docs/wiki/Input-And-Voice.md index 35c89830..7ad8ba15 100644 --- a/docs/wiki/Input-And-Voice.md +++ b/docs/wiki/Input-And-Voice.md @@ -82,9 +82,19 @@ sequences, so word jumps and history keys keep working. In **App Settings → Voice → Stop after silence**, choose 3, 6, 15 or 30 seconds, or **Manual** to keep listening until you tap the mic again (`Ctrl+Shift+V`). -**Provider default** preserves the existing timeout: 6 seconds for Claude and -3 seconds for Deepgram or Web Speech. The timeout counts from the latest transcript -update. Connection failures or microphone errors can still end a recording. +**Provider default** uses 6 seconds for Claude and 3 seconds for Deepgram or Web +Speech. Final transcript events no longer stop Deepgram or Web Speech immediately: +they wait for the configured silence deadline. The timeout counts from the latest +transcript update. A browser can still end its native recognition run; extended +and Manual dictation restart it while preserving the text and silence deadline. +Connection failures, microphone errors, or repeated empty recognition runs can +still end a recording. + +On Chrome for Android, live words replace earlier partial results without repeating +them in the input. The browser microphone bars indicate speech activity; streaming +providers show audio levels. After Stop, dictation briefly waits for the final +words before inserting text. Reconnecting preserves the available words and resets +the mic so another recording can start. ### Dictating through your Claude login diff --git a/src/web/public/settings-ui.js b/src/web/public/settings-ui.js index 6e7d9221..4c637b23 100644 --- a/src/web/public/settings-ui.js +++ b/src/web/public/settings-ui.js @@ -810,7 +810,7 @@ Object.assign(CodemanApp.prototype, { document.getElementById('voiceKeyterms').value = voiceCfg.keyterms || DEFAULT_VOICE_KEYTERMS; document.getElementById('voiceInsertMode').value = voiceCfg.insertMode || 'direct'; document.getElementById('voiceProvider').value = voiceCfg.provider || 'auto'; - document.getElementById('voiceSilenceTimeout').value = [0, 3, 6, 15, 30].includes(voiceCfg.silenceTimeout) + document.getElementById('voiceSilenceTimeout').value = VoiceInput.SILENCE_TIMEOUT_SECONDS.includes(voiceCfg.silenceTimeout) ? String(voiceCfg.silenceTimeout) : 'default'; document.getElementById('appSettingsClaudeVoice').checked = settings.claudeVoiceEnabled ?? false; // Reset key visibility to hidden diff --git a/src/web/public/voice-input.js b/src/web/public/voice-input.js index 27f0c2a3..b33b9a04 100644 --- a/src/web/public/voice-input.js +++ b/src/web/public/voice-input.js @@ -18,7 +18,7 @@ * indicator, level meter (AnalyserNode), and elapsed timer. Two insert modes: "direct" * (inject into local echo overlay or PTY) and "compose" (editable textarea overlay). * Includes a temporary green Send button that replaces the settings gear icon after voice input. - * Web Speech API has auto-retry (up to 2x) for premature onend and iOS Safari stability check. + * Web Speech API retries empty runs twice with backoff and preserves late results after Stop. * * @globals {object} ClaudeVoiceProvider * @globals {object} DeepgramProvider @@ -272,8 +272,8 @@ const DeepgramProvider = { * DeepgramProvider's recorder. The AudioContext is constructed at 16000 Hz so the * browser does the resampling. * - * ⚠️ Transcript frames carry the WHOLE running transcript, not deltas. Callers - * must replace, never concatenate. + * Transcript frames cover the current utterance. Replace interims and append + * each endpoint final once; an endpoint alone does not end the recording. */ const ClaudeVoiceProvider = { _ws: null, @@ -287,6 +287,8 @@ const ClaudeVoiceProvider = { _onError: null, _onEnd: null, _finalized: false, + _finalizeTimeout: null, + _generation: 0, /** How long without any transcript before the recording gives up on its own. */ SILENCE_MS: 6000, @@ -296,6 +298,8 @@ const ClaudeVoiceProvider = { * @param {object} opts - { language, keyterms[], onResult(text, isFinal), onError(msg), onEnd(), onStream(stream) } */ async start(opts) { + this._cleanup(); + const generation = this._generation; this._onResult = opts.onResult; this._onError = opts.onError; this._onEnd = opts.onEnd; @@ -307,10 +311,17 @@ const ClaudeVoiceProvider = { return; } try { - this._stream = await navigator.mediaDevices.getUserMedia({ + const stream = await navigator.mediaDevices.getUserMedia({ audio: { noiseSuppression: true, echoCancellation: true, autoGainControl: true } }); + // Permission prompts can resolve after Stop/reconnect or a newer recording. + if (generation !== this._generation || this._finalized) { + stream.getTracks().forEach(t => t.stop()); + return; + } + this._stream = stream; } catch (err) { + if (generation !== this._generation || this._finalized) return; const msg = err.name === 'NotAllowedError' ? 'Microphone access denied. Check browser settings.' : 'Microphone error: ' + err.message; @@ -334,10 +345,12 @@ const ClaudeVoiceProvider = { this._ws.binaryType = 'arraybuffer'; this._ws.onopen = () => { + if (generation !== this._generation || this._finalized) return; // Capture starts only once the socket is up: PCM buffered before that would // be the oldest audio, and dropping it keeps the transcript aligned with what // the user hears themselves saying. - this._startCapture().catch((err) => { + this._startCapture(generation).catch((err) => { + if (generation !== this._generation || this._finalized) return; this._onError?.('Microphone capture failed: ' + err.message); this.stop(); }); @@ -345,6 +358,7 @@ const ClaudeVoiceProvider = { }; this._ws.onmessage = (event) => { + if (generation !== this._generation) return; let msg; try { msg = JSON.parse(event.data); @@ -364,6 +378,7 @@ const ClaudeVoiceProvider = { }; this._ws.onclose = (event) => { + if (generation !== this._generation) return; if (event.code === 4004) { this._onError?.(this._unavailableMessage(event.reason)); } else if (event.code === 4008) { @@ -388,16 +403,19 @@ const ClaudeVoiceProvider = { }, /** Wire mic -> 16 kHz PCM16 frames -> WebSocket. */ - async _startCapture() { + async _startCapture(generation = this._generation) { + if (generation !== this._generation || this._finalized) return; const Ctx = window.AudioContext || window.webkitAudioContext; // Ask for 16 kHz directly so the browser resamples; Safari may hand back its // own rate, which _pcmFromFloat32 then downsamples to match. this._audioContext = new Ctx({ sampleRate: 16000 }); if (this._audioContext.state === 'suspended') await this._audioContext.resume(); + if (generation !== this._generation || this._finalized) return; this._sourceNode = this._audioContext.createMediaStreamSource(this._stream); if (this._audioContext.audioWorklet) { await this._audioContext.audioWorklet.addModule(this._workletUrl()); + if (generation !== this._generation || this._finalized) return; this._workletNode = new AudioWorkletNode(this._audioContext, 'pcm-frame-processor'); this._workletNode.port.onmessage = (event) => this._sendAudio(event.data); this._sourceNode.connect(this._workletNode); @@ -475,6 +493,13 @@ const ClaudeVoiceProvider = { if (this._finalized) return; this._finalized = true; this._stopCapture(); + // Bound both manual and silence-triggered finalization on a dead connection. + // Repeated stop requests return above, so they cannot extend this deadline. + this._finalizeTimeout = setTimeout(() => { + const onEnd = this._onEnd; + this._cleanup(); + onEnd?.(); + }, 5000); if (this._ws?.readyState === WebSocket.OPEN) { try { this._ws.send(JSON.stringify({ t: 'finalize' })); @@ -516,15 +541,19 @@ const ClaudeVoiceProvider = { /** Hard stop: drop the socket without waiting for a final transcript. */ _cleanup() { + this._generation++; this._finalized = true; + clearTimeout(this._finalizeTimeout); + this._finalizeTimeout = null; clearTimeout(this._silenceTimeout); this._silenceTimeout = null; this._stopCapture(); if (this._ws) { + this._ws.onopen = null; this._ws.onclose = null; this._ws.onmessage = null; this._ws.onerror = null; - if (this._ws.readyState === WebSocket.OPEN) { + if (this._ws.readyState === WebSocket.OPEN || this._ws.readyState === WebSocket.CONNECTING) { try { this._ws.close(1000); } catch (_e) { /* ignore */ } } this._ws = null; @@ -543,19 +572,25 @@ const ClaudeVoiceProvider = { * Inserts final text into the active session (user presses Enter to submit). */ const VoiceInput = { + SILENCE_TIMEOUT_SECONDS: Object.freeze([0, 3, 6, 15, 30]), recognition: null, isRecording: false, supported: false, silenceTimeout: null, previewEl: null, - _lastTranscript: '', - _stabilityTimer: null, _accumulatedFinal: '', _pendingInterim: '', - _activeProvider: null, // 'deepgram' | 'webspeech' | null + _webSpeechCommitted: '', // Text from completed native recognition runs. + _webSpeechSingleUtterance: false, + _webSpeechSoundActive: false, + _webSpeechActivityUntil: 0, + _webSpeechStopping: false, + _webSpeechStopTimer: null, + _webSpeechRestartTimer: null, + _activeProvider: null, // 'claude' | 'deepgram' | 'webspeech' | null _recordingStartedAt: 0, // timestamp when recording started - _retryCount: 0, // auto-retry counter for premature Web Speech API ends - _hasReceivedResult: false, // whether any speech result came in this session + _retryCount: 0, // consecutive empty Web Speech API runs + _hasReceivedResult: false, // whether this native recognition run received speech _durationInterval: null, // timer for updating elapsed time display _analyser: null, // AudioContext analyser for level meter _analyserSource: null, // MediaStreamSource for level meter @@ -609,7 +644,7 @@ const VoiceInput = { /** Seconds in settings; zero means stop only when the user asks. */ getSilenceTimeoutMs(fallback) { const seconds = this._getDeepgramConfig().silenceTimeout; - return [0, 3, 6, 15, 30].includes(seconds) ? seconds * 1000 : fallback; + return this.SILENCE_TIMEOUT_SECONDS.includes(seconds) ? seconds * 1000 : fallback; }, _saveDeepgramConfig(config) { @@ -661,7 +696,14 @@ const VoiceInput = { if (!this.supported) return; this.recognition = new SR(); - this.recognition.continuous = true; + // Android's continuous bridge promotes growing partials to separate finals. + // Single-utterance mode keeps refinements provisional; onend resumes the mic + // while our own silence deadline controls the overall dictation session. + // Android tablets in desktop-site mode may identify as touch-capable Linux. + this._webSpeechSingleUtterance = navigator.userAgentData?.platform === 'Android' + || /Android/i.test(navigator.userAgent || '') + || (navigator.maxTouchPoints > 0 && /Linux/i.test(navigator.platform || '')); + this.recognition.continuous = !this._webSpeechSingleUtterance; this.recognition.interimResults = true; this.recognition.lang = 'en-US'; this.recognition.maxAlternatives = 1; @@ -669,6 +711,8 @@ const VoiceInput = { this.recognition.onresult = (e) => this._onWebSpeechResult(e); this.recognition.onerror = (e) => this._onWebSpeechError(e); this.recognition.onend = () => this._onWebSpeechEnd(); + this.recognition.onsoundstart = this.recognition.onspeechstart = () => this._onWebSpeechSound(true); + this.recognition.onsoundend = this.recognition.onspeechend = this.recognition.onaudioend = () => this._onWebSpeechSound(false); }, toggle() { @@ -720,7 +764,6 @@ const VoiceInput = { this._activeProvider = 'claude'; this._accumulatedFinal = ''; this._pendingInterim = ''; - this._lastTranscript = ''; this._hasReceivedResult = false; this._recordingStartedAt = Date.now(); this._updateButtons('recording'); @@ -739,12 +782,15 @@ const VoiceInput = { onResult: (text, isFinal) => { if (!this.isRecording) return; this._hasReceivedResult = true; - // Each frame is the WHOLE running transcript, so replace rather than append. - this._accumulatedFinal = text; + // Interims replace the current utterance; each endpoint commits it once. if (isFinal) { - this.stop(true); + this._accumulatedFinal = [this._accumulatedFinal, text].filter(Boolean).join(' '); + this._pendingInterim = ''; + if (ClaudeVoiceProvider._finalized) this.stop(true); + else this._showPreview(this._accumulatedFinal, 'claude'); } else { - this._showPreview(text, 'claude'); + this._pendingInterim = (this._accumulatedFinal ? ' ' : '') + text; + this._showPreview(this._accumulatedFinal + this._pendingInterim, 'claude'); } }, onError: (msg) => { @@ -768,7 +814,6 @@ const VoiceInput = { this._activeProvider = 'deepgram'; this._accumulatedFinal = ''; this._pendingInterim = ''; - this._lastTranscript = ''; this._hasReceivedResult = false; this._recordingStartedAt = Date.now(); this._updateButtons('recording'); @@ -827,9 +872,12 @@ const VoiceInput = { } this.isRecording = true; this._activeProvider = 'webspeech'; + this._retryCount = 0; + this._webSpeechCommitted = ''; + this._webSpeechSoundActive = false; + this._webSpeechActivityUntil = 0; this._accumulatedFinal = ''; this._pendingInterim = ''; - this._lastTranscript = ''; this._hasReceivedResult = false; this._recordingStartedAt = Date.now(); this._updateButtons('recording'); @@ -840,42 +888,57 @@ const VoiceInput = { } catch (e) { // InvalidStateError = already started — ignore. Other errors = genuine failure. if (e.name !== 'InvalidStateError') { - this.stop(); + this.stop(true); app.showToast('Voice input failed to start: ' + e.message, 'error'); return; } } this._resetSilenceTimeout(); - // Get mic stream for level meter (non-blocking — level meter is cosmetic) - navigator.mediaDevices?.getUserMedia({ audio: true }).then(stream => { - if (this.isRecording && this._activeProvider === 'webspeech') { - this._webSpeechStream = stream; - this._startLevelMeter(stream); - } else { - stream.getTracks().forEach(t => t.stop()); - } - }).catch(() => { /* level meter just won't show */ }); + // SpeechRecognition owns its microphone. A second getUserMedia capture for + // a cosmetic meter can take audio away from Android's recognition service. + // Streaming providers still meter the same stream they transcribe. + this._drawLevelMeter(); // Haptic feedback on mobile if (navigator.vibrate) navigator.vibrate(50); }, - stop(claudeFinished = false) { + stop(providerFinished = false) { if (!this.isRecording) return; + if (this._webSpeechRestartTimer !== null) { + clearTimeout(this._webSpeechRestartTimer); + this._webSpeechRestartTimer = null; + // The recognizer already ended; no onend remains to wait for. + providerFinished = true; + } // Claude delivers its final transcript after finalize. Keep accepting it // until that frame (or onEnd) completes this recording. - if (this._activeProvider === 'claude' && !claudeFinished && ClaudeVoiceProvider._ws?.readyState === WebSocket.OPEN) { + if (this._activeProvider === 'claude' && !providerFinished && ClaudeVoiceProvider._ws?.readyState === WebSocket.OPEN) { ClaudeVoiceProvider.stop(); return; } + if (this._activeProvider === 'webspeech' && !providerFinished) { + if (this._webSpeechStopping) return; + this._webSpeechStopping = true; + clearTimeout(this.silenceTimeout); + // stop() requests a result; Android may deliver it only before onend. + // Keep accepting results, with a backstop for engines that never end. + this._webSpeechStopTimer = setTimeout(() => { + try { this.recognition?.abort(); } catch (_e) { /* already ended */ } + this.stop(true); + }, 3000); + try { this.recognition?.stop(); } catch (_e) { this.stop(true); } + return; + } + clearTimeout(this._webSpeechStopTimer); + this._webSpeechStopTimer = null; + this._webSpeechStopping = false; this.isRecording = false; const text = this._accumulatedFinal + this._pendingInterim; this._accumulatedFinal = ''; this._pendingInterim = ''; if (text) this._insertText(text); clearTimeout(this.silenceTimeout); - clearTimeout(this._stabilityTimer); this.silenceTimeout = null; - this._stabilityTimer = null; this._retryCount = 0; this._stopDurationTimer(); this._stopLevelMeter(); @@ -883,16 +946,11 @@ const VoiceInput = { this._hidePreview(); if (this._activeProvider === 'claude') { - // Finalize, don't hang up: the last transcript arrives after the audio does. - ClaudeVoiceProvider.stop(); + // Completion received, or forced teardown: callbacks must not outlive it. + ClaudeVoiceProvider._cleanup(); } else if (this._activeProvider === 'deepgram') { DeepgramProvider.stop(); } else if (this._activeProvider === 'webspeech') { - try { - this.recognition?.stop(); - } catch (_e) { - // Already stopped — ignore - } // Stop the mic stream we opened for the level meter if (this._webSpeechStream) { this._webSpeechStream.getTracks().forEach(t => t.stop()); @@ -905,36 +963,33 @@ const VoiceInput = { if (navigator.vibrate) navigator.vibrate([30, 50, 30]); }, + _onWebSpeechSound(active) { + if (!this.isRecording || this._activeProvider !== 'webspeech' || this._webSpeechStopping) return; + this._webSpeechSoundActive = active; + if (!active) this._webSpeechActivityUntil = 0; + }, + _onWebSpeechResult(event) { if (!this.isRecording) return; - this._hasReceivedResult = true; this._resetSilenceTimeout(); - let interim = ''; - let finalText = ''; - for (let i = event.resultIndex; i < event.results.length; i++) { - const transcript = event.results[i][0].transcript; - if (event.results[i].isFinal) { - finalText += transcript; - } else { - interim += transcript; - } - } - - if (finalText) { - clearTimeout(this._stabilityTimer); - this._stabilityTimer = null; - this._accumulatedFinal += finalText; - this._pendingInterim = interim; - this._showPreview(this._accumulatedFinal + interim); - } - if (interim) { - this._pendingInterim = interim; - const display = this._accumulatedFinal + interim; - this._showPreview(display); - // iOS Safari workaround: isFinal is always false. - // Detect when interim results stop changing for 750ms → treat as final. - this._iosStabilityCheck(interim); + // results is the current run's complete snapshot, not a list of new words. + // Rebuild by result position so repeats/corrections replace the same text. + // Never deduplicate by text: "go go" and repeated phrases can be intentional. + const finals = []; + const interims = []; + for (let i = 0; i < event.results.length; i++) { + const result = event.results[i]; + const text = result[0]?.transcript?.trim(); + if (text) (result.isFinal ? finals : interims).push(text); } + this._accumulatedFinal = [this._webSpeechCommitted, ...finals].filter(Boolean).join(' '); + const interim = interims.join(' '); + this._hasReceivedResult ||= finals.length > 0 || !!interim; + this._pendingInterim = interim ? (this._accumulatedFinal ? ' ' : '') + interim : ''; + this._showPreview(this._accumulatedFinal + this._pendingInterim); + // Some engines omit sound events; transcript updates are also real evidence + // of speech activity. This is an activity animation, not an amplitude meter. + if (finals.length || interim) this._webSpeechActivityUntil = Date.now() + 600; }, _onWebSpeechError(event) { @@ -945,7 +1000,7 @@ const VoiceInput = { if (event.error === 'no-speech' && this.getSilenceTimeoutMs(3000) !== 3000) return; const wasRecording = this.isRecording; - this.stop(); + this.stop(true); if (!wasRecording) return; switch (event.error) { @@ -969,41 +1024,40 @@ const VoiceInput = { _onWebSpeechEnd() { // Recognition ended (browser auto-stopped or we called stop()) if (!this.isRecording) return; - - const elapsed = Date.now() - this._recordingStartedAt; - // Web Speech API often fires onend prematurely on the first attempt (< 500ms, no results). - // Auto-retry up to 2 times to avoid the "needs two clicks" problem. - if (elapsed < 500 && !this._hasReceivedResult && this._retryCount < 2) { - this._retryCount++; - try { - this.recognition.start(); - } catch (_e) { - // If restart fails, fall through to stop - this.stop(); - } + this._onWebSpeechSound(false); + if (this._webSpeechStopping) { + this.stop(true); return; } - // Some browsers end recognition independently of our silence timer. For an - // extended/manual session, restart without resetting the original deadline. + const empty = !this._hasReceivedResult; + this._retryCount = empty ? this._retryCount + 1 : 0; + if (this._retryCount > 2) { + this.stop(true); + return; + } + + // Native endpoints must not extend the configured silence deadline. const delay = this.getSilenceTimeoutMs(3000); - if (delay === 0 || delay > 3000) { - try { - // Each recognition run starts a fresh result list. Retain Safari's - // interim-only phrase before the next run can replace it. - if (this._pendingInterim) { - this._accumulatedFinal += this._pendingInterim + ' '; - this._pendingInterim = ''; - } - this.recognition.start(); - return; - } catch (_e) { - // A failed restart ends recording normally. - } + if (empty || this._webSpeechSingleUtterance || delay === 0 || delay > 3000) { + // Commit this run once before the next run resets its result indexes. + this._webSpeechCommitted = (this._accumulatedFinal + this._pendingInterim).trim(); + this._accumulatedFinal = this._webSpeechCommitted; + this._pendingInterim = ''; + const restart = () => { + this._webSpeechRestartTimer = null; + if (!this.isRecording || this._webSpeechStopping) return; + this._hasReceivedResult = false; + try { this.recognition.start(); } catch (_e) { this.stop(true); } + }; + // A broken or unavailable recognizer must not spin forever in Manual mode. + if (empty) this._webSpeechRestartTimer = setTimeout(restart, 250); + else restart(); + return; } // Genuine end — finalize any accumulated text - this.stop(); + this.stop(true); }, /** The session this dictation belongs to (see _targetSessionId). */ @@ -1188,21 +1242,6 @@ const VoiceInput = { }, delay); }, - _iosStabilityCheck(transcript) { - // The Safari shortcut must not override the chosen thinking pause. - if (this.getSilenceTimeoutMs(3000) !== 3000) return; - if (transcript !== this._lastTranscript) { - this._lastTranscript = transcript; - clearTimeout(this._stabilityTimer); - this._stabilityTimer = setTimeout(() => { - if (this.isRecording) { - this._pendingInterim = transcript; - this.stop(); - } - }, 750); - } - }, - _startDurationTimer() { this._stopDurationTimer(); this._durationInterval = setInterval(() => { @@ -1239,6 +1278,8 @@ const VoiceInput = { }, _stopLevelMeter() { + this._webSpeechSoundActive = false; + this._webSpeechActivityUntil = 0; if (this._levelAnimFrame) { cancelAnimationFrame(this._levelAnimFrame); this._levelAnimFrame = null; @@ -1255,13 +1296,21 @@ const VoiceInput = { }, _drawLevelMeter() { - if (!this._analyser || !this.isRecording) return; - const dataArray = new Uint8Array(this._analyser.frequencyBinCount); - this._analyser.getByteFrequencyData(dataArray); - // Compute RMS level 0-1 - let sum = 0; - for (let i = 0; i < dataArray.length; i++) sum += dataArray[i] * dataArray[i]; - const rms = Math.sqrt(sum / dataArray.length) / 255; + if (!this.isRecording) return; + let rms = 0; + if (this._activeProvider === 'webspeech') { + // The speech API exposes activity events, but no microphone sample level. + const active = !this._webSpeechStopping + && (this._webSpeechSoundActive || Date.now() < this._webSpeechActivityUntil); + if (active) rms = 0.25 + 0.55 * (1 + Math.sin(Date.now() / 110)) / 2; + } else { + if (!this._analyser) return; + const dataArray = new Uint8Array(this._analyser.frequencyBinCount); + this._analyser.getByteFrequencyData(dataArray); + let sum = 0; + for (let i = 0; i < dataArray.length; i++) sum += dataArray[i] * dataArray[i]; + rms = Math.sqrt(sum / dataArray.length) / 255; + } // Update the level bars in the preview const barsEl = this.previewEl?.querySelector('.voice-level-bars'); if (barsEl) { @@ -1291,6 +1340,8 @@ const VoiceInput = { indicator.innerHTML = ''; const barsEl = document.createElement('span'); barsEl.className = 'voice-level-bars'; + barsEl.setAttribute('aria-label', this._activeProvider === 'webspeech' ? 'Speech activity' : 'Microphone level'); + barsEl.title = this._activeProvider === 'webspeech' ? 'Speech activity' : 'Microphone level'; for (let i = 0; i < 5; i++) { const bar = document.createElement('span'); bar.className = 'voice-level-bar'; @@ -1358,7 +1409,10 @@ const VoiceInput = { /** Cleanup on SSE reconnect or page unload */ cleanup() { - if (this.isRecording) this.stop(); + if (this.isRecording) { + this.stop(true); + try { this.recognition?.abort(); } catch (_e) { /* already ended */ } + } this._hideVoiceSendBtn(); DeepgramProvider._cleanup(); ClaudeVoiceProvider._cleanup(); @@ -1375,12 +1429,9 @@ const VoiceInput = { this.previewEl = null; } clearTimeout(this.silenceTimeout); - clearTimeout(this._stabilityTimer); this.silenceTimeout = null; - this._stabilityTimer = null; this._accumulatedFinal = ''; this._pendingInterim = ''; - this._lastTranscript = ''; this._retryCount = 0; this._hasReceivedResult = false; } diff --git a/src/web/voice-stream.ts b/src/web/voice-stream.ts index f6bd195b..d0991941 100644 --- a/src/web/voice-stream.ts +++ b/src/web/voice-stream.ts @@ -244,7 +244,7 @@ export class VoiceStreamRelay { switch (msg.type) { case 'TranscriptText': case 'TranscriptInterim': { - // Each frame is the whole running transcript, not a delta. + // Each frame is the current utterance so far, not a delta; endpoints commit it. if (typeof msg.data === 'string' && msg.data) { this.pendingTranscript = msg.data; this.opts.onTranscript(msg.data, false); diff --git a/test/voice-input-results.test.ts b/test/voice-input-results.test.ts new file mode 100644 index 00000000..772b6d3b --- /dev/null +++ b/test/voice-input-results.test.ts @@ -0,0 +1,443 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import vm from 'node:vm'; +import { test } from 'vitest'; +const source = fs.readFileSync(new URL('../src/web/public/voice-input.js', import.meta.url), 'utf8'); +function result(text, final = true) { + return Object.assign([{ transcript: text }], { isFinal: final }); +} +function harness({ + platform = 'Android', + ua = 'Mozilla/5.0 (Linux; Android 15) Chrome/141.0', + touch = 5, + delay = 0, +} = {}) { + let now = 1000, + next = 0, + starts = 0, + mic = 0; + const timers = new Map(), + frames = new Map(), + inserted = [], + previews = [], + attrs = {}; + const bars = Array.from({ length: 5 }, () => ({ + active: false, + classList: { + toggle(_name, on) { + this.owner.active = on; + }, + }, + })); + for (const bar of bars) bar.classList.owner = bar; + const meter = { + children: bars, + setAttribute(k, v) { + attrs[k] = v; + }, + }; + class Recognition { + start() { + starts++; + this.results = []; + } + stop() {} + abort() {} + // Chromium Android promotes every native partial into a new final when continuous=true. + partial(text) { + if (this.continuous) this.results.push(result(text)); + else this.results = [result(text, false)]; + this.onresult({ resultIndex: this.continuous ? this.results.length - 1 : 0, results: this.results }); + } + final(text) { + if (this.continuous) this.results.push(result(text)); + else this.results = [result(text)]; + this.onresult({ resultIndex: this.continuous ? this.results.length - 1 : 0, results: this.results }); + } + } + const context = vm.createContext({ + console, + window: { SpeechRecognition: Recognition }, + navigator: { + userAgent: ua, + platform: platform === 'Windows' ? 'Win32' : 'Linux aarch64', + userAgentData: { platform }, + maxTouchPoints: touch, + mediaDevices: { + getUserMedia() { + mic++; + throw Error('second microphone capture'); + }, + }, + }, + localStorage: { getItem: () => JSON.stringify({ silenceTimeout: delay }) }, + app: { showToast() {} }, + WebSocket: { OPEN: 1 }, + Date: class extends Date { + static now() { + return now; + } + }, + setTimeout(fn, ms) { + timers.set(++next, { fn, ms }); + return next; + }, + clearTimeout: (id) => timers.delete(id), + clearInterval() {}, + requestAnimationFrame(fn) { + frames.set(++next, fn); + return next; + }, + cancelAnimationFrame: (id) => frames.delete(id), + }); + const voice = vm.runInContext(source + '\nVoiceInput', context); + for (const name of [ + '_updateButtons', + '_startDurationTimer', + '_stopDurationTimer', + '_hidePreview', + '_hideVoiceSendBtn', + ]) + voice[name] = () => {}; + voice.previewEl = { querySelector: () => meter, remove() {} }; + voice._showPreview = (text) => previews.push(text); + voice._insertText = (text) => inserted.push(text); + voice._initRecognition(); + return { + voice, + inserted, + previews, + bars, + attrs, + timers, + frames, + get starts() { + return starts; + }, + get mic() { + return mic; + }, + frame(delta = 120) { + now += delta; + const jobs = [...frames.values()]; + frames.clear(); + for (const fn of jobs) fn(now); + }, + event(results, index = 0) { + voice._onWebSpeechResult({ results, resultIndex: index }); + }, + finish() { + voice.stop(); + voice._onWebSpeechEnd(); + }, + }; +} +test('Android growing partial phrases and final insert exactly five words', () => { + const h = harness(); + h.voice._startWebSpeech(); + for (const phrase of ['one', 'one two', 'one two three', 'one two three four', 'one two three four five']) + h.voice.recognition.partial(phrase); + h.voice.recognition.final('one two three four five'); + h.finish(); + assert.deepEqual(h.inserted, ['one two three four five']); + assert.equal(h.mic, 0); +}); +test('Android desktop-site mode also avoids continuous partial-to-final promotion', () => { + const h = harness({ platform: 'Linux', ua: 'Mozilla/5.0 (X11; Linux x86_64) Chrome/141.0' }); + assert.equal(h.voice.recognition.continuous, false); +}); +test('Windows keeps continuous recognition and repeated snapshots do not append twice', () => { + const h = harness({ platform: 'Windows', ua: 'Mozilla/5.0 (Windows NT 10.0) Chrome/141.0' }); + h.voice._startWebSpeech(); + assert.equal(h.voice.recognition.continuous, true); + h.event([result('yes yes'), result('do it', false)], 0); + h.event([result('yes yes'), result('do it', false)], 0); + h.event([result('yes yes'), result('do it')], 1); + h.event([result('yes yes'), result('do it')], 0); + h.finish(); + assert.deepEqual(h.inserted, ['yes yes do it']); +}); +test('an interim correction replaces the previous text, including removed interim results', () => { + const h = harness(); + h.voice._startWebSpeech(); + h.event([result('keep this'), result('wrong', false)]); + h.event([result('keep this'), result('correct', false)], 1); + assert.equal(h.previews.at(-1), 'keep this correct'); + h.event([result('keep this')], 1); + h.finish(); + assert.deepEqual(h.inserted, ['keep this']); +}); +for (const delay of [0, 3, 6, 15, 30]) + test(`Android preserves intentional repeated phrases across native ends with ${delay}s silence setting`, () => { + const h = harness({ delay }); + h.voice._startWebSpeech(); + h.voice.recognition.final('go go'); + h.voice._onWebSpeechEnd(); + assert.equal(h.starts, 2); + h.voice.recognition.partial('go go'); + h.voice.recognition.final('go go'); + h.finish(); + assert.deepEqual(h.inserted, ['go go go go']); + }); +test('speech events animate bars, silence clears them, and Stop cancels animation', () => { + const h = harness(); + h.voice._startWebSpeech(); + assert.equal(h.bars.filter((b) => b.active).length, 0); + assert.equal(typeof h.voice.recognition.onsoundstart, 'function'); + h.voice.recognition.onsoundstart(); + h.frame(); + const count = h.bars.filter((b) => b.active).length; + assert.ok(count > 0); + h.frame(300); + assert.notEqual(h.bars.filter((b) => b.active).length, count); + h.voice.recognition.onsoundend(); + h.frame(); + assert.equal(h.bars.filter((b) => b.active).length, 0); + h.voice.recognition.onspeechstart(); + h.frame(); + assert.ok(h.bars.some((b) => b.active)); + h.finish(); + assert.equal(h.frames.size, 0); + assert.equal(h.mic, 0); +}); +test('engines without sound events briefly animate on transcript updates', () => { + const h = harness(); + h.voice._startWebSpeech(); + h.voice.recognition.partial('hello'); + h.frame(); + assert.ok(h.bars.some((b) => b.active)); + h.frame(1000); + assert.equal(h.bars.filter((b) => b.active).length, 0); + h.finish(); +}); +test('a new dictation starts with no words from the previous dictation', () => { + const h = harness(); + h.voice._startWebSpeech(); + h.voice.recognition.final('first'); + h.finish(); + h.voice._startWebSpeech(); + h.voice.recognition.final('second'); + h.finish(); + assert.deepEqual(h.inserted, ['first', 'second']); +}); +test('default 3-second silence deadline does not stop Android after a 750ms stable interim', () => { + const h = harness({ delay: 3 }); + h.voice._startWebSpeech(); + h.voice.recognition.partial('five words spoken very clearly'); + assert.deepEqual( + [...h.timers.values()].map((t) => t.ms), + [3000] + ); +}); +test('interim-only native end cannot be reinserted by an old stability callback', () => { + const h = harness({ delay: 3 }); + h.voice._startWebSpeech(); + h.voice.recognition.partial('five words spoken very clearly'); + h.voice._onWebSpeechEnd(); + assert.equal(h.starts, 2); + // Execute the callbacks that were queued before the native end, in deadline order. + const pending = [...h.timers.entries()].sort((a, b) => a[1].ms - b[1].ms); + for (const [id, timer] of pending) { + if (h.timers.delete(id)) timer.fn(); + } + h.voice._onWebSpeechEnd(); + assert.deepEqual(h.inserted, ['five words spoken very clearly']); +}); + +function stopHarness() { + const timers = new Map(), + inserted = [], + previews = [], + toasts = []; + let next = 0, + micRequests = 0, + stops = 0, + starts = 0; + const context = vm.createContext({ + console, + app: { showToast: (text) => toasts.push(text) }, + navigator: { + userAgent: 'Mozilla/5.0 (Linux; Android 15) Chrome/141.0', + mediaDevices: { + getUserMedia: async () => { + micRequests++; + return { getTracks: () => [] }; + }, + }, + }, + localStorage: { getItem: () => JSON.stringify({ silenceTimeout: 0 }) }, + WebSocket: { OPEN: 1 }, + setTimeout: (fn, ms) => { + timers.set(++next, { fn, ms }); + return next; + }, + clearTimeout: (id) => timers.delete(id), + clearInterval() {}, + requestAnimationFrame() { + return 1; + }, + cancelAnimationFrame() {}, + }); + const [voice, claude] = vm.runInContext(source + '\n[VoiceInput,ClaudeVoiceProvider]', context); + voice.recognition = { + start() { + starts++; + }, + stop() { + stops++; + }, + abort() {}, + }; + voice.supported = true; + for (const name of [ + '_updateButtons', + '_startDurationTimer', + '_stopDurationTimer', + '_startLevelMeter', + '_stopLevelMeter', + '_hidePreview', + '_hideVoiceSendBtn', + ]) + voice[name] = () => {}; + voice._showPreview = (text) => previews.push(text); + voice._insertText = (text) => inserted.push(text); + return { + voice, + claude, + timers, + inserted, + previews, + toasts, + get micRequests() { + return micRequests; + }, + get stops() { + return stops; + }, + get starts() { + return starts; + }, + }; +} +function stopResult(voice, text, final = true) { + voice._onWebSpeechResult({ resultIndex: 0, results: [Object.assign([{ transcript: text }], { isFinal: final })] }); +} +test('browser recognition owns the microphone without a competing cosmetic recorder', async () => { + const h = stopHarness(); + h.voice._startWebSpeech(); + await Promise.resolve(); + assert.equal(h.starts, 1); + assert.equal(h.micRequests, 0); + stopResult(h.voice, 'live Android words', false); + assert.equal(h.previews.at(-1), 'live Android words'); +}); +test('tap Stop accepts a delayed final transcript and inserts once on end', () => { + const h = stopHarness(); + h.voice._startWebSpeech(); + h.voice.stop(); + assert.equal(h.stops, 1); + assert.deepEqual(h.inserted, []); + stopResult(h.voice, 'words arriving after stop'); + h.voice._onWebSpeechEnd(); + h.voice._onWebSpeechEnd(); + assert.deepEqual(h.inserted, ['words arriving after stop']); + assert.equal(h.voice.isRecording, false); + assert.equal(h.starts, 1); +}); +test('repeated Stop does not finalize early or restart recognition', () => { + const h = stopHarness(); + h.voice._startWebSpeech(); + h.voice.stop(); + h.voice.stop(); + stopResult(h.voice, 'last phrase'); + h.voice._onWebSpeechEnd(); + assert.deepEqual(h.inserted, ['last phrase']); + assert.equal(h.stops, 1); + assert.equal(h.starts, 1); +}); +test('stop timeout preserves interim words if the browser never ends', () => { + const h = stopHarness(); + h.voice._startWebSpeech(); + stopResult(h.voice, 'interim words', false); + h.voice.stop(); + const timeout = [...h.timers.values()].find((t) => t.ms === 3000); + assert.ok(timeout, 'bounded wait for the browser final result'); + timeout.fn(); + assert.deepEqual(h.inserted, ['interim words']); + assert.equal(h.voice.isRecording, false); + stopResult(h.voice, 'too late'); + assert.deepEqual(h.inserted, ['interim words']); +}); +test('manual thinking pauses still restart natural recognition ends and retain words', () => { + const h = stopHarness(); + h.voice._startWebSpeech(); + stopResult(h.voice, 'first phrase', false); + h.voice._onWebSpeechEnd(); + stopResult(h.voice, 'second phrase'); + h.voice.stop(); + h.voice._onWebSpeechEnd(); + assert.deepEqual(h.inserted, ['first phrase second phrase']); + assert.equal(h.starts, 2); +}); +test('Claude still accepts its final transcript after finalize', () => { + const h = stopHarness(); + h.voice.isRecording = true; + h.voice._activeProvider = 'claude'; + h.claude._ws = { readyState: 1 }; + let finalized = 0; + h.claude.stop = () => finalized++; + h.voice.stop(); + assert.equal(h.voice.isRecording, true); + h.voice._accumulatedFinal = 'Claude final'; + h.voice.stop(true); + assert.deepEqual(h.inserted, ['Claude final']); + assert.ok(finalized >= 1); +}); + +test('failed start ends immediately with an error instead of waiting for a final result', () => { + const h = stopHarness(); + h.voice.recognition.start = () => { + throw new Error('fixture failure'); + }; + h.voice._startWebSpeech(); + assert.equal(h.voice.isRecording, false); + assert.match(h.toasts[0], /fixture failure/); + assert.equal(h.timers.size, 0); +}); +test('failed stop still inserts available words and clears the finishing timer', () => { + const h = stopHarness(); + h.voice._startWebSpeech(); + stopResult(h.voice, 'existing words', false); + h.voice.recognition.stop = () => { + throw new Error('already stopped'); + }; + h.voice.stop(); + assert.deepEqual(h.inserted, ['existing words']); + assert.equal(h.voice.isRecording, false); + assert.equal(h.timers.size, 0); +}); +test('cleanup during finishing releases recognition and cannot insert a second result', () => { + const h = stopHarness(); + h.voice._startWebSpeech(); + stopResult(h.voice, 'keep these words', false); + h.voice.stop(); + h.voice.cleanup(); + assert.equal(h.voice.isRecording, false); + assert.equal(h.timers.size, 0); + stopResult(h.voice, 'stale result'); + h.voice._onWebSpeechEnd(); + assert.deepEqual(h.inserted, ['keep these words']); + assert.equal(h.voice.recognition, null); +}); +test('fatal browser error ends immediately without retrying or dropping existing words', () => { + const h = stopHarness(); + h.voice._startWebSpeech(); + stopResult(h.voice, 'existing words', false); + h.voice._onWebSpeechError({ error: 'network' }); + h.voice._onWebSpeechEnd(); + assert.equal(h.voice.isRecording, false); + assert.deepEqual(h.inserted, ['existing words']); + assert.equal(h.starts, 1); + assert.equal(h.timers.size, 0); + assert.match(h.toasts[0], /internet/); +}); diff --git a/test/voice-input.browser.test.ts b/test/voice-input.browser.test.ts new file mode 100644 index 00000000..fc3d1718 --- /dev/null +++ b/test/voice-input.browser.test.ts @@ -0,0 +1,85 @@ +// Real Chromium rendering/input; native Android/Windows recognition is simulated. +import { readFileSync } from 'node:fs'; +import assert from 'node:assert/strict'; +import { chromium, type Browser } from 'playwright'; +import { beforeAll, afterAll, it } from 'vitest'; +const source = readFileSync(new URL('../src/web/public/voice-input.js', import.meta.url), 'utf8'); +const styles = readFileSync(new URL('../src/web/public/styles.css', import.meta.url), 'utf8'); +let browser: Browser; +beforeAll(async () => { + browser = await chromium.launch(); +}); +afterAll(async () => { + await browser?.close(); +}); +it.each([ + [390, 'Android', 'Mozilla/5.0 (Linux; Android 15) Chrome/141.0'], + [820, 'Linux', 'Mozilla/5.0 (X11; Linux x86_64) Chrome/141.0'], + [1292, 'Windows', 'Mozilla/5.0 (Windows NT 10.0) Chrome/141.0'], +] as const)('dictation at %ipx (%s)', async (width, platform, ua) => { + const page = await browser.newPage({ viewport: { width, height: 900 }, hasTouch: true, userAgent: ua }); + const errors: string[] = []; + page.on('pageerror', (e) => errors.push(e.message)); + await page.setContent( + '
' + ); + await page.addStyleTag({ content: styles }); + await page.addScriptTag({ + content: `const platform = ${JSON.stringify(platform)}; + Object.defineProperty(navigator,'userAgentData',{value:{platform},configurable:true}); + Object.defineProperty(navigator,'platform',{value:platform==='Windows'?'Win32':'Linux x86_64',configurable:true}); + window.micRequests=0; + Object.defineProperty(navigator,'mediaDevices',{value:{getUserMedia(){window.micRequests++;return Promise.reject(Error('Unexpected second microphone'));}},configurable:true}); + window.SpeechRecognition=class{ + start(){this.results=[];} + partial(text){this.text=text;const r=Object.assign([{transcript:text}],{isFinal:platform!=='Windows'&&this.continuous});if(r.isFinal)this.results.push(r);else this.results=[r];this.onresult({resultIndex:r.isFinal?this.results.length-1:0,results:this.results});} + final(text){this.text=text;const r=Object.assign([{transcript:text}],{isFinal:true});if(platform!=='Windows'&&this.continuous)this.results.push(r);else this.results=[r];this.onresult({resultIndex:0,results:this.results});} + stop(){setTimeout(()=>{this.final(this.text);this.onend();},50);} + abort(){} + }; + window.app={activeSessionId:'fixture',showToast(){}}; +`, + }); + await page.addScriptTag({ + content: + source + + `\nwindow.voice=VoiceInput;VoiceInput._getDeepgramConfig=()=>({provider:'webspeech',insertMode:'compose',silenceTimeout:0});document.getElementById('mic').onclick=()=>VoiceInput.toggle();`, + }); + await page.click('#mic'); + assert.equal(await page.evaluate(() => (window as any).voice.recognition.continuous), platform === 'Windows'); + await page.evaluate(() => (window as any).voice.recognition.onsoundstart()); + await page.waitForFunction(() => document.querySelectorAll('.voice-level-bar.active').length > 0); + const counts = new Set(); + for (let i = 0; i < 8; i++) { + await page.waitForTimeout(90); + counts.add(await page.locator('.voice-level-bar.active').count()); + } + assert.ok(counts.size > 1, 'speech bars must animate'); + await page.evaluate(() => (window as any).voice.recognition.onsoundend()); + await page.waitForFunction(() => document.querySelectorAll('.voice-level-bar.active').length === 0); + await page.evaluate(() => { + const voice = (window as any).voice; + for (const text of ['one', 'one two', 'one two three', 'one two three four', 'one two three four five']) + voice.recognition.partial(text); + voice.recognition.final('one two three four five'); + }); + assert.equal( + await page.evaluate(() => (window as any).voice._previewTextNode.textContent), + 'one two three four five' + ); + await page.click('#mic'); + await page.waitForFunction(() => document.querySelector('textarea')?.value === 'one two three four five'); + assert.equal(await page.evaluate(() => (window as any).voice.isRecording), false); + await page.click('#mic'); + await page.evaluate(() => { + const voice = (window as any).voice; + voice.recognition.final('go go'); + voice.recognition.onend(); + voice.recognition.partial('go go'); + }); + await page.click('#mic'); + await page.waitForFunction(() => document.querySelector('textarea')?.value === 'go go go go'); + assert.equal(await page.evaluate(() => (window as any).micRequests), 0); + assert.deepEqual(errors, []); + await page.close(); +}); diff --git a/test/voice-silence-timeout.test.ts b/test/voice-silence-timeout.test.ts index d903b750..f68e2283 100644 --- a/test/voice-silence-timeout.test.ts +++ b/test/voice-silence-timeout.test.ts @@ -1,19 +1,27 @@ import { readFileSync } from 'node:fs'; import vm from 'node:vm'; -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { SettingsUpdateSchema } from '../src/web/schemas.js'; +afterEach(() => vi.useRealTimers()); + const source = readFileSync(new URL('../src/web/public/voice-input.js', import.meta.url), 'utf8'); -function load(silenceTimeout?: unknown) { +function load(silenceTimeout?: unknown, overrides: Record = {}) { vi.useRealTimers(); vi.useFakeTimers(); const context = vm.createContext({ setTimeout, clearTimeout, + setInterval, + clearInterval, + requestAnimationFrame: () => 1, + cancelAnimationFrame: () => {}, + app: { showToast: vi.fn() }, console, navigator: {}, WebSocket: { OPEN: 1 }, localStorage: { getItem: () => JSON.stringify({ silenceTimeout }) }, + ...overrides, }); const providers = vm.runInContext(`${source}\n[DeepgramProvider, ClaudeVoiceProvider, VoiceInput]`, context); return providers as Array>; @@ -62,12 +70,15 @@ describe('voice silence timeout', () => { const voice = load(setting)[2]; expect(voice.getSilenceTimeoutMs(6000)).toBe(6000); }); - it('does not let the iOS stability shortcut stop extended/manual dictation', () => { - for (const setting of [0, 15]) { + it('does not stop interim dictation at the former 750ms stability deadline', () => { + for (const setting of [0, 3, 15]) { const voice = load(setting)[2]; voice.isRecording = true; - voice._iosStabilityCheck('thinking'); - expect(vi.getTimerCount()).toBe(0); + voice._showPreview = vi.fn(); + voice.stop = vi.fn(); + voice._onWebSpeechResult({ results: [[{ transcript: 'thinking' }]] }); + vi.advanceTimersByTime(750); + expect(voice.stop).not.toHaveBeenCalled(); } }); it('restarts browser recognition without extending the silence deadline', () => { @@ -76,6 +87,7 @@ describe('voice silence timeout', () => { voice._recordingStartedAt = Date.now() - 1000; voice.recognition = { start: vi.fn() }; voice.stop = vi.fn(); + voice._hasReceivedResult = true; voice._resetSilenceTimeout(); vi.advanceTimersByTime(10000); voice._onWebSpeechEnd(); @@ -97,9 +109,11 @@ describe('voice silence timeout', () => { expect(voice.isRecording).toBe(true); expect(voice._insertText).not.toHaveBeenCalled(); const interim = [{ transcript: 'thinking aloud' }]; - voice._onWebSpeechResult({ resultIndex: 0, results: [interim] }); + voice._onWebSpeechResult({ resultIndex: 1, results: [result, interim] }); voice.stop(); voice.stop(); + expect(voice._insertText).not.toHaveBeenCalled(); + voice._onWebSpeechEnd(); expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('first phrase thinking aloud'); }); it('preserves mixed final/interim results across browser recognition restarts', () => { @@ -130,7 +144,7 @@ describe('voice silence timeout', () => { expect(voice._insertText).not.toHaveBeenCalled(); expect(voice.isRecording).toBe(true); }); - it('cancels Safari stability text when a final result supersedes it', () => { + it('keeps a corrected final through the configured silence deadline', () => { const voice = load(3)[2]; voice.isRecording = true; voice._showPreview = vi.fn(); @@ -151,3 +165,186 @@ describe('voice silence timeout', () => { expect(SettingsUpdateSchema.safeParse({ voiceSettings: { silenceTimeout: -1 } }).success).toBe(false); }); }); + +function claudeSession(setting: number) { + const [, provider, voice] = load(setting); + voice._claudeStatus = { available: true }; + for (const method of [ + '_showPreview', + '_startDurationTimer', + '_stopDurationTimer', + '_stopLevelMeter', + '_updateButtons', + '_hidePreview', + '_hideVoiceSendBtn', + ]) + voice[method] = vi.fn(); + voice._insertText = vi.fn(); + provider.start = vi.fn((opts) => { + provider._finalized = false; + provider._ws = { readyState: 1, send: vi.fn(), close: vi.fn() }; + provider._onResult = opts.onResult; + provider._onEnd = opts.onEnd; + provider._resetSilenceTimeout(); + }); + voice._startClaude(); + return { provider, voice }; +} + +describe('Claude utterances and teardown', () => { + it.each([0, 15])('keeps two final utterances through a thinking pause with setting %s', (setting) => { + const { provider, voice } = claudeSession(setting); + provider._onResult('first phrase', false); + provider._onResult('first phrase', true); + expect(voice.isRecording).toBe(true); + vi.advanceTimersByTime(2000); + provider._onResult('second phrase', false); + voice.stop(); + provider._onResult('second phrase', true); + expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('first phrase second phrase'); + expect(voice.isRecording).toBe(false); + }); + it('salvages partial text and leaves the mic idle on SSE cleanup', () => { + const { provider, voice } = claudeSession(0); + provider._onResult('keep this partial', false); + voice.cleanup(); + expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('keep this partial'); + expect(voice.isRecording).toBe(false); + expect(voice._activeProvider).toBeNull(); + expect(provider._ws).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + it('bounds finalization triggered by the provider silence timer too', () => { + const { provider, voice } = claudeSession(3); + provider._onResult('before silence', false); + vi.advanceTimersByTime(3000); + expect(provider._finalized).toBe(true); + expect(voice.isRecording).toBe(true); + vi.advanceTimersByTime(5000); + expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('before silence'); + expect(voice.isRecording).toBe(false); + expect(vi.getTimerCount()).toBe(0); + }); + it('bounds a dead-link finalization wait and repeated taps cannot extend it', () => { + const { provider, voice } = claudeSession(0); + provider._onResult('salvage on timeout', false); + voice.stop(); + vi.advanceTimersByTime(4000); + voice.stop(); + expect(voice.isRecording).toBe(true); + vi.advanceTimersByTime(1000); + expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('salvage on timeout'); + expect(voice.isRecording).toBe(false); + expect(provider._ws).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe('empty browser recognition runs', () => { + it('backs off and stops after repeated empty ends instead of a tight restart loop', () => { + const voice = load(0)[2]; + voice.isRecording = true; + voice._activeProvider = 'webspeech'; + voice.recognition = { start: vi.fn() }; + voice.stop = vi.fn(); + voice._onWebSpeechEnd(); + expect(voice.recognition.start).not.toHaveBeenCalled(); + vi.advanceTimersByTime(250); + expect(voice.recognition.start).toHaveBeenCalledTimes(1); + voice._onWebSpeechEnd(); + vi.advanceTimersByTime(250); + expect(voice.recognition.start).toHaveBeenCalledTimes(2); + voice._onWebSpeechEnd(); + vi.advanceTimersByTime(10000); + expect(voice.recognition.start).toHaveBeenCalledTimes(2); + expect(voice.stop).toHaveBeenCalledWith(true); + }); + it('resets the empty-run budget after actual speech', () => { + const voice = load(0)[2]; + voice.isRecording = true; + voice._activeProvider = 'webspeech'; + voice.recognition = { start: vi.fn() }; + voice._showPreview = vi.fn(); + voice.stop = vi.fn(); + voice._onWebSpeechEnd(); + vi.advanceTimersByTime(250); + voice._onWebSpeechResult({ results: [[{ transcript: 'a real phrase' }]] }); + voice._onWebSpeechEnd(); + voice._onWebSpeechEnd(); + vi.advanceTimersByTime(250); + voice._onWebSpeechEnd(); + vi.advanceTimersByTime(250); + expect(voice.recognition.start).toHaveBeenCalledTimes(4); + expect(voice.stop).not.toHaveBeenCalled(); + expect(voice._accumulatedFinal).toBe('a real phrase'); + }); + it('does not restart the microphone after Stop during an empty-run backoff', () => { + const voice = load(0)[2]; + voice.isRecording = true; + voice._activeProvider = 'webspeech'; + for (const method of ['_stopDurationTimer', '_stopLevelMeter', '_updateButtons', '_hidePreview']) + voice[method] = vi.fn(); + voice.recognition = { start: vi.fn(), stop: vi.fn() }; + voice._onWebSpeechEnd(); + voice.stop(); + vi.advanceTimersByTime(10000); + expect(voice.recognition.start).not.toHaveBeenCalled(); + expect(voice.isRecording).toBe(false); + }); +}); + +describe('Claude acquisition cancellation', () => { + function pendingMicrophone() { + let grant!: (value: unknown) => void; + const microphone = new Promise((resolve) => { + grant = resolve; + }); + const track = { stop: vi.fn() }; + const stream = { getTracks: () => [track] }; + const sockets: any[] = []; + class Socket { + static OPEN = 1; + static CONNECTING = 0; + readyState = 0; + close = vi.fn(); + constructor() { + sockets.push(this); + } + } + const [, provider] = load(0, { + navigator: { mediaDevices: { getUserMedia: () => microphone } }, + URLSearchParams, + WebSocket: Socket, + location: { protocol: 'https:', host: 'fixture.invalid' }, + window: {}, + }); + const onStream = vi.fn(); + const onError = vi.fn(); + return { provider, grant, track, stream, sockets, onStream, onError }; + } + it('releases a microphone granted after cleanup without opening a socket', async () => { + const h = pendingMicrophone(); + const started = h.provider.start({ onStream: h.onStream, onError: h.onError }); + h.provider._cleanup(); + h.grant(h.stream); + await started; + expect(h.track.stop).toHaveBeenCalledOnce(); + expect(h.sockets).toHaveLength(0); + expect(h.onStream).not.toHaveBeenCalled(); + expect(h.onError).not.toHaveBeenCalled(); + expect(h.provider._stream).toBeNull(); + }); + it('cancels a connecting socket and detaches its capture callback', async () => { + const h = pendingMicrophone(); + const started = h.provider.start({ onStream: h.onStream, onError: h.onError }); + h.grant(h.stream); + await started; + const socket = h.sockets[0]; + expect(socket.onopen).toBeTypeOf('function'); + h.provider._cleanup(); + expect(socket.onopen).toBeNull(); + expect(socket.close).toHaveBeenCalledOnce(); + expect(h.track.stop).toHaveBeenCalledOnce(); + expect(h.provider._ws).toBeNull(); + }); +});