fix(voice): preserve Android transcripts and Claude utterances

This commit is contained in:
Ludo Fourrage committed 2026-11-09 07:27:02 -08:00
1 parent 0cedeb2657
commit 2d7cd46565
11 files changed
+930 -139

No files matched your search

+5 -1
View File
@@ -1,5 +1,9 @@
---
"aicodeman": patch
'aicodeman': patch
---
Add a configurable voice silence timeout and manual-stop dictation so thinking pauses do not cut off recording.
Provider-default Deepgram and Web Speech final events now wait for the 3-second silence deadline instead of stopping immediately. Claude keeps recording across utterance endpoints and preserves final words after Stop, with a bounded wait on failed connections.
Fix repeated words and lost final transcripts on Chrome for Android, animate browser speech activity without a competing microphone capture, and preserve partial text when reconnecting. Bound repeated empty browser recognition restarts.
+1 -1
View File
@@ -216,7 +216,7 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph
**Agent inbox** (discussion #558; persistence `agentInboxPersistEnabled`, SYNCED, default OFF): per-session mailbox in `web/agent-inbox.ts`, routes `routes/inbox-routes.ts`, CLI `codeman agent post|inbox|ack`. Never writes to a pane, never in `toState()` (counts via `GET /api/agent-inbox/summary`). ⚠️ Read until ack by id; seen/ack only for the receiver resolved via `resolveParentSessionId()` (ack from anyone else = 403); `?wait=` parks while nothing is UNSEEN; full = 409, never drop the oldest; dropped on delete (`killMux`) only, boot prune only after a successful restore. → [architecture-invariants#agent-inbox](docs/architecture-invariants.md#agent-inbox)
**Voice dictation via Claude** (`claudeVoiceEnabled`, SYNCED, default OFF): the mic transcribes through this machine's Claude Code login (the CLI `/voice` backend) instead of Deepgram; the browser captures, `src/web/voice-stream.ts` relays to Anthropic. ⚠️ The OAuth token never reaches the page. ⚠️ Credentials are READ-ONLY (`src/claude-credentials.ts`): never refresh them (it rotates the refresh token and can sign the user out of their CLI). ⚠️ Capture must be linear16/16 kHz/mono via an AudioWorklet; `voice-pcm-worklet.js` borrows voice-input.js's `?v=` token, so edit the two together. ⚠️ Claude transcript frames are cumulative: replace, never append. → [architecture-invariants#voice-dictation-via-claude](docs/architecture-invariants.md#voice-dictation-via-claude)
**Voice dictation via Claude** (`claudeVoiceEnabled`, SYNCED, default OFF): the mic transcribes through this machine's Claude Code login (the CLI `/voice` backend) instead of Deepgram; the browser captures, `src/web/voice-stream.ts` relays to Anthropic. ⚠️ The OAuth token never reaches the page. ⚠️ Credentials are READ-ONLY (`src/claude-credentials.ts`): never refresh them (it rotates the refresh token and can sign the user out of their CLI). ⚠️ Capture must be linear16/16 kHz/mono via an AudioWorklet; `voice-pcm-worklet.js` borrows voice-input.js's `?v=` token, so edit the two together. ⚠️ Claude transcript frames are cumulative within each utterance: replace interims, append endpoint finals once, and keep recording until Stop or the configured silence deadline. → [architecture-invariants#voice-dictation-via-claude](docs/architecture-invariants.md#voice-dictation-via-claude)
**Agent Teams**: `TeamWatcher` polls `~/.claude/teams/`, matches to sessions via `leadSessionId`. Teammates are in-process threads appearing as subagents. Enable: `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`. See `docs/agent-teams/`.
+1
View File
@@ -21,6 +21,7 @@
export const BROWSER_TEST_GLOBS = [
'test/wrapped-file-paths.browser.test.ts',
'test/command-palette-agent-arrows.browser.test.ts',
'test/voice-input.browser.test.ts',
'test/tmux-capture-sgr.browser.test.ts',
'test/tab-rail-resize.browser.test.ts',
'test/tab-activation.browser.test.ts',
+1 -1
View File
@@ -782,7 +782,7 @@ Frontend `readmymind-ui.js`: header 🧠 marker-hidden (`btn-readmymind--hidden`
⚠️ Capture MUST be linear16/16 kHz/mono, so it uses an **AudioWorklet**, not MediaRecorder (which cannot emit raw PCM); `voice-pcm-worklet.js` is fetched from JS, so it is invisible to `cacheBustAssets` and borrows voice-input.js's `?v=` token — **edit the two together**.
⚠️ Transcript frames carry the WHOLE running transcript, not deltas: the Claude path replaces where the Deepgram path appends. Provider choice is `voiceSettings.provider` (`auto` prefers Claude → Deepgram → Web Speech). → `docs/claude-voice-plan.md`
⚠️ Claude transcript frames are cumulative within the current utterance: replace interims, then append each endpoint final once. An endpoint does not stop dictation unless finalization was requested. Finalization has a 5-second client backstop; reconnect cleanup salvages accumulated/interim text and releases the microphone immediately. Provider choice is `voiceSettings.provider` (`auto` prefers Claude → Deepgram → Web Speech). → `docs/claude-voice-plan.md`
### Files panel search
+13 -3
View File
@@ -82,9 +82,19 @@ sequences, so word jumps and history keys keep working.
In **App Settings → Voice → Stop after silence**, choose 3, 6, 15 or 30 seconds,
or **Manual** to keep listening until you tap the mic again (`Ctrl+Shift+V`).
**Provider default** preserves the existing timeout: 6 seconds for Claude and
3 seconds for Deepgram or Web Speech. The timeout counts from the latest transcript
update. Connection failures or microphone errors can still end a recording.
**Provider default** uses 6 seconds for Claude and 3 seconds for Deepgram or Web
Speech. Final transcript events no longer stop Deepgram or Web Speech immediately:
they wait for the configured silence deadline. The timeout counts from the latest
transcript update. A browser can still end its native recognition run; extended
and Manual dictation restart it while preserving the text and silence deadline.
Connection failures, microphone errors, or repeated empty recognition runs can
still end a recording.
On Chrome for Android, live words replace earlier partial results without repeating
them in the input. The browser microphone bars indicate speech activity; streaming
providers show audio levels. After Stop, dictation briefly waits for the final
words before inserting text. Reconnecting preserves the available words and resets
the mic so another recording can start.
### Dictating through your Claude login
+1 -1
View File
@@ -810,7 +810,7 @@ Object.assign(CodemanApp.prototype, {
document.getElementById('voiceKeyterms').value = voiceCfg.keyterms || DEFAULT_VOICE_KEYTERMS;
document.getElementById('voiceInsertMode').value = voiceCfg.insertMode || 'direct';
document.getElementById('voiceProvider').value = voiceCfg.provider || 'auto';
document.getElementById('voiceSilenceTimeout').value = [0, 3, 6, 15, 30].includes(voiceCfg.silenceTimeout)
document.getElementById('voiceSilenceTimeout').value = VoiceInput.SILENCE_TIMEOUT_SECONDS.includes(voiceCfg.silenceTimeout)
? String(voiceCfg.silenceTimeout) : 'default';
document.getElementById('appSettingsClaudeVoice').checked = settings.claudeVoiceEnabled ?? false;
// Reset key visibility to hidden
+174 -123
View File
@@ -18,7 +18,7 @@
* indicator, level meter (AnalyserNode), and elapsed timer. Two insert modes: "direct"
* (inject into local echo overlay or PTY) and "compose" (editable textarea overlay).
* Includes a temporary green Send button that replaces the settings gear icon after voice input.
* Web Speech API has auto-retry (up to 2x) for premature onend and iOS Safari stability check.
* Web Speech API retries empty runs twice with backoff and preserves late results after Stop.
*
* @globals {object} ClaudeVoiceProvider
* @globals {object} DeepgramProvider
@@ -272,8 +272,8 @@ const DeepgramProvider = {
* DeepgramProvider's recorder. The AudioContext is constructed at 16000 Hz so the
* browser does the resampling.
*
* ⚠️ Transcript frames carry the WHOLE running transcript, not deltas. Callers
* must replace, never concatenate.
* Transcript frames cover the current utterance. Replace interims and append
* each endpoint final once; an endpoint alone does not end the recording.
*/
const ClaudeVoiceProvider = {
_ws: null,
@@ -287,6 +287,8 @@ const ClaudeVoiceProvider = {
_onError: null,
_onEnd: null,
_finalized: false,
_finalizeTimeout: null,
_generation: 0,
/** How long without any transcript before the recording gives up on its own. */
SILENCE_MS: 6000,
@@ -296,6 +298,8 @@ const ClaudeVoiceProvider = {
* @param {object} opts - { language, keyterms[], onResult(text, isFinal), onError(msg), onEnd(), onStream(stream) }
*/
async start(opts) {
this._cleanup();
const generation = this._generation;
this._onResult = opts.onResult;
this._onError = opts.onError;
this._onEnd = opts.onEnd;
@@ -307,10 +311,17 @@ const ClaudeVoiceProvider = {
return;
}
try {
this._stream = await navigator.mediaDevices.getUserMedia({
const stream = await navigator.mediaDevices.getUserMedia({
audio: { noiseSuppression: true, echoCancellation: true, autoGainControl: true }
});
// Permission prompts can resolve after Stop/reconnect or a newer recording.
if (generation !== this._generation || this._finalized) {
stream.getTracks().forEach(t => t.stop());
return;
}
this._stream = stream;
} catch (err) {
if (generation !== this._generation || this._finalized) return;
const msg = err.name === 'NotAllowedError'
? 'Microphone access denied. Check browser settings.'
: 'Microphone error: ' + err.message;
@@ -334,10 +345,12 @@ const ClaudeVoiceProvider = {
this._ws.binaryType = 'arraybuffer';
this._ws.onopen = () => {
if (generation !== this._generation || this._finalized) return;
// Capture starts only once the socket is up: PCM buffered before that would
// be the oldest audio, and dropping it keeps the transcript aligned with what
// the user hears themselves saying.
this._startCapture().catch((err) => {
this._startCapture(generation).catch((err) => {
if (generation !== this._generation || this._finalized) return;
this._onError?.('Microphone capture failed: ' + err.message);
this.stop();
});
@@ -345,6 +358,7 @@ const ClaudeVoiceProvider = {
};
this._ws.onmessage = (event) => {
if (generation !== this._generation) return;
let msg;
try {
msg = JSON.parse(event.data);
@@ -364,6 +378,7 @@ const ClaudeVoiceProvider = {
};
this._ws.onclose = (event) => {
if (generation !== this._generation) return;
if (event.code === 4004) {
this._onError?.(this._unavailableMessage(event.reason));
} else if (event.code === 4008) {
@@ -388,16 +403,19 @@ const ClaudeVoiceProvider = {
},
/** Wire mic -> 16 kHz PCM16 frames -> WebSocket. */
async _startCapture() {
async _startCapture(generation = this._generation) {
if (generation !== this._generation || this._finalized) return;
const Ctx = window.AudioContext || window.webkitAudioContext;
// Ask for 16 kHz directly so the browser resamples; Safari may hand back its
// own rate, which _pcmFromFloat32 then downsamples to match.
this._audioContext = new Ctx({ sampleRate: 16000 });
if (this._audioContext.state === 'suspended') await this._audioContext.resume();
if (generation !== this._generation || this._finalized) return;
this._sourceNode = this._audioContext.createMediaStreamSource(this._stream);
if (this._audioContext.audioWorklet) {
await this._audioContext.audioWorklet.addModule(this._workletUrl());
if (generation !== this._generation || this._finalized) return;
this._workletNode = new AudioWorkletNode(this._audioContext, 'pcm-frame-processor');
this._workletNode.port.onmessage = (event) => this._sendAudio(event.data);
this._sourceNode.connect(this._workletNode);
@@ -475,6 +493,13 @@ const ClaudeVoiceProvider = {
if (this._finalized) return;
this._finalized = true;
this._stopCapture();
// Bound both manual and silence-triggered finalization on a dead connection.
// Repeated stop requests return above, so they cannot extend this deadline.
this._finalizeTimeout = setTimeout(() => {
const onEnd = this._onEnd;
this._cleanup();
onEnd?.();
}, 5000);
if (this._ws?.readyState === WebSocket.OPEN) {
try {
this._ws.send(JSON.stringify({ t: 'finalize' }));
@@ -516,15 +541,19 @@ const ClaudeVoiceProvider = {
/** Hard stop: drop the socket without waiting for a final transcript. */
_cleanup() {
this._generation++;
this._finalized = true;
clearTimeout(this._finalizeTimeout);
this._finalizeTimeout = null;
clearTimeout(this._silenceTimeout);
this._silenceTimeout = null;
this._stopCapture();
if (this._ws) {
this._ws.onopen = null;
this._ws.onclose = null;
this._ws.onmessage = null;
this._ws.onerror = null;
if (this._ws.readyState === WebSocket.OPEN) {
if (this._ws.readyState === WebSocket.OPEN || this._ws.readyState === WebSocket.CONNECTING) {
try { this._ws.close(1000); } catch (_e) { /* ignore */ }
}
this._ws = null;
@@ -543,19 +572,25 @@ const ClaudeVoiceProvider = {
* Inserts final text into the active session (user presses Enter to submit).
*/
const VoiceInput = {
SILENCE_TIMEOUT_SECONDS: Object.freeze([0, 3, 6, 15, 30]),
recognition: null,
isRecording: false,
supported: false,
silenceTimeout: null,
previewEl: null,
_lastTranscript: '',
_stabilityTimer: null,
_accumulatedFinal: '',
_pendingInterim: '',
_activeProvider: null, // 'deepgram' | 'webspeech' | null
_webSpeechCommitted: '', // Text from completed native recognition runs.
_webSpeechSingleUtterance: false,
_webSpeechSoundActive: false,
_webSpeechActivityUntil: 0,
_webSpeechStopping: false,
_webSpeechStopTimer: null,
_webSpeechRestartTimer: null,
_activeProvider: null, // 'claude' | 'deepgram' | 'webspeech' | null
_recordingStartedAt: 0, // timestamp when recording started
_retryCount: 0, // auto-retry counter for premature Web Speech API ends
_hasReceivedResult: false, // whether any speech result came in this session
_retryCount: 0, // consecutive empty Web Speech API runs
_hasReceivedResult: false, // whether this native recognition run received speech
_durationInterval: null, // timer for updating elapsed time display
_analyser: null, // AudioContext analyser for level meter
_analyserSource: null, // MediaStreamSource for level meter
@@ -609,7 +644,7 @@ const VoiceInput = {
/** Seconds in settings; zero means stop only when the user asks. */
getSilenceTimeoutMs(fallback) {
const seconds = this._getDeepgramConfig().silenceTimeout;
return [0, 3, 6, 15, 30].includes(seconds) ? seconds * 1000 : fallback;
return this.SILENCE_TIMEOUT_SECONDS.includes(seconds) ? seconds * 1000 : fallback;
},
_saveDeepgramConfig(config) {
@@ -661,7 +696,14 @@ const VoiceInput = {
if (!this.supported) return;
this.recognition = new SR();
this.recognition.continuous = true;
// Android's continuous bridge promotes growing partials to separate finals.
// Single-utterance mode keeps refinements provisional; onend resumes the mic
// while our own silence deadline controls the overall dictation session.
// Android tablets in desktop-site mode may identify as touch-capable Linux.
this._webSpeechSingleUtterance = navigator.userAgentData?.platform === 'Android'
|| /Android/i.test(navigator.userAgent || '')
|| (navigator.maxTouchPoints > 0 && /Linux/i.test(navigator.platform || ''));
this.recognition.continuous = !this._webSpeechSingleUtterance;
this.recognition.interimResults = true;
this.recognition.lang = 'en-US';
this.recognition.maxAlternatives = 1;
@@ -669,6 +711,8 @@ const VoiceInput = {
this.recognition.onresult = (e) => this._onWebSpeechResult(e);
this.recognition.onerror = (e) => this._onWebSpeechError(e);
this.recognition.onend = () => this._onWebSpeechEnd();
this.recognition.onsoundstart = this.recognition.onspeechstart = () => this._onWebSpeechSound(true);
this.recognition.onsoundend = this.recognition.onspeechend = this.recognition.onaudioend = () => this._onWebSpeechSound(false);
},
toggle() {
@@ -720,7 +764,6 @@ const VoiceInput = {
this._activeProvider = 'claude';
this._accumulatedFinal = '';
this._pendingInterim = '';
this._lastTranscript = '';
this._hasReceivedResult = false;
this._recordingStartedAt = Date.now();
this._updateButtons('recording');
@@ -739,12 +782,15 @@ const VoiceInput = {
onResult: (text, isFinal) => {
if (!this.isRecording) return;
this._hasReceivedResult = true;
// Each frame is the WHOLE running transcript, so replace rather than append.
this._accumulatedFinal = text;
// Interims replace the current utterance; each endpoint commits it once.
if (isFinal) {
this.stop(true);
this._accumulatedFinal = [this._accumulatedFinal, text].filter(Boolean).join(' ');
this._pendingInterim = '';
if (ClaudeVoiceProvider._finalized) this.stop(true);
else this._showPreview(this._accumulatedFinal, 'claude');
} else {
this._showPreview(text, 'claude');
this._pendingInterim = (this._accumulatedFinal ? ' ' : '') + text;
this._showPreview(this._accumulatedFinal + this._pendingInterim, 'claude');
}
},
onError: (msg) => {
@@ -768,7 +814,6 @@ const VoiceInput = {
this._activeProvider = 'deepgram';
this._accumulatedFinal = '';
this._pendingInterim = '';
this._lastTranscript = '';
this._hasReceivedResult = false;
this._recordingStartedAt = Date.now();
this._updateButtons('recording');
@@ -827,9 +872,12 @@ const VoiceInput = {
}
this.isRecording = true;
this._activeProvider = 'webspeech';
this._retryCount = 0;
this._webSpeechCommitted = '';
this._webSpeechSoundActive = false;
this._webSpeechActivityUntil = 0;
this._accumulatedFinal = '';
this._pendingInterim = '';
this._lastTranscript = '';
this._hasReceivedResult = false;
this._recordingStartedAt = Date.now();
this._updateButtons('recording');
@@ -840,42 +888,57 @@ const VoiceInput = {
} catch (e) {
// InvalidStateError = already started — ignore. Other errors = genuine failure.
if (e.name !== 'InvalidStateError') {
this.stop();
this.stop(true);
app.showToast('Voice input failed to start: ' + e.message, 'error');
return;
}
}
this._resetSilenceTimeout();
// Get mic stream for level meter (non-blocking — level meter is cosmetic)
navigator.mediaDevices?.getUserMedia({ audio: true }).then(stream => {
if (this.isRecording && this._activeProvider === 'webspeech') {
this._webSpeechStream = stream;
this._startLevelMeter(stream);
} else {
stream.getTracks().forEach(t => t.stop());
}
}).catch(() => { /* level meter just won't show */ });
// SpeechRecognition owns its microphone. A second getUserMedia capture for
// a cosmetic meter can take audio away from Android's recognition service.
// Streaming providers still meter the same stream they transcribe.
this._drawLevelMeter();
// Haptic feedback on mobile
if (navigator.vibrate) navigator.vibrate(50);
},
stop(claudeFinished = false) {
stop(providerFinished = false) {
if (!this.isRecording) return;
if (this._webSpeechRestartTimer !== null) {
clearTimeout(this._webSpeechRestartTimer);
this._webSpeechRestartTimer = null;
// The recognizer already ended; no onend remains to wait for.
providerFinished = true;
}
// Claude delivers its final transcript after finalize. Keep accepting it
// until that frame (or onEnd) completes this recording.
if (this._activeProvider === 'claude' && !claudeFinished && ClaudeVoiceProvider._ws?.readyState === WebSocket.OPEN) {
if (this._activeProvider === 'claude' && !providerFinished && ClaudeVoiceProvider._ws?.readyState === WebSocket.OPEN) {
ClaudeVoiceProvider.stop();
return;
}
if (this._activeProvider === 'webspeech' && !providerFinished) {
if (this._webSpeechStopping) return;
this._webSpeechStopping = true;
clearTimeout(this.silenceTimeout);
// stop() requests a result; Android may deliver it only before onend.
// Keep accepting results, with a backstop for engines that never end.
this._webSpeechStopTimer = setTimeout(() => {
try { this.recognition?.abort(); } catch (_e) { /* already ended */ }
this.stop(true);
}, 3000);
try { this.recognition?.stop(); } catch (_e) { this.stop(true); }
return;
}
clearTimeout(this._webSpeechStopTimer);
this._webSpeechStopTimer = null;
this._webSpeechStopping = false;
this.isRecording = false;
const text = this._accumulatedFinal + this._pendingInterim;
this._accumulatedFinal = '';
this._pendingInterim = '';
if (text) this._insertText(text);
clearTimeout(this.silenceTimeout);
clearTimeout(this._stabilityTimer);
this.silenceTimeout = null;
this._stabilityTimer = null;
this._retryCount = 0;
this._stopDurationTimer();
this._stopLevelMeter();
@@ -883,16 +946,11 @@ const VoiceInput = {
this._hidePreview();
if (this._activeProvider === 'claude') {
// Finalize, don't hang up: the last transcript arrives after the audio does.
ClaudeVoiceProvider.stop();
// Completion received, or forced teardown: callbacks must not outlive it.
ClaudeVoiceProvider._cleanup();
} else if (this._activeProvider === 'deepgram') {
DeepgramProvider.stop();
} else if (this._activeProvider === 'webspeech') {
try {
this.recognition?.stop();
} catch (_e) {
// Already stopped — ignore
}
// Stop the mic stream we opened for the level meter
if (this._webSpeechStream) {
this._webSpeechStream.getTracks().forEach(t => t.stop());
@@ -905,36 +963,33 @@ const VoiceInput = {
if (navigator.vibrate) navigator.vibrate([30, 50, 30]);
},
_onWebSpeechSound(active) {
if (!this.isRecording || this._activeProvider !== 'webspeech' || this._webSpeechStopping) return;
this._webSpeechSoundActive = active;
if (!active) this._webSpeechActivityUntil = 0;
},
_onWebSpeechResult(event) {
if (!this.isRecording) return;
this._hasReceivedResult = true;
this._resetSilenceTimeout();
let interim = '';
let finalText = '';
for (let i = event.resultIndex; i < event.results.length; i++) {
const transcript = event.results[i][0].transcript;
if (event.results[i].isFinal) {
finalText += transcript;
} else {
interim += transcript;
}
}
if (finalText) {
clearTimeout(this._stabilityTimer);
this._stabilityTimer = null;
this._accumulatedFinal += finalText;
this._pendingInterim = interim;
this._showPreview(this._accumulatedFinal + interim);
}
if (interim) {
this._pendingInterim = interim;
const display = this._accumulatedFinal + interim;
this._showPreview(display);
// iOS Safari workaround: isFinal is always false.
// Detect when interim results stop changing for 750ms → treat as final.
this._iosStabilityCheck(interim);
// results is the current run's complete snapshot, not a list of new words.
// Rebuild by result position so repeats/corrections replace the same text.
// Never deduplicate by text: "go go" and repeated phrases can be intentional.
const finals = [];
const interims = [];
for (let i = 0; i < event.results.length; i++) {
const result = event.results[i];
const text = result[0]?.transcript?.trim();
if (text) (result.isFinal ? finals : interims).push(text);
}
this._accumulatedFinal = [this._webSpeechCommitted, ...finals].filter(Boolean).join(' ');
const interim = interims.join(' ');
this._hasReceivedResult ||= finals.length > 0 || !!interim;
this._pendingInterim = interim ? (this._accumulatedFinal ? ' ' : '') + interim : '';
this._showPreview(this._accumulatedFinal + this._pendingInterim);
// Some engines omit sound events; transcript updates are also real evidence
// of speech activity. This is an activity animation, not an amplitude meter.
if (finals.length || interim) this._webSpeechActivityUntil = Date.now() + 600;
},
_onWebSpeechError(event) {
@@ -945,7 +1000,7 @@ const VoiceInput = {
if (event.error === 'no-speech' && this.getSilenceTimeoutMs(3000) !== 3000) return;
const wasRecording = this.isRecording;
this.stop();
this.stop(true);
if (!wasRecording) return;
switch (event.error) {
@@ -969,41 +1024,40 @@ const VoiceInput = {
_onWebSpeechEnd() {
// Recognition ended (browser auto-stopped or we called stop())
if (!this.isRecording) return;
const elapsed = Date.now() - this._recordingStartedAt;
// Web Speech API often fires onend prematurely on the first attempt (< 500ms, no results).
// Auto-retry up to 2 times to avoid the "needs two clicks" problem.
if (elapsed < 500 && !this._hasReceivedResult && this._retryCount < 2) {
this._retryCount++;
try {
this.recognition.start();
} catch (_e) {
// If restart fails, fall through to stop
this.stop();
}
this._onWebSpeechSound(false);
if (this._webSpeechStopping) {
this.stop(true);
return;
}
// Some browsers end recognition independently of our silence timer. For an
// extended/manual session, restart without resetting the original deadline.
const empty = !this._hasReceivedResult;
this._retryCount = empty ? this._retryCount + 1 : 0;
if (this._retryCount > 2) {
this.stop(true);
return;
}
// Native endpoints must not extend the configured silence deadline.
const delay = this.getSilenceTimeoutMs(3000);
if (delay === 0 || delay > 3000) {
try {
// Each recognition run starts a fresh result list. Retain Safari's
// interim-only phrase before the next run can replace it.
if (this._pendingInterim) {
this._accumulatedFinal += this._pendingInterim + ' ';
this._pendingInterim = '';
}
this.recognition.start();
return;
} catch (_e) {
// A failed restart ends recording normally.
}
if (empty || this._webSpeechSingleUtterance || delay === 0 || delay > 3000) {
// Commit this run once before the next run resets its result indexes.
this._webSpeechCommitted = (this._accumulatedFinal + this._pendingInterim).trim();
this._accumulatedFinal = this._webSpeechCommitted;
this._pendingInterim = '';
const restart = () => {
this._webSpeechRestartTimer = null;
if (!this.isRecording || this._webSpeechStopping) return;
this._hasReceivedResult = false;
try { this.recognition.start(); } catch (_e) { this.stop(true); }
};
// A broken or unavailable recognizer must not spin forever in Manual mode.
if (empty) this._webSpeechRestartTimer = setTimeout(restart, 250);
else restart();
return;
}
// Genuine end — finalize any accumulated text
this.stop();
this.stop(true);
},
/** The session this dictation belongs to (see _targetSessionId). */
@@ -1188,21 +1242,6 @@ const VoiceInput = {
}, delay);
},
_iosStabilityCheck(transcript) {
// The Safari shortcut must not override the chosen thinking pause.
if (this.getSilenceTimeoutMs(3000) !== 3000) return;
if (transcript !== this._lastTranscript) {
this._lastTranscript = transcript;
clearTimeout(this._stabilityTimer);
this._stabilityTimer = setTimeout(() => {
if (this.isRecording) {
this._pendingInterim = transcript;
this.stop();
}
}, 750);
}
},
_startDurationTimer() {
this._stopDurationTimer();
this._durationInterval = setInterval(() => {
@@ -1239,6 +1278,8 @@ const VoiceInput = {
},
_stopLevelMeter() {
this._webSpeechSoundActive = false;
this._webSpeechActivityUntil = 0;
if (this._levelAnimFrame) {
cancelAnimationFrame(this._levelAnimFrame);
this._levelAnimFrame = null;
@@ -1255,13 +1296,21 @@ const VoiceInput = {
},
_drawLevelMeter() {
if (!this._analyser || !this.isRecording) return;
const dataArray = new Uint8Array(this._analyser.frequencyBinCount);
this._analyser.getByteFrequencyData(dataArray);
// Compute RMS level 0-1
let sum = 0;
for (let i = 0; i < dataArray.length; i++) sum += dataArray[i] * dataArray[i];
const rms = Math.sqrt(sum / dataArray.length) / 255;
if (!this.isRecording) return;
let rms = 0;
if (this._activeProvider === 'webspeech') {
// The speech API exposes activity events, but no microphone sample level.
const active = !this._webSpeechStopping
&& (this._webSpeechSoundActive || Date.now() < this._webSpeechActivityUntil);
if (active) rms = 0.25 + 0.55 * (1 + Math.sin(Date.now() / 110)) / 2;
} else {
if (!this._analyser) return;
const dataArray = new Uint8Array(this._analyser.frequencyBinCount);
this._analyser.getByteFrequencyData(dataArray);
let sum = 0;
for (let i = 0; i < dataArray.length; i++) sum += dataArray[i] * dataArray[i];
rms = Math.sqrt(sum / dataArray.length) / 255;
}
// Update the level bars in the preview
const barsEl = this.previewEl?.querySelector('.voice-level-bars');
if (barsEl) {
@@ -1291,6 +1340,8 @@ const VoiceInput = {
indicator.innerHTML = '<span class="voice-rec-dot"></span>';
const barsEl = document.createElement('span');
barsEl.className = 'voice-level-bars';
barsEl.setAttribute('aria-label', this._activeProvider === 'webspeech' ? 'Speech activity' : 'Microphone level');
barsEl.title = this._activeProvider === 'webspeech' ? 'Speech activity' : 'Microphone level';
for (let i = 0; i < 5; i++) {
const bar = document.createElement('span');
bar.className = 'voice-level-bar';
@@ -1358,7 +1409,10 @@ const VoiceInput = {
/** Cleanup on SSE reconnect or page unload */
cleanup() {
if (this.isRecording) this.stop();
if (this.isRecording) {
this.stop(true);
try { this.recognition?.abort(); } catch (_e) { /* already ended */ }
}
this._hideVoiceSendBtn();
DeepgramProvider._cleanup();
ClaudeVoiceProvider._cleanup();
@@ -1375,12 +1429,9 @@ const VoiceInput = {
this.previewEl = null;
}
clearTimeout(this.silenceTimeout);
clearTimeout(this._stabilityTimer);
this.silenceTimeout = null;
this._stabilityTimer = null;
this._accumulatedFinal = '';
this._pendingInterim = '';
this._lastTranscript = '';
this._retryCount = 0;
this._hasReceivedResult = false;
}
+1 -1
View File
@@ -244,7 +244,7 @@ export class VoiceStreamRelay {
switch (msg.type) {
case 'TranscriptText':
case 'TranscriptInterim': {
// Each frame is the whole running transcript, not a delta.
// Each frame is the current utterance so far, not a delta; endpoints commit it.
if (typeof msg.data === 'string' && msg.data) {
this.pendingTranscript = msg.data;
this.opts.onTranscript(msg.data, false);
+443
View File
@@ -0,0 +1,443 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import vm from 'node:vm';
import { test } from 'vitest';
const source = fs.readFileSync(new URL('../src/web/public/voice-input.js', import.meta.url), 'utf8');
function result(text, final = true) {
return Object.assign([{ transcript: text }], { isFinal: final });
}
function harness({
platform = 'Android',
ua = 'Mozilla/5.0 (Linux; Android 15) Chrome/141.0',
touch = 5,
delay = 0,
} = {}) {
let now = 1000,
next = 0,
starts = 0,
mic = 0;
const timers = new Map(),
frames = new Map(),
inserted = [],
previews = [],
attrs = {};
const bars = Array.from({ length: 5 }, () => ({
active: false,
classList: {
toggle(_name, on) {
this.owner.active = on;
},
},
}));
for (const bar of bars) bar.classList.owner = bar;
const meter = {
children: bars,
setAttribute(k, v) {
attrs[k] = v;
},
};
class Recognition {
start() {
starts++;
this.results = [];
}
stop() {}
abort() {}
// Chromium Android promotes every native partial into a new final when continuous=true.
partial(text) {
if (this.continuous) this.results.push(result(text));
else this.results = [result(text, false)];
this.onresult({ resultIndex: this.continuous ? this.results.length - 1 : 0, results: this.results });
}
final(text) {
if (this.continuous) this.results.push(result(text));
else this.results = [result(text)];
this.onresult({ resultIndex: this.continuous ? this.results.length - 1 : 0, results: this.results });
}
}
const context = vm.createContext({
console,
window: { SpeechRecognition: Recognition },
navigator: {
userAgent: ua,
platform: platform === 'Windows' ? 'Win32' : 'Linux aarch64',
userAgentData: { platform },
maxTouchPoints: touch,
mediaDevices: {
getUserMedia() {
mic++;
throw Error('second microphone capture');
},
},
},
localStorage: { getItem: () => JSON.stringify({ silenceTimeout: delay }) },
app: { showToast() {} },
WebSocket: { OPEN: 1 },
Date: class extends Date {
static now() {
return now;
}
},
setTimeout(fn, ms) {
timers.set(++next, { fn, ms });
return next;
},
clearTimeout: (id) => timers.delete(id),
clearInterval() {},
requestAnimationFrame(fn) {
frames.set(++next, fn);
return next;
},
cancelAnimationFrame: (id) => frames.delete(id),
});
const voice = vm.runInContext(source + '\nVoiceInput', context);
for (const name of [
'_updateButtons',
'_startDurationTimer',
'_stopDurationTimer',
'_hidePreview',
'_hideVoiceSendBtn',
])
voice[name] = () => {};
voice.previewEl = { querySelector: () => meter, remove() {} };
voice._showPreview = (text) => previews.push(text);
voice._insertText = (text) => inserted.push(text);
voice._initRecognition();
return {
voice,
inserted,
previews,
bars,
attrs,
timers,
frames,
get starts() {
return starts;
},
get mic() {
return mic;
},
frame(delta = 120) {
now += delta;
const jobs = [...frames.values()];
frames.clear();
for (const fn of jobs) fn(now);
},
event(results, index = 0) {
voice._onWebSpeechResult({ results, resultIndex: index });
},
finish() {
voice.stop();
voice._onWebSpeechEnd();
},
};
}
test('Android growing partial phrases and final insert exactly five words', () => {
const h = harness();
h.voice._startWebSpeech();
for (const phrase of ['one', 'one two', 'one two three', 'one two three four', 'one two three four five'])
h.voice.recognition.partial(phrase);
h.voice.recognition.final('one two three four five');
h.finish();
assert.deepEqual(h.inserted, ['one two three four five']);
assert.equal(h.mic, 0);
});
test('Android desktop-site mode also avoids continuous partial-to-final promotion', () => {
const h = harness({ platform: 'Linux', ua: 'Mozilla/5.0 (X11; Linux x86_64) Chrome/141.0' });
assert.equal(h.voice.recognition.continuous, false);
});
test('Windows keeps continuous recognition and repeated snapshots do not append twice', () => {
const h = harness({ platform: 'Windows', ua: 'Mozilla/5.0 (Windows NT 10.0) Chrome/141.0' });
h.voice._startWebSpeech();
assert.equal(h.voice.recognition.continuous, true);
h.event([result('yes yes'), result('do it', false)], 0);
h.event([result('yes yes'), result('do it', false)], 0);
h.event([result('yes yes'), result('do it')], 1);
h.event([result('yes yes'), result('do it')], 0);
h.finish();
assert.deepEqual(h.inserted, ['yes yes do it']);
});
test('an interim correction replaces the previous text, including removed interim results', () => {
const h = harness();
h.voice._startWebSpeech();
h.event([result('keep this'), result('wrong', false)]);
h.event([result('keep this'), result('correct', false)], 1);
assert.equal(h.previews.at(-1), 'keep this correct');
h.event([result('keep this')], 1);
h.finish();
assert.deepEqual(h.inserted, ['keep this']);
});
for (const delay of [0, 3, 6, 15, 30])
test(`Android preserves intentional repeated phrases across native ends with ${delay}s silence setting`, () => {
const h = harness({ delay });
h.voice._startWebSpeech();
h.voice.recognition.final('go go');
h.voice._onWebSpeechEnd();
assert.equal(h.starts, 2);
h.voice.recognition.partial('go go');
h.voice.recognition.final('go go');
h.finish();
assert.deepEqual(h.inserted, ['go go go go']);
});
test('speech events animate bars, silence clears them, and Stop cancels animation', () => {
const h = harness();
h.voice._startWebSpeech();
assert.equal(h.bars.filter((b) => b.active).length, 0);
assert.equal(typeof h.voice.recognition.onsoundstart, 'function');
h.voice.recognition.onsoundstart();
h.frame();
const count = h.bars.filter((b) => b.active).length;
assert.ok(count > 0);
h.frame(300);
assert.notEqual(h.bars.filter((b) => b.active).length, count);
h.voice.recognition.onsoundend();
h.frame();
assert.equal(h.bars.filter((b) => b.active).length, 0);
h.voice.recognition.onspeechstart();
h.frame();
assert.ok(h.bars.some((b) => b.active));
h.finish();
assert.equal(h.frames.size, 0);
assert.equal(h.mic, 0);
});
test('engines without sound events briefly animate on transcript updates', () => {
const h = harness();
h.voice._startWebSpeech();
h.voice.recognition.partial('hello');
h.frame();
assert.ok(h.bars.some((b) => b.active));
h.frame(1000);
assert.equal(h.bars.filter((b) => b.active).length, 0);
h.finish();
});
test('a new dictation starts with no words from the previous dictation', () => {
const h = harness();
h.voice._startWebSpeech();
h.voice.recognition.final('first');
h.finish();
h.voice._startWebSpeech();
h.voice.recognition.final('second');
h.finish();
assert.deepEqual(h.inserted, ['first', 'second']);
});
test('default 3-second silence deadline does not stop Android after a 750ms stable interim', () => {
const h = harness({ delay: 3 });
h.voice._startWebSpeech();
h.voice.recognition.partial('five words spoken very clearly');
assert.deepEqual(
[...h.timers.values()].map((t) => t.ms),
[3000]
);
});
test('interim-only native end cannot be reinserted by an old stability callback', () => {
const h = harness({ delay: 3 });
h.voice._startWebSpeech();
h.voice.recognition.partial('five words spoken very clearly');
h.voice._onWebSpeechEnd();
assert.equal(h.starts, 2);
// Execute the callbacks that were queued before the native end, in deadline order.
const pending = [...h.timers.entries()].sort((a, b) => a[1].ms - b[1].ms);
for (const [id, timer] of pending) {
if (h.timers.delete(id)) timer.fn();
}
h.voice._onWebSpeechEnd();
assert.deepEqual(h.inserted, ['five words spoken very clearly']);
});
function stopHarness() {
const timers = new Map(),
inserted = [],
previews = [],
toasts = [];
let next = 0,
micRequests = 0,
stops = 0,
starts = 0;
const context = vm.createContext({
console,
app: { showToast: (text) => toasts.push(text) },
navigator: {
userAgent: 'Mozilla/5.0 (Linux; Android 15) Chrome/141.0',
mediaDevices: {
getUserMedia: async () => {
micRequests++;
return { getTracks: () => [] };
},
},
},
localStorage: { getItem: () => JSON.stringify({ silenceTimeout: 0 }) },
WebSocket: { OPEN: 1 },
setTimeout: (fn, ms) => {
timers.set(++next, { fn, ms });
return next;
},
clearTimeout: (id) => timers.delete(id),
clearInterval() {},
requestAnimationFrame() {
return 1;
},
cancelAnimationFrame() {},
});
const [voice, claude] = vm.runInContext(source + '\n[VoiceInput,ClaudeVoiceProvider]', context);
voice.recognition = {
start() {
starts++;
},
stop() {
stops++;
},
abort() {},
};
voice.supported = true;
for (const name of [
'_updateButtons',
'_startDurationTimer',
'_stopDurationTimer',
'_startLevelMeter',
'_stopLevelMeter',
'_hidePreview',
'_hideVoiceSendBtn',
])
voice[name] = () => {};
voice._showPreview = (text) => previews.push(text);
voice._insertText = (text) => inserted.push(text);
return {
voice,
claude,
timers,
inserted,
previews,
toasts,
get micRequests() {
return micRequests;
},
get stops() {
return stops;
},
get starts() {
return starts;
},
};
}
function stopResult(voice, text, final = true) {
voice._onWebSpeechResult({ resultIndex: 0, results: [Object.assign([{ transcript: text }], { isFinal: final })] });
}
test('browser recognition owns the microphone without a competing cosmetic recorder', async () => {
const h = stopHarness();
h.voice._startWebSpeech();
await Promise.resolve();
assert.equal(h.starts, 1);
assert.equal(h.micRequests, 0);
stopResult(h.voice, 'live Android words', false);
assert.equal(h.previews.at(-1), 'live Android words');
});
test('tap Stop accepts a delayed final transcript and inserts once on end', () => {
const h = stopHarness();
h.voice._startWebSpeech();
h.voice.stop();
assert.equal(h.stops, 1);
assert.deepEqual(h.inserted, []);
stopResult(h.voice, 'words arriving after stop');
h.voice._onWebSpeechEnd();
h.voice._onWebSpeechEnd();
assert.deepEqual(h.inserted, ['words arriving after stop']);
assert.equal(h.voice.isRecording, false);
assert.equal(h.starts, 1);
});
test('repeated Stop does not finalize early or restart recognition', () => {
const h = stopHarness();
h.voice._startWebSpeech();
h.voice.stop();
h.voice.stop();
stopResult(h.voice, 'last phrase');
h.voice._onWebSpeechEnd();
assert.deepEqual(h.inserted, ['last phrase']);
assert.equal(h.stops, 1);
assert.equal(h.starts, 1);
});
test('stop timeout preserves interim words if the browser never ends', () => {
const h = stopHarness();
h.voice._startWebSpeech();
stopResult(h.voice, 'interim words', false);
h.voice.stop();
const timeout = [...h.timers.values()].find((t) => t.ms === 3000);
assert.ok(timeout, 'bounded wait for the browser final result');
timeout.fn();
assert.deepEqual(h.inserted, ['interim words']);
assert.equal(h.voice.isRecording, false);
stopResult(h.voice, 'too late');
assert.deepEqual(h.inserted, ['interim words']);
});
test('manual thinking pauses still restart natural recognition ends and retain words', () => {
const h = stopHarness();
h.voice._startWebSpeech();
stopResult(h.voice, 'first phrase', false);
h.voice._onWebSpeechEnd();
stopResult(h.voice, 'second phrase');
h.voice.stop();
h.voice._onWebSpeechEnd();
assert.deepEqual(h.inserted, ['first phrase second phrase']);
assert.equal(h.starts, 2);
});
test('Claude still accepts its final transcript after finalize', () => {
const h = stopHarness();
h.voice.isRecording = true;
h.voice._activeProvider = 'claude';
h.claude._ws = { readyState: 1 };
let finalized = 0;
h.claude.stop = () => finalized++;
h.voice.stop();
assert.equal(h.voice.isRecording, true);
h.voice._accumulatedFinal = 'Claude final';
h.voice.stop(true);
assert.deepEqual(h.inserted, ['Claude final']);
assert.ok(finalized >= 1);
});
test('failed start ends immediately with an error instead of waiting for a final result', () => {
const h = stopHarness();
h.voice.recognition.start = () => {
throw new Error('fixture failure');
};
h.voice._startWebSpeech();
assert.equal(h.voice.isRecording, false);
assert.match(h.toasts[0], /fixture failure/);
assert.equal(h.timers.size, 0);
});
test('failed stop still inserts available words and clears the finishing timer', () => {
const h = stopHarness();
h.voice._startWebSpeech();
stopResult(h.voice, 'existing words', false);
h.voice.recognition.stop = () => {
throw new Error('already stopped');
};
h.voice.stop();
assert.deepEqual(h.inserted, ['existing words']);
assert.equal(h.voice.isRecording, false);
assert.equal(h.timers.size, 0);
});
test('cleanup during finishing releases recognition and cannot insert a second result', () => {
const h = stopHarness();
h.voice._startWebSpeech();
stopResult(h.voice, 'keep these words', false);
h.voice.stop();
h.voice.cleanup();
assert.equal(h.voice.isRecording, false);
assert.equal(h.timers.size, 0);
stopResult(h.voice, 'stale result');
h.voice._onWebSpeechEnd();
assert.deepEqual(h.inserted, ['keep these words']);
assert.equal(h.voice.recognition, null);
});
test('fatal browser error ends immediately without retrying or dropping existing words', () => {
const h = stopHarness();
h.voice._startWebSpeech();
stopResult(h.voice, 'existing words', false);
h.voice._onWebSpeechError({ error: 'network' });
h.voice._onWebSpeechEnd();
assert.equal(h.voice.isRecording, false);
assert.deepEqual(h.inserted, ['existing words']);
assert.equal(h.starts, 1);
assert.equal(h.timers.size, 0);
assert.match(h.toasts[0], /internet/);
});
+85
View File
@@ -0,0 +1,85 @@
// Real Chromium rendering/input; native Android/Windows recognition is simulated.
import { readFileSync } from 'node:fs';
import assert from 'node:assert/strict';
import { chromium, type Browser } from 'playwright';
import { beforeAll, afterAll, it } from 'vitest';
const source = readFileSync(new URL('../src/web/public/voice-input.js', import.meta.url), 'utf8');
const styles = readFileSync(new URL('../src/web/public/styles.css', import.meta.url), 'utf8');
let browser: Browser;
beforeAll(async () => {
browser = await chromium.launch();
});
afterAll(async () => {
await browser?.close();
});
it.each([
[390, 'Android', 'Mozilla/5.0 (Linux; Android 15) Chrome/141.0'],
[820, 'Linux', 'Mozilla/5.0 (X11; Linux x86_64) Chrome/141.0'],
[1292, 'Windows', 'Mozilla/5.0 (Windows NT 10.0) Chrome/141.0'],
] as const)('dictation at %ipx (%s)', async (width, platform, ua) => {
const page = await browser.newPage({ viewport: { width, height: 900 }, hasTouch: true, userAgent: ua });
const errors: string[] = [];
page.on('pageerror', (e) => errors.push(e.message));
await page.setContent(
'<button id="mic" style="position:fixed;top:20px;left:20px">Microphone</button><div class="voice-compose-overlay"><textarea class="paste-textarea" aria-label="Transcript"></textarea></div>'
);
await page.addStyleTag({ content: styles });
await page.addScriptTag({
content: `const platform = ${JSON.stringify(platform)};
Object.defineProperty(navigator,'userAgentData',{value:{platform},configurable:true});
Object.defineProperty(navigator,'platform',{value:platform==='Windows'?'Win32':'Linux x86_64',configurable:true});
window.micRequests=0;
Object.defineProperty(navigator,'mediaDevices',{value:{getUserMedia(){window.micRequests++;return Promise.reject(Error('Unexpected second microphone'));}},configurable:true});
window.SpeechRecognition=class{
start(){this.results=[];}
partial(text){this.text=text;const r=Object.assign([{transcript:text}],{isFinal:platform!=='Windows'&&this.continuous});if(r.isFinal)this.results.push(r);else this.results=[r];this.onresult({resultIndex:r.isFinal?this.results.length-1:0,results:this.results});}
final(text){this.text=text;const r=Object.assign([{transcript:text}],{isFinal:true});if(platform!=='Windows'&&this.continuous)this.results.push(r);else this.results=[r];this.onresult({resultIndex:0,results:this.results});}
stop(){setTimeout(()=>{this.final(this.text);this.onend();},50);}
abort(){}
};
window.app={activeSessionId:'fixture',showToast(){}};
`,
});
await page.addScriptTag({
content:
source +
`\nwindow.voice=VoiceInput;VoiceInput._getDeepgramConfig=()=>({provider:'webspeech',insertMode:'compose',silenceTimeout:0});document.getElementById('mic').onclick=()=>VoiceInput.toggle();`,
});
await page.click('#mic');
assert.equal(await page.evaluate(() => (window as any).voice.recognition.continuous), platform === 'Windows');
await page.evaluate(() => (window as any).voice.recognition.onsoundstart());
await page.waitForFunction(() => document.querySelectorAll('.voice-level-bar.active').length > 0);
const counts = new Set();
for (let i = 0; i < 8; i++) {
await page.waitForTimeout(90);
counts.add(await page.locator('.voice-level-bar.active').count());
}
assert.ok(counts.size > 1, 'speech bars must animate');
await page.evaluate(() => (window as any).voice.recognition.onsoundend());
await page.waitForFunction(() => document.querySelectorAll('.voice-level-bar.active').length === 0);
await page.evaluate(() => {
const voice = (window as any).voice;
for (const text of ['one', 'one two', 'one two three', 'one two three four', 'one two three four five'])
voice.recognition.partial(text);
voice.recognition.final('one two three four five');
});
assert.equal(
await page.evaluate(() => (window as any).voice._previewTextNode.textContent),
'one two three four five'
);
await page.click('#mic');
await page.waitForFunction(() => document.querySelector('textarea')?.value === 'one two three four five');
assert.equal(await page.evaluate(() => (window as any).voice.isRecording), false);
await page.click('#mic');
await page.evaluate(() => {
const voice = (window as any).voice;
voice.recognition.final('go go');
voice.recognition.onend();
voice.recognition.partial('go go');
});
await page.click('#mic');
await page.waitForFunction(() => document.querySelector('textarea')?.value === 'go go go go');
assert.equal(await page.evaluate(() => (window as any).micRequests), 0);
assert.deepEqual(errors, []);
await page.close();
});
+205 -8
View File
@@ -1,19 +1,27 @@
import { readFileSync } from 'node:fs';
import vm from 'node:vm';
import { describe, expect, it, vi } from 'vitest';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { SettingsUpdateSchema } from '../src/web/schemas.js';
afterEach(() => vi.useRealTimers());
const source = readFileSync(new URL('../src/web/public/voice-input.js', import.meta.url), 'utf8');
function load(silenceTimeout?: unknown) {
function load(silenceTimeout?: unknown, overrides: Record<string, unknown> = {}) {
vi.useRealTimers();
vi.useFakeTimers();
const context = vm.createContext({
setTimeout,
clearTimeout,
setInterval,
clearInterval,
requestAnimationFrame: () => 1,
cancelAnimationFrame: () => {},
app: { showToast: vi.fn() },
console,
navigator: {},
WebSocket: { OPEN: 1 },
localStorage: { getItem: () => JSON.stringify({ silenceTimeout }) },
...overrides,
});
const providers = vm.runInContext(`${source}\n[DeepgramProvider, ClaudeVoiceProvider, VoiceInput]`, context);
return providers as Array<Record<string, any>>;
@@ -62,12 +70,15 @@ describe('voice silence timeout', () => {
const voice = load(setting)[2];
expect(voice.getSilenceTimeoutMs(6000)).toBe(6000);
});
it('does not let the iOS stability shortcut stop extended/manual dictation', () => {
for (const setting of [0, 15]) {
it('does not stop interim dictation at the former 750ms stability deadline', () => {
for (const setting of [0, 3, 15]) {
const voice = load(setting)[2];
voice.isRecording = true;
voice._iosStabilityCheck('thinking');
expect(vi.getTimerCount()).toBe(0);
voice._showPreview = vi.fn();
voice.stop = vi.fn();
voice._onWebSpeechResult({ results: [[{ transcript: 'thinking' }]] });
vi.advanceTimersByTime(750);
expect(voice.stop).not.toHaveBeenCalled();
}
});
it('restarts browser recognition without extending the silence deadline', () => {
@@ -76,6 +87,7 @@ describe('voice silence timeout', () => {
voice._recordingStartedAt = Date.now() - 1000;
voice.recognition = { start: vi.fn() };
voice.stop = vi.fn();
voice._hasReceivedResult = true;
voice._resetSilenceTimeout();
vi.advanceTimersByTime(10000);
voice._onWebSpeechEnd();
@@ -97,9 +109,11 @@ describe('voice silence timeout', () => {
expect(voice.isRecording).toBe(true);
expect(voice._insertText).not.toHaveBeenCalled();
const interim = [{ transcript: 'thinking aloud' }];
voice._onWebSpeechResult({ resultIndex: 0, results: [interim] });
voice._onWebSpeechResult({ resultIndex: 1, results: [result, interim] });
voice.stop();
voice.stop();
expect(voice._insertText).not.toHaveBeenCalled();
voice._onWebSpeechEnd();
expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('first phrase thinking aloud');
});
it('preserves mixed final/interim results across browser recognition restarts', () => {
@@ -130,7 +144,7 @@ describe('voice silence timeout', () => {
expect(voice._insertText).not.toHaveBeenCalled();
expect(voice.isRecording).toBe(true);
});
it('cancels Safari stability text when a final result supersedes it', () => {
it('keeps a corrected final through the configured silence deadline', () => {
const voice = load(3)[2];
voice.isRecording = true;
voice._showPreview = vi.fn();
@@ -151,3 +165,186 @@ describe('voice silence timeout', () => {
expect(SettingsUpdateSchema.safeParse({ voiceSettings: { silenceTimeout: -1 } }).success).toBe(false);
});
});
function claudeSession(setting: number) {
const [, provider, voice] = load(setting);
voice._claudeStatus = { available: true };
for (const method of [
'_showPreview',
'_startDurationTimer',
'_stopDurationTimer',
'_stopLevelMeter',
'_updateButtons',
'_hidePreview',
'_hideVoiceSendBtn',
])
voice[method] = vi.fn();
voice._insertText = vi.fn();
provider.start = vi.fn((opts) => {
provider._finalized = false;
provider._ws = { readyState: 1, send: vi.fn(), close: vi.fn() };
provider._onResult = opts.onResult;
provider._onEnd = opts.onEnd;
provider._resetSilenceTimeout();
});
voice._startClaude();
return { provider, voice };
}
describe('Claude utterances and teardown', () => {
it.each([0, 15])('keeps two final utterances through a thinking pause with setting %s', (setting) => {
const { provider, voice } = claudeSession(setting);
provider._onResult('first phrase', false);
provider._onResult('first phrase', true);
expect(voice.isRecording).toBe(true);
vi.advanceTimersByTime(2000);
provider._onResult('second phrase', false);
voice.stop();
provider._onResult('second phrase', true);
expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('first phrase second phrase');
expect(voice.isRecording).toBe(false);
});
it('salvages partial text and leaves the mic idle on SSE cleanup', () => {
const { provider, voice } = claudeSession(0);
provider._onResult('keep this partial', false);
voice.cleanup();
expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('keep this partial');
expect(voice.isRecording).toBe(false);
expect(voice._activeProvider).toBeNull();
expect(provider._ws).toBeNull();
expect(vi.getTimerCount()).toBe(0);
});
it('bounds finalization triggered by the provider silence timer too', () => {
const { provider, voice } = claudeSession(3);
provider._onResult('before silence', false);
vi.advanceTimersByTime(3000);
expect(provider._finalized).toBe(true);
expect(voice.isRecording).toBe(true);
vi.advanceTimersByTime(5000);
expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('before silence');
expect(voice.isRecording).toBe(false);
expect(vi.getTimerCount()).toBe(0);
});
it('bounds a dead-link finalization wait and repeated taps cannot extend it', () => {
const { provider, voice } = claudeSession(0);
provider._onResult('salvage on timeout', false);
voice.stop();
vi.advanceTimersByTime(4000);
voice.stop();
expect(voice.isRecording).toBe(true);
vi.advanceTimersByTime(1000);
expect(voice._insertText).toHaveBeenCalledExactlyOnceWith('salvage on timeout');
expect(voice.isRecording).toBe(false);
expect(provider._ws).toBeNull();
expect(vi.getTimerCount()).toBe(0);
});
});
describe('empty browser recognition runs', () => {
it('backs off and stops after repeated empty ends instead of a tight restart loop', () => {
const voice = load(0)[2];
voice.isRecording = true;
voice._activeProvider = 'webspeech';
voice.recognition = { start: vi.fn() };
voice.stop = vi.fn();
voice._onWebSpeechEnd();
expect(voice.recognition.start).not.toHaveBeenCalled();
vi.advanceTimersByTime(250);
expect(voice.recognition.start).toHaveBeenCalledTimes(1);
voice._onWebSpeechEnd();
vi.advanceTimersByTime(250);
expect(voice.recognition.start).toHaveBeenCalledTimes(2);
voice._onWebSpeechEnd();
vi.advanceTimersByTime(10000);
expect(voice.recognition.start).toHaveBeenCalledTimes(2);
expect(voice.stop).toHaveBeenCalledWith(true);
});
it('resets the empty-run budget after actual speech', () => {
const voice = load(0)[2];
voice.isRecording = true;
voice._activeProvider = 'webspeech';
voice.recognition = { start: vi.fn() };
voice._showPreview = vi.fn();
voice.stop = vi.fn();
voice._onWebSpeechEnd();
vi.advanceTimersByTime(250);
voice._onWebSpeechResult({ results: [[{ transcript: 'a real phrase' }]] });
voice._onWebSpeechEnd();
voice._onWebSpeechEnd();
vi.advanceTimersByTime(250);
voice._onWebSpeechEnd();
vi.advanceTimersByTime(250);
expect(voice.recognition.start).toHaveBeenCalledTimes(4);
expect(voice.stop).not.toHaveBeenCalled();
expect(voice._accumulatedFinal).toBe('a real phrase');
});
it('does not restart the microphone after Stop during an empty-run backoff', () => {
const voice = load(0)[2];
voice.isRecording = true;
voice._activeProvider = 'webspeech';
for (const method of ['_stopDurationTimer', '_stopLevelMeter', '_updateButtons', '_hidePreview'])
voice[method] = vi.fn();
voice.recognition = { start: vi.fn(), stop: vi.fn() };
voice._onWebSpeechEnd();
voice.stop();
vi.advanceTimersByTime(10000);
expect(voice.recognition.start).not.toHaveBeenCalled();
expect(voice.isRecording).toBe(false);
});
});
describe('Claude acquisition cancellation', () => {
function pendingMicrophone() {
let grant!: (value: unknown) => void;
const microphone = new Promise((resolve) => {
grant = resolve;
});
const track = { stop: vi.fn() };
const stream = { getTracks: () => [track] };
const sockets: any[] = [];
class Socket {
static OPEN = 1;
static CONNECTING = 0;
readyState = 0;
close = vi.fn();
constructor() {
sockets.push(this);
}
}
const [, provider] = load(0, {
navigator: { mediaDevices: { getUserMedia: () => microphone } },
URLSearchParams,
WebSocket: Socket,
location: { protocol: 'https:', host: 'fixture.invalid' },
window: {},
});
const onStream = vi.fn();
const onError = vi.fn();
return { provider, grant, track, stream, sockets, onStream, onError };
}
it('releases a microphone granted after cleanup without opening a socket', async () => {
const h = pendingMicrophone();
const started = h.provider.start({ onStream: h.onStream, onError: h.onError });
h.provider._cleanup();
h.grant(h.stream);
await started;
expect(h.track.stop).toHaveBeenCalledOnce();
expect(h.sockets).toHaveLength(0);
expect(h.onStream).not.toHaveBeenCalled();
expect(h.onError).not.toHaveBeenCalled();
expect(h.provider._stream).toBeNull();
});
it('cancels a connecting socket and detaches its capture callback', async () => {
const h = pendingMicrophone();
const started = h.provider.start({ onStream: h.onStream, onError: h.onError });
h.grant(h.stream);
await started;
const socket = h.sockets[0];
expect(socket.onopen).toBeTypeOf('function');
h.provider._cleanup();
expect(socket.onopen).toBeNull();
expect(socket.close).toHaveBeenCalledOnce();
expect(h.track.stop).toHaveBeenCalledOnce();
expect(h.provider._ws).toBeNull();
});
});