mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-11-15 02:49:33 +01:00
Merge #705: fix(voice): preserve thinking pauses and Android dictation
This commit is contained in:
Codeman maintainer committed
commit
7f35f798ee
18 files changed
+1204
-146
No files matched your search
@@ -3456,6 +3456,20 @@
|
||||
<option value="compose">Compose dialog</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="set-row has-field" data-search="voice silence pause timeout manual stop recording">
|
||||
<div class="set-row-text">
|
||||
<label class="set-row-label" for="voiceSilenceTimeout">Stop after silence</label>
|
||||
<span class="set-row-desc">Allow time to think between phrases. Manual keeps listening until you tap the mic again.</span>
|
||||
</div>
|
||||
<select id="voiceSilenceTimeout" class="set-select">
|
||||
<option value="default">Provider default</option>
|
||||
<option value="3">3 seconds</option>
|
||||
<option value="6">6 seconds</option>
|
||||
<option value="15">15 seconds</option>
|
||||
<option value="30">30 seconds</option>
|
||||
<option value="0">Manual</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="set-row has-field" data-search="voice language">
|
||||
<div class="set-row-text">
|
||||
<span class="set-row-label">Language</span>
|
||||
|
||||
@@ -811,6 +811,8 @@ Object.assign(CodemanApp.prototype, {
|
||||
document.getElementById('voiceKeyterms').value = voiceCfg.keyterms || DEFAULT_VOICE_KEYTERMS;
|
||||
document.getElementById('voiceInsertMode').value = voiceCfg.insertMode || 'direct';
|
||||
document.getElementById('voiceProvider').value = voiceCfg.provider || 'auto';
|
||||
document.getElementById('voiceSilenceTimeout').value = VoiceInput.SILENCE_TIMEOUT_SECONDS.includes(voiceCfg.silenceTimeout)
|
||||
? String(voiceCfg.silenceTimeout) : 'default';
|
||||
document.getElementById('appSettingsClaudeVoice').checked = settings.claudeVoiceEnabled ?? false;
|
||||
// Reset key visibility to hidden
|
||||
const keyInput = document.getElementById('voiceDeepgramKey');
|
||||
@@ -3299,6 +3301,8 @@ Object.assign(CodemanApp.prototype, {
|
||||
language: document.getElementById('voiceLanguage').value,
|
||||
keyterms: document.getElementById('voiceKeyterms').value.trim(),
|
||||
insertMode: document.getElementById('voiceInsertMode').value,
|
||||
silenceTimeout: document.getElementById('voiceSilenceTimeout').value === 'default'
|
||||
? undefined : Number(document.getElementById('voiceSilenceTimeout').value),
|
||||
};
|
||||
VoiceInput._saveDeepgramConfig(voiceSettings);
|
||||
|
||||
|
||||
+230
-137
@@ -14,11 +14,11 @@
|
||||
* auto-detects MIME type (opus/webm/mp4), and supports custom key terms for dev vocabulary.
|
||||
*
|
||||
* - VoiceInput — High-level voice input controller. Toggle mode: tap mic to start, tap
|
||||
* again to stop. Auto-stops after 3s silence. Shows floating preview overlay with recording
|
||||
* again to stop. Auto-stops after configurable silence. Shows floating preview overlay with recording
|
||||
* indicator, level meter (AnalyserNode), and elapsed timer. Two insert modes: "direct"
|
||||
* (inject into local echo overlay or PTY) and "compose" (editable textarea overlay).
|
||||
* Includes a temporary green Send button that replaces the settings gear icon after voice input.
|
||||
* Web Speech API has auto-retry (up to 2x) for premature onend and iOS Safari stability check.
|
||||
* Web Speech API retries empty runs twice with backoff and preserves late results after Stop.
|
||||
*
|
||||
* @globals {object} ClaudeVoiceProvider
|
||||
* @globals {object} DeepgramProvider
|
||||
@@ -216,9 +216,12 @@ const DeepgramProvider = {
|
||||
|
||||
_resetSilenceTimeout() {
|
||||
clearTimeout(this._silenceTimeout);
|
||||
this._silenceTimeout = null;
|
||||
const delay = VoiceInput.getSilenceTimeoutMs(3000);
|
||||
if (delay === 0) return;
|
||||
this._silenceTimeout = setTimeout(() => {
|
||||
this.stop();
|
||||
}, 3000);
|
||||
}, delay);
|
||||
},
|
||||
|
||||
stop() {
|
||||
@@ -269,8 +272,8 @@ const DeepgramProvider = {
|
||||
* DeepgramProvider's recorder. The AudioContext is constructed at 16000 Hz so the
|
||||
* browser does the resampling.
|
||||
*
|
||||
* ⚠️ Transcript frames carry the WHOLE running transcript, not deltas. Callers
|
||||
* must replace, never concatenate.
|
||||
* Transcript frames cover the current utterance. Replace interims and append
|
||||
* each endpoint final once; an endpoint alone does not end the recording.
|
||||
*/
|
||||
const ClaudeVoiceProvider = {
|
||||
_ws: null,
|
||||
@@ -284,6 +287,8 @@ const ClaudeVoiceProvider = {
|
||||
_onError: null,
|
||||
_onEnd: null,
|
||||
_finalized: false,
|
||||
_finalizeTimeout: null,
|
||||
_generation: 0,
|
||||
|
||||
/** How long without any transcript before the recording gives up on its own. */
|
||||
SILENCE_MS: 6000,
|
||||
@@ -293,6 +298,8 @@ const ClaudeVoiceProvider = {
|
||||
* @param {object} opts - { language, keyterms[], onResult(text, isFinal), onError(msg), onEnd(), onStream(stream) }
|
||||
*/
|
||||
async start(opts) {
|
||||
this._cleanup();
|
||||
const generation = this._generation;
|
||||
this._onResult = opts.onResult;
|
||||
this._onError = opts.onError;
|
||||
this._onEnd = opts.onEnd;
|
||||
@@ -304,10 +311,17 @@ const ClaudeVoiceProvider = {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this._stream = await navigator.mediaDevices.getUserMedia({
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { noiseSuppression: true, echoCancellation: true, autoGainControl: true }
|
||||
});
|
||||
// Permission prompts can resolve after Stop/reconnect or a newer recording.
|
||||
if (generation !== this._generation || this._finalized) {
|
||||
stream.getTracks().forEach(t => t.stop());
|
||||
return;
|
||||
}
|
||||
this._stream = stream;
|
||||
} catch (err) {
|
||||
if (generation !== this._generation || this._finalized) return;
|
||||
const msg = err.name === 'NotAllowedError'
|
||||
? 'Microphone access denied. Check browser settings.'
|
||||
: 'Microphone error: ' + err.message;
|
||||
@@ -331,10 +345,12 @@ const ClaudeVoiceProvider = {
|
||||
this._ws.binaryType = 'arraybuffer';
|
||||
|
||||
this._ws.onopen = () => {
|
||||
if (generation !== this._generation || this._finalized) return;
|
||||
// Capture starts only once the socket is up: PCM buffered before that would
|
||||
// be the oldest audio, and dropping it keeps the transcript aligned with what
|
||||
// the user hears themselves saying.
|
||||
this._startCapture().catch((err) => {
|
||||
this._startCapture(generation).catch((err) => {
|
||||
if (generation !== this._generation || this._finalized) return;
|
||||
this._onError?.('Microphone capture failed: ' + err.message);
|
||||
this.stop();
|
||||
});
|
||||
@@ -342,6 +358,7 @@ const ClaudeVoiceProvider = {
|
||||
};
|
||||
|
||||
this._ws.onmessage = (event) => {
|
||||
if (generation !== this._generation) return;
|
||||
let msg;
|
||||
try {
|
||||
msg = JSON.parse(event.data);
|
||||
@@ -361,6 +378,7 @@ const ClaudeVoiceProvider = {
|
||||
};
|
||||
|
||||
this._ws.onclose = (event) => {
|
||||
if (generation !== this._generation) return;
|
||||
if (event.code === 4004) {
|
||||
this._onError?.(this._unavailableMessage(event.reason));
|
||||
} else if (event.code === 4008) {
|
||||
@@ -385,16 +403,19 @@ const ClaudeVoiceProvider = {
|
||||
},
|
||||
|
||||
/** Wire mic -> 16 kHz PCM16 frames -> WebSocket. */
|
||||
async _startCapture() {
|
||||
async _startCapture(generation = this._generation) {
|
||||
if (generation !== this._generation || this._finalized) return;
|
||||
const Ctx = window.AudioContext || window.webkitAudioContext;
|
||||
// Ask for 16 kHz directly so the browser resamples; Safari may hand back its
|
||||
// own rate, which _pcmFromFloat32 then downsamples to match.
|
||||
this._audioContext = new Ctx({ sampleRate: 16000 });
|
||||
if (this._audioContext.state === 'suspended') await this._audioContext.resume();
|
||||
if (generation !== this._generation || this._finalized) return;
|
||||
this._sourceNode = this._audioContext.createMediaStreamSource(this._stream);
|
||||
|
||||
if (this._audioContext.audioWorklet) {
|
||||
await this._audioContext.audioWorklet.addModule(this._workletUrl());
|
||||
if (generation !== this._generation || this._finalized) return;
|
||||
this._workletNode = new AudioWorkletNode(this._audioContext, 'pcm-frame-processor');
|
||||
this._workletNode.port.onmessage = (event) => this._sendAudio(event.data);
|
||||
this._sourceNode.connect(this._workletNode);
|
||||
@@ -455,7 +476,10 @@ const ClaudeVoiceProvider = {
|
||||
|
||||
_resetSilenceTimeout() {
|
||||
clearTimeout(this._silenceTimeout);
|
||||
this._silenceTimeout = setTimeout(() => this.stop(), this.SILENCE_MS);
|
||||
this._silenceTimeout = null;
|
||||
const delay = VoiceInput.getSilenceTimeoutMs(this.SILENCE_MS);
|
||||
if (delay === 0) return;
|
||||
this._silenceTimeout = setTimeout(() => this.stop(), delay);
|
||||
},
|
||||
|
||||
/**
|
||||
@@ -469,6 +493,13 @@ const ClaudeVoiceProvider = {
|
||||
if (this._finalized) return;
|
||||
this._finalized = true;
|
||||
this._stopCapture();
|
||||
// Bound both manual and silence-triggered finalization on a dead connection.
|
||||
// Repeated stop requests return above, so they cannot extend this deadline.
|
||||
this._finalizeTimeout = setTimeout(() => {
|
||||
const onEnd = this._onEnd;
|
||||
this._cleanup();
|
||||
onEnd?.();
|
||||
}, 5000);
|
||||
if (this._ws?.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
this._ws.send(JSON.stringify({ t: 'finalize' }));
|
||||
@@ -510,15 +541,19 @@ const ClaudeVoiceProvider = {
|
||||
|
||||
/** Hard stop: drop the socket without waiting for a final transcript. */
|
||||
_cleanup() {
|
||||
this._generation++;
|
||||
this._finalized = true;
|
||||
clearTimeout(this._finalizeTimeout);
|
||||
this._finalizeTimeout = null;
|
||||
clearTimeout(this._silenceTimeout);
|
||||
this._silenceTimeout = null;
|
||||
this._stopCapture();
|
||||
if (this._ws) {
|
||||
this._ws.onopen = null;
|
||||
this._ws.onclose = null;
|
||||
this._ws.onmessage = null;
|
||||
this._ws.onerror = null;
|
||||
if (this._ws.readyState === WebSocket.OPEN) {
|
||||
if (this._ws.readyState === WebSocket.OPEN || this._ws.readyState === WebSocket.CONNECTING) {
|
||||
try { this._ws.close(1000); } catch (_e) { /* ignore */ }
|
||||
}
|
||||
this._ws = null;
|
||||
@@ -537,18 +572,27 @@ const ClaudeVoiceProvider = {
|
||||
* Inserts final text into the active session (user presses Enter to submit).
|
||||
*/
|
||||
const VoiceInput = {
|
||||
SILENCE_TIMEOUT_SECONDS: Object.freeze([0, 3, 6, 15, 30]),
|
||||
WEB_SPEECH_INSTANT_END_MS: 1000, // an empty run shorter than this counts toward the retry cap
|
||||
recognition: null,
|
||||
isRecording: false,
|
||||
supported: false,
|
||||
silenceTimeout: null,
|
||||
previewEl: null,
|
||||
_lastTranscript: '',
|
||||
_stabilityTimer: null,
|
||||
_accumulatedFinal: '',
|
||||
_activeProvider: null, // 'deepgram' | 'webspeech' | null
|
||||
_pendingInterim: '',
|
||||
_webSpeechCommitted: '', // Text from completed native recognition runs.
|
||||
_webSpeechSingleUtterance: false,
|
||||
_webSpeechSoundActive: false,
|
||||
_webSpeechActivityUntil: 0,
|
||||
_webSpeechStopping: false,
|
||||
_webSpeechStopTimer: null,
|
||||
_webSpeechRestartTimer: null,
|
||||
_activeProvider: null, // 'claude' | 'deepgram' | 'webspeech' | null
|
||||
_recordingStartedAt: 0, // timestamp when recording started
|
||||
_retryCount: 0, // auto-retry counter for premature Web Speech API ends
|
||||
_hasReceivedResult: false, // whether any speech result came in this session
|
||||
_retryCount: 0, // consecutive Web Speech runs that ended empty right after starting
|
||||
_hasReceivedResult: false, // whether this native recognition run received speech
|
||||
_webSpeechRunStartedAt: 0, // when the current native recognition run was started
|
||||
_durationInterval: null, // timer for updating elapsed time display
|
||||
_analyser: null, // AudioContext analyser for level meter
|
||||
_analyserSource: null, // MediaStreamSource for level meter
|
||||
@@ -599,6 +643,12 @@ const VoiceInput = {
|
||||
}
|
||||
},
|
||||
|
||||
/** Seconds in settings; zero means stop only when the user asks. */
|
||||
getSilenceTimeoutMs(fallback) {
|
||||
const seconds = this._getDeepgramConfig().silenceTimeout;
|
||||
return this.SILENCE_TIMEOUT_SECONDS.includes(seconds) ? seconds * 1000 : fallback;
|
||||
},
|
||||
|
||||
_saveDeepgramConfig(config) {
|
||||
localStorage.setItem('codeman-voice-settings', JSON.stringify(config));
|
||||
},
|
||||
@@ -648,7 +698,18 @@ const VoiceInput = {
|
||||
if (!this.supported) return;
|
||||
|
||||
this.recognition = new SR();
|
||||
this.recognition.continuous = true;
|
||||
// Android's continuous bridge promotes growing partials to separate finals.
|
||||
// Single-utterance mode keeps refinements provisional; onend resumes the mic
|
||||
// while our own silence deadline controls the overall dictation session.
|
||||
// Android tablets in desktop-site mode may identify as touch-capable Linux.
|
||||
// ChromeOS also reports Linux but says CrOS, so it keeps continuous mode.
|
||||
// A Linux laptop with a touchscreen still matches; single-utterance works
|
||||
// there too, with one native restart per phrase.
|
||||
const ua = navigator.userAgent || '';
|
||||
this._webSpeechSingleUtterance = navigator.userAgentData?.platform === 'Android'
|
||||
|| /Android/i.test(ua)
|
||||
|| (navigator.maxTouchPoints > 0 && /Linux/i.test(navigator.platform || '') && !/CrOS/.test(ua));
|
||||
this.recognition.continuous = !this._webSpeechSingleUtterance;
|
||||
this.recognition.interimResults = true;
|
||||
this.recognition.lang = 'en-US';
|
||||
this.recognition.maxAlternatives = 1;
|
||||
@@ -656,6 +717,8 @@ const VoiceInput = {
|
||||
this.recognition.onresult = (e) => this._onWebSpeechResult(e);
|
||||
this.recognition.onerror = (e) => this._onWebSpeechError(e);
|
||||
this.recognition.onend = () => this._onWebSpeechEnd();
|
||||
this.recognition.onsoundstart = this.recognition.onspeechstart = () => this._onWebSpeechSound(true);
|
||||
this.recognition.onsoundend = this.recognition.onspeechend = this.recognition.onaudioend = () => this._onWebSpeechSound(false);
|
||||
},
|
||||
|
||||
toggle() {
|
||||
@@ -706,7 +769,7 @@ const VoiceInput = {
|
||||
this.isRecording = true;
|
||||
this._activeProvider = 'claude';
|
||||
this._accumulatedFinal = '';
|
||||
this._lastTranscript = '';
|
||||
this._pendingInterim = '';
|
||||
this._hasReceivedResult = false;
|
||||
this._recordingStartedAt = Date.now();
|
||||
this._updateButtons('recording');
|
||||
@@ -725,25 +788,25 @@ const VoiceInput = {
|
||||
onResult: (text, isFinal) => {
|
||||
if (!this.isRecording) return;
|
||||
this._hasReceivedResult = true;
|
||||
// Each frame is the WHOLE running transcript, so replace rather than append.
|
||||
this._accumulatedFinal = text;
|
||||
// Interims replace the current utterance; each endpoint commits it once.
|
||||
if (isFinal) {
|
||||
this._hidePreview();
|
||||
this._insertText(text);
|
||||
this.stop();
|
||||
this._accumulatedFinal = [this._accumulatedFinal, text].filter(Boolean).join(' ');
|
||||
this._pendingInterim = '';
|
||||
if (ClaudeVoiceProvider._finalized) this.stop(true);
|
||||
else this._showPreview(this._accumulatedFinal, 'claude');
|
||||
} else {
|
||||
this._showPreview(text, 'claude');
|
||||
this._pendingInterim = (this._accumulatedFinal ? ' ' : '') + text;
|
||||
this._showPreview(this._accumulatedFinal + this._pendingInterim, 'claude');
|
||||
}
|
||||
},
|
||||
onError: (msg) => {
|
||||
const wasRecording = this.isRecording;
|
||||
this.stop();
|
||||
this.stop(true);
|
||||
if (wasRecording) app.showToast(msg, 'error');
|
||||
},
|
||||
onEnd: () => {
|
||||
if (this.isRecording) {
|
||||
if (this._accumulatedFinal) this._insertText(this._accumulatedFinal);
|
||||
this.stop();
|
||||
this.stop(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -756,7 +819,7 @@ const VoiceInput = {
|
||||
this.isRecording = true;
|
||||
this._activeProvider = 'deepgram';
|
||||
this._accumulatedFinal = '';
|
||||
this._lastTranscript = '';
|
||||
this._pendingInterim = '';
|
||||
this._hasReceivedResult = false;
|
||||
this._recordingStartedAt = Date.now();
|
||||
this._updateButtons('recording');
|
||||
@@ -777,12 +840,12 @@ const VoiceInput = {
|
||||
if (!this.isRecording) return;
|
||||
this._hasReceivedResult = true;
|
||||
if (isFinal) {
|
||||
this._accumulatedFinal += text;
|
||||
this._hidePreview();
|
||||
this._insertText(this._accumulatedFinal);
|
||||
this.stop();
|
||||
this._accumulatedFinal += (this._accumulatedFinal ? ' ' : '') + text;
|
||||
this._pendingInterim = '';
|
||||
this._showPreview(this._accumulatedFinal, 'deepgram');
|
||||
} else {
|
||||
const display = this._accumulatedFinal + text;
|
||||
this._pendingInterim = (this._accumulatedFinal ? ' ' : '') + text;
|
||||
const display = this._accumulatedFinal + this._pendingInterim;
|
||||
this._showPreview(display, 'deepgram');
|
||||
}
|
||||
},
|
||||
@@ -793,9 +856,6 @@ const VoiceInput = {
|
||||
},
|
||||
onEnd: () => {
|
||||
if (this.isRecording) {
|
||||
if (this._accumulatedFinal) {
|
||||
this._insertText(this._accumulatedFinal);
|
||||
}
|
||||
this.stop();
|
||||
}
|
||||
}
|
||||
@@ -818,44 +878,74 @@ const VoiceInput = {
|
||||
}
|
||||
this.isRecording = true;
|
||||
this._activeProvider = 'webspeech';
|
||||
this._retryCount = 0;
|
||||
this._webSpeechCommitted = '';
|
||||
this._webSpeechSoundActive = false;
|
||||
this._webSpeechActivityUntil = 0;
|
||||
this._accumulatedFinal = '';
|
||||
this._lastTranscript = '';
|
||||
this._pendingInterim = '';
|
||||
this._hasReceivedResult = false;
|
||||
this._recordingStartedAt = Date.now();
|
||||
this._updateButtons('recording');
|
||||
this._showPreview('Listening...');
|
||||
this._startDurationTimer();
|
||||
try {
|
||||
this._webSpeechRunStartedAt = Date.now();
|
||||
this.recognition.start();
|
||||
} catch (e) {
|
||||
// InvalidStateError = already started — ignore. Other errors = genuine failure.
|
||||
if (e.name !== 'InvalidStateError') {
|
||||
this.stop();
|
||||
this.stop(true);
|
||||
app.showToast('Voice input failed to start: ' + e.message, 'error');
|
||||
return;
|
||||
}
|
||||
}
|
||||
this._resetSilenceTimeout();
|
||||
// Get mic stream for level meter (non-blocking — level meter is cosmetic)
|
||||
navigator.mediaDevices?.getUserMedia({ audio: true }).then(stream => {
|
||||
if (this.isRecording && this._activeProvider === 'webspeech') {
|
||||
this._webSpeechStream = stream;
|
||||
this._startLevelMeter(stream);
|
||||
} else {
|
||||
stream.getTracks().forEach(t => t.stop());
|
||||
}
|
||||
}).catch(() => { /* level meter just won't show */ });
|
||||
// SpeechRecognition owns its microphone. A second getUserMedia capture for
|
||||
// a cosmetic meter can take audio away from Android's recognition service.
|
||||
// Streaming providers still meter the same stream they transcribe.
|
||||
this._drawLevelMeter();
|
||||
// Haptic feedback on mobile
|
||||
if (navigator.vibrate) navigator.vibrate(50);
|
||||
},
|
||||
|
||||
stop() {
|
||||
stop(providerFinished = false) {
|
||||
if (!this.isRecording) return;
|
||||
if (this._webSpeechRestartTimer !== null) {
|
||||
clearTimeout(this._webSpeechRestartTimer);
|
||||
this._webSpeechRestartTimer = null;
|
||||
// The recognizer already ended; no onend remains to wait for.
|
||||
providerFinished = true;
|
||||
}
|
||||
// Claude delivers its final transcript after finalize. Keep accepting it
|
||||
// until that frame (or onEnd) completes this recording.
|
||||
if (this._activeProvider === 'claude' && !providerFinished && ClaudeVoiceProvider._ws?.readyState === WebSocket.OPEN) {
|
||||
ClaudeVoiceProvider.stop();
|
||||
return;
|
||||
}
|
||||
if (this._activeProvider === 'webspeech' && !providerFinished) {
|
||||
if (this._webSpeechStopping) return;
|
||||
this._webSpeechStopping = true;
|
||||
clearTimeout(this.silenceTimeout);
|
||||
// stop() requests a result; Android may deliver it only before onend.
|
||||
// Keep accepting results, with a backstop for engines that never end.
|
||||
this._webSpeechStopTimer = setTimeout(() => {
|
||||
try { this.recognition?.abort(); } catch (_e) { /* already ended */ }
|
||||
this.stop(true);
|
||||
}, 3000);
|
||||
try { this.recognition?.stop(); } catch (_e) { this.stop(true); }
|
||||
return;
|
||||
}
|
||||
clearTimeout(this._webSpeechStopTimer);
|
||||
this._webSpeechStopTimer = null;
|
||||
this._webSpeechStopping = false;
|
||||
this.isRecording = false;
|
||||
const text = this._accumulatedFinal + this._pendingInterim;
|
||||
this._accumulatedFinal = '';
|
||||
this._pendingInterim = '';
|
||||
if (text) this._insertText(text);
|
||||
clearTimeout(this.silenceTimeout);
|
||||
clearTimeout(this._stabilityTimer);
|
||||
this.silenceTimeout = null;
|
||||
this._stabilityTimer = null;
|
||||
this._retryCount = 0;
|
||||
this._stopDurationTimer();
|
||||
this._stopLevelMeter();
|
||||
@@ -863,21 +953,10 @@ const VoiceInput = {
|
||||
this._hidePreview();
|
||||
|
||||
if (this._activeProvider === 'claude') {
|
||||
// Finalize, don't hang up: the last transcript arrives after the audio does.
|
||||
ClaudeVoiceProvider.stop();
|
||||
// Completion received, or forced teardown: callbacks must not outlive it.
|
||||
ClaudeVoiceProvider._cleanup();
|
||||
} else if (this._activeProvider === 'deepgram') {
|
||||
DeepgramProvider.stop();
|
||||
} else if (this._activeProvider === 'webspeech') {
|
||||
try {
|
||||
this.recognition?.stop();
|
||||
} catch (_e) {
|
||||
// Already stopped — ignore
|
||||
}
|
||||
// Stop the mic stream we opened for the level meter
|
||||
if (this._webSpeechStream) {
|
||||
this._webSpeechStream.getTracks().forEach(t => t.stop());
|
||||
this._webSpeechStream = null;
|
||||
}
|
||||
}
|
||||
this._activeProvider = null;
|
||||
|
||||
@@ -885,41 +964,44 @@ const VoiceInput = {
|
||||
if (navigator.vibrate) navigator.vibrate([30, 50, 30]);
|
||||
},
|
||||
|
||||
_onWebSpeechSound(active) {
|
||||
if (!this.isRecording || this._activeProvider !== 'webspeech' || this._webSpeechStopping) return;
|
||||
this._webSpeechSoundActive = active;
|
||||
if (!active) this._webSpeechActivityUntil = 0;
|
||||
},
|
||||
|
||||
_onWebSpeechResult(event) {
|
||||
if (!this.isRecording) return;
|
||||
this._hasReceivedResult = true;
|
||||
this._resetSilenceTimeout();
|
||||
let interim = '';
|
||||
let finalText = '';
|
||||
for (let i = event.resultIndex; i < event.results.length; i++) {
|
||||
const transcript = event.results[i][0].transcript;
|
||||
if (event.results[i].isFinal) {
|
||||
finalText += transcript;
|
||||
} else {
|
||||
interim += transcript;
|
||||
}
|
||||
}
|
||||
|
||||
if (finalText) {
|
||||
this._accumulatedFinal += finalText;
|
||||
this._hidePreview();
|
||||
this._insertText(this._accumulatedFinal);
|
||||
this.stop();
|
||||
} else if (interim) {
|
||||
const display = this._accumulatedFinal + interim;
|
||||
this._showPreview(display);
|
||||
// iOS Safari workaround: isFinal is always false.
|
||||
// Detect when interim results stop changing for 750ms → treat as final.
|
||||
this._iosStabilityCheck(interim);
|
||||
// results is the current run's complete snapshot, not a list of new words.
|
||||
// Rebuild by result position so repeats/corrections replace the same text.
|
||||
// Never deduplicate by text: "go go" and repeated phrases can be intentional.
|
||||
const finals = [];
|
||||
const interims = [];
|
||||
for (let i = 0; i < event.results.length; i++) {
|
||||
const result = event.results[i];
|
||||
const text = result[0]?.transcript?.trim();
|
||||
if (text) (result.isFinal ? finals : interims).push(text);
|
||||
}
|
||||
this._accumulatedFinal = [this._webSpeechCommitted, ...finals].filter(Boolean).join(' ');
|
||||
const interim = interims.join(' ');
|
||||
this._hasReceivedResult ||= finals.length > 0 || !!interim;
|
||||
this._pendingInterim = interim ? (this._accumulatedFinal ? ' ' : '') + interim : '';
|
||||
this._showPreview(this._accumulatedFinal + this._pendingInterim);
|
||||
// Some engines omit sound events; transcript updates are also real evidence
|
||||
// of speech activity. This is an activity animation, not an amplitude meter.
|
||||
if (finals.length || interim) this._webSpeechActivityUntil = Date.now() + 600;
|
||||
},
|
||||
|
||||
_onWebSpeechError(event) {
|
||||
// During auto-retry, 'aborted' and 'no-speech' errors are expected — ignore them
|
||||
if (this._retryCount > 0 && (event.error === 'aborted' || event.error === 'no-speech')) return;
|
||||
|
||||
// Browser silence events must not override an extended/manual deadline.
|
||||
if (event.error === 'no-speech' && this.getSilenceTimeoutMs(3000) !== 3000) return;
|
||||
|
||||
const wasRecording = this.isRecording;
|
||||
this.stop();
|
||||
this.stop(true);
|
||||
if (!wasRecording) return;
|
||||
|
||||
switch (event.error) {
|
||||
@@ -943,27 +1025,45 @@ const VoiceInput = {
|
||||
_onWebSpeechEnd() {
|
||||
// Recognition ended (browser auto-stopped or we called stop())
|
||||
if (!this.isRecording) return;
|
||||
this._onWebSpeechSound(false);
|
||||
if (this._webSpeechStopping) {
|
||||
this.stop(true);
|
||||
return;
|
||||
}
|
||||
|
||||
const elapsed = Date.now() - this._recordingStartedAt;
|
||||
// Web Speech API often fires onend prematurely on the first attempt (< 500ms, no results).
|
||||
// Auto-retry up to 2 times to avoid the "needs two clicks" problem.
|
||||
if (elapsed < 500 && !this._hasReceivedResult && this._retryCount < 2) {
|
||||
this._retryCount++;
|
||||
try {
|
||||
this.recognition.start();
|
||||
} catch (_e) {
|
||||
// If restart fails, fall through to stop
|
||||
if (this._accumulatedFinal) this._insertText(this._accumulatedFinal);
|
||||
this.stop();
|
||||
}
|
||||
// Only a run that ends empty right after start() counts toward the cap: that
|
||||
// is a recognizer that cannot listen. A silent run of normal length is the
|
||||
// engine's own no-speech window, and 30 s or Manual must outlast many of them.
|
||||
const empty = !this._hasReceivedResult;
|
||||
const instant = empty && Date.now() - this._webSpeechRunStartedAt < this.WEB_SPEECH_INSTANT_END_MS;
|
||||
this._retryCount = instant ? this._retryCount + 1 : 0;
|
||||
if (this._retryCount > 2) {
|
||||
this.stop(true);
|
||||
return;
|
||||
}
|
||||
|
||||
// Native endpoints must not extend the configured silence deadline.
|
||||
const delay = this.getSilenceTimeoutMs(3000);
|
||||
if (empty || this._webSpeechSingleUtterance || delay === 0 || delay > 3000) {
|
||||
// Commit this run once before the next run resets its result indexes.
|
||||
this._webSpeechCommitted = (this._accumulatedFinal + this._pendingInterim).trim();
|
||||
this._accumulatedFinal = this._webSpeechCommitted;
|
||||
this._pendingInterim = '';
|
||||
const restart = () => {
|
||||
this._webSpeechRestartTimer = null;
|
||||
if (!this.isRecording || this._webSpeechStopping) return;
|
||||
this._hasReceivedResult = false;
|
||||
this._webSpeechRunStartedAt = Date.now();
|
||||
try { this.recognition.start(); } catch (_e) { this.stop(true); }
|
||||
};
|
||||
// A broken or unavailable recognizer must not spin forever in Manual mode.
|
||||
if (empty) this._webSpeechRestartTimer = setTimeout(restart, 250);
|
||||
else restart();
|
||||
return;
|
||||
}
|
||||
|
||||
// Genuine end — finalize any accumulated text
|
||||
if (this._accumulatedFinal) {
|
||||
this._insertText(this._accumulatedFinal);
|
||||
}
|
||||
this.stop();
|
||||
this.stop(true);
|
||||
},
|
||||
|
||||
/** The session this dictation belongs to (see _targetSessionId). */
|
||||
@@ -1138,30 +1238,14 @@ const VoiceInput = {
|
||||
|
||||
_resetSilenceTimeout() {
|
||||
clearTimeout(this.silenceTimeout);
|
||||
this.silenceTimeout = null;
|
||||
const delay = this.getSilenceTimeoutMs(3000);
|
||||
if (delay === 0) return;
|
||||
this.silenceTimeout = setTimeout(() => {
|
||||
if (this.isRecording) {
|
||||
// Finalize any accumulated text before stopping
|
||||
if (this._accumulatedFinal) {
|
||||
this._insertText(this._accumulatedFinal);
|
||||
}
|
||||
this.stop();
|
||||
}
|
||||
}, 3000);
|
||||
},
|
||||
|
||||
_iosStabilityCheck(transcript) {
|
||||
if (transcript !== this._lastTranscript) {
|
||||
this._lastTranscript = transcript;
|
||||
clearTimeout(this._stabilityTimer);
|
||||
this._stabilityTimer = setTimeout(() => {
|
||||
if (this.isRecording) {
|
||||
const finalText = this._accumulatedFinal + transcript;
|
||||
this._hidePreview();
|
||||
this._insertText(finalText);
|
||||
this.stop();
|
||||
}
|
||||
}, 750);
|
||||
}
|
||||
}, delay);
|
||||
},
|
||||
|
||||
_startDurationTimer() {
|
||||
@@ -1200,6 +1284,8 @@ const VoiceInput = {
|
||||
},
|
||||
|
||||
_stopLevelMeter() {
|
||||
this._webSpeechSoundActive = false;
|
||||
this._webSpeechActivityUntil = 0;
|
||||
if (this._levelAnimFrame) {
|
||||
cancelAnimationFrame(this._levelAnimFrame);
|
||||
this._levelAnimFrame = null;
|
||||
@@ -1216,13 +1302,21 @@ const VoiceInput = {
|
||||
},
|
||||
|
||||
_drawLevelMeter() {
|
||||
if (!this._analyser || !this.isRecording) return;
|
||||
const dataArray = new Uint8Array(this._analyser.frequencyBinCount);
|
||||
this._analyser.getByteFrequencyData(dataArray);
|
||||
// Compute RMS level 0-1
|
||||
let sum = 0;
|
||||
for (let i = 0; i < dataArray.length; i++) sum += dataArray[i] * dataArray[i];
|
||||
const rms = Math.sqrt(sum / dataArray.length) / 255;
|
||||
if (!this.isRecording) return;
|
||||
let rms = 0;
|
||||
if (this._activeProvider === 'webspeech') {
|
||||
// The speech API exposes activity events, but no microphone sample level.
|
||||
const active = !this._webSpeechStopping
|
||||
&& (this._webSpeechSoundActive || Date.now() < this._webSpeechActivityUntil);
|
||||
if (active) rms = 0.25 + 0.55 * (1 + Math.sin(Date.now() / 110)) / 2;
|
||||
} else {
|
||||
if (!this._analyser) return;
|
||||
const dataArray = new Uint8Array(this._analyser.frequencyBinCount);
|
||||
this._analyser.getByteFrequencyData(dataArray);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < dataArray.length; i++) sum += dataArray[i] * dataArray[i];
|
||||
rms = Math.sqrt(sum / dataArray.length) / 255;
|
||||
}
|
||||
// Update the level bars in the preview
|
||||
const barsEl = this.previewEl?.querySelector('.voice-level-bars');
|
||||
if (barsEl) {
|
||||
@@ -1252,6 +1346,8 @@ const VoiceInput = {
|
||||
indicator.innerHTML = '<span class="voice-rec-dot"></span>';
|
||||
const barsEl = document.createElement('span');
|
||||
barsEl.className = 'voice-level-bars';
|
||||
barsEl.setAttribute('aria-label', this._activeProvider === 'webspeech' ? 'Speech activity' : 'Microphone level');
|
||||
barsEl.title = this._activeProvider === 'webspeech' ? 'Speech activity' : 'Microphone level';
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const bar = document.createElement('span');
|
||||
bar.className = 'voice-level-bar';
|
||||
@@ -1319,7 +1415,10 @@ const VoiceInput = {
|
||||
|
||||
/** Cleanup on SSE reconnect or page unload */
|
||||
cleanup() {
|
||||
if (this.isRecording) this.stop();
|
||||
if (this.isRecording) {
|
||||
this.stop(true);
|
||||
try { this.recognition?.abort(); } catch (_e) { /* already ended */ }
|
||||
}
|
||||
this._hideVoiceSendBtn();
|
||||
DeepgramProvider._cleanup();
|
||||
ClaudeVoiceProvider._cleanup();
|
||||
@@ -1327,20 +1426,14 @@ const VoiceInput = {
|
||||
this._activeProvider = null;
|
||||
this._stopDurationTimer();
|
||||
this._stopLevelMeter();
|
||||
if (this._webSpeechStream) {
|
||||
this._webSpeechStream.getTracks().forEach(t => t.stop());
|
||||
this._webSpeechStream = null;
|
||||
}
|
||||
if (this.previewEl) {
|
||||
this.previewEl.remove();
|
||||
this.previewEl = null;
|
||||
}
|
||||
clearTimeout(this.silenceTimeout);
|
||||
clearTimeout(this._stabilityTimer);
|
||||
this.silenceTimeout = null;
|
||||
this._stabilityTimer = null;
|
||||
this._accumulatedFinal = '';
|
||||
this._lastTranscript = '';
|
||||
this._pendingInterim = '';
|
||||
this._retryCount = 0;
|
||||
this._hasReceivedResult = false;
|
||||
}
|
||||
|
||||
@@ -1795,6 +1795,8 @@ export const SettingsUpdateSchema = z
|
||||
language: z.string().max(20).optional(),
|
||||
keyterms: z.string().max(500).optional(),
|
||||
insertMode: z.string().max(20).optional(),
|
||||
/** Silence timeout in seconds; 0 disables automatic stopping. Omitted keeps provider defaults. */
|
||||
silenceTimeout: z.union([z.literal(0), z.literal(3), z.literal(6), z.literal(15), z.literal(30)]).optional(),
|
||||
})
|
||||
.optional(),
|
||||
// Run mode preference (cross-device sync)
|
||||
|
||||
@@ -244,7 +244,7 @@ export class VoiceStreamRelay {
|
||||
switch (msg.type) {
|
||||
case 'TranscriptText':
|
||||
case 'TranscriptInterim': {
|
||||
// Each frame is the whole running transcript, not a delta.
|
||||
// Each frame is the current utterance so far, not a delta; endpoints commit it.
|
||||
if (typeof msg.data === 'string' && msg.data) {
|
||||
this.pendingTranscript = msg.data;
|
||||
this.opts.onTranscript(msg.data, false);
|
||||
|
||||
Reference in new issue
Block a user