(function (global) { "use strict"; const namespace = (global.Speakr = global.Speakr || {}); function appendInt16Samples(current, incoming) { const merged = new Int16Array(current.length + incoming.length); merged.set(current); merged.set(incoming, current.length); return merged; } function createPcmStreamer({ label, recorder, isActive, sendChunk, chunkSize, warnSamples, hardLimitSamples, outputSampleRate, }) { let sampleBuffer = new Int16Array(); function reset() { sampleBuffer = new Int16Array(); } function onProcess(buffer, powerLevel, bufferDuration, bufferSampleRate) { if (!isActive()) { return; } const latestFrame = buffer && buffer[buffer.length - 1]; if (!latestFrame) { return; } const resampled = recorder.SampleData( [latestFrame], bufferSampleRate, outputSampleRate ).data; sampleBuffer = appendInt16Samples(sampleBuffer, resampled); if (sampleBuffer.length > hardLimitSamples) { const overflowSamples = sampleBuffer.length - hardLimitSamples; console.warn( `[${label}] Dropping stale local audio buffer: ${overflowSamples} samples` ); sampleBuffer = sampleBuffer.slice(-hardLimitSamples); } while (sampleBuffer.length >= chunkSize) { const chunk = sampleBuffer.slice(0, chunkSize); const accepted = sendChunk(chunk); if (!accepted) { break; } sampleBuffer = sampleBuffer.slice(chunkSize); } if (sampleBuffer.length > warnSamples) { console.warn( `[${label}] Audio buffer backlog: ${sampleBuffer.length} samples (${( sampleBuffer.length / outputSampleRate ).toFixed(2)}s)` ); } } return { onProcess, reset, }; } namespace.createRealtimeAsrSession = function createRealtimeAsrSession({ Recorder, WebSocketConnectMethod, getMicStream, getSystemStream, getPausedTranscription, setMainText, setPreviewText, setOfflineText, getLastTimer, setLastTimer, getConfigWords, shouldShowSpeaker, getSegments, setSegments, getSegmentRenderStartIndex, getRealtimeEpoch, getTimestampBarrier, setTimestampBarrier, getEditCarryover, setEditCarryover, setPendingCorrectionCount, onActiveChange, }) { const ASR_CHUNK_SIZE = 960; const ASR_BUFFER_WARN_SAMPLES = 20000; const ASR_BUFFER_HARD_LIMIT_SAMPLES = 16000 * 5; const ASR_SAMPLE_RATE = 16000; let webReader = null; let webReader2 = null; let rec = null; let rec2 = null; let asrAudioContext = null; let asrAudioContext2 = null; let isActive = false; let resetTranscriptionFlag = false; let recText = ""; let offlineText = ""; let recTextOnline = ""; let offlineSentenceHandoffText = ""; let recTextOffline = ""; let pendingSentenceQueue = []; let pendingSentenceSequence = 0; let lastOnlineTimestamp = null; let pendingCorrectionCount = 0; let onlineStreamer = null; let offlineStreamer = null; function setActive(nextValue) { isActive = !!nextValue; if (typeof onActiveChange === "function") { onActiveChange(isActive); } } function updatePendingCorrectionCount(delta) { pendingCorrectionCount = Math.max(0, pendingCorrectionCount + delta); if (typeof setPendingCorrectionCount === "function") { setPendingCorrectionCount(pendingCorrectionCount); } } function extractTimestampMs(timestampPayload) { if (!timestampPayload) { return null; } try { const timestampData = typeof timestampPayload === "string" ? JSON.parse(timestampPayload) : timestampPayload; if (!Array.isArray(timestampData) || timestampData.length === 0) { return null; } const lastItem = timestampData[timestampData.length - 1]; if (!Array.isArray(lastItem) || lastItem.length === 0) { return null; } const endOrStart = lastItem.length > 1 ? Number(lastItem[1]) : Number(lastItem[0]); return Number.isFinite(endOrStart) ? endOrStart : null; } catch (error) { return null; } } function getRealtimeCarryoverTail(text, maxLength = 160) { return (text || "").slice(-maxLength); } function buildRealtimeEditCarryover(baseText) { const tailText = getRealtimeCarryoverTail(baseText); if (tailText.length < 4) { return null; } return { tailText, }; } function trimOfflineTextWithCarryover(rawText) { const incomingText = rawText || ""; const realtimeEditCarryover = getEditCarryover ? getEditCarryover() : null; if (!realtimeEditCarryover || !incomingText) { return incomingText; } const tailText = realtimeEditCarryover.tailText || ""; const minOverlapLength = 4; const maxOverlapLength = Math.min(80, tailText.length, incomingText.length); for (let size = maxOverlapLength; size >= minOverlapLength; size--) { const overlapText = tailText.slice(-size); const matchIndex = incomingText.lastIndexOf(overlapText); if (matchIndex === -1) { continue; } const remainingText = incomingText.slice(matchIndex + overlapText.length); setEditCarryover({ tailText: getRealtimeCarryoverTail(tailText + remainingText), }); return remainingText; } setEditCarryover(null); return incomingText; } function isStaleRealtimeMessage( messageTimestampMs, messageEpoch = getRealtimeEpoch() ) { if (messageEpoch !== getRealtimeEpoch()) { return true; } const realtimeTimestampBarrierMs = getTimestampBarrier(); return ( Number.isFinite(messageTimestampMs) && realtimeTimestampBarrierMs >= 0 && messageTimestampMs <= realtimeTimestampBarrierMs ); } function getLatestRealtimeTimestamp() { const latestSegmentTimestamp = getSegments().reduce( (latest, segment) => Number.isFinite(segment && segment.timestampMs) ? Math.max(latest, segment.timestampMs) : latest, -1 ); const latestPendingTimestamp = pendingSentenceQueue.reduce( (latest, item) => Number.isFinite(item.timestampMs) ? Math.max(latest, item.timestampMs) : latest, -1 ); const latestOnlineTimestamp = Number.isFinite(lastOnlineTimestamp) ? lastOnlineTimestamp : -1; return Math.max( latestSegmentTimestamp, latestPendingTimestamp, latestOnlineTimestamp ); } function getPendingSentencePreviewText() { return pendingSentenceQueue .filter((item) => item.epoch === getRealtimeEpoch()) .map((item) => item.correctedText || item.rawText) .join(""); } function syncRealtimePreview() { setPreviewText( getPendingSentencePreviewText() + offlineSentenceHandoffText + recTextOnline ); } function clearRealtimePipeline({ preserveTimestampBarrier = false } = {}) { if (onlineStreamer) { onlineStreamer.reset(); } if (offlineStreamer) { offlineStreamer.reset(); } recText = ""; offlineText = ""; recTextOnline = ""; offlineSentenceHandoffText = ""; recTextOffline = ""; pendingSentenceQueue = []; pendingSentenceSequence = 0; lastOnlineTimestamp = null; setPreviewText(""); setOfflineText(""); if (!preserveTimestampBarrier) { setTimestampBarrier(-1); } syncRealtimePreview(); } function findPendingSentenceBySource( sourceText, pendingTimestamp, messageEpoch = getRealtimeEpoch() ) { const normalizedSourceText = sourceText || ""; const normalizedTimestamp = pendingTimestamp || ""; return ( pendingSentenceQueue.find( (item) => item.epoch === messageEpoch && item.sourceText === normalizedSourceText && item.timestamp === normalizedTimestamp ) || getSegments().find( (item) => item.epoch === messageEpoch && item.sourceText === normalizedSourceText && item.timestamp === normalizedTimestamp ) || null ); } function ensurePendingSentence( sourceText, pendingTimestamp, messageEpoch = getRealtimeEpoch() ) { const timestampMs = extractTimestampMs(pendingTimestamp); if (isStaleRealtimeMessage(timestampMs, messageEpoch)) { return null; } const existingItem = findPendingSentenceBySource( sourceText, pendingTimestamp, messageEpoch ); if (existingItem) { return existingItem; } const trimmedText = trimOfflineTextWithCarryover(sourceText || ""); if (!trimmedText) { syncRealtimePreview(); return null; } const pendingItem = { id: pendingSentenceSequence++, sourceText: sourceText || "", rawText: trimmedText, timestamp: pendingTimestamp || "", timestampMs, correctedText: "", processed: false, correctionRequested: false, epoch: messageEpoch, }; pendingSentenceQueue.push(pendingItem); syncRealtimePreview(); return pendingItem; } function addSegment(segment) { const segments = getSegments(); segments.push(segment); setSegments(segments); } function flushProcessedPendingSentences() { let didFlush = false; while (pendingSentenceQueue.length > 0) { if (pendingSentenceQueue[0].epoch !== getRealtimeEpoch()) { pendingSentenceQueue.shift(); continue; } if (!pendingSentenceQueue[0].processed) { break; } const item = pendingSentenceQueue.shift(); addSegment({ text: item.correctedText || item.rawText, sourceText: item.sourceText, timestamp: item.timestamp, timestampMs: item.timestampMs, speaker: null, processed: true, epoch: item.epoch, }); didFlush = true; } if (didFlush) { renderFullText(); } else { syncRealtimePreview(); } } function finalizePendingSentence( pendingSentenceId, finalText, messageEpoch = getRealtimeEpoch() ) { const target = pendingSentenceQueue.find( (item) => item.id === pendingSentenceId && item.epoch === messageEpoch ); if (!target) { return; } target.correctedText = finalText || target.rawText || ""; target.processed = true; target.correctionRequested = false; flushProcessedPendingSentences(); } function addNewlineAfterPeriod(str) { return str.replace(/[。?!]/g, "$&\n"); } function handleWithTimestamp(tmptext, tmptime, keywordStr, timeThreshold) { if (!tmptime || tmptext.length <= 0) return tmptext; const keywordList = keywordStr .split(",") .map((k) => k.trim()) .filter((k) => k.length > 0); const keywordsAsRegex = keywordList.map((k) => new RegExp(k)); const segments = tmptext.match(/[^。!?]+[。!?]?/g) || []; const jsontime = JSON.parse(tmptime); let charIndex = 0; let textWithTime = ""; function removeLeadingPunctuation(text) { return text.replace(/^[,。!?、;:,\.!?;:]+/, ""); } for (let i = 0; i < segments.length; i++) { const segment = segments[i]; if (!segment || segment === "undefined") continue; const nowTime = jsontime[charIndex][0] / 1000; if ( getLastTimer() > 0 && nowTime - getLastTimer() > timeThreshold ) { const lastChar = textWithTime.slice(-1); const endsWithPunc = /[。!?]/.test(lastChar); const processedSegment = removeLeadingPunctuation(segment); textWithTime += (endsWithPunc ? "\n" : "。\n") + processedSegment; } else { const hitKeyword = keywordsAsRegex.some((reg) => reg.test(segment)); if (hitKeyword) { removeLeadingPunctuation(segment); } else { textWithTime += segment; } } setLastTimer(nowTime); charIndex++; } return textWithTime; } function getSpeakerBeforeIndex(endIndex) { let previousSpeaker = null; for (let i = 0; i < endIndex; i++) { const segment = getSegments()[i]; if (segment && segment.speaker) { previousSpeaker = segment.speaker; } } return previousSpeaker; } function buildSegmentText(segments, initialSpeaker = null) { let displayText = ""; let lastRenderedSpeaker = initialSpeaker; segments.forEach((segment) => { if (shouldShowSpeaker()) { if ( segment.speaker && segment.speaker !== lastRenderedSpeaker && !segment.speaker.includes("SPK") ) { displayText += (displayText ? "\n" : "") + segment.speaker + ":\n"; lastRenderedSpeaker = segment.speaker; } else if ( segment.speaker && segment.speaker !== lastRenderedSpeaker ) { displayText += (displayText ? "\n" : "") + segment.speaker + ":\n"; lastRenderedSpeaker = segment.speaker; } } const textContent = handleWithTimestamp( segment.text, segment.timestamp, getConfigWords(), 6 ); displayText += textContent; }); return displayText; } function renderFullText() { const segmentRenderStartIndex = getSegmentRenderStartIndex(); const initialSpeaker = segmentRenderStartIndex > 0 ? getSpeakerBeforeIndex(segmentRenderStartIndex) : null; const displayText = buildSegmentText( getSegments().slice(segmentRenderStartIndex), initialSpeaker ); recText = displayText; setMainText(getPausedTranscription() + recText); syncRealtimePreview(); } function resetIfRequested() { if (resetTranscriptionFlag) { console.log("重置实时转录状态(暂停/恢复后)"); clearRealtimePipeline(); resetTranscriptionFlag = false; } } function handleOnlineMessage(jsonMsg) { const parsedMessage = JSON.parse(jsonMsg.data); const text = "" + parsedMessage["text"]; const asrModel = parsedMessage["mode"]; const timestamp = parsedMessage["timestamp"]; resetIfRequested(); const messageEpoch = getRealtimeEpoch(); if (asrModel === "2pass-online") { const currentTimestamp = extractTimestampMs(timestamp); if (isStaleRealtimeMessage(currentTimestamp, messageEpoch)) { syncRealtimePreview(); return; } let shouldBreak = false; if (lastOnlineTimestamp !== null && currentTimestamp !== null) { const timeDiff = (currentTimestamp - lastOnlineTimestamp) / 1000; if (timeDiff >= 3 && /。\s*$/.test(text)) { shouldBreak = true; } } recTextOnline += shouldBreak ? text + "\n" : text; lastOnlineTimestamp = currentTimestamp; } else if (asrModel === "2pass-offline") { recTextOnline = ""; offlineSentenceHandoffText = text || recTextOnline || ""; recTextOnline = ""; lastOnlineTimestamp = null; } syncRealtimePreview(); } async function handleOfflineMessage(jsonMsg) { const parsedMessage = JSON.parse(jsonMsg.data); const asrModelType = parsedMessage["mode"]; const timestamp = parsedMessage["timestamp"]; const messageEpoch = getRealtimeEpoch(); const messageTimestampMs = extractTimestampMs(timestamp); if (asrModelType === "2pass-offline") { if (isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) { return; } const text = "" + parsedMessage["text"]; recTextOnline = ""; lastOnlineTimestamp = null; const pendingSentence = ensurePendingSentence( text, timestamp, messageEpoch ); offlineSentenceHandoffText = ""; syncRealtimePreview(); if ( !pendingSentence || pendingSentence.processed || pendingSentence.correctionRequested ) { return; } pendingSentence.correctionRequested = true; updatePendingCorrectionCount(1); try { const response = await fetch(`/tool/speakr/api/asr/correct`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text: pendingSentence.rawText }), }); if (!response.ok) { throw new Error("ASR correction failed"); } const data = await response.json(); const correctedText = data.corrected_text || pendingSentence.rawText; finalizePendingSentence( pendingSentence.id, correctedText, messageEpoch ); console.log("LLM矫正完成"); } catch (error) { finalizePendingSentence( pendingSentence.id, pendingSentence.rawText, messageEpoch ); console.error("LLM矫正出错:", error); } finally { updatePendingCorrectionCount(-1); } } else if (asrModelType === "offline-speaker") { if (isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) { return; } const nowSpeaker = (parsedMessage["segments"] && parsedMessage["segments"][0] && parsedMessage["segments"][0].speaker) || "未命名演讲者"; const segments = getSegments(); let targetIndex = -1; for (let i = segments.length - 1; i >= 0; i--) { const segment = segments[i]; if (!segment || segment.epoch !== messageEpoch) { continue; } if ( segment.speaker === null && Number.isFinite(messageTimestampMs) && Number.isFinite(segment.timestampMs) && segment.timestampMs === messageTimestampMs ) { targetIndex = i; break; } if (targetIndex === -1 && segment.speaker === null) { targetIndex = i; } } if (targetIndex !== -1) { segments[targetIndex].speaker = nowSpeaker; setSegments(segments); renderFullText(); } } } async function getMicrophoneInfo() { if ( !navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices ) { console.warn("当前浏览器不支持获取设备信息"); return null; } const devices = await navigator.mediaDevices.enumerateDevices(); const mics = devices.filter((device) => device.kind === "audioinput"); const defaultMic = mics.find((device) => device.deviceId === "default") || mics[0]; return { name: (defaultMic && defaultMic.label) || "未知麦克风", deviceId: (defaultMic && defaultMic.deviceId) || "", all: mics.map((device) => ({ name: device.label, deviceId: device.deviceId, })), }; } async function openMixedStream(channel) { try { if (channel === "online") { const micInfo = await getMicrophoneInfo(); if (micInfo) { console.log("🎤 当前麦克风:", micInfo.name, micInfo); } } const micStream = getMicStream(); const systemStream = getSystemStream(); let contextRef = channel === "online" ? asrAudioContext : asrAudioContext2; if (contextRef) { try { contextRef.close(); } catch (error) {} } contextRef = new (global.AudioContext || global.webkitAudioContext)(); if (channel === "online") { asrAudioContext = contextRef; } else { asrAudioContext2 = contextRef; } const destination = contextRef.createMediaStreamDestination(); const micSource = contextRef.createMediaStreamSource(micStream); micSource.connect(destination); if (systemStream && systemStream.getAudioTracks().length > 0) { const sysSource = contextRef.createMediaStreamSource(systemStream); sysSource.connect(destination); } else { console.log("⚠️ 未检测到系统音频,使用麦克风录音"); } const recorder = channel === "online" ? rec : rec2; recorder.open( () => { recorder.start(destination.stream); }, (msg) => { console.error("❌ 录音权限被拒绝:", msg); } ); } catch (error) { console.error("初始化录音失败:", error); } } function stop() { setActive(false); if (webReader) { webReader.wsStop(); } if (webReader2) { webReader2.wsStop(); } } async function finish(timeoutMs = 3500) { setActive(false); const finishPromises = []; if (webReader && typeof webReader.wsFinish === "function") { finishPromises.push(webReader.wsFinish(timeoutMs)); } else if (webReader) { webReader.wsStop(); } if (webReader2 && typeof webReader2.wsFinish === "function") { finishPromises.push(webReader2.wsFinish(timeoutMs)); } else if (webReader2) { webReader2.wsStop(); } if (finishPromises.length > 0) { await Promise.allSettled(finishPromises); } } function cleanup() { stop(); if (rec) { try { rec.close(); } catch (error) { console.warn("关闭旧rec失败:", error); } rec = null; } if (rec2) { try { rec2.close(); } catch (error) { console.warn("关闭旧rec2失败:", error); } rec2 = null; } if (asrAudioContext) { try { asrAudioContext.close(); } catch (error) {} asrAudioContext = null; } if (asrAudioContext2) { try { asrAudioContext2.close(); } catch (error) {} asrAudioContext2 = null; } } function createRecorders() { onlineStreamer = createPcmStreamer({ label: "Online", recorder: Recorder, isActive: () => isActive, sendChunk: (chunk) => webReader && webReader.wsSend(chunk), chunkSize: ASR_CHUNK_SIZE, warnSamples: ASR_BUFFER_WARN_SAMPLES, hardLimitSamples: ASR_BUFFER_HARD_LIMIT_SAMPLES, outputSampleRate: ASR_SAMPLE_RATE, }); offlineStreamer = createPcmStreamer({ label: "Offline", recorder: Recorder, isActive: () => isActive, sendChunk: (chunk) => webReader2 && webReader2.wsSend(chunk), chunkSize: ASR_CHUNK_SIZE, warnSamples: ASR_BUFFER_WARN_SAMPLES, hardLimitSamples: ASR_BUFFER_HARD_LIMIT_SAMPLES, outputSampleRate: ASR_SAMPLE_RATE, }); rec = Recorder({ type: "pcm", bitRate: 16, sampleRate: ASR_SAMPLE_RATE, onProcess: onlineStreamer.onProcess, }); rec2 = Recorder({ type: "pcm", bitRate: 16, sampleRate: ASR_SAMPLE_RATE, onProcess: offlineStreamer.onProcess, }); } function setupConnections() { webReader = new WebSocketConnectMethod({ msgHandle: handleOnlineMessage, stateHandle(connState) { if (connState === 0) { openMixedStream("online"); } else if (connState === 2) { stop(); } }, }); webReader2 = new WebSocketConnectMethod({ msgHandle: handleOfflineMessage, stateHandle(connState) { if (connState === 0) { openMixedStream("offline"); } else if (connState === 2) { stop(); } }, }); } function start() { setupConnections(); createRecorders(); const onlineStarted = webReader.wsStart(); const offlineStarted = webReader2.wsStart("offline"); if (onlineStarted === 1 || offlineStarted === 1) { setActive(true); } return onlineStarted === 1 && offlineStarted === 1; } function restart() { resetTranscriptionFlag = true; setPreviewText(""); let started = false; if (webReader) { const ret = webReader.wsStart(); if (ret === 1) { started = true; } } if (webReader2) { try { const ret = webReader2.wsStart("offline"); if (ret === 1) { started = true; } } catch (error) { console.warn("Failed to restart offline WS:", error); } } if (started) { setActive(true); } return started; } return { start, restart, stop, finish, cleanup, clearRealtimePipeline, getLatestRealtimeTimestamp, buildRealtimeEditCarryover, renderFullText, getPendingCorrectionCount() { return pendingCorrectionCount; }, }; }; })(window);