// ============================================================================= // 文件来源:从 src/main-old.js 拆分而来(实时 ASR 链路独立成类) // 与原代码(main-old.js)相比的【行为差异】,需在迁移/回归时特别注意: // // 1. WebSocket state===2(断连)处理 // - 原代码:getConnState/getConnState2 在 state===2 时调用 stop(),但 // main-old.js 里并没有定义 stop 函数,触发时会抛 ReferenceError(原 bug)。 // - 新代码:handleOnlineState(2)/handleOfflineState(2) 调用 this.stop(), // 真正关闭 ws + Recorder + AudioContext,但【不会】停止 MediaRecorder // 或把 isRecording 置 false(pipeline 只管 ASR 通道)。 // - 含义:ws 断连不再报错,但整体录音会话不会因此停止。若产品希望 ws // 断连时也停掉 MediaRecorder,需在 main.js 的 onActiveChange(false) // 或新增 onDisconnect 回调里做额外处理(当前未做)。 // // 2. stopRecording 会立即关闭 ASR Recorder // - 原代码:stopRecording 只调 webReader.wsFinish/wsStop,rec/rec2 直到 // 切走 recording view 才 close。 // - 新代码:pipeline.finish() 会 closeRecorders()。恢复录音时 // ensureRecorders() 会重建,逻辑自洽。若存在“停止后不切 view 直接再录” // 的路径需确认。 // // 3. correct 接口新增 scene 字段 // - 原代码:/api/asr/correct body 只传 { text }。 // - 新代码:handleOfflineMessage 调 correct 时多传 { text, scene }。 // - 前置条件:后端 /tool/speakr/api/asr/correct 必须容忍/使用 scene 字段。 // // 4. resumeRecording 不再设置 resetTranscriptionFlag // - 原代码:resume 时设 resetTranscriptionFlag=true,等下一条 ws 消息进来 // 时 getJsonMessage 再 clearRealtimePipeline()。 // - 新代码:main.js 的 resumeRecording 开头直接 resetRealtimeTranscript(), // 立刻清。结果等价,时序不同(更干净)。 // // 5. continueDraftRecording 现在会调 resetRealtimeTranscript // - 原代码:加载草稿只重置 segmentRenderStartIndex=0。 // - 新代码:额外清空 pendingSentenceQueue 并 startNewEpoch。更彻底, // 基本是改进,但若依赖 pending 队列残留需注意。 // // 6. wsconnecter 现在收到 getScene // - 原代码:new WebSocketConnectMethod({msgHandle, stateHandle}) 不传 // getScene,wsconnecter 走默认 "cg"。 // - 新代码:createSockets() 传 getScene: () => this.getScene(),按 // selectedScene 路由到不同后端 ASR URL。新增场景切换能力。 // // 7. 性能优化(行为等价) // - appendInt16 替代 Int16Array.from([...sampleBuf, ...data_16k]),避免 // 高频 onProcess 里用展开运算符复制大数组。结果等价。 // // 8. 新增 dispose() 屏障 // - 原代码:离开 recording view 后,迟到的 ws/LLM 回调仍可能写入页面状态。 // - 新代码:dispose() 设 disposed=true,handleOnlineMessage/handleOfflineMessage // /processRecorderFrame 入口都检查 disposed,迟到回调被忽略。 // ============================================================================= import { Recorder } from "../vendor-globals.js"; import { WebSocketConnectMethod } from "./wsconnecter.js"; // FunASR 实时 websocket 需要持续收到短 PCM 帧。 // 16 kHz 下 960 个 sample 约等于 60ms 音频,沿用原链路的分片粒度。 const DEFAULT_CHUNK_SIZE = 960; // 软告警:只提示 websocket 发送速度已经落后于采集速度。 const DEFAULT_BUFFER_WARN_SAMPLES = 20000; // 硬上限:最多保留最近 5 秒音频,丢弃更旧数据,避免内存持续增长。 const DEFAULT_BUFFER_HARD_LIMIT_SAMPLES = 16000 * 5; function noop() {} // Recorder.SampleData 返回 Int16Array;这里显式拼接,避免在高频 onProcess // 回调里用展开运算符复制大数组。 // 【行为差异 #7】替代原 main-old.js 的 Int16Array.from([...sampleBuf, ...data_16k]), // 行为等价,性能更好。 function appendInt16(left, right) { const merged = new Int16Array(left.length + right.length); merged.set(left, 0); merged.set(right, left.length); return merged; } function createAudioContext() { const AudioContextCtor = window.AudioContext || window.webkitAudioContext; return new AudioContextCtor(); } /** * 浏览器实时 ASR 完整管线: * Recorder -> websocket -> 回包解析 -> LLM 矫正 -> 页面文本回调。 * * Vue 仍然负责页面 ref、媒体流和录音控制;本类只通过 getter/callback * 读写这些依赖,因此 ASR 链路可以从 main.js 中独立出来,而不引入 Vue。 */ export class RealtimeAsrPipeline { constructor({ getMicStream, getSystemStream, getPausedTranscription, setPausedTranscription, getShowSpeaker, getConfigWords, getScene = () => "cg", getLastTimer, setLastTimer, onMainTextChange = noop, onPreviewTextChange = noop, onOfflineTextChange = noop, onActiveChange = noop, onPendingCorrectionCountChange = noop, onError = noop, fetchImpl = window.fetch.bind(window), correctUrl = "/tool/speakr/api/asr/correct", sourceKind = "mixed", }) { // 页面拥有的依赖。管线通过 getter 读取媒体流,通过 callback 回写显示文本, // 不直接触碰 Vue ref。 this.getMicStream = getMicStream; this.getSystemStream = getSystemStream; this.getPausedTranscription = getPausedTranscription; this.setPausedTranscription = setPausedTranscription; this.getShowSpeaker = getShowSpeaker; this.getConfigWords = getConfigWords; this.getScene = getScene; this.getLastTimer = getLastTimer; this.setLastTimer = setLastTimer; this.onMainTextChange = onMainTextChange; this.onPreviewTextChange = onPreviewTextChange; this.onOfflineTextChange = onOfflineTextChange; this.onActiveChange = onActiveChange; this.onPendingCorrectionCountChange = onPendingCorrectionCountChange; this.onError = onError; this.fetchImpl = fetchImpl; this.correctUrl = correctUrl; this.sourceKind = sourceKind; // 运行期资源。online websocket 负责低延迟预览;offline websocket 负责 // 正式句子、说话人回填和 LLM 矫正。 this.wsOnline = null; this.wsOffline = null; this.recOnline = null; this.recOffline = null; this.audioContextOnline = null; this.audioContextOffline = null; // PCM 发送缓冲、pending 队列和 segmentList 都是管线内部状态。 // segmentList 保存已按顺序矫正完成的正式转录片段。 this.sampleBufOnline = new Int16Array(); this.sampleBufOffline = new Int16Array(); this.segmentList = []; this.segmentRenderStartIndex = 0; this.pendingSentenceQueue = []; this.pendingSentenceSequence = 0; this.pendingCorrectionCount = 0; // recText 是已正式落到页面的文本;recTextOnline 和 handoff 文本是临时预览, // 用于 offline/LLM 矫正返回前让用户先看到内容。 this.recText = ""; this.recTextOnline = ""; this.offlineSentenceHandoffText = ""; this.lastOnlineTimestamp = null; this.liveTranscriptionEpoch = 0; this.realtimeTimestampBarrierMs = -1; this.realtimeEditCarryover = null; this.editModeBufferStartIndex = 0; this.active = false; this.disposed = false; this.createSockets(); } // 预先创建两条 websocket 客户端。传输对象负责排队和发送;本管线负责解释回包。 // 【行为差异 #6】原 main-old.js 创建 WebSocketConnectMethod 时不传 getScene, // wsconnecter 走默认 "cg";这里传 getScene: () => this.getScene(),按 // selectedScene 路由到不同后端 ASR URL,支持场景切换。 createSockets() { this.wsOnline = new WebSocketConnectMethod({ msgHandle: (event) => this.handleOnlineMessage(event), stateHandle: (state) => this.handleOnlineState(state), getScene: () => this.getScene(), }); this.wsOffline = new WebSocketConnectMethod({ msgHandle: (event) => this.handleOfflineMessage(event), stateHandle: (state) => this.handleOfflineState(state), getScene: () => this.getScene(), }); } // 启动/重启实时 ASR。Recorder 不在此处创建——它需要 sourceStream, // 而音频流要在 openMixedStream 时才能确定。ws open 后会回调 // handleOnlineState/handleOfflineState → openMixedStream 动态创建 Recorder。 start() { this.disposed = false; const onlineStarted = this.wsOnline?.wsStart() === 1; const offlineStarted = this.wsOffline?.wsStart("offline") === 1; this.setActive(onlineStarted || offlineStarted); return onlineStarted || offlineStarted ? 1 : 0; } // 立即停止:用于暂停或离开录音视图。关闭 socket 和 Recorder 回调, // 避免旧 onProcess 帧继续向已关闭 websocket 发送数据。 stop() { this.setActive(false); this.wsOnline?.wsStop(); this.wsOffline?.wsStop(); this.closeRecorders(); } // 优雅停止:用于用户结束录音。wsFinish 发送 stop 帧并短暂等待后端 // flush 最后的 offline/LLM 结果。 // 【行为差异 #2】原 main-old.js 的 stopRecording 只调 webReader.wsFinish/wsStop, // rec/rec2 直到切走 recording view 才 close;这里 finish() 会立即 // closeRecorders()。恢复录音时 ensureRecorders() 会重建,逻辑自洽。 async finish(timeoutMs = 3000) { this.setActive(false); this.closeRecorders(); const finishPromises = []; if (this.wsOnline && typeof this.wsOnline.wsFinish === "function") { finishPromises.push(this.wsOnline.wsFinish(timeoutMs)); } else { this.wsOnline?.wsStop(); } if (this.wsOffline && typeof this.wsOffline.wsFinish === "function") { finishPromises.push(this.wsOffline.wsFinish(timeoutMs)); } else { this.wsOffline?.wsStop(); } if (finishPromises.length > 0) { await Promise.allSettled(finishPromises); } } // dispose 比 stop 更彻底:离开录音视图后,迟到的 websocket/矫正回调 // 应被忽略,不能再写入新的页面状态。 // 【行为差异 #8】新增 disposed 屏障。原 main-old.js 离开 view 后迟到的 // ws/LLM 回调仍可能写入页面状态;这里 handleOnlineMessage/handleOfflineMessage // /processRecorderFrame 入口都检查 this.disposed,迟到回调被忽略。 dispose() { this.disposed = true; this.stop(); this.wsOnline = null; this.wsOffline = null; } isActive() { return this.active; } setActive(active) { this.active = active; this.onActiveChange(active); } setPendingCorrectionCount(count) { this.pendingCorrectionCount = Math.max(0, count); this.onPendingCorrectionCountChange(this.pendingCorrectionCount); } getPendingCorrectionCount() { return this.pendingCorrectionCount; } // 暂停/保存草稿前要等待正在进行的 LLM 矫正;否则草稿可能保存 raw offline 文本, // 而正确句子马上就要返回。 async waitForCorrections(timeoutMs = 8000, pollMs = 200) { const waitStart = Date.now(); while ( this.pendingCorrectionCount > 0 && Date.now() - waitStart < timeoutMs ) { await new Promise((resolve) => setTimeout(resolve, pollMs)); } return this.pendingCorrectionCount; } // epoch 是旧消息屏障。录音周期变化后,旧 websocket/LLM 异步结果都会被忽略。 startNewEpoch({ barrierTimestampMs = null, clearTimestampBarrier = false, clearEditCarryover = true, } = {}) { this.liveTranscriptionEpoch += 1; if (clearEditCarryover) { this.realtimeEditCarryover = null; } if (clearTimestampBarrier) { this.realtimeTimestampBarrierMs = -1; } else if (Number.isFinite(barrierTimestampMs)) { this.realtimeTimestampBarrierMs = Math.max( this.realtimeTimestampBarrierMs, barrierTimestampMs ); } this.clearRealtimePipeline({ preserveTimestampBarrier: !clearTimestampBarrier, }); } // 只清理实时转写内部状态,不清空页面历史文本。草稿恢复依赖 // pausedTranscription 保持可见,新片段继续追加在它后面。 resetTranscript({ clearTimestampBarrier = false } = {}) { this.segmentList = []; this.segmentRenderStartIndex = 0; this.setPendingCorrectionCount(0); this.startNewEpoch({ clearTimestampBarrier }); } // 准备恢复录音:保留历史转写数据,但清空内部缓冲区准备接收新数据 // pausedTranscription 保持可见,新片段从 segmentList 末尾继续追加 prepareForResume() { // 保留 segmentList(包含说话人、时间戳等元数据),只调整渲染起点 this.segmentRenderStartIndex = this.segmentList.length; this.editModeBufferStartIndex = this.segmentList.length; // 清空内部缓冲区,准备接收新的实时数据 this.sampleBufOnline = new Int16Array(); this.sampleBufOffline = new Int16Array(); this.recText = ""; this.recTextOnline = ""; this.offlineSentenceHandoffText = ""; this.pendingSentenceQueue = []; this.pendingSentenceSequence = 0; this.lastOnlineTimestamp = null; // 清空预览文本 this.onPreviewTextChange(""); this.onOfflineTextChange(""); // 增加 epoch 以过滤掉旧消息,但保留时间戳屏障 this.liveTranscriptionEpoch += 1; // 重新渲染,确保显示完整的历史文本 this.renderFullText(); this.syncRealtimePreview(); } // 手动编辑后,以当前页面文本作为新的基准。记录当前 segment 边界, // 后续 ASR 结果只追加到编辑后的文本之后。 enterEditMode(baseText) { this.editModeBufferStartIndex = this.segmentList.length; const barrierTimestampMs = this.getLatestRealtimeTimestamp(); this.startNewEpoch({ barrierTimestampMs, clearEditCarryover: false, }); this.realtimeEditCarryover = this.buildRealtimeEditCarryover(baseText); } // 立即应用用户编辑,并让后续矫正片段从保存的编辑边界后继续渲染, // 防止旧 ASR 文本覆盖用户手改内容。 applyEditedText(nextText) { this.setPausedTranscription(nextText); this.segmentRenderStartIndex = this.editModeBufferStartIndex; this.recText = ""; this.recTextOnline = ""; this.offlineSentenceHandoffText = ""; this.onPreviewTextChange(""); this.onOfflineTextChange(""); this.onMainTextChange(nextText); this.renderFullText(); } // 这里的 Recorder 只负责实时 PCM 推流;它和页面里用于保存/上传 webm // 成品音频的 MediaRecorder 是两套东西。 // Recorder 不再提前创建。sourceStream 必须在构造时传入,而音频流在 // openMixedStream 时才确定,因此 Recorder 实例改在 openMixedStream 中创建。 // 此方法保留为空操作,避免外部调用方报错。 ensureRecorders() { // no-op: Recorder 实例现在在 openMixedStream 中按需创建 } closeRecorders() { if (this.recOnline) { try { this.recOnline.close(); } catch (error) { // 关闭失败不影响整体清理,继续释放后续资源。 console.warn("Failed to close online Recorder:", error); } this.recOnline = null; } if (this.recOffline) { try { this.recOffline.close(); } catch (error) { console.warn("Failed to close offline Recorder:", error); } this.recOffline = null; } this.closeAudioContexts(); } closeAudioContexts() { if (this.audioContextOnline) { try { this.audioContextOnline.close(); } catch (_) {} this.audioContextOnline = null; } if (this.audioContextOffline) { try { this.audioContextOffline.close(); } catch (_) {} this.audioContextOffline = null; } } handleOnlineState(state) { if (state === 0) { this.openMixedStream("online"); } else if (state === 2) { // 【行为差异 #1】原 main-old.js 在 getConnState state===2 时调用未定义的 // stop() 会抛 ReferenceError;这里改为优雅关闭 ASR 通道(ws + Recorder + // AudioContext),但【不会】停止 MediaRecorder 或 isRecording——pipeline // 只管 ASR 通道。若需在 ws 断连时整体停录,需在 main.js 的回调里补处理。 this.stop(); } } handleOfflineState(state) { if (state === 0) { this.openMixedStream("offline"); } else if (state === 2) { // 【行为差异 #1】同 handleOnlineState(2)。 this.stop(); } } // 为指定通道构造混合 MediaStream。online/offline 监听同一组麦克风/系统音源, // 但各自维护独立 AudioContext 和 Recorder。 async openMixedStream(channel) { try { const micInfo = channel === "online" ? await this.getMicrophoneInfo() : null; if (micInfo) { console.log("Current microphone:", micInfo.name, micInfo); } const micStream = this.getMicStream(); const systemStream = this.getSystemStream(); const hasMic = micStream && micStream.getAudioTracks().length > 0; const hasSystem = systemStream && systemStream.getAudioTracks().length > 0; if (this.sourceKind === "mic" && !hasMic) { throw new Error("No microphone stream is available for realtime ASR."); } if (this.sourceKind === "system" && !hasSystem) { throw new Error("No system audio stream is available for realtime ASR."); } if (this.sourceKind === "mixed" && !hasMic && !hasSystem) { throw new Error("No audio stream is available for realtime ASR."); } if (channel === "online" && this.audioContextOnline) { try { this.audioContextOnline.close(); } catch (_) {} } if (channel === "offline" && this.audioContextOffline) { try { this.audioContextOffline.close(); } catch (_) {} } const audioContext = createAudioContext(); if (channel === "online") { this.audioContextOnline = audioContext; } else { this.audioContextOffline = audioContext; } const destination = audioContext.createMediaStreamDestination(); if ((this.sourceKind === "mixed" || this.sourceKind === "mic") && hasMic) { audioContext.createMediaStreamSource(micStream).connect(destination); } if ((this.sourceKind === "mixed" || this.sourceKind === "system") && hasSystem) { audioContext.createMediaStreamSource(systemStream).connect(destination); } else if (this.sourceKind === "mixed" && !hasMic) { console.log("No system audio detected for realtime ASR."); } // 【关键修复】Recorder 必须在构造时传入 sourceStream,否则 open() 会 // 自行调用 getUserMedia({audio:true}) 获取麦克风流,导致系统音频被忽略。 // 在此处动态创建 Recorder,将混音流作为 sourceStream 传入。 const existingRecorder = channel === "online" ? this.recOnline : this.recOffline; if (existingRecorder) { try { existingRecorder.close(); } catch (_) {} } // 创建一个实时音频采集器,用于语音识别 const recorder = Recorder({ type: "pcm", // 输出未经压缩的原始音频数据,方便后续处理 bitRate: 16, sampleRate: 16000, // 16kHz 是语音识别模型的标准采样率 sourceStream: destination.stream, // 传入混音后的音频流(麦克风+系统音频混合),而不是让 Recorder 自己去调 getUserMedia 获取麦克风 onProcess: (...args) => this.processRecorderFrame(channel, ...args), }); if (channel === "online") { this.recOnline = recorder; } else { this.recOffline = recorder; } recorder.open( () => { recorder.start(); }, (message) => { console.error("Recorder permission denied:", message); this.onError(message); } ); } catch (error) { console.error("Failed to initialize realtime ASR recorder:", error); this.onError(error); } } // Recorder onProcess 入口:把浏览器采样率下的新 PCM 片段重采样到 16k, // 进入缓冲区,再切成 websocket 分片发送到对应通道。 processRecorderFrame(channel, buffer, _powerLevel, _duration, sampleRate) { if (!this.active || this.disposed || !buffer || buffer.length === 0) { return; } const sourceChunk = buffer[buffer.length - 1]; // Recorder.SampleData 需要传入片段数组,并返回 ASR 所需的 16k PCM。 const data16k = Recorder.SampleData([sourceChunk], sampleRate, 16000).data; const isOnline = channel === "online"; const ws = isOnline ? this.wsOnline : this.wsOffline; const label = isOnline ? "Online" : "Offline"; let sampleBuf = isOnline ? this.sampleBufOnline : this.sampleBufOffline; sampleBuf = appendInt16(sampleBuf, data16k); // 如果 websocket 发送跟不上采集,只保留最近语音,丢弃最旧音频。 if (sampleBuf.length > DEFAULT_BUFFER_HARD_LIMIT_SAMPLES) { const overflowSamples = sampleBuf.length - DEFAULT_BUFFER_HARD_LIMIT_SAMPLES; console.warn( `[${label}] Dropping excess realtime ASR audio buffer: ${overflowSamples} samples` ); sampleBuf = sampleBuf.slice(-DEFAULT_BUFFER_HARD_LIMIT_SAMPLES); } // 发送所有完整分片;不足一个分片的尾巴继续留在通道缓冲里。 while (sampleBuf.length >= DEFAULT_CHUNK_SIZE) { const sendBuf = sampleBuf.slice(0, DEFAULT_CHUNK_SIZE); const sendAccepted = ws?.wsSend(sendBuf); if (!sendAccepted) { break; } sampleBuf = sampleBuf.slice(DEFAULT_CHUNK_SIZE); } if (sampleBuf.length > DEFAULT_BUFFER_WARN_SAMPLES) { console.warn( `[${label}] Realtime ASR audio buffer is high: ${sampleBuf.length} samples (${( sampleBuf.length / 16000 ).toFixed(2)}s)` ); } if (isOnline) { this.sampleBufOnline = sampleBuf; } else { this.sampleBufOffline = sampleBuf; } } async getMicrophoneInfo() { if ( !navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices ) { console.warn("Browser does not support enumerating audio devices."); return null; } const devices = await navigator.mediaDevices.enumerateDevices(); const mics = devices.filter((device) => device.kind === "audioinput"); const defaultMic = mics.find((device) => device.deviceId === "default") || mics[0]; return { name: defaultMic?.label || "Unknown microphone", deviceId: defaultMic?.deviceId || "", all: mics.map((device) => ({ name: device.label, deviceId: device.deviceId, })), }; } // 清理临时缓冲和预览文本。手动编辑时可保留时间戳屏障, // 防止编辑点之前的旧 websocket 消息重新进入。 clearRealtimePipeline({ preserveTimestampBarrier = false } = {}) { this.sampleBufOnline = new Int16Array(); this.sampleBufOffline = new Int16Array(); this.recText = ""; this.recTextOnline = ""; this.offlineSentenceHandoffText = ""; this.pendingSentenceQueue = []; this.pendingSentenceSequence = 0; this.lastOnlineTimestamp = null; this.onPreviewTextChange(""); this.onOfflineTextChange(""); if (!preserveTimestampBarrier) { this.realtimeTimestampBarrierMs = -1; } this.syncRealtimePreview(); } // FunASR timestamp 可能是 JSON 字符串,也可能已经是数组。这里取最后一段 // 结束时间,用于去重、旧消息过滤和后续说话人对齐。 extractTimestampMs(timestampPayload) { if (!timestampPayload) { return null; } try { const timestampData = typeof timestampPayload === "string" ? JSON.parse(timestampPayload) : timestampPayload; if (!Array.isArray(timestampData) || timestampData.length === 0) { return null; } const lastItem = timestampData[timestampData.length - 1]; if (!Array.isArray(lastItem) || lastItem.length === 0) { return null; } const endOrStart = lastItem.length > 1 ? Number(lastItem[1]) : Number(lastItem[0]); return Number.isFinite(endOrStart) ? endOrStart : null; } catch (_) { return null; } } // 手动编辑前获取最新 ASR 时间点,用它建立旧消息屏障。 getLatestRealtimeTimestamp() { const latestSegmentTimestamp = this.segmentList.reduce( (latest, segment) => Number.isFinite(segment?.timestampMs) ? Math.max(latest, segment.timestampMs) : latest, -1 ); const latestPendingTimestamp = this.pendingSentenceQueue.reduce( (latest, item) => Number.isFinite(item.timestampMs) ? Math.max(latest, item.timestampMs) : latest, -1 ); const latestOnlineTimestamp = Number.isFinite(this.lastOnlineTimestamp) ? this.lastOnlineTimestamp : -1; return Math.max( latestSegmentTimestamp, latestPendingTimestamp, latestOnlineTimestamp ); } getRealtimeCarryoverTail(text, maxLength = 160) { return (text || "").slice(-maxLength); } // 保存编辑后文本的一小段尾巴。下一条 offline 结果可能与它重叠, // trimOfflineTextWithCarryover 会用它裁掉重复内容。 buildRealtimeEditCarryover(baseText) { const tailText = this.getRealtimeCarryoverTail(baseText); if (tailText.length < 4) { return null; } return { tailText }; } // 手动编辑后,offline ASR 可能返回与编辑尾巴重复的文本。这里只保留新后缀, // 避免页面出现重复句子。 trimOfflineTextWithCarryover(rawText) { const incomingText = rawText || ""; if (!this.realtimeEditCarryover || !incomingText) { return incomingText; } const tailText = this.realtimeEditCarryover.tailText || ""; const minOverlapLength = 4; const maxOverlapLength = Math.min( 80, tailText.length, incomingText.length ); for (let size = maxOverlapLength; size >= minOverlapLength; size--) { const overlapText = tailText.slice(-size); const matchIndex = incomingText.lastIndexOf(overlapText); if (matchIndex === -1) { continue; } const remainingText = incomingText.slice(matchIndex + overlapText.length); this.realtimeEditCarryover.tailText = this.getRealtimeCarryoverTail( tailText + remainingText ); return remainingText; } this.realtimeEditCarryover = null; return incomingText; } // 丢弃旧录音周期或编辑时间屏障之前的消息。这是暂停/恢复竞态的主要保护。 isStaleRealtimeMessage(messageTimestampMs, messageEpoch) { if (messageEpoch !== this.liveTranscriptionEpoch) { return true; } return ( Number.isFinite(messageTimestampMs) && this.realtimeTimestampBarrierMs >= 0 && messageTimestampMs <= this.realtimeTimestampBarrierMs ); } // pending 句子会先用 raw/corrected 文本显示在预览区;只有 LLM 矫正完成并按队首 // 顺序 flush 后,才会进入正式 segmentList。 // pendingSentenceQueue 已按 timestampMs 有序插入,预览自然按说话顺序显示 getPendingSentencePreviewText() { return this.pendingSentenceQueue .filter((item) => item.epoch === this.liveTranscriptionEpoch) .map((item) => item.correctedText || item.rawText) .join(""); } // 预览文本 = 待矫正 offline 文本 + online/offline 交接文本 + 当前 online 临时文本。 syncRealtimePreview() { // 【兜底去重】无论前面逻辑是否产生重复标点,最终渲染前统一清理 const previewText = this._deduplicateConsecutivePunctuation( this.getPendingSentencePreviewText() + this.offlineSentenceHandoffText + this.recTextOnline ); this.onPreviewTextChange(previewText); } // 幂等查找:同一 source/timestamp 的重复 websocket 回包不能触发第二次矫正, // 也不能重复落到页面。 findPendingSentenceBySource(sourceText, pendingTimestamp, messageEpoch) { const normalizedSourceText = sourceText || ""; const normalizedTimestamp = pendingTimestamp || ""; return ( this.pendingSentenceQueue.find( (item) => item.epoch === messageEpoch && item.sourceText === normalizedSourceText && item.timestamp === normalizedTimestamp ) || this.segmentList.find( (item) => item.epoch === messageEpoch && item.sourceText === normalizedSourceText && item.timestamp === normalizedTimestamp ) || null ); } // 2pass-offline 文本进入 LLM 矫正前的唯一入口。这里负责旧消息过滤、 // 重复回包去重、编辑重叠裁剪,以及创建 pending 队列项。 ensurePendingSentence(sourceText, pendingTimestamp, messageEpoch) { const timestampMs = this.extractTimestampMs(pendingTimestamp); if (this.isStaleRealtimeMessage(timestampMs, messageEpoch)) { return null; } // 去重 const existingItem = this.findPendingSentenceBySource( sourceText, pendingTimestamp, messageEpoch ); if (existingItem) { return existingItem; } const trimmedText = this.trimOfflineTextWithCarryover(sourceText || ""); if (!trimmedText) { this.syncRealtimePreview(); return null; } // pendingItem 是 LLM 矫正队列的最小单元。它会一直 pending 到 // /api/asr/correct 返回,再由 flushProcessedPendingSentences 按顺序正式落句。 const pendingItem = { id: this.pendingSentenceSequence++, sourceText: sourceText || "", rawText: trimmedText, timestamp: pendingTimestamp || "", timestampMs, correctedText: "", processed: false, correctionRequested: false, epoch: messageEpoch, }; // 按 timestampMs 顺序插入,保持队列按音频时间有序 // 这样 flush 时按时间顺序处理,预览也按说话顺序显示 const insertIndex = this.pendingSentenceQueue.findIndex( (item) => item.timestampMs > timestampMs ); if (insertIndex === -1) { this.pendingSentenceQueue.push(pendingItem); } else { this.pendingSentenceQueue.splice(insertIndex, 0, pendingItem); } this.syncRealtimePreview(); return pendingItem; } // 即使 LLM 请求乱序返回,也必须保持句子顺序:只有队首已 processed 的项 // 才能进入 segmentList。 // pendingSentenceQueue 已按 timestampMs 有序插入,segmentList 也按 timestampMs 有序插入 flushProcessedPendingSentences() { let didFlush = false; while (this.pendingSentenceQueue.length > 0) { if (this.pendingSentenceQueue[0].epoch !== this.liveTranscriptionEpoch) { this.pendingSentenceQueue.shift(); continue; } if (!this.pendingSentenceQueue[0].processed) { break; } const item = this.pendingSentenceQueue.shift(); let correctedText = item.correctedText || item.rawText || ""; // 构造 segment 项 const segment = { text: correctedText, sourceText: item.sourceText, timestamp: item.timestamp, timestampMs: item.timestampMs, speaker: null, processed: true, epoch: item.epoch, }; // 按 timestampMs 有序插入 segmentList,确保最终渲染顺序与音频时间一致 const insertIndex = this.segmentList.findIndex( (seg) => seg.timestampMs > segment.timestampMs ); if (insertIndex === -1) { this.segmentList.push(segment); } else { this.segmentList.splice(insertIndex, 0, segment); } // 相邻句子首尾标点去重: // 场景:前一句经 /api/asr/correct 添加句号后, // WebSocket 返回的下一句开头又带句号(模型补全),导致双句号 // 规则:前一句末尾有标点 + 当前句开头也有标点 → 移除当前句开头标点 // 注意:这里基于插入后的物理位置检查相邻关系 const insertedIdx = this.segmentList.indexOf(segment); if (insertedIdx > 0) { const prevItem = this.segmentList[insertedIdx - 1]; const prevText = prevItem.text || ""; const prevEndingPunct = prevText.match(/[。!?.!?]+$/)?.[0]; const currentStartingPunct = segment.text.match(/^[,。!?、;:,.!?;:]+/)?.[0]; if (prevEndingPunct && currentStartingPunct) { // 移除当前句开头的标点,避免与前一句末尾标点重复 segment.text = segment.text.replace(/^[,。!?、;:,.!?;:]+/, ""); } } didFlush = true; } if (didFlush) { this.renderFullText(); } else { this.syncRealtimePreview(); } } // 将一条 pending 句子标记为矫正完成或 raw fallback,然后尝试 flush 队列。 finalizePendingSentence(pendingSentenceId, finalText, messageEpoch) { const target = this.pendingSentenceQueue.find( (item) => item.id === pendingSentenceId && item.epoch === messageEpoch ); if (!target) { return; } let correctedText = finalText || target.rawText || ""; // 去除矫正结果末尾的连续重复标点,以 /api/asr/correct 返回结果为准 // 示例:"你好。。" → "你好。" ; "你好。!" → "你好!" correctedText = this._deduplicateEndingPunctuation(correctedText); target.correctedText = correctedText; target.processed = true; target.correctionRequested = false; this.flushProcessedPendingSentences(); } // 对单条 pending 句子发起 LLM 矫正请求,完成后 finalize。 // 调用方需先设置 correctionRequested=true 并增加 pendingCorrectionCount。 async _requestCorrection(pendingSentence) { const messageEpoch = pendingSentence.epoch; try { const response = await this.fetchImpl(this.correctUrl, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text: pendingSentence.rawText, scene: this.getScene(), }), }); if (!response.ok) { throw new Error("ASR correction failed"); } const data = await response.json(); const correctedText = data.corrected_text || pendingSentence.rawText; this.finalizePendingSentence( pendingSentence.id, correctedText, messageEpoch ); } catch (error) { this.finalizePendingSentence( pendingSentence.id, pendingSentence.rawText, messageEpoch ); console.error("ASR correction failed:", error); } finally { this.setPendingCorrectionCount(this.pendingCorrectionCount - 1); } } // 暂停录音时调用:把所有未矫正的预览文本立即拿去矫正, // 等待矫正完成后加入 segmentList,确保历史转写结构完整。 async finalizeRemainingSentences(timeoutMs = 8000) { // 1. 对所有尚未请求矫正的 pending 句子立即发起矫正 const unrequested = this.pendingSentenceQueue.filter( (item) => item.epoch === this.liveTranscriptionEpoch && !item.correctionRequested && !item.processed ); for (const item of unrequested) { item.correctionRequested = true; this.setPendingCorrectionCount(this.pendingCorrectionCount + 1); // 不 await,让所有矫正并行进行 this._requestCorrection(item); } // 2. 如果还有尚未形成 offline 句子的 online 预览文本,作为 raw 句子追加 const remainingOnlineText = (this.recTextOnline || "") + (this.offlineSentenceHandoffText || ""); if (remainingOnlineText.trim()) { const messageEpoch = this.liveTranscriptionEpoch; // timestampMs 用一个大值,确保插入到 segmentList 末尾而不是开头 const fallbackTimestampMs = this.getLatestRealtimeTimestamp() || Date.now(); const pendingItem = { id: this.pendingSentenceSequence++, sourceText: remainingOnlineText, rawText: remainingOnlineText, timestamp: "", timestampMs: fallbackTimestampMs, correctedText: "", processed: false, correctionRequested: true, epoch: messageEpoch, }; this.pendingSentenceQueue.push(pendingItem); this.setPendingCorrectionCount(this.pendingCorrectionCount + 1); this._requestCorrection(pendingItem); } // 3. 等待所有矫正完成 await this.waitForCorrections(timeoutMs); // 4. 对超时仍未完成的句子,用 raw 文本强制 finalize const stuck = this.pendingSentenceQueue.filter( (item) => item.epoch === this.liveTranscriptionEpoch && !item.processed ); for (const item of stuck) { this.finalizePendingSentence(item.id, item.rawText, item.epoch); this.setPendingCorrectionCount(Math.max(0, this.pendingCorrectionCount - 1)); } // 5. 显式 flush 残留的已处理句子,确保全部进入 segmentList this.flushProcessedPendingSentences(); // 6. 清空残余的 online 预览缓冲,并最终渲染一次确保显示框更新 this.recTextOnline = ""; this.offlineSentenceHandoffText = ""; this.renderFullText(); this.syncRealtimePreview(); } // 去除矫正文本末尾的连续重复标点符号 // 原则:以 /api/asr/correct 返回结果为准,保留其最后一个标点 // 无论符号是否相同,只要末尾有连续多个标点,就缩减为最后一个 // 示例:corrected="你好。。" → "你好。" // 示例:corrected="你好。!" → "你好!" (保留最后一个) _deduplicateEndingPunctuation(correctedText) { if (!correctedText) { return correctedText; } // 匹配末尾连续标点(中英文标点) const punctPattern = /[。!?.!?]+$/; const correctedPunct = correctedText.match(punctPattern)?.[0] || ""; // 如果末尾有2个或以上连续标点,只保留最后一个(以矫正结果为准) if (correctedPunct.length >= 2) { const lastPunct = correctedPunct.slice(-1); return correctedText.replace(punctPattern, lastPunct); } return correctedText; } // 【兜底方案 A】去除文本中任意位置出现的连续标点(保留第一个) // 用于捕捉前面逻辑(换行加标点、相邻句首尾标点等)漏掉的重复标点问题 _deduplicateConsecutivePunctuation(text) { if (!text) return text; // 第一步:处理跨换行的连续标点(如 "?\n。" → "?\n"、"。\n。" → "。\n") // 任意标点 + 可选空白 + 换行 + 可选空白 + 任意标点 → 只保留第一个标点和换行 // 注意:不用 \1 反向引用,因为需要匹配不同标点相邻的情况(如 ?。、,。) let result = text.replace(/([。!?.!?,、;:,;:])\s*\n\s*[。!?.!?,、;:,;:]/g, "$1\n"); // 第二步:处理同行内的连续标点(如 "?。" → "?"、"。。" → "。") result = result.replace(/([。!?.!?,、;:,;:])(?:[。!?.!?,、;:,;:])+/g, "$1"); return result; } // 根据字符级 timestamp 重建更适合阅读的断句。长停顿或配置的结束短语 // 可以触发换行,但不修改 ASR 原文内容。 // prevChar: 上一个 segment 返回文本的末尾字符,用于跨 segment 调用时 // textWithTime 为空的情况下正确判断是否需要补句号。 handleWithTimestamp(text, timestamp, keywordStr, timeThreshold, prevChar = "") { if (!timestamp || text.length <= 0) { return text; } const keywordList = (keywordStr || "") .split(",") .map((keyword) => keyword.trim()) .filter((keyword) => keyword.length > 0); const keywordsAsRegex = keywordList.map((keyword) => new RegExp(keyword)); const segments = text.match(/[^。!?.!?]+[。!?.!?]?/g) || []; const timeData = JSON.parse(timestamp); let charIndex = 0; let textWithTime = ""; const removeLeadingPunctuation = (value) => value.replace(/^[,。!?、;:,\.!?;:]+/, ""); for (let i = 0; i < segments.length; i++) { const segment = segments[i]; if (!segment || segment === "undefined") { continue; } const nowTime = timeData[charIndex][0] / 1000; const previousTime = this.getLastTimer(); if (previousTime > 0 && nowTime - previousTime > timeThreshold) { // textWithTime 为空时(跨 segment 首次进入),fallback 到上一个 // segment 的末尾字符,避免对已有标点的行尾再补句号 const lastChar = textWithTime.slice(-1) || prevChar; const endsWithPunc = /[。!?.!?,、;:,;:]/.test(lastChar); const processedSegment = removeLeadingPunctuation(segment); const segmentStartsWithPunc = /^[,。!?、;:,.!?;:]+/.test(processedSegment); textWithTime += (endsWithPunc || segmentStartsWithPunc ? "\n" : "。\n") + processedSegment; } else { const hitKeyword = keywordsAsRegex.some((regex) => regex.test(segment)); if (!hitKeyword) { textWithTime += segment; } } this.setLastTimer(nowTime); charIndex++; } return textWithTime; } getSpeakerBeforeIndex(endIndex) { let previousSpeaker = null; for (let i = 0; i < endIndex; i++) { if (this.segmentList[i] && this.segmentList[i].speaker) { previousSpeaker = this.segmentList[i].speaker; } } return previousSpeaker; } // 将已提交 segment 转成页面文本。只有说话人变化时才插入 speaker header, // 避免每句话都重复显示说话人。 buildSegmentText(segments, initialSpeaker = null) { let displayText = ""; let lastRenderedSpeaker = initialSpeaker; let lastContentChar = ""; segments.forEach((segment) => { if (this.getShowSpeaker()) { if ( segment.speaker && segment.speaker !== lastRenderedSpeaker && !segment.speaker.includes("SPK") ) { displayText += (displayText ? "\n" : "") + segment.speaker + ":\n"; lastRenderedSpeaker = segment.speaker; } else if ( segment.speaker && segment.speaker !== lastRenderedSpeaker ) { displayText += (displayText ? "\n" : "") + segment.speaker + ":\n"; lastRenderedSpeaker = segment.speaker; } } const segText = this.handleWithTimestamp( segment.text, segment.timestamp, this.getConfigWords(), 6, lastContentChar ); displayText += segText; // 跟踪上一个 segment 的末尾非空白字符,传给下一次 handleWithTimestamp const trimmed = segText.replace(/\s+$/, ""); if (trimmed.length > 0) { lastContentChar = trimmed.slice(-1); } }); return displayText; } // 将已矫正并提交的 segment 渲染到主转录区。pending/online 预览文本保持独立, // 主文本更新后再同步预览区。 renderFullText() { const initialSpeaker = this.segmentRenderStartIndex > 0 ? this.getSpeakerBeforeIndex(this.segmentRenderStartIndex) : null; const displayText = this.buildSegmentText( this.segmentList.slice(this.segmentRenderStartIndex), initialSpeaker ); this.recText = displayText; // 【兜底去重】清理 pausedTranscription + recText 中可能产生的连续重复标点 const mainText = this._deduplicateConsecutivePunctuation( this.getPausedTranscription() + this.recText ); this.onMainTextChange(mainText); this.syncRealtimePreview(); } // online websocket 返回低延迟片段,只更新预览区;正式 offline 句子统一交给 // handleOfflineMessage,避免双通路重复落句。 handleOnlineMessage(event) { if (this.disposed) { return; } const parsedMessage = JSON.parse(event.data); const text = "" + parsedMessage.text; const mode = parsedMessage.mode; const timestamp = parsedMessage.timestamp; const messageEpoch = this.liveTranscriptionEpoch; if (mode === "2pass-online") { // 低延迟预览文本可能不完整,因此不进入正式 segmentList。 const currentTimestamp = this.extractTimestampMs(timestamp); if (this.isStaleRealtimeMessage(currentTimestamp, messageEpoch)) { this.syncRealtimePreview(); return; } let shouldBreak = false; if (this.lastOnlineTimestamp !== null && currentTimestamp !== null) { const timeDiff = (currentTimestamp - this.lastOnlineTimestamp) / 1000; if (timeDiff >= 3 && /。\s*$/.test(text)) { shouldBreak = true; } } if (parsedMessage.is_full_text) { this.recTextOnline = shouldBreak ? text + "\n" : text; } else { this.recTextOnline += shouldBreak ? text + "\n" : text; } this.lastOnlineTimestamp = currentTimestamp; } else if (mode === "2pass-offline") { // online 通道也可能回显 offline 句子,这里只作为交接预览保存。 // 正式句子只通过 offline 通道进入矫正流程。 this.recTextOnline = ""; this.offlineSentenceHandoffText = text || this.recTextOnline || ""; this.recTextOnline = ""; this.lastOnlineTimestamp = null; } this.syncRealtimePreview(); } // offline websocket 才负责正式转录路径:2pass-offline 文本进入 LLM 矫正, // offline-speaker 后续只补说话人信息。 async handleOfflineMessage(event) { if (this.disposed) { return; } const parsedMessage = JSON.parse(event.data); const mode = parsedMessage.mode; const timestamp = parsedMessage.timestamp; const messageEpoch = this.liveTranscriptionEpoch; const messageTimestampMs = this.extractTimestampMs(timestamp); if (mode === "2pass-offline") { // 文本先到:先放入 pending 队列刷新预览,再调用 LLM 矫正接口, // 矫正完成后才进入正式 segmentList。 if (this.isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) { return; } const text = "" + parsedMessage.text; this.recTextOnline = ""; this.lastOnlineTimestamp = null; const pendingSentence = this.ensurePendingSentence( text, timestamp, messageEpoch ); this.offlineSentenceHandoffText = ""; this.syncRealtimePreview(); if ( !pendingSentence || pendingSentence.processed || pendingSentence.correctionRequested ) { return; } pendingSentence.correctionRequested = true; this.setPendingCorrectionCount(this.pendingCorrectionCount + 1); await this._requestCorrection(pendingSentence); } else if (mode === "offline-speaker") { // 说话人标签可能晚于矫正文本到达。这里只更新 speaker 元数据, // 绝不替换 text,避免已矫正句子跳回 raw ASR 文本。 if (this.isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) { return; } const speaker = (parsedMessage.segments && parsedMessage.segments[0] && parsedMessage.segments[0].speaker) || "未命名演讲者"; let targetIndex = -1; for (let i = this.segmentList.length - 1; i >= 0; i--) { const segment = this.segmentList[i]; if (!segment || segment.epoch !== messageEpoch) { continue; } if ( segment.speaker === null && Number.isFinite(messageTimestampMs) && Number.isFinite(segment.timestampMs) && segment.timestampMs === messageTimestampMs ) { targetIndex = i; break; } if (targetIndex === -1 && segment.speaker === null) { targetIndex = i; } } if (targetIndex !== -1) { this.segmentList[targetIndex].speaker = speaker; this.renderFullText(); } } } }