1321 lines
49 KiB
JavaScript
1321 lines
49 KiB
JavaScript
// =============================================================================
|
||
// 文件来源：从 src/main-old.js 拆分而来（实时 ASR 链路独立成类）
|
||
// 与原代码（main-old.js）相比的【行为差异】，需在迁移/回归时特别注意：
|
||
//
|
||
// 1. WebSocket state===2（断连）处理
|
||
// - 原代码：getConnState/getConnState2 在 state===2 时调用 stop()，但
|
||
// main-old.js 里并没有定义 stop 函数，触发时会抛 ReferenceError（原 bug）。
|
||
// - 新代码：handleOnlineState(2)/handleOfflineState(2) 调用 this.stop()，
|
||
// 真正关闭 ws + Recorder + AudioContext，但【不会】停止 MediaRecorder
|
||
// 或把 isRecording 置 false（pipeline 只管 ASR 通道）。
|
||
// - 含义：ws 断连不再报错，但整体录音会话不会因此停止。若产品希望 ws
|
||
// 断连时也停掉 MediaRecorder，需在 main.js 的 onActiveChange(false)
|
||
// 或新增 onDisconnect 回调里做额外处理（当前未做）。
|
||
//
|
||
// 2. stopRecording 会立即关闭 ASR Recorder
|
||
// - 原代码：stopRecording 只调 webReader.wsFinish/wsStop，rec/rec2 直到
|
||
// 切走 recording view 才 close。
|
||
// - 新代码：pipeline.finish() 会 closeRecorders()。恢复录音时
|
||
// ensureRecorders() 会重建，逻辑自洽。若存在“停止后不切 view 直接再录”
|
||
// 的路径需确认。
|
||
//
|
||
// 3. correct 接口新增 scene 字段
|
||
// - 原代码：/api/asr/correct body 只传 { text }。
|
||
// - 新代码：handleOfflineMessage 调 correct 时多传 { text, scene }。
|
||
// - 前置条件：后端 /tool/speakr/api/asr/correct 必须容忍/使用 scene 字段。
|
||
//
|
||
// 4. resumeRecording 不再设置 resetTranscriptionFlag
|
||
// - 原代码：resume 时设 resetTranscriptionFlag=true，等下一条 ws 消息进来
|
||
// 时 getJsonMessage 再 clearRealtimePipeline()。
|
||
// - 新代码：main.js 的 resumeRecording 开头直接 resetRealtimeTranscript()，
|
||
// 立刻清。结果等价，时序不同（更干净）。
|
||
//
|
||
// 5. continueDraftRecording 现在会调 resetRealtimeTranscript
|
||
// - 原代码：加载草稿只重置 segmentRenderStartIndex=0。
|
||
// - 新代码：额外清空 pendingSentenceQueue 并 startNewEpoch。更彻底，
|
||
// 基本是改进，但若依赖 pending 队列残留需注意。
|
||
//
|
||
// 6. wsconnecter 现在收到 getScene
|
||
// - 原代码：new WebSocketConnectMethod({msgHandle, stateHandle}) 不传
|
||
// getScene，wsconnecter 走默认 "cg"。
|
||
// - 新代码：createSockets() 传 getScene: () => this.getScene()，按
|
||
// selectedScene 路由到不同后端 ASR URL。新增场景切换能力。
|
||
//
|
||
// 7. 性能优化（行为等价）
|
||
// - appendInt16 替代 Int16Array.from([...sampleBuf, ...data_16k])，避免
|
||
// 高频 onProcess 里用展开运算符复制大数组。结果等价。
|
||
//
|
||
// 8. 新增 dispose() 屏障
|
||
// - 原代码：离开 recording view 后，迟到的 ws/LLM 回调仍可能写入页面状态。
|
||
// - 新代码：dispose() 设 disposed=true，handleOnlineMessage/handleOfflineMessage
|
||
// /processRecorderFrame 入口都检查 disposed，迟到回调被忽略。
|
||
// =============================================================================
|
||
|
||
import { Recorder } from "../vendor-globals.js";
|
||
import { WebSocketConnectMethod } from "./wsconnecter.js";
|
||
|
||
// FunASR 实时 websocket 需要持续收到短 PCM 帧。
|
||
// 16 kHz 下 960 个 sample 约等于 60ms 音频，沿用原链路的分片粒度。
|
||
const DEFAULT_CHUNK_SIZE = 960;
|
||
// 软告警：只提示 websocket 发送速度已经落后于采集速度。
|
||
const DEFAULT_BUFFER_WARN_SAMPLES = 20000;
|
||
// 硬上限：最多保留最近 5 秒音频，丢弃更旧数据，避免内存持续增长。
|
||
const DEFAULT_BUFFER_HARD_LIMIT_SAMPLES = 16000 * 5;
|
||
|
||
function noop() {}
|
||
|
||
// Recorder.SampleData 返回 Int16Array；这里显式拼接，避免在高频 onProcess
|
||
// 回调里用展开运算符复制大数组。
|
||
// 【行为差异 #7】替代原 main-old.js 的 Int16Array.from([...sampleBuf, ...data_16k])，
|
||
// 行为等价，性能更好。
|
||
function appendInt16(left, right) {
|
||
const merged = new Int16Array(left.length + right.length);
|
||
merged.set(left, 0);
|
||
merged.set(right, left.length);
|
||
return merged;
|
||
}
|
||
|
||
function createAudioContext() {
|
||
const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
|
||
return new AudioContextCtor();
|
||
}
|
||
|
||
/**
|
||
* 浏览器实时 ASR 完整管线：
|
||
* Recorder -> websocket -> 回包解析 -> LLM 矫正 -> 页面文本回调。
|
||
*
|
||
* Vue 仍然负责页面 ref、媒体流和录音控制；本类只通过 getter/callback
|
||
* 读写这些依赖，因此 ASR 链路可以从 main.js 中独立出来，而不引入 Vue。
|
||
*/
|
||
export class RealtimeAsrPipeline {
|
||
constructor({
|
||
getMicStream,
|
||
getSystemStream,
|
||
getPausedTranscription,
|
||
setPausedTranscription,
|
||
getShowSpeaker,
|
||
getConfigWords,
|
||
getScene = () => "cg",
|
||
getLastTimer,
|
||
setLastTimer,
|
||
onMainTextChange = noop,
|
||
onPreviewTextChange = noop,
|
||
onOfflineTextChange = noop,
|
||
onActiveChange = noop,
|
||
onPendingCorrectionCountChange = noop,
|
||
onError = noop,
|
||
fetchImpl = window.fetch.bind(window),
|
||
correctUrl = "/tool/speakr/api/asr/correct",
|
||
sourceKind = "mixed",
|
||
}) {
|
||
// 页面拥有的依赖。管线通过 getter 读取媒体流，通过 callback 回写显示文本，
|
||
// 不直接触碰 Vue ref。
|
||
this.getMicStream = getMicStream;
|
||
this.getSystemStream = getSystemStream;
|
||
this.getPausedTranscription = getPausedTranscription;
|
||
this.setPausedTranscription = setPausedTranscription;
|
||
this.getShowSpeaker = getShowSpeaker;
|
||
this.getConfigWords = getConfigWords;
|
||
this.getScene = getScene;
|
||
this.getLastTimer = getLastTimer;
|
||
this.setLastTimer = setLastTimer;
|
||
this.onMainTextChange = onMainTextChange;
|
||
this.onPreviewTextChange = onPreviewTextChange;
|
||
this.onOfflineTextChange = onOfflineTextChange;
|
||
this.onActiveChange = onActiveChange;
|
||
this.onPendingCorrectionCountChange = onPendingCorrectionCountChange;
|
||
this.onError = onError;
|
||
this.fetchImpl = fetchImpl;
|
||
this.correctUrl = correctUrl;
|
||
this.sourceKind = sourceKind;
|
||
|
||
// 运行期资源。online websocket 负责低延迟预览；offline websocket 负责
|
||
// 正式句子、说话人回填和 LLM 矫正。
|
||
this.wsOnline = null;
|
||
this.wsOffline = null;
|
||
this.recOnline = null;
|
||
this.recOffline = null;
|
||
this.audioContextOnline = null;
|
||
this.audioContextOffline = null;
|
||
|
||
// PCM 发送缓冲、pending 队列和 segmentList 都是管线内部状态。
|
||
// segmentList 保存已按顺序矫正完成的正式转录片段。
|
||
this.sampleBufOnline = new Int16Array();
|
||
this.sampleBufOffline = new Int16Array();
|
||
this.segmentList = [];
|
||
this.segmentRenderStartIndex = 0;
|
||
this.pendingSentenceQueue = [];
|
||
this.pendingSentenceSequence = 0;
|
||
this.pendingCorrectionCount = 0;
|
||
|
||
// recText 是已正式落到页面的文本；recTextOnline 和 handoff 文本是临时预览，
|
||
// 用于 offline/LLM 矫正返回前让用户先看到内容。
|
||
this.recText = "";
|
||
this.recTextOnline = "";
|
||
this.offlineSentenceHandoffText = "";
|
||
this.lastOnlineTimestamp = null;
|
||
this.liveTranscriptionEpoch = 0;
|
||
this.realtimeTimestampBarrierMs = -1;
|
||
this.realtimeEditCarryover = null;
|
||
this.editModeBufferStartIndex = 0;
|
||
this.active = false;
|
||
this.disposed = false;
|
||
|
||
this.createSockets();
|
||
}
|
||
|
||
// 预先创建两条 websocket 客户端。传输对象负责排队和发送；本管线负责解释回包。
|
||
// 【行为差异 #6】原 main-old.js 创建 WebSocketConnectMethod 时不传 getScene，
|
||
// wsconnecter 走默认 "cg"；这里传 getScene: () => this.getScene()，按
|
||
// selectedScene 路由到不同后端 ASR URL，支持场景切换。
|
||
createSockets() {
|
||
this.wsOnline = new WebSocketConnectMethod({
|
||
msgHandle: (event) => this.handleOnlineMessage(event),
|
||
stateHandle: (state) => this.handleOnlineState(state),
|
||
getScene: () => this.getScene(),
|
||
});
|
||
this.wsOffline = new WebSocketConnectMethod({
|
||
msgHandle: (event) => this.handleOfflineMessage(event),
|
||
stateHandle: (state) => this.handleOfflineState(state),
|
||
getScene: () => this.getScene(),
|
||
});
|
||
}
|
||
|
||
// 启动/重启实时 ASR。Recorder 不在此处创建——它需要 sourceStream，
|
||
// 而音频流要在 openMixedStream 时才能确定。ws open 后会回调
|
||
// handleOnlineState/handleOfflineState → openMixedStream 动态创建 Recorder。
|
||
start() {
|
||
this.disposed = false;
|
||
|
||
const onlineStarted = this.wsOnline?.wsStart() === 1;
|
||
const offlineStarted = this.wsOffline?.wsStart("offline") === 1;
|
||
this.setActive(onlineStarted || offlineStarted);
|
||
return onlineStarted || offlineStarted ? 1 : 0;
|
||
}
|
||
|
||
// 立即停止：用于暂停或离开录音视图。关闭 socket 和 Recorder 回调，
|
||
// 避免旧 onProcess 帧继续向已关闭 websocket 发送数据。
|
||
stop() {
|
||
this.setActive(false);
|
||
this.wsOnline?.wsStop();
|
||
this.wsOffline?.wsStop();
|
||
this.closeRecorders();
|
||
}
|
||
|
||
// 优雅停止：用于用户结束录音。wsFinish 发送 stop 帧并短暂等待后端
|
||
// flush 最后的 offline/LLM 结果。
|
||
// 【行为差异 #2】原 main-old.js 的 stopRecording 只调 webReader.wsFinish/wsStop，
|
||
// rec/rec2 直到切走 recording view 才 close；这里 finish() 会立即
|
||
// closeRecorders()。恢复录音时 ensureRecorders() 会重建，逻辑自洽。
|
||
async finish(timeoutMs = 3000) {
|
||
this.setActive(false);
|
||
this.closeRecorders();
|
||
|
||
const finishPromises = [];
|
||
if (this.wsOnline && typeof this.wsOnline.wsFinish === "function") {
|
||
finishPromises.push(this.wsOnline.wsFinish(timeoutMs));
|
||
} else {
|
||
this.wsOnline?.wsStop();
|
||
}
|
||
|
||
if (this.wsOffline && typeof this.wsOffline.wsFinish === "function") {
|
||
finishPromises.push(this.wsOffline.wsFinish(timeoutMs));
|
||
} else {
|
||
this.wsOffline?.wsStop();
|
||
}
|
||
|
||
if (finishPromises.length > 0) {
|
||
await Promise.allSettled(finishPromises);
|
||
}
|
||
}
|
||
|
||
// dispose 比 stop 更彻底：离开录音视图后，迟到的 websocket/矫正回调
|
||
// 应被忽略，不能再写入新的页面状态。
|
||
// 【行为差异 #8】新增 disposed 屏障。原 main-old.js 离开 view 后迟到的
|
||
// ws/LLM 回调仍可能写入页面状态；这里 handleOnlineMessage/handleOfflineMessage
|
||
// /processRecorderFrame 入口都检查 this.disposed，迟到回调被忽略。
|
||
dispose() {
|
||
this.disposed = true;
|
||
this.stop();
|
||
this.wsOnline = null;
|
||
this.wsOffline = null;
|
||
}
|
||
|
||
isActive() {
|
||
return this.active;
|
||
}
|
||
|
||
setActive(active) {
|
||
this.active = active;
|
||
this.onActiveChange(active);
|
||
}
|
||
|
||
setPendingCorrectionCount(count) {
|
||
this.pendingCorrectionCount = Math.max(0, count);
|
||
this.onPendingCorrectionCountChange(this.pendingCorrectionCount);
|
||
}
|
||
|
||
getPendingCorrectionCount() {
|
||
return this.pendingCorrectionCount;
|
||
}
|
||
|
||
// 暂停/保存草稿前要等待正在进行的 LLM 矫正；否则草稿可能保存 raw offline 文本，
|
||
// 而正确句子马上就要返回。
|
||
async waitForCorrections(timeoutMs = 8000, pollMs = 200) {
|
||
const waitStart = Date.now();
|
||
while (
|
||
this.pendingCorrectionCount > 0 &&
|
||
Date.now() - waitStart < timeoutMs
|
||
) {
|
||
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
||
}
|
||
return this.pendingCorrectionCount;
|
||
}
|
||
|
||
// epoch 是旧消息屏障。录音周期变化后，旧 websocket/LLM 异步结果都会被忽略。
|
||
startNewEpoch({
|
||
barrierTimestampMs = null,
|
||
clearTimestampBarrier = false,
|
||
clearEditCarryover = true,
|
||
} = {}) {
|
||
this.liveTranscriptionEpoch += 1;
|
||
|
||
if (clearEditCarryover) {
|
||
this.realtimeEditCarryover = null;
|
||
}
|
||
|
||
if (clearTimestampBarrier) {
|
||
this.realtimeTimestampBarrierMs = -1;
|
||
} else if (Number.isFinite(barrierTimestampMs)) {
|
||
this.realtimeTimestampBarrierMs = Math.max(
|
||
this.realtimeTimestampBarrierMs,
|
||
barrierTimestampMs
|
||
);
|
||
}
|
||
|
||
this.clearRealtimePipeline({
|
||
preserveTimestampBarrier: !clearTimestampBarrier,
|
||
});
|
||
}
|
||
|
||
// 只清理实时转写内部状态，不清空页面历史文本。草稿恢复依赖
|
||
// pausedTranscription 保持可见，新片段继续追加在它后面。
|
||
resetTranscript({ clearTimestampBarrier = false } = {}) {
|
||
this.segmentList = [];
|
||
this.segmentRenderStartIndex = 0;
|
||
this.setPendingCorrectionCount(0);
|
||
this.startNewEpoch({ clearTimestampBarrier });
|
||
}
|
||
|
||
// 准备恢复录音：保留历史转写数据，但清空内部缓冲区准备接收新数据
|
||
// pausedTranscription 保持可见，新片段从 segmentList 末尾继续追加
|
||
prepareForResume() {
|
||
// 保留 segmentList（包含说话人、时间戳等元数据），只调整渲染起点
|
||
this.segmentRenderStartIndex = this.segmentList.length;
|
||
this.editModeBufferStartIndex = this.segmentList.length;
|
||
|
||
// 清空内部缓冲区，准备接收新的实时数据
|
||
this.sampleBufOnline = new Int16Array();
|
||
this.sampleBufOffline = new Int16Array();
|
||
this.recText = "";
|
||
this.recTextOnline = "";
|
||
this.offlineSentenceHandoffText = "";
|
||
this.pendingSentenceQueue = [];
|
||
this.pendingSentenceSequence = 0;
|
||
this.lastOnlineTimestamp = null;
|
||
|
||
// 清空预览文本
|
||
this.onPreviewTextChange("");
|
||
this.onOfflineTextChange("");
|
||
|
||
// 增加 epoch 以过滤掉旧消息，但保留时间戳屏障
|
||
this.liveTranscriptionEpoch += 1;
|
||
|
||
// 重新渲染，确保显示完整的历史文本
|
||
this.renderFullText();
|
||
this.syncRealtimePreview();
|
||
}
|
||
|
||
// 手动编辑后，以当前页面文本作为新的基准。记录当前 segment 边界，
|
||
// 后续 ASR 结果只追加到编辑后的文本之后。
|
||
enterEditMode(baseText) {
|
||
this.editModeBufferStartIndex = this.segmentList.length;
|
||
const barrierTimestampMs = this.getLatestRealtimeTimestamp();
|
||
this.startNewEpoch({
|
||
barrierTimestampMs,
|
||
clearEditCarryover: false,
|
||
});
|
||
this.realtimeEditCarryover = this.buildRealtimeEditCarryover(baseText);
|
||
}
|
||
|
||
// 立即应用用户编辑，并让后续矫正片段从保存的编辑边界后继续渲染，
|
||
// 防止旧 ASR 文本覆盖用户手改内容。
|
||
applyEditedText(nextText) {
|
||
this.setPausedTranscription(nextText);
|
||
this.segmentRenderStartIndex = this.editModeBufferStartIndex;
|
||
this.recText = "";
|
||
this.recTextOnline = "";
|
||
this.offlineSentenceHandoffText = "";
|
||
this.onPreviewTextChange("");
|
||
this.onOfflineTextChange("");
|
||
this.onMainTextChange(nextText);
|
||
this.renderFullText();
|
||
}
|
||
|
||
// 这里的 Recorder 只负责实时 PCM 推流；它和页面里用于保存/上传 webm
|
||
// 成品音频的 MediaRecorder 是两套东西。
|
||
// Recorder 不再提前创建。sourceStream 必须在构造时传入，而音频流在
|
||
// openMixedStream 时才确定，因此 Recorder 实例改在 openMixedStream 中创建。
|
||
// 此方法保留为空操作，避免外部调用方报错。
|
||
ensureRecorders() {
|
||
// no-op: Recorder 实例现在在 openMixedStream 中按需创建
|
||
}
|
||
|
||
closeRecorders() {
|
||
if (this.recOnline) {
|
||
try {
|
||
this.recOnline.close();
|
||
} catch (error) {
|
||
// 关闭失败不影响整体清理，继续释放后续资源。
|
||
console.warn("Failed to close online Recorder:", error);
|
||
}
|
||
this.recOnline = null;
|
||
}
|
||
|
||
if (this.recOffline) {
|
||
try {
|
||
this.recOffline.close();
|
||
} catch (error) {
|
||
console.warn("Failed to close offline Recorder:", error);
|
||
}
|
||
this.recOffline = null;
|
||
}
|
||
|
||
this.closeAudioContexts();
|
||
}
|
||
|
||
closeAudioContexts() {
|
||
if (this.audioContextOnline) {
|
||
try {
|
||
this.audioContextOnline.close();
|
||
} catch (_) {}
|
||
this.audioContextOnline = null;
|
||
}
|
||
|
||
if (this.audioContextOffline) {
|
||
try {
|
||
this.audioContextOffline.close();
|
||
} catch (_) {}
|
||
this.audioContextOffline = null;
|
||
}
|
||
}
|
||
|
||
handleOnlineState(state) {
|
||
if (state === 0) {
|
||
this.openMixedStream("online");
|
||
} else if (state === 2) {
|
||
// 【行为差异 #1】原 main-old.js 在 getConnState state===2 时调用未定义的
|
||
// stop() 会抛 ReferenceError；这里改为优雅关闭 ASR 通道（ws + Recorder +
|
||
// AudioContext），但【不会】停止 MediaRecorder 或 isRecording——pipeline
|
||
// 只管 ASR 通道。若需在 ws 断连时整体停录，需在 main.js 的回调里补处理。
|
||
this.stop();
|
||
}
|
||
}
|
||
|
||
handleOfflineState(state) {
|
||
if (state === 0) {
|
||
this.openMixedStream("offline");
|
||
} else if (state === 2) {
|
||
// 【行为差异 #1】同 handleOnlineState(2)。
|
||
this.stop();
|
||
}
|
||
}
|
||
|
||
// 为指定通道构造混合 MediaStream。online/offline 监听同一组麦克风/系统音源，
|
||
// 但各自维护独立 AudioContext 和 Recorder。
|
||
async openMixedStream(channel) {
|
||
try {
|
||
const micInfo = channel === "online" ? await this.getMicrophoneInfo() : null;
|
||
if (micInfo) {
|
||
console.log("Current microphone:", micInfo.name, micInfo);
|
||
}
|
||
|
||
const micStream = this.getMicStream();
|
||
const systemStream = this.getSystemStream();
|
||
const hasMic = micStream && micStream.getAudioTracks().length > 0;
|
||
const hasSystem =
|
||
systemStream && systemStream.getAudioTracks().length > 0;
|
||
|
||
if (this.sourceKind === "mic" && !hasMic) {
|
||
throw new Error("No microphone stream is available for realtime ASR.");
|
||
}
|
||
if (this.sourceKind === "system" && !hasSystem) {
|
||
throw new Error("No system audio stream is available for realtime ASR.");
|
||
}
|
||
if (this.sourceKind === "mixed" && !hasMic && !hasSystem) {
|
||
throw new Error("No audio stream is available for realtime ASR.");
|
||
}
|
||
|
||
if (channel === "online" && this.audioContextOnline) {
|
||
try {
|
||
this.audioContextOnline.close();
|
||
} catch (_) {}
|
||
}
|
||
if (channel === "offline" && this.audioContextOffline) {
|
||
try {
|
||
this.audioContextOffline.close();
|
||
} catch (_) {}
|
||
}
|
||
|
||
const audioContext = createAudioContext();
|
||
if (channel === "online") {
|
||
this.audioContextOnline = audioContext;
|
||
} else {
|
||
this.audioContextOffline = audioContext;
|
||
}
|
||
|
||
const destination = audioContext.createMediaStreamDestination();
|
||
|
||
if ((this.sourceKind === "mixed" || this.sourceKind === "mic") && hasMic) {
|
||
audioContext.createMediaStreamSource(micStream).connect(destination);
|
||
}
|
||
|
||
if ((this.sourceKind === "mixed" || this.sourceKind === "system") && hasSystem) {
|
||
audioContext.createMediaStreamSource(systemStream).connect(destination);
|
||
} else if (this.sourceKind === "mixed" && !hasMic) {
|
||
console.log("No system audio detected for realtime ASR.");
|
||
}
|
||
|
||
// 【关键修复】Recorder 必须在构造时传入 sourceStream，否则 open() 会
|
||
// 自行调用 getUserMedia({audio:true}) 获取麦克风流，导致系统音频被忽略。
|
||
// 在此处动态创建 Recorder，将混音流作为 sourceStream 传入。
|
||
const existingRecorder = channel === "online" ? this.recOnline : this.recOffline;
|
||
if (existingRecorder) {
|
||
try { existingRecorder.close(); } catch (_) {}
|
||
}
|
||
|
||
// 创建一个实时音频采集器，用于语音识别
|
||
const recorder = Recorder({
|
||
type: "pcm", // 输出未经压缩的原始音频数据，方便后续处理
|
||
bitRate: 16,
|
||
sampleRate: 16000, // 16kHz 是语音识别模型的标准采样率
|
||
sourceStream: destination.stream, // 传入混音后的音频流（麦克风+系统音频混合），而不是让 Recorder 自己去调 getUserMedia 获取麦克风
|
||
onProcess: (...args) => this.processRecorderFrame(channel, ...args),
|
||
});
|
||
if (channel === "online") {
|
||
this.recOnline = recorder;
|
||
} else {
|
||
this.recOffline = recorder;
|
||
}
|
||
|
||
recorder.open(
|
||
() => {
|
||
recorder.start();
|
||
},
|
||
(message) => {
|
||
console.error("Recorder permission denied:", message);
|
||
this.onError(message);
|
||
}
|
||
);
|
||
} catch (error) {
|
||
console.error("Failed to initialize realtime ASR recorder:", error);
|
||
this.onError(error);
|
||
}
|
||
}
|
||
|
||
// Recorder onProcess 入口：把浏览器采样率下的新 PCM 片段重采样到 16k，
|
||
// 进入缓冲区，再切成 websocket 分片发送到对应通道。
|
||
processRecorderFrame(channel, buffer, _powerLevel, _duration, sampleRate) {
|
||
if (!this.active || this.disposed || !buffer || buffer.length === 0) {
|
||
return;
|
||
}
|
||
|
||
const sourceChunk = buffer[buffer.length - 1];
|
||
// Recorder.SampleData 需要传入片段数组，并返回 ASR 所需的 16k PCM。
|
||
const data16k = Recorder.SampleData([sourceChunk], sampleRate, 16000).data;
|
||
const isOnline = channel === "online";
|
||
const ws = isOnline ? this.wsOnline : this.wsOffline;
|
||
const label = isOnline ? "Online" : "Offline";
|
||
let sampleBuf = isOnline ? this.sampleBufOnline : this.sampleBufOffline;
|
||
|
||
sampleBuf = appendInt16(sampleBuf, data16k);
|
||
|
||
// 如果 websocket 发送跟不上采集，只保留最近语音，丢弃最旧音频。
|
||
if (sampleBuf.length > DEFAULT_BUFFER_HARD_LIMIT_SAMPLES) {
|
||
const overflowSamples =
|
||
sampleBuf.length - DEFAULT_BUFFER_HARD_LIMIT_SAMPLES;
|
||
console.warn(
|
||
`[${label}] Dropping excess realtime ASR audio buffer: ${overflowSamples} samples`
|
||
);
|
||
sampleBuf = sampleBuf.slice(-DEFAULT_BUFFER_HARD_LIMIT_SAMPLES);
|
||
}
|
||
|
||
// 发送所有完整分片；不足一个分片的尾巴继续留在通道缓冲里。
|
||
while (sampleBuf.length >= DEFAULT_CHUNK_SIZE) {
|
||
const sendBuf = sampleBuf.slice(0, DEFAULT_CHUNK_SIZE);
|
||
const sendAccepted = ws?.wsSend(sendBuf);
|
||
if (!sendAccepted) {
|
||
break;
|
||
}
|
||
sampleBuf = sampleBuf.slice(DEFAULT_CHUNK_SIZE);
|
||
}
|
||
|
||
if (sampleBuf.length > DEFAULT_BUFFER_WARN_SAMPLES) {
|
||
console.warn(
|
||
`[${label}] Realtime ASR audio buffer is high: ${sampleBuf.length} samples (${(
|
||
sampleBuf.length / 16000
|
||
).toFixed(2)}s)`
|
||
);
|
||
}
|
||
|
||
if (isOnline) {
|
||
this.sampleBufOnline = sampleBuf;
|
||
} else {
|
||
this.sampleBufOffline = sampleBuf;
|
||
}
|
||
}
|
||
|
||
async getMicrophoneInfo() {
|
||
if (
|
||
!navigator.mediaDevices ||
|
||
!navigator.mediaDevices.enumerateDevices
|
||
) {
|
||
console.warn("Browser does not support enumerating audio devices.");
|
||
return null;
|
||
}
|
||
|
||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||
const mics = devices.filter((device) => device.kind === "audioinput");
|
||
const defaultMic =
|
||
mics.find((device) => device.deviceId === "default") || mics[0];
|
||
|
||
return {
|
||
name: defaultMic?.label || "Unknown microphone",
|
||
deviceId: defaultMic?.deviceId || "",
|
||
all: mics.map((device) => ({
|
||
name: device.label,
|
||
deviceId: device.deviceId,
|
||
})),
|
||
};
|
||
}
|
||
|
||
// 清理临时缓冲和预览文本。手动编辑时可保留时间戳屏障，
|
||
// 防止编辑点之前的旧 websocket 消息重新进入。
|
||
clearRealtimePipeline({ preserveTimestampBarrier = false } = {}) {
|
||
this.sampleBufOnline = new Int16Array();
|
||
this.sampleBufOffline = new Int16Array();
|
||
this.recText = "";
|
||
this.recTextOnline = "";
|
||
this.offlineSentenceHandoffText = "";
|
||
this.pendingSentenceQueue = [];
|
||
this.pendingSentenceSequence = 0;
|
||
this.lastOnlineTimestamp = null;
|
||
this.onPreviewTextChange("");
|
||
this.onOfflineTextChange("");
|
||
|
||
if (!preserveTimestampBarrier) {
|
||
this.realtimeTimestampBarrierMs = -1;
|
||
}
|
||
|
||
this.syncRealtimePreview();
|
||
}
|
||
|
||
// FunASR timestamp 可能是 JSON 字符串，也可能已经是数组。这里取最后一段
|
||
// 结束时间，用于去重、旧消息过滤和后续说话人对齐。
|
||
extractTimestampMs(timestampPayload) {
|
||
if (!timestampPayload) {
|
||
return null;
|
||
}
|
||
|
||
try {
|
||
const timestampData =
|
||
typeof timestampPayload === "string"
|
||
? JSON.parse(timestampPayload)
|
||
: timestampPayload;
|
||
|
||
if (!Array.isArray(timestampData) || timestampData.length === 0) {
|
||
return null;
|
||
}
|
||
|
||
const lastItem = timestampData[timestampData.length - 1];
|
||
if (!Array.isArray(lastItem) || lastItem.length === 0) {
|
||
return null;
|
||
}
|
||
|
||
const endOrStart =
|
||
lastItem.length > 1 ? Number(lastItem[1]) : Number(lastItem[0]);
|
||
return Number.isFinite(endOrStart) ? endOrStart : null;
|
||
} catch (_) {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
// 手动编辑前获取最新 ASR 时间点，用它建立旧消息屏障。
|
||
getLatestRealtimeTimestamp() {
|
||
const latestSegmentTimestamp = this.segmentList.reduce(
|
||
(latest, segment) =>
|
||
Number.isFinite(segment?.timestampMs)
|
||
? Math.max(latest, segment.timestampMs)
|
||
: latest,
|
||
-1
|
||
);
|
||
const latestPendingTimestamp = this.pendingSentenceQueue.reduce(
|
||
(latest, item) =>
|
||
Number.isFinite(item.timestampMs)
|
||
? Math.max(latest, item.timestampMs)
|
||
: latest,
|
||
-1
|
||
);
|
||
const latestOnlineTimestamp = Number.isFinite(this.lastOnlineTimestamp)
|
||
? this.lastOnlineTimestamp
|
||
: -1;
|
||
|
||
return Math.max(
|
||
latestSegmentTimestamp,
|
||
latestPendingTimestamp,
|
||
latestOnlineTimestamp
|
||
);
|
||
}
|
||
|
||
getRealtimeCarryoverTail(text, maxLength = 160) {
|
||
return (text || "").slice(-maxLength);
|
||
}
|
||
|
||
// 保存编辑后文本的一小段尾巴。下一条 offline 结果可能与它重叠，
|
||
// trimOfflineTextWithCarryover 会用它裁掉重复内容。
|
||
buildRealtimeEditCarryover(baseText) {
|
||
const tailText = this.getRealtimeCarryoverTail(baseText);
|
||
if (tailText.length < 4) {
|
||
return null;
|
||
}
|
||
|
||
return { tailText };
|
||
}
|
||
|
||
// 手动编辑后，offline ASR 可能返回与编辑尾巴重复的文本。这里只保留新后缀，
|
||
// 避免页面出现重复句子。
|
||
trimOfflineTextWithCarryover(rawText) {
|
||
const incomingText = rawText || "";
|
||
if (!this.realtimeEditCarryover || !incomingText) {
|
||
return incomingText;
|
||
}
|
||
|
||
const tailText = this.realtimeEditCarryover.tailText || "";
|
||
const minOverlapLength = 4;
|
||
const maxOverlapLength = Math.min(
|
||
80,
|
||
tailText.length,
|
||
incomingText.length
|
||
);
|
||
|
||
for (let size = maxOverlapLength; size >= minOverlapLength; size--) {
|
||
const overlapText = tailText.slice(-size);
|
||
const matchIndex = incomingText.lastIndexOf(overlapText);
|
||
if (matchIndex === -1) {
|
||
continue;
|
||
}
|
||
|
||
const remainingText = incomingText.slice(matchIndex + overlapText.length);
|
||
this.realtimeEditCarryover.tailText = this.getRealtimeCarryoverTail(
|
||
tailText + remainingText
|
||
);
|
||
return remainingText;
|
||
}
|
||
|
||
this.realtimeEditCarryover = null;
|
||
return incomingText;
|
||
}
|
||
|
||
// 丢弃旧录音周期或编辑时间屏障之前的消息。这是暂停/恢复竞态的主要保护。
|
||
isStaleRealtimeMessage(messageTimestampMs, messageEpoch) {
|
||
if (messageEpoch !== this.liveTranscriptionEpoch) {
|
||
return true;
|
||
}
|
||
|
||
return (
|
||
Number.isFinite(messageTimestampMs) &&
|
||
this.realtimeTimestampBarrierMs >= 0 &&
|
||
messageTimestampMs <= this.realtimeTimestampBarrierMs
|
||
);
|
||
}
|
||
|
||
// pending 句子会先用 raw/corrected 文本显示在预览区；只有 LLM 矫正完成并按队首
|
||
// 顺序 flush 后，才会进入正式 segmentList。
|
||
// pendingSentenceQueue 已按 timestampMs 有序插入，预览自然按说话顺序显示
|
||
getPendingSentencePreviewText() {
|
||
return this.pendingSentenceQueue
|
||
.filter((item) => item.epoch === this.liveTranscriptionEpoch)
|
||
.map((item) => item.correctedText || item.rawText)
|
||
.join("");
|
||
}
|
||
|
||
// 预览文本 = 待矫正 offline 文本 + online/offline 交接文本 + 当前 online 临时文本。
|
||
syncRealtimePreview() {
|
||
// 【兜底去重】无论前面逻辑是否产生重复标点，最终渲染前统一清理
|
||
const previewText = this._deduplicateConsecutivePunctuation(
|
||
this.getPendingSentencePreviewText() +
|
||
this.offlineSentenceHandoffText +
|
||
this.recTextOnline
|
||
);
|
||
this.onPreviewTextChange(previewText);
|
||
}
|
||
|
||
// 幂等查找：同一 source/timestamp 的重复 websocket 回包不能触发第二次矫正，
|
||
// 也不能重复落到页面。
|
||
findPendingSentenceBySource(sourceText, pendingTimestamp, messageEpoch) {
|
||
const normalizedSourceText = sourceText || "";
|
||
const normalizedTimestamp = pendingTimestamp || "";
|
||
return (
|
||
this.pendingSentenceQueue.find(
|
||
(item) =>
|
||
item.epoch === messageEpoch &&
|
||
item.sourceText === normalizedSourceText &&
|
||
item.timestamp === normalizedTimestamp
|
||
) ||
|
||
this.segmentList.find(
|
||
(item) =>
|
||
item.epoch === messageEpoch &&
|
||
item.sourceText === normalizedSourceText &&
|
||
item.timestamp === normalizedTimestamp
|
||
) ||
|
||
null
|
||
);
|
||
}
|
||
|
||
// 2pass-offline 文本进入 LLM 矫正前的唯一入口。这里负责旧消息过滤、
|
||
// 重复回包去重、编辑重叠裁剪，以及创建 pending 队列项。
|
||
ensurePendingSentence(sourceText, pendingTimestamp, messageEpoch) {
|
||
const timestampMs = this.extractTimestampMs(pendingTimestamp);
|
||
if (this.isStaleRealtimeMessage(timestampMs, messageEpoch)) {
|
||
return null;
|
||
}
|
||
|
||
// 去重
|
||
const existingItem = this.findPendingSentenceBySource(
|
||
sourceText,
|
||
pendingTimestamp,
|
||
messageEpoch
|
||
);
|
||
if (existingItem) {
|
||
return existingItem;
|
||
}
|
||
|
||
const trimmedText = this.trimOfflineTextWithCarryover(sourceText || "");
|
||
if (!trimmedText) {
|
||
this.syncRealtimePreview();
|
||
return null;
|
||
}
|
||
|
||
// pendingItem 是 LLM 矫正队列的最小单元。它会一直 pending 到
|
||
// /api/asr/correct 返回，再由 flushProcessedPendingSentences 按顺序正式落句。
|
||
const pendingItem = {
|
||
id: this.pendingSentenceSequence++,
|
||
sourceText: sourceText || "",
|
||
rawText: trimmedText,
|
||
timestamp: pendingTimestamp || "",
|
||
timestampMs,
|
||
correctedText: "",
|
||
processed: false,
|
||
correctionRequested: false,
|
||
epoch: messageEpoch,
|
||
};
|
||
|
||
// 按 timestampMs 顺序插入，保持队列按音频时间有序
|
||
// 这样 flush 时按时间顺序处理，预览也按说话顺序显示
|
||
const insertIndex = this.pendingSentenceQueue.findIndex(
|
||
(item) => item.timestampMs > timestampMs
|
||
);
|
||
if (insertIndex === -1) {
|
||
this.pendingSentenceQueue.push(pendingItem);
|
||
} else {
|
||
this.pendingSentenceQueue.splice(insertIndex, 0, pendingItem);
|
||
}
|
||
|
||
this.syncRealtimePreview();
|
||
return pendingItem;
|
||
}
|
||
|
||
// 即使 LLM 请求乱序返回，也必须保持句子顺序：只有队首已 processed 的项
|
||
// 才能进入 segmentList。
|
||
// pendingSentenceQueue 已按 timestampMs 有序插入，segmentList 也按 timestampMs 有序插入
|
||
flushProcessedPendingSentences() {
|
||
let didFlush = false;
|
||
while (this.pendingSentenceQueue.length > 0) {
|
||
if (this.pendingSentenceQueue[0].epoch !== this.liveTranscriptionEpoch) {
|
||
this.pendingSentenceQueue.shift();
|
||
continue;
|
||
}
|
||
if (!this.pendingSentenceQueue[0].processed) {
|
||
break;
|
||
}
|
||
|
||
const item = this.pendingSentenceQueue.shift();
|
||
let correctedText = item.correctedText || item.rawText || "";
|
||
|
||
// 构造 segment 项
|
||
const segment = {
|
||
text: correctedText,
|
||
sourceText: item.sourceText,
|
||
timestamp: item.timestamp,
|
||
timestampMs: item.timestampMs,
|
||
speaker: null,
|
||
processed: true,
|
||
epoch: item.epoch,
|
||
};
|
||
|
||
// 按 timestampMs 有序插入 segmentList，确保最终渲染顺序与音频时间一致
|
||
const insertIndex = this.segmentList.findIndex(
|
||
(seg) => seg.timestampMs > segment.timestampMs
|
||
);
|
||
if (insertIndex === -1) {
|
||
this.segmentList.push(segment);
|
||
} else {
|
||
this.segmentList.splice(insertIndex, 0, segment);
|
||
}
|
||
|
||
// 相邻句子首尾标点去重：
|
||
// 场景：前一句经 /api/asr/correct 添加句号后，
|
||
// WebSocket 返回的下一句开头又带句号（模型补全），导致双句号
|
||
// 规则：前一句末尾有标点 + 当前句开头也有标点 → 移除当前句开头标点
|
||
// 注意：这里基于插入后的物理位置检查相邻关系
|
||
const insertedIdx = this.segmentList.indexOf(segment);
|
||
if (insertedIdx > 0) {
|
||
const prevItem = this.segmentList[insertedIdx - 1];
|
||
const prevText = prevItem.text || "";
|
||
const prevEndingPunct = prevText.match(/[。！？.!?]+$/)?.[0];
|
||
const currentStartingPunct = segment.text.match(/^[，。！？、；：,.!?;:]+/)?.[0];
|
||
|
||
if (prevEndingPunct && currentStartingPunct) {
|
||
// 移除当前句开头的标点，避免与前一句末尾标点重复
|
||
segment.text = segment.text.replace(/^[，。！？、；：,.!?;:]+/, "");
|
||
}
|
||
}
|
||
|
||
didFlush = true;
|
||
}
|
||
|
||
if (didFlush) {
|
||
this.renderFullText();
|
||
} else {
|
||
this.syncRealtimePreview();
|
||
}
|
||
}
|
||
|
||
// 将一条 pending 句子标记为矫正完成或 raw fallback，然后尝试 flush 队列。
|
||
finalizePendingSentence(pendingSentenceId, finalText, messageEpoch) {
|
||
const target = this.pendingSentenceQueue.find(
|
||
(item) => item.id === pendingSentenceId && item.epoch === messageEpoch
|
||
);
|
||
if (!target) {
|
||
return;
|
||
}
|
||
|
||
let correctedText = finalText || target.rawText || "";
|
||
|
||
// 去除矫正结果末尾的连续重复标点，以 /api/asr/correct 返回结果为准
|
||
// 示例："你好。。" → "你好。" ； "你好。！" → "你好！"
|
||
correctedText = this._deduplicateEndingPunctuation(correctedText);
|
||
|
||
target.correctedText = correctedText;
|
||
target.processed = true;
|
||
target.correctionRequested = false;
|
||
this.flushProcessedPendingSentences();
|
||
}
|
||
|
||
// 对单条 pending 句子发起 LLM 矫正请求，完成后 finalize。
|
||
// 调用方需先设置 correctionRequested=true 并增加 pendingCorrectionCount。
|
||
async _requestCorrection(pendingSentence) {
|
||
const messageEpoch = pendingSentence.epoch;
|
||
try {
|
||
const response = await this.fetchImpl(this.correctUrl, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
text: pendingSentence.rawText,
|
||
scene: this.getScene(),
|
||
}),
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error("ASR correction failed");
|
||
}
|
||
|
||
const data = await response.json();
|
||
const correctedText = data.corrected_text || pendingSentence.rawText;
|
||
|
||
this.finalizePendingSentence(
|
||
pendingSentence.id,
|
||
correctedText,
|
||
messageEpoch
|
||
);
|
||
} catch (error) {
|
||
this.finalizePendingSentence(
|
||
pendingSentence.id,
|
||
pendingSentence.rawText,
|
||
messageEpoch
|
||
);
|
||
console.error("ASR correction failed:", error);
|
||
} finally {
|
||
this.setPendingCorrectionCount(this.pendingCorrectionCount - 1);
|
||
}
|
||
}
|
||
|
||
// 暂停录音时调用：把所有未矫正的预览文本立即拿去矫正，
|
||
// 等待矫正完成后加入 segmentList，确保历史转写结构完整。
|
||
async finalizeRemainingSentences(timeoutMs = 8000) {
|
||
// 1. 对所有尚未请求矫正的 pending 句子立即发起矫正
|
||
const unrequested = this.pendingSentenceQueue.filter(
|
||
(item) =>
|
||
item.epoch === this.liveTranscriptionEpoch &&
|
||
!item.correctionRequested &&
|
||
!item.processed
|
||
);
|
||
|
||
for (const item of unrequested) {
|
||
item.correctionRequested = true;
|
||
this.setPendingCorrectionCount(this.pendingCorrectionCount + 1);
|
||
// 不 await，让所有矫正并行进行
|
||
this._requestCorrection(item);
|
||
}
|
||
|
||
// 2. 如果还有尚未形成 offline 句子的 online 预览文本，作为 raw 句子追加
|
||
const remainingOnlineText = (this.recTextOnline || "") + (this.offlineSentenceHandoffText || "");
|
||
if (remainingOnlineText.trim()) {
|
||
const messageEpoch = this.liveTranscriptionEpoch;
|
||
// timestampMs 用一个大值，确保插入到 segmentList 末尾而不是开头
|
||
const fallbackTimestampMs = this.getLatestRealtimeTimestamp() || Date.now();
|
||
const pendingItem = {
|
||
id: this.pendingSentenceSequence++,
|
||
sourceText: remainingOnlineText,
|
||
rawText: remainingOnlineText,
|
||
timestamp: "",
|
||
timestampMs: fallbackTimestampMs,
|
||
correctedText: "",
|
||
processed: false,
|
||
correctionRequested: true,
|
||
epoch: messageEpoch,
|
||
};
|
||
this.pendingSentenceQueue.push(pendingItem);
|
||
this.setPendingCorrectionCount(this.pendingCorrectionCount + 1);
|
||
this._requestCorrection(pendingItem);
|
||
}
|
||
|
||
// 3. 等待所有矫正完成
|
||
await this.waitForCorrections(timeoutMs);
|
||
|
||
// 4. 对超时仍未完成的句子，用 raw 文本强制 finalize
|
||
const stuck = this.pendingSentenceQueue.filter(
|
||
(item) =>
|
||
item.epoch === this.liveTranscriptionEpoch && !item.processed
|
||
);
|
||
for (const item of stuck) {
|
||
this.finalizePendingSentence(item.id, item.rawText, item.epoch);
|
||
this.setPendingCorrectionCount(Math.max(0, this.pendingCorrectionCount - 1));
|
||
}
|
||
|
||
// 5. 显式 flush 残留的已处理句子，确保全部进入 segmentList
|
||
this.flushProcessedPendingSentences();
|
||
|
||
// 6. 清空残余的 online 预览缓冲，并最终渲染一次确保显示框更新
|
||
this.recTextOnline = "";
|
||
this.offlineSentenceHandoffText = "";
|
||
this.renderFullText();
|
||
this.syncRealtimePreview();
|
||
}
|
||
|
||
// 去除矫正文本末尾的连续重复标点符号
|
||
// 原则：以 /api/asr/correct 返回结果为准，保留其最后一个标点
|
||
// 无论符号是否相同，只要末尾有连续多个标点，就缩减为最后一个
|
||
// 示例：corrected="你好。。" → "你好。"
|
||
// 示例：corrected="你好。！" → "你好！" （保留最后一个）
|
||
_deduplicateEndingPunctuation(correctedText) {
|
||
if (!correctedText) {
|
||
return correctedText;
|
||
}
|
||
|
||
// 匹配末尾连续标点（中英文标点）
|
||
const punctPattern = /[。！？.!?]+$/;
|
||
const correctedPunct = correctedText.match(punctPattern)?.[0] || "";
|
||
|
||
// 如果末尾有2个或以上连续标点，只保留最后一个（以矫正结果为准）
|
||
if (correctedPunct.length >= 2) {
|
||
const lastPunct = correctedPunct.slice(-1);
|
||
return correctedText.replace(punctPattern, lastPunct);
|
||
}
|
||
|
||
return correctedText;
|
||
}
|
||
|
||
// 【兜底方案 A】去除文本中任意位置出现的连续标点（保留第一个）
|
||
// 用于捕捉前面逻辑（换行加标点、相邻句首尾标点等）漏掉的重复标点问题
|
||
_deduplicateConsecutivePunctuation(text) {
|
||
if (!text) return text;
|
||
// 第一步：处理跨换行的连续标点（如 "？\n。" → "？\n"、"。\n。" → "。\n"）
|
||
// 任意标点 + 可选空白 + 换行 + 可选空白 + 任意标点 → 只保留第一个标点和换行
|
||
// 注意：不用 \1 反向引用，因为需要匹配不同标点相邻的情况（如 ？。、，。）
|
||
let result = text.replace(/([。！？.!?，、；：,;:])\s*\n\s*[。！？.!?，、；：,;:]/g, "$1\n");
|
||
// 第二步：处理同行内的连续标点（如 "？。" → "？"、"。。" → "。"）
|
||
result = result.replace(/([。！？.!?，、；：,;:])(?:[。！？.!?，、；：,;:])+/g, "$1");
|
||
return result;
|
||
}
|
||
|
||
// 根据字符级 timestamp 重建更适合阅读的断句。长停顿或配置的结束短语
|
||
// 可以触发换行，但不修改 ASR 原文内容。
|
||
// prevChar: 上一个 segment 返回文本的末尾字符，用于跨 segment 调用时
|
||
// textWithTime 为空的情况下正确判断是否需要补句号。
|
||
handleWithTimestamp(text, timestamp, keywordStr, timeThreshold, prevChar = "") {
|
||
if (!timestamp || text.length <= 0) {
|
||
return text;
|
||
}
|
||
|
||
const keywordList = (keywordStr || "")
|
||
.split(",")
|
||
.map((keyword) => keyword.trim())
|
||
.filter((keyword) => keyword.length > 0);
|
||
const keywordsAsRegex = keywordList.map((keyword) => new RegExp(keyword));
|
||
const segments = text.match(/[^。！？.!?]+[。！？.!?]?/g) || [];
|
||
const timeData = JSON.parse(timestamp);
|
||
let charIndex = 0;
|
||
let textWithTime = "";
|
||
|
||
const removeLeadingPunctuation = (value) =>
|
||
value.replace(/^[，。！？、；：,\.!?;:]+/, "");
|
||
|
||
for (let i = 0; i < segments.length; i++) {
|
||
const segment = segments[i];
|
||
if (!segment || segment === "undefined") {
|
||
continue;
|
||
}
|
||
|
||
const nowTime = timeData[charIndex][0] / 1000;
|
||
const previousTime = this.getLastTimer();
|
||
|
||
if (previousTime > 0 && nowTime - previousTime > timeThreshold) {
|
||
// textWithTime 为空时（跨 segment 首次进入），fallback 到上一个
|
||
// segment 的末尾字符，避免对已有标点的行尾再补句号
|
||
const lastChar = textWithTime.slice(-1) || prevChar;
|
||
const endsWithPunc = /[。！？.!?，、；：,;:]/.test(lastChar);
|
||
const processedSegment = removeLeadingPunctuation(segment);
|
||
const segmentStartsWithPunc = /^[，。！？、；：,.!?;:]+/.test(processedSegment);
|
||
textWithTime +=
|
||
(endsWithPunc || segmentStartsWithPunc ? "\n" : "。\n") + processedSegment;
|
||
} else {
|
||
const hitKeyword = keywordsAsRegex.some((regex) => regex.test(segment));
|
||
if (!hitKeyword) {
|
||
textWithTime += segment;
|
||
}
|
||
}
|
||
|
||
this.setLastTimer(nowTime);
|
||
charIndex++;
|
||
}
|
||
|
||
return textWithTime;
|
||
}
|
||
|
||
getSpeakerBeforeIndex(endIndex) {
|
||
let previousSpeaker = null;
|
||
for (let i = 0; i < endIndex; i++) {
|
||
if (this.segmentList[i] && this.segmentList[i].speaker) {
|
||
previousSpeaker = this.segmentList[i].speaker;
|
||
}
|
||
}
|
||
return previousSpeaker;
|
||
}
|
||
|
||
// 将已提交 segment 转成页面文本。只有说话人变化时才插入 speaker header，
|
||
// 避免每句话都重复显示说话人。
|
||
buildSegmentText(segments, initialSpeaker = null) {
|
||
let displayText = "";
|
||
let lastRenderedSpeaker = initialSpeaker;
|
||
let lastContentChar = "";
|
||
|
||
segments.forEach((segment) => {
|
||
if (this.getShowSpeaker()) {
|
||
if (
|
||
segment.speaker &&
|
||
segment.speaker !== lastRenderedSpeaker &&
|
||
!segment.speaker.includes("SPK")
|
||
) {
|
||
displayText +=
|
||
(displayText ? "\n" : "") + segment.speaker + ":\n";
|
||
lastRenderedSpeaker = segment.speaker;
|
||
} else if (
|
||
segment.speaker &&
|
||
segment.speaker !== lastRenderedSpeaker
|
||
) {
|
||
displayText +=
|
||
(displayText ? "\n" : "") + segment.speaker + ":\n";
|
||
lastRenderedSpeaker = segment.speaker;
|
||
}
|
||
}
|
||
|
||
const segText = this.handleWithTimestamp(
|
||
segment.text,
|
||
segment.timestamp,
|
||
this.getConfigWords(),
|
||
6,
|
||
lastContentChar
|
||
);
|
||
displayText += segText;
|
||
|
||
// 跟踪上一个 segment 的末尾非空白字符，传给下一次 handleWithTimestamp
|
||
const trimmed = segText.replace(/\s+$/, "");
|
||
if (trimmed.length > 0) {
|
||
lastContentChar = trimmed.slice(-1);
|
||
}
|
||
});
|
||
|
||
return displayText;
|
||
}
|
||
|
||
// 将已矫正并提交的 segment 渲染到主转录区。pending/online 预览文本保持独立，
|
||
// 主文本更新后再同步预览区。
|
||
renderFullText() {
|
||
const initialSpeaker =
|
||
this.segmentRenderStartIndex > 0
|
||
? this.getSpeakerBeforeIndex(this.segmentRenderStartIndex)
|
||
: null;
|
||
const displayText = this.buildSegmentText(
|
||
this.segmentList.slice(this.segmentRenderStartIndex),
|
||
initialSpeaker
|
||
);
|
||
|
||
this.recText = displayText;
|
||
// 【兜底去重】清理 pausedTranscription + recText 中可能产生的连续重复标点
|
||
const mainText = this._deduplicateConsecutivePunctuation(
|
||
this.getPausedTranscription() + this.recText
|
||
);
|
||
this.onMainTextChange(mainText);
|
||
this.syncRealtimePreview();
|
||
}
|
||
// online websocket 返回低延迟片段，只更新预览区；正式 offline 句子统一交给
|
||
// handleOfflineMessage，避免双通路重复落句。
|
||
handleOnlineMessage(event) {
|
||
if (this.disposed) {
|
||
return;
|
||
}
|
||
|
||
const parsedMessage = JSON.parse(event.data);
|
||
const text = "" + parsedMessage.text;
|
||
const mode = parsedMessage.mode;
|
||
const timestamp = parsedMessage.timestamp;
|
||
const messageEpoch = this.liveTranscriptionEpoch;
|
||
|
||
if (mode === "2pass-online") {
|
||
// 低延迟预览文本可能不完整，因此不进入正式 segmentList。
|
||
const currentTimestamp = this.extractTimestampMs(timestamp);
|
||
if (this.isStaleRealtimeMessage(currentTimestamp, messageEpoch)) {
|
||
this.syncRealtimePreview();
|
||
return;
|
||
}
|
||
|
||
let shouldBreak = false;
|
||
if (this.lastOnlineTimestamp !== null && currentTimestamp !== null) {
|
||
const timeDiff = (currentTimestamp - this.lastOnlineTimestamp) / 1000;
|
||
if (timeDiff >= 3 && /。\s*$/.test(text)) {
|
||
shouldBreak = true;
|
||
}
|
||
}
|
||
|
||
if (parsedMessage.is_full_text) {
|
||
this.recTextOnline = shouldBreak ? text + "\n" : text;
|
||
} else {
|
||
this.recTextOnline += shouldBreak ? text + "\n" : text;
|
||
}
|
||
this.lastOnlineTimestamp = currentTimestamp;
|
||
} else if (mode === "2pass-offline") {
|
||
// online 通道也可能回显 offline 句子，这里只作为交接预览保存。
|
||
// 正式句子只通过 offline 通道进入矫正流程。
|
||
this.recTextOnline = "";
|
||
this.offlineSentenceHandoffText = text || this.recTextOnline || "";
|
||
this.recTextOnline = "";
|
||
this.lastOnlineTimestamp = null;
|
||
}
|
||
|
||
this.syncRealtimePreview();
|
||
}
|
||
|
||
// offline websocket 才负责正式转录路径：2pass-offline 文本进入 LLM 矫正，
|
||
// offline-speaker 后续只补说话人信息。
|
||
async handleOfflineMessage(event) {
|
||
if (this.disposed) {
|
||
return;
|
||
}
|
||
|
||
const parsedMessage = JSON.parse(event.data);
|
||
const mode = parsedMessage.mode;
|
||
const timestamp = parsedMessage.timestamp;
|
||
const messageEpoch = this.liveTranscriptionEpoch;
|
||
const messageTimestampMs = this.extractTimestampMs(timestamp);
|
||
|
||
if (mode === "2pass-offline") {
|
||
// 文本先到：先放入 pending 队列刷新预览，再调用 LLM 矫正接口，
|
||
// 矫正完成后才进入正式 segmentList。
|
||
if (this.isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) {
|
||
return;
|
||
}
|
||
|
||
const text = "" + parsedMessage.text;
|
||
this.recTextOnline = "";
|
||
this.lastOnlineTimestamp = null;
|
||
const pendingSentence = this.ensurePendingSentence(
|
||
text,
|
||
timestamp,
|
||
messageEpoch
|
||
);
|
||
this.offlineSentenceHandoffText = "";
|
||
this.syncRealtimePreview();
|
||
|
||
if (
|
||
!pendingSentence ||
|
||
pendingSentence.processed ||
|
||
pendingSentence.correctionRequested
|
||
) {
|
||
return;
|
||
}
|
||
|
||
pendingSentence.correctionRequested = true;
|
||
this.setPendingCorrectionCount(this.pendingCorrectionCount + 1);
|
||
await this._requestCorrection(pendingSentence);
|
||
} else if (mode === "offline-speaker") {
|
||
// 说话人标签可能晚于矫正文本到达。这里只更新 speaker 元数据，
|
||
// 绝不替换 text，避免已矫正句子跳回 raw ASR 文本。
|
||
if (this.isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) {
|
||
return;
|
||
}
|
||
|
||
const speaker =
|
||
(parsedMessage.segments &&
|
||
parsedMessage.segments[0] &&
|
||
parsedMessage.segments[0].speaker) ||
|
||
"未命名演讲者";
|
||
|
||
let targetIndex = -1;
|
||
for (let i = this.segmentList.length - 1; i >= 0; i--) {
|
||
const segment = this.segmentList[i];
|
||
if (!segment || segment.epoch !== messageEpoch) {
|
||
continue;
|
||
}
|
||
if (
|
||
segment.speaker === null &&
|
||
Number.isFinite(messageTimestampMs) &&
|
||
Number.isFinite(segment.timestampMs) &&
|
||
segment.timestampMs === messageTimestampMs
|
||
) {
|
||
targetIndex = i;
|
||
break;
|
||
}
|
||
if (targetIndex === -1 && segment.speaker === null) {
|
||
targetIndex = i;
|
||
}
|
||
}
|
||
|
||
if (targetIndex !== -1) {
|
||
this.segmentList[targetIndex].speaker = speaker;
|
||
this.renderFullText();
|
||
}
|
||
}
|
||
}
|
||
}
|