Ascend/SmartMeeting
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve realtime speaker display and transcript spacing

Browse Source
This commit is contained in:
Xulilong 2026-07-16 09:50:29 +08:00
parent 4ec8bfb93b
commit 67edea1b4b
5 changed files with 188 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fastapi_wss/src/a2a_wss.py
View File

@ -606,7 +606,9 @@ class A2AOrchestrator:
def build_acoustic_speaker_message(data: Dict) -> Optional[Dict]: def build_acoustic_speaker_message(data: Dict) -> Optional[Dict]:
text = (data.get("text") or "").strip() text = (data.get("text") or "").strip()
speaker = (data.get("spk_name") or "").strip() speaker = (data.get("spk_name") or "").strip()
if not text or not speaker or speaker.lower() == "unknown": if not speaker or speaker.lower() == "unknown":
speaker = get_env("REALTIME_DEFAULT_SPEAKER", "SPEAKER_00")
if not text or not speaker:
return None return None
segment = { segment = {

62
speakr/static/vite/index/assets/index.js
View File

File diff suppressed because one or more lines are too long

2
speakr/static/vite/index/assets/index.js.map
View File

File diff suppressed because one or more lines are too long

96
speakr/templates/index.html
View File

@ -1105,30 +1105,30 @@
<div v-if="isDualRealtimeTranscription" ref="dualTranscriptContainer" <div v-if="isDualRealtimeTranscription" ref="dualTranscriptContainer"
class="dual-transcript-stack" style="height: 650px;"> class="dual-transcript-stack" style="height: 650px;">
<textarea v-if="isEditingTranscription" <textarea v-if="isEditingTranscription"
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 text-lg leading-8 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
:style="{ height: dualTranscriptTopPercent + '%' }" :style="{ height: dualTranscriptTopPercent + '%' }"
v-model="editableText"></textarea> v-model="editableText"></textarea>
<textarea v-else ref="asrResultTextareaOnline" <textarea v-else ref="asrResultTextareaOnline"
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 text-lg leading-8 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
:style="{ height: dualTranscriptTopPercent + '%' }" :style="{ height: dualTranscriptTopPercent + '%' }"
:value="fullRealtimeTranscription" readonly></textarea> :value="fullRealtimeTranscription" readonly></textarea>
<div class="dual-transcript-divider" @mousedown="startDualTranscriptResize"></div> <div class="dual-transcript-divider" @mousedown="startDualTranscriptResize"></div>
<textarea v-if="isEditingTranscription" <textarea v-if="isEditingTranscription"
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 text-lg leading-8 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
:style="{ height: (100 - dualTranscriptTopPercent) + '%' }" :style="{ height: (100 - dualTranscriptTopPercent) + '%' }"
v-model="systemEditableText"></textarea> v-model="systemEditableText"></textarea>
<textarea v-else ref="systemAsrResultTextareaOnline" <textarea v-else ref="systemAsrResultTextareaOnline"
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 text-lg leading-8 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
:style="{ height: (100 - dualTranscriptTopPercent) + '%' }" :style="{ height: (100 - dualTranscriptTopPercent) + '%' }"
:value="fullSystemRealtimeTranscription" readonly></textarea> :value="fullSystemRealtimeTranscription" readonly></textarea>
</div> </div>
<textarea v-else-if="isEditingTranscription" style="height: 650px;" <textarea v-else-if="isEditingTranscription" style="height: 650px;"
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 text-lg leading-8 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
v-model="editableText"></textarea> v-model="editableText"></textarea>
<textarea v-else ref="asrResultTextareaOnline" style="height: 650px;" <textarea v-else ref="asrResultTextareaOnline" style="height: 650px;"
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 text-lg leading-8 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
:value="fullRealtimeTranscription" readonly></textarea> :value="fullRealtimeTranscription" readonly></textarea>
<!-- <textarea <!-- <textarea
ref="asrTextarea" ref="asrTextarea"
@ -1353,32 +1353,32 @@ ${item.name}
<!-- 弹框主体 --> <!-- 弹框主体 -->
<div class="p-4"> <div class="p-4">
<div v-if="isDualRealtimeTranscription" class="dual-transcript-stack" style="height: 90vh;"> <div v-if="isDualRealtimeTranscription" class="dual-transcript-stack" style="height: 90vh;">
<textarea v-if="isEditingTranscription" <textarea v-if="isEditingTranscription"
:style="{ fontSize: baseInfo.fontSize + 'px', height: dualTranscriptTopPercent + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }" :style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height: dualTranscriptTopPercent + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
v-model="editableText"></textarea> v-model="editableText"></textarea>
<textarea v-else ref="asrResultTextarea" <textarea v-else ref="asrResultTextarea"
:style="{ fontSize: baseInfo.fontSize + 'px', height: dualTranscriptTopPercent + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }" :style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height: dualTranscriptTopPercent + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
:value="fullRealtimeTranscription" readonly></textarea> :value="fullRealtimeTranscription" readonly></textarea>
<div class="dual-transcript-divider" @mousedown="startDualTranscriptResize"></div> <div class="dual-transcript-divider" @mousedown="startDualTranscriptResize"></div>
<textarea v-if="isEditingTranscription" <textarea v-if="isEditingTranscription"
:style="{ fontSize: baseInfo.fontSize + 'px', height: (100 - dualTranscriptTopPercent) + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }" :style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height: (100 - dualTranscriptTopPercent) + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
v-model="systemEditableText"></textarea> v-model="systemEditableText"></textarea>
<textarea v-else ref="systemAsrResultTextarea" <textarea v-else ref="systemAsrResultTextarea"
:style="{ fontSize: baseInfo.fontSize + 'px', height: (100 - dualTranscriptTopPercent) + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }" :style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height: (100 - dualTranscriptTopPercent) + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full p-4 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
:value="fullSystemRealtimeTranscription" readonly></textarea> :value="fullSystemRealtimeTranscription" readonly></textarea>
</div> </div>
<textarea v-else-if="isEditingTranscription" <textarea v-else-if="isEditingTranscription"
:style="{ fontSize: baseInfo.fontSize + 'px',height:'90vh',backgroundColor:'rgba(0, 94, 170, 0.8)' }" :style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height:'90vh',backgroundColor:'rgba(0, 94, 170, 0.8)' }"
class="w-full h-60 p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full h-60 p-4 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
v-model="editableText"></textarea> v-model="editableText"></textarea>
<textarea v-else ref="asrResultTextarea" <textarea v-else ref="asrResultTextarea"
:style="{ fontSize: baseInfo.fontSize + 'px',height:'90vh',backgroundColor:'rgba(0, 94, 170, 0.8)' }" :style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height:'90vh',backgroundColor:'rgba(0, 94, 170, 0.8)' }"
class="w-full h-60 p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto" class="w-full h-60 p-4 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
:value="fullRealtimeTranscription" readonly></textarea> :value="fullRealtimeTranscription" readonly></textarea>
</div> </div>
</div> </div>

172
speakr/templates/index/src/asr/realtime-asr-pipeline.js
View File

@ -848,15 +848,15 @@ export class RealtimeAsrPipeline {
let correctedText = item.correctedText || item.rawText || ""; let correctedText = item.correctedText || item.rawText || "";
// 构造 segment 项 // 构造 segment 项
const segment = { const segment = {
text: correctedText, text: correctedText,
sourceText: item.sourceText, sourceText: item.sourceText,
timestamp: item.timestamp, timestamp: item.timestamp,
timestampMs: item.timestampMs, timestampMs: item.timestampMs,
speaker: null, speaker: item.speaker || null,
processed: true, processed: true,
epoch: item.epoch, epoch: item.epoch,
}; };
// 按 timestampMs 有序插入 segmentList，确保最终渲染顺序与音频时间一致 // 按 timestampMs 有序插入 segmentList，确保最终渲染顺序与音频时间一致
const insertIndex = this.segmentList.findIndex( const insertIndex = this.segmentList.findIndex(
@ -1125,7 +1125,7 @@ export class RealtimeAsrPipeline {
let lastRenderedSpeaker = initialSpeaker; let lastRenderedSpeaker = initialSpeaker;
let lastContentChar = ""; let lastContentChar = "";
segments.forEach((segment) => { segments.forEach((segment, index) => {
if (this.getShowSpeaker()) { if (this.getShowSpeaker()) {
if ( if (
segment.speaker && segment.speaker &&
@ -1145,16 +1145,19 @@ export class RealtimeAsrPipeline {
} }
} }
const segText = this.handleWithTimestamp( const segText = this.handleWithTimestamp(
segment.text, segment.text,
segment.timestamp, segment.timestamp,
this.getConfigWords(), this.getConfigWords(),
6, 6,
lastContentChar lastContentChar
); );
displayText += segText; displayText += segText;
if (index < segments.length - 1 && !/\n\s*$/.test(displayText)) {
// 跟踪上一个 segment 的末尾非空白字符，传给下一次 handleWithTimestamp displayText += "\n\n";
}
// 跟踪上一个 segment 的末尾非空白字符，传给下一次 handleWithTimestamp
const trimmed = segText.replace(/\s+$/, ""); const trimmed = segText.replace(/\s+$/, "");
if (trimmed.length > 0) { if (trimmed.length > 0) {
lastContentChar = trimmed.slice(-1); lastContentChar = trimmed.slice(-1);
@ -1166,7 +1169,7 @@ export class RealtimeAsrPipeline {
// 将已矫正并提交的 segment 渲染到主转录区。pending/online 预览文本保持独立， // 将已矫正并提交的 segment 渲染到主转录区。pending/online 预览文本保持独立，
// 主文本更新后再同步预览区。 // 主文本更新后再同步预览区。
renderFullText() { renderFullText() {
const initialSpeaker = const initialSpeaker =
this.segmentRenderStartIndex > 0 this.segmentRenderStartIndex > 0
? this.getSpeakerBeforeIndex(this.segmentRenderStartIndex) ? this.getSpeakerBeforeIndex(this.segmentRenderStartIndex)
@ -1181,12 +1184,76 @@ export class RealtimeAsrPipeline {
const mainText = this._deduplicateConsecutivePunctuation( const mainText = this._deduplicateConsecutivePunctuation(
this.getPausedTranscription() + this.recText this.getPausedTranscription() + this.recText
); );
this.onMainTextChange(mainText); this.onMainTextChange(mainText);
this.syncRealtimePreview(); this.syncRealtimePreview();
} }
// online websocket 返回低延迟片段，只更新预览区；正式 offline 句子统一交给
// handleOfflineMessage，避免双通路重复落句。 applySpeakerMessage(parsedMessage, messageEpoch) {
handleOnlineMessage(event) { const timestamp = parsedMessage.timestamp;
const messageTimestampMs = this.extractTimestampMs(timestamp);
if (this.isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) {
return;
}
const speaker =
(parsedMessage.segments &&
parsedMessage.segments[0] &&
parsedMessage.segments[0].speaker) ||
parsedMessage.speaker ||
"SPEAKER_00";
let didApply = false;
for (let i = this.pendingSentenceQueue.length - 1; i >= 0; i--) {
const pending = this.pendingSentenceQueue[i];
if (!pending || pending.epoch !== messageEpoch || pending.speaker) {
continue;
}
if (
Number.isFinite(messageTimestampMs) &&
Number.isFinite(pending.timestampMs) &&
pending.timestampMs !== messageTimestampMs
) {
continue;
}
pending.speaker = speaker;
didApply = true;
break;
}
let targetIndex = -1;
for (let i = this.segmentList.length - 1; i >= 0; i--) {
const segment = this.segmentList[i];
if (!segment || segment.epoch !== messageEpoch) {
continue;
}
if (
segment.speaker === null &&
Number.isFinite(messageTimestampMs) &&
Number.isFinite(segment.timestampMs) &&
segment.timestampMs === messageTimestampMs
) {
targetIndex = i;
break;
}
if (targetIndex === -1 && segment.speaker === null) {
targetIndex = i;
}
}
if (targetIndex !== -1) {
this.segmentList[targetIndex].speaker = speaker;
this.renderFullText();
return;
}
if (didApply) {
this.syncRealtimePreview();
}
}
// online websocket 返回低延迟片段，只更新预览区；正式 offline 句子统一交给
// handleOfflineMessage，避免双通路重复落句。
handleOnlineMessage(event) {
if (this.disposed) { if (this.disposed) {
return; return;
} }
@ -1238,10 +1305,12 @@ export class RealtimeAsrPipeline {
this._requestCorrection(pendingSentence); this._requestCorrection(pendingSentence);
} }
} }
} } else if (mode === "offline-speaker") {
this.applySpeakerMessage(parsedMessage, messageEpoch);
this.syncRealtimePreview(); }
}
this.syncRealtimePreview();
}
// offline websocket 才负责正式转录路径：2pass-offline 文本进入 LLM 矫正， // offline websocket 才负责正式转录路径：2pass-offline 文本进入 LLM 矫正，
// offline-speaker 后续只补说话人信息。 // offline-speaker 后续只补说话人信息。
@ -1287,43 +1356,10 @@ export class RealtimeAsrPipeline {
pendingSentence.correctionRequested = true; pendingSentence.correctionRequested = true;
this.setPendingCorrectionCount(this.pendingCorrectionCount + 1); this.setPendingCorrectionCount(this.pendingCorrectionCount + 1);
await this._requestCorrection(pendingSentence); await this._requestCorrection(pendingSentence);
} else if (mode === "offline-speaker") { } else if (mode === "offline-speaker") {
// 说话人标签可能晚于矫正文本到达。这里只更新 speaker 元数据， // 说话人标签可能早于或晚于矫正文本到达。这里只更新 speaker 元数据，
// 绝不替换 text，避免已矫正句子跳回 raw ASR 文本。 // 绝不替换 text，避免已矫正句子跳回 raw ASR 文本。
if (this.isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) { this.applySpeakerMessage(parsedMessage, messageEpoch);
return; }
} }
}
const speaker =
(parsedMessage.segments &&
parsedMessage.segments[0] &&
parsedMessage.segments[0].speaker) ||
"未命名演讲者";
let targetIndex = -1;
for (let i = this.segmentList.length - 1; i >= 0; i--) {
const segment = this.segmentList[i];
if (!segment || segment.epoch !== messageEpoch) {
continue;
}
if (
segment.speaker === null &&
Number.isFinite(messageTimestampMs) &&
Number.isFinite(segment.timestampMs) &&
segment.timestampMs === messageTimestampMs
) {
targetIndex = i;
break;
}
if (targetIndex === -1 && segment.speaker === null) {
targetIndex = i;
}
}
if (targetIndex !== -1) {
this.segmentList[targetIndex].speaker = speaker;
this.renderFullText();
}
}
}
}