Improve realtime speaker display and transcript spacing
This commit is contained in:
parent
4ec8bfb93b
commit
67edea1b4b
@ -606,7 +606,9 @@ class A2AOrchestrator:
|
||||
def build_acoustic_speaker_message(data: Dict) -> Optional[Dict]:
|
||||
text = (data.get("text") or "").strip()
|
||||
speaker = (data.get("spk_name") or "").strip()
|
||||
if not text or not speaker or speaker.lower() == "unknown":
|
||||
if not speaker or speaker.lower() == "unknown":
|
||||
speaker = get_env("REALTIME_DEFAULT_SPEAKER", "SPEAKER_00")
|
||||
if not text or not speaker:
|
||||
return None
|
||||
|
||||
segment = {
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -1105,30 +1105,30 @@
|
||||
|
||||
<div v-if="isDualRealtimeTranscription" ref="dualTranscriptContainer"
|
||||
class="dual-transcript-stack" style="height: 650px;">
|
||||
<textarea v-if="isEditingTranscription"
|
||||
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:style="{ height: dualTranscriptTopPercent + '%' }"
|
||||
v-model="editableText"></textarea>
|
||||
<textarea v-else ref="asrResultTextareaOnline"
|
||||
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:style="{ height: dualTranscriptTopPercent + '%' }"
|
||||
:value="fullRealtimeTranscription" readonly></textarea>
|
||||
<textarea v-if="isEditingTranscription"
|
||||
class="w-full p-4 text-lg leading-8 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:style="{ height: dualTranscriptTopPercent + '%' }"
|
||||
v-model="editableText"></textarea>
|
||||
<textarea v-else ref="asrResultTextareaOnline"
|
||||
class="w-full p-4 text-lg leading-8 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:style="{ height: dualTranscriptTopPercent + '%' }"
|
||||
:value="fullRealtimeTranscription" readonly></textarea>
|
||||
<div class="dual-transcript-divider" @mousedown="startDualTranscriptResize"></div>
|
||||
<textarea v-if="isEditingTranscription"
|
||||
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:style="{ height: (100 - dualTranscriptTopPercent) + '%' }"
|
||||
v-model="systemEditableText"></textarea>
|
||||
<textarea v-else ref="systemAsrResultTextareaOnline"
|
||||
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:style="{ height: (100 - dualTranscriptTopPercent) + '%' }"
|
||||
:value="fullSystemRealtimeTranscription" readonly></textarea>
|
||||
</div>
|
||||
<textarea v-else-if="isEditingTranscription" style="height: 650px;"
|
||||
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
v-model="editableText"></textarea>
|
||||
<textarea v-else ref="asrResultTextareaOnline" style="height: 650px;"
|
||||
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:value="fullRealtimeTranscription" readonly></textarea>
|
||||
<textarea v-if="isEditingTranscription"
|
||||
class="w-full p-4 text-lg leading-8 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:style="{ height: (100 - dualTranscriptTopPercent) + '%' }"
|
||||
v-model="systemEditableText"></textarea>
|
||||
<textarea v-else ref="systemAsrResultTextareaOnline"
|
||||
class="w-full p-4 text-lg leading-8 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:style="{ height: (100 - dualTranscriptTopPercent) + '%' }"
|
||||
:value="fullSystemRealtimeTranscription" readonly></textarea>
|
||||
</div>
|
||||
<textarea v-else-if="isEditingTranscription" style="height: 650px;"
|
||||
class="w-full p-4 text-lg leading-8 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
v-model="editableText"></textarea>
|
||||
<textarea v-else ref="asrResultTextareaOnline" style="height: 650px;"
|
||||
class="w-full p-4 text-lg leading-8 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:value="fullRealtimeTranscription" readonly></textarea>
|
||||
|
||||
<!-- <textarea
|
||||
ref="asrTextarea"
|
||||
@ -1353,32 +1353,32 @@ ${item.name}
|
||||
<!-- 弹框主体 -->
|
||||
<div class="p-4">
|
||||
<div v-if="isDualRealtimeTranscription" class="dual-transcript-stack" style="height: 90vh;">
|
||||
<textarea v-if="isEditingTranscription"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', height: dualTranscriptTopPercent + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
v-model="editableText"></textarea>
|
||||
<textarea v-else ref="asrResultTextarea"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', height: dualTranscriptTopPercent + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:value="fullRealtimeTranscription" readonly></textarea>
|
||||
<textarea v-if="isEditingTranscription"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height: dualTranscriptTopPercent + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full p-4 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
v-model="editableText"></textarea>
|
||||
<textarea v-else ref="asrResultTextarea"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height: dualTranscriptTopPercent + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full p-4 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:value="fullRealtimeTranscription" readonly></textarea>
|
||||
<div class="dual-transcript-divider" @mousedown="startDualTranscriptResize"></div>
|
||||
<textarea v-if="isEditingTranscription"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', height: (100 - dualTranscriptTopPercent) + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
v-model="systemEditableText"></textarea>
|
||||
<textarea v-else ref="systemAsrResultTextarea"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', height: (100 - dualTranscriptTopPercent) + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:value="fullSystemRealtimeTranscription" readonly></textarea>
|
||||
</div>
|
||||
<textarea v-else-if="isEditingTranscription"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px',height:'90vh',backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full h-60 p-2 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
v-model="editableText"></textarea>
|
||||
<textarea v-else ref="asrResultTextarea"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px',height:'90vh',backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full h-60 p-2 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:value="fullRealtimeTranscription" readonly></textarea>
|
||||
<textarea v-if="isEditingTranscription"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height: (100 - dualTranscriptTopPercent) + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full p-4 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
v-model="systemEditableText"></textarea>
|
||||
<textarea v-else ref="systemAsrResultTextarea"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height: (100 - dualTranscriptTopPercent) + '%', backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full p-4 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:value="fullSystemRealtimeTranscription" readonly></textarea>
|
||||
</div>
|
||||
<textarea v-else-if="isEditingTranscription"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height:'90vh',backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full h-60 p-4 border-2 border-blue-500 rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
v-model="editableText"></textarea>
|
||||
<textarea v-else ref="asrResultTextarea"
|
||||
:style="{ fontSize: baseInfo.fontSize + 'px', lineHeight: 1.45, height:'90vh',backgroundColor:'rgba(0, 94, 170, 0.8)' }"
|
||||
class="w-full h-60 p-4 border border-[var(--border-primary)] rounded-md bg-[var(--bg-secondary)] text-[var(--text-primary)] resize-none overflow-y-auto"
|
||||
:value="fullRealtimeTranscription" readonly></textarea>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
@ -848,15 +848,15 @@ export class RealtimeAsrPipeline {
|
||||
let correctedText = item.correctedText || item.rawText || "";
|
||||
|
||||
// 构造 segment 项
|
||||
const segment = {
|
||||
text: correctedText,
|
||||
sourceText: item.sourceText,
|
||||
timestamp: item.timestamp,
|
||||
timestampMs: item.timestampMs,
|
||||
speaker: null,
|
||||
processed: true,
|
||||
epoch: item.epoch,
|
||||
};
|
||||
const segment = {
|
||||
text: correctedText,
|
||||
sourceText: item.sourceText,
|
||||
timestamp: item.timestamp,
|
||||
timestampMs: item.timestampMs,
|
||||
speaker: item.speaker || null,
|
||||
processed: true,
|
||||
epoch: item.epoch,
|
||||
};
|
||||
|
||||
// 按 timestampMs 有序插入 segmentList，确保最终渲染顺序与音频时间一致
|
||||
const insertIndex = this.segmentList.findIndex(
|
||||
@ -1125,7 +1125,7 @@ export class RealtimeAsrPipeline {
|
||||
let lastRenderedSpeaker = initialSpeaker;
|
||||
let lastContentChar = "";
|
||||
|
||||
segments.forEach((segment) => {
|
||||
segments.forEach((segment, index) => {
|
||||
if (this.getShowSpeaker()) {
|
||||
if (
|
||||
segment.speaker &&
|
||||
@ -1145,16 +1145,19 @@ export class RealtimeAsrPipeline {
|
||||
}
|
||||
}
|
||||
|
||||
const segText = this.handleWithTimestamp(
|
||||
segment.text,
|
||||
segment.timestamp,
|
||||
const segText = this.handleWithTimestamp(
|
||||
segment.text,
|
||||
segment.timestamp,
|
||||
this.getConfigWords(),
|
||||
6,
|
||||
lastContentChar
|
||||
);
|
||||
displayText += segText;
|
||||
|
||||
// 跟踪上一个 segment 的末尾非空白字符，传给下一次 handleWithTimestamp
|
||||
);
|
||||
displayText += segText;
|
||||
if (index < segments.length - 1 && !/\n\s*$/.test(displayText)) {
|
||||
displayText += "\n\n";
|
||||
}
|
||||
|
||||
// 跟踪上一个 segment 的末尾非空白字符，传给下一次 handleWithTimestamp
|
||||
const trimmed = segText.replace(/\s+$/, "");
|
||||
if (trimmed.length > 0) {
|
||||
lastContentChar = trimmed.slice(-1);
|
||||
@ -1166,7 +1169,7 @@ export class RealtimeAsrPipeline {
|
||||
|
||||
// 将已矫正并提交的 segment 渲染到主转录区。pending/online 预览文本保持独立，
|
||||
// 主文本更新后再同步预览区。
|
||||
renderFullText() {
|
||||
renderFullText() {
|
||||
const initialSpeaker =
|
||||
this.segmentRenderStartIndex > 0
|
||||
? this.getSpeakerBeforeIndex(this.segmentRenderStartIndex)
|
||||
@ -1181,12 +1184,76 @@ export class RealtimeAsrPipeline {
|
||||
const mainText = this._deduplicateConsecutivePunctuation(
|
||||
this.getPausedTranscription() + this.recText
|
||||
);
|
||||
this.onMainTextChange(mainText);
|
||||
this.syncRealtimePreview();
|
||||
}
|
||||
// online websocket 返回低延迟片段，只更新预览区；正式 offline 句子统一交给
|
||||
// handleOfflineMessage，避免双通路重复落句。
|
||||
handleOnlineMessage(event) {
|
||||
this.onMainTextChange(mainText);
|
||||
this.syncRealtimePreview();
|
||||
}
|
||||
|
||||
applySpeakerMessage(parsedMessage, messageEpoch) {
|
||||
const timestamp = parsedMessage.timestamp;
|
||||
const messageTimestampMs = this.extractTimestampMs(timestamp);
|
||||
if (this.isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const speaker =
|
||||
(parsedMessage.segments &&
|
||||
parsedMessage.segments[0] &&
|
||||
parsedMessage.segments[0].speaker) ||
|
||||
parsedMessage.speaker ||
|
||||
"SPEAKER_00";
|
||||
|
||||
let didApply = false;
|
||||
for (let i = this.pendingSentenceQueue.length - 1; i >= 0; i--) {
|
||||
const pending = this.pendingSentenceQueue[i];
|
||||
if (!pending || pending.epoch !== messageEpoch || pending.speaker) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
Number.isFinite(messageTimestampMs) &&
|
||||
Number.isFinite(pending.timestampMs) &&
|
||||
pending.timestampMs !== messageTimestampMs
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
pending.speaker = speaker;
|
||||
didApply = true;
|
||||
break;
|
||||
}
|
||||
|
||||
let targetIndex = -1;
|
||||
for (let i = this.segmentList.length - 1; i >= 0; i--) {
|
||||
const segment = this.segmentList[i];
|
||||
if (!segment || segment.epoch !== messageEpoch) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
segment.speaker === null &&
|
||||
Number.isFinite(messageTimestampMs) &&
|
||||
Number.isFinite(segment.timestampMs) &&
|
||||
segment.timestampMs === messageTimestampMs
|
||||
) {
|
||||
targetIndex = i;
|
||||
break;
|
||||
}
|
||||
if (targetIndex === -1 && segment.speaker === null) {
|
||||
targetIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (targetIndex !== -1) {
|
||||
this.segmentList[targetIndex].speaker = speaker;
|
||||
this.renderFullText();
|
||||
return;
|
||||
}
|
||||
|
||||
if (didApply) {
|
||||
this.syncRealtimePreview();
|
||||
}
|
||||
}
|
||||
|
||||
// online websocket 返回低延迟片段，只更新预览区；正式 offline 句子统一交给
|
||||
// handleOfflineMessage，避免双通路重复落句。
|
||||
handleOnlineMessage(event) {
|
||||
if (this.disposed) {
|
||||
return;
|
||||
}
|
||||
@ -1238,10 +1305,12 @@ export class RealtimeAsrPipeline {
|
||||
this._requestCorrection(pendingSentence);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.syncRealtimePreview();
|
||||
}
|
||||
} else if (mode === "offline-speaker") {
|
||||
this.applySpeakerMessage(parsedMessage, messageEpoch);
|
||||
}
|
||||
|
||||
this.syncRealtimePreview();
|
||||
}
|
||||
|
||||
// offline websocket 才负责正式转录路径：2pass-offline 文本进入 LLM 矫正，
|
||||
// offline-speaker 后续只补说话人信息。
|
||||
@ -1287,43 +1356,10 @@ export class RealtimeAsrPipeline {
|
||||
pendingSentence.correctionRequested = true;
|
||||
this.setPendingCorrectionCount(this.pendingCorrectionCount + 1);
|
||||
await this._requestCorrection(pendingSentence);
|
||||
} else if (mode === "offline-speaker") {
|
||||
// 说话人标签可能晚于矫正文本到达。这里只更新 speaker 元数据，
|
||||
// 绝不替换 text，避免已矫正句子跳回 raw ASR 文本。
|
||||
if (this.isStaleRealtimeMessage(messageTimestampMs, messageEpoch)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const speaker =
|
||||
(parsedMessage.segments &&
|
||||
parsedMessage.segments[0] &&
|
||||
parsedMessage.segments[0].speaker) ||
|
||||
"未命名演讲者";
|
||||
|
||||
let targetIndex = -1;
|
||||
for (let i = this.segmentList.length - 1; i >= 0; i--) {
|
||||
const segment = this.segmentList[i];
|
||||
if (!segment || segment.epoch !== messageEpoch) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
segment.speaker === null &&
|
||||
Number.isFinite(messageTimestampMs) &&
|
||||
Number.isFinite(segment.timestampMs) &&
|
||||
segment.timestampMs === messageTimestampMs
|
||||
) {
|
||||
targetIndex = i;
|
||||
break;
|
||||
}
|
||||
if (targetIndex === -1 && segment.speaker === null) {
|
||||
targetIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (targetIndex !== -1) {
|
||||
this.segmentList[targetIndex].speaker = speaker;
|
||||
this.renderFullText();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (mode === "offline-speaker") {
|
||||
// 说话人标签可能早于或晚于矫正文本到达。这里只更新 speaker 元数据，
|
||||
// 绝不替换 text，避免已矫正句子跳回 raw ASR 文本。
|
||||
this.applySpeakerMessage(parsedMessage, messageEpoch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user