1641 lines
49 KiB
Python
1641 lines
49 KiB
Python
"""
|
||
工作流公共工具函数
|
||
提取自 workflow_fault_diagnosis.py 和 workflow_operate.py 的共享代码
|
||
"""
|
||
|
||
from typing import TypedDict, Optional, Dict, Any, List
|
||
from difflib import SequenceMatcher
|
||
from modelsAPI.model_api import OpenaiAPI
|
||
from utils.function_tracker import get_all_callbacks, get_current_stream_handler
|
||
from utils.image_utils import extract_image_paths, normalize_markdown_images
|
||
import asyncio
|
||
import json
|
||
import re
|
||
import random
|
||
import inspect
|
||
import unicodedata
|
||
|
||
|
||
RAG_DEDUP_SIMILARITY_THRESHOLD = 0.86
|
||
RAG_DEDUP_CONTAINMENT_THRESHOLD = 0.92
|
||
RAG_DEDUP_MIN_TEXT_LENGTH = 20
|
||
|
||
|
||
def safe_json_extract(text: str) -> Optional[Any]:
|
||
if not text:
|
||
return None
|
||
cleaned = text.strip()
|
||
cleaned = re.sub(r"```json\s*", "", cleaned, flags=re.IGNORECASE)
|
||
cleaned = re.sub(r"```\s*", "", cleaned)
|
||
m = re.search(r"\[[\s\S]*\]", cleaned)
|
||
if m:
|
||
try:
|
||
return json.loads(m.group(0))
|
||
except Exception:
|
||
pass
|
||
m = re.search(r"\{[\s\S]*\}", cleaned)
|
||
if m:
|
||
try:
|
||
return json.loads(m.group(0))
|
||
except Exception:
|
||
pass
|
||
try:
|
||
return json.loads(cleaned)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
from workflows.history_manager import parse_history, build_history_str, filter_image_urls, init_history # noqa: F401
|
||
|
||
import re
|
||
|
||
|
||
def normalize_latex_spacing(text: str) -> str:
|
||
"""
|
||
规范 Markdown 中的 LaTeX 公式。
|
||
|
||
规则：
|
||
1. 行内公式 $...$
|
||
- 清除公式定界符内部首尾空格
|
||
- 公式前后只要存在非空白字符，就补一个空格
|
||
- ($T$) 会转换为 ( $T$ )
|
||
|
||
2. 段落公式 $$...$$
|
||
- 多行公式完整保留，不会删除结尾的 $$
|
||
- 单行 $$公式$$ 转换为标准多行段落公式
|
||
- Markdown 表格中的 $$公式$$ 不拆行
|
||
|
||
3. 转换：
|
||
- \\(...\\) 转换为 $...$
|
||
- \\[...\\] 转换为 $$...$$
|
||
|
||
4. 不处理：
|
||
- Markdown 围栏代码块
|
||
- Markdown 行内代码
|
||
- HTML code、pre、script、style
|
||
- HTML 注释
|
||
- Markdown 链接地址
|
||
- 转义美元符号 \\$
|
||
"""
|
||
if not isinstance(text, str):
|
||
raise TypeError(
|
||
f"text 必须是 str，实际类型为 {type(text).__name__}"
|
||
)
|
||
|
||
if not text:
|
||
return text
|
||
|
||
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
||
|
||
protected_parts: list[str] = []
|
||
|
||
def protect(content: str) -> str:
|
||
token = (
|
||
f"\uE000LATEX_PROTECTED_"
|
||
f"{len(protected_parts)}"
|
||
f"\uE001"
|
||
)
|
||
protected_parts.append(content)
|
||
return token
|
||
|
||
def restore(content: str) -> str:
|
||
# 保护内容可能再次包含更早生成的占位符，必须按后进先出恢复。
|
||
for index in reversed(range(len(protected_parts))):
|
||
original = protected_parts[index]
|
||
token = f"\uE000LATEX_PROTECTED_{index}\uE001"
|
||
content = content.replace(token, original)
|
||
|
||
return content
|
||
|
||
def is_escaped(content: str, position: int) -> bool:
|
||
"""
|
||
判断 content[position] 是否被奇数个反斜杠转义。
|
||
"""
|
||
slash_count = 0
|
||
position -= 1
|
||
|
||
while position >= 0 and content[position] == "\\":
|
||
slash_count += 1
|
||
position -= 1
|
||
|
||
return slash_count % 2 == 1
|
||
|
||
# ================================================================
|
||
# 1. 保护 Markdown 围栏代码块
|
||
# ================================================================
|
||
|
||
lines = text.splitlines(keepends=True)
|
||
protected_lines: list[str] = []
|
||
line_index = 0
|
||
|
||
while line_index < len(lines):
|
||
line = lines[line_index]
|
||
|
||
opening_match = re.match(
|
||
r"^[ \t]{0,3}(`{3,}|~{3,})",
|
||
line,
|
||
)
|
||
|
||
if not opening_match:
|
||
protected_lines.append(line)
|
||
line_index += 1
|
||
continue
|
||
|
||
opening_fence = opening_match.group(1)
|
||
fence_character = opening_fence[0]
|
||
fence_length = len(opening_fence)
|
||
|
||
closing_pattern = re.compile(
|
||
rf"^[ \t]{{0,3}}"
|
||
rf"{re.escape(fence_character)}"
|
||
rf"{{{fence_length},}}"
|
||
rf"[ \t]*(?:\n|$)"
|
||
)
|
||
|
||
block_lines = [line]
|
||
line_index += 1
|
||
|
||
while line_index < len(lines):
|
||
current_line = lines[line_index]
|
||
block_lines.append(current_line)
|
||
line_index += 1
|
||
|
||
if closing_pattern.match(current_line):
|
||
break
|
||
|
||
protected_lines.append(
|
||
protect("".join(block_lines))
|
||
)
|
||
|
||
text = "".join(protected_lines)
|
||
|
||
# ================================================================
|
||
# 2. 保护 HTML 代码区域和注释
|
||
# ================================================================
|
||
|
||
text = re.sub(
|
||
r"(?is)"
|
||
r"<(?P<tag>pre|code|script|style)\b[^>]*>"
|
||
r".*?"
|
||
r"</(?P=tag)\s*>",
|
||
lambda match: protect(match.group(0)),
|
||
text,
|
||
)
|
||
|
||
text = re.sub(
|
||
r"(?s)<!--.*?-->",
|
||
lambda match: protect(match.group(0)),
|
||
text,
|
||
)
|
||
|
||
# ================================================================
|
||
# 3. 保护 Markdown 链接地址
|
||
# ================================================================
|
||
|
||
text = re.sub(
|
||
r"(!?\[[^\]\n]*\]\()([^)\n]*)(\))",
|
||
lambda match: (
|
||
match.group(1)
|
||
+ protect(match.group(2))
|
||
+ match.group(3)
|
||
),
|
||
text,
|
||
)
|
||
|
||
text = re.sub(
|
||
r"(?im)"
|
||
r"^([ \t]{0,3}\[[^\]\n]+\]:[ \t]*)"
|
||
r"(\S+(?:[ \t]+.*)?)$",
|
||
lambda match: (
|
||
match.group(1)
|
||
+ protect(match.group(2))
|
||
),
|
||
text,
|
||
)
|
||
|
||
# ================================================================
|
||
# 4. 保护 Markdown 行内代码
|
||
# ================================================================
|
||
|
||
inline_code_result: list[str] = []
|
||
position = 0
|
||
|
||
while position < len(text):
|
||
if text[position] != "`":
|
||
inline_code_result.append(text[position])
|
||
position += 1
|
||
continue
|
||
|
||
delimiter_end = position
|
||
|
||
while (
|
||
delimiter_end < len(text)
|
||
and text[delimiter_end] == "`"
|
||
):
|
||
delimiter_end += 1
|
||
|
||
delimiter = text[position:delimiter_end]
|
||
search_position = delimiter_end
|
||
closing_position = -1
|
||
|
||
while search_position < len(text):
|
||
candidate = text.find(
|
||
delimiter,
|
||
search_position,
|
||
)
|
||
|
||
if candidate < 0:
|
||
break
|
||
|
||
previous_character = (
|
||
text[candidate - 1]
|
||
if candidate > 0
|
||
else ""
|
||
)
|
||
|
||
candidate_end = candidate + len(delimiter)
|
||
|
||
next_character = (
|
||
text[candidate_end]
|
||
if candidate_end < len(text)
|
||
else ""
|
||
)
|
||
|
||
if (
|
||
previous_character != "`"
|
||
and next_character != "`"
|
||
):
|
||
closing_position = candidate
|
||
break
|
||
|
||
search_position = candidate + 1
|
||
|
||
if closing_position < 0:
|
||
inline_code_result.append(delimiter)
|
||
position = delimiter_end
|
||
continue
|
||
|
||
end_position = closing_position + len(delimiter)
|
||
|
||
inline_code_result.append(
|
||
protect(text[position:end_position])
|
||
)
|
||
|
||
position = end_position
|
||
|
||
text = "".join(inline_code_result)
|
||
|
||
# ================================================================
|
||
# 5. 转换其他 LaTeX 公式定界符
|
||
# ================================================================
|
||
|
||
# \( x + y \) -> $x + y$
|
||
text = re.sub(
|
||
r"(?<!\\)\\\((.*?)(?<!\\)\\\)",
|
||
lambda match: (
|
||
"$" + match.group(1).strip() + "$"
|
||
),
|
||
text,
|
||
)
|
||
|
||
# \[ x + y \] -> $$x + y$$
|
||
text = re.sub(
|
||
r"(?s)(?<!\\)\\\[(.*?)(?<!\\)\\\]",
|
||
lambda match: (
|
||
"$$" + match.group(1).strip() + "$$"
|
||
),
|
||
text,
|
||
)
|
||
|
||
# ================================================================
|
||
# 6. 将复杂、较长的 $...$ 行内公式提升为段落公式
|
||
# ================================================================
|
||
|
||
def should_promote_inline_formula(formula_body: str) -> bool:
|
||
compact_body = re.sub(r"\s+", "", formula_body)
|
||
if len(compact_body) >= 46:
|
||
return True
|
||
|
||
has_relation = bool(
|
||
re.search(
|
||
r"(?<!\\)[=<>]"
|
||
r"|\\(?:leq?|geq?|approx|sim|equiv|propto|rightarrow|mapsto)\b",
|
||
formula_body,
|
||
)
|
||
)
|
||
has_large_operator = bool(
|
||
re.search(
|
||
r"\\(?:sum|int|iint|iiint|prod|lim|begin|cases|matrix)\b",
|
||
formula_body,
|
||
)
|
||
)
|
||
has_structured_expression = bool(
|
||
re.search(
|
||
r"\\(?:d?frac|sqrt|left|right|overbrace|underbrace)\b",
|
||
formula_body,
|
||
)
|
||
)
|
||
|
||
return (
|
||
has_large_operator
|
||
or (
|
||
has_relation
|
||
and has_structured_expression
|
||
and len(compact_body) >= 24
|
||
)
|
||
)
|
||
|
||
promoted_result: list[str] = []
|
||
position = 0
|
||
|
||
while position < len(text):
|
||
is_inline_opening = (
|
||
text[position] == "$"
|
||
and not is_escaped(text, position)
|
||
and not text.startswith("$$", position)
|
||
and not (
|
||
position > 0
|
||
and text[position - 1] == "$"
|
||
)
|
||
)
|
||
|
||
if not is_inline_opening:
|
||
promoted_result.append(text[position])
|
||
position += 1
|
||
continue
|
||
|
||
opening_position = position
|
||
closing_position = -1
|
||
search_position = position + 1
|
||
|
||
while search_position < len(text):
|
||
candidate = text.find("$", search_position)
|
||
if candidate < 0:
|
||
break
|
||
if "\n" in text[opening_position + 1:candidate]:
|
||
break
|
||
if is_escaped(text, candidate):
|
||
search_position = candidate + 1
|
||
continue
|
||
if text.startswith("$$", candidate):
|
||
search_position = candidate + 2
|
||
continue
|
||
closing_position = candidate
|
||
break
|
||
|
||
if closing_position < 0:
|
||
promoted_result.append("$")
|
||
position += 1
|
||
continue
|
||
|
||
formula_body = text[
|
||
opening_position + 1:closing_position
|
||
].strip()
|
||
line_start = text.rfind("\n", 0, opening_position) + 1
|
||
line_end = text.find("\n", closing_position + 1)
|
||
if line_end < 0:
|
||
line_end = len(text)
|
||
stripped_line = text[line_start:line_end].strip()
|
||
is_markdown_table_row = (
|
||
stripped_line.startswith("|")
|
||
and stripped_line.endswith("|")
|
||
)
|
||
|
||
if (
|
||
not formula_body
|
||
or is_markdown_table_row
|
||
or not should_promote_inline_formula(formula_body)
|
||
):
|
||
promoted_result.append(
|
||
text[opening_position:closing_position + 1]
|
||
)
|
||
position = closing_position + 1
|
||
continue
|
||
|
||
before_text = "".join(promoted_result)
|
||
if before_text and not before_text.endswith("\n\n"):
|
||
before_text = before_text.rstrip(" \t\n") + "\n\n"
|
||
promoted_result = [before_text] if before_text else []
|
||
promoted_result.append(
|
||
"$$\n" + formula_body + "\n$$"
|
||
)
|
||
|
||
position = closing_position + 1
|
||
while position < len(text) and text[position] in " \t":
|
||
position += 1
|
||
|
||
# 独立公式后的句号不再单独占一段；逗号和冒号仍保留语义。
|
||
if position < len(text) and text[position] in "。.":
|
||
position += 1
|
||
|
||
if position < len(text):
|
||
while position < len(text) and text[position] == "\n":
|
||
position += 1
|
||
promoted_result.append("\n\n")
|
||
|
||
text = "".join(promoted_result)
|
||
|
||
# ================================================================
|
||
# 7. 完整提取并保护 $$...$$ 段落公式
|
||
# ================================================================
|
||
|
||
display_result: list[str] = []
|
||
position = 0
|
||
|
||
while position < len(text):
|
||
if not (
|
||
text.startswith("$$", position)
|
||
and not is_escaped(text, position)
|
||
):
|
||
display_result.append(text[position])
|
||
position += 1
|
||
continue
|
||
|
||
opening_position = position
|
||
search_position = position + 2
|
||
closing_position = -1
|
||
|
||
while search_position < len(text):
|
||
candidate = text.find(
|
||
"$$",
|
||
search_position,
|
||
)
|
||
|
||
if candidate < 0:
|
||
break
|
||
|
||
if not is_escaped(text, candidate):
|
||
closing_position = candidate
|
||
break
|
||
|
||
search_position = candidate + 2
|
||
|
||
# 未找到闭合 $$，剩余内容原样保留。
|
||
if closing_position < 0:
|
||
display_result.append(
|
||
text[opening_position:]
|
||
)
|
||
position = len(text)
|
||
break
|
||
|
||
complete_formula = text[
|
||
opening_position:closing_position + 2
|
||
]
|
||
|
||
formula_body = text[
|
||
opening_position + 2:closing_position
|
||
]
|
||
|
||
# ------------------------------------------------------------
|
||
# 多行段落公式也统一重建，并确保公式块前后各有一个空行。
|
||
# 相邻公式若只有一个换行，部分 Markdown/KaTeX 解析器会把它们
|
||
# 合并到同一个显示块中，导致两条公式横向排在同一行。
|
||
# ------------------------------------------------------------
|
||
if "\n" in complete_formula:
|
||
formula_lines = [
|
||
line.strip()
|
||
for line in formula_body.splitlines()
|
||
if line.strip()
|
||
]
|
||
contains_multiline_environment = bool(
|
||
re.search(
|
||
r"\\(?:begin|end|substack)\b"
|
||
r"|\\\\",
|
||
formula_body,
|
||
)
|
||
)
|
||
lines_are_independent_equations = (
|
||
len(formula_lines) > 1
|
||
and not contains_multiline_environment
|
||
and all(
|
||
re.search(
|
||
r"(?<!\\)[=<>]"
|
||
r"|\\(?:leq?|geq?|approx|sim|equiv|propto)\b",
|
||
line,
|
||
)
|
||
for line in formula_lines
|
||
)
|
||
)
|
||
|
||
if lines_are_independent_equations:
|
||
normalized_formula = "\n\n".join(
|
||
"$$\n" + line + "\n$$"
|
||
for line in formula_lines
|
||
)
|
||
else:
|
||
normalized_formula = (
|
||
"$$\n"
|
||
+ formula_body.strip()
|
||
+ "\n$$"
|
||
)
|
||
before_text = "".join(display_result)
|
||
if before_text and not before_text.endswith("\n\n"):
|
||
before_text = before_text.rstrip(" \t\n") + "\n\n"
|
||
display_result = [before_text] if before_text else []
|
||
display_result.append(
|
||
protect(normalized_formula)
|
||
)
|
||
position = closing_position + 2
|
||
|
||
while (
|
||
position < len(text)
|
||
and text[position] in " \t"
|
||
):
|
||
position += 1
|
||
|
||
if position < len(text):
|
||
while (
|
||
position < len(text)
|
||
and text[position] == "\n"
|
||
):
|
||
position += 1
|
||
display_result.append("\n\n")
|
||
continue
|
||
|
||
# ------------------------------------------------------------
|
||
# 判断公式是否位于 Markdown 表格内。
|
||
# 表格中的公式不能强制拆成多行。
|
||
# ------------------------------------------------------------
|
||
|
||
line_start = text.rfind(
|
||
"\n",
|
||
0,
|
||
opening_position,
|
||
) + 1
|
||
|
||
line_end = text.find(
|
||
"\n",
|
||
closing_position + 2,
|
||
)
|
||
|
||
if line_end < 0:
|
||
line_end = len(text)
|
||
|
||
current_line = text[line_start:line_end]
|
||
clean_body = formula_body.strip()
|
||
|
||
stripped_line = current_line.strip()
|
||
is_markdown_table_row = (
|
||
stripped_line.startswith("|")
|
||
and stripped_line.endswith("|")
|
||
)
|
||
|
||
if is_markdown_table_row:
|
||
display_result.append(
|
||
protect(
|
||
"$$" + clean_body + "$$"
|
||
)
|
||
)
|
||
|
||
position = closing_position + 2
|
||
continue
|
||
|
||
# ------------------------------------------------------------
|
||
# 单行段落公式转换成标准多行格式。
|
||
#
|
||
# $$R = K$$
|
||
#
|
||
# 转换为：
|
||
#
|
||
# $$
|
||
# R = K
|
||
# $$
|
||
# ------------------------------------------------------------
|
||
|
||
normalized_formula = (
|
||
"$$\n"
|
||
+ clean_body
|
||
+ "\n$$"
|
||
)
|
||
|
||
before_text = "".join(display_result)
|
||
|
||
# 公式前存在普通正文时，确保前面有空行。
|
||
if before_text and not before_text.endswith("\n\n"):
|
||
before_text = before_text.rstrip(" \t\n") + "\n\n"
|
||
|
||
display_result = [before_text] if before_text else []
|
||
|
||
display_result.append(
|
||
protect(normalized_formula)
|
||
)
|
||
|
||
position = closing_position + 2
|
||
|
||
# 清理公式后已有的普通空格。
|
||
while (
|
||
position < len(text)
|
||
and text[position] in " \t"
|
||
):
|
||
position += 1
|
||
|
||
# 后面还有内容时，确保公式后有空行。
|
||
if position < len(text):
|
||
if text[position] == "\n":
|
||
while (
|
||
position < len(text)
|
||
and text[position] == "\n"
|
||
):
|
||
position += 1
|
||
|
||
display_result.append("\n\n")
|
||
|
||
text = "".join(display_result)
|
||
|
||
# ================================================================
|
||
# 8. 规范 $...$ 行内公式
|
||
# ================================================================
|
||
|
||
inline_result: list[str] = []
|
||
position = 0
|
||
|
||
while position < len(text):
|
||
is_inline_opening = (
|
||
text[position] == "$"
|
||
and not is_escaped(text, position)
|
||
and not text.startswith("$$", position)
|
||
and not (
|
||
position > 0
|
||
and text[position - 1] == "$"
|
||
)
|
||
)
|
||
|
||
if not is_inline_opening:
|
||
inline_result.append(text[position])
|
||
position += 1
|
||
continue
|
||
|
||
opening_position = position
|
||
|
||
# 避免把 $100 识别成行内公式开始。
|
||
next_opening_character = (
|
||
text[position + 1]
|
||
if position + 1 < len(text)
|
||
else ""
|
||
)
|
||
|
||
if next_opening_character.isdigit():
|
||
inline_result.append("$")
|
||
position += 1
|
||
continue
|
||
|
||
search_position = position + 1
|
||
closing_position = -1
|
||
|
||
while search_position < len(text):
|
||
candidate = text.find(
|
||
"$",
|
||
search_position,
|
||
)
|
||
|
||
if candidate < 0:
|
||
break
|
||
|
||
# 行内公式不能跨行。
|
||
if "\n" in text[
|
||
opening_position + 1:candidate
|
||
]:
|
||
break
|
||
|
||
if is_escaped(text, candidate):
|
||
search_position = candidate + 1
|
||
continue
|
||
|
||
# 跳过属于 $$ 的美元符号。
|
||
if text.startswith("$$", candidate):
|
||
search_position = candidate + 2
|
||
continue
|
||
|
||
closing_position = candidate
|
||
break
|
||
|
||
# 未找到闭合符时原样保留。
|
||
if closing_position < 0:
|
||
inline_result.append("$")
|
||
position += 1
|
||
continue
|
||
|
||
formula_body = text[
|
||
opening_position + 1:closing_position
|
||
].strip()
|
||
|
||
# 空公式不处理。
|
||
if not formula_body:
|
||
inline_result.append(
|
||
text[
|
||
opening_position:closing_position + 1
|
||
]
|
||
)
|
||
|
||
position = closing_position + 1
|
||
continue
|
||
|
||
# 折叠公式内部普通空格，不修改 LaTeX 命令。
|
||
formula_body = re.sub(
|
||
r"[ \t]+",
|
||
" ",
|
||
formula_body,
|
||
)
|
||
|
||
previous_character = ""
|
||
|
||
for part in reversed(inline_result):
|
||
if part:
|
||
previous_character = part[-1]
|
||
break
|
||
|
||
next_character = (
|
||
text[closing_position + 1]
|
||
if closing_position + 1 < len(text)
|
||
else ""
|
||
)
|
||
|
||
# 公式前只要存在非空白字符，就补空格。
|
||
#
|
||
# 参数$T$ -> 参数 $T$
|
||
# ($T$) -> ( $T$
|
||
# （$T$） -> （ $T$
|
||
if (
|
||
previous_character
|
||
and not previous_character.isspace()
|
||
):
|
||
inline_result.append(" ")
|
||
|
||
inline_result.append(
|
||
"$" + formula_body + "$"
|
||
)
|
||
|
||
# 公式后只要存在非空白字符，就补空格。
|
||
#
|
||
# $T$参数 -> $T$ 参数
|
||
# ($T$) -> $T$ )
|
||
# （$T$） -> $T$ ）
|
||
if (
|
||
next_character
|
||
and not next_character.isspace()
|
||
):
|
||
inline_result.append(" ")
|
||
|
||
position = closing_position + 1
|
||
|
||
return restore("".join(inline_result))
|
||
|
||
|
||
def convert_display_math_for_frontend(text: str) -> str:
|
||
"""将段落公式改为 16000 前端能正确识别的块级定界符。"""
|
||
return re.sub(
|
||
r"(?ms)^[ \t]*\$\$[ \t]*\n"
|
||
r"(.*?)"
|
||
r"\n[ \t]*\$\$[ \t]*$",
|
||
lambda match: (
|
||
"\\[\n"
|
||
+ match.group(1).strip()
|
||
+ "\n\\]"
|
||
),
|
||
text,
|
||
)
|
||
|
||
|
||
def remove_duplicate_content(text: str) -> str:
|
||
"""
|
||
删除普通 Markdown 正文中的完全重复行。
|
||
|
||
不对以下内容进行去重：
|
||
- $$ ... $$ 段落公式
|
||
- \\[ ... \\] 段落公式
|
||
- equation、align、gather 等 LaTeX 环境
|
||
- Markdown 围栏代码块
|
||
- Markdown 缩进代码块
|
||
- Markdown 标题、列表、引用、表格等结构行
|
||
|
||
这样不会删除段落公式末尾的第二个 $$。
|
||
"""
|
||
if not text or len(text) < 10:
|
||
return text
|
||
|
||
if not isinstance(text, str):
|
||
raise TypeError(
|
||
f"text 必须是 str，实际类型为 {type(text).__name__}"
|
||
)
|
||
|
||
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
||
lines = text.split("\n")
|
||
|
||
result: list[str] = []
|
||
seen_segments: set[str] = set()
|
||
|
||
in_code_block = False
|
||
code_fence_character = ""
|
||
code_fence_length = 0
|
||
|
||
in_dollar_math = False
|
||
in_bracket_math = False
|
||
|
||
latex_environment_stack: list[str] = []
|
||
|
||
display_environments = {
|
||
"equation",
|
||
"equation*",
|
||
"align",
|
||
"align*",
|
||
"alignat",
|
||
"alignat*",
|
||
"gather",
|
||
"gather*",
|
||
"multline",
|
||
"multline*",
|
||
"flalign",
|
||
"flalign*",
|
||
"eqnarray",
|
||
"eqnarray*",
|
||
"displaymath",
|
||
"math",
|
||
}
|
||
|
||
def is_escaped(value: str, position: int) -> bool:
|
||
slash_count = 0
|
||
position -= 1
|
||
|
||
while position >= 0 and value[position] == "\\":
|
||
slash_count += 1
|
||
position -= 1
|
||
|
||
return slash_count % 2 == 1
|
||
|
||
def count_unescaped_double_dollars(value: str) -> int:
|
||
"""
|
||
统计一行中未转义的 $$ 数量。
|
||
"""
|
||
count = 0
|
||
position = 0
|
||
|
||
while position < len(value) - 1:
|
||
if (
|
||
value[position:position + 2] == "$$"
|
||
and not is_escaped(value, position)
|
||
):
|
||
count += 1
|
||
position += 2
|
||
else:
|
||
position += 1
|
||
|
||
return count
|
||
|
||
def is_markdown_structure_line(value: str) -> bool:
|
||
"""
|
||
Markdown 结构行不参与去重，避免破坏语义。
|
||
"""
|
||
stripped = value.strip()
|
||
|
||
if not stripped:
|
||
return True
|
||
|
||
patterns = (
|
||
# 标题
|
||
r"^#{1,6}\s+",
|
||
|
||
# 无序列表
|
||
r"^[-+*]\s+",
|
||
|
||
# 有序列表
|
||
r"^\d+[.)]\s+",
|
||
|
||
# 引用
|
||
r"^>\s*",
|
||
|
||
# 任务列表
|
||
r"^[-+*]\s+\[[ xX]\]\s+",
|
||
|
||
# 水平分隔线
|
||
r"^(?:-{3,}|\*{3,}|_{3,})$",
|
||
|
||
# 表格分隔行
|
||
r"^\|?\s*:?-{3,}:?"
|
||
r"(?:\s*\|\s*:?-{3,}:?)+\s*\|?$",
|
||
|
||
# 普通表格行
|
||
r"^\|.*\|$",
|
||
|
||
# HTML 标签
|
||
r"^</?[A-Za-z][^>]*>$",
|
||
|
||
# LaTeX 环境边界
|
||
r"^\\(?:begin|end)\{[^}]+\}",
|
||
|
||
# 公式定界符
|
||
r"^(?:\$\$|\\\[|\\\])$",
|
||
|
||
# Markdown 链接引用定义
|
||
r"^\[[^\]]+\]:\s*\S+",
|
||
)
|
||
|
||
return any(
|
||
re.match(pattern, stripped)
|
||
for pattern in patterns
|
||
)
|
||
|
||
for line in lines:
|
||
line = line.rstrip()
|
||
stripped = line.strip()
|
||
|
||
# ============================================================
|
||
# 1. Markdown 围栏代码块
|
||
# ============================================================
|
||
|
||
fence_match = re.match(
|
||
r"^[ \t]{0,3}(`{3,}|~{3,})",
|
||
line,
|
||
)
|
||
|
||
if not in_code_block and fence_match:
|
||
fence = fence_match.group(1)
|
||
|
||
in_code_block = True
|
||
code_fence_character = fence[0]
|
||
code_fence_length = len(fence)
|
||
|
||
result.append(line)
|
||
continue
|
||
|
||
if in_code_block:
|
||
result.append(line)
|
||
|
||
closing_pattern = re.compile(
|
||
rf"^[ \t]{{0,3}}"
|
||
rf"{re.escape(code_fence_character)}"
|
||
rf"{{{code_fence_length},}}"
|
||
rf"[ \t]*$"
|
||
)
|
||
|
||
if closing_pattern.match(line):
|
||
in_code_block = False
|
||
code_fence_character = ""
|
||
code_fence_length = 0
|
||
|
||
continue
|
||
|
||
# Markdown 缩进代码块不参与去重。
|
||
if re.match(r"^(?:\t| {4})", line):
|
||
result.append(line)
|
||
continue
|
||
|
||
# ============================================================
|
||
# 2. \[ ... \] 段落公式
|
||
# ============================================================
|
||
|
||
if in_bracket_math:
|
||
result.append(line)
|
||
|
||
if re.search(r"(?<!\\)\\\]", line):
|
||
in_bracket_math = False
|
||
|
||
continue
|
||
|
||
bracket_open_count = len(
|
||
re.findall(r"(?<!\\)\\\[", line)
|
||
)
|
||
|
||
bracket_close_count = len(
|
||
re.findall(r"(?<!\\)\\\]", line)
|
||
)
|
||
|
||
if bracket_open_count:
|
||
result.append(line)
|
||
|
||
if bracket_open_count > bracket_close_count:
|
||
in_bracket_math = True
|
||
|
||
continue
|
||
|
||
# ============================================================
|
||
# 3. LaTeX display 环境
|
||
# ============================================================
|
||
|
||
begin_matches = re.findall(
|
||
r"\\begin\{([^}]+)\}",
|
||
line,
|
||
)
|
||
|
||
end_matches = re.findall(
|
||
r"\\end\{([^}]+)\}",
|
||
line,
|
||
)
|
||
|
||
relevant_begins = [
|
||
environment
|
||
for environment in begin_matches
|
||
if environment in display_environments
|
||
]
|
||
|
||
relevant_ends = [
|
||
environment
|
||
for environment in end_matches
|
||
if environment in display_environments
|
||
]
|
||
|
||
if latex_environment_stack:
|
||
result.append(line)
|
||
|
||
for environment in relevant_begins:
|
||
latex_environment_stack.append(environment)
|
||
|
||
for environment in relevant_ends:
|
||
if (
|
||
latex_environment_stack
|
||
and latex_environment_stack[-1]
|
||
== environment
|
||
):
|
||
latex_environment_stack.pop()
|
||
elif environment in latex_environment_stack:
|
||
latex_environment_stack.remove(environment)
|
||
|
||
continue
|
||
|
||
if relevant_begins:
|
||
result.append(line)
|
||
|
||
for environment in relevant_begins:
|
||
latex_environment_stack.append(environment)
|
||
|
||
for environment in relevant_ends:
|
||
if (
|
||
latex_environment_stack
|
||
and latex_environment_stack[-1]
|
||
== environment
|
||
):
|
||
latex_environment_stack.pop()
|
||
elif environment in latex_environment_stack:
|
||
latex_environment_stack.remove(environment)
|
||
|
||
continue
|
||
|
||
# ============================================================
|
||
# 4. $$ ... $$ 段落公式
|
||
# ============================================================
|
||
|
||
double_dollar_count = (
|
||
count_unescaped_double_dollars(line)
|
||
)
|
||
|
||
if in_dollar_math:
|
||
# 公式块中的内容全部保留，包括最后一行的 $$。
|
||
result.append(line)
|
||
|
||
if double_dollar_count % 2 == 1:
|
||
in_dollar_math = False
|
||
|
||
continue
|
||
|
||
if double_dollar_count > 0:
|
||
# 含有 $$ 的行永远不参与去重。
|
||
result.append(line)
|
||
|
||
# 奇数个 $$ 代表开启了跨行段落公式。
|
||
if double_dollar_count % 2 == 1:
|
||
in_dollar_math = True
|
||
|
||
continue
|
||
|
||
# ============================================================
|
||
# 5. 空行和 Markdown 结构行
|
||
# ============================================================
|
||
|
||
if not stripped:
|
||
result.append(line)
|
||
continue
|
||
|
||
if is_markdown_structure_line(line):
|
||
result.append(line)
|
||
continue
|
||
|
||
# ============================================================
|
||
# 6. 仅对普通正文行去重
|
||
# ============================================================
|
||
|
||
normalized_line = re.sub(
|
||
r"\s+",
|
||
" ",
|
||
stripped.lower(),
|
||
)
|
||
|
||
if normalized_line in seen_segments:
|
||
continue
|
||
|
||
seen_segments.add(normalized_line)
|
||
result.append(line)
|
||
|
||
return "\n".join(result)
|
||
|
||
|
||
|
||
|
||
# 推荐调用顺序：
|
||
#
|
||
# text = remove_duplicate_content(text)
|
||
# text = normalize_latex_spacing(text)
|
||
|
||
|
||
|
||
def _extract_rag_text(item: Any) -> str:
|
||
if isinstance(item, dict):
|
||
return str(item.get("text") or item.get("content") or "")
|
||
if isinstance(item, str):
|
||
return item
|
||
return ""
|
||
|
||
|
||
def _extract_rag_score(item: Any) -> float:
|
||
if not isinstance(item, dict):
|
||
return 0.0
|
||
try:
|
||
return float(item.get("score") or item.get("socre") or 0.0)
|
||
except (TypeError, ValueError):
|
||
return 0.0
|
||
|
||
|
||
def _normalize_rag_text_for_dedupe(text: Any) -> str:
|
||
value = unicodedata.normalize("NFKC", str(text or ""))
|
||
value = re.sub(r"!\[[^\]]*\]\([^)]+\)", "", value)
|
||
value = re.sub(r"\[([^\]]*)\]\([^)]+\)", r"\1", value)
|
||
value = re.sub(r"https?://\S+", "", value, flags=re.IGNORECASE)
|
||
value = re.sub(r"[/\\]?picture[/\\]\S+", "", value, flags=re.IGNORECASE)
|
||
value = re.sub(r"\s+", "", value).lower()
|
||
return "".join(ch for ch in value if ch.isalnum())
|
||
|
||
|
||
def _char_ngrams(text: str, n: int) -> set:
|
||
if len(text) <= n:
|
||
return {text} if text else set()
|
||
return {text[i:i + n] for i in range(len(text) - n + 1)}
|
||
|
||
|
||
def _rag_texts_are_similar(text_a: str, text_b: str, threshold: float) -> bool:
|
||
if not text_a or not text_b:
|
||
return False
|
||
if text_a == text_b:
|
||
return True
|
||
|
||
min_len = min(len(text_a), len(text_b))
|
||
max_len = max(len(text_a), len(text_b))
|
||
if min_len < 8:
|
||
return False
|
||
|
||
if min_len >= RAG_DEDUP_MIN_TEXT_LENGTH and (text_a in text_b or text_b in text_a):
|
||
return (min_len / max_len) >= RAG_DEDUP_CONTAINMENT_THRESHOLD
|
||
|
||
if min_len < RAG_DEDUP_MIN_TEXT_LENGTH:
|
||
return SequenceMatcher(None, text_a, text_b).ratio() >= 0.95
|
||
|
||
ngram_size = 3 if min_len >= 30 else 2
|
||
grams_a = _char_ngrams(text_a, ngram_size)
|
||
grams_b = _char_ngrams(text_b, ngram_size)
|
||
if not grams_a or not grams_b:
|
||
return False
|
||
|
||
jaccard = len(grams_a & grams_b) / len(grams_a | grams_b)
|
||
if jaccard >= threshold:
|
||
return True
|
||
|
||
return SequenceMatcher(None, text_a, text_b).ratio() >= threshold
|
||
|
||
|
||
def dedupe_rag_results(
|
||
items: Any,
|
||
limit: Optional[int] = None,
|
||
similarity_threshold: float = RAG_DEDUP_SIMILARITY_THRESHOLD,
|
||
) -> Any:
|
||
"""
|
||
Deduplicate RAG result chunks while preserving the original item shape.
|
||
|
||
Higher-score items are kept first. Graph results are intentionally not handled
|
||
here; callers should pass only text RAG chunks.
|
||
"""
|
||
if not isinstance(items, list):
|
||
return items
|
||
if len(items) < 2:
|
||
return items[:limit] if limit else items
|
||
|
||
prepared = []
|
||
for idx, item in enumerate(items):
|
||
prepared.append({
|
||
"item": item,
|
||
"index": idx,
|
||
"score": _extract_rag_score(item),
|
||
"normalized_text": _normalize_rag_text_for_dedupe(_extract_rag_text(item)),
|
||
})
|
||
|
||
prepared.sort(key=lambda entry: (-entry["score"], entry["index"]))
|
||
|
||
kept = []
|
||
for entry in prepared:
|
||
current_text = entry["normalized_text"]
|
||
is_duplicate = False
|
||
if current_text:
|
||
for kept_entry in kept:
|
||
if _rag_texts_are_similar(
|
||
current_text,
|
||
kept_entry["normalized_text"],
|
||
similarity_threshold,
|
||
):
|
||
is_duplicate = True
|
||
break
|
||
if not is_duplicate:
|
||
kept.append(entry)
|
||
|
||
deduped = [entry["item"] for entry in kept]
|
||
return deduped[:limit] if limit else deduped
|
||
|
||
|
||
def convert_rag_result(rag_result):
|
||
if not rag_result or not rag_result.get("success", False):
|
||
return []
|
||
source_citation = rag_result.get("sourceCitation", {})
|
||
flat_results = []
|
||
for resource, items in source_citation.items():
|
||
for item in items:
|
||
flat_results.append(item)
|
||
return flat_results
|
||
|
||
|
||
def calculate_info_completeness(weights: Dict[str, float], values: Dict[str, str]) -> float:
|
||
score = 0.0
|
||
for field, weight in weights.items():
|
||
value = values.get(field, "") or ""
|
||
if value and str(value).strip():
|
||
score += weight
|
||
return round(score, 2)
|
||
|
||
|
||
def emit_callback_event(title_options: List[str], details: str):
|
||
start_event = {"type": "function_execution", "title": random.choice(title_options), "details": details}
|
||
for callback in get_all_callbacks():
|
||
try:
|
||
callback(start_event)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
async def resolve_ship_number_for_workflow(
|
||
ship_number: str,
|
||
context: str = "workflow",
|
||
) -> tuple[str, Dict[str, Any]]:
|
||
"""
|
||
工作流内舷号/舰名标准化。
|
||
|
||
能映射到具体舷号时返回映射后的舷号；失败或未匹配时返回原值。
|
||
"""
|
||
original_ship_number = str(ship_number or "").strip()
|
||
if not original_ship_number:
|
||
return original_ship_number, {}
|
||
|
||
try:
|
||
from utils.ship_number_search import smart_ship_number_mapping
|
||
|
||
matched_ship_number, matched_model, all_numbers, match_type = await smart_ship_number_mapping(
|
||
original_ship_number
|
||
)
|
||
result = {
|
||
"matched_ship_number": matched_ship_number,
|
||
"matched_model": matched_model,
|
||
"all_numbers": all_numbers,
|
||
"match_type": match_type,
|
||
}
|
||
if match_type in ["exact_number", "exact_name", "embedding"] and matched_ship_number:
|
||
mapped_ship_number = str(matched_ship_number).strip()
|
||
if mapped_ship_number and mapped_ship_number != original_ship_number:
|
||
print(
|
||
f"[{context}] 舷号映射: '{original_ship_number}' -> "
|
||
f"'{mapped_ship_number}'，匹配类型={match_type}"
|
||
)
|
||
return mapped_ship_number or original_ship_number, result
|
||
|
||
print(f"[{context}] 舷号未映射，继续使用原值: {original_ship_number}")
|
||
return original_ship_number, result
|
||
except Exception as e:
|
||
print(f"[{context}] 舷号映射异常，继续使用原值: {original_ship_number}; 错误: {str(e)}")
|
||
return original_ship_number, {}
|
||
|
||
|
||
async def resolve_ship_scope_for_workflow(
|
||
ship_number: str,
|
||
context: str = "workflow",
|
||
) -> tuple[str, List[str], Dict[str, Any]]:
|
||
"""
|
||
Resolve user ship input into a display hull number and a hull-number scope.
|
||
|
||
Hull number and ship-name matches collapse to one hull. Model matches keep the
|
||
user's original value for display but return every hull number under that model
|
||
so shared device normalization and system lookup can search the whole scope.
|
||
"""
|
||
original_ship_number = str(ship_number or "").strip()
|
||
if not original_ship_number:
|
||
return original_ship_number, [], {}
|
||
|
||
try:
|
||
from utils.ship_number_search import smart_ship_number_mapping
|
||
|
||
matched_ship_number, matched_model, all_numbers, match_type = await smart_ship_number_mapping(
|
||
original_ship_number
|
||
)
|
||
scope_numbers = [str(num).strip() for num in (all_numbers or []) if str(num or "").strip()]
|
||
result = {
|
||
"matched_ship_number": matched_ship_number,
|
||
"matched_model": matched_model,
|
||
"all_numbers": scope_numbers,
|
||
"match_type": match_type,
|
||
}
|
||
|
||
if match_type in ["exact_number", "exact_name", "embedding"] and matched_ship_number:
|
||
mapped_ship_number = str(matched_ship_number).strip()
|
||
return mapped_ship_number or original_ship_number, [mapped_ship_number], result
|
||
|
||
if match_type == "exact_model" and scope_numbers:
|
||
print(
|
||
f"[{context}] ship model scope resolved: '{original_ship_number}' -> "
|
||
f"{matched_model}, hull_numbers={scope_numbers}"
|
||
)
|
||
return original_ship_number, scope_numbers, result
|
||
|
||
print(f"[{context}] ship scope not resolved, continuing with original value: {original_ship_number}")
|
||
return original_ship_number, [], result
|
||
except Exception as e:
|
||
print(f"[{context}] ship scope resolve failed, continuing with original value: {original_ship_number}; error: {str(e)}")
|
||
return original_ship_number, [], {}
|
||
|
||
|
||
async def normalize_device_name_for_workflow(
|
||
device_name: str,
|
||
ship_number: str = "",
|
||
ship_numbers: Optional[List[str]] = None,
|
||
context: str = "workflow",
|
||
) -> tuple[str, Dict[str, Any]]:
|
||
"""
|
||
工作流内设备名称标准化。
|
||
|
||
标准化失败时返回原设备名，避免检索流程被 Neo4j 或 embedding 服务异常阻断。
|
||
"""
|
||
original_device_name = str(device_name or "").strip()
|
||
if not original_device_name:
|
||
return original_device_name, {}
|
||
|
||
try:
|
||
from tools.graph_tools import normalize_device_name_by_ship_graph
|
||
|
||
print(
|
||
f"[{context}] 开始设备名称标准化: device='{original_device_name}', "
|
||
f"ship_number='{ship_number or '未提供'}'"
|
||
)
|
||
call_kwargs = {}
|
||
if "ship_number" in inspect.signature(normalize_device_name_by_ship_graph).parameters:
|
||
call_kwargs["ship_number"] = str(ship_number or "").strip() or None
|
||
if "ship_numbers" in inspect.signature(normalize_device_name_by_ship_graph).parameters:
|
||
scoped_numbers = [str(num).strip() for num in (ship_numbers or []) if str(num or "").strip()]
|
||
call_kwargs["ship_numbers"] = scoped_numbers or None
|
||
elif ship_number:
|
||
print(f"[{context}] 当前进程加载的设备标准化函数不支持 ship_number，请重启服务加载最新 graph_tools.py")
|
||
|
||
result = await normalize_device_name_by_ship_graph(original_device_name, **call_kwargs)
|
||
if result.get("success", False):
|
||
normalized_device_name = str(result.get("device_name") or original_device_name).strip()
|
||
print(
|
||
f"[{context}] 设备名称标准化完成: '{original_device_name}' -> "
|
||
f"'{normalized_device_name or original_device_name}'，舷号={ship_number or '未提供'}"
|
||
)
|
||
emit_callback_event(
|
||
["🔎 设备名称标准化", "🔧 设备匹配"],
|
||
f"{original_device_name} -> {normalized_device_name or original_device_name}"
|
||
)
|
||
return normalized_device_name or original_device_name, result
|
||
|
||
print(
|
||
f"[{context}] 设备名称标准化失败，继续使用原设备名: {original_device_name}; "
|
||
f"原因: {result.get('error', '未知错误')}"
|
||
)
|
||
return original_device_name, result
|
||
except Exception as e:
|
||
print(f"[{context}] 设备名称标准化异常，继续使用原设备名: {original_device_name}; 错误: {str(e)}")
|
||
return original_device_name, {}
|
||
|
||
|
||
async def resolve_device_system_for_workflow(
|
||
device_name: str,
|
||
ship_number: str = "",
|
||
ship_numbers: Optional[List[str]] = None,
|
||
context: str = "workflow",
|
||
) -> tuple[str, Dict[str, Any]]:
|
||
"""
|
||
工作流内根据设备名称反推所属系统。
|
||
|
||
失败时返回空字符串，调用方可继续执行主流程。
|
||
"""
|
||
device_name = str(device_name or "").strip()
|
||
if not device_name:
|
||
return "", {}
|
||
|
||
try:
|
||
print(
|
||
f"[{context}] 开始设备反推系统: device='{device_name}', "
|
||
f"ship_number='{ship_number or '未提供'}'"
|
||
)
|
||
from tools.graph_tools import find_system_by_device
|
||
|
||
call_kwargs = {"device_name": device_name}
|
||
if "ship_number" in inspect.signature(find_system_by_device).parameters:
|
||
call_kwargs["ship_number"] = str(ship_number or "").strip() or None
|
||
if "ship_numbers" in inspect.signature(find_system_by_device).parameters:
|
||
scoped_numbers = [str(num).strip() for num in (ship_numbers or []) if str(num or "").strip()]
|
||
call_kwargs["ship_numbers"] = scoped_numbers or None
|
||
elif ship_number:
|
||
print(f"[{context}] 当前进程加载的设备反推系统函数不支持 ship_number，请重启服务加载最新 graph_tools.py")
|
||
|
||
result = await find_system_by_device(**call_kwargs)
|
||
if not result.get("success", False):
|
||
print(f"[{context}] 设备反推系统失败: {result.get('error', '未知错误')}")
|
||
return "", result
|
||
|
||
systems = result.get("systems") or []
|
||
first_system = systems[0] if systems else {}
|
||
system_name = ""
|
||
if isinstance(first_system, dict):
|
||
system_name = str(first_system.get("name") or first_system.get("系统名称") or first_system.get("名称") or "").strip()
|
||
|
||
if system_name:
|
||
print(f"[{context}] 设备反推系统完成: {device_name} -> {system_name}")
|
||
emit_callback_event(
|
||
["🔎 设备所属系统识别", "🧭 系统反查"],
|
||
f"{device_name} -> {system_name}"
|
||
)
|
||
else:
|
||
print(f"[{context}] 设备反推系统无结果: {device_name}")
|
||
return system_name, result
|
||
except Exception as e:
|
||
print(f"[{context}] 设备反推系统异常: {str(e)}")
|
||
return "", {}
|
||
|
||
|
||
def build_detail_info(fields: Dict[str, str]) -> str:
|
||
detail_info = ""
|
||
for label, value in fields.items():
|
||
if value:
|
||
detail_info += f"- {label}：{value}\n"
|
||
return detail_info
|
||
|
||
|
||
def append_atlas_section(answer: str, atlas_results: Dict[str, Any]) -> str:
|
||
if not atlas_results:
|
||
return answer
|
||
atlas_section = "\n\n---\n\n**参考图册：**\n"
|
||
for node_name, node_data in atlas_results.items():
|
||
if isinstance(node_data, dict):
|
||
for title, urls in node_data.items():
|
||
if isinstance(urls, list) and urls:
|
||
for url in urls:
|
||
atlas_section += f"- {title}: [ `{title}` ]({url})\n"
|
||
return answer + atlas_section
|
||
|
||
|
||
|
||
|
||
async def stream_generate_and_postprocess(
|
||
system_prompt: str,
|
||
messages: List[Dict[str, Any]],
|
||
original_image_paths: List[str],
|
||
temperature: float = 0.3,
|
||
fallback: str = "抱歉，无法生成回答。",
|
||
) -> str:
|
||
"""非流式生成完整答案，统一校正后再按安全语义块模拟流式发送。
|
||
|
||
公式、代码块、表格和图片会作为完整单元发送，避免前端在内容尚未闭合时
|
||
错误解析。函数返回值与已经发送的所有片段严格一致。
|
||
"""
|
||
stream_handler = get_current_stream_handler()
|
||
|
||
raw_answer = await OpenaiAPI.open_api_chat_without_thinking(
|
||
query=None,
|
||
model=None,
|
||
system_prompt=system_prompt,
|
||
messages=messages,
|
||
enable_thinking=False,
|
||
temperature=temperature,
|
||
)
|
||
|
||
answer = (raw_answer or fallback).strip()
|
||
answer = re.sub(
|
||
r"ynchroneg>.*?ost switching>",
|
||
"",
|
||
answer,
|
||
flags=re.DOTALL,
|
||
)
|
||
# 修复模型把下一个无序列表项接在上一句末尾的情况。
|
||
answer = re.sub(
|
||
r"(?<=[。！？；：])\s+\*\s+(?=\S)",
|
||
"\n* ",
|
||
answer,
|
||
)
|
||
answer = remove_duplicate_content(answer)
|
||
answer = normalize_markdown_images(
|
||
answer,
|
||
original_paths=original_image_paths,
|
||
)
|
||
answer = normalize_latex_spacing(answer).strip() or fallback
|
||
# 16000 前端同时注册了两套 KaTeX 插件，其中旧插件会抢先把标准
|
||
# $$...$$ 识别为 displayMode=False。改用同为标准 LaTeX 的 \[...\]
|
||
# 可命中新插件的块级规则，生成真正居中且独占一行的 katex-display。
|
||
answer = convert_display_math_for_frontend(answer)
|
||
|
||
def split_prose_line(line: str) -> List[str]:
|
||
"""在闭合的行内公式/代码之外按中文语义停顿切分。"""
|
||
if len(line) <= 52:
|
||
return [line]
|
||
|
||
pieces: List[str] = []
|
||
start = 0
|
||
for index, character in enumerate(line):
|
||
length = index + 1 - start
|
||
is_sentence_end = character in "。！？；"
|
||
is_long_pause = character in "，," and length >= 92
|
||
if length < 36 or not (is_sentence_end or is_long_pause):
|
||
continue
|
||
|
||
candidate = line[start:index + 1]
|
||
if candidate.count("$") % 2 or candidate.count("`") % 2:
|
||
continue
|
||
pieces.append(candidate)
|
||
start = index + 1
|
||
|
||
if start < len(line):
|
||
pieces.append(line[start:])
|
||
return pieces or [line]
|
||
|
||
def build_fake_stream_chunks(content: str) -> List[str]:
|
||
"""构造不会截断特殊 Markdown 单元、且可原样拼回正文的片段。"""
|
||
lines = content.splitlines(keepends=True)
|
||
chunks: List[str] = []
|
||
index = 0
|
||
|
||
while index < len(lines):
|
||
line = lines[index]
|
||
stripped = line.strip()
|
||
|
||
if stripped in ("$$", "\\["):
|
||
closing_delimiter = (
|
||
"$$" if stripped == "$$" else "\\]"
|
||
)
|
||
block = line
|
||
index += 1
|
||
while index < len(lines):
|
||
block += lines[index]
|
||
closing = (
|
||
lines[index].strip()
|
||
== closing_delimiter
|
||
)
|
||
index += 1
|
||
if closing:
|
||
break
|
||
chunks.append(block)
|
||
continue
|
||
|
||
if stripped.startswith("```"):
|
||
block = line
|
||
index += 1
|
||
while index < len(lines):
|
||
block += lines[index]
|
||
closing = lines[index].strip().startswith("```")
|
||
index += 1
|
||
if closing:
|
||
break
|
||
chunks.append(block)
|
||
continue
|
||
|
||
if stripped.startswith("|"):
|
||
block = line
|
||
index += 1
|
||
while index < len(lines) and lines[index].strip().startswith("|"):
|
||
block += lines[index]
|
||
index += 1
|
||
chunks.append(block)
|
||
continue
|
||
|
||
if "![" in line:
|
||
chunks.append(line)
|
||
else:
|
||
chunks.extend(split_prose_line(line))
|
||
index += 1
|
||
|
||
return [chunk for chunk in chunks if chunk]
|
||
|
||
chunks = build_fake_stream_chunks(answer)
|
||
for index, chunk in enumerate(chunks):
|
||
if stream_handler:
|
||
stream_handler.send_stream_content(chunk)
|
||
|
||
if index < len(chunks) - 1:
|
||
visible_length = len(re.sub(r"\s+", "", chunk))
|
||
# 总体约 120 字/秒，并保留 160~600ms 的自然停顿。
|
||
delay = max(0.16, min(0.60, visible_length / 120))
|
||
if chunk.lstrip().startswith(("$$", "\\[", "```", "|", "![")):
|
||
delay = max(delay, 0.22)
|
||
await asyncio.sleep(delay)
|
||
|
||
return answer
|