更新 workflows/workflow_utils.py
This commit is contained in:
parent
c76138604f
commit
e242d9caa8
@ -46,41 +46,890 @@ def safe_json_extract(text: str) -> Optional[Any]:
|
||||
|
||||
from workflows.history_manager import parse_history, build_history_str, filter_image_urls, init_history # noqa: F401
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def normalize_latex_spacing(text: str) -> str:
|
||||
placeholder = "\x00DOUBLE_DOLLAR\x00"
|
||||
text = text.replace("$$", placeholder)
|
||||
text = re.sub(r'(?<!\s)\$', r' $', text)
|
||||
text = re.sub(r'\$(?!\s)', r'$ ', text)
|
||||
text = text.replace(placeholder, "$$")
|
||||
text = re.sub(r'(?<!\s)\$\$', r' $$', text)
|
||||
text = re.sub(r'\$\$(?!\s)', r'$$ ', text)
|
||||
"""
|
||||
规范 Markdown 中的 LaTeX 公式。
|
||||
|
||||
规则：
|
||||
1. 行内公式 $...$
|
||||
- 清除公式定界符内部首尾空格
|
||||
- 公式前后只要存在非空白字符，就补一个空格
|
||||
- ($T$) 会转换为 ( $T$ )
|
||||
|
||||
2. 段落公式 $$...$$
|
||||
- 多行公式完整保留，不会删除结尾的 $$
|
||||
- 单行 $$公式$$ 转换为标准多行段落公式
|
||||
- Markdown 表格中的 $$公式$$ 不拆行
|
||||
|
||||
3. 转换：
|
||||
- \\(...\\) 转换为 $...$
|
||||
- \\[...\\] 转换为 $$...$$
|
||||
|
||||
4. 不处理：
|
||||
- Markdown 围栏代码块
|
||||
- Markdown 行内代码
|
||||
- HTML code、pre、script、style
|
||||
- HTML 注释
|
||||
- Markdown 链接地址
|
||||
- 转义美元符号 \\$
|
||||
"""
|
||||
if not isinstance(text, str):
|
||||
raise TypeError(
|
||||
f"text 必须是 str，实际类型为 {type(text).__name__}"
|
||||
)
|
||||
|
||||
if not text:
|
||||
return text
|
||||
|
||||
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
|
||||
protected_parts: list[str] = []
|
||||
|
||||
def protect(content: str) -> str:
|
||||
token = (
|
||||
f"\uE000LATEX_PROTECTED_"
|
||||
f"{len(protected_parts)}"
|
||||
f"\uE001"
|
||||
)
|
||||
protected_parts.append(content)
|
||||
return token
|
||||
|
||||
def restore(content: str) -> str:
|
||||
for index, original in enumerate(protected_parts):
|
||||
token = f"\uE000LATEX_PROTECTED_{index}\uE001"
|
||||
content = content.replace(token, original)
|
||||
|
||||
return content
|
||||
|
||||
def is_escaped(content: str, position: int) -> bool:
|
||||
"""
|
||||
判断 content[position] 是否被奇数个反斜杠转义。
|
||||
"""
|
||||
slash_count = 0
|
||||
position -= 1
|
||||
|
||||
while position >= 0 and content[position] == "\\":
|
||||
slash_count += 1
|
||||
position -= 1
|
||||
|
||||
return slash_count % 2 == 1
|
||||
|
||||
# ================================================================
|
||||
# 1. 保护 Markdown 围栏代码块
|
||||
# ================================================================
|
||||
|
||||
lines = text.splitlines(keepends=True)
|
||||
protected_lines: list[str] = []
|
||||
line_index = 0
|
||||
|
||||
while line_index < len(lines):
|
||||
line = lines[line_index]
|
||||
|
||||
opening_match = re.match(
|
||||
r"^[ \t]{0,3}(`{3,}|~{3,})",
|
||||
line,
|
||||
)
|
||||
|
||||
if not opening_match:
|
||||
protected_lines.append(line)
|
||||
line_index += 1
|
||||
continue
|
||||
|
||||
opening_fence = opening_match.group(1)
|
||||
fence_character = opening_fence[0]
|
||||
fence_length = len(opening_fence)
|
||||
|
||||
closing_pattern = re.compile(
|
||||
rf"^[ \t]{{0,3}}"
|
||||
rf"{re.escape(fence_character)}"
|
||||
rf"{{{fence_length},}}"
|
||||
rf"[ \t]*(?:\n|$)"
|
||||
)
|
||||
|
||||
block_lines = [line]
|
||||
line_index += 1
|
||||
|
||||
while line_index < len(lines):
|
||||
current_line = lines[line_index]
|
||||
block_lines.append(current_line)
|
||||
line_index += 1
|
||||
|
||||
if closing_pattern.match(current_line):
|
||||
break
|
||||
|
||||
protected_lines.append(
|
||||
protect("".join(block_lines))
|
||||
)
|
||||
|
||||
text = "".join(protected_lines)
|
||||
|
||||
# ================================================================
|
||||
# 2. 保护 HTML 代码区域和注释
|
||||
# ================================================================
|
||||
|
||||
text = re.sub(
|
||||
r"(?is)"
|
||||
r"<(?P<tag>pre|code|script|style)\b[^>]*>"
|
||||
r".*?"
|
||||
r"</(?P=tag)\s*>",
|
||||
lambda match: protect(match.group(0)),
|
||||
text,
|
||||
)
|
||||
|
||||
text = re.sub(
|
||||
r"(?s)<!--.*?-->",
|
||||
lambda match: protect(match.group(0)),
|
||||
text,
|
||||
)
|
||||
|
||||
# ================================================================
|
||||
# 3. 保护 Markdown 链接地址
|
||||
# ================================================================
|
||||
|
||||
text = re.sub(
|
||||
r"(!?\[[^\]\n]*\]\()([^)\n]*)(\))",
|
||||
lambda match: (
|
||||
match.group(1)
|
||||
+ protect(match.group(2))
|
||||
+ match.group(3)
|
||||
),
|
||||
text,
|
||||
)
|
||||
|
||||
text = re.sub(
|
||||
r"(?im)"
|
||||
r"^([ \t]{0,3}\[[^\]\n]+\]:[ \t]*)"
|
||||
r"(\S+(?:[ \t]+.*)?)$",
|
||||
lambda match: (
|
||||
match.group(1)
|
||||
+ protect(match.group(2))
|
||||
),
|
||||
text,
|
||||
)
|
||||
|
||||
# ================================================================
|
||||
# 4. 保护 Markdown 行内代码
|
||||
# ================================================================
|
||||
|
||||
inline_code_result: list[str] = []
|
||||
position = 0
|
||||
|
||||
while position < len(text):
|
||||
if text[position] != "`":
|
||||
inline_code_result.append(text[position])
|
||||
position += 1
|
||||
continue
|
||||
|
||||
delimiter_end = position
|
||||
|
||||
while (
|
||||
delimiter_end < len(text)
|
||||
and text[delimiter_end] == "`"
|
||||
):
|
||||
delimiter_end += 1
|
||||
|
||||
delimiter = text[position:delimiter_end]
|
||||
search_position = delimiter_end
|
||||
closing_position = -1
|
||||
|
||||
while search_position < len(text):
|
||||
candidate = text.find(
|
||||
delimiter,
|
||||
search_position,
|
||||
)
|
||||
|
||||
if candidate < 0:
|
||||
break
|
||||
|
||||
previous_character = (
|
||||
text[candidate - 1]
|
||||
if candidate > 0
|
||||
else ""
|
||||
)
|
||||
|
||||
candidate_end = candidate + len(delimiter)
|
||||
|
||||
next_character = (
|
||||
text[candidate_end]
|
||||
if candidate_end < len(text)
|
||||
else ""
|
||||
)
|
||||
|
||||
if (
|
||||
previous_character != "`"
|
||||
and next_character != "`"
|
||||
):
|
||||
closing_position = candidate
|
||||
break
|
||||
|
||||
search_position = candidate + 1
|
||||
|
||||
if closing_position < 0:
|
||||
inline_code_result.append(delimiter)
|
||||
position = delimiter_end
|
||||
continue
|
||||
|
||||
end_position = closing_position + len(delimiter)
|
||||
|
||||
inline_code_result.append(
|
||||
protect(text[position:end_position])
|
||||
)
|
||||
|
||||
position = end_position
|
||||
|
||||
text = "".join(inline_code_result)
|
||||
|
||||
# ================================================================
|
||||
# 5. 转换其他 LaTeX 公式定界符
|
||||
# ================================================================
|
||||
|
||||
# \( x + y \) -> $x + y$
|
||||
text = re.sub(
|
||||
r"(?<!\\)\\\((.*?)(?<!\\)\\\)",
|
||||
lambda match: (
|
||||
"$" + match.group(1).strip() + "$"
|
||||
),
|
||||
text,
|
||||
)
|
||||
|
||||
# \[ x + y \] -> $$x + y$$
|
||||
text = re.sub(
|
||||
r"(?s)(?<!\\)\\\[(.*?)(?<!\\)\\\]",
|
||||
lambda match: (
|
||||
"$$" + match.group(1).strip() + "$$"
|
||||
),
|
||||
text,
|
||||
)
|
||||
|
||||
# ================================================================
|
||||
# 6. 完整提取并保护 $$...$$ 段落公式
|
||||
# ================================================================
|
||||
|
||||
display_result: list[str] = []
|
||||
position = 0
|
||||
|
||||
while position < len(text):
|
||||
if not (
|
||||
text.startswith("$$", position)
|
||||
and not is_escaped(text, position)
|
||||
):
|
||||
display_result.append(text[position])
|
||||
position += 1
|
||||
continue
|
||||
|
||||
opening_position = position
|
||||
search_position = position + 2
|
||||
closing_position = -1
|
||||
|
||||
while search_position < len(text):
|
||||
candidate = text.find(
|
||||
"$$",
|
||||
search_position,
|
||||
)
|
||||
|
||||
if candidate < 0:
|
||||
break
|
||||
|
||||
if not is_escaped(text, candidate):
|
||||
closing_position = candidate
|
||||
break
|
||||
|
||||
search_position = candidate + 2
|
||||
|
||||
# 未找到闭合 $$，剩余内容原样保留。
|
||||
if closing_position < 0:
|
||||
display_result.append(
|
||||
text[opening_position:]
|
||||
)
|
||||
position = len(text)
|
||||
break
|
||||
|
||||
complete_formula = text[
|
||||
opening_position:closing_position + 2
|
||||
]
|
||||
|
||||
formula_body = text[
|
||||
opening_position + 2:closing_position
|
||||
]
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# 已经是多行段落公式时，完整原样保留。
|
||||
#
|
||||
# $$
|
||||
# R = K
|
||||
# $$
|
||||
#
|
||||
# 不拆解、不重建，避免结尾 $$ 丢失。
|
||||
# ------------------------------------------------------------
|
||||
|
||||
if "\n" in complete_formula:
|
||||
display_result.append(
|
||||
protect(complete_formula)
|
||||
)
|
||||
position = closing_position + 2
|
||||
continue
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# 判断公式是否位于 Markdown 表格内。
|
||||
# 表格中的公式不能强制拆成多行。
|
||||
# ------------------------------------------------------------
|
||||
|
||||
line_start = text.rfind(
|
||||
"\n",
|
||||
0,
|
||||
opening_position,
|
||||
) + 1
|
||||
|
||||
line_end = text.find(
|
||||
"\n",
|
||||
closing_position + 2,
|
||||
)
|
||||
|
||||
if line_end < 0:
|
||||
line_end = len(text)
|
||||
|
||||
current_line = text[line_start:line_end]
|
||||
clean_body = formula_body.strip()
|
||||
|
||||
if "|" in current_line:
|
||||
display_result.append(
|
||||
protect(
|
||||
"$$" + clean_body + "$$"
|
||||
)
|
||||
)
|
||||
|
||||
position = closing_position + 2
|
||||
continue
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# 单行段落公式转换成标准多行格式。
|
||||
#
|
||||
# $$R = K$$
|
||||
#
|
||||
# 转换为：
|
||||
#
|
||||
# $$
|
||||
# R = K
|
||||
# $$
|
||||
# ------------------------------------------------------------
|
||||
|
||||
normalized_formula = (
|
||||
"$$\n"
|
||||
+ clean_body
|
||||
+ "\n$$"
|
||||
)
|
||||
|
||||
before_text = "".join(display_result)
|
||||
|
||||
# 公式前存在普通正文时，确保前面有空行。
|
||||
if before_text and not before_text.endswith("\n\n"):
|
||||
before_text = before_text.rstrip(" \t\n") + "\n\n"
|
||||
|
||||
display_result = [before_text] if before_text else []
|
||||
|
||||
display_result.append(
|
||||
protect(normalized_formula)
|
||||
)
|
||||
|
||||
position = closing_position + 2
|
||||
|
||||
# 清理公式后已有的普通空格。
|
||||
while (
|
||||
position < len(text)
|
||||
and text[position] in " \t"
|
||||
):
|
||||
position += 1
|
||||
|
||||
# 后面还有内容时，确保公式后有空行。
|
||||
if position < len(text):
|
||||
if text[position] == "\n":
|
||||
while (
|
||||
position < len(text)
|
||||
and text[position] == "\n"
|
||||
):
|
||||
position += 1
|
||||
|
||||
display_result.append("\n\n")
|
||||
|
||||
text = "".join(display_result)
|
||||
|
||||
# ================================================================
|
||||
# 7. 规范 $...$ 行内公式
|
||||
# ================================================================
|
||||
|
||||
inline_result: list[str] = []
|
||||
position = 0
|
||||
|
||||
while position < len(text):
|
||||
is_inline_opening = (
|
||||
text[position] == "$"
|
||||
and not is_escaped(text, position)
|
||||
and not text.startswith("$$", position)
|
||||
and not (
|
||||
position > 0
|
||||
and text[position - 1] == "$"
|
||||
)
|
||||
)
|
||||
|
||||
if not is_inline_opening:
|
||||
inline_result.append(text[position])
|
||||
position += 1
|
||||
continue
|
||||
|
||||
opening_position = position
|
||||
|
||||
# 避免把 $100 识别成行内公式开始。
|
||||
next_opening_character = (
|
||||
text[position + 1]
|
||||
if position + 1 < len(text)
|
||||
else ""
|
||||
)
|
||||
|
||||
if next_opening_character.isdigit():
|
||||
inline_result.append("$")
|
||||
position += 1
|
||||
continue
|
||||
|
||||
search_position = position + 1
|
||||
closing_position = -1
|
||||
|
||||
while search_position < len(text):
|
||||
candidate = text.find(
|
||||
"$",
|
||||
search_position,
|
||||
)
|
||||
|
||||
if candidate < 0:
|
||||
break
|
||||
|
||||
# 行内公式不能跨行。
|
||||
if "\n" in text[
|
||||
opening_position + 1:candidate
|
||||
]:
|
||||
break
|
||||
|
||||
if is_escaped(text, candidate):
|
||||
search_position = candidate + 1
|
||||
continue
|
||||
|
||||
# 跳过属于 $$ 的美元符号。
|
||||
if text.startswith("$$", candidate):
|
||||
search_position = candidate + 2
|
||||
continue
|
||||
|
||||
closing_position = candidate
|
||||
break
|
||||
|
||||
# 未找到闭合符时原样保留。
|
||||
if closing_position < 0:
|
||||
inline_result.append("$")
|
||||
position += 1
|
||||
continue
|
||||
|
||||
formula_body = text[
|
||||
opening_position + 1:closing_position
|
||||
].strip()
|
||||
|
||||
# 空公式不处理。
|
||||
if not formula_body:
|
||||
inline_result.append(
|
||||
text[
|
||||
opening_position:closing_position + 1
|
||||
]
|
||||
)
|
||||
|
||||
position = closing_position + 1
|
||||
continue
|
||||
|
||||
# 折叠公式内部普通空格，不修改 LaTeX 命令。
|
||||
formula_body = re.sub(
|
||||
r"[ \t]+",
|
||||
" ",
|
||||
formula_body,
|
||||
)
|
||||
|
||||
previous_character = ""
|
||||
|
||||
for part in reversed(inline_result):
|
||||
if part:
|
||||
previous_character = part[-1]
|
||||
break
|
||||
|
||||
next_character = (
|
||||
text[closing_position + 1]
|
||||
if closing_position + 1 < len(text)
|
||||
else ""
|
||||
)
|
||||
|
||||
# 公式前只要存在非空白字符，就补空格。
|
||||
#
|
||||
# 参数$T$ -> 参数 $T$
|
||||
# ($T$) -> ( $T$
|
||||
# （$T$） -> （ $T$
|
||||
if (
|
||||
previous_character
|
||||
and not previous_character.isspace()
|
||||
):
|
||||
inline_result.append(" ")
|
||||
|
||||
inline_result.append(
|
||||
"$" + formula_body + "$"
|
||||
)
|
||||
|
||||
# 公式后只要存在非空白字符，就补空格。
|
||||
#
|
||||
# $T$参数 -> $T$ 参数
|
||||
# ($T$) -> $T$ )
|
||||
# （$T$） -> $T$ ）
|
||||
if (
|
||||
next_character
|
||||
and not next_character.isspace()
|
||||
):
|
||||
inline_result.append(" ")
|
||||
|
||||
position = closing_position + 1
|
||||
|
||||
return restore("".join(inline_result))
|
||||
|
||||
|
||||
def remove_duplicate_content(text: str) -> str:
|
||||
"""
|
||||
删除普通 Markdown 正文中的完全重复行。
|
||||
|
||||
不对以下内容进行去重：
|
||||
- $$ ... $$ 段落公式
|
||||
- \\[ ... \\] 段落公式
|
||||
- equation、align、gather 等 LaTeX 环境
|
||||
- Markdown 围栏代码块
|
||||
- Markdown 缩进代码块
|
||||
- Markdown 标题、列表、引用、表格等结构行
|
||||
|
||||
这样不会删除段落公式末尾的第二个 $$。
|
||||
"""
|
||||
if not text or len(text) < 10:
|
||||
return text
|
||||
|
||||
lines = text.split('\n')
|
||||
result = []
|
||||
seen_segments = set()
|
||||
if not isinstance(text, str):
|
||||
raise TypeError(
|
||||
f"text 必须是 str，实际类型为 {type(text).__name__}"
|
||||
)
|
||||
|
||||
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
lines = text.split("\n")
|
||||
|
||||
result: list[str] = []
|
||||
seen_segments: set[str] = set()
|
||||
|
||||
in_code_block = False
|
||||
code_fence_character = ""
|
||||
code_fence_length = 0
|
||||
|
||||
in_dollar_math = False
|
||||
in_bracket_math = False
|
||||
|
||||
latex_environment_stack: list[str] = []
|
||||
|
||||
display_environments = {
|
||||
"equation",
|
||||
"equation*",
|
||||
"align",
|
||||
"align*",
|
||||
"alignat",
|
||||
"alignat*",
|
||||
"gather",
|
||||
"gather*",
|
||||
"multline",
|
||||
"multline*",
|
||||
"flalign",
|
||||
"flalign*",
|
||||
"eqnarray",
|
||||
"eqnarray*",
|
||||
"displaymath",
|
||||
"math",
|
||||
}
|
||||
|
||||
def is_escaped(value: str, position: int) -> bool:
|
||||
slash_count = 0
|
||||
position -= 1
|
||||
|
||||
while position >= 0 and value[position] == "\\":
|
||||
slash_count += 1
|
||||
position -= 1
|
||||
|
||||
return slash_count % 2 == 1
|
||||
|
||||
def count_unescaped_double_dollars(value: str) -> int:
|
||||
"""
|
||||
统计一行中未转义的 $$ 数量。
|
||||
"""
|
||||
count = 0
|
||||
position = 0
|
||||
|
||||
while position < len(value) - 1:
|
||||
if (
|
||||
value[position:position + 2] == "$$"
|
||||
and not is_escaped(value, position)
|
||||
):
|
||||
count += 1
|
||||
position += 2
|
||||
else:
|
||||
position += 1
|
||||
|
||||
return count
|
||||
|
||||
def is_markdown_structure_line(value: str) -> bool:
|
||||
"""
|
||||
Markdown 结构行不参与去重，避免破坏语义。
|
||||
"""
|
||||
stripped = value.strip()
|
||||
|
||||
if not stripped:
|
||||
return True
|
||||
|
||||
patterns = (
|
||||
# 标题
|
||||
r"^#{1,6}\s+",
|
||||
|
||||
# 无序列表
|
||||
r"^[-+*]\s+",
|
||||
|
||||
# 有序列表
|
||||
r"^\d+[.)]\s+",
|
||||
|
||||
# 引用
|
||||
r"^>\s*",
|
||||
|
||||
# 任务列表
|
||||
r"^[-+*]\s+\[[ xX]\]\s+",
|
||||
|
||||
# 水平分隔线
|
||||
r"^(?:-{3,}|\*{3,}|_{3,})$",
|
||||
|
||||
# 表格分隔行
|
||||
r"^\|?\s*:?-{3,}:?"
|
||||
r"(?:\s*\|\s*:?-{3,}:?)+\s*\|?$",
|
||||
|
||||
# 普通表格行
|
||||
r"^\|.*\|$",
|
||||
|
||||
# HTML 标签
|
||||
r"^</?[A-Za-z][^>]*>$",
|
||||
|
||||
# LaTeX 环境边界
|
||||
r"^\\(?:begin|end)\{[^}]+\}",
|
||||
|
||||
# 公式定界符
|
||||
r"^(?:\$\$|\\\[|\\\])$",
|
||||
|
||||
# Markdown 链接引用定义
|
||||
r"^\[[^\]]+\]:\s*\S+",
|
||||
)
|
||||
|
||||
return any(
|
||||
re.match(pattern, stripped)
|
||||
for pattern in patterns
|
||||
)
|
||||
|
||||
for line in lines:
|
||||
line = line.rstrip()
|
||||
if not line:
|
||||
stripped = line.strip()
|
||||
|
||||
# ============================================================
|
||||
# 1. Markdown 围栏代码块
|
||||
# ============================================================
|
||||
|
||||
fence_match = re.match(
|
||||
r"^[ \t]{0,3}(`{3,}|~{3,})",
|
||||
line,
|
||||
)
|
||||
|
||||
if not in_code_block and fence_match:
|
||||
fence = fence_match.group(1)
|
||||
|
||||
in_code_block = True
|
||||
code_fence_character = fence[0]
|
||||
code_fence_length = len(fence)
|
||||
|
||||
result.append(line)
|
||||
continue
|
||||
|
||||
normalized_line = line.strip().lower()
|
||||
if in_code_block:
|
||||
result.append(line)
|
||||
|
||||
closing_pattern = re.compile(
|
||||
rf"^[ \t]{{0,3}}"
|
||||
rf"{re.escape(code_fence_character)}"
|
||||
rf"{{{code_fence_length},}}"
|
||||
rf"[ \t]*$"
|
||||
)
|
||||
|
||||
if closing_pattern.match(line):
|
||||
in_code_block = False
|
||||
code_fence_character = ""
|
||||
code_fence_length = 0
|
||||
|
||||
continue
|
||||
|
||||
# Markdown 缩进代码块不参与去重。
|
||||
if re.match(r"^(?:\t| {4})", line):
|
||||
result.append(line)
|
||||
continue
|
||||
|
||||
# ============================================================
|
||||
# 2. \[ ... \] 段落公式
|
||||
# ============================================================
|
||||
|
||||
if in_bracket_math:
|
||||
result.append(line)
|
||||
|
||||
if re.search(r"(?<!\\)\\\]", line):
|
||||
in_bracket_math = False
|
||||
|
||||
continue
|
||||
|
||||
bracket_open_count = len(
|
||||
re.findall(r"(?<!\\)\\\[", line)
|
||||
)
|
||||
|
||||
bracket_close_count = len(
|
||||
re.findall(r"(?<!\\)\\\]", line)
|
||||
)
|
||||
|
||||
if bracket_open_count:
|
||||
result.append(line)
|
||||
|
||||
if bracket_open_count > bracket_close_count:
|
||||
in_bracket_math = True
|
||||
|
||||
continue
|
||||
|
||||
# ============================================================
|
||||
# 3. LaTeX display 环境
|
||||
# ============================================================
|
||||
|
||||
begin_matches = re.findall(
|
||||
r"\\begin\{([^}]+)\}",
|
||||
line,
|
||||
)
|
||||
|
||||
end_matches = re.findall(
|
||||
r"\\end\{([^}]+)\}",
|
||||
line,
|
||||
)
|
||||
|
||||
relevant_begins = [
|
||||
environment
|
||||
for environment in begin_matches
|
||||
if environment in display_environments
|
||||
]
|
||||
|
||||
relevant_ends = [
|
||||
environment
|
||||
for environment in end_matches
|
||||
if environment in display_environments
|
||||
]
|
||||
|
||||
if latex_environment_stack:
|
||||
result.append(line)
|
||||
|
||||
for environment in relevant_begins:
|
||||
latex_environment_stack.append(environment)
|
||||
|
||||
for environment in relevant_ends:
|
||||
if (
|
||||
latex_environment_stack
|
||||
and latex_environment_stack[-1]
|
||||
== environment
|
||||
):
|
||||
latex_environment_stack.pop()
|
||||
elif environment in latex_environment_stack:
|
||||
latex_environment_stack.remove(environment)
|
||||
|
||||
continue
|
||||
|
||||
if relevant_begins:
|
||||
result.append(line)
|
||||
|
||||
for environment in relevant_begins:
|
||||
latex_environment_stack.append(environment)
|
||||
|
||||
for environment in relevant_ends:
|
||||
if (
|
||||
latex_environment_stack
|
||||
and latex_environment_stack[-1]
|
||||
== environment
|
||||
):
|
||||
latex_environment_stack.pop()
|
||||
elif environment in latex_environment_stack:
|
||||
latex_environment_stack.remove(environment)
|
||||
|
||||
continue
|
||||
|
||||
# ============================================================
|
||||
# 4. $$ ... $$ 段落公式
|
||||
# ============================================================
|
||||
|
||||
double_dollar_count = (
|
||||
count_unescaped_double_dollars(line)
|
||||
)
|
||||
|
||||
if in_dollar_math:
|
||||
# 公式块中的内容全部保留，包括最后一行的 $$。
|
||||
result.append(line)
|
||||
|
||||
if double_dollar_count % 2 == 1:
|
||||
in_dollar_math = False
|
||||
|
||||
continue
|
||||
|
||||
if double_dollar_count > 0:
|
||||
# 含有 $$ 的行永远不参与去重。
|
||||
result.append(line)
|
||||
|
||||
# 奇数个 $$ 代表开启了跨行段落公式。
|
||||
if double_dollar_count % 2 == 1:
|
||||
in_dollar_math = True
|
||||
|
||||
continue
|
||||
|
||||
# ============================================================
|
||||
# 5. 空行和 Markdown 结构行
|
||||
# ============================================================
|
||||
|
||||
if not stripped:
|
||||
result.append(line)
|
||||
continue
|
||||
|
||||
if is_markdown_structure_line(line):
|
||||
result.append(line)
|
||||
continue
|
||||
|
||||
# ============================================================
|
||||
# 6. 仅对普通正文行去重
|
||||
# ============================================================
|
||||
|
||||
normalized_line = re.sub(
|
||||
r"\s+",
|
||||
" ",
|
||||
stripped.lower(),
|
||||
)
|
||||
|
||||
if normalized_line in seen_segments:
|
||||
continue
|
||||
|
||||
seen_segments.add(normalized_line)
|
||||
result.append(line)
|
||||
|
||||
cleaned_text = '\n'.join(result)
|
||||
return cleaned_text
|
||||
return "\n".join(result)
|
||||
|
||||
|
||||
# 推荐调用顺序：
|
||||
#
|
||||
# text = remove_duplicate_content(text)
|
||||
# text = normalize_latex_spacing(text)
|
||||
|
||||
|
||||
def _extract_rag_text(item: Any) -> str:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user