更新 workflows/workflow_utils.py

workflows/workflow_utils.py

@@ -46,41 +46,890 @@ def safe_json_extract(text: str) -> Optional[Any]:
 
 from workflows.history_manager import parse_history, build_history_str, filter_image_urls, init_history  # noqa: F401
 
+import re
+
 
 def normalize_latex_spacing(text: str) -> str:
-    placeholder = "\x00DOUBLE_DOLLAR\x00"
+    """
-    text = text.replace("$$", placeholder)
+    规范 Markdown 中的 LaTeX 公式。
-    text = re.sub(r'(?<!\s)\$', r' $', text)
+
-    text = re.sub(r'\$(?!\s)', r'$ ', text)
+    规则：
-    text = text.replace(placeholder, "$$")
+    1. 行内公式 $...$
-    text = re.sub(r'(?<!\s)\$\$', r' $$', text)
+       - 清除公式定界符内部首尾空格
-    text = re.sub(r'\$\$(?!\s)', r'$$ ', text)
+       - 公式前后只要存在非空白字符，就补一个空格
-    return text
+       - ($T$) 会转换为 ( $T$ )
+
+    2. 段落公式 $$...$$
+       - 多行公式完整保留，不会删除结尾的 $$
+       - 单行 $$公式$$ 转换为标准多行段落公式
+       - Markdown 表格中的 $$公式$$ 不拆行
+
+    3. 转换：
+       - \\(...\\) 转换为 $...$
+       - \\[...\\] 转换为 $$...$$
+
+    4. 不处理：
+       - Markdown 围栏代码块
+       - Markdown 行内代码
+       - HTML code、pre、script、style
+       - HTML 注释
+       - Markdown 链接地址
+       - 转义美元符号 \\$
+    """
+    if not isinstance(text, str):
+        raise TypeError(
+            f"text 必须是 str，实际类型为 {type(text).__name__}"
+        )
+
+    if not text:
+        return text
+
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+
+    protected_parts: list[str] = []
+
+    def protect(content: str) -> str:
+        token = (
+            f"\uE000LATEX_PROTECTED_"
+            f"{len(protected_parts)}"
+            f"\uE001"
+        )
+        protected_parts.append(content)
+        return token
+
+    def restore(content: str) -> str:
+        for index, original in enumerate(protected_parts):
+            token = f"\uE000LATEX_PROTECTED_{index}\uE001"
+            content = content.replace(token, original)
+
+        return content
+
+    def is_escaped(content: str, position: int) -> bool:
+        """
+        判断 content[position] 是否被奇数个反斜杠转义。
+        """
+        slash_count = 0
+        position -= 1
+
+        while position >= 0 and content[position] == "\\":
+            slash_count += 1
+            position -= 1
+
+        return slash_count % 2 == 1
+
+    # ================================================================
+    # 1. 保护 Markdown 围栏代码块
+    # ================================================================
+
+    lines = text.splitlines(keepends=True)
+    protected_lines: list[str] = []
+    line_index = 0
+
+    while line_index < len(lines):
+        line = lines[line_index]
+
+        opening_match = re.match(
+            r"^[ \t]{0,3}(`{3,}|~{3,})",
+            line,
+        )
+
+        if not opening_match:
+            protected_lines.append(line)
+            line_index += 1
+            continue
+
+        opening_fence = opening_match.group(1)
+        fence_character = opening_fence[0]
+        fence_length = len(opening_fence)
+
+        closing_pattern = re.compile(
+            rf"^[ \t]{{0,3}}"
+            rf"{re.escape(fence_character)}"
+            rf"{{{fence_length},}}"
+            rf"[ \t]*(?:\n|$)"
+        )
+
+        block_lines = [line]
+        line_index += 1
+
+        while line_index < len(lines):
+            current_line = lines[line_index]
+            block_lines.append(current_line)
+            line_index += 1
+
+            if closing_pattern.match(current_line):
+                break
+
+        protected_lines.append(
+            protect("".join(block_lines))
+        )
+
+    text = "".join(protected_lines)
+
+    # ================================================================
+    # 2. 保护 HTML 代码区域和注释
+    # ================================================================
+
+    text = re.sub(
+        r"(?is)"
+        r"<(?P<tag>pre|code|script|style)\b[^>]*>"
+        r".*?"
+        r"</(?P=tag)\s*>",
+        lambda match: protect(match.group(0)),
+        text,
+    )
+
+    text = re.sub(
+        r"(?s)<!--.*?-->",
+        lambda match: protect(match.group(0)),
+        text,
+    )
+
+    # ================================================================
+    # 3. 保护 Markdown 链接地址
+    # ================================================================
+
+    text = re.sub(
+        r"(!?\[[^\]\n]*\]\()([^)\n]*)(\))",
+        lambda match: (
+            match.group(1)
+            + protect(match.group(2))
+            + match.group(3)
+        ),
+        text,
+    )
+
+    text = re.sub(
+        r"(?im)"
+        r"^([ \t]{0,3}\[[^\]\n]+\]:[ \t]*)"
+        r"(\S+(?:[ \t]+.*)?)$",
+        lambda match: (
+            match.group(1)
+            + protect(match.group(2))
+        ),
+        text,
+    )
+
+    # ================================================================
+    # 4. 保护 Markdown 行内代码
+    # ================================================================
+
+    inline_code_result: list[str] = []
+    position = 0
+
+    while position < len(text):
+        if text[position] != "`":
+            inline_code_result.append(text[position])
+            position += 1
+            continue
+
+        delimiter_end = position
+
+        while (
+            delimiter_end < len(text)
+            and text[delimiter_end] == "`"
+        ):
+            delimiter_end += 1
+
+        delimiter = text[position:delimiter_end]
+        search_position = delimiter_end
+        closing_position = -1
+
+        while search_position < len(text):
+            candidate = text.find(
+                delimiter,
+                search_position,
+            )
+
+            if candidate < 0:
+                break
+
+            previous_character = (
+                text[candidate - 1]
+                if candidate > 0
+                else ""
+            )
+
+            candidate_end = candidate + len(delimiter)
+
+            next_character = (
+                text[candidate_end]
+                if candidate_end < len(text)
+                else ""
+            )
+
+            if (
+                previous_character != "`"
+                and next_character != "`"
+            ):
+                closing_position = candidate
+                break
+
+            search_position = candidate + 1
+
+        if closing_position < 0:
+            inline_code_result.append(delimiter)
+            position = delimiter_end
+            continue
+
+        end_position = closing_position + len(delimiter)
+
+        inline_code_result.append(
+            protect(text[position:end_position])
+        )
+
+        position = end_position
+
+    text = "".join(inline_code_result)
+
+    # ================================================================
+    # 5. 转换其他 LaTeX 公式定界符
+    # ================================================================
+
+    # \( x + y \) -> $x + y$
+    text = re.sub(
+        r"(?<!\\)\\\((.*?)(?<!\\)\\\)",
+        lambda match: (
+            "$" + match.group(1).strip() + "$"
+        ),
+        text,
+    )
+
+    # \[ x + y \] -> $$x + y$$
+    text = re.sub(
+        r"(?s)(?<!\\)\\\[(.*?)(?<!\\)\\\]",
+        lambda match: (
+            "$$" + match.group(1).strip() + "$$"
+        ),
+        text,
+    )
+
+    # ================================================================
+    # 6. 完整提取并保护 $$...$$ 段落公式
+    # ================================================================
+
+    display_result: list[str] = []
+    position = 0
+
+    while position < len(text):
+        if not (
+            text.startswith("$$", position)
+            and not is_escaped(text, position)
+        ):
+            display_result.append(text[position])
+            position += 1
+            continue
+
+        opening_position = position
+        search_position = position + 2
+        closing_position = -1
+
+        while search_position < len(text):
+            candidate = text.find(
+                "$$",
+                search_position,
+            )
+
+            if candidate < 0:
+                break
+
+            if not is_escaped(text, candidate):
+                closing_position = candidate
+                break
+
+            search_position = candidate + 2
+
+        # 未找到闭合 $$，剩余内容原样保留。
+        if closing_position < 0:
+            display_result.append(
+                text[opening_position:]
+            )
+            position = len(text)
+            break
+
+        complete_formula = text[
+            opening_position:closing_position + 2
+        ]
+
+        formula_body = text[
+            opening_position + 2:closing_position
+        ]
+
+        # ------------------------------------------------------------
+        # 已经是多行段落公式时，完整原样保留。
+        #
+        # $$
+        # R = K
+        # $$
+        #
+        # 不拆解、不重建，避免结尾 $$ 丢失。
+        # ------------------------------------------------------------
+
+        if "\n" in complete_formula:
+            display_result.append(
+                protect(complete_formula)
+            )
+            position = closing_position + 2
+            continue
+
+        # ------------------------------------------------------------
+        # 判断公式是否位于 Markdown 表格内。
+        # 表格中的公式不能强制拆成多行。
+        # ------------------------------------------------------------
+
+        line_start = text.rfind(
+            "\n",
+            0,
+            opening_position,
+        ) + 1
+
+        line_end = text.find(
+            "\n",
+            closing_position + 2,
+        )
+
+        if line_end < 0:
+            line_end = len(text)
+
+        current_line = text[line_start:line_end]
+        clean_body = formula_body.strip()
+
+        if "|" in current_line:
+            display_result.append(
+                protect(
+                    "$$" + clean_body + "$$"
+                )
+            )
+
+            position = closing_position + 2
+            continue
+
+        # ------------------------------------------------------------
+        # 单行段落公式转换成标准多行格式。
+        #
+        # $$R = K$$
+        #
+        # 转换为：
+        #
+        # $$
+        # R = K
+        # $$
+        # ------------------------------------------------------------
+
+        normalized_formula = (
+            "$$\n"
+            + clean_body
+            + "\n$$"
+        )
+
+        before_text = "".join(display_result)
+
+        # 公式前存在普通正文时，确保前面有空行。
+        if before_text and not before_text.endswith("\n\n"):
+            before_text = before_text.rstrip(" \t\n") + "\n\n"
+
+        display_result = [before_text] if before_text else []
+
+        display_result.append(
+            protect(normalized_formula)
+        )
+
+        position = closing_position + 2
+
+        # 清理公式后已有的普通空格。
+        while (
+            position < len(text)
+            and text[position] in " \t"
+        ):
+            position += 1
+
+        # 后面还有内容时，确保公式后有空行。
+        if position < len(text):
+            if text[position] == "\n":
+                while (
+                    position < len(text)
+                    and text[position] == "\n"
+                ):
+                    position += 1
+
+            display_result.append("\n\n")
+
+    text = "".join(display_result)
+
+    # ================================================================
+    # 7. 规范 $...$ 行内公式
+    # ================================================================
+
+    inline_result: list[str] = []
+    position = 0
+
+    while position < len(text):
+        is_inline_opening = (
+            text[position] == "$"
+            and not is_escaped(text, position)
+            and not text.startswith("$$", position)
+            and not (
+                position > 0
+                and text[position - 1] == "$"
+            )
+        )
+
+        if not is_inline_opening:
+            inline_result.append(text[position])
+            position += 1
+            continue
+
+        opening_position = position
+
+        # 避免把 $100 识别成行内公式开始。
+        next_opening_character = (
+            text[position + 1]
+            if position + 1 < len(text)
+            else ""
+        )
+
+        if next_opening_character.isdigit():
+            inline_result.append("$")
+            position += 1
+            continue
+
+        search_position = position + 1
+        closing_position = -1
+
+        while search_position < len(text):
+            candidate = text.find(
+                "$",
+                search_position,
+            )
+
+            if candidate < 0:
+                break
+
+            # 行内公式不能跨行。
+            if "\n" in text[
+                opening_position + 1:candidate
+            ]:
+                break
+
+            if is_escaped(text, candidate):
+                search_position = candidate + 1
+                continue
+
+            # 跳过属于 $$ 的美元符号。
+            if text.startswith("$$", candidate):
+                search_position = candidate + 2
+                continue
+
+            closing_position = candidate
+            break
+
+        # 未找到闭合符时原样保留。
+        if closing_position < 0:
+            inline_result.append("$")
+            position += 1
+            continue
+
+        formula_body = text[
+            opening_position + 1:closing_position
+        ].strip()
+
+        # 空公式不处理。
+        if not formula_body:
+            inline_result.append(
+                text[
+                    opening_position:closing_position + 1
+                ]
+            )
+
+            position = closing_position + 1
+            continue
+
+        # 折叠公式内部普通空格，不修改 LaTeX 命令。
+        formula_body = re.sub(
+            r"[ \t]+",
+            " ",
+            formula_body,
+        )
+
+        previous_character = ""
+
+        for part in reversed(inline_result):
+            if part:
+                previous_character = part[-1]
+                break
+
+        next_character = (
+            text[closing_position + 1]
+            if closing_position + 1 < len(text)
+            else ""
+        )
+
+        # 公式前只要存在非空白字符，就补空格。
+        #
+        # 参数$T$     -> 参数 $T$
+        # ($T$)       -> ( $T$
+        # （$T$）     -> （ $T$
+        if (
+            previous_character
+            and not previous_character.isspace()
+        ):
+            inline_result.append(" ")
+
+        inline_result.append(
+            "$" + formula_body + "$"
+        )
+
+        # 公式后只要存在非空白字符，就补空格。
+        #
+        # $T$参数     -> $T$ 参数
+        # ($T$)       -> $T$ )
+        # （$T$）     -> $T$ ）
+        if (
+            next_character
+            and not next_character.isspace()
+        ):
+            inline_result.append(" ")
+
+        position = closing_position + 1
+
+    return restore("".join(inline_result))
 
 
 def remove_duplicate_content(text: str) -> str:
+    """
+    删除普通 Markdown 正文中的完全重复行。
+
+    不对以下内容进行去重：
+    - $$ ... $$ 段落公式
+    - \\[ ... \\] 段落公式
+    - equation、align、gather 等 LaTeX 环境
+    - Markdown 围栏代码块
+    - Markdown 缩进代码块
+    - Markdown 标题、列表、引用、表格等结构行
+
+    这样不会删除段落公式末尾的第二个 $$。
+    """
     if not text or len(text) < 10:
         return text
 
-    lines = text.split('\n')
+    if not isinstance(text, str):
-    result = []
+        raise TypeError(
-    seen_segments = set()
+            f"text 必须是 str，实际类型为 {type(text).__name__}"
+        )
+
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    lines = text.split("\n")
+
+    result: list[str] = []
+    seen_segments: set[str] = set()
+
+    in_code_block = False
+    code_fence_character = ""
+    code_fence_length = 0
+
+    in_dollar_math = False
+    in_bracket_math = False
+
+    latex_environment_stack: list[str] = []
+
+    display_environments = {
+        "equation",
+        "equation*",
+        "align",
+        "align*",
+        "alignat",
+        "alignat*",
+        "gather",
+        "gather*",
+        "multline",
+        "multline*",
+        "flalign",
+        "flalign*",
+        "eqnarray",
+        "eqnarray*",
+        "displaymath",
+        "math",
+    }
+
+    def is_escaped(value: str, position: int) -> bool:
+        slash_count = 0
+        position -= 1
+
+        while position >= 0 and value[position] == "\\":
+            slash_count += 1
+            position -= 1
+
+        return slash_count % 2 == 1
+
+    def count_unescaped_double_dollars(value: str) -> int:
+        """
+        统计一行中未转义的 $$ 数量。
+        """
+        count = 0
+        position = 0
+
+        while position < len(value) - 1:
+            if (
+                value[position:position + 2] == "$$"
+                and not is_escaped(value, position)
+            ):
+                count += 1
+                position += 2
+            else:
+                position += 1
+
+        return count
+
+    def is_markdown_structure_line(value: str) -> bool:
+        """
+        Markdown 结构行不参与去重，避免破坏语义。
+        """
+        stripped = value.strip()
+
+        if not stripped:
+            return True
+
+        patterns = (
+            # 标题
+            r"^#{1,6}\s+",
+
+            # 无序列表
+            r"^[-+*]\s+",
+
+            # 有序列表
+            r"^\d+[.)]\s+",
+
+            # 引用
+            r"^>\s*",
+
+            # 任务列表
+            r"^[-+*]\s+\[[ xX]\]\s+",
+
+            # 水平分隔线
+            r"^(?:-{3,}|\*{3,}|_{3,})$",
+
+            # 表格分隔行
+            r"^\|?\s*:?-{3,}:?"
+            r"(?:\s*\|\s*:?-{3,}:?)+\s*\|?$",
+
+            # 普通表格行
+            r"^\|.*\|$",
+
+            # HTML 标签
+            r"^</?[A-Za-z][^>]*>$",
+
+            # LaTeX 环境边界
+            r"^\\(?:begin|end)\{[^}]+\}",
+
+            # 公式定界符
+            r"^(?:\$\$|\\\[|\\\])$",
+
+            # Markdown 链接引用定义
+            r"^\[[^\]]+\]:\s*\S+",
+        )
+
+        return any(
+            re.match(pattern, stripped)
+            for pattern in patterns
+        )
 
     for line in lines:
         line = line.rstrip()
-        if not line:
+        stripped = line.strip()
+
+        # ============================================================
+        # 1. Markdown 围栏代码块
+        # ============================================================
+
+        fence_match = re.match(
+            r"^[ \t]{0,3}(`{3,}|~{3,})",
+            line,
+        )
+
+        if not in_code_block and fence_match:
+            fence = fence_match.group(1)
+
+            in_code_block = True
+            code_fence_character = fence[0]
+            code_fence_length = len(fence)
+
             result.append(line)
             continue
 
-        normalized_line = line.strip().lower()
+        if in_code_block:
+            result.append(line)
+
+            closing_pattern = re.compile(
+                rf"^[ \t]{{0,3}}"
+                rf"{re.escape(code_fence_character)}"
+                rf"{{{code_fence_length},}}"
+                rf"[ \t]*$"
+            )
+
+            if closing_pattern.match(line):
+                in_code_block = False
+                code_fence_character = ""
+                code_fence_length = 0
+
+            continue
+
+        # Markdown 缩进代码块不参与去重。
+        if re.match(r"^(?:\t| {4})", line):
+            result.append(line)
+            continue
+
+        # ============================================================
+        # 2. \[ ... \] 段落公式
+        # ============================================================
+
+        if in_bracket_math:
+            result.append(line)
+
+            if re.search(r"(?<!\\)\\\]", line):
+                in_bracket_math = False
+
+            continue
+
+        bracket_open_count = len(
+            re.findall(r"(?<!\\)\\\[", line)
+        )
+
+        bracket_close_count = len(
+            re.findall(r"(?<!\\)\\\]", line)
+        )
+
+        if bracket_open_count:
+            result.append(line)
+
+            if bracket_open_count > bracket_close_count:
+                in_bracket_math = True
+
+            continue
+
+        # ============================================================
+        # 3. LaTeX display 环境
+        # ============================================================
+
+        begin_matches = re.findall(
+            r"\\begin\{([^}]+)\}",
+            line,
+        )
+
+        end_matches = re.findall(
+            r"\\end\{([^}]+)\}",
+            line,
+        )
+
+        relevant_begins = [
+            environment
+            for environment in begin_matches
+            if environment in display_environments
+        ]
+
+        relevant_ends = [
+            environment
+            for environment in end_matches
+            if environment in display_environments
+        ]
+
+        if latex_environment_stack:
+            result.append(line)
+
+            for environment in relevant_begins:
+                latex_environment_stack.append(environment)
+
+            for environment in relevant_ends:
+                if (
+                    latex_environment_stack
+                    and latex_environment_stack[-1]
+                    == environment
+                ):
+                    latex_environment_stack.pop()
+                elif environment in latex_environment_stack:
+                    latex_environment_stack.remove(environment)
+
+            continue
+
+        if relevant_begins:
+            result.append(line)
+
+            for environment in relevant_begins:
+                latex_environment_stack.append(environment)
+
+            for environment in relevant_ends:
+                if (
+                    latex_environment_stack
+                    and latex_environment_stack[-1]
+                    == environment
+                ):
+                    latex_environment_stack.pop()
+                elif environment in latex_environment_stack:
+                    latex_environment_stack.remove(environment)
+
+            continue
+
+        # ============================================================
+        # 4. $$ ... $$ 段落公式
+        # ============================================================
+
+        double_dollar_count = (
+            count_unescaped_double_dollars(line)
+        )
+
+        if in_dollar_math:
+            # 公式块中的内容全部保留，包括最后一行的 $$。
+            result.append(line)
+
+            if double_dollar_count % 2 == 1:
+                in_dollar_math = False
+
+            continue
+
+        if double_dollar_count > 0:
+            # 含有 $$ 的行永远不参与去重。
+            result.append(line)
+
+            # 奇数个 $$ 代表开启了跨行段落公式。
+            if double_dollar_count % 2 == 1:
+                in_dollar_math = True
+
+            continue
+
+        # ============================================================
+        # 5. 空行和 Markdown 结构行
+        # ============================================================
+
+        if not stripped:
+            result.append(line)
+            continue
+
+        if is_markdown_structure_line(line):
+            result.append(line)
+            continue
+
+        # ============================================================
+        # 6. 仅对普通正文行去重
+        # ============================================================
+
+        normalized_line = re.sub(
+            r"\s+",
+            " ",
+            stripped.lower(),
+        )
+
         if normalized_line in seen_segments:
             continue
 
         seen_segments.add(normalized_line)
         result.append(line)
 
-    cleaned_text = '\n'.join(result)
+    return "\n".join(result)
-    return cleaned_text
+
+
+# 推荐调用顺序：
+#
+# text = remove_duplicate_content(text)
+# text = normalize_latex_spacing(text)
 
 
 def _extract_rag_text(item: Any) -> str: