""" 工作流公共工具函数 提取自 workflow_fault_diagnosis.py 和 workflow_operate.py 的共享代码 """ from typing import TypedDict, Optional, Dict, Any, List from difflib import SequenceMatcher from modelsAPI.model_api import OpenaiAPI from utils.function_tracker import get_all_callbacks, get_current_stream_handler from utils.image_utils import extract_image_paths, normalize_markdown_images import json import re import random import inspect import unicodedata RAG_DEDUP_SIMILARITY_THRESHOLD = 0.86 RAG_DEDUP_CONTAINMENT_THRESHOLD = 0.92 RAG_DEDUP_MIN_TEXT_LENGTH = 20 def safe_json_extract(text: str) -> Optional[Any]: if not text: return None cleaned = text.strip() cleaned = re.sub(r"```json\s*", "", cleaned, flags=re.IGNORECASE) cleaned = re.sub(r"```\s*", "", cleaned) m = re.search(r"\[[\s\S]*\]", cleaned) if m: try: return json.loads(m.group(0)) except Exception: pass m = re.search(r"\{[\s\S]*\}", cleaned) if m: try: return json.loads(m.group(0)) except Exception: pass try: return json.loads(cleaned) except Exception: return None from workflows.history_manager import parse_history, build_history_str, filter_image_urls, init_history # noqa: F401 def normalize_latex_spacing(text: str) -> str: placeholder = "\x00DOUBLE_DOLLAR\x00" text = text.replace("$$", placeholder) text = re.sub(r'(? str: if not text or len(text) < 10: return text lines = text.split('\n') result = [] seen_segments = set() for line in lines: line = line.rstrip() if not line: result.append(line) continue normalized_line = line.strip().lower() if normalized_line in seen_segments: continue seen_segments.add(normalized_line) result.append(line) cleaned_text = '\n'.join(result) return cleaned_text def _extract_rag_text(item: Any) -> str: if isinstance(item, dict): return str(item.get("text") or item.get("content") or "") if isinstance(item, str): return item return "" def _extract_rag_score(item: Any) -> float: if not isinstance(item, dict): return 0.0 try: return float(item.get("score") or item.get("socre") or 0.0) except (TypeError, ValueError): return 0.0 def _normalize_rag_text_for_dedupe(text: Any) -> str: value = unicodedata.normalize("NFKC", str(text or "")) value = re.sub(r"!\[[^\]]*\]\([^)]+\)", "", value) value = re.sub(r"\[([^\]]*)\]\([^)]+\)", r"\1", value) value = re.sub(r"https?://\S+", "", value, flags=re.IGNORECASE) value = re.sub(r"[/\\]?picture[/\\]\S+", "", value, flags=re.IGNORECASE) value = re.sub(r"\s+", "", value).lower() return "".join(ch for ch in value if ch.isalnum()) def _char_ngrams(text: str, n: int) -> set: if len(text) <= n: return {text} if text else set() return {text[i:i + n] for i in range(len(text) - n + 1)} def _rag_texts_are_similar(text_a: str, text_b: str, threshold: float) -> bool: if not text_a or not text_b: return False if text_a == text_b: return True min_len = min(len(text_a), len(text_b)) max_len = max(len(text_a), len(text_b)) if min_len < 8: return False if min_len >= RAG_DEDUP_MIN_TEXT_LENGTH and (text_a in text_b or text_b in text_a): return (min_len / max_len) >= RAG_DEDUP_CONTAINMENT_THRESHOLD if min_len < RAG_DEDUP_MIN_TEXT_LENGTH: return SequenceMatcher(None, text_a, text_b).ratio() >= 0.95 ngram_size = 3 if min_len >= 30 else 2 grams_a = _char_ngrams(text_a, ngram_size) grams_b = _char_ngrams(text_b, ngram_size) if not grams_a or not grams_b: return False jaccard = len(grams_a & grams_b) / len(grams_a | grams_b) if jaccard >= threshold: return True return SequenceMatcher(None, text_a, text_b).ratio() >= threshold def dedupe_rag_results( items: Any, limit: Optional[int] = None, similarity_threshold: float = RAG_DEDUP_SIMILARITY_THRESHOLD, ) -> Any: """ Deduplicate RAG result chunks while preserving the original item shape. Higher-score items are kept first. Graph results are intentionally not handled here; callers should pass only text RAG chunks. """ if not isinstance(items, list): return items if len(items) < 2: return items[:limit] if limit else items prepared = [] for idx, item in enumerate(items): prepared.append({ "item": item, "index": idx, "score": _extract_rag_score(item), "normalized_text": _normalize_rag_text_for_dedupe(_extract_rag_text(item)), }) prepared.sort(key=lambda entry: (-entry["score"], entry["index"])) kept = [] for entry in prepared: current_text = entry["normalized_text"] is_duplicate = False if current_text: for kept_entry in kept: if _rag_texts_are_similar( current_text, kept_entry["normalized_text"], similarity_threshold, ): is_duplicate = True break if not is_duplicate: kept.append(entry) deduped = [entry["item"] for entry in kept] return deduped[:limit] if limit else deduped def convert_rag_result(rag_result): if not rag_result or not rag_result.get("success", False): return [] source_citation = rag_result.get("sourceCitation", {}) flat_results = [] for resource, items in source_citation.items(): for item in items: flat_results.append(item) return flat_results def calculate_info_completeness(weights: Dict[str, float], values: Dict[str, str]) -> float: score = 0.0 for field, weight in weights.items(): value = values.get(field, "") or "" if value and str(value).strip(): score += weight return round(score, 2) def emit_callback_event(title_options: List[str], details: str): start_event = {"type": "function_execution", "title": random.choice(title_options), "details": details} for callback in get_all_callbacks(): try: callback(start_event) except Exception: pass async def resolve_ship_number_for_workflow( ship_number: str, context: str = "workflow", ) -> tuple[str, Dict[str, Any]]: """ 工作流内舷号/舰名标准化。 能映射到具体舷号时返回映射后的舷号；失败或未匹配时返回原值。 """ original_ship_number = str(ship_number or "").strip() if not original_ship_number: return original_ship_number, {} try: from utils.ship_number_search import smart_ship_number_mapping matched_ship_number, matched_model, all_numbers, match_type = await smart_ship_number_mapping( original_ship_number ) result = { "matched_ship_number": matched_ship_number, "matched_model": matched_model, "all_numbers": all_numbers, "match_type": match_type, } if match_type in ["exact_number", "exact_name", "embedding"] and matched_ship_number: mapped_ship_number = str(matched_ship_number).strip() if mapped_ship_number and mapped_ship_number != original_ship_number: print( f"[{context}] 舷号映射: '{original_ship_number}' -> " f"'{mapped_ship_number}'，匹配类型={match_type}" ) return mapped_ship_number or original_ship_number, result print(f"[{context}] 舷号未映射，继续使用原值: {original_ship_number}") return original_ship_number, result except Exception as e: print(f"[{context}] 舷号映射异常，继续使用原值: {original_ship_number}; 错误: {str(e)}") return original_ship_number, {} async def resolve_ship_scope_for_workflow( ship_number: str, context: str = "workflow", ) -> tuple[str, List[str], Dict[str, Any]]: """ Resolve user ship input into a display hull number and a hull-number scope. Hull number and ship-name matches collapse to one hull. Model matches keep the user's original value for display but return every hull number under that model so shared device normalization and system lookup can search the whole scope. """ original_ship_number = str(ship_number or "").strip() if not original_ship_number: return original_ship_number, [], {} try: from utils.ship_number_search import smart_ship_number_mapping matched_ship_number, matched_model, all_numbers, match_type = await smart_ship_number_mapping( original_ship_number ) scope_numbers = [str(num).strip() for num in (all_numbers or []) if str(num or "").strip()] result = { "matched_ship_number": matched_ship_number, "matched_model": matched_model, "all_numbers": scope_numbers, "match_type": match_type, } if match_type in ["exact_number", "exact_name", "embedding"] and matched_ship_number: mapped_ship_number = str(matched_ship_number).strip() return mapped_ship_number or original_ship_number, [mapped_ship_number], result if match_type == "exact_model" and scope_numbers: print( f"[{context}] ship model scope resolved: '{original_ship_number}' -> " f"{matched_model}, hull_numbers={scope_numbers}" ) return original_ship_number, scope_numbers, result print(f"[{context}] ship scope not resolved, continuing with original value: {original_ship_number}") return original_ship_number, [], result except Exception as e: print(f"[{context}] ship scope resolve failed, continuing with original value: {original_ship_number}; error: {str(e)}") return original_ship_number, [], {} async def normalize_device_name_for_workflow( device_name: str, ship_number: str = "", ship_numbers: Optional[List[str]] = None, context: str = "workflow", ) -> tuple[str, Dict[str, Any]]: """ 工作流内设备名称标准化。 标准化失败时返回原设备名，避免检索流程被 Neo4j 或 embedding 服务异常阻断。 """ original_device_name = str(device_name or "").strip() if not original_device_name: return original_device_name, {} try: from tools.graph_tools import normalize_device_name_by_ship_graph print( f"[{context}] 开始设备名称标准化: device='{original_device_name}', " f"ship_number='{ship_number or '未提供'}'" ) call_kwargs = {} if "ship_number" in inspect.signature(normalize_device_name_by_ship_graph).parameters: call_kwargs["ship_number"] = str(ship_number or "").strip() or None if "ship_numbers" in inspect.signature(normalize_device_name_by_ship_graph).parameters: scoped_numbers = [str(num).strip() for num in (ship_numbers or []) if str(num or "").strip()] call_kwargs["ship_numbers"] = scoped_numbers or None elif ship_number: print(f"[{context}] 当前进程加载的设备标准化函数不支持 ship_number，请重启服务加载最新 graph_tools.py") result = await normalize_device_name_by_ship_graph(original_device_name, **call_kwargs) if result.get("success", False): normalized_device_name = str(result.get("device_name") or original_device_name).strip() print( f"[{context}] 设备名称标准化完成: '{original_device_name}' -> " f"'{normalized_device_name or original_device_name}'，舷号={ship_number or '未提供'}" ) emit_callback_event( ["🔎 设备名称标准化", "🔧 设备匹配"], f"{original_device_name} -> {normalized_device_name or original_device_name}" ) return normalized_device_name or original_device_name, result print( f"[{context}] 设备名称标准化失败，继续使用原设备名: {original_device_name}; " f"原因: {result.get('error', '未知错误')}" ) return original_device_name, result except Exception as e: print(f"[{context}] 设备名称标准化异常，继续使用原设备名: {original_device_name}; 错误: {str(e)}") return original_device_name, {} async def resolve_device_system_for_workflow( device_name: str, ship_number: str = "", ship_numbers: Optional[List[str]] = None, context: str = "workflow", ) -> tuple[str, Dict[str, Any]]: """ 工作流内根据设备名称反推所属系统。 失败时返回空字符串，调用方可继续执行主流程。 """ device_name = str(device_name or "").strip() if not device_name: return "", {} try: print( f"[{context}] 开始设备反推系统: device='{device_name}', " f"ship_number='{ship_number or '未提供'}'" ) from tools.graph_tools import find_system_by_device call_kwargs = {"device_name": device_name} if "ship_number" in inspect.signature(find_system_by_device).parameters: call_kwargs["ship_number"] = str(ship_number or "").strip() or None if "ship_numbers" in inspect.signature(find_system_by_device).parameters: scoped_numbers = [str(num).strip() for num in (ship_numbers or []) if str(num or "").strip()] call_kwargs["ship_numbers"] = scoped_numbers or None elif ship_number: print(f"[{context}] 当前进程加载的设备反推系统函数不支持 ship_number，请重启服务加载最新 graph_tools.py") result = await find_system_by_device(**call_kwargs) if not result.get("success", False): print(f"[{context}] 设备反推系统失败: {result.get('error', '未知错误')}") return "", result systems = result.get("systems") or [] first_system = systems[0] if systems else {} system_name = "" if isinstance(first_system, dict): system_name = str(first_system.get("name") or first_system.get("系统名称") or first_system.get("名称") or "").strip() if system_name: print(f"[{context}] 设备反推系统完成: {device_name} -> {system_name}") emit_callback_event( ["🔎 设备所属系统识别", "🧭 系统反查"], f"{device_name} -> {system_name}" ) else: print(f"[{context}] 设备反推系统无结果: {device_name}") return system_name, result except Exception as e: print(f"[{context}] 设备反推系统异常: {str(e)}") return "", {} def build_detail_info(fields: Dict[str, str]) -> str: detail_info = "" for label, value in fields.items(): if value: detail_info += f"- {label}：{value}\n" return detail_info def append_atlas_section(answer: str, atlas_results: Dict[str, Any]) -> str: if not atlas_results: return answer atlas_section = "\n\n---\n\n**参考图册：**\n" for node_name, node_data in atlas_results.items(): if isinstance(node_data, dict): for title, urls in node_data.items(): if isinstance(urls, list) and urls: for url in urls: atlas_section += f"- {title}: [ `{title}` ]({url})\n" return answer + atlas_section async def stream_generate_and_postprocess( system_prompt: str, messages: List[Dict[str, Any]], original_image_paths: List[str], temperature: float = 0.3, fallback: str = "抱歉，无法生成回答。", ) -> str: """单次流式生成最终回复，并在本地做稳定的格式与图片校验。""" stream_handler = get_current_stream_handler() answer = "" accumulated_content = "" async for chunk in OpenaiAPI.open_api_chat_stream( model=None, system_prompt=system_prompt, messages=messages, temperature=temperature, ): answer += chunk accumulated_content += chunk if stream_handler: stream_handler.send_stream_content(accumulated_content) answer = answer.strip() if answer else fallback answer = re.sub(r'ynchroneg>.*?ost switching>', '', answer, flags=re.DOTALL) answer = remove_duplicate_content(answer) answer = normalize_markdown_images(answer, original_paths=original_image_paths) answer = normalize_latex_spacing(answer) return answer.strip() if answer else fallback