477 lines
17 KiB
Python
477 lines
17 KiB
Python
"""
|
||
工作流公共工具函数
|
||
提取自 workflow_fault_diagnosis.py 和 workflow_operate.py 的共享代码
|
||
"""
|
||
|
||
from typing import TypedDict, Optional, Dict, Any, List
|
||
from difflib import SequenceMatcher
|
||
from modelsAPI.model_api import OpenaiAPI
|
||
from utils.function_tracker import get_all_callbacks, get_current_stream_handler
|
||
from utils.image_utils import extract_image_paths, normalize_markdown_images
|
||
import json
|
||
import re
|
||
import random
|
||
import inspect
|
||
import unicodedata
|
||
|
||
|
||
RAG_DEDUP_SIMILARITY_THRESHOLD = 0.86
|
||
RAG_DEDUP_CONTAINMENT_THRESHOLD = 0.92
|
||
RAG_DEDUP_MIN_TEXT_LENGTH = 20
|
||
|
||
|
||
def safe_json_extract(text: str) -> Optional[Any]:
|
||
if not text:
|
||
return None
|
||
cleaned = text.strip()
|
||
cleaned = re.sub(r"```json\s*", "", cleaned, flags=re.IGNORECASE)
|
||
cleaned = re.sub(r"```\s*", "", cleaned)
|
||
m = re.search(r"\[[\s\S]*\]", cleaned)
|
||
if m:
|
||
try:
|
||
return json.loads(m.group(0))
|
||
except Exception:
|
||
pass
|
||
m = re.search(r"\{[\s\S]*\}", cleaned)
|
||
if m:
|
||
try:
|
||
return json.loads(m.group(0))
|
||
except Exception:
|
||
pass
|
||
try:
|
||
return json.loads(cleaned)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
from workflows.history_manager import parse_history, build_history_str, filter_image_urls, init_history # noqa: F401
|
||
|
||
|
||
def normalize_latex_spacing(text: str) -> str:
|
||
placeholder = "\x00DOUBLE_DOLLAR\x00"
|
||
text = text.replace("$$", placeholder)
|
||
text = re.sub(r'(?<!\s)\$', r' $', text)
|
||
text = re.sub(r'\$(?!\s)', r'$ ', text)
|
||
text = text.replace(placeholder, "$$")
|
||
text = re.sub(r'(?<!\s)\$\$', r' $$', text)
|
||
text = re.sub(r'\$\$(?!\s)', r'$$ ', text)
|
||
return text
|
||
|
||
|
||
def remove_duplicate_content(text: str) -> str:
|
||
if not text or len(text) < 10:
|
||
return text
|
||
|
||
lines = text.split('\n')
|
||
result = []
|
||
seen_segments = set()
|
||
|
||
for line in lines:
|
||
line = line.rstrip()
|
||
if not line:
|
||
result.append(line)
|
||
continue
|
||
|
||
normalized_line = line.strip().lower()
|
||
if normalized_line in seen_segments:
|
||
continue
|
||
|
||
seen_segments.add(normalized_line)
|
||
result.append(line)
|
||
|
||
cleaned_text = '\n'.join(result)
|
||
return cleaned_text
|
||
|
||
|
||
def _extract_rag_text(item: Any) -> str:
|
||
if isinstance(item, dict):
|
||
return str(item.get("text") or item.get("content") or "")
|
||
if isinstance(item, str):
|
||
return item
|
||
return ""
|
||
|
||
|
||
def _extract_rag_score(item: Any) -> float:
|
||
if not isinstance(item, dict):
|
||
return 0.0
|
||
try:
|
||
return float(item.get("score") or item.get("socre") or 0.0)
|
||
except (TypeError, ValueError):
|
||
return 0.0
|
||
|
||
|
||
def _normalize_rag_text_for_dedupe(text: Any) -> str:
|
||
value = unicodedata.normalize("NFKC", str(text or ""))
|
||
value = re.sub(r"!\[[^\]]*\]\([^)]+\)", "", value)
|
||
value = re.sub(r"\[([^\]]*)\]\([^)]+\)", r"\1", value)
|
||
value = re.sub(r"https?://\S+", "", value, flags=re.IGNORECASE)
|
||
value = re.sub(r"[/\\]?picture[/\\]\S+", "", value, flags=re.IGNORECASE)
|
||
value = re.sub(r"\s+", "", value).lower()
|
||
return "".join(ch for ch in value if ch.isalnum())
|
||
|
||
|
||
def _char_ngrams(text: str, n: int) -> set:
|
||
if len(text) <= n:
|
||
return {text} if text else set()
|
||
return {text[i:i + n] for i in range(len(text) - n + 1)}
|
||
|
||
|
||
def _rag_texts_are_similar(text_a: str, text_b: str, threshold: float) -> bool:
|
||
if not text_a or not text_b:
|
||
return False
|
||
if text_a == text_b:
|
||
return True
|
||
|
||
min_len = min(len(text_a), len(text_b))
|
||
max_len = max(len(text_a), len(text_b))
|
||
if min_len < 8:
|
||
return False
|
||
|
||
if min_len >= RAG_DEDUP_MIN_TEXT_LENGTH and (text_a in text_b or text_b in text_a):
|
||
return (min_len / max_len) >= RAG_DEDUP_CONTAINMENT_THRESHOLD
|
||
|
||
if min_len < RAG_DEDUP_MIN_TEXT_LENGTH:
|
||
return SequenceMatcher(None, text_a, text_b).ratio() >= 0.95
|
||
|
||
ngram_size = 3 if min_len >= 30 else 2
|
||
grams_a = _char_ngrams(text_a, ngram_size)
|
||
grams_b = _char_ngrams(text_b, ngram_size)
|
||
if not grams_a or not grams_b:
|
||
return False
|
||
|
||
jaccard = len(grams_a & grams_b) / len(grams_a | grams_b)
|
||
if jaccard >= threshold:
|
||
return True
|
||
|
||
return SequenceMatcher(None, text_a, text_b).ratio() >= threshold
|
||
|
||
|
||
def dedupe_rag_results(
|
||
items: Any,
|
||
limit: Optional[int] = None,
|
||
similarity_threshold: float = RAG_DEDUP_SIMILARITY_THRESHOLD,
|
||
) -> Any:
|
||
"""
|
||
Deduplicate RAG result chunks while preserving the original item shape.
|
||
|
||
Higher-score items are kept first. Graph results are intentionally not handled
|
||
here; callers should pass only text RAG chunks.
|
||
"""
|
||
if not isinstance(items, list):
|
||
return items
|
||
if len(items) < 2:
|
||
return items[:limit] if limit else items
|
||
|
||
prepared = []
|
||
for idx, item in enumerate(items):
|
||
prepared.append({
|
||
"item": item,
|
||
"index": idx,
|
||
"score": _extract_rag_score(item),
|
||
"normalized_text": _normalize_rag_text_for_dedupe(_extract_rag_text(item)),
|
||
})
|
||
|
||
prepared.sort(key=lambda entry: (-entry["score"], entry["index"]))
|
||
|
||
kept = []
|
||
for entry in prepared:
|
||
current_text = entry["normalized_text"]
|
||
is_duplicate = False
|
||
if current_text:
|
||
for kept_entry in kept:
|
||
if _rag_texts_are_similar(
|
||
current_text,
|
||
kept_entry["normalized_text"],
|
||
similarity_threshold,
|
||
):
|
||
is_duplicate = True
|
||
break
|
||
if not is_duplicate:
|
||
kept.append(entry)
|
||
|
||
deduped = [entry["item"] for entry in kept]
|
||
return deduped[:limit] if limit else deduped
|
||
|
||
|
||
def convert_rag_result(rag_result):
|
||
if not rag_result or not rag_result.get("success", False):
|
||
return []
|
||
source_citation = rag_result.get("sourceCitation", {})
|
||
flat_results = []
|
||
for resource, items in source_citation.items():
|
||
for item in items:
|
||
flat_results.append(item)
|
||
return flat_results
|
||
|
||
|
||
def calculate_info_completeness(weights: Dict[str, float], values: Dict[str, str]) -> float:
|
||
score = 0.0
|
||
for field, weight in weights.items():
|
||
value = values.get(field, "") or ""
|
||
if value and str(value).strip():
|
||
score += weight
|
||
return round(score, 2)
|
||
|
||
|
||
def emit_callback_event(title_options: List[str], details: str):
|
||
start_event = {"type": "function_execution", "title": random.choice(title_options), "details": details}
|
||
for callback in get_all_callbacks():
|
||
try:
|
||
callback(start_event)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
async def resolve_ship_number_for_workflow(
|
||
ship_number: str,
|
||
context: str = "workflow",
|
||
) -> tuple[str, Dict[str, Any]]:
|
||
"""
|
||
工作流内舷号/舰名标准化。
|
||
|
||
能映射到具体舷号时返回映射后的舷号；失败或未匹配时返回原值。
|
||
"""
|
||
original_ship_number = str(ship_number or "").strip()
|
||
if not original_ship_number:
|
||
return original_ship_number, {}
|
||
|
||
try:
|
||
from utils.ship_number_search import smart_ship_number_mapping
|
||
|
||
matched_ship_number, matched_model, all_numbers, match_type = await smart_ship_number_mapping(
|
||
original_ship_number
|
||
)
|
||
result = {
|
||
"matched_ship_number": matched_ship_number,
|
||
"matched_model": matched_model,
|
||
"all_numbers": all_numbers,
|
||
"match_type": match_type,
|
||
}
|
||
if match_type in ["exact_number", "exact_name", "embedding"] and matched_ship_number:
|
||
mapped_ship_number = str(matched_ship_number).strip()
|
||
if mapped_ship_number and mapped_ship_number != original_ship_number:
|
||
print(
|
||
f"[{context}] 舷号映射: '{original_ship_number}' -> "
|
||
f"'{mapped_ship_number}'，匹配类型={match_type}"
|
||
)
|
||
return mapped_ship_number or original_ship_number, result
|
||
|
||
print(f"[{context}] 舷号未映射，继续使用原值: {original_ship_number}")
|
||
return original_ship_number, result
|
||
except Exception as e:
|
||
print(f"[{context}] 舷号映射异常，继续使用原值: {original_ship_number}; 错误: {str(e)}")
|
||
return original_ship_number, {}
|
||
|
||
|
||
async def resolve_ship_scope_for_workflow(
|
||
ship_number: str,
|
||
context: str = "workflow",
|
||
) -> tuple[str, List[str], Dict[str, Any]]:
|
||
"""
|
||
Resolve user ship input into a display hull number and a hull-number scope.
|
||
|
||
Hull number and ship-name matches collapse to one hull. Model matches keep the
|
||
user's original value for display but return every hull number under that model
|
||
so shared device normalization and system lookup can search the whole scope.
|
||
"""
|
||
original_ship_number = str(ship_number or "").strip()
|
||
if not original_ship_number:
|
||
return original_ship_number, [], {}
|
||
|
||
try:
|
||
from utils.ship_number_search import smart_ship_number_mapping
|
||
|
||
matched_ship_number, matched_model, all_numbers, match_type = await smart_ship_number_mapping(
|
||
original_ship_number
|
||
)
|
||
scope_numbers = [str(num).strip() for num in (all_numbers or []) if str(num or "").strip()]
|
||
result = {
|
||
"matched_ship_number": matched_ship_number,
|
||
"matched_model": matched_model,
|
||
"all_numbers": scope_numbers,
|
||
"match_type": match_type,
|
||
}
|
||
|
||
if match_type in ["exact_number", "exact_name", "embedding"] and matched_ship_number:
|
||
mapped_ship_number = str(matched_ship_number).strip()
|
||
return mapped_ship_number or original_ship_number, [mapped_ship_number], result
|
||
|
||
if match_type == "exact_model" and scope_numbers:
|
||
print(
|
||
f"[{context}] ship model scope resolved: '{original_ship_number}' -> "
|
||
f"{matched_model}, hull_numbers={scope_numbers}"
|
||
)
|
||
return original_ship_number, scope_numbers, result
|
||
|
||
print(f"[{context}] ship scope not resolved, continuing with original value: {original_ship_number}")
|
||
return original_ship_number, [], result
|
||
except Exception as e:
|
||
print(f"[{context}] ship scope resolve failed, continuing with original value: {original_ship_number}; error: {str(e)}")
|
||
return original_ship_number, [], {}
|
||
|
||
|
||
async def normalize_device_name_for_workflow(
|
||
device_name: str,
|
||
ship_number: str = "",
|
||
ship_numbers: Optional[List[str]] = None,
|
||
context: str = "workflow",
|
||
) -> tuple[str, Dict[str, Any]]:
|
||
"""
|
||
工作流内设备名称标准化。
|
||
|
||
标准化失败时返回原设备名，避免检索流程被 Neo4j 或 embedding 服务异常阻断。
|
||
"""
|
||
original_device_name = str(device_name or "").strip()
|
||
if not original_device_name:
|
||
return original_device_name, {}
|
||
|
||
try:
|
||
from tools.graph_tools import normalize_device_name_by_ship_graph
|
||
|
||
print(
|
||
f"[{context}] 开始设备名称标准化: device='{original_device_name}', "
|
||
f"ship_number='{ship_number or '未提供'}'"
|
||
)
|
||
call_kwargs = {}
|
||
if "ship_number" in inspect.signature(normalize_device_name_by_ship_graph).parameters:
|
||
call_kwargs["ship_number"] = str(ship_number or "").strip() or None
|
||
if "ship_numbers" in inspect.signature(normalize_device_name_by_ship_graph).parameters:
|
||
scoped_numbers = [str(num).strip() for num in (ship_numbers or []) if str(num or "").strip()]
|
||
call_kwargs["ship_numbers"] = scoped_numbers or None
|
||
elif ship_number:
|
||
print(f"[{context}] 当前进程加载的设备标准化函数不支持 ship_number，请重启服务加载最新 graph_tools.py")
|
||
|
||
result = await normalize_device_name_by_ship_graph(original_device_name, **call_kwargs)
|
||
if result.get("success", False):
|
||
normalized_device_name = str(result.get("device_name") or original_device_name).strip()
|
||
print(
|
||
f"[{context}] 设备名称标准化完成: '{original_device_name}' -> "
|
||
f"'{normalized_device_name or original_device_name}'，舷号={ship_number or '未提供'}"
|
||
)
|
||
emit_callback_event(
|
||
["🔎 设备名称标准化", "🔧 设备匹配"],
|
||
f"{original_device_name} -> {normalized_device_name or original_device_name}"
|
||
)
|
||
return normalized_device_name or original_device_name, result
|
||
|
||
print(
|
||
f"[{context}] 设备名称标准化失败，继续使用原设备名: {original_device_name}; "
|
||
f"原因: {result.get('error', '未知错误')}"
|
||
)
|
||
return original_device_name, result
|
||
except Exception as e:
|
||
print(f"[{context}] 设备名称标准化异常，继续使用原设备名: {original_device_name}; 错误: {str(e)}")
|
||
return original_device_name, {}
|
||
|
||
|
||
async def resolve_device_system_for_workflow(
|
||
device_name: str,
|
||
ship_number: str = "",
|
||
ship_numbers: Optional[List[str]] = None,
|
||
context: str = "workflow",
|
||
) -> tuple[str, Dict[str, Any]]:
|
||
"""
|
||
工作流内根据设备名称反推所属系统。
|
||
|
||
失败时返回空字符串，调用方可继续执行主流程。
|
||
"""
|
||
device_name = str(device_name or "").strip()
|
||
if not device_name:
|
||
return "", {}
|
||
|
||
try:
|
||
print(
|
||
f"[{context}] 开始设备反推系统: device='{device_name}', "
|
||
f"ship_number='{ship_number or '未提供'}'"
|
||
)
|
||
from tools.graph_tools import find_system_by_device
|
||
|
||
call_kwargs = {"device_name": device_name}
|
||
if "ship_number" in inspect.signature(find_system_by_device).parameters:
|
||
call_kwargs["ship_number"] = str(ship_number or "").strip() or None
|
||
if "ship_numbers" in inspect.signature(find_system_by_device).parameters:
|
||
scoped_numbers = [str(num).strip() for num in (ship_numbers or []) if str(num or "").strip()]
|
||
call_kwargs["ship_numbers"] = scoped_numbers or None
|
||
elif ship_number:
|
||
print(f"[{context}] 当前进程加载的设备反推系统函数不支持 ship_number，请重启服务加载最新 graph_tools.py")
|
||
|
||
result = await find_system_by_device(**call_kwargs)
|
||
if not result.get("success", False):
|
||
print(f"[{context}] 设备反推系统失败: {result.get('error', '未知错误')}")
|
||
return "", result
|
||
|
||
systems = result.get("systems") or []
|
||
first_system = systems[0] if systems else {}
|
||
system_name = ""
|
||
if isinstance(first_system, dict):
|
||
system_name = str(first_system.get("name") or first_system.get("系统名称") or first_system.get("名称") or "").strip()
|
||
|
||
if system_name:
|
||
print(f"[{context}] 设备反推系统完成: {device_name} -> {system_name}")
|
||
emit_callback_event(
|
||
["🔎 设备所属系统识别", "🧭 系统反查"],
|
||
f"{device_name} -> {system_name}"
|
||
)
|
||
else:
|
||
print(f"[{context}] 设备反推系统无结果: {device_name}")
|
||
return system_name, result
|
||
except Exception as e:
|
||
print(f"[{context}] 设备反推系统异常: {str(e)}")
|
||
return "", {}
|
||
|
||
|
||
def build_detail_info(fields: Dict[str, str]) -> str:
|
||
detail_info = ""
|
||
for label, value in fields.items():
|
||
if value:
|
||
detail_info += f"- {label}：{value}\n"
|
||
return detail_info
|
||
|
||
|
||
def append_atlas_section(answer: str, atlas_results: Dict[str, Any]) -> str:
|
||
if not atlas_results:
|
||
return answer
|
||
atlas_section = "\n\n---\n\n**参考图册：**\n"
|
||
for node_name, node_data in atlas_results.items():
|
||
if isinstance(node_data, dict):
|
||
for title, urls in node_data.items():
|
||
if isinstance(urls, list) and urls:
|
||
for url in urls:
|
||
atlas_section += f"- {title}: [ `{title}` ]({url})\n"
|
||
return answer + atlas_section
|
||
|
||
|
||
|
||
|
||
async def stream_generate_and_postprocess(
|
||
system_prompt: str,
|
||
messages: List[Dict[str, Any]],
|
||
original_image_paths: List[str],
|
||
temperature: float = 0.3,
|
||
fallback: str = "抱歉，无法生成回答。",
|
||
) -> str:
|
||
"""单次流式生成最终回复，并在本地做稳定的格式与图片校验。"""
|
||
stream_handler = get_current_stream_handler()
|
||
answer = ""
|
||
accumulated_content = ""
|
||
|
||
async for chunk in OpenaiAPI.open_api_chat_stream(
|
||
model=None,
|
||
system_prompt=system_prompt,
|
||
messages=messages,
|
||
temperature=temperature,
|
||
):
|
||
answer += chunk
|
||
accumulated_content += chunk
|
||
if stream_handler:
|
||
stream_handler.send_stream_content(accumulated_content)
|
||
|
||
answer = answer.strip() if answer else fallback
|
||
answer = re.sub(r'ynchroneg>.*?ost switching>', '', answer, flags=re.DOTALL)
|
||
answer = remove_duplicate_content(answer)
|
||
answer = normalize_markdown_images(answer, original_paths=original_image_paths)
|
||
answer = normalize_latex_spacing(answer)
|
||
|
||
return answer.strip() if answer else fallback
|
||
|