import re from openai import OpenAI from typing import List, Dict,Tuple import json import time import os from kg_build_v2.Prompt import Prompt_guzhang,Prompt_compose,Prompt_compose_tujie,Prompt_compose_mulu_tujie from kg_build_v2.extract_html import extract_repairproject_tables_html,extract_operation_tables_html,extract_tables,extract_guzhang_tables # from modelsAPI.model_api import OpenaiAPI def read_markdown_file(file_path): """ 读取 Markdown 文件并返回其全部内容作为字符串。 参数: file_path (str): Markdown 文件的路径（支持相对或绝对路径） 返回: str 或 None: 成功时返回文件内容；出错时返回 None """ try: with open(file_path, 'r', encoding='utf-8') as f: content = f.read() return content except FileNotFoundError: print(f"错误：文件未找到 - {file_path}") except PermissionError: print(f"错误：没有权限读取文件 - {file_path}") except UnicodeDecodeError: print(f"错误：文件编码不是 UTF-8，无法读取 - {file_path}") except Exception as e: print(f"读取文件时发生未知错误: {e}") return None def openai_chat(query:str, model="Qwen3.5-35B-A3B"): """调用OpenAI API""" try: client = OpenAI( api_key= "none", base_url= "http://192.168.0.46:59800/v1" ) response = client.chat.completions.create( model=model, messages=[ {"role": "user", "content": query}, ], temperature=0.1, stream=False, response_format = {"type": "json_object"}, extra_body={"chat_template_kwargs": {"enable_thinking": False}} ) return response.choices[0].message.content except Exception as e: print(f"调用OpenAI API时出错: {e}") return None def safe_json_loads(s: str): try: return json.loads(s) except json.JSONDecodeError as e: # 尝试修复常见问题 s = s.replace("\\n", "\\\n").replace("\r", "\\r").replace("\t", "\\t") # 或者直接清理非法字符 s = re.sub(r'[\x00-\x1f\x7f]', '', s) # 移除控制字符 try: return json.loads(s) except: raise e #def openai_chat(query: str, model="Qwen3-32B"): # """调用OpenAI API""" # try: # client = OpenAI( # api_key="none", # base_url="http://192.168.0.46:59800/v1" # ) # response = client.chat.completions.create( # model=model, # messages=[ # {"role": "user", "content": query}, # ], # temperature=0.1, # stream=False, # response_format={"type": "json_object"}, # extra_body={"chat_template_kwargs": {"enable_thinking": False}} # ) # return response.choices[0].message.content # except Exception as e: # print(f"调用OpenAI API时出错: {e}") # return None # def extract_guzhang_table_entity(chunks): # """ # 提取故障类实体 # """ # all_results = [] # all_entity_results = [] # all_relation_results = [] # # 确定起始索引 # print("提取故障类实体和关系") # for idx, block in enumerate(chunks, 1): # # 跳过第一个表格 # print(f"=== 故障类 Block {idx} ===") # print(block[:100]) # 只打印前500字符 # print("\n" + "="*60 + "\n") # final_prompt = Prompt_guzhang.format( # text=block # ) # raw_response = openai_chat(final_prompt) # if raw_response is None: # print(f" ⚠️ 第 {idx} 切片 API 调用失败") # continue # # 尝试解析 JSON # try: # cleaned = raw_response.strip() # if cleaned.startswith("```json"): # cleaned = cleaned[7:].lstrip() # if cleaned.endswith("```"): # cleaned = cleaned[:-3].rstrip() # json_result = safe_json_loads(cleaned) # entities = json_result.get("entities", []) # relationships = json_result.get("relationships", []) # for entity in entities: # # 确保 entity 有 "properties" 键，且其值为字典 # if "properties" in entity and isinstance(entity["properties"], dict): # entity["properties"]["切片"] = block # # 打印美化版 # all_entity_results += entities # all_relation_results += relationships # # all_results.append({ # # "entities": entities, # 已添加切片属性的实体列表 # # "relationships": relationships, # 关系列表 # # }) # # 打印美化版 # except Exception as e: # print(f" ⚠️ 第 {idx} 切片 JSON 解析失败: {e}") # print(f" 原始响应: {repr(raw_response[:500])}") # json_result = None # continue # all_results.append({ # "entities": all_entity_results, # 已添加切片属性的实体列表 # "relationships": all_relation_results, # 关系列表 # }) # return all_results def extract_guzhang_table_entity(chunks): """ 提取故障类实体和关系，将所有切片的结果合并为两个大列表 返回: { "entities": [...], "relationships": [...] } """ all_entities = [] all_relationships = [] print("提取故障类实体和关系") for idx, block in enumerate(chunks, 1): print(f"=== 故障类 Block {idx} ===") print(block[:100]) print("\n" + "="*60 + "\n") final_prompt = Prompt_guzhang.format(text=block) raw_response =openai_chat(final_prompt) if raw_response is None: print(f" ⚠️ 第 {idx} 切片 API 调用失败") continue try: cleaned = raw_response.strip() if cleaned.startswith("```json"): cleaned = cleaned[7:].lstrip() if cleaned.endswith("```"): cleaned = cleaned[:-3].rstrip() json_result = safe_json_loads(cleaned) if json_result is None: print(f" ⚠️ 第 {idx} 切片 JSON 解析失败: 解析返回 None") continue entities = json_result.get("entities", []) relationships = json_result.get("relationships", []) # 为每个实体添加来源切片信息 for entity in entities: if "properties" in entity and isinstance(entity["properties"], dict): entity["properties"]["切片"] = block # 拼接到总列表 all_entities.extend(entities) all_relationships.extend(relationships) print(f" ✅ 第 {idx} 切片解析成功: {len(entities)} 个实体, {len(relationships)} 个关系") except Exception as e: print(f" ⚠️ 第 {idx} 切片 JSON 解析失败: {e}") print(f" 原始响应: {repr(raw_response[:500])}") continue # 最终结果：返回一个字典，包含拼接后的两个列表 result = { "entities": all_entities, "relationships": all_relationships } return result # def extract_compose_table_entity(chunks): # """ # 提取组成表的设备、零部件实体以及关系 # """ # print("提取故障类实体和关系") # all_results = [] # all_entity_results = [] # all_relation_results = [] # # 确定起始索引 # for idx, block in enumerate(chunks, 1): # # 跳过第一个表格 # print(f"=== 组成表 Block {idx} ===") # print(block[:100]) # 只打印前500字符 # print("\n" + "="*60 + "\n") # final_prompt = Prompt_compose.format( # text=block # ) # raw_response = openai_chat(final_prompt) # if raw_response is None: # print(f" ⚠️ 第 {idx} 切片 API 调用失败") # continue # # 尝试解析 JSON # try: # cleaned = raw_response.strip() # if cleaned.startswith("```json"): # cleaned = cleaned[7:].lstrip() # if cleaned.endswith("```"): # cleaned = cleaned[:-3].rstrip() # json_result = safe_json_loads(cleaned) # entities = json_result.get("entities", []) # relationships = json_result.get("relationships", []) # for entity in entities: # # 确保 entity 有 "properties" 键，且其值为字典 # if "properties" in entity and isinstance(entity["properties"], dict): # entity["properties"]["切片"] = block # all_entity_results += entities # all_relation_results += relationships # # 打印美化版 # print(json.dumps(json_result, ensure_ascii=False, indent=2)) # # all_results.append({ # # "entities": entities, # 已添加切片属性的实体列表 # # "relationships": relationships, # 关系列表 # # }) # except Exception as e: # print(f" ⚠️ 第 {idx} 切片 JSON 解析失败: {e}") # print(f" 原始响应: {repr(raw_response[:500])}") # json_result = None # continue # all_results.append({ # "entities": all_entity_results, # 已添加切片属性的实体列表 # "relationships": all_relation_results, # 关系列表 # }) # return all_results def extract_compose_table_entity(chunks): """ 提取组成表的设备、零部件实体以及关系 """ print("提取组成表实体和关系") # 建议：将 all_results 改为复数形式以符合习惯 all_entities = [] all_relationships = [] # 确定起始索引 for idx, block in enumerate(chunks, 1): # 跳过第一个表格（如果需要的话，逻辑保持不变） print(f"=== 组成表 Block {idx} ===") print(block[:100]) # 只打印前100字符预览 print("\n" + "="*60 + "\n") final_prompt = Prompt_compose.format(text=block) raw_response =openai_chat(final_prompt) if raw_response is None: print(f" ⚠️ 第 {idx} 切片 API 调用失败") continue # 尝试解析 JSON try: cleaned = raw_response.strip() if cleaned.startswith("```json"): cleaned = cleaned[7:].lstrip() if cleaned.endswith("```"): cleaned = cleaned[:-3].rstrip() json_result = safe_json_loads(cleaned) # === 核心修改点：处理实体并添加切片信息 === entities = json_result.get("entities", []) relationships = json_result.get("relationships", []) # 为当前切片的实体添加属性 for entity in entities: if "properties" in entity and isinstance(entity["properties"], dict): entity["properties"]["切片"] = block # 构建当前块的结果对象 # current_result = { # "entities": entities, # "relationships": relationships, # } all_entities.extend(entities) all_relationships.extend(relationships) # 添加到总结果列表 # 打印美化版 # print(json.dumps(json_result, ensure_ascii=False, indent=2)) except Exception as e: print(f" ⚠️ 第 {idx} 切片 JSON 解析失败: {e}") print(f" 原始响应: {repr(raw_response[:500])}") continue result = { "entities": all_entities, "relationships": all_relationships } # 返回按切片分组的结果列表 return result # def compose_table_tujie_entity(chunks): # """ # 提取图解目录中设备、零部件类实体以及相关关系 # """ # all_results = [] # all_entity_results = [] # all_relation_results = [] # # 确定起始索引 # for idx, block in enumerate(chunks, 1): # # 跳过第一个表格 # print(f"=== 图解目录 Block {idx} ===") # print(block[:100]) # 只打印前500字符 # print("\n" + "="*60 + "\n") # final_prompt = Prompt_compose_tujie.format( # text=block # ) # raw_response = openai_chat(final_prompt) # if raw_response is None: # print(f" ⚠️ 第 {idx} 切片 API 调用失败") # continue # # 尝试解析 JSON # try: # cleaned = raw_response.strip() # if cleaned.startswith("```json"): # cleaned = cleaned[7:].lstrip() # if cleaned.endswith("```"): # cleaned = cleaned[:-3].rstrip() # json_result = safe_json_loads(cleaned) # entities = json_result.get("entities", []) # relationships = json_result.get("relationships", []) # for entity in entities: # # 确保 entity 有 "properties" 键，且其值为字典 # if "properties" in entity and isinstance(entity["properties"], dict): # entity["properties"]["切片"] = block # # 打印美化版 # all_entity_results += entities # all_relation_results += relationships # print(json.dumps(json_result, ensure_ascii=False, indent=2)) # # all_results.append({ # # "entities": entities, # 已添加切片属性的实体列表 # # "relationships": relationships, # 关系列表 # # }) # # 打印美化版 # print(json.dumps(json_result, ensure_ascii=False, indent=2)) # success = True # except Exception as e: # print(f" ⚠️ 第 {idx} 切片 JSON 解析失败: {e}") # print(f" 原始响应: {repr(raw_response[:500])}") # success = False # json_result = None # continue # all_results.append({ # "entities": all_entity_results, # 已添加切片属性的实体列表 # "relationships": all_relation_results, # 关系列表 # }) # return all_results def compose_table_tujie_entity(chunks): """ 提取图解目录中设备、零部件类实体以及相关关系 """ all_entities = [] all_relationships = [] for idx, block in enumerate(chunks, 1): print(f"=== 图解目录 Block {idx} ===") print(block[:100]) # 只打印前100字符预览 print("\n" + "="*60 + "\n") final_prompt = Prompt_compose_tujie.format(text=block) raw_response = openai_chat(final_prompt) if raw_response is None: print(f" ⚠️ 第 {idx} 切片 API 调用失败") continue # 尝试解析 JSON try: # 清理响应中的 Markdown 代码块标记 cleaned = raw_response.strip() if cleaned.startswith("```json"): cleaned = cleaned[7:].lstrip() if cleaned.endswith("```"): cleaned = cleaned[:-3].rstrip() json_result = safe_json_loads(cleaned) # 提取实体和关系 entities = json_result.get("entities", []) relationships = json_result.get("relationships", []) # 为当前切片的实体添加“切片”属性 for entity in entities: if "properties" in entity and isinstance(entity["properties"], dict): entity["properties"]["切片"] = block # 构建当前切片的结果对象 all_entities.extend(entities) all_relationships.extend(relationships) # 将当前结果追加到总列表 # 打印美化版（打印 current_result 而不是原始的 json_result，确保包含修改） # print(json.dumps(json_result, ensure_ascii=False, indent=2)) except Exception as e: print(f" ⚠️ 第 {idx} 切片 JSON 解析失败: {e}") print(f" 原始响应: {repr(raw_response[:500])}") continue result = { "entities": all_entities, "relationships": all_relationships } return result def extract_compose_table_mulu_tujie(chunks): """ 提取图解目录中设备属性信息 """ all_results = [] # 确定起始索引 for idx, block in enumerate(chunks, 1): # 跳过第一个表格 print(f"=== Repair Block {idx} ===") print(block[:100]) # 只打印前500字符 print("\n" + "="*60 + "\n") final_prompt = Prompt_compose_mulu_tujie.format( text=block ) raw_response = openai_chat(final_prompt) if raw_response is None: print(f" ⚠️ 第 {idx} 切片 API 调用失败") continue all_results.append({ "chunk_index": idx, "structured_result": raw_response, "success": "success" }) return all_results # def get_nromal_node_rela(mdcontent): # print("开始提取常规表格实体和关系") # # 提取各类表格 # chunks_repair, cleaned_content = extract_repairproject_tables_html(mdcontent) # chunks_operation, cleaned_content_operation = extract_operation_tables_html(cleaned_content) # slices, contexts, cleaned_content_normal = extract_tables(cleaned_content_operation) # # 分类表格 # compose_table = [] # compose_table_tujie = [] # compose_table_mulu_tujie = [] # guzhang_table = [] # for ins in contexts: # if '组成部分' in ins and '功能描述' in ins and '数量' in ins: # compose_table.append(ins) # elif '组成编码' in ins and '名称' in ins and '是否为关重件' in ins and '是否为寿命件' in ins: # compose_table_tujie.append(ins) # elif '组成编码' in ins and '名称' in ins and '数量' in ins and '重量' in ins: # compose_table_mulu_tujie.append(ins) # elif '故障现象' in ins and '故障原因' in ins and '维修项目' in ins: # guzhang_table.append(ins) # # 初始化结果 # all_entities = [] # all_relationships = [] # # 提取并合并故障表 # if guzhang_table: # result = extract_guzhang_table_entity(guzhang_table) # all_entities.extend(result.get('entities', [])) # all_relationships.extend(result.get('relationships', [])) # print("输出故障类实体和关系") # print(len(result['entities'])) # print(len(result['relationships'])) # # 提取并合并普通组成表 # if compose_table: # result = extract_compose_table_entity(compose_table) # all_entities.extend(result.get('entities', [])) # all_relationships.extend(result.get('relationships', [])) # print("输出组成表类实体和关系") # print(len(result['entities'])) # print(len(result['relationships'])) # # 提取并合并图解组成表 # if compose_table_tujie: # result = compose_table_tujie_entity(compose_table_tujie) # all_entities.extend(result.get('entities', [])) # all_relationships.extend(result.get('relationships', [])) # print("输出图解组成表类实体和关系") # print(len(result['entities'])) # print(len(result['relationships'])) # print(11111111111111111111) # print(result) # print(f"常规提取完成：共提取 {len(all_entities)} 个实体，{len(all_relationships)} 个关系") # # 构造返回结果 # result = [{ # "entities": all_entities, # "relationships": all_relationships, # }] # return result, cleaned_content_normal def get_nromal_node_rela(mdcontent): print("开始提取常规表格实体和关系") # 初始化最终返回的清洗后内容 cleaned_content_normal = mdcontent try: # 提取各类表格 chunks_repair, cleaned_content = extract_repairproject_tables_html(mdcontent) chunks_operation, cleaned_content_operation = extract_operation_tables_html(cleaned_content) slices, contexts, cleaned_content_normal = extract_tables(cleaned_content_operation) guzhangtable = extract_guzhang_tables(mdcontent) # 分类表格 compose_table = [] compose_table_tujie = [] compose_table_mulu_tujie = [] guzhang_table = [] for ins in contexts: if '组成部分' in ins and '功能描述' in ins and '数量' in ins: compose_table.append(ins) elif '组成编码' in ins and '名称' in ins and '是否为关重件' in ins and '是否为寿命件' in ins: compose_table_tujie.append(ins) elif '组成编码' in ins and '名称' in ins and '数量' in ins and '重量' in ins: compose_table_mulu_tujie.append(ins) elif '故障现象' in ins and '故障原因' in ins and '维修项目' in ins: guzhang_table.append(ins) except Exception as e: print(f"预处理表格分类时发生错误: {e}") # 如果预处理出错，初始化的 contexts 为空，后续不会提取数据 contexts = [] # 初始化结果 all_entities = [] all_relationships = [] if guzhangtable: try: result = extract_guzhang_table_entity(guzhangtable) # 确保 result 是字典且包含键 if isinstance(result, dict): entities = result.get('entities', []) relationships = result.get('relationships', []) if entities or relationships: # 确保有数据 all_entities.extend(entities) all_relationships.extend(relationships) print(all_entities) print(all_relationships) print("输出故障类实体和关系") print(len(entities), len(relationships)) else: print("警告: extract_guzhang_table_entity 返回值不是字典") except Exception as e: print(f"提取故障表时发生错误: {e}") # --- 提取并合并故障表 (带容错) --- if guzhang_table: try: result = extract_guzhang_table_entity(guzhang_table) # 确保 result 是字典且包含键 if isinstance(result, dict): entities = result.get('entities', []) relationships = result.get('relationships', []) if entities or relationships: # 确保有数据 all_entities.extend(entities) all_relationships.extend(relationships) print("输出故障类实体和关系") print(len(entities), len(relationships)) else: print("警告: extract_guzhang_table_entity 返回值不是字典") except Exception as e: print(f"提取故障表时发生错误: {e}") # --- 提取并合并普通组成表 (带容错) --- if compose_table: try: result = extract_compose_table_entity(compose_table) if isinstance(result, dict): entities = result.get('entities', []) relationships = result.get('relationships', []) if entities or relationships: all_entities.extend(entities) all_relationships.extend(relationships) print("输出组成表类实体和关系") print(len(entities), len(relationships)) else: print("警告: extract_compose_table_entity 返回值不是字典") except Exception as e: print(f"提取普通组成表时发生错误: {e}") # --- 提取并合并图解组成表 (带容错) --- if compose_table_tujie: try: result = compose_table_tujie_entity(compose_table_tujie) if isinstance(result, dict): entities = result.get('entities', []) relationships = result.get('relationships', []) if entities or relationships: all_entities.extend(entities) all_relationships.extend(relationships) print("输出图解组成表类实体和关系") print(len(entities), len(relationships)) else: print("警告: compose_table_tujie_entity 返回值不是字典") except Exception as e: print(f"提取图解组成表时发生错误: {e}") print(f"常规提取完成：共提取 {len(all_entities)} 个实体，{len(all_relationships)} 个关系") # 构造返回结果 (保持原有格式 [{...}]) result = [{ "entities": all_entities, "relationships": all_relationships, }] return result, cleaned_content_normal # if __name__ == "__main__": # input_directory = "F:\zklnlp\HJ\hj_kg_code\kgrag\extract_html_json\测试LXJwx手册_shenghna_output11_full_content.md" # # 调用目录处理函数 # mdcontent = read_markdown_file(input_directory) # if mdcontent is None: # print("无法读取文件，程序退出") # exit(1) # tables, cleaned_content = get_nromal_node_rela(mdcontent=mdcontent) # for ins in tables: # print(ins) # print(11111111111111111111)