diff --git a/chunk_text.py b/chunk_text.py index 865454e..4f06be1 100644 --- a/chunk_text.py +++ b/chunk_text.py @@ -16,6 +16,7 @@ import threading import aiofiles import httpx import logging +from config import SHIP_MODEL_NAME logger = logging.getLogger(__name__) API_URLS = [ @@ -52,6 +53,7 @@ def format_entity_text(data): # 2. 遍历 entities 列表 for entity in data.get('entities', []): entity_type = entity.get('type') + properties = entity.get('properties', {}) # 3. 如果当前实体类型在映射表中，则进行拼接 @@ -269,44 +271,6 @@ def extract_operation_groups_and_remaining(records, max_following_for_last=23): return groups, remaining_records -# def extract_wxanli(records): -# # 主触发词：表示一个全新故障案例的开始 - -# # 所有触发词（用于识别是否属于元信息或段落标题，但不用于切片起点） -# all_keywords = ["一、故障现象", "单位:", "故障名称：", "一、基本情况", "单位：", "故障现象及原因："] - -# def is_main_trigger(record): -# text = record.get("text", "") -# return any(trigger in text for trigger in all_keywords) - -# extracted_slices = [] -# remaining_records = [] -# i = 0 -# n = len(records) - -# while i < n: -# record = records[i] -# if is_main_trigger(record): -# # 开始一个新的故障案例 slice -# current_slice = [record] -# i += 1 -# # 继续收集，直到遇到下一个主触发词 或 结束 -# while i < n and not is_main_trigger(records[i]): -# current_slice.append(records[i]) -# i += 1 -# extracted_slices.append(current_slice) -# else: -# # 不是以主触发词开头的记录，保留在 remaining -# remaining_records.append(record) -# i += 1 - -# # ✨ 优化：将每一组（除了最后一组）的最后一个记录移到下一组的开头 -# for i in range(len(extracted_slices) - 1): # 遍历到倒数第二组 -# if len(extracted_slices[i]) > 1: # 确保当前组至少有2条记录 -# last_record = extracted_slices[i].pop() # 移除当前组的最后一条 -# extracted_slices[i + 1].insert(0, last_record) # 插入到下一组的开头 - -# return extracted_slices, remaining_records def extract_wxanli(records): keywords = [ "一、故障现象", @@ -484,44 +448,6 @@ def split_content_bbox(data, max_length=8000): return contents, highlight_lists -# def extract_wxanli(records): -# # 主触发词：表示一个全新故障案例的开始 - -# # 所有触发词（用于识别是否属于元信息或段落标题，但不用于切片起点） -# all_keywords = ["一、故障现象", "单位:", "故障名称：", "一、基本情况", "单位：", "故障现象及原因："] - -# def is_main_trigger(record): -# text = record.get("text", "") -# return any(trigger in text for trigger in all_keywords) - -# extracted_slices = [] -# remaining_records = [] -# i = 0 -# n = len(records) - -# while i < n: -# record = records[i] -# if is_main_trigger(record): -# # 开始一个新的故障案例 slice -# current_slice = [record] -# i += 1 -# # 继续收集，直到遇到下一个主触发词 或 结束 -# while i < n and not is_main_trigger(records[i]): -# current_slice.append(records[i]) -# i += 1 -# extracted_slices.append(current_slice) -# else: -# # 不是以主触发词开头的记录，保留在 remaining -# remaining_records.append(record) -# i += 1 - -# # ✨ 优化：将每一组（除了最后一组）的最后一个记录移到下一组的开头 -# for i in range(len(extracted_slices) - 1): # 遍历到倒数第二组 -# if len(extracted_slices[i]) > 1: # 确保当前组至少有2条记录 -# last_record = extracted_slices[i].pop() # 移除当前组的最后一条 -# extracted_slices[i + 1].insert(0, last_record) # 插入到下一组的开头 - -# return extracted_slices, remaining_records def group_records(records): @@ -948,6 +874,45 @@ def chunk_check(data,MAX_length=8000): "positions": positions # 共享原始位置信息 }) return slice_reset + +def find_ship_info_by_hull(json_file_path, data): + """ + 从NER结果中提取舷号，再从JSON文件中查找对应的舰船信息 + :param json_file_path: JSON文件路径 + :param data: get_entity() 返回的NER结果字典 + :return: 包含 model_name 和 ship_name 的字典，未找到返回 None + """ + # 1. 提取舷号 + hulls = [ + entity['properties']['舷号'] + for entity in data.get('entities', []) + if 'properties' in entity and '舷号' in entity['properties'] + ] + if not hulls: + return None + + target_hull = hulls[0] + + # 2. 根据舷号查找舰船信息 + try: + with open(json_file_path, 'r', encoding='utf-8') as f: + ship_data = json.load(f) + + for item in ship_data: + if str(item.get('hull_number')) == str(target_hull): + return { + "model_name": item.get('model_name'), + "ship_name": item.get('ship_name') + } + + return None + + except FileNotFoundError: + print(f"错误：找不到文件 {json_file_path}") + return None + except json.JSONDecodeError: + print("错误：JSON 文件格式不正确") + return None def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动机-维修手册.pdf"): """ 合并过短的切片： @@ -1005,10 +970,15 @@ def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动 }) try: final_result = get_entity(filename) + print(final_result) + xinghao = find_ship_info_by_hull(SHIP_MODEL_NAME,final_result) + if xinghao.get('model_name'): + model_name = xinghao['model_name'] + else: + model_name = "" # lower_entity = final_result.get("low_level") otehr_data = format_entity_text(final_result) logger.info(f"文件名实体提取成功: {otehr_data}") - logger.info("11111111111111111111111111111111111111111") except Exception as e: logger.warning(f"文件名实体提取失败: {e}") otehr_data = "" # 建议加个兜底,避免后面 NameError @@ -1018,12 +988,15 @@ def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动 if not otehr_data: continue newline_idx = content.find('\n') + # 拼接成一个括号：otehr_data + 型号为xxx + info = otehr_data + if model_name: + info += f'，型号为{model_name}' + suffix = f'({info})' if newline_idx == -1: - # 没有换行符,整个 content 就是第一行,直接追加 - ins['content'] = content + f'({otehr_data})' + ins['content'] = content + suffix else: - # 在第一个换行符之前插入 (lower_entity) - ins['content'] = content[:newline_idx] + f'({otehr_data})' + content[newline_idx:] + ins['content'] = content[:newline_idx] + suffix + content[newline_idx:] return result def find_and_read_content_list(directory, original_filename, encoding='utf-8'): """ @@ -1155,7 +1128,7 @@ async def process_other_file(pdf_path: Path, image_prefix: str, filename: str) - release_api_url(api_url) if __name__ == "__main__": - filepath = r"E:\ZKYNLP\HJproject\知识库\122-06A0014-B01001_发动机-维修手册_content_list.json" + filepath = r"E:\ZKYNLP\Hjunproject\project0506\kgrag\122-06A0014-B01001_发动机-维修手册_content_list.json" try: with open(filepath, 'r', encoding='utf-8') as file: data = json.load(file) @@ -1172,8 +1145,8 @@ if __name__ == "__main__": slices_2 = merge_short_slices(slices_1, min_length=30) # ← 新增这一行 - # for ins in slices_2: - # print(ins) + for ins in slices_2: + print(ins) diff --git a/config.py b/config.py index ccd2309..dea117b 100644 --- a/config.py +++ b/config.py @@ -72,4 +72,6 @@ API_OTHER_URLS = [ #"http://192.168.0.111:9979/analyze-otherfile", #"http://192.168.0.111:9980/analyze-otherfile", # "http://192.168.0.111:9975/analyze-otherfile", -] \ No newline at end of file +] + +SHIP_MODEL_NAME = "E:\ZKYNLP\Hjunproject\project0506\kgrag\ship_xinghao.json" \ No newline at end of file diff --git a/ship_xinghao.json b/ship_xinghao.json new file mode 100644 index 0000000..bb16ca8 --- /dev/null +++ b/ship_xinghao.json @@ -0,0 +1,104 @@ +[ + { + "id": 1, + "model_name": "055型驱逐舰", + "hull_number": 102, + "ship_name": "拉萨舰" + }, + { + "id": 2, + "model_name": "055型驱逐舰", + "hull_number": 103, + "ship_name": "鞍山舰" + }, + { + "id": 3, + "model_name": "055型驱逐舰", + "hull_number": 104, + "ship_name": "无锡舰" + }, + { + "id": 4, + "model_name": "055型驱逐舰", + "hull_number": 105, + "ship_name": "大连舰" + }, + { + "id": 5, + "model_name": "055型驱逐舰", + "hull_number": 106, + "ship_name": "延安舰" + }, + { + "id": 6, + "model_name": "055型驱逐舰", + "hull_number": 107, + "ship_name": "遵义舰" + }, + { + "id": 7, + "model_name": "055型驱逐舰", + "hull_number": 108, + "ship_name": "咸阳舰" + }, + { + "id": 8, + "model_name": "052D型驱逐舰", + "hull_number": 172, + "ship_name": "昆明舰" + }, + { + "id": 9, + "model_name": "052D型驱逐舰", + "hull_number": 173, + "ship_name": "长沙舰" + }, + { + "id": 10, + "model_name": "052D型驱逐舰", + "hull_number": 174, + "ship_name": "合肥舰" + }, + { + "id": 11, + "model_name": "052D型驱逐舰", + "hull_number": 175, + "ship_name": "银川舰" + }, + { + "id": 12, + "model_name": "052D型驱逐舰", + "hull_number": 163, + "ship_name": "南昌舰" + }, + { + "id": 13, + "model_name": "054A型护卫舰", + "hull_number": 529, + "ship_name": "舟山舰" + }, + { + "id": 14, + "model_name": "054A型护卫舰", + "hull_number": 530, + "ship_name": "徐州舰" + }, + { + "id": 15, + "model_name": "054A型护卫舰", + "hull_number": 547, + "ship_name": "临沂舰" + }, + { + "id": 16, + "model_name": "054A型护卫舰", + "hull_number": 568, + "ship_name": "衡阳舰" + }, + { + "id": 17, + "model_name": "054A型护卫舰", + "hull_number": 570, + "ship_name": "黄山舰" + } +] \ No newline at end of file