增加chunk_text.py的切片型号信息

chunk_text.py

@@ -16,6 +16,7 @@ import threading
 import aiofiles
 import httpx
 import logging
+from config import SHIP_MODEL_NAME
 logger = logging.getLogger(__name__)
 
 API_URLS = [
@@ -52,6 +53,7 @@ def format_entity_text(data):
     # 2. 遍历 entities 列表
     for entity in data.get('entities', []):
         entity_type = entity.get('type')
+
         properties = entity.get('properties', {})
         
         # 3. 如果当前实体类型在映射表中，则进行拼接
@@ -269,44 +271,6 @@ def extract_operation_groups_and_remaining(records, max_following_for_last=23):
     return groups, remaining_records
 
 
-# def extract_wxanli(records):
-#     # 主触发词：表示一个全新故障案例的开始
-    
-#     # 所有触发词（用于识别是否属于元信息或段落标题，但不用于切片起点）
-#     all_keywords = ["一、故障现象", "单位:", "故障名称：", "一、基本情况", "单位：", "故障现象及原因："]
-
-#     def is_main_trigger(record):
-#         text = record.get("text", "")
-#         return any(trigger in text for trigger in all_keywords)
-
-#     extracted_slices = []
-#     remaining_records = []
-#     i = 0
-#     n = len(records)
-
-#     while i < n:
-#         record = records[i]
-#         if is_main_trigger(record):
-#             # 开始一个新的故障案例 slice
-#             current_slice = [record]
-#             i += 1
-#             # 继续收集，直到遇到下一个主触发词 或 结束
-#             while i < n and not is_main_trigger(records[i]):
-#                 current_slice.append(records[i])
-#                 i += 1
-#             extracted_slices.append(current_slice)
-#         else:
-#             # 不是以主触发词开头的记录，保留在 remaining
-#             remaining_records.append(record)
-#             i += 1
-
-#     # ✨ 优化：将每一组（除了最后一组）的最后一个记录移到下一组的开头
-#     for i in range(len(extracted_slices) - 1):  # 遍历到倒数第二组
-#         if len(extracted_slices[i]) > 1:  # 确保当前组至少有2条记录
-#             last_record = extracted_slices[i].pop()  # 移除当前组的最后一条
-#             extracted_slices[i + 1].insert(0, last_record)  # 插入到下一组的开头
-
-#     return extracted_slices, remaining_records
 def extract_wxanli(records):
     keywords = [
         "一、故障现象",
@@ -484,44 +448,6 @@ def split_content_bbox(data, max_length=8000):
 
     return contents, highlight_lists
 
-# def extract_wxanli(records):
-#     # 主触发词：表示一个全新故障案例的开始
-    
-#     # 所有触发词（用于识别是否属于元信息或段落标题，但不用于切片起点）
-#     all_keywords = ["一、故障现象", "单位:", "故障名称：", "一、基本情况", "单位：", "故障现象及原因："]
-
-#     def is_main_trigger(record):
-#         text = record.get("text", "")
-#         return any(trigger in text for trigger in all_keywords)
-
-#     extracted_slices = []
-#     remaining_records = []
-#     i = 0
-#     n = len(records)
-
-#     while i < n:
-#         record = records[i]
-#         if is_main_trigger(record):
-#             # 开始一个新的故障案例 slice
-#             current_slice = [record]
-#             i += 1
-#             # 继续收集，直到遇到下一个主触发词 或 结束
-#             while i < n and not is_main_trigger(records[i]):
-#                 current_slice.append(records[i])
-#                 i += 1
-#             extracted_slices.append(current_slice)
-#         else:
-#             # 不是以主触发词开头的记录，保留在 remaining
-#             remaining_records.append(record)
-#             i += 1
-
-#     # ✨ 优化：将每一组（除了最后一组）的最后一个记录移到下一组的开头
-#     for i in range(len(extracted_slices) - 1):  # 遍历到倒数第二组
-#         if len(extracted_slices[i]) > 1:  # 确保当前组至少有2条记录
-#             last_record = extracted_slices[i].pop()  # 移除当前组的最后一条
-#             extracted_slices[i + 1].insert(0, last_record)  # 插入到下一组的开头
-
-#     return extracted_slices, remaining_records
 
 
 def group_records(records):
@@ -948,6 +874,45 @@ def chunk_check(data,MAX_length=8000):
                     "positions": positions  # 共享原始位置信息
                 })
     return slice_reset
+
+def find_ship_info_by_hull(json_file_path, data):
+    """
+    从NER结果中提取舷号，再从JSON文件中查找对应的舰船信息
+    :param json_file_path: JSON文件路径
+    :param data: get_entity() 返回的NER结果字典
+    :return: 包含 model_name 和 ship_name 的字典，未找到返回 None
+    """
+    # 1. 提取舷号
+    hulls = [
+        entity['properties']['舷号']
+        for entity in data.get('entities', [])
+        if 'properties' in entity and '舷号' in entity['properties']
+    ]
+    if not hulls:
+        return None
+
+    target_hull = hulls[0]
+
+    # 2. 根据舷号查找舰船信息
+    try:
+        with open(json_file_path, 'r', encoding='utf-8') as f:
+            ship_data = json.load(f)
+
+        for item in ship_data:
+            if str(item.get('hull_number')) == str(target_hull):
+                return {
+                    "model_name": item.get('model_name'),
+                    "ship_name": item.get('ship_name')
+                }
+
+        return None
+
+    except FileNotFoundError:
+        print(f"错误：找不到文件 {json_file_path}")
+        return None
+    except json.JSONDecodeError:
+        print("错误：JSON 文件格式不正确")
+        return None
 def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动机-维修手册.pdf"):
     """
     合并过短的切片：
@@ -1005,10 +970,15 @@ def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动
             })
     try:
         final_result = get_entity(filename)
+        print(final_result)
+        xinghao = find_ship_info_by_hull(SHIP_MODEL_NAME,final_result)
+        if xinghao.get('model_name'):
+            model_name = xinghao['model_name']
+        else:
+            model_name = ""
         # lower_entity = final_result.get("low_level")
         otehr_data = format_entity_text(final_result)
         logger.info(f"文件名实体提取成功: {otehr_data}")
-        logger.info("11111111111111111111111111111111111111111")
     except Exception as e:
         logger.warning(f"文件名实体提取失败: {e}")
         otehr_data = ""  # 建议加个兜底,避免后面 NameError
@@ -1018,12 +988,15 @@ def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动
         if not otehr_data:
             continue
         newline_idx = content.find('\n')
+        # 拼接成一个括号：otehr_data + 型号为xxx
+        info = otehr_data
+        if model_name:
+            info += f'，型号为{model_name}'
+        suffix = f'({info})'
         if newline_idx == -1:
-            # 没有换行符,整个 content 就是第一行,直接追加
+            ins['content'] = content + suffix
-            ins['content'] = content + f'({otehr_data})'
         else:
-            # 在第一个换行符之前插入 (lower_entity)
+            ins['content'] = content[:newline_idx] + suffix + content[newline_idx:]
-            ins['content'] = content[:newline_idx] + f'({otehr_data})' + content[newline_idx:]
     return result
 def find_and_read_content_list(directory, original_filename, encoding='utf-8'):
     """
@@ -1155,7 +1128,7 @@ async def process_other_file(pdf_path: Path, image_prefix: str, filename: str) -
         release_api_url(api_url)
 if __name__ == "__main__":
 
-    filepath = r"E:\ZKYNLP\HJproject\知识库\122-06A0014-B01001_发动机-维修手册_content_list.json"
+    filepath = r"E:\ZKYNLP\Hjunproject\project0506\kgrag\122-06A0014-B01001_发动机-维修手册_content_list.json"
     try:
         with open(filepath, 'r', encoding='utf-8') as file:
             data = json.load(file)
@@ -1172,8 +1145,8 @@ if __name__ == "__main__":
     slices_2 = merge_short_slices(slices_1, min_length=30)   # ← 新增这一行
 
 
-    # for ins in slices_2:
+    for ins in slices_2:
-    #     print(ins)
+        print(ins)
 
 
 

config.py

@@ -73,3 +73,5 @@ API_OTHER_URLS = [
     #"http://192.168.0.111:9980/analyze-otherfile",
     # "http://192.168.0.111:9975/analyze-otherfile",
 ]
+
+SHIP_MODEL_NAME = "E:\ZKYNLP\Hjunproject\project0506\kgrag\ship_xinghao.json"

ship_xinghao.json

@@ -0,0 +1,104 @@
+[
+  {
+    "id": 1,
+    "model_name": "055型驱逐舰",
+    "hull_number": 102,
+    "ship_name": "拉萨舰"
+  },
+  {
+    "id": 2,
+    "model_name": "055型驱逐舰",
+    "hull_number": 103,
+    "ship_name": "鞍山舰"
+  },
+  {
+    "id": 3,
+    "model_name": "055型驱逐舰",
+    "hull_number": 104,
+    "ship_name": "无锡舰"
+  },
+  {
+    "id": 4,
+    "model_name": "055型驱逐舰",
+    "hull_number": 105,
+    "ship_name": "大连舰"
+  },
+  {
+    "id": 5,
+    "model_name": "055型驱逐舰",
+    "hull_number": 106,
+    "ship_name": "延安舰"
+  },
+  {
+    "id": 6,
+    "model_name": "055型驱逐舰",
+    "hull_number": 107,
+    "ship_name": "遵义舰"
+  },
+  {
+    "id": 7,
+    "model_name": "055型驱逐舰",
+    "hull_number": 108,
+    "ship_name": "咸阳舰"
+  },
+  {
+    "id": 8,
+    "model_name": "052D型驱逐舰",
+    "hull_number": 172,
+    "ship_name": "昆明舰"
+  },
+  {
+    "id": 9,
+    "model_name": "052D型驱逐舰",
+    "hull_number": 173,
+    "ship_name": "长沙舰"
+  },
+  {
+    "id": 10,
+    "model_name": "052D型驱逐舰",
+    "hull_number": 174,
+    "ship_name": "合肥舰"
+  },
+  {
+    "id": 11,
+    "model_name": "052D型驱逐舰",
+    "hull_number": 175,
+    "ship_name": "银川舰"
+  },
+  {
+    "id": 12,
+    "model_name": "052D型驱逐舰",
+    "hull_number": 163,
+    "ship_name": "南昌舰"
+  },
+  {
+    "id": 13,
+    "model_name": "054A型护卫舰",
+    "hull_number": 529,
+    "ship_name": "舟山舰"
+  },
+  {
+    "id": 14,
+    "model_name": "054A型护卫舰",
+    "hull_number": 530,
+    "ship_name": "徐州舰"
+  },
+  {
+    "id": 15,
+    "model_name": "054A型护卫舰",
+    "hull_number": 547,
+    "ship_name": "临沂舰"
+  },
+  {
+    "id": 16,
+    "model_name": "054A型护卫舰",
+    "hull_number": 568,
+    "ship_name": "衡阳舰"
+  },
+  {
+    "id": 17,
+    "model_name": "054A型护卫舰",
+    "hull_number": 570,
+    "ship_name": "黄山舰"
+  }
+]