增加chunk_text.py的切片型号信息
This commit is contained in:
parent
17d5ebe535
commit
98896c3f2f
141
chunk_text.py
141
chunk_text.py
@ -16,6 +16,7 @@ import threading
|
||||
import aiofiles
|
||||
import httpx
|
||||
import logging
|
||||
from config import SHIP_MODEL_NAME
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
API_URLS = [
|
||||
@ -52,6 +53,7 @@ def format_entity_text(data):
|
||||
# 2. 遍历 entities 列表
|
||||
for entity in data.get('entities', []):
|
||||
entity_type = entity.get('type')
|
||||
|
||||
properties = entity.get('properties', {})
|
||||
|
||||
# 3. 如果当前实体类型在映射表中，则进行拼接
|
||||
@ -269,44 +271,6 @@ def extract_operation_groups_and_remaining(records, max_following_for_last=23):
|
||||
return groups, remaining_records
|
||||
|
||||
|
||||
# def extract_wxanli(records):
|
||||
# # 主触发词：表示一个全新故障案例的开始
|
||||
|
||||
# # 所有触发词（用于识别是否属于元信息或段落标题，但不用于切片起点）
|
||||
# all_keywords = ["一、故障现象", "单位:", "故障名称：", "一、基本情况", "单位：", "故障现象及原因："]
|
||||
|
||||
# def is_main_trigger(record):
|
||||
# text = record.get("text", "")
|
||||
# return any(trigger in text for trigger in all_keywords)
|
||||
|
||||
# extracted_slices = []
|
||||
# remaining_records = []
|
||||
# i = 0
|
||||
# n = len(records)
|
||||
|
||||
# while i < n:
|
||||
# record = records[i]
|
||||
# if is_main_trigger(record):
|
||||
# # 开始一个新的故障案例 slice
|
||||
# current_slice = [record]
|
||||
# i += 1
|
||||
# # 继续收集，直到遇到下一个主触发词 或 结束
|
||||
# while i < n and not is_main_trigger(records[i]):
|
||||
# current_slice.append(records[i])
|
||||
# i += 1
|
||||
# extracted_slices.append(current_slice)
|
||||
# else:
|
||||
# # 不是以主触发词开头的记录，保留在 remaining
|
||||
# remaining_records.append(record)
|
||||
# i += 1
|
||||
|
||||
# # ✨ 优化：将每一组（除了最后一组）的最后一个记录移到下一组的开头
|
||||
# for i in range(len(extracted_slices) - 1): # 遍历到倒数第二组
|
||||
# if len(extracted_slices[i]) > 1: # 确保当前组至少有2条记录
|
||||
# last_record = extracted_slices[i].pop() # 移除当前组的最后一条
|
||||
# extracted_slices[i + 1].insert(0, last_record) # 插入到下一组的开头
|
||||
|
||||
# return extracted_slices, remaining_records
|
||||
def extract_wxanli(records):
|
||||
keywords = [
|
||||
"一、故障现象",
|
||||
@ -484,44 +448,6 @@ def split_content_bbox(data, max_length=8000):
|
||||
|
||||
return contents, highlight_lists
|
||||
|
||||
# def extract_wxanli(records):
|
||||
# # 主触发词：表示一个全新故障案例的开始
|
||||
|
||||
# # 所有触发词（用于识别是否属于元信息或段落标题，但不用于切片起点）
|
||||
# all_keywords = ["一、故障现象", "单位:", "故障名称：", "一、基本情况", "单位：", "故障现象及原因："]
|
||||
|
||||
# def is_main_trigger(record):
|
||||
# text = record.get("text", "")
|
||||
# return any(trigger in text for trigger in all_keywords)
|
||||
|
||||
# extracted_slices = []
|
||||
# remaining_records = []
|
||||
# i = 0
|
||||
# n = len(records)
|
||||
|
||||
# while i < n:
|
||||
# record = records[i]
|
||||
# if is_main_trigger(record):
|
||||
# # 开始一个新的故障案例 slice
|
||||
# current_slice = [record]
|
||||
# i += 1
|
||||
# # 继续收集，直到遇到下一个主触发词 或 结束
|
||||
# while i < n and not is_main_trigger(records[i]):
|
||||
# current_slice.append(records[i])
|
||||
# i += 1
|
||||
# extracted_slices.append(current_slice)
|
||||
# else:
|
||||
# # 不是以主触发词开头的记录，保留在 remaining
|
||||
# remaining_records.append(record)
|
||||
# i += 1
|
||||
|
||||
# # ✨ 优化：将每一组（除了最后一组）的最后一个记录移到下一组的开头
|
||||
# for i in range(len(extracted_slices) - 1): # 遍历到倒数第二组
|
||||
# if len(extracted_slices[i]) > 1: # 确保当前组至少有2条记录
|
||||
# last_record = extracted_slices[i].pop() # 移除当前组的最后一条
|
||||
# extracted_slices[i + 1].insert(0, last_record) # 插入到下一组的开头
|
||||
|
||||
# return extracted_slices, remaining_records
|
||||
|
||||
|
||||
def group_records(records):
|
||||
@ -948,6 +874,45 @@ def chunk_check(data,MAX_length=8000):
|
||||
"positions": positions # 共享原始位置信息
|
||||
})
|
||||
return slice_reset
|
||||
|
||||
def find_ship_info_by_hull(json_file_path, data):
|
||||
"""
|
||||
从NER结果中提取舷号，再从JSON文件中查找对应的舰船信息
|
||||
:param json_file_path: JSON文件路径
|
||||
:param data: get_entity() 返回的NER结果字典
|
||||
:return: 包含 model_name 和 ship_name 的字典，未找到返回 None
|
||||
"""
|
||||
# 1. 提取舷号
|
||||
hulls = [
|
||||
entity['properties']['舷号']
|
||||
for entity in data.get('entities', [])
|
||||
if 'properties' in entity and '舷号' in entity['properties']
|
||||
]
|
||||
if not hulls:
|
||||
return None
|
||||
|
||||
target_hull = hulls[0]
|
||||
|
||||
# 2. 根据舷号查找舰船信息
|
||||
try:
|
||||
with open(json_file_path, 'r', encoding='utf-8') as f:
|
||||
ship_data = json.load(f)
|
||||
|
||||
for item in ship_data:
|
||||
if str(item.get('hull_number')) == str(target_hull):
|
||||
return {
|
||||
"model_name": item.get('model_name'),
|
||||
"ship_name": item.get('ship_name')
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
except FileNotFoundError:
|
||||
print(f"错误：找不到文件 {json_file_path}")
|
||||
return None
|
||||
except json.JSONDecodeError:
|
||||
print("错误：JSON 文件格式不正确")
|
||||
return None
|
||||
def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动机-维修手册.pdf"):
|
||||
"""
|
||||
合并过短的切片：
|
||||
@ -1005,10 +970,15 @@ def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动
|
||||
})
|
||||
try:
|
||||
final_result = get_entity(filename)
|
||||
print(final_result)
|
||||
xinghao = find_ship_info_by_hull(SHIP_MODEL_NAME,final_result)
|
||||
if xinghao.get('model_name'):
|
||||
model_name = xinghao['model_name']
|
||||
else:
|
||||
model_name = ""
|
||||
# lower_entity = final_result.get("low_level")
|
||||
otehr_data = format_entity_text(final_result)
|
||||
logger.info(f"文件名实体提取成功: {otehr_data}")
|
||||
logger.info("11111111111111111111111111111111111111111")
|
||||
except Exception as e:
|
||||
logger.warning(f"文件名实体提取失败: {e}")
|
||||
otehr_data = "" # 建议加个兜底,避免后面 NameError
|
||||
@ -1018,12 +988,15 @@ def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动
|
||||
if not otehr_data:
|
||||
continue
|
||||
newline_idx = content.find('\n')
|
||||
# 拼接成一个括号：otehr_data + 型号为xxx
|
||||
info = otehr_data
|
||||
if model_name:
|
||||
info += f'，型号为{model_name}'
|
||||
suffix = f'({info})'
|
||||
if newline_idx == -1:
|
||||
# 没有换行符,整个 content 就是第一行,直接追加
|
||||
ins['content'] = content + f'({otehr_data})'
|
||||
ins['content'] = content + suffix
|
||||
else:
|
||||
# 在第一个换行符之前插入 (lower_entity)
|
||||
ins['content'] = content[:newline_idx] + f'({otehr_data})' + content[newline_idx:]
|
||||
ins['content'] = content[:newline_idx] + suffix + content[newline_idx:]
|
||||
return result
|
||||
def find_and_read_content_list(directory, original_filename, encoding='utf-8'):
|
||||
"""
|
||||
@ -1155,7 +1128,7 @@ async def process_other_file(pdf_path: Path, image_prefix: str, filename: str) -
|
||||
release_api_url(api_url)
|
||||
if __name__ == "__main__":
|
||||
|
||||
filepath = r"E:\ZKYNLP\HJproject\知识库\122-06A0014-B01001_发动机-维修手册_content_list.json"
|
||||
filepath = r"E:\ZKYNLP\Hjunproject\project0506\kgrag\122-06A0014-B01001_发动机-维修手册_content_list.json"
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as file:
|
||||
data = json.load(file)
|
||||
@ -1172,8 +1145,8 @@ if __name__ == "__main__":
|
||||
slices_2 = merge_short_slices(slices_1, min_length=30) # ← 新增这一行
|
||||
|
||||
|
||||
# for ins in slices_2:
|
||||
# print(ins)
|
||||
for ins in slices_2:
|
||||
print(ins)
|
||||
|
||||
|
||||
|
||||
|
||||
@ -72,4 +72,6 @@ API_OTHER_URLS = [
|
||||
#"http://192.168.0.111:9979/analyze-otherfile",
|
||||
#"http://192.168.0.111:9980/analyze-otherfile",
|
||||
# "http://192.168.0.111:9975/analyze-otherfile",
|
||||
]
|
||||
]
|
||||
|
||||
SHIP_MODEL_NAME = "E:\ZKYNLP\Hjunproject\project0506\kgrag\ship_xinghao.json"
|
||||
104
ship_xinghao.json
Normal file
104
ship_xinghao.json
Normal file
@ -0,0 +1,104 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"model_name": "055型驱逐舰",
|
||||
"hull_number": 102,
|
||||
"ship_name": "拉萨舰"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"model_name": "055型驱逐舰",
|
||||
"hull_number": 103,
|
||||
"ship_name": "鞍山舰"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"model_name": "055型驱逐舰",
|
||||
"hull_number": 104,
|
||||
"ship_name": "无锡舰"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"model_name": "055型驱逐舰",
|
||||
"hull_number": 105,
|
||||
"ship_name": "大连舰"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"model_name": "055型驱逐舰",
|
||||
"hull_number": 106,
|
||||
"ship_name": "延安舰"
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"model_name": "055型驱逐舰",
|
||||
"hull_number": 107,
|
||||
"ship_name": "遵义舰"
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"model_name": "055型驱逐舰",
|
||||
"hull_number": 108,
|
||||
"ship_name": "咸阳舰"
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"model_name": "052D型驱逐舰",
|
||||
"hull_number": 172,
|
||||
"ship_name": "昆明舰"
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"model_name": "052D型驱逐舰",
|
||||
"hull_number": 173,
|
||||
"ship_name": "长沙舰"
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"model_name": "052D型驱逐舰",
|
||||
"hull_number": 174,
|
||||
"ship_name": "合肥舰"
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"model_name": "052D型驱逐舰",
|
||||
"hull_number": 175,
|
||||
"ship_name": "银川舰"
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"model_name": "052D型驱逐舰",
|
||||
"hull_number": 163,
|
||||
"ship_name": "南昌舰"
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"model_name": "054A型护卫舰",
|
||||
"hull_number": 529,
|
||||
"ship_name": "舟山舰"
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"model_name": "054A型护卫舰",
|
||||
"hull_number": 530,
|
||||
"ship_name": "徐州舰"
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"model_name": "054A型护卫舰",
|
||||
"hull_number": 547,
|
||||
"ship_name": "临沂舰"
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"model_name": "054A型护卫舰",
|
||||
"hull_number": 568,
|
||||
"ship_name": "衡阳舰"
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"model_name": "054A型护卫舰",
|
||||
"hull_number": 570,
|
||||
"ship_name": "黄山舰"
|
||||
}
|
||||
]
|
||||
Loading…
x
Reference in New Issue
Block a user