Ascend/kgrag
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新 chunk_text.py

Browse Source 
增加型号切片的异常错误捕捉
This commit is contained in:
Defeng 2026-07-07 17:45:40 +08:00
parent eb07576567
commit 47b3a27812
1 changed files with 27 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
chunk_text.py
View File

@ -898,7 +898,7 @@ def find_ship_info_by_hull(json_file_path, data):
except json.JSONDecodeError: except json.JSONDecodeError:
print("错误：JSON 文件格式不正确") print("错误：JSON 文件格式不正确")
return None return None
def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动机-维修手册.pdf"): def merge_short_slices(slices, min_length=30,filename="122-06A0014-B01003_雷达-使用说明书.pdf"):
""" """
合并过短的切片： 合并过短的切片：
- 如果某切片 content 长度 <= min_length，则将其合并到下一个切片的开头 - 如果某切片 content 长度 <= min_length，则将其合并到下一个切片的开头
@ -908,6 +908,8 @@ def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动
Args: Args:
slices: [{"content": str, "positions": [...]}] slices: [{"content": str, "positions": [...]}]
min_length: 短切片的字符长度阈值（含） min_length: 短切片的字符长度阈值（含）
filename: 用于实体提取的文件名
Returns: Returns:
合并后的 slices 列表 合并后的 slices 列表
""" """
@ -934,6 +936,7 @@ def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动
positions = pending_positions + positions positions = pending_positions + positions
pending_contents = [] pending_contents = []
pending_positions = [] pending_positions = []
result.append({ result.append({
"content": content, "content": content,
"positions": positions "positions": positions
@ -953,35 +956,43 @@ def merge_short_slices(slices, min_length=30,filename="163-06A0014-B01001_发动
"content": merged_suffix, "content": merged_suffix,
"positions": pending_positions "positions": pending_positions
}) })
try: try:
final_result = get_entity(filename) final_result = get_entity(filename)
print(final_result) xinghao = find_ship_info_by_hull(SHIP_MODEL_NAME, final_result)
xinghao = find_ship_info_by_hull(SHIP_MODEL_NAME,final_result) if xinghao is None: # 确保 xinghao 为 None 时不会报错
if xinghao.get('model_name'): xinghao = {"model_name": "", "ship_name": ""}
model_name = xinghao['model_name'] print("未找到匹配的舰船信息")
else:
# 安全处理 xinghao 为 None 的情况
model_name = "" model_name = ""
# lower_entity = final_result.get("low_level") if xinghao and 'model_name' in xinghao:
otehr_data = format_entity_text(final_result) model_name = xinghao['model_name']
logger.info(f"文件名实体提取成功: {otehr_data}")
other_data = format_entity_text(final_result)
logger.info(f"文件名实体提取成功: {other_data}")
except Exception as e: except Exception as e:
logger.warning(f"文件名实体提取失败: {e}") logger.warning(f"文件名实体提取失败: {e}")
otehr_data = "" # 建议加个兜底,避免后面 NameError other_data = ""
model_name = "" # 确保异常时 model_name 有定义
# 将提取的信息（如舰艇名、型号）注入到每个切片的开头
for ins in result: for ins in result:
content = ins['content'] content = ins['content']
if not otehr_data: # 如果没有提取到有效数据，则跳过注入
continue
newline_idx = content.find('\n') newline_idx = content.find('\n')
# 拼接成一个括号：otehr_data + 型号为xxx info = other_data
info = otehr_data
if model_name: if model_name:
info += f'，型号为{model_name}' info += f'，型号为{model_name}'
suffix = f'({info})' suffix = f'({info})'
# 将信息插入到第一行末尾
if newline_idx == -1: if newline_idx == -1:
ins['content'] = content + suffix ins['content'] = content + suffix
else: else:
ins['content'] = content[:newline_idx] + suffix + content[newline_idx:] ins['content'] = content[:newline_idx] + suffix + content[newline_idx:]
return result return result
def find_and_read_content_list(directory, original_filename, encoding='utf-8'): def find_and_read_content_list(directory, original_filename, encoding='utf-8'):
""" """