225 lines
8.6 KiB
Python
225 lines
8.6 KiB
Python
import re
|
||
from typing import List, Tuple
|
||
#from chunk_text import _split_markdown,read_markdown_file
|
||
|
||
# def extract_tables(content: str) -> tuple[list[str], str]:
|
||
# """
|
||
# 提取所有 HTML 表格，并从原文中移除它们。
|
||
|
||
# :param content: 原始 HTML/文本内容
|
||
# :return: (tables: List[str], cleaned_content: str)
|
||
# """
|
||
# pattern = r'<table\b[^>]*>.*?</table>'
|
||
|
||
# # 提取所有表格
|
||
# tables = re.findall(pattern, content, flags=re.DOTALL | re.IGNORECASE)
|
||
|
||
# # 删除所有表格（替换为空）
|
||
# cleaned_content = re.sub(pattern, '', content, flags=re.DOTALL | re.IGNORECASE)
|
||
|
||
# return tables, cleaned_content
|
||
|
||
|
||
|
||
|
||
#提取常规表格内容
|
||
def extract_tables(content: str, start_len: int = 300,end_len:int = 50) -> Tuple[List[str], List[str], str]:
|
||
"""
|
||
提取所有 HTML 表格及其前后上下文（各最多 context_len 个字符），并从原文中移除表格及上下文。
|
||
|
||
:param content: 原始 HTML/文本内容
|
||
:param context_len: 上下文长度（默认 100 字符）
|
||
:return: (tables: List[str], contexts: List[str], cleaned_content: str)
|
||
其中 contexts[i] 是 tables[i] 对应的 "<前文> + <table>... + <后文>" 字符串
|
||
"""
|
||
# 构造正则：捕获最多 context_len 个任意字符（非贪婪）+ table 标签 + 最多 context_len 个任意字符
|
||
pattern = rf'(.{{0,{start_len}}}?)(<table\b[^>]*>.*?</table>)(.{{0,{end_len}}})'
|
||
|
||
matches = re.finditer(pattern, content, flags=re.DOTALL | re.IGNORECASE)
|
||
|
||
tables = []
|
||
contexts = []
|
||
cleaned_content = content
|
||
|
||
# 从后往前替换，避免位置偏移问题
|
||
for match in reversed(list(matches)):
|
||
pre_context = match.group(1)
|
||
table = match.group(2)
|
||
post_context = match.group(3)
|
||
|
||
full_context = pre_context + table + post_context
|
||
contexts.append(full_context)
|
||
tables.append(table)
|
||
|
||
# 从原文中删除整个上下文片段（包括前后文和表格）
|
||
cleaned_content = cleaned_content[:match.start()] + cleaned_content[match.end():]
|
||
|
||
# 因为是从后往前删的，所以 tables 和 contexts 顺序是反的，需要反转回来
|
||
tables.reverse()
|
||
contexts.reverse()
|
||
|
||
return tables, contexts, cleaned_content
|
||
|
||
|
||
#提取特殊维修项目表
|
||
def extract_repair_tables_keep_first(content: str) -> tuple[list[str], str]:
|
||
"""
|
||
提取所有维修表格，但只在原文中保留第一个，其余全部删除。
|
||
|
||
:param content: 原始 HTML/文本内容
|
||
:return: (tables: List[str], cleaned_content: str)
|
||
- tables: 所有提取到的表格（包括第一个）
|
||
- cleaned_content: 原文中仅保留第一个表格，其余已删除
|
||
"""
|
||
# 构建起始和结束模式
|
||
start_literal = (
|
||
'<table><tr>'
|
||
'<td rowspan=1 colspan=1>维修项目编号</td>'
|
||
'<td rowspan=1 colspan=1>组成编码</td>'
|
||
'<td rowspan=1 colspan=1>名称</td>'
|
||
'<td rowspan=1 colspan=1>维修项目</td>'
|
||
'<td rowspan=1 colspan=1>维修间隔期</td>'
|
||
)
|
||
start_pattern = re.escape(start_literal)
|
||
end_pattern = r'<table><tr><td>页码：\d+/\d+</td><td>版本号：\d+</td></tr></table>'
|
||
|
||
full_pattern = f'({start_pattern}.*?{end_pattern})'
|
||
|
||
# 查找所有匹配（带位置信息）
|
||
matches = list(re.finditer(full_pattern, content, flags=re.DOTALL))
|
||
|
||
# 提取所有表格内容
|
||
tables = [match.group(0) for match in matches]
|
||
|
||
# 如果没有匹配或只有一个，直接返回
|
||
if len(matches) <= 1:
|
||
return tables, content
|
||
|
||
# 从后往前删除第2个及之后的表格（索引从1开始）
|
||
cleaned = content
|
||
# 注意：reversed(matches[1:]) 会按原始顺序的逆序删除，避免偏移
|
||
for match in reversed(matches[1:]):
|
||
start, end = match.span()
|
||
cleaned = cleaned[:start] + cleaned[end:]
|
||
|
||
return tables, cleaned
|
||
|
||
|
||
|
||
# def extract_operation_tables(content: str) -> Tuple[List[str], str]:
|
||
# """
|
||
# 提取所有以“项目编号 + 操作项目”为表头的操作表格（兼容多种HTML格式），
|
||
# 但只在原文中保留第一个，其余全部删除。
|
||
# """
|
||
# # 通用起始模式：匹配任意 <td> 标签，只要文本是“项目编号”和“操作项目”
|
||
# start_pattern = r'''
|
||
# <table [^>]*> \s*
|
||
# <tr [^>]*> \s*
|
||
# <td [^>]*> \s* 项目编号 \s* </td> \s*
|
||
# <td [^>]*> \s* 操作项目 \s* </td> \s*
|
||
# </tr> \s* <tr [^>]*>
|
||
# '''
|
||
|
||
# # 结尾模式：页码+版本号（动态数字）
|
||
# end_pattern = r'<table [^>]*> \s* <tr [^>]*> \s* <td [^>]*> 页码：\d+/\d+ </td> \s* <td [^>]*> 版本号：\d+ </td> \s* </tr> \s* </table>'
|
||
|
||
# # 合并为完整非贪婪匹配
|
||
# full_pattern = f'({start_pattern}.*?{end_pattern})'
|
||
|
||
# # 使用 VERBOSE 忽略空格和注释，DOTALL 让 . 匹配换行符
|
||
# matches = list(re.finditer(full_pattern, content, flags=re.DOTALL | re.VERBOSE))
|
||
|
||
# tables = [match.group(0) for match in matches]
|
||
|
||
# if len(matches) <= 1:
|
||
# return tables, content
|
||
|
||
# # 从后往前删除第2个及之后的表格
|
||
# cleaned = content
|
||
# for match in reversed(matches[1:]):
|
||
# start, end = match.span()
|
||
# cleaned = cleaned[:start] + cleaned[end:]
|
||
|
||
# return tables, cleaned
|
||
|
||
|
||
#
|
||
def extract_operation_tables(content: str) -> Tuple[List[str], str]:
|
||
"""
|
||
提取所有以“项目编号 + 操作项目”为表头的操作表格。
|
||
每个表格的范围：从匹配的 start_pattern 开始，到下一个 start_pattern 之前（或文档结尾）。
|
||
同时生成切片：「前100字上下文 + 表格内容」。
|
||
cleaned_content 中仅保留第一个表格（原始形式），其余删除。
|
||
"""
|
||
# 起始模式（修复版）
|
||
start_pattern = r'''
|
||
<table\b[^>]*> \s*
|
||
<tr\b[^>]*> \s*
|
||
<td\b[^>]*> \s* 项目编号 \s* </td> \s*
|
||
<td\b[^>]*> \s* 操作项目 \s* </td> \s*
|
||
</tr> \s*
|
||
<tr\b[^>]*>
|
||
'''
|
||
|
||
# 查找所有起始位置（只关心 .start()）
|
||
matches = list(re.finditer(start_pattern, content, flags=re.DOTALL | re.VERBOSE))
|
||
|
||
if not matches:
|
||
return [], content
|
||
|
||
# 构建每个表格的完整范围：[start_i, start_{i+1}) 或 [start_i, end)
|
||
table_ranges = []
|
||
for i in range(len(matches)):
|
||
start_pos = matches[i].start()
|
||
if i + 1 < len(matches):
|
||
end_pos = matches[i + 1].start() # 不包含下一个表格
|
||
else:
|
||
end_pos = len(content) # 最后一个到文档结尾
|
||
table_ranges.append((start_pos, end_pos))
|
||
|
||
# 提取每个表格的 HTML 内容
|
||
tables_html = [content[start:end] for start, end in table_ranges]
|
||
|
||
# 构建 slices: 前100字 + 表格HTML
|
||
slices = []
|
||
for (start, _), html in zip(table_ranges, tables_html):
|
||
prefix_start = max(0, start - 100)
|
||
prefix = content[prefix_start:start]
|
||
content = prefix + html
|
||
content = content.replace("# 操作步骤..","操作步骤..")
|
||
content = content.replace("操作步骤..","# 操作步骤")
|
||
|
||
slices.append(content)
|
||
|
||
# 构建 cleaned_content：只保留第一个表格（从 start0 到 end0），其余删除
|
||
if len(table_ranges) <= 1:
|
||
cleaned = content
|
||
else:
|
||
# 保留 [0, end0)
|
||
first_end = table_ranges[0][1]
|
||
cleaned = content[:first_end]
|
||
cleaned = content
|
||
# 从后往前删除第2个及之后的表格区域
|
||
for start, end in reversed(table_ranges[1:]):
|
||
cleaned = cleaned[:start] + cleaned[end:]
|
||
|
||
return slices, cleaned
|
||
# if __name__ == "__main__":
|
||
# file_path = r"F:\zklnlp\HJ\hj_kg_code\file\output_caozuoshiyongsc.md"
|
||
# output_log_file = r"F:\zklnlp\HJ\code\extraction_results_wxsc.txt" # 保存结果的 txt 文件路径
|
||
|
||
# markdown_content = read_markdown_file(file_path)
|
||
# if markdown_content is None:
|
||
# print("无法读取 Markdown 文件，程序退出。")
|
||
# exit(1)
|
||
|
||
# content, headers = _split_markdown(markdown_content, chunk_size=3000, chunk_overlap=100)
|
||
# # all_results,replacecontent = extract_wxindextable(wx_table,markdown_content)
|
||
# wx_tables, cleaned_wx_content = extract_repair_tables_keep_first(content=markdown_content)
|
||
# opration_tables, cleaned_opration_content = extract_operation_tables(cleaned_wx_content)
|
||
# nromaltabel,contexts,cleaned_normal_content = extract_tables(cleaned_opration_content)
|
||
# for ins in opration_tables:
|
||
# print(ins)
|
||
# print(111111111111111)
|
||
|