76 lines
2.9 KiB
Python
76 lines
2.9 KiB
Python
|
|
from langchain_text_splitters import CharacterTextSplitter,TokenTextSplitter,RecursiveCharacterTextSplitter
|
|
from markdown_splits import convert_numbered_headings,parse_markdown,header_postprocess
|
|
|
|
def split_01(md_Document,chunk_length=256,chunk_overlap=50):
|
|
md_header_json = parse_markdown(md_Document)
|
|
# print(md_header_json)
|
|
#合并处理
|
|
md_header_list = header_postprocess(md_header_json)
|
|
for i ,md in enumerate(md_header_list):
|
|
if md["text"]:
|
|
header_1 = md["Header_1"]
|
|
header_2 = md["Header_2"]
|
|
header_3 = md["Header_3"]
|
|
text = md["text"]
|
|
text_splitter = RecursiveCharacterTextSplitter(
|
|
chunk_size=chunk_length,
|
|
chunk_overlap=chunk_overlap,
|
|
length_function=len,
|
|
separators=[" "]
|
|
)
|
|
texts = text_splitter.split_text(text)
|
|
for txt in texts:
|
|
print("Header_1:",header_1)
|
|
print("Header_2:",header_2)
|
|
print("Header_3:",header_3)
|
|
print("text:",txt)
|
|
print("======================================")
|
|
return md_header_list
|
|
|
|
|
|
def split_02(md_Document,chunk_length=256,chunk_overlap=50):
|
|
md_header_json = parse_markdown(md_Document)
|
|
# print(md_header_json)
|
|
#合并处理
|
|
md_header_list = header_postprocess(md_header_json)
|
|
for i ,md in enumerate(md_header_list):
|
|
if md["text"]:
|
|
header_1 = md["Header_1"]
|
|
header_2 = md["Header_2"]
|
|
header_3 = md["Header_3"]
|
|
text = md["text"]
|
|
print("Header_1:",header_1)
|
|
print("Header_2:",header_2)
|
|
print("Header_3:",header_3)
|
|
print("text:",text)
|
|
print("======================================")
|
|
return md_header_list
|
|
|
|
def split_03(md_Document,chunk_length=256,chunk_overlap=50):
|
|
# text_splitter = text_splitter = CharacterTextSplitter(
|
|
# separator="\n\n",
|
|
# chunk_size=chunk_length,
|
|
# chunk_overlap=chunk_overlap,
|
|
# length_function=len,
|
|
# is_separator_regex=False,
|
|
# )
|
|
text_splitter = TokenTextSplitter(chunk_size=chunk_length, chunk_overlap=chunk_overlap)
|
|
# text_splitter = RecursiveCharacterTextSplitter(
|
|
# chunk_size=chunk_length,
|
|
# chunk_overlap=chunk_overlap,
|
|
# length_function=len,
|
|
# separators=[""]
|
|
# )
|
|
texts = text_splitter.split_text(md_Document)
|
|
return texts
|
|
|
|
if __name__ == "__main__":
|
|
with open("邱茜茜.md","r",encoding="utf-8") as f:
|
|
md_Document = f.read()
|
|
# texts = split_03(md_Document)
|
|
# for text in texts:
|
|
# print("______________")
|
|
# print(text)
|
|
# split_02(md_Document)
|
|
split_01(md_Document) |