from langchain_text_splitters import CharacterTextSplitter,TokenTextSplitter,RecursiveCharacterTextSplitter
from markdown_splits import convert_numbered_headings,parse_markdown,header_postprocess

def split_01(md_Document,chunk_length=256,chunk_overlap=50):
    md_header_json = parse_markdown(md_Document)
    # print(md_header_json)
    #合并处理
    md_header_list = header_postprocess(md_header_json)
    for i ,md in enumerate(md_header_list):
            if md["text"]:
                header_1 = md["Header_1"]
                header_2 = md["Header_2"]
                header_3 = md["Header_3"]
                text = md["text"]
                text_splitter = RecursiveCharacterTextSplitter(
                    chunk_size=chunk_length,
                    chunk_overlap=chunk_overlap,
                    length_function=len,
                    separators=["  "]
                )
                texts = text_splitter.split_text(text)
                for txt in texts:              
                    print("Header_1:",header_1)
                    print("Header_2:",header_2)
                    print("Header_3:",header_3)
                    print("text:",txt)
                    print("======================================")
    return md_header_list


def split_02(md_Document,chunk_length=256,chunk_overlap=50):
    md_header_json = parse_markdown(md_Document)
    # print(md_header_json)
    #合并处理
    md_header_list = header_postprocess(md_header_json)
    for i ,md in enumerate(md_header_list):
            if md["text"]:
                header_1 = md["Header_1"]
                header_2 = md["Header_2"]
                header_3 = md["Header_3"]
                text = md["text"]
                print("Header_1:",header_1)
                print("Header_2:",header_2)
                print("Header_3:",header_3)
                print("text:",text)
                print("======================================")
    return md_header_list

def split_03(md_Document,chunk_length=256,chunk_overlap=50):
#     text_splitter = text_splitter = CharacterTextSplitter(
#     separator="\n\n",
#     chunk_size=chunk_length,
#     chunk_overlap=chunk_overlap,
#     length_function=len,
#     is_separator_regex=False,
# )
    text_splitter = TokenTextSplitter(chunk_size=chunk_length, chunk_overlap=chunk_overlap)
    # text_splitter = RecursiveCharacterTextSplitter(
    #         chunk_size=chunk_length,
    #         chunk_overlap=chunk_overlap,
    #         length_function=len,
    #         separators=[""]
    #     )
    texts = text_splitter.split_text(md_Document)
    return texts

if __name__ == "__main__":
    with open("邱茜茜.md","r",encoding="utf-8") as f:
        md_Document = f.read()
        # texts = split_03(md_Document)
        # for text in texts:
        #     print("______________")
        #     print(text)
        # split_02(md_Document)
        split_01(md_Document)