63 lines
3.0 KiB
Python
63 lines
3.0 KiB
Python
from modelsAPI.model_api import OpenaiAPI
|
|
import os
|
|
def context_chunk_process(markdown_context,chunks):
|
|
def prepare_chunk(WHOLE_DOCUMENT,CHUNK_CONTENT):
|
|
prompt = f"""
|
|
<document>
|
|
{WHOLE_DOCUMENT}
|
|
</document>
|
|
Here is the chunk we want to situate within the whole document
|
|
<chunk>
|
|
{CHUNK_CONTENT}
|
|
</chunk>
|
|
Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else.
|
|
"""
|
|
chinese_prompt = f"""
|
|
<文档>
|
|
{WHOLE_DOCUMENT}
|
|
</文档>
|
|
以下是需要定位到整体文档中的片段
|
|
<片段>
|
|
{CHUNK_CONTENT}
|
|
</片段>
|
|
请简要说明该片段在整体文档中的上下文关系，以提升片段搜索效果。只需给出简洁的上下文说明，无需其他内容。
|
|
"""
|
|
return chinese_prompt
|
|
for chunk in chunks:
|
|
text = chunk.get("text","")
|
|
CONTEXT_CHUNK = prepare_chunk(markdown_context,text)
|
|
NEW_CHUNK = OpenaiAPI().open_api_caht_without_thinking(query=CONTEXT_CHUNK,model=os.getenv("OPENAI_MODEL","qwen3-32b"))
|
|
chunk["text"] = NEW_CHUNK + text
|
|
chunk["origin_text"] = text
|
|
return chunks
|
|
|
|
def abstract_chunk(markdown_context,chunks):
|
|
def prepare_chunk(WHOLE_DOCUMENT,CHUNK_CONTENT):
|
|
prompt = f"""
|
|
<document>
|
|
{WHOLE_DOCUMENT}
|
|
</document>
|
|
Here is the chunk we want to situate within the whole document
|
|
<chunk>
|
|
{CHUNK_CONTENT}
|
|
</chunk>
|
|
Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else.
|
|
"""
|
|
chinese_prompt = f"""
|
|
<文档>
|
|
{WHOLE_DOCUMENT}
|
|
</文档>
|
|
以下是需要定位到整体文档中的片段
|
|
<片段>
|
|
{CHUNK_CONTENT}
|
|
</片段>
|
|
请简要说明该片段在整体文档中的上下文关系，以提升片段搜索效果。只需给出简洁的上下文说明，无需其他内容。
|
|
"""
|
|
return chinese_prompt
|
|
for chunk in chunks:
|
|
text = chunk.get("text","")
|
|
CONTEXT_CHUNK = prepare_chunk(markdown_context,text)
|
|
NEW_CHUNK = OpenaiAPI().open_api_caht_without_thinking(query=CONTEXT_CHUNK,model=os.getenv("OPENAI_MODEL","qwen3-32b"))
|
|
chunk["text"] = NEW_CHUNK
|
|
chunk["origin_text"] = NEW_CHUNK + text
|
|
return chunks |