Ascend/kgrag
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新 chunk_text.py

Browse Source 
更新API_URLS从config.py导入
This commit is contained in:
Defeng 2026-07-07 14:10:09 +08:00
parent 98896c3f2f
commit 0dad49a050
1 changed files with 6 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
chunk_text.py
View File

@ -16,25 +16,10 @@ import threading
import aiofiles import aiofiles
import httpx import httpx
import logging import logging
from config import SHIP_MODEL_NAME from config import SHIP_MODEL_NAME,API_URLS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
API_URLS = [
"http://192.168.1.108:9988/analyze-pdf",
# "http://192.168.0.111:9977/analyze-pdf",
# "http://192.168.0.111:9978/analyze-pdf",
#"http://192.168.0.111:9979/analyze-pdf",
#"http://192.168.0.111:9980/analyze-pdf",
# "http://192.168.0.111:9975/analyze-pdf",
]
API_other_URLS = [
"http://192.168.1.108:59988/analyze-otherfile",
# "http://192.168.0.111:9977/analyze-otherfile",
# "http://192.168.0.111:9978/analyze-otherfile",
#"http://192.168.0.111:9979/analyze-otherfile",
#"http://192.168.0.111:9980/analyze-otherfile",
# "http://192.168.0.111:9975/analyze-otherfile",
]
_api_url_inflight = [0] * len(API_URLS) _api_url_inflight = [0] * len(API_URLS)
_api_url_lock = threading.Lock() _api_url_lock = threading.Lock()
@ -75,7 +60,7 @@ def get_api_url(task_id: str = "") -> tuple:
idx = _api_url_inflight.index(min(_api_url_inflight)) idx = _api_url_inflight.index(min(_api_url_inflight))
_api_url_inflight[idx] += 1 _api_url_inflight[idx] += 1
base = API_URLS[idx] base = API_URLS[idx]
return base, base.replace("/analyze-pdf", "/analyze-otherfile") return base
def release_api_url(url: str): def release_api_url(url: str):
@ -1040,7 +1025,7 @@ async def process_pdf_file(pdf_path: Path, image_prefix: str,filename:str) -> Op
slices_check = merge_short_slices(slices_check, min_length=30,filename=filename) slices_check = merge_short_slices(slices_check, min_length=30,filename=filename)
return {"slices": slices_check, "images": images} return {"slices": slices_check, "images": images}
api_url, _ = get_api_url() api_url = get_api_url()
try: try:
async with aiofiles.open(pdf_path, "rb") as f: async with aiofiles.open(pdf_path, "rb") as f:
file_content = await f.read() file_content = await f.read()
@ -1086,7 +1071,7 @@ async def process_other_file(pdf_path: Path, image_prefix: str, filename: str) -
slices_check = chunk_check(slices, 8000) slices_check = chunk_check(slices, 8000)
return {"slices": slices_check, "images": images} return {"slices": slices_check, "images": images}
api_url, _ = get_api_url() api_url = get_api_url()
analyze_other_url = api_url.replace("/analyze-pdf", "/analyze-otherfile") analyze_other_url = api_url.replace("/analyze-pdf", "/analyze-otherfile")
mime_type, _ = mimetypes.guess_type(pdf_path.name) mime_type, _ = mimetypes.guess_type(pdf_path.name)
@ -1126,6 +1111,7 @@ async def process_other_file(pdf_path: Path, image_prefix: str, filename: str) -
return None return None
finally: finally:
release_api_url(api_url) release_api_url(api_url)
if __name__ == "__main__": if __name__ == "__main__":
filepath = r"E:\ZKYNLP\Hjunproject\project0506\kgrag\122-06A0014-B01001_发动机-维修手册_content_list.json" filepath = r"E:\ZKYNLP\Hjunproject\project0506\kgrag\122-06A0014-B01001_发动机-维修手册_content_list.json"