diff --git a/app.py b/app.py index 80e92c5..ab2da2b 100644 --- a/app.py +++ b/app.py @@ -92,7 +92,7 @@ from default_ontology_config import ( get_relationships, get_relationship_types, ) -from config import NEO4J_CONFIG +from config import NEO4J_CONFIG,URL,API_URLS,API_OTHER_URLS converter = Doc2PDF() DATA_DIR = "/app/files" # ================== 全局资源容器 ================== @@ -199,24 +199,11 @@ NEO4J_URI = NEO4J_CONFIG["uri"] NEO4J_USER = NEO4J_CONFIG["username"] NEO4J_PASSWORD = NEO4J_CONFIG["password"] # 不设默认值，强制要求提供 NEO4J_DATABASE =NEO4J_CONFIG["database"] -indexing_url = "http://192.168.1.108:9085/neo4j_indexing" # 根据实际部署调整 -PREFIX_URL = os.getenv("KGRAG_PREFIX_URL", "http://192.168.1.108:9085") -API_URLS = [ - "http://192.168.1.64:18000/analyze-pdf", - # "http://192.168.0.111:9977/analyze-pdf", - # "http://192.168.0.111:9978/analyze-pdf", - #"http://192.168.0.111:9979/analyze-pdf", - #"http://192.168.0.111:9980/analyze-pdf", - # "http://192.168.0.111:9975/analyze-pdf", -] -API_other_URLS = [ - "http://192.168.1.64:18000/analyze-otherfile", - # "http://192.168.0.111:9977/analyze-otherfile", - # "http://192.168.0.111:9978/analyze-otherfile", - #"http://192.168.0.111:9979/analyze-otherfile", - #"http://192.168.0.111:9980/analyze-otherfile", - # "http://192.168.0.111:9975/analyze-otherfile", -] +indexing_url = URL['indexing_url'] # 根据实际部署调整 +PREFIX_URL = URL['prefix_url'] +API_URLS = API_URLS +API_other_URLS = API_OTHER_URLS + _api_url_inflight = [0] * len(API_URLS) _api_url_lock = threading.Lock() diff --git a/config.py b/config.py index 570189f..ccd2309 100644 --- a/config.py +++ b/config.py @@ -5,26 +5,25 @@ import os # ==================== 大模型配置 ==================== LLM_CONFIG = { "model": "46-qwen3.5-35B", - "base_url": "https://openai.zkzdht.com/v1", + "base_url": "http://192.168.0.46:59800/v1", "api_key": "gpustack_dee9ca823290886c_5edfc86aeeeceb1e9ee5162941cb2cb5", "temperature": 0.1, - "max_tokens": 25096, + "max_tokens": 55096, "timeout": 30 } # ==================== 嵌入模型配置 ==================== EMBEDDING_CONFIG = { "model": "bge-m3", - "base_url": "http://embedding:9700/v1/embeddings", + "base_url": "http://192.168.0.46:59700/v1/embeddings", "api_key": "gpustack_dee9ca823290886c_5edfc86aeeeceb1e9ee5162941cb2cb5", - "base_url_v2": "http://embedding:9700/v1", } # ==================== Neo4j 图数据库配置 ==================== NEO4J_CONFIG = { # 优先读取环境变量，如果没有则使用默认值 # 注意：本地开发通常用 localhost，部署时用服务名 neo4j - "uri": "bolt://neo4j:7687", + "uri": "neo4j://192.168.0.46:57687", "username": "neo4j", "password": "zdht123@", "database": "neo4j", # 新增 database 配置 @@ -38,12 +37,39 @@ SEARCH_CONFIG = { "search_label": "Searchable", "vector_index_name": "global_searchable_embedding", "fulltext_index_name": "global_searchable_content_search", - "fulltext_field_index_name": "searchable_fulltext", + "fulltext_field_index_name": "global_searchable_fulltext_search", + "global_entity_embedding" : "global_entity_embedding", + "global_entity_content_search": "global_entity_content_search", "excluded_business_labels": [ "Entity", "Chunk", "Document", "_Bloom_Perspective_", "Searchable" ], "vector_dimension": 1024, "jieba_pos_whitelist": ["n", "nr", "ns", "nt", "nz", "vn", "eng"], + "embed_batch_size" : 64, + "embed_thread_num" : 4, + "node_fetch_page" : 10000, } +INDEXING_ULR = "http://192.168.0.46:59085/neo4j_indexing" +URL = { + "indexing_url" : "http://192.168.0.46:59085/neo4j_indexing", + "prefix_url" : "http://192.168.0.46:59085", +} + +API_URLS = [ + "http://192.168.0.64:59988/analyze-pdf", + # "http://192.168.0.111:9977/analyze-pdf", + # "http://192.168.0.111:9978/analyze-pdf", + #"http://192.168.0.111:9979/analyze-pdf", + #"http://192.168.0.111:9980/analyze-pdf", + # "http://192.168.0.111:9975/analyze-pdf", +] +API_OTHER_URLS = [ + "http://192.168.0.46:59988/analyze-otherfile", + # "http://192.168.0.111:9977/analyze-otherfile", + # "http://192.168.0.111:9978/analyze-otherfile", + #"http://192.168.0.111:9979/analyze-otherfile", + #"http://192.168.0.111:9980/analyze-otherfile", + # "http://192.168.0.111:9975/analyze-otherfile", +] \ No newline at end of file diff --git a/graph_search/Hybrid_graphsearch.py b/graph_search/Hybrid_graphsearch.py index c156d29..5f9f7be 100644 --- a/graph_search/Hybrid_graphsearch.py +++ b/graph_search/Hybrid_graphsearch.py @@ -3,8 +3,10 @@ import httpx import json import asyncio import warnings +import socket from typing import List, Set, Tuple, Dict, Optional import ast +from urllib.parse import quote, urlsplit, urlunsplit warnings.filterwarnings("ignore", category=UserWarning, module='jieba._compat') import re import jieba @@ -21,6 +23,12 @@ if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="replace") sys.stderr.reconfigure(encoding="utf-8", errors="replace") sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +try: + from dotenv import load_dotenv + load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env")) + load_dotenv() +except ImportError: + pass from config import LLM_CONFIG, EMBEDDING_CONFIG, NEO4J_CONFIG, SEARCH_CONFIG # ✅ 2. 【关键】全局清除代理环境变量，防止内网请求走代理 @@ -30,14 +38,22 @@ for _key in ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']: print(f"[Init] Removed environment variable: {_key}") -# ================== 2. 基础配置 ================== -# 如果 config.py 中没有 Neo4j 配置，这里暂时保留硬编码，建议也移入 config.py # ================== 2. 基础配置 ================== # 从配置文件中加载参数 -URI = os.getenv("NEO4J_URI", NEO4J_CONFIG["uri"]) -AUTH = (NEO4J_CONFIG["username"], NEO4J_CONFIG["password"]) -# URI = "bolt://192.168.1.164:7687" -# AUTH = ("neo4j", "zdht123@") +def _first_env(*names: str) -> Optional[str]: + for name in names: + value = os.getenv(name) + if value: + return value + return None + + +URI = NEO4J_CONFIG["uri"] +AUTH = ( + NEO4J_CONFIG["username"], + NEO4J_CONFIG["password"], +) +RERANK_URL = EMBEDDING_CONFIG["base_url"] NAME_PROPERTY = SEARCH_CONFIG["name_property"] FULLTEXT_PROPERTY = SEARCH_CONFIG["fulltext_property"] EMBEDDING_PROPERTY = SEARCH_CONFIG["embedding_property"] @@ -67,9 +83,9 @@ _SCHEMA_LOCK = asyncio.Lock() class LocalBgeM3Embeddings(Embedder): def __init__(self, base_url: str = None, api_key: str = None, model_name: str = None): # 使用默认配置，允许外部覆盖 - self.base_url = base_url or EMBEDDING_CONFIG["base_url"] - self.api_key = api_key or EMBEDDING_CONFIG["api_key"] - self.model_name = model_name or EMBEDDING_CONFIG["model"] + self.base_url = EMBEDDING_CONFIG["base_url"] + self.api_key = EMBEDDING_CONFIG["api_key"] + self.model_name = EMBEDDING_CONFIG["model"] def embed_query(self, text: str) -> List[float]: return self.embed_documents([text])[0] @@ -100,6 +116,7 @@ class LocalBgeM3Embeddings(Embedder): return embedding_list + async def openai_chat_aysnc_nothink(query: str, timeout: int = None) -> str | None: """异步调用 OpenAI 兼容 API""" global _LLM_CLIENT @@ -164,7 +181,6 @@ def _build_retriever(sync_driver, embedder) -> HybridCypherRetriever: """ ) - # ================== 6. Schema 解析与 N 跳标签扩展 ================== def parse_schema_relationships(schema_str: str) -> List[Tuple[str, str, str]]: pattern = re.compile(r"\(:([^)]+)\)-\[:([^\]]+)\]->\(:([^)]+)\)") @@ -249,7 +265,7 @@ async def rerank_query(query: str, documents: List[str], top_n: int = 3) -> List "Authorization": f'Bearer {os.getenv("OPENAI_API_KEY", "none")}', } data = { - "model": "bge-rerank", + "model": _first_env("RERANK_MODEL", "OPENAI_RERANKER_MODEL") or "bge-rerank", "query": query, "top_n": top_n, "documents": documents, @@ -257,7 +273,7 @@ async def rerank_query(query: str, documents: List[str], top_n: int = 3) -> List async with httpx.AsyncClient() as client: response = await client.post( - "http://rerank:9600/v1/rerank", + RERANK_URL, headers=headers, json=data, timeout=30.0, diff --git a/neo4j_indexing.py b/neo4j_indexing.py index db7bf01..c490788 100644 --- a/neo4j_indexing.py +++ b/neo4j_indexing.py @@ -14,6 +14,7 @@ from neo4j_graphrag.indexes import ( from modelsAPI.model_api import OpenaiAPI from dotenv import load_dotenv from typing import List, Set, Tuple, Dict, Optional +from config import LLM_CONFIG, EMBEDDING_CONFIG, NEO4J_CONFIG, SEARCH_CONFIG # 加载 .env 文件中的环境变量 load_dotenv() @@ -26,38 +27,38 @@ if not logger.handlers: handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) - -vector_dim = 1024 # OpenAI bge-m3 模型的嵌入向量维度 -embed_batch_size = 64 # 嵌入向量计算批次大小 -EMBED_MAX_WORKERS = 4 # embedding 批次并行线程数（受下游 OpenaiAPI 限流约束，勿过大） -NODE_FETCH_PAGE = 10000 # 单次拉取待生成 embedding 节点的分页大小 + +VECTOR_DIMENSION = SEARCH_CONFIG["vector_dimension"] +vector_dim = SEARCH_CONFIG["vector_dimension"] # OpenAI bge-m3 模型的嵌入向量维度 +embed_batch_size = SEARCH_CONFIG["embed_batch_size"] # 嵌入向量计算批次大小 +EMBED_MAX_WORKERS = SEARCH_CONFIG["embed_thread_num"] # embedding 批次并行线程数（受下游 OpenaiAPI 限流约束，勿过大） +NODE_FETCH_PAGE = SEARCH_CONFIG["node_fetch_page"] # 单次拉取待生成 embedding 节点的分页大小 # 全局索引名称（覆盖所有节点标签） -GLOBAL_VECTOR_INDEX_NAME = "global_entity_embedding" -GLOBAL_FULLTEXT_INDEX_NAME = "global_entity_content_search" -DEFAULT_URI = "bolt://192.168.0.46:57687" -DEFAULT_AUTH = ("neo4j", "zdht123@") +GLOBAL_VECTOR_INDEX_NAME = SEARCH_CONFIG["global_entity_embedding"] +GLOBAL_FULLTEXT_INDEX_NAME = SEARCH_CONFIG["global_entity_content_search"] +DEFAULT_URI = NEO4J_CONFIG["uri"] +DEFAULT_AUTH = ( + NEO4J_CONFIG["username"], + NEO4J_CONFIG["password"], +) -NAME_PROPERTY = "名称" -FULLTEXT_PROPERTY = "fulltext" -EMBEDDING_PROPERTY = "embedding" -SEARCH_LABEL = "Searchable" +NAME_PROPERTY = SEARCH_CONFIG["name_property"] +FULLTEXT_PROPERTY = SEARCH_CONFIG["fulltext_property"] +EMBEDDING_PROPERTY = SEARCH_CONFIG["embedding_property"] +SEARCH_LABEL = SEARCH_CONFIG["search_label"] -VECTOR_INDEX_NAME = "global_searchable_embedding" -FULLTEXT_INDEX_NAME = "global_searchable_content_search" -FULLTEXT_FIELD_INDEX_NAME = "global_searchable_fulltext_search" +VECTOR_INDEX_NAME = SEARCH_CONFIG["vector_index_name"] +FULLTEXT_INDEX_NAME = SEARCH_CONFIG["fulltext_index_name"] +FULLTEXT_FIELD_INDEX_NAME = SEARCH_CONFIG["fulltext_field_index_name"] -VECTOR_DIMENSION: int = 1024 - -EXCLUDED_BUSINESS_LABELS: Set[str] = { - "Entity", "Chunk", "Document", "_Bloom_Perspective_", SEARCH_LABEL, -} +EXCLUDED_BUSINESS_LABELS: Set[str] = set(SEARCH_CONFIG["excluded_business_labels"]) OLD_INDEX_NAMES: List[str] = [ VECTOR_INDEX_NAME, FULLTEXT_INDEX_NAME, FULLTEXT_FIELD_INDEX_NAME, - "global_entity_embedding", - "global_entity_content_search", + GLOBAL_VECTOR_INDEX_NAME, + GLOBAL_FULLTEXT_INDEX_NAME, ]