app.py,neo4j_indexing.py、Hybrid_graphsearch.py里面配置信息统一从app.py导入

This commit is contained in:
Defeng 2026-07-06 15:32:47 +08:00
parent 3dff801c5d
commit 17d5ebe535
4 changed files with 90 additions and 60 deletions

25
app.py
View File

@ -92,7 +92,7 @@ from default_ontology_config import (
get_relationships, get_relationships,
get_relationship_types, get_relationship_types,
) )
from config import NEO4J_CONFIG from config import NEO4J_CONFIG,URL,API_URLS,API_OTHER_URLS
converter = Doc2PDF() converter = Doc2PDF()
DATA_DIR = "/app/files" DATA_DIR = "/app/files"
# ================== 全局资源容器 ================== # ================== 全局资源容器 ==================
@ -199,24 +199,11 @@ NEO4J_URI = NEO4J_CONFIG["uri"]
NEO4J_USER = NEO4J_CONFIG["username"] NEO4J_USER = NEO4J_CONFIG["username"]
NEO4J_PASSWORD = NEO4J_CONFIG["password"] # 不设默认值,强制要求提供 NEO4J_PASSWORD = NEO4J_CONFIG["password"] # 不设默认值,强制要求提供
NEO4J_DATABASE =NEO4J_CONFIG["database"] NEO4J_DATABASE =NEO4J_CONFIG["database"]
indexing_url = "http://192.168.1.108:9085/neo4j_indexing" # 根据实际部署调整 indexing_url = URL['indexing_url'] # 根据实际部署调整
PREFIX_URL = os.getenv("KGRAG_PREFIX_URL", "http://192.168.1.108:9085") PREFIX_URL = URL['prefix_url']
API_URLS = [ API_URLS = API_URLS
"http://192.168.1.64:18000/analyze-pdf", API_other_URLS = API_OTHER_URLS
# "http://192.168.0.111:9977/analyze-pdf",
# "http://192.168.0.111:9978/analyze-pdf",
#"http://192.168.0.111:9979/analyze-pdf",
#"http://192.168.0.111:9980/analyze-pdf",
# "http://192.168.0.111:9975/analyze-pdf",
]
API_other_URLS = [
"http://192.168.1.64:18000/analyze-otherfile",
# "http://192.168.0.111:9977/analyze-otherfile",
# "http://192.168.0.111:9978/analyze-otherfile",
#"http://192.168.0.111:9979/analyze-otherfile",
#"http://192.168.0.111:9980/analyze-otherfile",
# "http://192.168.0.111:9975/analyze-otherfile",
]
_api_url_inflight = [0] * len(API_URLS) _api_url_inflight = [0] * len(API_URLS)
_api_url_lock = threading.Lock() _api_url_lock = threading.Lock()

View File

@ -5,26 +5,25 @@ import os
# ==================== 大模型配置 ==================== # ==================== 大模型配置 ====================
LLM_CONFIG = { LLM_CONFIG = {
"model": "46-qwen3.5-35B", "model": "46-qwen3.5-35B",
"base_url": "https://openai.zkzdht.com/v1", "base_url": "http://192.168.0.46:59800/v1",
"api_key": "gpustack_dee9ca823290886c_5edfc86aeeeceb1e9ee5162941cb2cb5", "api_key": "gpustack_dee9ca823290886c_5edfc86aeeeceb1e9ee5162941cb2cb5",
"temperature": 0.1, "temperature": 0.1,
"max_tokens": 25096, "max_tokens": 55096,
"timeout": 30 "timeout": 30
} }
# ==================== 嵌入模型配置 ==================== # ==================== 嵌入模型配置 ====================
EMBEDDING_CONFIG = { EMBEDDING_CONFIG = {
"model": "bge-m3", "model": "bge-m3",
"base_url": "http://embedding:9700/v1/embeddings", "base_url": "http://192.168.0.46:59700/v1/embeddings",
"api_key": "gpustack_dee9ca823290886c_5edfc86aeeeceb1e9ee5162941cb2cb5", "api_key": "gpustack_dee9ca823290886c_5edfc86aeeeceb1e9ee5162941cb2cb5",
"base_url_v2": "http://embedding:9700/v1",
} }
# ==================== Neo4j 图数据库配置 ==================== # ==================== Neo4j 图数据库配置 ====================
NEO4J_CONFIG = { NEO4J_CONFIG = {
# 优先读取环境变量,如果没有则使用默认值 # 优先读取环境变量,如果没有则使用默认值
# 注意:本地开发通常用 localhost部署时用服务名 neo4j # 注意:本地开发通常用 localhost部署时用服务名 neo4j
"uri": "bolt://neo4j:7687", "uri": "neo4j://192.168.0.46:57687",
"username": "neo4j", "username": "neo4j",
"password": "zdht123@", "password": "zdht123@",
"database": "neo4j", # 新增 database 配置 "database": "neo4j", # 新增 database 配置
@ -38,12 +37,39 @@ SEARCH_CONFIG = {
"search_label": "Searchable", "search_label": "Searchable",
"vector_index_name": "global_searchable_embedding", "vector_index_name": "global_searchable_embedding",
"fulltext_index_name": "global_searchable_content_search", "fulltext_index_name": "global_searchable_content_search",
"fulltext_field_index_name": "searchable_fulltext", "fulltext_field_index_name": "global_searchable_fulltext_search",
"global_entity_embedding" : "global_entity_embedding",
"global_entity_content_search": "global_entity_content_search",
"excluded_business_labels": [ "excluded_business_labels": [
"Entity", "Chunk", "Document", "_Bloom_Perspective_", "Searchable" "Entity", "Chunk", "Document", "_Bloom_Perspective_", "Searchable"
], ],
"vector_dimension": 1024, "vector_dimension": 1024,
"jieba_pos_whitelist": ["n", "nr", "ns", "nt", "nz", "vn", "eng"], "jieba_pos_whitelist": ["n", "nr", "ns", "nt", "nz", "vn", "eng"],
"embed_batch_size" : 64,
"embed_thread_num" : 4,
"node_fetch_page" : 10000,
} }
INDEXING_ULR = "http://192.168.0.46:59085/neo4j_indexing"
URL = {
"indexing_url" : "http://192.168.0.46:59085/neo4j_indexing",
"prefix_url" : "http://192.168.0.46:59085",
}
API_URLS = [
"http://192.168.0.64:59988/analyze-pdf",
# "http://192.168.0.111:9977/analyze-pdf",
# "http://192.168.0.111:9978/analyze-pdf",
#"http://192.168.0.111:9979/analyze-pdf",
#"http://192.168.0.111:9980/analyze-pdf",
# "http://192.168.0.111:9975/analyze-pdf",
]
API_OTHER_URLS = [
"http://192.168.0.46:59988/analyze-otherfile",
# "http://192.168.0.111:9977/analyze-otherfile",
# "http://192.168.0.111:9978/analyze-otherfile",
#"http://192.168.0.111:9979/analyze-otherfile",
#"http://192.168.0.111:9980/analyze-otherfile",
# "http://192.168.0.111:9975/analyze-otherfile",
]

View File

@ -3,8 +3,10 @@ import httpx
import json import json
import asyncio import asyncio
import warnings import warnings
import socket
from typing import List, Set, Tuple, Dict, Optional from typing import List, Set, Tuple, Dict, Optional
import ast import ast
from urllib.parse import quote, urlsplit, urlunsplit
warnings.filterwarnings("ignore", category=UserWarning, module='jieba._compat') warnings.filterwarnings("ignore", category=UserWarning, module='jieba._compat')
import re import re
import jieba import jieba
@ -21,6 +23,12 @@ if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace") sys.stdout.reconfigure(encoding="utf-8", errors="replace")
sys.stderr.reconfigure(encoding="utf-8", errors="replace") sys.stderr.reconfigure(encoding="utf-8", errors="replace")
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
try:
from dotenv import load_dotenv
load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env"))
load_dotenv()
except ImportError:
pass
from config import LLM_CONFIG, EMBEDDING_CONFIG, NEO4J_CONFIG, SEARCH_CONFIG from config import LLM_CONFIG, EMBEDDING_CONFIG, NEO4J_CONFIG, SEARCH_CONFIG
# ✅ 2. 【关键】全局清除代理环境变量,防止内网请求走代理 # ✅ 2. 【关键】全局清除代理环境变量,防止内网请求走代理
@ -30,14 +38,22 @@ for _key in ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']:
print(f"[Init] Removed environment variable: {_key}") print(f"[Init] Removed environment variable: {_key}")
# ================== 2. 基础配置 ==================
# 如果 config.py 中没有 Neo4j 配置,这里暂时保留硬编码,建议也移入 config.py
# ================== 2. 基础配置 ================== # ================== 2. 基础配置 ==================
# 从配置文件中加载参数 # 从配置文件中加载参数
URI = os.getenv("NEO4J_URI", NEO4J_CONFIG["uri"]) def _first_env(*names: str) -> Optional[str]:
AUTH = (NEO4J_CONFIG["username"], NEO4J_CONFIG["password"]) for name in names:
# URI = "bolt://192.168.1.164:7687" value = os.getenv(name)
# AUTH = ("neo4j", "zdht123@") if value:
return value
return None
URI = NEO4J_CONFIG["uri"]
AUTH = (
NEO4J_CONFIG["username"],
NEO4J_CONFIG["password"],
)
RERANK_URL = EMBEDDING_CONFIG["base_url"]
NAME_PROPERTY = SEARCH_CONFIG["name_property"] NAME_PROPERTY = SEARCH_CONFIG["name_property"]
FULLTEXT_PROPERTY = SEARCH_CONFIG["fulltext_property"] FULLTEXT_PROPERTY = SEARCH_CONFIG["fulltext_property"]
EMBEDDING_PROPERTY = SEARCH_CONFIG["embedding_property"] EMBEDDING_PROPERTY = SEARCH_CONFIG["embedding_property"]
@ -67,9 +83,9 @@ _SCHEMA_LOCK = asyncio.Lock()
class LocalBgeM3Embeddings(Embedder): class LocalBgeM3Embeddings(Embedder):
def __init__(self, base_url: str = None, api_key: str = None, model_name: str = None): def __init__(self, base_url: str = None, api_key: str = None, model_name: str = None):
# 使用默认配置,允许外部覆盖 # 使用默认配置,允许外部覆盖
self.base_url = base_url or EMBEDDING_CONFIG["base_url"] self.base_url = EMBEDDING_CONFIG["base_url"]
self.api_key = api_key or EMBEDDING_CONFIG["api_key"] self.api_key = EMBEDDING_CONFIG["api_key"]
self.model_name = model_name or EMBEDDING_CONFIG["model"] self.model_name = EMBEDDING_CONFIG["model"]
def embed_query(self, text: str) -> List[float]: def embed_query(self, text: str) -> List[float]:
return self.embed_documents([text])[0] return self.embed_documents([text])[0]
@ -100,6 +116,7 @@ class LocalBgeM3Embeddings(Embedder):
return embedding_list return embedding_list
async def openai_chat_aysnc_nothink(query: str, timeout: int = None) -> str | None: async def openai_chat_aysnc_nothink(query: str, timeout: int = None) -> str | None:
"""异步调用 OpenAI 兼容 API""" """异步调用 OpenAI 兼容 API"""
global _LLM_CLIENT global _LLM_CLIENT
@ -164,7 +181,6 @@ def _build_retriever(sync_driver, embedder) -> HybridCypherRetriever:
""" """
) )
# ================== 6. Schema 解析与 N 跳标签扩展 ================== # ================== 6. Schema 解析与 N 跳标签扩展 ==================
def parse_schema_relationships(schema_str: str) -> List[Tuple[str, str, str]]: def parse_schema_relationships(schema_str: str) -> List[Tuple[str, str, str]]:
pattern = re.compile(r"\(:([^)]+)\)-\[:([^\]]+)\]->\(:([^)]+)\)") pattern = re.compile(r"\(:([^)]+)\)-\[:([^\]]+)\]->\(:([^)]+)\)")
@ -249,7 +265,7 @@ async def rerank_query(query: str, documents: List[str], top_n: int = 3) -> List
"Authorization": f'Bearer {os.getenv("OPENAI_API_KEY", "none")}', "Authorization": f'Bearer {os.getenv("OPENAI_API_KEY", "none")}',
} }
data = { data = {
"model": "bge-rerank", "model": _first_env("RERANK_MODEL", "OPENAI_RERANKER_MODEL") or "bge-rerank",
"query": query, "query": query,
"top_n": top_n, "top_n": top_n,
"documents": documents, "documents": documents,
@ -257,7 +273,7 @@ async def rerank_query(query: str, documents: List[str], top_n: int = 3) -> List
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
response = await client.post( response = await client.post(
"http://rerank:9600/v1/rerank", RERANK_URL,
headers=headers, headers=headers,
json=data, json=data,
timeout=30.0, timeout=30.0,

View File

@ -14,6 +14,7 @@ from neo4j_graphrag.indexes import (
from modelsAPI.model_api import OpenaiAPI from modelsAPI.model_api import OpenaiAPI
from dotenv import load_dotenv from dotenv import load_dotenv
from typing import List, Set, Tuple, Dict, Optional from typing import List, Set, Tuple, Dict, Optional
from config import LLM_CONFIG, EMBEDDING_CONFIG, NEO4J_CONFIG, SEARCH_CONFIG
# 加载 .env 文件中的环境变量 # 加载 .env 文件中的环境变量
load_dotenv() load_dotenv()
@ -26,38 +27,38 @@ if not logger.handlers:
handler = logging.StreamHandler() handler = logging.StreamHandler()
handler.setFormatter(formatter) handler.setFormatter(formatter)
logger.addHandler(handler) logger.addHandler(handler)
vector_dim = 1024 # OpenAI bge-m3 模型的嵌入向量维度 VECTOR_DIMENSION = SEARCH_CONFIG["vector_dimension"]
embed_batch_size = 64 # 嵌入向量计算批次大小 vector_dim = SEARCH_CONFIG["vector_dimension"] # OpenAI bge-m3 模型的嵌入向量维度
EMBED_MAX_WORKERS = 4 # embedding 批次并行线程数(受下游 OpenaiAPI 限流约束,勿过大) embed_batch_size = SEARCH_CONFIG["embed_batch_size"] # 嵌入向量计算批次大小
NODE_FETCH_PAGE = 10000 # 单次拉取待生成 embedding 节点的分页大小 EMBED_MAX_WORKERS = SEARCH_CONFIG["embed_thread_num"] # embedding 批次并行线程数(受下游 OpenaiAPI 限流约束,勿过大)
NODE_FETCH_PAGE = SEARCH_CONFIG["node_fetch_page"] # 单次拉取待生成 embedding 节点的分页大小
# 全局索引名称(覆盖所有节点标签) # 全局索引名称(覆盖所有节点标签)
GLOBAL_VECTOR_INDEX_NAME = "global_entity_embedding" GLOBAL_VECTOR_INDEX_NAME = SEARCH_CONFIG["global_entity_embedding"]
GLOBAL_FULLTEXT_INDEX_NAME = "global_entity_content_search" GLOBAL_FULLTEXT_INDEX_NAME = SEARCH_CONFIG["global_entity_content_search"]
DEFAULT_URI = "bolt://192.168.0.46:57687" DEFAULT_URI = NEO4J_CONFIG["uri"]
DEFAULT_AUTH = ("neo4j", "zdht123@") DEFAULT_AUTH = (
NEO4J_CONFIG["username"],
NEO4J_CONFIG["password"],
)
NAME_PROPERTY = "名称" NAME_PROPERTY = SEARCH_CONFIG["name_property"]
FULLTEXT_PROPERTY = "fulltext" FULLTEXT_PROPERTY = SEARCH_CONFIG["fulltext_property"]
EMBEDDING_PROPERTY = "embedding" EMBEDDING_PROPERTY = SEARCH_CONFIG["embedding_property"]
SEARCH_LABEL = "Searchable" SEARCH_LABEL = SEARCH_CONFIG["search_label"]
VECTOR_INDEX_NAME = "global_searchable_embedding" VECTOR_INDEX_NAME = SEARCH_CONFIG["vector_index_name"]
FULLTEXT_INDEX_NAME = "global_searchable_content_search" FULLTEXT_INDEX_NAME = SEARCH_CONFIG["fulltext_index_name"]
FULLTEXT_FIELD_INDEX_NAME = "global_searchable_fulltext_search" FULLTEXT_FIELD_INDEX_NAME = SEARCH_CONFIG["fulltext_field_index_name"]
VECTOR_DIMENSION: int = 1024 EXCLUDED_BUSINESS_LABELS: Set[str] = set(SEARCH_CONFIG["excluded_business_labels"])
EXCLUDED_BUSINESS_LABELS: Set[str] = {
"Entity", "Chunk", "Document", "_Bloom_Perspective_", SEARCH_LABEL,
}
OLD_INDEX_NAMES: List[str] = [ OLD_INDEX_NAMES: List[str] = [
VECTOR_INDEX_NAME, VECTOR_INDEX_NAME,
FULLTEXT_INDEX_NAME, FULLTEXT_INDEX_NAME,
FULLTEXT_FIELD_INDEX_NAME, FULLTEXT_FIELD_INDEX_NAME,
"global_entity_embedding", GLOBAL_VECTOR_INDEX_NAME,
"global_entity_content_search", GLOBAL_FULLTEXT_INDEX_NAME,
] ]