app.py，neo4j_indexing.py、Hybrid_graphsearch.py里面配置信息统一从app.py导入
This commit is contained in:
parent
3dff801c5d
commit
17d5ebe535
25
app.py
25
app.py
@ -92,7 +92,7 @@ from default_ontology_config import (
|
||||
get_relationships,
|
||||
get_relationship_types,
|
||||
)
|
||||
from config import NEO4J_CONFIG
|
||||
from config import NEO4J_CONFIG,URL,API_URLS,API_OTHER_URLS
|
||||
converter = Doc2PDF()
|
||||
DATA_DIR = "/app/files"
|
||||
# ================== 全局资源容器 ==================
|
||||
@ -199,24 +199,11 @@ NEO4J_URI = NEO4J_CONFIG["uri"]
|
||||
NEO4J_USER = NEO4J_CONFIG["username"]
|
||||
NEO4J_PASSWORD = NEO4J_CONFIG["password"] # 不设默认值，强制要求提供
|
||||
NEO4J_DATABASE =NEO4J_CONFIG["database"]
|
||||
indexing_url = "http://192.168.1.108:9085/neo4j_indexing" # 根据实际部署调整
|
||||
PREFIX_URL = os.getenv("KGRAG_PREFIX_URL", "http://192.168.1.108:9085")
|
||||
API_URLS = [
|
||||
"http://192.168.1.64:18000/analyze-pdf",
|
||||
# "http://192.168.0.111:9977/analyze-pdf",
|
||||
# "http://192.168.0.111:9978/analyze-pdf",
|
||||
#"http://192.168.0.111:9979/analyze-pdf",
|
||||
#"http://192.168.0.111:9980/analyze-pdf",
|
||||
# "http://192.168.0.111:9975/analyze-pdf",
|
||||
]
|
||||
API_other_URLS = [
|
||||
"http://192.168.1.64:18000/analyze-otherfile",
|
||||
# "http://192.168.0.111:9977/analyze-otherfile",
|
||||
# "http://192.168.0.111:9978/analyze-otherfile",
|
||||
#"http://192.168.0.111:9979/analyze-otherfile",
|
||||
#"http://192.168.0.111:9980/analyze-otherfile",
|
||||
# "http://192.168.0.111:9975/analyze-otherfile",
|
||||
]
|
||||
indexing_url = URL['indexing_url'] # 根据实际部署调整
|
||||
PREFIX_URL = URL['prefix_url']
|
||||
API_URLS = API_URLS
|
||||
API_other_URLS = API_OTHER_URLS
|
||||
|
||||
_api_url_inflight = [0] * len(API_URLS)
|
||||
_api_url_lock = threading.Lock()
|
||||
|
||||
|
||||
38
config.py
38
config.py
@ -5,26 +5,25 @@ import os
|
||||
# ==================== 大模型配置 ====================
|
||||
LLM_CONFIG = {
|
||||
"model": "46-qwen3.5-35B",
|
||||
"base_url": "https://openai.zkzdht.com/v1",
|
||||
"base_url": "http://192.168.0.46:59800/v1",
|
||||
"api_key": "gpustack_dee9ca823290886c_5edfc86aeeeceb1e9ee5162941cb2cb5",
|
||||
"temperature": 0.1,
|
||||
"max_tokens": 25096,
|
||||
"max_tokens": 55096,
|
||||
"timeout": 30
|
||||
}
|
||||
|
||||
# ==================== 嵌入模型配置 ====================
|
||||
EMBEDDING_CONFIG = {
|
||||
"model": "bge-m3",
|
||||
"base_url": "http://embedding:9700/v1/embeddings",
|
||||
"base_url": "http://192.168.0.46:59700/v1/embeddings",
|
||||
"api_key": "gpustack_dee9ca823290886c_5edfc86aeeeceb1e9ee5162941cb2cb5",
|
||||
"base_url_v2": "http://embedding:9700/v1",
|
||||
}
|
||||
|
||||
# ==================== Neo4j 图数据库配置 ====================
|
||||
NEO4J_CONFIG = {
|
||||
# 优先读取环境变量，如果没有则使用默认值
|
||||
# 注意：本地开发通常用 localhost，部署时用服务名 neo4j
|
||||
"uri": "bolt://neo4j:7687",
|
||||
"uri": "neo4j://192.168.0.46:57687",
|
||||
"username": "neo4j",
|
||||
"password": "zdht123@",
|
||||
"database": "neo4j", # 新增 database 配置
|
||||
@ -38,12 +37,39 @@ SEARCH_CONFIG = {
|
||||
"search_label": "Searchable",
|
||||
"vector_index_name": "global_searchable_embedding",
|
||||
"fulltext_index_name": "global_searchable_content_search",
|
||||
"fulltext_field_index_name": "searchable_fulltext",
|
||||
"fulltext_field_index_name": "global_searchable_fulltext_search",
|
||||
"global_entity_embedding" : "global_entity_embedding",
|
||||
"global_entity_content_search": "global_entity_content_search",
|
||||
"excluded_business_labels": [
|
||||
"Entity", "Chunk", "Document", "_Bloom_Perspective_", "Searchable"
|
||||
],
|
||||
"vector_dimension": 1024,
|
||||
"jieba_pos_whitelist": ["n", "nr", "ns", "nt", "nz", "vn", "eng"],
|
||||
"embed_batch_size" : 64,
|
||||
"embed_thread_num" : 4,
|
||||
"node_fetch_page" : 10000,
|
||||
}
|
||||
|
||||
INDEXING_ULR = "http://192.168.0.46:59085/neo4j_indexing"
|
||||
|
||||
URL = {
|
||||
"indexing_url" : "http://192.168.0.46:59085/neo4j_indexing",
|
||||
"prefix_url" : "http://192.168.0.46:59085",
|
||||
}
|
||||
|
||||
API_URLS = [
|
||||
"http://192.168.0.64:59988/analyze-pdf",
|
||||
# "http://192.168.0.111:9977/analyze-pdf",
|
||||
# "http://192.168.0.111:9978/analyze-pdf",
|
||||
#"http://192.168.0.111:9979/analyze-pdf",
|
||||
#"http://192.168.0.111:9980/analyze-pdf",
|
||||
# "http://192.168.0.111:9975/analyze-pdf",
|
||||
]
|
||||
API_OTHER_URLS = [
|
||||
"http://192.168.0.46:59988/analyze-otherfile",
|
||||
# "http://192.168.0.111:9977/analyze-otherfile",
|
||||
# "http://192.168.0.111:9978/analyze-otherfile",
|
||||
#"http://192.168.0.111:9979/analyze-otherfile",
|
||||
#"http://192.168.0.111:9980/analyze-otherfile",
|
||||
# "http://192.168.0.111:9975/analyze-otherfile",
|
||||
]
|
||||
@ -3,8 +3,10 @@ import httpx
|
||||
import json
|
||||
import asyncio
|
||||
import warnings
|
||||
import socket
|
||||
from typing import List, Set, Tuple, Dict, Optional
|
||||
import ast
|
||||
from urllib.parse import quote, urlsplit, urlunsplit
|
||||
warnings.filterwarnings("ignore", category=UserWarning, module='jieba._compat')
|
||||
import re
|
||||
import jieba
|
||||
@ -21,6 +23,12 @@ if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env"))
|
||||
load_dotenv()
|
||||
except ImportError:
|
||||
pass
|
||||
from config import LLM_CONFIG, EMBEDDING_CONFIG, NEO4J_CONFIG, SEARCH_CONFIG
|
||||
|
||||
# ✅ 2. 【关键】全局清除代理环境变量，防止内网请求走代理
|
||||
@ -30,14 +38,22 @@ for _key in ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']:
|
||||
print(f"[Init] Removed environment variable: {_key}")
|
||||
|
||||
|
||||
# ================== 2. 基础配置 ==================
|
||||
# 如果 config.py 中没有 Neo4j 配置，这里暂时保留硬编码，建议也移入 config.py
|
||||
# ================== 2. 基础配置 ==================
|
||||
# 从配置文件中加载参数
|
||||
URI = os.getenv("NEO4J_URI", NEO4J_CONFIG["uri"])
|
||||
AUTH = (NEO4J_CONFIG["username"], NEO4J_CONFIG["password"])
|
||||
# URI = "bolt://192.168.1.164:7687"
|
||||
# AUTH = ("neo4j", "zdht123@")
|
||||
def _first_env(*names: str) -> Optional[str]:
|
||||
for name in names:
|
||||
value = os.getenv(name)
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
URI = NEO4J_CONFIG["uri"]
|
||||
AUTH = (
|
||||
NEO4J_CONFIG["username"],
|
||||
NEO4J_CONFIG["password"],
|
||||
)
|
||||
RERANK_URL = EMBEDDING_CONFIG["base_url"]
|
||||
NAME_PROPERTY = SEARCH_CONFIG["name_property"]
|
||||
FULLTEXT_PROPERTY = SEARCH_CONFIG["fulltext_property"]
|
||||
EMBEDDING_PROPERTY = SEARCH_CONFIG["embedding_property"]
|
||||
@ -67,9 +83,9 @@ _SCHEMA_LOCK = asyncio.Lock()
|
||||
class LocalBgeM3Embeddings(Embedder):
|
||||
def __init__(self, base_url: str = None, api_key: str = None, model_name: str = None):
|
||||
# 使用默认配置，允许外部覆盖
|
||||
self.base_url = base_url or EMBEDDING_CONFIG["base_url"]
|
||||
self.api_key = api_key or EMBEDDING_CONFIG["api_key"]
|
||||
self.model_name = model_name or EMBEDDING_CONFIG["model"]
|
||||
self.base_url = EMBEDDING_CONFIG["base_url"]
|
||||
self.api_key = EMBEDDING_CONFIG["api_key"]
|
||||
self.model_name = EMBEDDING_CONFIG["model"]
|
||||
|
||||
def embed_query(self, text: str) -> List[float]:
|
||||
return self.embed_documents([text])[0]
|
||||
@ -100,6 +116,7 @@ class LocalBgeM3Embeddings(Embedder):
|
||||
|
||||
return embedding_list
|
||||
|
||||
|
||||
async def openai_chat_aysnc_nothink(query: str, timeout: int = None) -> str | None:
|
||||
"""异步调用 OpenAI 兼容 API"""
|
||||
global _LLM_CLIENT
|
||||
@ -164,7 +181,6 @@ def _build_retriever(sync_driver, embedder) -> HybridCypherRetriever:
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
# ================== 6. Schema 解析与 N 跳标签扩展 ==================
|
||||
def parse_schema_relationships(schema_str: str) -> List[Tuple[str, str, str]]:
|
||||
pattern = re.compile(r"\(:([^)]+)\)-\[:([^\]]+)\]->\(:([^)]+)\)")
|
||||
@ -249,7 +265,7 @@ async def rerank_query(query: str, documents: List[str], top_n: int = 3) -> List
|
||||
"Authorization": f'Bearer {os.getenv("OPENAI_API_KEY", "none")}',
|
||||
}
|
||||
data = {
|
||||
"model": "bge-rerank",
|
||||
"model": _first_env("RERANK_MODEL", "OPENAI_RERANKER_MODEL") or "bge-rerank",
|
||||
"query": query,
|
||||
"top_n": top_n,
|
||||
"documents": documents,
|
||||
@ -257,7 +273,7 @@ async def rerank_query(query: str, documents: List[str], top_n: int = 3) -> List
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(
|
||||
"http://rerank:9600/v1/rerank",
|
||||
RERANK_URL,
|
||||
headers=headers,
|
||||
json=data,
|
||||
timeout=30.0,
|
||||
|
||||
@ -14,6 +14,7 @@ from neo4j_graphrag.indexes import (
|
||||
from modelsAPI.model_api import OpenaiAPI
|
||||
from dotenv import load_dotenv
|
||||
from typing import List, Set, Tuple, Dict, Optional
|
||||
from config import LLM_CONFIG, EMBEDDING_CONFIG, NEO4J_CONFIG, SEARCH_CONFIG
|
||||
|
||||
# 加载 .env 文件中的环境变量
|
||||
load_dotenv()
|
||||
@ -27,37 +28,37 @@ if not logger.handlers:
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
vector_dim = 1024 # OpenAI bge-m3 模型的嵌入向量维度
|
||||
embed_batch_size = 64 # 嵌入向量计算批次大小
|
||||
EMBED_MAX_WORKERS = 4 # embedding 批次并行线程数（受下游 OpenaiAPI 限流约束，勿过大）
|
||||
NODE_FETCH_PAGE = 10000 # 单次拉取待生成 embedding 节点的分页大小
|
||||
VECTOR_DIMENSION = SEARCH_CONFIG["vector_dimension"]
|
||||
vector_dim = SEARCH_CONFIG["vector_dimension"] # OpenAI bge-m3 模型的嵌入向量维度
|
||||
embed_batch_size = SEARCH_CONFIG["embed_batch_size"] # 嵌入向量计算批次大小
|
||||
EMBED_MAX_WORKERS = SEARCH_CONFIG["embed_thread_num"] # embedding 批次并行线程数（受下游 OpenaiAPI 限流约束，勿过大）
|
||||
NODE_FETCH_PAGE = SEARCH_CONFIG["node_fetch_page"] # 单次拉取待生成 embedding 节点的分页大小
|
||||
|
||||
# 全局索引名称（覆盖所有节点标签）
|
||||
GLOBAL_VECTOR_INDEX_NAME = "global_entity_embedding"
|
||||
GLOBAL_FULLTEXT_INDEX_NAME = "global_entity_content_search"
|
||||
DEFAULT_URI = "bolt://192.168.0.46:57687"
|
||||
DEFAULT_AUTH = ("neo4j", "zdht123@")
|
||||
GLOBAL_VECTOR_INDEX_NAME = SEARCH_CONFIG["global_entity_embedding"]
|
||||
GLOBAL_FULLTEXT_INDEX_NAME = SEARCH_CONFIG["global_entity_content_search"]
|
||||
DEFAULT_URI = NEO4J_CONFIG["uri"]
|
||||
DEFAULT_AUTH = (
|
||||
NEO4J_CONFIG["username"],
|
||||
NEO4J_CONFIG["password"],
|
||||
)
|
||||
|
||||
NAME_PROPERTY = "名称"
|
||||
FULLTEXT_PROPERTY = "fulltext"
|
||||
EMBEDDING_PROPERTY = "embedding"
|
||||
SEARCH_LABEL = "Searchable"
|
||||
NAME_PROPERTY = SEARCH_CONFIG["name_property"]
|
||||
FULLTEXT_PROPERTY = SEARCH_CONFIG["fulltext_property"]
|
||||
EMBEDDING_PROPERTY = SEARCH_CONFIG["embedding_property"]
|
||||
SEARCH_LABEL = SEARCH_CONFIG["search_label"]
|
||||
|
||||
VECTOR_INDEX_NAME = "global_searchable_embedding"
|
||||
FULLTEXT_INDEX_NAME = "global_searchable_content_search"
|
||||
FULLTEXT_FIELD_INDEX_NAME = "global_searchable_fulltext_search"
|
||||
VECTOR_INDEX_NAME = SEARCH_CONFIG["vector_index_name"]
|
||||
FULLTEXT_INDEX_NAME = SEARCH_CONFIG["fulltext_index_name"]
|
||||
FULLTEXT_FIELD_INDEX_NAME = SEARCH_CONFIG["fulltext_field_index_name"]
|
||||
|
||||
VECTOR_DIMENSION: int = 1024
|
||||
|
||||
EXCLUDED_BUSINESS_LABELS: Set[str] = {
|
||||
"Entity", "Chunk", "Document", "_Bloom_Perspective_", SEARCH_LABEL,
|
||||
}
|
||||
EXCLUDED_BUSINESS_LABELS: Set[str] = set(SEARCH_CONFIG["excluded_business_labels"])
|
||||
OLD_INDEX_NAMES: List[str] = [
|
||||
VECTOR_INDEX_NAME,
|
||||
FULLTEXT_INDEX_NAME,
|
||||
FULLTEXT_FIELD_INDEX_NAME,
|
||||
"global_entity_embedding",
|
||||
"global_entity_content_search",
|
||||
GLOBAL_VECTOR_INDEX_NAME,
|
||||
GLOBAL_FULLTEXT_INDEX_NAME,
|
||||
]
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user