From 209b69964065d572ea7fa20536dd6653322aa311 Mon Sep 17 00:00:00 2001
From: zc <2649023789@qq.com>
Date: Wed, 22 Jul 2026 10:46:29 +0800
Subject: [PATCH] feat: add 49088 wiki service snapshot
---
.env.example | 9 +
.gitignore | 2 +
README.md | 24 +
admin_routes.py | 39 +
app.py | 11 +
config.py | 57 +
docker-compose.yml | 28 +
modelsAPI/model_api.py | 85 +-
prompts.py | 81 +-
tools/agent_registration.py | 86 ++
tools/conversation_cleanup.py | 106 ++
tools/graph_tools.py | 2 +
tools/wiki_tools.py | 139 +++
wiki_engine/__init__.py | 6 +
wiki_engine/chunk_enhancer.py | 100 ++
wiki_engine/db.py | 1798 ++++++++++++++++++++++++++++++
wiki_engine/htknow_client.py | 132 +++
wiki_engine/routes.py | 230 ++++
wiki_engine/service.py | 967 ++++++++++++++++
wiki_engine/wiki_builder.py | 1317 ++++++++++++++++++++++
wiki_engine/worker.py | 138 +++
workflow_registry.py | 10 +
workflows/workflow_baike.py | 976 +++++++++-------
workflows/workflow_utils.py | 547 ++++-----
workflows/workflow_wiki_admin.py | 493 ++++++++
25 files changed, 6610 insertions(+), 773 deletions(-)
create mode 100644 .env.example
create mode 100644 admin_routes.py
create mode 100644 docker-compose.yml
create mode 100644 tools/agent_registration.py
create mode 100644 tools/conversation_cleanup.py
create mode 100644 tools/wiki_tools.py
create mode 100644 wiki_engine/__init__.py
create mode 100644 wiki_engine/chunk_enhancer.py
create mode 100644 wiki_engine/db.py
create mode 100644 wiki_engine/htknow_client.py
create mode 100644 wiki_engine/routes.py
create mode 100644 wiki_engine/service.py
create mode 100644 wiki_engine/wiki_builder.py
create mode 100644 wiki_engine/worker.py
create mode 100644 workflows/workflow_wiki_admin.py
diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..102ef20
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,9 @@
+OPENAI_API_BASE=http://qwen3-14b:8000/v1
+OPENAI_API_EMBEDDING_BASE=http://bge-m3:8000/v1
+Milvus_URI=http://standalone:19350
+OPENAI_EMBEDDING_MODEL=bge-m3
+OPENAI_API_RERANKER_BASE=http://bge-rerank:8000/v1
+OPENAI_MODEL=Qwen3-14B
+OPENAI_API_KEY=replace_me
+CONVERT_DOC_URL=http://libreoffice:8000
+OPENAI_RERANKER_MODEL=bge-rerank
diff --git a/.gitignore b/.gitignore
index 4927469..dbda1af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ env/
# Local env files
.env
.env.*
+!.env.example
# Runtime data and generated outputs
data/*.db
@@ -21,6 +22,7 @@ image_output/
# Logs and temporary files
*.log
*.tmp
+*.swp
.pytest_cache/
# OS/editor files
diff --git a/README.md b/README.md
index e69de29..c2e6802 100644
--- a/README.md
+++ b/README.md
@@ -0,0 +1,24 @@
+# 46 Wiki Agent
+
+本分支保存服务器 `192.168.0.46` 上、宿主机端口 `49088` 对应的
+`46-wx-agent-wiki-49088` 服务源码快照。
+
+## Docker 启动
+
+运行前需要满足以下条件：
+
+- 本机已有镜像 `wx-agent:hj_v3`。
+- 已存在外部 Docker 网络 `weixiu-hj_test-network`，并且该网络中可以访问
+ Qwen、BGE、Milvus 和 LibreOffice 等依赖服务。
+- 将 `.env.example` 复制为 `.env`，并填写有效的 `OPENAI_API_KEY`。
+
+启动服务：
+
+```bash
+docker compose up -d
+```
+
+服务监听容器内的 `9088` 端口，并映射到宿主机的 `49088` 端口。
+
+运行期数据库、缓存、生成的 Word/PDF 文件、Python 字节码及编辑器临时文件
+不会纳入 Git。
diff --git a/admin_routes.py b/admin_routes.py
new file mode 100644
index 0000000..1d76e51
--- /dev/null
+++ b/admin_routes.py
@@ -0,0 +1,39 @@
+from datetime import datetime
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from tools.conversation_cleanup import (
+ delete_conversation_by_chat_id,
+ delete_conversations_by_time,
+)
+
+
+router = APIRouter(prefix="/api/admin", tags=["admin"])
+
+
+class DeleteConversationByChatRequest(BaseModel):
+ chat_id: str
+
+
+class DeleteConversationsByTimeRequest(BaseModel):
+ before: Optional[datetime] = None
+ after: Optional[datetime] = None
+
+
+@router.post("/conversations/delete/by-chat")
+async def delete_by_chat(request: DeleteConversationByChatRequest):
+ if not request.chat_id.strip():
+ raise HTTPException(status_code=400, detail="chat_id is empty")
+ return await delete_conversation_by_chat_id(request.chat_id)
+
+
+@router.post("/conversations/delete/by-time")
+async def delete_by_time(request: DeleteConversationsByTimeRequest):
+ if request.before is None and request.after is None:
+ raise HTTPException(status_code=400, detail="before or after is required")
+ return await delete_conversations_by_time(
+ before=request.before,
+ after=request.after,
+ )
diff --git a/app.py b/app.py
index 1d76b11..f28840a 100644
--- a/app.py
+++ b/app.py
@@ -35,6 +35,8 @@ from checkpointer_config import (
CheckpointerManager,
checkpointer_manager
)
+from admin_routes import router as admin_router
+from wiki_engine.routes import router as wiki_router
# 创建 FastAPI 应用
app = FastAPI(max_request_size=1024 * 1024 * 10)
@@ -53,6 +55,9 @@ app.add_middleware(
allow_headers=["*"],
)
+app.include_router(wiki_router)
+app.include_router(admin_router)
+
# app.mount("/picture", StaticFiles(directory="/app/picture"), name="pictures")
@@ -63,6 +68,12 @@ async def startup_event():
# 初始化智能体使用统计表
from tools.agent_usage_statistics import init_agent_usage_table
await init_agent_usage_table()
+ from wiki_engine.db import init_wiki_tables
+ await init_wiki_tables()
+ from wiki_engine.worker import start_wiki_queue_workers
+ await start_wiki_queue_workers()
+ from tools.agent_registration import register_wiki_admin_agent
+ await register_wiki_admin_agent()
logger.info("应用启动完成，PostgreSQL Checkpointer 已配置")
diff --git a/config.py b/config.py
index 96e8e04..ec7ae54 100644
--- a/config.py
+++ b/config.py
@@ -90,6 +90,63 @@ class DynamicRAGConfig(dict):
# 使用动态配置类，支持多用户并发时自动读取各自的配置
RAG_CONFIG = DynamicRAGConfig()
+# ==================== Wiki pointer layer config ====================
+_HTKNOW_BASE_URL_DEFAULT = (
+ _RAG_CONFIG_DEFAULT.get("search_endpoint", "").split("/api/v1/knowledge/search")[0].rstrip("/")
+ or "http://htknow:8080"
+)
+WIKI_CONFIG = {
+ "enabled": os.getenv("WIKI_ENABLED", "true").lower() == "true",
+ "htknow_base_url": os.getenv("HTKNOW_BASE_URL", _HTKNOW_BASE_URL_DEFAULT).rstrip("/"),
+ "default_user_id": os.getenv("WIKI_DEFAULT_USER_ID", "1"),
+ "default_user_name": os.getenv("WIKI_DEFAULT_USER_NAME", "testuser"),
+ "default_role": os.getenv("WIKI_DEFAULT_ROLE", "admin"),
+ "default_sync_limit": int(os.getenv("WIKI_DEFAULT_SYNC_LIMIT", "20")),
+ "max_slices_per_file": int(os.getenv("WIKI_MAX_SLICES_PER_FILE", "300")),
+ "max_pages_per_file": int(os.getenv("WIKI_MAX_PAGES_PER_FILE", "25")),
+ "llm_summary_enabled": os.getenv("WIKI_LLM_SUMMARY_ENABLED", "true").lower() == "true",
+ "embedding_enabled": os.getenv("WIKI_EMBEDDING_ENABLED", "false").lower() == "true",
+ "embedding_batch_size": int(os.getenv("WIKI_EMBEDDING_BATCH_SIZE", "16")),
+ "embedding_vector_dim": int(os.getenv("WIKI_EMBEDDING_VECTOR_DIM", "1024")),
+ "embedding_text_max_chars": int(os.getenv("WIKI_EMBEDDING_TEXT_MAX_CHARS", "2200")),
+ "vector_search_top_k": int(os.getenv("WIKI_VECTOR_SEARCH_TOP_K", "32")),
+ "slice_text_search_top_k": int(os.getenv("WIKI_SLICE_TEXT_SEARCH_TOP_K", "32")),
+ "page_search_top_k": int(os.getenv("WIKI_PAGE_SEARCH_TOP_K", "12")),
+ "vector_min_score": float(os.getenv("WIKI_VECTOR_MIN_SCORE", "0.2")),
+ "max_vector_candidates": int(os.getenv("WIKI_MAX_VECTOR_CANDIDATES", "50000")),
+ "rrf_k": int(os.getenv("WIKI_RRF_K", "60")),
+ "evidence_slices_per_page": int(os.getenv("WIKI_EVIDENCE_SLICES_PER_PAGE", "6")),
+ "language": os.getenv("WIKI_LANGUAGE", "Chinese"),
+ "extraction_granularity": os.getenv("WIKI_EXTRACTION_GRANULARITY", "standard"),
+ "full_content_max_chars": int(os.getenv("WIKI_FULL_CONTENT_MAX_CHARS", "120000")),
+ "citation_concurrency": int(os.getenv("WIKI_CITATION_CONCURRENCY", "3")),
+ "reduce_concurrency": int(os.getenv("WIKI_REDUCE_CONCURRENCY", "3")),
+ "llm_concurrency": int(os.getenv("WIKI_LLM_CONCURRENCY", "4")),
+ "llm_cache_enabled": os.getenv("WIKI_LLM_CACHE_ENABLED", "true").lower() == "true",
+ "llm_cache_ttl_seconds": int(os.getenv("WIKI_LLM_CACHE_TTL_SECONDS", str(7 * 24 * 3600))),
+ "llm_retry_count": int(os.getenv("WIKI_LLM_RETRY_COUNT", "2")),
+ "structured_template_enabled": os.getenv("WIKI_STRUCTURED_TEMPLATE_ENABLED", "true").lower() == "true",
+ "stage_progress_enabled": os.getenv("WIKI_STAGE_PROGRESS_ENABLED", "true").lower() == "true",
+ "finalize_enabled": os.getenv("WIKI_FINALIZE_ENABLED", "true").lower() == "true",
+ "finalize_debounce_seconds": int(os.getenv("WIKI_FINALIZE_DEBOUNCE_SECONDS", "20")),
+ "slug_lock_ttl_seconds": int(os.getenv("WIKI_SLUG_LOCK_TTL_SECONDS", "600")),
+ "queue_enabled": os.getenv("WIKI_QUEUE_ENABLED", "true").lower() == "true",
+ "queue_worker_concurrency": int(os.getenv("WIKI_QUEUE_WORKER_CONCURRENCY", "2")),
+ "queue_per_kb_concurrency": int(os.getenv("WIKI_QUEUE_PER_KB_CONCURRENCY", "2")),
+ "queue_claim_stale_seconds": int(os.getenv("WIKI_QUEUE_CLAIM_STALE_SECONDS", "1800")),
+ "queue_max_retries": int(os.getenv("WIKI_QUEUE_MAX_RETRIES", "3")),
+}
+
+# ==================== Rerank 服务配置 ====================
+RERANK_CONFIG = {
+ "enabled": os.getenv("RERANK_ENABLED", "true").lower() == "true",
+ "endpoint": os.getenv("RERANK_ENDPOINT", "http://192.168.0.46:59600/v1/rerank"),
+ "model": os.getenv("RERANK_MODEL", "bge-rerank"),
+ "timeout": float(os.getenv("RERANK_TIMEOUT", "30")),
+ "max_candidates": int(os.getenv("RERANK_MAX_CANDIDATES", "24")),
+ "top_k": int(os.getenv("RERANK_TOP_K", "10")),
+}
+
def set_request_user_config(x_user_id: Optional[str] = None,
x_user_name: Optional[str] = None,
x_role: Optional[str] = None,
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..752537a
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,28 @@
+services:
+ wx-agent:
+ image: wx-agent:hj_v3
+ container_name: 46-wx-agent-wiki-49088
+ working_dir: /app
+ tty: true
+ stdin_open: true
+ restart: unless-stopped
+ ports:
+ - "49088:9088"
+ volumes:
+ - .:/app
+ env_file:
+ - .env
+ command:
+ - uvicorn
+ - app:app
+ - --host
+ - 0.0.0.0
+ - --port
+ - "9088"
+ networks:
+ - weixiu-hj_test-network
+
+networks:
+ weixiu-hj_test-network:
+ external: true
+ name: weixiu-hj_test-network
diff --git a/modelsAPI/model_api.py b/modelsAPI/model_api.py
index e55e56b..78d1d2a 100644
--- a/modelsAPI/model_api.py
+++ b/modelsAPI/model_api.py
@@ -4,7 +4,7 @@ from typing import List, Union
import os
import numpy as np
from openai import OpenAI
-from config import LLM_CONFIG, EMBEDDING_CONFIG
+from config import LLM_CONFIG, EMBEDDING_CONFIG, RERANK_CONFIG
from neo4j_graphrag.embeddings.base import Embedder
from openai import AsyncOpenAI
@@ -24,6 +24,25 @@ def _get_async_client() -> AsyncOpenAI:
return _async_client
+def _log_model_messages(label: str, model: str, message_list: list) -> None:
+ if os.getenv("LLM_LOG_FULL_PROMPTS", "false").lower() == "true":
+ print(label, message_list)
+ return
+ if os.getenv("LLM_LOG_CALLS", "true").lower() != "true":
+ return
+ summary = []
+ total_chars = 0
+ for message in message_list or []:
+ content = message.get("content") if isinstance(message, dict) else ""
+ if isinstance(content, str):
+ chars = len(content)
+ else:
+ chars = len(str(content))
+ total_chars += chars
+ summary.append({"role": message.get("role", ""), "chars": chars})
+ print(label, {"model": model, "messages": summary, "total_chars": total_chars})
+
+
class OpenaiAPI:
@staticmethod
async def open_api_chat_without_thinking(
@@ -64,7 +83,7 @@ class OpenaiAPI:
if json_output:
kwargs["response_format"] = {"type": "json_object"}
- print("model API. history message", message_list)
+ _log_model_messages("model API call", model, message_list)
response = await client.chat.completions.create(**kwargs)
return response.choices[0].message.content
@@ -112,7 +131,7 @@ class OpenaiAPI:
if json_output:
kwargs["response_format"] = {"type": "json_object"}
- print("model API stream. history message", message_list)
+ _log_model_messages("model API stream call", model, message_list)
stream = await client.chat.completions.create(**kwargs)
async for chunk in stream:
@@ -224,6 +243,66 @@ class OpenaiAPI:
embedding = response.json()["data"][0]["embedding"]
return embedding
+ @staticmethod
+ async def rerank(
+ query: str,
+ documents: List[str],
+ top_k: int = None,
+ model: str = None,
+ ) -> List[dict]:
+ """
+ 调用 59600 rerank 服务。
+
+ 返回格式:
+ [
+ {"index": 0, "relevance_score": 0.98},
+ ...
+ ]
+ """
+ if not RERANK_CONFIG.get("enabled", True) or not query or not documents:
+ return []
+
+ max_candidates = int(RERANK_CONFIG.get("max_candidates") or 24)
+ selected_documents = [str(x or "")[:4000] for x in documents[:max_candidates]]
+
+ try:
+ async with httpx.AsyncClient(
+ timeout=float(RERANK_CONFIG.get("timeout") or 30),
+ verify=False,
+ trust_env=False,
+ ) as client:
+ response = await client.post(
+ str(RERANK_CONFIG["endpoint"]),
+ json={
+ "model": model or RERANK_CONFIG.get("model") or "bge-rerank",
+ "query": query,
+ "documents": selected_documents,
+ },
+ )
+ response.raise_for_status()
+ payload = response.json()
+ except Exception as exc:
+ print(f"[rerank] rerank service failed: {exc}")
+ return []
+
+ rows: List[dict] = []
+ for row in payload.get("results", []):
+ try:
+ idx = int(row.get("index"))
+ except Exception:
+ continue
+ if idx < 0 or idx >= len(selected_documents):
+ continue
+ rows.append({
+ "index": idx,
+ "relevance_score": float(row.get("relevance_score") or 0.0),
+ })
+
+ rows.sort(key=lambda x: float(x.get("relevance_score") or 0), reverse=True)
+ if top_k is not None:
+ rows = rows[:top_k]
+ return rows
+
@staticmethod
async def get_embeddings_batch_async(texts: List[str]) -> List[List[float]]:
embedding_base_url = EMBEDDING_CONFIG["base_url"]
diff --git a/prompts.py b/prompts.py
index 3a47400..6c8b63d 100644
--- a/prompts.py
+++ b/prompts.py
@@ -427,29 +427,19 @@ BAIKE_PROMPTS = {
# 输入变量: history_messages, original_query
# ------------------------------------------------------------------
"judge_need_retrieval": (
- "你是一个专业的问题分析助手，请判断用户的问题是否需要检索知识库才能回答。\n"
+ "You are an intent classifier for a knowledge-base QA assistant. Decide whether the user question needs retrieval.\n"
"\n"
- "【判断标准】\n"
- "需要检索的情况：\n"
- "- 询问特定设备、技术、规程、规范的具体内容\n"
- "- 需要专业知识或特定数据支持的问题\n"
- "- 询问故障诊断、维修方法等专业领域问题\n"
- "- 询问历史事件、法规条款、技术标准等需要查证的内容\n"
- "- 例如发动机的组成等\n"
+ "Return true when the question asks about documents, files, knowledge-base content, equipment, systems, faults, operations, standards, procedures, meeting notes, database-related materials, or any factual/domain-specific information.\n"
+ "Return false only for pure greetings, thanks, cancellation/confirmation with no substantive question, or a follow-up that can be fully answered from the conversation history alone.\n"
+ "Default rule: when unsure, return true.\n"
"\n"
- "不需要检索的情况：\n"
- "- 简单的问候、寒暄\n"
- "- 常识性问题（如常识性知识、简单计算等）\n"
- "- 明确不需要专业知识的问题\n"
- "- 对之前对话的简单追问，上下文已足够回答\n"
- "\n"
- "【对话历史】\n"
+ "Conversation history:\n"
"{history_messages}\n"
"\n"
- "【当前用户问题】\n"
+ "User question:\n"
"{original_query}\n"
"\n"
- "请仅输出 \"true\" 或 \"false\"（不带引号），true 表示需要检索，false 表示不需要检索。"
+ "Output only true or false."
),
# ------------------------------------------------------------------
@@ -457,29 +447,30 @@ BAIKE_PROMPTS = {
# 输入变量: last_user_query, history_str, original_query
# ------------------------------------------------------------------
"rewrite_query_with_history": (
- "你是一个专业的检索query重写助手，负责根据对话历史和当前用户问题，生成**适合向量检索/RAG 的单条查询语句**。\n"
+ "You are a query understanding assistant for a Chinese knowledge-base retrieval system.\n"
"\n"
- "请严格遵守以下要求：\n"
- "1. 只融合与当前问题语义强相关的**最近几轮对话信息**，不要机械拼接全部历史。\n"
- "2. 消除指代和省略（例如：这个、刚才那个问题、上面说的故障等），补全成自洽、完整的描述。\n"
- "3. 不要加入与用户问题无关的背景信息，不要过度扩展检索范围。\n"
- "4. 输出必须是一句自然的中文问句或陈述句，便于检索；不要包含条目符号、编号或解释性文字。\n"
- "5. 只输出改写后的query本身：\n"
- " - 不要输出说明文字\n"
- " - 不要包含「改写结果：」「检索query：」等前缀\n"
- " - 不要添加引号\n"
+ "Tasks:\n"
+ "1. Rewrite the current user question into a self-contained Chinese retrieval query.\n"
+ "2. Generate 2-5 short search queries for hybrid retrieval.\n"
"\n"
- "【特别提示】\n"
- "- 请优先参考**上一轮用户的问题**来重写本轮query。\n"
- "- 上一轮用户的问题是：{last_user_query}\n"
+ "Rules:\n"
+ "- Resolve references and ellipsis using only relevant recent history.\n"
+ "- Preserve concrete entities, file names, database/table terms, equipment names, fault names, standards, procedure names, and exact keywords.\n"
+ "- Do not output meta instructions such as “在知识库中查找”. Output the actual search content.\n"
+ "- Include compact keyword queries that help exact matching, for example document titles, equipment names, or core nouns.\n"
+ "- The rewritten query and search queries must be in Chinese unless the original keyword is English.\n"
"\n"
- "【对话历史（JSON 格式，已按时间排序）】\n"
+ "Last user question:\n"
+ "{last_user_query}\n"
+ "\n"
+ "Conversation history JSON:\n"
"{history_str}\n"
"\n"
- "【当前用户问题】\n"
+ "Current user question:\n"
"{original_query}\n"
"\n"
- "请给出最终用于检索的单条中文query。"
+ "Output ONLY valid JSON:\n"
+ "{{\"rewrite_query\":\"string\",\"search_queries\":[\"string\"]}}"
),
# ------------------------------------------------------------------
@@ -515,7 +506,13 @@ BAIKE_PROMPTS = {
# 输入变量: domain_desc
# ------------------------------------------------------------------
"generate_answer_with_retrieval_system": (
- "我是一个专业的{domain_desc}知识问答助手，根据用户的问题综合检索结果，提供准确、专业、完整的回答。回答应围绕问题组织知识点、依据和必要图示，把参考资料中的相关图片作为资料证据放在对应说明附近。只能使用参考资料中真实存在的图片链接，不得使用占位链接或自行构造图片链接。"
+ "You are a professional knowledge-base assistant for {domain_desc}.\n"
+ "Answer ONLY from the retrieved evidence provided by the user message. Do not use prior knowledge as factual evidence.\n"
+ "If the evidence is insufficient, say clearly what is missing and what can be confirmed from the evidence.\n"
+ "Prefer concrete facts, names, parameters, steps, conclusions, and source relationships from the evidence.\n"
+ "When sources conflict, explain the conflict instead of inventing a merged fact.\n"
+ "Use images only when their URLs appear in the retrieved evidence; never fabricate or repair image URLs.\n"
+ "Always respond in Chinese. Do not expose internal tool names or implementation details."
),
# ------------------------------------------------------------------
@@ -523,20 +520,20 @@ BAIKE_PROMPTS = {
# 输入变量: extracted_text, rag_count, all_source_text
# ------------------------------------------------------------------
"generate_answer_with_retrieval_user": (
+ "用户问题：\n"
"{extracted_text}\n"
"\n"
- "# 检索到的相关信息（共 {rag_count} 条）：\n"
+ "# Retrieved evidence ({rag_count} items)\n"
"\n"
"{all_source_text}\n"
"\n"
"【输出要求】\n"
- "1. 先直接回答用户问题，再按需要补充结构、原理、参数、应用场景、注意事项或资料依据\n"
- "2. 参考资料中真实存在的图片可在当前问题、设备结构、部件位置、流程步骤或现象说明对应段落附近自然展示；图片规则不要写入正文\n"
- "3. 图片只作为资料引用输出为单个 Markdown 图片标签，URL 和文件名必须逐字来自参考资料，不要改写、拆分、补全、连续重复或自行构造；同一图片资源重复出现时只引用一次\n"
- "4. 如果参考资料包含 LaTeX 公式、链接等，请在对应位置正常使用，不得修改或删除\n"
- "5. 请专注于内容的准确性、完整性和逻辑性，不要担心格式的美观性\n"
- "6. 可以适当使用简单的段落分隔，但不要使用复杂的Markdown格式\n"
- "7. 如果参考资料包含表格，请不要输出表格，而要直接描述表格的内容\n"
+ "1. 先直接回答用户问题，再补充必要依据。\n"
+ "2. 每个关键事实都必须能在 Retrieved evidence 中找到依据；不要补充资料外结论。\n"
+ "3. 如果资料只能说明部分内容，请明确说“资料中只能确认...”和“资料中未提供...”。\n"
+ "4. 优先使用 evidence 中的文件名、标题、File ID、Slice ID 做简短来源说明，但不要暴露无意义的内部参数。\n"
+ "5. 参考资料中真实存在的图片可在对应段落附近展示；URL 必须逐字来自资料。\n"
+ "6. 表格内容请转成自然语言描述，不要输出复杂表格。\n"
"\n"
"请给出回复："
),
diff --git a/tools/agent_registration.py b/tools/agent_registration.py
new file mode 100644
index 0000000..250e703
--- /dev/null
+++ b/tools/agent_registration.py
@@ -0,0 +1,86 @@
+import json
+import time
+from typing import Any, Dict
+
+from config import POSTGRES_CONNECTION_STRING
+
+
+WIKI_ADMIN_AGENT_ID = "6f0b5c4f-7d4a-4db4-9c4c-8f89b2a4f7a1"
+WIKI_ADMIN_ROLE_ID = "5"
+
+
+WIKI_ADMIN_AGENT: Dict[str, Any] = {
+ "id": WIKI_ADMIN_AGENT_ID,
+ "name": "Wiki知识库管理",
+ "description": "管理 Wiki 指针层的同步、检查、重建、清理和删除，帮助维护 11000 知识库与 Wiki 数据的一致性。",
+ "prompt": "您好，我是 Wiki 知识库管理助手。您可以直接用知识库名称让我检查同步情况、同步指定文件或知识库、清理 11000 已删除但 Wiki 仍存在的数据。我会先生成待执行计划，列出知识库、文件和动作；您确认后才会执行。删除类操作只删除 Wiki 指针层数据，不会删除 11000 原始知识库文件。",
+ "background": "Wiki 管理助手用于维护 wx-agent 内置 Wiki 指针层，不负责回答业务知识问题。",
+ "access_control": {},
+ "default_query": [
+ {"title": "检查知识库同步情况 →", "content": "检查知识库 163舰 的 Wiki 同步情况"},
+ {"title": "同步指定知识库 →", "content": "同步知识库 163舰 的前 20 个文件"},
+ {"title": "清理脏数据预览 →", "content": "清理知识库 163舰 中 11000 已删除但 Wiki 还存在的数据"},
+ {"title": "确认执行计划 →", "content": "确认执行"},
+ ],
+}
+
+
+async def register_wiki_admin_agent() -> Dict[str, Any]:
+ """Register the Wiki admin agent in an_webui tables and bind it to role 5."""
+ try:
+ from psycopg_pool import AsyncConnectionPool
+ except ImportError:
+ print("[agent_registration] psycopg_pool is not installed; skip wiki admin agent registration")
+ return {"success": False, "error": "psycopg_pool is not installed"}
+
+ now = int(time.time())
+ async with AsyncConnectionPool(
+ POSTGRES_CONNECTION_STRING,
+ kwargs={"autocommit": True},
+ ) as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO agent (
+ id, name, description, prompt, background, access_control,
+ created_time, updated_time, default_query
+ )
+ VALUES (%s, %s, %s, %s, %s, %s::json, %s, %s, %s::json)
+ ON CONFLICT (id) DO UPDATE SET
+ name = EXCLUDED.name,
+ description = EXCLUDED.description,
+ prompt = EXCLUDED.prompt,
+ background = EXCLUDED.background,
+ access_control = EXCLUDED.access_control,
+ updated_time = EXCLUDED.updated_time,
+ default_query = EXCLUDED.default_query
+ """,
+ (
+ WIKI_ADMIN_AGENT["id"],
+ WIKI_ADMIN_AGENT["name"],
+ WIKI_ADMIN_AGENT["description"],
+ WIKI_ADMIN_AGENT["prompt"],
+ WIKI_ADMIN_AGENT["background"],
+ json.dumps(WIKI_ADMIN_AGENT["access_control"], ensure_ascii=False),
+ now,
+ now,
+ json.dumps(WIKI_ADMIN_AGENT["default_query"], ensure_ascii=False),
+ ),
+ )
+ await cur.execute(
+ """
+ INSERT INTO agent_role_association (agent_id, agent_role_id, is_active)
+ VALUES (%s, %s, TRUE)
+ ON CONFLICT (agent_id, agent_role_id) DO UPDATE SET
+ is_active = EXCLUDED.is_active
+ """,
+ (WIKI_ADMIN_AGENT_ID, WIKI_ADMIN_ROLE_ID),
+ )
+
+ print(f"[agent_registration] wiki admin agent registered: {WIKI_ADMIN_AGENT_ID} -> role {WIKI_ADMIN_ROLE_ID}")
+ return {
+ "success": True,
+ "agent_id": WIKI_ADMIN_AGENT_ID,
+ "agent_role_id": WIKI_ADMIN_ROLE_ID,
+ }
diff --git a/tools/conversation_cleanup.py b/tools/conversation_cleanup.py
new file mode 100644
index 0000000..fdb98bf
--- /dev/null
+++ b/tools/conversation_cleanup.py
@@ -0,0 +1,106 @@
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from config import POSTGRES_CONNECTION_STRING
+
+
+async def delete_conversation_by_chat_id(chat_id: str) -> Dict[str, Any]:
+ """Delete wx-agent LangGraph checkpoint state for one chat_id."""
+ chat_id = str(chat_id or "").strip()
+ if not chat_id:
+ raise ValueError("chat_id is empty")
+
+ counts = {
+ "checkpoint_writes": 0,
+ "checkpoint_blobs": 0,
+ "checkpoints": 0,
+ }
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await _delete_checkpoint_threads(cur, chat_id, counts)
+ return {"success": True, "chat_id": chat_id, "deleted": counts}
+
+
+async def delete_conversations_by_time(
+ before: Optional[datetime] = None,
+ after: Optional[datetime] = None,
+) -> Dict[str, Any]:
+ """Delete LangGraph checkpoint state whose checkpoint timestamp falls in a time range."""
+ if before is None and after is None:
+ raise ValueError("before or after is required")
+
+ where_parts = []
+ params: List[Any] = []
+ if before is not None:
+ where_parts.append("(checkpoint->>'ts')::timestamptz < %s")
+ params.append(before)
+ if after is not None:
+ where_parts.append("(checkpoint->>'ts')::timestamptz >= %s")
+ params.append(after)
+ where_sql = " AND ".join(where_parts)
+
+ counts = {
+ "checkpoint_writes": 0,
+ "checkpoint_blobs": 0,
+ "checkpoints": 0,
+ }
+ deleted_thread_ids: List[str] = []
+
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"SELECT DISTINCT thread_id FROM checkpoints WHERE {where_sql}",
+ params,
+ )
+ deleted_thread_ids = [str(row[0]) for row in await cur.fetchall() if row and row[0]]
+
+ for thread_id in deleted_thread_ids:
+ await _delete_exact_thread(cur, thread_id, counts)
+
+ return {
+ "success": True,
+ "before": before.isoformat() if before else "",
+ "after": after.isoformat() if after else "",
+ "thread_count": len(deleted_thread_ids),
+ "thread_ids": deleted_thread_ids,
+ "deleted": counts,
+ }
+
+
+async def _delete_checkpoint_threads(cur, chat_id: str, counts: Dict[str, int]) -> None:
+ thread_like = f"{chat_id}:%"
+ await cur.execute(
+ "DELETE FROM checkpoint_writes WHERE thread_id = %s OR thread_id LIKE %s",
+ (chat_id, thread_like),
+ )
+ counts["checkpoint_writes"] += cur.rowcount or 0
+ await cur.execute(
+ "DELETE FROM checkpoint_blobs WHERE thread_id = %s OR thread_id LIKE %s",
+ (chat_id, thread_like),
+ )
+ counts["checkpoint_blobs"] += cur.rowcount or 0
+ await cur.execute(
+ "DELETE FROM checkpoints WHERE thread_id = %s OR thread_id LIKE %s",
+ (chat_id, thread_like),
+ )
+ counts["checkpoints"] += cur.rowcount or 0
+
+
+async def _delete_exact_thread(cur, thread_id: str, counts: Dict[str, int]) -> None:
+ await cur.execute("DELETE FROM checkpoint_writes WHERE thread_id = %s", (thread_id,))
+ counts["checkpoint_writes"] += cur.rowcount or 0
+ await cur.execute("DELETE FROM checkpoint_blobs WHERE thread_id = %s", (thread_id,))
+ counts["checkpoint_blobs"] += cur.rowcount or 0
+ await cur.execute("DELETE FROM checkpoints WHERE thread_id = %s", (thread_id,))
+ counts["checkpoints"] += cur.rowcount or 0
+
+
+async def _connect_pool():
+ from psycopg_pool import AsyncConnectionPool
+
+ return AsyncConnectionPool(
+ POSTGRES_CONNECTION_STRING,
+ kwargs={"autocommit": True},
+ )
diff --git a/tools/graph_tools.py b/tools/graph_tools.py
index 05f11ff..df61bb8 100644
--- a/tools/graph_tools.py
+++ b/tools/graph_tools.py
@@ -536,6 +536,7 @@ def _node_record_to_dict(record: Any) -> Dict[str, Any]:
return result
+@track_function_calls
async def normalize_device_name_by_ship_graph(
device_name: str,
fulltext_index: str = GLOBAL_FULLTEXT_INDEX,
@@ -902,6 +903,7 @@ def _build_fault_graph_results_text(
return "\n".join(parts)
+@track_function_calls
async def find_system_by_device(
device_name: str,
min_hops: int = 2,
diff --git a/tools/wiki_tools.py b/tools/wiki_tools.py
new file mode 100644
index 0000000..ade015a
--- /dev/null
+++ b/tools/wiki_tools.py
@@ -0,0 +1,139 @@
+from typing import Any, Dict, Optional
+
+from langchain_core.tools import tool
+
+from wiki_engine.service import (
+ delete_by_file,
+ delete_by_kb,
+ diff_kb,
+ evidence_search,
+ get_stats,
+ reconcile_kb,
+ resolve_kb_reference,
+ search_wiki,
+ sync_file,
+ sync_kb,
+)
+
+
+async def wiki_pointer_search(
+ query: str,
+ kb_id: Optional[str] = None,
+ limit: int = 5,
+) -> Dict[str, Any]:
+ """Search local Wiki pointer pages and return related 11000 slice ids."""
+ if not query or not query.strip():
+ return {"success": False, "error": "query is empty", "items": []}
+ return await search_wiki(query=query.strip(), kb_id=kb_id, limit=limit, include_slices=True)
+
+
+async def wiki_evidence_search(
+ query: str,
+ kb_id: Optional[str] = None,
+ limit: int = 5,
+) -> Dict[str, Any]:
+ """Return Wiki results in the unified evidence-like format."""
+ if not query or not query.strip():
+ return {"success": False, "error": "query is empty", "items": []}
+ return await evidence_search(query=query.strip(), kb_id=kb_id, limit=limit)
+
+
+async def wiki_admin_stats() -> Dict[str, Any]:
+ return {"success": True, "stats": await get_stats()}
+
+
+async def wiki_admin_sync_file(file_id: str, kb_id: Optional[str] = None, use_llm: bool = False) -> Dict[str, Any]:
+ return await sync_file(file_id=file_id, kb_id=kb_id, use_llm=use_llm)
+
+
+async def wiki_admin_sync_kb(
+ kb_id: Optional[str] = None,
+ kb_name: Optional[str] = None,
+ limit: int = 20,
+ use_llm: bool = False,
+) -> Dict[str, Any]:
+ target = await _resolve_kb_target(kb_id=kb_id, kb_name=kb_name)
+ if not target.get("success"):
+ return target
+ result = await sync_kb(kb_id=target["kb_id"], limit=limit, use_llm=use_llm)
+ result.update({"kb_name": target.get("kb_name", ""), "kb_path": target.get("kb_path", "")})
+ return result
+
+
+async def wiki_admin_diff_kb(
+ kb_id: Optional[str] = None,
+ kb_name: Optional[str] = None,
+ include_children: bool = False,
+ limit: int = 200,
+) -> Dict[str, Any]:
+ target = await _resolve_kb_target(kb_id=kb_id, kb_name=kb_name)
+ if not target.get("success"):
+ return target
+ result = await diff_kb(kb_id=target["kb_id"], include_children=include_children, limit=limit)
+ result.update({"kb_name": target.get("kb_name", ""), "kb_path": target.get("kb_path", "")})
+ return result
+
+
+async def wiki_admin_reconcile_kb(
+ kb_id: Optional[str] = None,
+ kb_name: Optional[str] = None,
+ include_children: bool = False,
+ limit: int = 200,
+ sync_missing: bool = True,
+ rebuild_stale: bool = True,
+ delete_orphan: bool = False,
+ use_llm: bool = False,
+ dry_run: bool = True,
+) -> Dict[str, Any]:
+ target = await _resolve_kb_target(kb_id=kb_id, kb_name=kb_name)
+ if not target.get("success"):
+ return target
+ result = await reconcile_kb(
+ kb_id=target["kb_id"],
+ include_children=include_children,
+ limit=limit,
+ sync_missing=sync_missing,
+ rebuild_stale=rebuild_stale,
+ delete_orphan=delete_orphan,
+ use_llm=use_llm,
+ dry_run=dry_run,
+ )
+ result.update({"kb_name": target.get("kb_name", ""), "kb_path": target.get("kb_path", "")})
+ return result
+
+
+async def wiki_admin_delete_file(file_id: str) -> Dict[str, Any]:
+ return await delete_by_file(file_id)
+
+
+async def wiki_admin_delete_kb(kb_id: Optional[str] = None, kb_name: Optional[str] = None) -> Dict[str, Any]:
+ target = await _resolve_kb_target(kb_id=kb_id, kb_name=kb_name)
+ if not target.get("success"):
+ return target
+ result = await delete_by_kb(target["kb_id"])
+ result.update({"kb_name": target.get("kb_name", ""), "kb_path": target.get("kb_path", "")})
+ return result
+
+
+async def _resolve_kb_target(kb_id: Optional[str] = None, kb_name: Optional[str] = None) -> Dict[str, Any]:
+ if kb_id and str(kb_id).strip():
+ return {"success": True, "kb_id": str(kb_id).strip(), "kb_name": "", "kb_path": ""}
+ if kb_name and str(kb_name).strip():
+ return await resolve_kb_reference(str(kb_name).strip())
+ return {"success": False, "error": "kb_name is required", "matches": []}
+
+
+@tool
+async def wiki_pointer_search_tool(query: str, limit: int = 5) -> str:
+ """Search Wiki pointer pages and format them for workflow prompts."""
+ result = await wiki_pointer_search(query=query, limit=limit)
+ if not result.get("success"):
+ return ""
+ parts = []
+ for item in result.get("items", []):
+ title = item.get("title", "")
+ summary = item.get("summary", "")
+ slice_ids = ", ".join(item.get("related_slice_ids", [])[:8])
+ if title or summary:
+ parts.append(f"[Wiki] {title}\n{summary}\nrelated_slice_ids: {slice_ids}")
+ return "\n\n".join(parts)
diff --git a/wiki_engine/__init__.py b/wiki_engine/__init__.py
new file mode 100644
index 0000000..719a47a
--- /dev/null
+++ b/wiki_engine/__init__.py
@@ -0,0 +1,6 @@
+"""Lightweight Wiki pointer layer for wx-agent.
+
+The module stores topic pages and enhanced slice pointers in PostgreSQL. It does
+not copy original files or become the source of truth; source text remains in
+the 11000 knowledge service.
+"""
diff --git a/wiki_engine/chunk_enhancer.py b/wiki_engine/chunk_enhancer.py
new file mode 100644
index 0000000..364f35a
--- /dev/null
+++ b/wiki_engine/chunk_enhancer.py
@@ -0,0 +1,100 @@
+import hashlib
+import re
+from collections import Counter
+from typing import Any, Dict, Iterable, List
+
+
+STOPWORDS = {
+ "的", "了", "和", "与", "及", "或", "在", "中", "对", "为", "是", "有",
+ "进行", "使用", "相关", "根据", "如下", "可以", "需要", "应当", "以及",
+}
+
+
+def enhance_slices(
+ slices: Iterable[Dict[str, Any]],
+ file_info: Dict[str, Any],
+ max_slices: int,
+) -> List[Dict[str, Any]]:
+ refs: List[Dict[str, Any]] = []
+ filename = str(file_info.get("filename") or file_info.get("name") or "")
+ kb_id = str(file_info.get("kb_id") or "")
+ file_id = str(file_info.get("file_id") or file_info.get("id") or "")
+
+ for idx, raw in enumerate(list(slices)[:max_slices]):
+ text = _slice_text(raw)
+ if not text.strip():
+ continue
+ slice_id = str(raw.get("id") or raw.get("slice_id") or raw.get("uuid") or f"{file_id}:{idx}")
+ title_path = _infer_title_path(text, filename)
+ keywords = extract_keywords(text, limit=12)
+ refs.append({
+ "kb_id": str(raw.get("kb_id") or kb_id),
+ "file_id": str(raw.get("file_id") or file_id),
+ "slice_id": slice_id,
+ "ordinal": int(raw.get("ordinal") or raw.get("index") or idx),
+ "filename": str(raw.get("filename") or filename),
+ "title_path": title_path,
+ "keywords": keywords,
+ "brief": _brief(text),
+ "chunk_type": _infer_chunk_type(text),
+ "content_hash": hashlib.sha1(text.encode("utf-8", errors="ignore")).hexdigest(),
+ "_text": text,
+ })
+ return refs
+
+
+def extract_keywords(text: str, limit: int = 10) -> List[str]:
+ text = normalize_text(text)
+ candidates = re.findall(r"[\u4e00-\u9fff]{2,12}|[A-Za-z0-9][A-Za-z0-9_\-]{1,30}", text)
+ normalized: List[str] = []
+ for token in candidates:
+ token = token.strip("：:，,。.;；、（）()[]【】")
+ if len(token) < 2 or token in STOPWORDS:
+ continue
+ if re.fullmatch(r"\d+", token):
+ continue
+ normalized.append(token)
+ counts = Counter(normalized)
+ return [term for term, _ in counts.most_common(limit)]
+
+
+def normalize_text(text: str) -> str:
+ return re.sub(r"\s+", " ", text or "").strip()
+
+
+def _slice_text(raw: Dict[str, Any]) -> str:
+ for key in ("content", "text", "page_content", "slice_content", "markdown"):
+ value = raw.get(key)
+ if isinstance(value, str) and value.strip():
+ return value
+ return ""
+
+
+def _brief(text: str, limit: int = 360) -> str:
+ return normalize_text(text)[:limit]
+
+
+def _infer_title_path(text: str, filename: str) -> List[str]:
+ path = []
+ if filename:
+ path.append(filename)
+ lines = [line.strip() for line in (text or "").splitlines() if line.strip()]
+ for line in lines[:6]:
+ line = re.sub(r"^#+\s*", "", line).strip()
+ if 2 <= len(line) <= 50 and not line.endswith(("。", ".", "；", ";")):
+ path.append(line)
+ break
+ return path[:4] if path else []
+
+
+def _infer_chunk_type(text: str) -> str:
+ compact = normalize_text(text)
+ if "|" in text and "---" in text:
+ return "table"
+ if re.search(r"(步骤|流程|操作|检查|确认|打开|关闭|启动|停止)", compact):
+ return "procedure"
+ if re.search(r"(故障|异常|报警|失效|原因|处理|排查)", compact):
+ return "fault"
+ if re.search(r"!\[.*?\]\(.*?\)", text):
+ return "image"
+ return "text"
diff --git a/wiki_engine/db.py b/wiki_engine/db.py
new file mode 100644
index 0000000..a2ee884
--- /dev/null
+++ b/wiki_engine/db.py
@@ -0,0 +1,1798 @@
+import json
+import re
+from datetime import datetime, timedelta
+from typing import Any, Dict, Iterable, List, Optional, Sequence
+
+from config import POSTGRES_CONNECTION_STRING, WIKI_CONFIG
+
+
+async def _connect_pool():
+ from psycopg_pool import AsyncConnectionPool
+
+ return AsyncConnectionPool(
+ POSTGRES_CONNECTION_STRING,
+ kwargs={"autocommit": True},
+ )
+
+
+async def init_wiki_tables() -> None:
+ """Create Wiki pointer tables in the wx-agent PostgreSQL database."""
+ try:
+ pool = await _connect_pool()
+ except ImportError:
+ print("[wiki_engine] psycopg_pool is not installed; skip wiki table init")
+ return
+
+ async with pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_sync_jobs (
+ id TEXT PRIMARY KEY,
+ target_type TEXT NOT NULL DEFAULT '',
+ target_id TEXT NOT NULL DEFAULT '',
+ kb_id TEXT NOT NULL DEFAULT '',
+ status TEXT NOT NULL DEFAULT 'pending',
+ total_files INTEGER NOT NULL DEFAULT 0,
+ processed_files INTEGER NOT NULL DEFAULT 0,
+ total_slices INTEGER NOT NULL DEFAULT 0,
+ skipped_files INTEGER NOT NULL DEFAULT 0,
+ page_count INTEGER NOT NULL DEFAULT 0,
+ stage TEXT NOT NULL DEFAULT '',
+ current_file_id TEXT NOT NULL DEFAULT '',
+ current_filename TEXT NOT NULL DEFAULT '',
+ error TEXT NOT NULL DEFAULT '',
+ created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW()
+ )
+ """)
+ await cur.execute("ALTER TABLE wiki_sync_jobs ADD COLUMN IF NOT EXISTS skipped_files INTEGER NOT NULL DEFAULT 0")
+ await cur.execute("ALTER TABLE wiki_sync_jobs ADD COLUMN IF NOT EXISTS page_count INTEGER NOT NULL DEFAULT 0")
+ await cur.execute("ALTER TABLE wiki_sync_jobs ADD COLUMN IF NOT EXISTS stage TEXT NOT NULL DEFAULT ''")
+ await cur.execute("ALTER TABLE wiki_sync_jobs ADD COLUMN IF NOT EXISTS current_file_id TEXT NOT NULL DEFAULT ''")
+ await cur.execute("ALTER TABLE wiki_sync_jobs ADD COLUMN IF NOT EXISTS current_filename TEXT NOT NULL DEFAULT ''")
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_file_state (
+ id SERIAL PRIMARY KEY,
+ kb_id TEXT NOT NULL DEFAULT '',
+ file_id TEXT NOT NULL UNIQUE,
+ filename TEXT NOT NULL DEFAULT '',
+ file_hash TEXT NOT NULL DEFAULT '',
+ status TEXT NOT NULL DEFAULT '',
+ slice_count INTEGER NOT NULL DEFAULT 0,
+ source_updated_at TEXT NOT NULL DEFAULT '',
+ metadata JSONB NOT NULL DEFAULT '{}',
+ last_synced_at TIMESTAMP NOT NULL DEFAULT NOW()
+ )
+ """)
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_slice_refs (
+ id SERIAL PRIMARY KEY,
+ kb_id TEXT NOT NULL DEFAULT '',
+ file_id TEXT NOT NULL DEFAULT '',
+ slice_id TEXT NOT NULL UNIQUE,
+ ordinal INTEGER NOT NULL DEFAULT 0,
+ filename TEXT NOT NULL DEFAULT '',
+ title_path JSONB NOT NULL DEFAULT '[]',
+ keywords JSONB NOT NULL DEFAULT '[]',
+ brief TEXT NOT NULL DEFAULT '',
+ content TEXT NOT NULL DEFAULT '',
+ chunk_type TEXT NOT NULL DEFAULT 'text',
+ content_hash TEXT NOT NULL DEFAULT '',
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW()
+ )
+ """)
+ await cur.execute("ALTER TABLE wiki_slice_refs ADD COLUMN IF NOT EXISTS content TEXT NOT NULL DEFAULT ''")
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_pages (
+ id SERIAL PRIMARY KEY,
+ kb_id TEXT NOT NULL DEFAULT '',
+ slug TEXT NOT NULL DEFAULT '',
+ title TEXT NOT NULL DEFAULT '',
+ page_type TEXT NOT NULL DEFAULT 'topic',
+ summary TEXT NOT NULL DEFAULT '',
+ content TEXT NOT NULL DEFAULT '',
+ keywords JSONB NOT NULL DEFAULT '[]',
+ aliases JSONB NOT NULL DEFAULT '[]',
+ related_file_ids JSONB NOT NULL DEFAULT '[]',
+ source_refs JSONB NOT NULL DEFAULT '[]',
+ chunk_refs JSONB NOT NULL DEFAULT '[]',
+ source_kind TEXT NOT NULL DEFAULT 'rule',
+ score_hint REAL NOT NULL DEFAULT 1.0,
+ created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ UNIQUE(kb_id, title, page_type)
+ )
+ """)
+ await cur.execute("ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS slug TEXT NOT NULL DEFAULT ''")
+ await cur.execute("ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS content TEXT NOT NULL DEFAULT ''")
+ await cur.execute("ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS aliases JSONB NOT NULL DEFAULT '[]'")
+ await cur.execute("ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS source_refs JSONB NOT NULL DEFAULT '[]'")
+ await cur.execute("ALTER TABLE wiki_pages ADD COLUMN IF NOT EXISTS chunk_refs JSONB NOT NULL DEFAULT '[]'")
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_page_slices (
+ id SERIAL PRIMARY KEY,
+ wiki_page_id INTEGER NOT NULL REFERENCES wiki_pages(id) ON DELETE CASCADE,
+ file_id TEXT NOT NULL DEFAULT '',
+ slice_id TEXT NOT NULL DEFAULT '',
+ weight REAL NOT NULL DEFAULT 1.0,
+ reason TEXT NOT NULL DEFAULT '',
+ UNIQUE(wiki_page_id, slice_id)
+ )
+ """)
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_query_logs (
+ id SERIAL PRIMARY KEY,
+ query TEXT NOT NULL DEFAULT '',
+ kb_id TEXT NOT NULL DEFAULT '',
+ result_count INTEGER NOT NULL DEFAULT 0,
+ created_at TIMESTAMP NOT NULL DEFAULT NOW()
+ )
+ """)
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_pending_ops (
+ id BIGSERIAL PRIMARY KEY,
+ job_id TEXT NOT NULL DEFAULT '',
+ kb_id TEXT NOT NULL DEFAULT '',
+ file_id TEXT NOT NULL DEFAULT '',
+ op TEXT NOT NULL DEFAULT 'sync_file',
+ dedup_key TEXT NOT NULL DEFAULT '',
+ payload JSONB NOT NULL DEFAULT '{}',
+ status TEXT NOT NULL DEFAULT 'pending',
+ fail_count INTEGER NOT NULL DEFAULT 0,
+ error TEXT NOT NULL DEFAULT '',
+ claimed_at TIMESTAMP,
+ enqueued_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW()
+ )
+ """)
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_dead_letters (
+ id BIGSERIAL PRIMARY KEY,
+ job_id TEXT NOT NULL DEFAULT '',
+ kb_id TEXT NOT NULL DEFAULT '',
+ file_id TEXT NOT NULL DEFAULT '',
+ op TEXT NOT NULL DEFAULT '',
+ payload JSONB NOT NULL DEFAULT '{}',
+ last_error TEXT NOT NULL DEFAULT '',
+ fail_count INTEGER NOT NULL DEFAULT 0,
+ failed_at TIMESTAMP NOT NULL DEFAULT NOW()
+ )
+ """)
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_llm_cache (
+ cache_key TEXT PRIMARY KEY,
+ response TEXT NOT NULL DEFAULT '',
+ hit_count INTEGER NOT NULL DEFAULT 0,
+ created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW()
+ )
+ """)
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_page_locks (
+ kb_id TEXT NOT NULL DEFAULT '',
+ slug TEXT NOT NULL DEFAULT '',
+ owner TEXT NOT NULL DEFAULT '',
+ locked_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ expires_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (kb_id, slug)
+ )
+ """)
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_finalize_ops (
+ id BIGSERIAL PRIMARY KEY,
+ job_id TEXT NOT NULL DEFAULT '',
+ kb_id TEXT NOT NULL DEFAULT '',
+ status TEXT NOT NULL DEFAULT 'pending',
+ payload JSONB NOT NULL DEFAULT '{}',
+ claimed_at TIMESTAMP,
+ run_after TIMESTAMP NOT NULL DEFAULT NOW(),
+ enqueued_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW()
+ )
+ """)
+ await cur.execute("""
+ CREATE TABLE IF NOT EXISTS wiki_admin_plans (
+ id TEXT PRIMARY KEY,
+ chat_id TEXT NOT NULL DEFAULT '',
+ status TEXT NOT NULL DEFAULT 'pending',
+ intent TEXT NOT NULL DEFAULT '',
+ scope TEXT NOT NULL DEFAULT '',
+ kb_id TEXT NOT NULL DEFAULT '',
+ kb_name TEXT NOT NULL DEFAULT '',
+ summary JSONB NOT NULL DEFAULT '{}',
+ actions JSONB NOT NULL DEFAULT '[]',
+ executed JSONB NOT NULL DEFAULT '[]',
+ metadata JSONB NOT NULL DEFAULT '{}',
+ created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ expires_at TIMESTAMP NOT NULL DEFAULT NOW()
+ )
+ """)
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_file_state_kb_idx ON wiki_file_state(kb_id)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_slice_refs_kb_file_idx ON wiki_slice_refs(kb_id, file_id)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_slice_refs_file_idx ON wiki_slice_refs(file_id)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_pages_kb_idx ON wiki_pages(kb_id)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_pages_title_idx ON wiki_pages(title)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_pages_slug_idx ON wiki_pages(slug)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_page_slices_slice_idx ON wiki_page_slices(slice_id)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_pending_ops_status_idx ON wiki_pending_ops(status, claimed_at, id)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_pending_ops_kb_idx ON wiki_pending_ops(kb_id, status, id)")
+ await cur.execute("CREATE UNIQUE INDEX IF NOT EXISTS wiki_pending_ops_job_file_op_idx ON wiki_pending_ops(job_id, file_id, op)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_dead_letters_kb_idx ON wiki_dead_letters(kb_id, failed_at DESC)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_llm_cache_updated_idx ON wiki_llm_cache(updated_at)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_page_locks_expire_idx ON wiki_page_locks(expires_at)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_finalize_ops_status_idx ON wiki_finalize_ops(status, run_after, id)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_finalize_ops_kb_idx ON wiki_finalize_ops(kb_id, status)")
+ await cur.execute("CREATE INDEX IF NOT EXISTS wiki_admin_plans_chat_status_idx ON wiki_admin_plans(chat_id, status, created_at DESC)")
+ print("[wiki_engine] wiki tables initialized")
+
+
+async def create_job(job_id: str, target_type: str, target_id: str = "", kb_id: str = "") -> None:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO wiki_sync_jobs (id, target_type, target_id, kb_id, status, created_at, updated_at)
+ VALUES (%s, %s, %s, %s, 'running', %s, %s)
+ ON CONFLICT (id) DO UPDATE SET
+ status = EXCLUDED.status,
+ updated_at = EXCLUDED.updated_at
+ """,
+ (job_id, target_type, target_id or "", kb_id or "", datetime.now(), datetime.now()),
+ )
+
+
+async def update_job(job_id: str, **fields: Any) -> None:
+ if not fields:
+ return
+ allowed = {
+ "status", "total_files", "processed_files", "total_slices", "error",
+ "skipped_files", "page_count", "stage", "current_file_id", "current_filename",
+ }
+ parts = []
+ params: List[Any] = []
+ for key, value in fields.items():
+ if key in allowed:
+ parts.append(f"{key} = %s")
+ params.append(value)
+ if not parts:
+ return
+ parts.append("updated_at = %s")
+ params.append(datetime.now())
+ params.append(job_id)
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(f"UPDATE wiki_sync_jobs SET {', '.join(parts)} WHERE id = %s", params)
+
+
+async def get_job(job_id: str) -> Optional[Dict[str, Any]]:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT id, target_type, target_id, kb_id, status, total_files,
+ processed_files, total_slices, skipped_files, page_count,
+ stage, current_file_id, current_filename, error,
+ created_at, updated_at
+ FROM wiki_sync_jobs WHERE id = %s
+ """,
+ (job_id,),
+ )
+ row = await cur.fetchone()
+ if not row:
+ return None
+ keys = [
+ "id", "target_type", "target_id", "kb_id", "status", "total_files",
+ "processed_files", "total_slices", "skipped_files", "page_count",
+ "stage", "current_file_id", "current_filename", "error",
+ "created_at", "updated_at",
+ ]
+ return _row_to_dict(keys, row)
+
+
+async def set_job_totals(job_id: str, total_files: int) -> None:
+ await update_job(
+ job_id,
+ total_files=max(int(total_files or 0), 0),
+ processed_files=0,
+ total_slices=0,
+ skipped_files=0,
+ page_count=0,
+ stage="queued",
+ current_file_id="",
+ current_filename="",
+ )
+
+
+async def increment_job_progress(
+ job_id: str,
+ processed_delta: int = 1,
+ slice_delta: int = 0,
+ skipped_delta: int = 0,
+ page_delta: int = 0,
+ error: str = "",
+ stage: str = "",
+ current_file_id: str = "",
+ current_filename: str = "",
+) -> None:
+ processed_delta = max(int(processed_delta or 0), 0)
+ slice_delta = max(int(slice_delta or 0), 0)
+ skipped_delta = max(int(skipped_delta or 0), 0)
+ page_delta = max(int(page_delta or 0), 0)
+ error = str(error or "").strip()
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE wiki_sync_jobs
+ SET processed_files = processed_files + %s,
+ total_slices = total_slices + %s,
+ skipped_files = skipped_files + %s,
+ page_count = page_count + %s,
+ stage = CASE WHEN %s <> '' THEN %s ELSE stage END,
+ current_file_id = CASE WHEN %s <> '' THEN %s ELSE current_file_id END,
+ current_filename = CASE WHEN %s <> '' THEN %s ELSE current_filename END,
+ error = CASE
+ WHEN %s <> '' THEN LEFT(
+ CASE WHEN error <> '' THEN error || E'\n' || %s ELSE %s END,
+ 8000
+ )
+ ELSE error
+ END,
+ status = CASE
+ WHEN processed_files + %s >= total_files THEN
+ CASE WHEN error <> '' OR %s <> '' THEN 'failed' ELSE 'success' END
+ ELSE status
+ END,
+ updated_at = %s
+ WHERE id = %s
+ """,
+ (
+ processed_delta,
+ slice_delta,
+ skipped_delta,
+ page_delta,
+ str(stage or ""),
+ str(stage or ""),
+ str(current_file_id or ""),
+ str(current_file_id or ""),
+ str(current_filename or ""),
+ str(current_filename or ""),
+ error,
+ error,
+ error,
+ processed_delta,
+ error,
+ datetime.now(),
+ job_id,
+ ),
+ )
+
+
+async def enqueue_wiki_file_op(
+ job_id: str,
+ kb_id: str,
+ file_id: str,
+ file_info: Optional[Dict[str, Any]] = None,
+ use_llm: bool = False,
+ force: bool = False,
+) -> None:
+ payload = {
+ "job_id": str(job_id or ""),
+ "kb_id": str(kb_id or ""),
+ "file_id": str(file_id or ""),
+ "file_info": file_info or {},
+ "use_llm": bool(use_llm),
+ "force": bool(force),
+ }
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO wiki_pending_ops (
+ job_id, kb_id, file_id, op, dedup_key, payload,
+ status, fail_count, error, claimed_at, enqueued_at, updated_at
+ )
+ VALUES (%s, %s, %s, 'sync_file', %s, %s::jsonb,
+ 'pending', 0, '', NULL, %s, %s)
+ ON CONFLICT (job_id, file_id, op) DO UPDATE SET
+ kb_id = EXCLUDED.kb_id,
+ dedup_key = EXCLUDED.dedup_key,
+ payload = EXCLUDED.payload,
+ status = 'pending',
+ error = '',
+ claimed_at = NULL,
+ updated_at = EXCLUDED.updated_at
+ """,
+ (
+ str(job_id or ""),
+ str(kb_id or ""),
+ str(file_id or ""),
+ str(file_id or ""),
+ json.dumps(payload, ensure_ascii=False),
+ datetime.now(),
+ datetime.now(),
+ ),
+ )
+
+
+async def claim_wiki_file_ops(
+ limit: int = 1,
+ stale_seconds: int = 1800,
+ per_kb_concurrency: int = 1,
+) -> List[Dict[str, Any]]:
+ limit = max(1, min(int(limit or 1), 50))
+ stale_seconds = max(int(stale_seconds or 1800), 60)
+ per_kb_concurrency = max(int(per_kb_concurrency or 1), 1)
+ stale_before = datetime.now() - timedelta(seconds=stale_seconds)
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.transaction():
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT id
+ FROM wiki_pending_ops q
+ WHERE q.op = 'sync_file'
+ AND (
+ q.status = 'pending'
+ OR (q.status = 'running' AND (q.claimed_at IS NULL OR q.claimed_at < %s))
+ )
+ AND (
+ SELECT COUNT(*) FROM wiki_pending_ops active
+ WHERE active.kb_id = q.kb_id
+ AND active.status = 'running'
+ AND active.claimed_at IS NOT NULL
+ AND active.claimed_at >= %s
+ AND active.id <> q.id
+ ) < %s
+ AND NOT EXISTS (
+ SELECT 1 FROM wiki_pending_ops active_file
+ WHERE active_file.file_id = q.file_id
+ AND active_file.status = 'running'
+ AND active_file.claimed_at IS NOT NULL
+ AND active_file.claimed_at >= %s
+ AND active_file.id <> q.id
+ )
+ ORDER BY q.id ASC
+ LIMIT %s
+ FOR UPDATE SKIP LOCKED
+ """,
+ (stale_before, stale_before, per_kb_concurrency, stale_before, limit),
+ )
+ ids = [int(row[0]) for row in await cur.fetchall()]
+ if not ids:
+ return []
+ await cur.execute(
+ """
+ UPDATE wiki_pending_ops
+ SET status = 'running', claimed_at = %s, updated_at = %s
+ WHERE id = ANY(%s)
+ RETURNING id, job_id, kb_id, file_id, op, payload, fail_count
+ """,
+ (datetime.now(), datetime.now(), ids),
+ )
+ rows = await cur.fetchall()
+ keys = ["id", "job_id", "kb_id", "file_id", "op", "payload", "fail_count"]
+ return [_row_to_dict(keys, row) for row in rows]
+
+
+async def complete_wiki_op(op_id: int) -> None:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("DELETE FROM wiki_pending_ops WHERE id = %s", (int(op_id),))
+
+
+async def fail_wiki_op(op: Dict[str, Any], error: str, max_retries: int) -> bool:
+ op_id = int(op.get("id") or 0)
+ max_retries = max(int(max_retries or 1), 1)
+ error = str(error or "")[:4000]
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE wiki_pending_ops
+ SET fail_count = fail_count + 1,
+ error = %s,
+ status = CASE WHEN fail_count + 1 >= %s THEN 'dead' ELSE 'pending' END,
+ claimed_at = NULL,
+ updated_at = %s
+ WHERE id = %s
+ RETURNING fail_count, status, payload
+ """,
+ (error, max_retries, datetime.now(), op_id),
+ )
+ row = await cur.fetchone()
+ if not row:
+ return True
+ fail_count, status, payload = row
+ if status == "dead":
+ await cur.execute(
+ """
+ INSERT INTO wiki_dead_letters (
+ job_id, kb_id, file_id, op, payload, last_error, fail_count, failed_at
+ )
+ VALUES (%s, %s, %s, %s, %s::jsonb, %s, %s, %s)
+ """,
+ (
+ str(op.get("job_id") or ""),
+ str(op.get("kb_id") or ""),
+ str(op.get("file_id") or ""),
+ str(op.get("op") or ""),
+ json.dumps(payload or {}, ensure_ascii=False),
+ error,
+ int(fail_count or 0),
+ datetime.now(),
+ ),
+ )
+ await cur.execute("DELETE FROM wiki_pending_ops WHERE id = %s", (op_id,))
+ return True
+ return False
+
+
+async def pending_wiki_op_count(job_id: Optional[str] = None, kb_id: Optional[str] = None) -> int:
+ where = ["status IN ('pending', 'running')"]
+ params: List[Any] = []
+ if job_id:
+ where.append("job_id = %s")
+ params.append(str(job_id))
+ if kb_id:
+ where.append("kb_id = %s")
+ params.append(str(kb_id))
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(f"SELECT COUNT(*) FROM wiki_pending_ops WHERE {' AND '.join(where)}", params)
+ row = await cur.fetchone()
+ return int(row[0] if row else 0)
+
+
+async def get_wiki_queue_stats() -> Dict[str, Any]:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT status, COUNT(*)
+ FROM wiki_pending_ops
+ GROUP BY status
+ """
+ )
+ rows = await cur.fetchall()
+ await cur.execute("SELECT COUNT(*) FROM wiki_dead_letters")
+ dead = (await cur.fetchone())[0]
+ stats = {str(status): int(count) for status, count in rows}
+ stats["dead_letters"] = int(dead or 0)
+ return stats
+
+
+async def get_llm_cache(cache_key: str, ttl_seconds: int) -> Optional[str]:
+ if not cache_key:
+ return None
+ ttl_seconds = int(ttl_seconds or 0)
+ where = "cache_key = %s"
+ params: List[Any] = [cache_key]
+ if ttl_seconds > 0:
+ where += " AND updated_at >= NOW() - (%s * INTERVAL '1 second')"
+ params.append(ttl_seconds)
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(f"SELECT response FROM wiki_llm_cache WHERE {where}", params)
+ row = await cur.fetchone()
+ if not row:
+ return None
+ await cur.execute(
+ "UPDATE wiki_llm_cache SET hit_count = hit_count + 1, updated_at = %s WHERE cache_key = %s",
+ (datetime.now(), cache_key),
+ )
+ return str(row[0] or "")
+
+
+async def set_llm_cache(cache_key: str, response: str) -> None:
+ if not cache_key:
+ return
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO wiki_llm_cache (cache_key, response, hit_count, created_at, updated_at)
+ VALUES (%s, %s, 0, %s, %s)
+ ON CONFLICT (cache_key) DO UPDATE SET
+ response = EXCLUDED.response,
+ updated_at = EXCLUDED.updated_at
+ """,
+ (cache_key, response or "", datetime.now(), datetime.now()),
+ )
+
+
+async def create_admin_plan(
+ plan_id: str,
+ chat_id: str,
+ intent: str,
+ scope: str,
+ kb_id: str = "",
+ kb_name: str = "",
+ summary: Optional[Dict[str, Any]] = None,
+ actions: Optional[List[Dict[str, Any]]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ ttl_hours: int = 24,
+) -> Dict[str, Any]:
+ now = datetime.now()
+ expires_at = now + timedelta(hours=max(int(ttl_hours or 24), 1))
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO wiki_admin_plans (
+ id, chat_id, status, intent, scope, kb_id, kb_name,
+ summary, actions, executed, metadata, created_at, updated_at, expires_at
+ )
+ VALUES (%s, %s, 'pending', %s, %s, %s, %s, %s::jsonb, %s::jsonb, '[]'::jsonb, %s::jsonb, %s, %s, %s)
+ ON CONFLICT (id) DO UPDATE SET
+ chat_id = EXCLUDED.chat_id,
+ status = EXCLUDED.status,
+ intent = EXCLUDED.intent,
+ scope = EXCLUDED.scope,
+ kb_id = EXCLUDED.kb_id,
+ kb_name = EXCLUDED.kb_name,
+ summary = EXCLUDED.summary,
+ actions = EXCLUDED.actions,
+ executed = EXCLUDED.executed,
+ metadata = EXCLUDED.metadata,
+ updated_at = EXCLUDED.updated_at,
+ expires_at = EXCLUDED.expires_at
+ RETURNING id, chat_id, status, intent, scope, kb_id, kb_name,
+ summary, actions, executed, metadata, created_at, updated_at, expires_at
+ """,
+ (
+ plan_id,
+ chat_id or "",
+ intent or "",
+ scope or "",
+ kb_id or "",
+ kb_name or "",
+ json.dumps(summary or {}, ensure_ascii=False),
+ json.dumps(actions or [], ensure_ascii=False),
+ json.dumps(metadata or {}, ensure_ascii=False),
+ now,
+ now,
+ expires_at,
+ ),
+ )
+ row = await cur.fetchone()
+ return _admin_plan_from_row(row)
+
+
+async def get_latest_pending_admin_plan(chat_id: str) -> Optional[Dict[str, Any]]:
+ if not chat_id:
+ return None
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT id, chat_id, status, intent, scope, kb_id, kb_name,
+ summary, actions, executed, metadata, created_at, updated_at, expires_at
+ FROM wiki_admin_plans
+ WHERE chat_id = %s AND status = 'pending' AND expires_at >= NOW()
+ ORDER BY created_at DESC
+ LIMIT 1
+ """,
+ (chat_id,),
+ )
+ row = await cur.fetchone()
+ return _admin_plan_from_row(row) if row else None
+
+
+async def update_admin_plan_status(
+ plan_id: str,
+ status: str,
+ executed: Optional[List[Dict[str, Any]]] = None,
+) -> Optional[Dict[str, Any]]:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ if executed is None:
+ await cur.execute(
+ """
+ UPDATE wiki_admin_plans
+ SET status = %s, updated_at = %s
+ WHERE id = %s
+ RETURNING id, chat_id, status, intent, scope, kb_id, kb_name,
+ summary, actions, executed, metadata, created_at, updated_at, expires_at
+ """,
+ (status, datetime.now(), plan_id),
+ )
+ else:
+ await cur.execute(
+ """
+ UPDATE wiki_admin_plans
+ SET status = %s, executed = %s::jsonb, updated_at = %s
+ WHERE id = %s
+ RETURNING id, chat_id, status, intent, scope, kb_id, kb_name,
+ summary, actions, executed, metadata, created_at, updated_at, expires_at
+ """,
+ (status, json.dumps(executed, ensure_ascii=False), datetime.now(), plan_id),
+ )
+ row = await cur.fetchone()
+ return _admin_plan_from_row(row) if row else None
+
+
+async def upsert_file_state(file_info: Dict[str, Any], slice_count: int) -> None:
+ metadata = dict(file_info or {})
+ nested_metadata = file_info.get("metadata") if isinstance(file_info.get("metadata"), dict) else {}
+ metadata.update(nested_metadata)
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO wiki_file_state (
+ kb_id, file_id, filename, file_hash, status, slice_count,
+ source_updated_at, metadata, last_synced_at
+ )
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s)
+ ON CONFLICT (file_id) DO UPDATE SET
+ kb_id = EXCLUDED.kb_id,
+ filename = EXCLUDED.filename,
+ file_hash = EXCLUDED.file_hash,
+ status = EXCLUDED.status,
+ slice_count = EXCLUDED.slice_count,
+ source_updated_at = EXCLUDED.source_updated_at,
+ metadata = EXCLUDED.metadata,
+ last_synced_at = EXCLUDED.last_synced_at
+ """,
+ (
+ str(file_info.get("kb_id") or ""),
+ str(file_info.get("file_id") or file_info.get("id") or ""),
+ str(file_info.get("filename") or file_info.get("name") or ""),
+ str(file_info.get("hash") or ""),
+ str(file_info.get("status") or ""),
+ slice_count,
+ str(file_info.get("updated_at") or ""),
+ json.dumps(metadata, ensure_ascii=False),
+ datetime.now(),
+ ),
+ )
+
+
+async def get_file_state(file_id: str) -> Optional[Dict[str, Any]]:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT kb_id, file_id, filename, file_hash, status, slice_count,
+ source_updated_at, metadata, last_synced_at
+ FROM wiki_file_state
+ WHERE file_id = %s
+ """,
+ (str(file_id or ""),),
+ )
+ row = await cur.fetchone()
+ if not row:
+ return None
+ keys = [
+ "kb_id", "file_id", "filename", "file_hash", "status", "slice_count",
+ "source_updated_at", "metadata", "last_synced_at",
+ ]
+ return _row_to_dict(keys, row)
+
+
+async def count_file_pages(file_id: str) -> int:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "SELECT COUNT(DISTINCT wiki_page_id) FROM wiki_page_slices WHERE file_id = %s",
+ (str(file_id or ""),),
+ )
+ row = await cur.fetchone()
+ return int(row[0] or 0) if row else 0
+
+
+async def replace_file_slice_refs(file_id: str, refs: Sequence[Dict[str, Any]]) -> None:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("DELETE FROM wiki_page_slices WHERE file_id = %s", (str(file_id),))
+ await cur.execute("DELETE FROM wiki_slice_refs WHERE file_id = %s", (str(file_id),))
+ await _delete_optional_table_rows(cur, "wiki_slice_embeddings", "file_id = %s", [str(file_id)])
+ for ref in refs:
+ await cur.execute(
+ """
+ INSERT INTO wiki_slice_refs (
+ kb_id, file_id, slice_id, ordinal, filename, title_path,
+ keywords, brief, content, chunk_type, content_hash, updated_at
+ )
+ VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s::jsonb, %s, %s, %s, %s, %s)
+ ON CONFLICT (slice_id) DO UPDATE SET
+ kb_id = EXCLUDED.kb_id,
+ file_id = EXCLUDED.file_id,
+ ordinal = EXCLUDED.ordinal,
+ filename = EXCLUDED.filename,
+ title_path = EXCLUDED.title_path,
+ keywords = EXCLUDED.keywords,
+ brief = EXCLUDED.brief,
+ content = EXCLUDED.content,
+ chunk_type = EXCLUDED.chunk_type,
+ content_hash = EXCLUDED.content_hash,
+ updated_at = EXCLUDED.updated_at
+ """,
+ (
+ str(ref.get("kb_id") or ""),
+ str(ref.get("file_id") or ""),
+ str(ref.get("slice_id") or ""),
+ int(ref.get("ordinal") or 0),
+ str(ref.get("filename") or ""),
+ json.dumps(ref.get("title_path") or [], ensure_ascii=False),
+ json.dumps(ref.get("keywords") or [], ensure_ascii=False),
+ str(ref.get("brief") or ""),
+ str(ref.get("content") or ref.get("_text") or ""),
+ str(ref.get("chunk_type") or "text"),
+ str(ref.get("content_hash") or ""),
+ datetime.now(),
+ ),
+ )
+
+
+async def upsert_wiki_page(page: Dict[str, Any]) -> int:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ kb_id = str(page.get("kb_id") or "")
+ slug = str(page.get("slug") or "")[:300]
+ title = str(page.get("title") or "")[:300]
+ page_type = str(page.get("page_type") or "topic")
+ params = (
+ kb_id,
+ slug,
+ title,
+ page_type,
+ str(page.get("summary") or ""),
+ str(page.get("content") or ""),
+ json.dumps(page.get("keywords") or [], ensure_ascii=False),
+ json.dumps(page.get("aliases") or [], ensure_ascii=False),
+ json.dumps(page.get("related_file_ids") or [], ensure_ascii=False),
+ json.dumps(page.get("source_refs") or [], ensure_ascii=False),
+ json.dumps(page.get("chunk_refs") or [], ensure_ascii=False),
+ str(page.get("source_kind") or "rule"),
+ float(page.get("score_hint") or 1.0),
+ datetime.now(),
+ )
+ if slug:
+ await cur.execute(
+ """
+ UPDATE wiki_pages
+ SET title = %s,
+ page_type = %s,
+ summary = %s,
+ content = %s,
+ keywords = %s::jsonb,
+ aliases = %s::jsonb,
+ related_file_ids = %s::jsonb,
+ source_refs = %s::jsonb,
+ chunk_refs = %s::jsonb,
+ source_kind = %s,
+ score_hint = %s,
+ updated_at = %s
+ WHERE kb_id = %s AND slug = %s
+ RETURNING id
+ """,
+ (
+ title,
+ page_type,
+ str(page.get("summary") or ""),
+ str(page.get("content") or ""),
+ json.dumps(page.get("keywords") or [], ensure_ascii=False),
+ json.dumps(page.get("aliases") or [], ensure_ascii=False),
+ json.dumps(page.get("related_file_ids") or [], ensure_ascii=False),
+ json.dumps(page.get("source_refs") or [], ensure_ascii=False),
+ json.dumps(page.get("chunk_refs") or [], ensure_ascii=False),
+ str(page.get("source_kind") or "rule"),
+ float(page.get("score_hint") or 1.0),
+ datetime.now(),
+ kb_id,
+ slug,
+ ),
+ )
+ row = await cur.fetchone()
+ if row:
+ return int(row[0])
+ await cur.execute(
+ """
+ INSERT INTO wiki_pages (
+ kb_id, slug, title, page_type, summary, content, keywords,
+ aliases, related_file_ids, source_refs, chunk_refs,
+ source_kind, score_hint, created_at, updated_at
+ )
+ VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb, %s::jsonb, %s::jsonb, %s::jsonb, %s::jsonb, %s, %s, %s, %s)
+ ON CONFLICT (kb_id, title, page_type) DO UPDATE SET
+ slug = EXCLUDED.slug,
+ summary = EXCLUDED.summary,
+ content = EXCLUDED.content,
+ keywords = EXCLUDED.keywords,
+ aliases = EXCLUDED.aliases,
+ related_file_ids = EXCLUDED.related_file_ids,
+ source_refs = EXCLUDED.source_refs,
+ chunk_refs = EXCLUDED.chunk_refs,
+ source_kind = EXCLUDED.source_kind,
+ score_hint = EXCLUDED.score_hint,
+ updated_at = EXCLUDED.updated_at
+ RETURNING id
+ """,
+ params + (datetime.now(),),
+ )
+ row = await cur.fetchone()
+ return int(row[0])
+
+
+async def replace_page_slices(page_id: int, links: Iterable[Dict[str, Any]]) -> None:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("DELETE FROM wiki_page_slices WHERE wiki_page_id = %s", (page_id,))
+ for link in links:
+ await cur.execute(
+ """
+ INSERT INTO wiki_page_slices (wiki_page_id, file_id, slice_id, weight, reason)
+ VALUES (%s, %s, %s, %s, %s)
+ ON CONFLICT (wiki_page_id, slice_id) DO UPDATE SET
+ weight = EXCLUDED.weight,
+ reason = EXCLUDED.reason
+ """,
+ (
+ page_id,
+ str(link.get("file_id") or ""),
+ str(link.get("slice_id") or ""),
+ float(link.get("weight") or 1.0),
+ str(link.get("reason") or ""),
+ ),
+ )
+
+
+async def acquire_wiki_page_lock(kb_id: str, slug: str, owner: str, ttl_seconds: int = 600) -> bool:
+ kb_id = str(kb_id or "")
+ slug = str(slug or "")[:300]
+ owner = str(owner or "")
+ ttl_seconds = max(int(ttl_seconds or 600), 30)
+ now = datetime.now()
+ expires_at = now + timedelta(seconds=ttl_seconds)
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "DELETE FROM wiki_page_locks WHERE expires_at < %s",
+ (now,),
+ )
+ await cur.execute(
+ """
+ INSERT INTO wiki_page_locks (kb_id, slug, owner, locked_at, expires_at)
+ VALUES (%s, %s, %s, %s, %s)
+ ON CONFLICT (kb_id, slug) DO NOTHING
+ """,
+ (kb_id, slug, owner, now, expires_at),
+ )
+ return bool(cur.rowcount)
+
+
+async def release_wiki_page_lock(kb_id: str, slug: str, owner: str) -> None:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "DELETE FROM wiki_page_locks WHERE kb_id = %s AND slug = %s AND owner = %s",
+ (str(kb_id or ""), str(slug or "")[:300], str(owner or "")),
+ )
+
+
+async def enqueue_wiki_finalize_op(job_id: str, kb_id: str, delay_seconds: int = 20) -> Optional[int]:
+ kb_id = str(kb_id or "")
+ if not kb_id:
+ return None
+ now = datetime.now()
+ run_after = now + timedelta(seconds=max(int(delay_seconds or 0), 0))
+ payload = {"job_id": str(job_id or ""), "kb_id": kb_id}
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE wiki_finalize_ops
+ SET job_id = %s,
+ payload = %s::jsonb,
+ run_after = GREATEST(run_after, %s),
+ updated_at = %s
+ WHERE kb_id = %s AND status IN ('pending', 'running')
+ RETURNING id
+ """,
+ (str(job_id or ""), json.dumps(payload, ensure_ascii=False), run_after, now, kb_id),
+ )
+ row = await cur.fetchone()
+ if row:
+ return int(row[0])
+ await cur.execute(
+ """
+ INSERT INTO wiki_finalize_ops (
+ job_id, kb_id, status, payload, claimed_at, run_after, enqueued_at, updated_at
+ )
+ VALUES (%s, %s, 'pending', %s::jsonb, NULL, %s, %s, %s)
+ RETURNING id
+ """,
+ (str(job_id or ""), kb_id, json.dumps(payload, ensure_ascii=False), run_after, now, now),
+ )
+ row = await cur.fetchone()
+ return int(row[0]) if row else None
+
+
+async def claim_wiki_finalize_ops(limit: int = 1) -> List[Dict[str, Any]]:
+ limit = max(1, min(int(limit or 1), 10))
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.transaction():
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT id
+ FROM wiki_finalize_ops
+ WHERE status = 'pending' AND run_after <= %s
+ ORDER BY run_after ASC, id ASC
+ LIMIT %s
+ FOR UPDATE SKIP LOCKED
+ """,
+ (datetime.now(), limit),
+ )
+ ids = [int(row[0]) for row in await cur.fetchall()]
+ if not ids:
+ return []
+ await cur.execute(
+ """
+ UPDATE wiki_finalize_ops
+ SET status = 'running', claimed_at = %s, updated_at = %s
+ WHERE id = ANY(%s)
+ RETURNING id, job_id, kb_id, payload
+ """,
+ (datetime.now(), datetime.now(), ids),
+ )
+ rows = await cur.fetchall()
+ keys = ["id", "job_id", "kb_id", "payload"]
+ return [_row_to_dict(keys, row) for row in rows]
+
+
+async def complete_wiki_finalize_op(op_id: int) -> None:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "UPDATE wiki_finalize_ops SET status = 'done', updated_at = %s WHERE id = %s",
+ (datetime.now(), int(op_id)),
+ )
+
+
+async def fail_wiki_finalize_op(op_id: int, error: str) -> None:
+ payload = {"error": str(error or "")[:4000]}
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ UPDATE wiki_finalize_ops
+ SET status = 'failed', payload = payload || %s::jsonb, updated_at = %s
+ WHERE id = %s
+ """,
+ (json.dumps(payload, ensure_ascii=False), datetime.now(), int(op_id)),
+ )
+
+
+async def count_pages_for_kb(kb_id: str) -> int:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("SELECT COUNT(*) FROM wiki_pages WHERE kb_id = %s", (str(kb_id or ""),))
+ row = await cur.fetchone()
+ return int(row[0] or 0) if row else 0
+
+
+async def get_slice_refs_by_ids(slice_ids: Sequence[str]) -> List[Dict[str, Any]]:
+ ids = [str(x) for x in slice_ids if str(x)]
+ if not ids:
+ return []
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT kb_id, file_id, slice_id, ordinal, filename, title_path,
+ keywords, brief, content, chunk_type, content_hash, updated_at
+ FROM wiki_slice_refs
+ WHERE slice_id = ANY(%s)
+ """,
+ (ids,),
+ )
+ rows = await cur.fetchall()
+ keys = [
+ "kb_id", "file_id", "slice_id", "ordinal", "filename", "title_path",
+ "keywords", "brief", "content", "chunk_type", "content_hash", "updated_at",
+ ]
+ by_id = {str(item.get("slice_id")): item for item in (_row_to_dict(keys, row) for row in rows)}
+ return [by_id[sid] for sid in ids if sid in by_id]
+
+
+async def search_slices_text(query: str, kb_id: Optional[str] = None, limit: int = 32) -> List[Dict[str, Any]]:
+ terms = [t for t in _extract_search_terms(query) if t]
+ if not terms:
+ return []
+ where = []
+ params: List[Any] = []
+ if kb_id:
+ where.append("sr.kb_id = %s")
+ params.append(str(kb_id))
+
+ term_clauses = []
+ for term in terms[:8]:
+ like = f"%{term}%"
+ term_clauses.append(
+ "(sr.filename ILIKE %s OR sr.title_path::text ILIKE %s OR sr.keywords::text ILIKE %s "
+ "OR sr.brief ILIKE %s OR sr.content ILIKE %s)"
+ )
+ params.extend([like, like, like, like, like])
+ where.append("(" + " OR ".join(term_clauses) + ")")
+ params.append(max(int(limit or 32) * 4, int(limit or 32)))
+
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"""
+ SELECT sr.kb_id, sr.file_id, sr.slice_id, sr.ordinal, sr.filename,
+ sr.title_path, sr.keywords, sr.brief, sr.content, sr.chunk_type,
+ sr.content_hash, sr.updated_at
+ FROM wiki_slice_refs sr
+ WHERE {" AND ".join(where)}
+ ORDER BY sr.updated_at DESC
+ LIMIT %s
+ """,
+ params,
+ )
+ rows = await cur.fetchall()
+ keys = [
+ "kb_id", "file_id", "slice_id", "ordinal", "filename",
+ "title_path", "keywords", "brief", "content", "chunk_type",
+ "content_hash", "updated_at",
+ ]
+ results = []
+ for row in rows:
+ item = _row_to_dict(keys, row)
+ item["score"] = _score_slice(query, item)
+ results.append(item)
+ results.sort(key=lambda x: float(x.get("score") or 0.0), reverse=True)
+ return results[: max(int(limit or 32), 1)]
+
+
+async def search_pages(query: str, kb_id: Optional[str] = None, limit: int = 8) -> List[Dict[str, Any]]:
+ terms = [t for t in _extract_search_terms(query) if t]
+ where = []
+ params: List[Any] = []
+ rank_parts = []
+ rank_params: List[Any] = []
+ if kb_id:
+ where.append("p.kb_id = %s")
+ params.append(str(kb_id))
+ if terms:
+ term_clauses = []
+ for term in terms[:6]:
+ like = f"%{term}%"
+ term_clauses.append(
+ "(p.title ILIKE %s OR p.slug ILIKE %s OR p.summary ILIKE %s "
+ "OR p.content ILIKE %s OR p.keywords::text ILIKE %s OR p.aliases::text ILIKE %s)"
+ )
+ params.extend([like, like, like, like, like, like])
+ weight = _term_weight(str(term).lower())
+ rank_parts.append(
+ "(CASE "
+ "WHEN p.title ILIKE %s THEN %s "
+ "WHEN p.slug ILIKE %s THEN %s "
+ "WHEN p.aliases::text ILIKE %s THEN %s "
+ "WHEN p.keywords::text ILIKE %s THEN %s "
+ "WHEN p.summary ILIKE %s THEN %s "
+ "WHEN p.content ILIKE %s THEN %s "
+ "ELSE 0 END)"
+ )
+ rank_params.extend([
+ like, 40.0 * weight,
+ like, 25.0 * weight,
+ like, 18.0 * weight,
+ like, 12.0 * weight,
+ like, 6.0 * weight,
+ like, 1.0 * weight,
+ ])
+ where.append("(" + " OR ".join(term_clauses) + ")")
+ where_sql = " AND ".join(where) if where else "1=1"
+ rank_sql = " + ".join(rank_parts) if rank_parts else "0"
+ execute_params = rank_params + params + [max(limit * 10, limit, 80)]
+
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"""
+ SELECT p.id, p.kb_id, p.slug, p.title, p.page_type, p.summary, p.content,
+ p.keywords, p.aliases, p.related_file_ids, p.source_refs, p.chunk_refs,
+ p.source_kind, p.score_hint, p.updated_at,
+ ({rank_sql}) AS db_match_rank
+ FROM wiki_pages p
+ WHERE {where_sql}
+ ORDER BY db_match_rank DESC, p.score_hint DESC, p.updated_at DESC
+ LIMIT %s
+ """,
+ execute_params,
+ )
+ rows = await cur.fetchall()
+
+ keys = [
+ "id", "kb_id", "slug", "title", "page_type", "summary", "content",
+ "keywords", "aliases", "related_file_ids", "source_refs", "chunk_refs",
+ "source_kind", "score_hint", "updated_at", "db_match_rank",
+ ]
+ results = []
+ for row in rows:
+ item = _row_to_dict(keys, row)
+ item["score"] = _score_page(query, item)
+ results.append(item)
+ results.sort(key=lambda x: x.get("score", 0), reverse=True)
+ return results[:limit]
+
+
+async def get_page(page_id: int) -> Optional[Dict[str, Any]]:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT id, kb_id, slug, title, page_type, summary, content,
+ keywords, aliases, related_file_ids, source_refs, chunk_refs,
+ source_kind, score_hint, updated_at
+ FROM wiki_pages WHERE id = %s
+ """,
+ (page_id,),
+ )
+ row = await cur.fetchone()
+ if not row:
+ return None
+ keys = [
+ "id", "kb_id", "slug", "title", "page_type", "summary", "content",
+ "keywords", "aliases", "related_file_ids", "source_refs", "chunk_refs",
+ "source_kind", "score_hint", "updated_at",
+ ]
+ page = _row_to_dict(keys, row)
+ await cur.execute(
+ """
+ SELECT ps.file_id, ps.slice_id, ps.weight, ps.reason,
+ sr.filename, sr.title_path, sr.keywords, sr.brief, sr.content, sr.chunk_type
+ FROM wiki_page_slices ps
+ LEFT JOIN wiki_slice_refs sr ON sr.slice_id = ps.slice_id
+ WHERE ps.wiki_page_id = %s
+ ORDER BY ps.weight DESC
+ LIMIT 50
+ """,
+ (page_id,),
+ )
+ links = await cur.fetchall()
+ link_keys = ["file_id", "slice_id", "weight", "reason", "filename", "title_path", "keywords", "brief", "content", "chunk_type"]
+ page["slices"] = [_row_to_dict(link_keys, row) for row in links]
+ return page
+
+
+async def list_wiki_pages_for_kb(kb_id: str) -> List[Dict[str, Any]]:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT id, kb_id, slug, title, page_type, summary, content,
+ keywords, aliases, related_file_ids, source_refs, chunk_refs,
+ source_kind, score_hint, updated_at
+ FROM wiki_pages
+ WHERE kb_id = %s
+ ORDER BY updated_at DESC
+ """,
+ (str(kb_id or ""),),
+ )
+ rows = await cur.fetchall()
+ keys = [
+ "id", "kb_id", "slug", "title", "page_type", "summary", "content",
+ "keywords", "aliases", "related_file_ids", "source_refs", "chunk_refs",
+ "source_kind", "score_hint", "updated_at",
+ ]
+ pages = [_row_to_dict(keys, row) for row in rows]
+ if not pages:
+ return []
+ page_ids = [int(page["id"]) for page in pages if page.get("id")]
+ placeholders = ",".join(["%s"] * len(page_ids))
+ await cur.execute(
+ f"""
+ SELECT ps.wiki_page_id, sr.kb_id, ps.file_id, ps.slice_id, ps.weight, ps.reason,
+ sr.filename, sr.title_path, sr.keywords, sr.brief, sr.content, sr.chunk_type
+ FROM wiki_page_slices ps
+ LEFT JOIN wiki_slice_refs sr ON sr.slice_id = ps.slice_id
+ WHERE ps.wiki_page_id IN ({placeholders})
+ ORDER BY ps.wiki_page_id, ps.weight DESC
+ """,
+ page_ids,
+ )
+ link_rows = await cur.fetchall()
+ link_keys = [
+ "wiki_page_id", "kb_id", "file_id", "slice_id", "weight", "reason",
+ "filename", "title_path", "keywords", "brief", "content", "chunk_type",
+ ]
+ links_by_page: Dict[int, List[Dict[str, Any]]] = {}
+ for row in link_rows:
+ item = _row_to_dict(link_keys, row)
+ page_id = int(item.pop("wiki_page_id"))
+ links_by_page.setdefault(page_id, []).append(item)
+ for page in pages:
+ page["slices"] = links_by_page.get(int(page["id"]), [])
+ return pages
+
+
+async def get_page_slices(page_ids: Sequence[int], max_slices_per_page: int = 8) -> Dict[int, List[Dict[str, Any]]]:
+ if not page_ids:
+ return {}
+ placeholders = ",".join(["%s"] * len(page_ids))
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"""
+ SELECT ps.wiki_page_id, sr.kb_id, ps.file_id, ps.slice_id, ps.weight, ps.reason,
+ sr.filename, sr.title_path, sr.keywords, sr.brief, sr.content, sr.chunk_type
+ FROM wiki_page_slices ps
+ LEFT JOIN wiki_slice_refs sr ON sr.slice_id = ps.slice_id
+ WHERE ps.wiki_page_id IN ({placeholders})
+ ORDER BY ps.wiki_page_id, ps.weight DESC
+ """,
+ list(page_ids),
+ )
+ rows = await cur.fetchall()
+ out: Dict[int, List[Dict[str, Any]]] = {}
+ keys = ["wiki_page_id", "kb_id", "file_id", "slice_id", "weight", "reason", "filename", "title_path", "keywords", "brief", "content", "chunk_type"]
+ for row in rows:
+ item = _row_to_dict(keys, row)
+ page_id = int(item.pop("wiki_page_id"))
+ bucket = out.setdefault(page_id, [])
+ if len(bucket) < max_slices_per_page:
+ bucket.append(item)
+ return out
+
+
+async def log_query(query: str, kb_id: Optional[str], result_count: int) -> None:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ INSERT INTO wiki_query_logs (query, kb_id, result_count, created_at)
+ VALUES (%s, %s, %s, %s)
+ """,
+ (query, kb_id or "", result_count, datetime.now()),
+ )
+
+
+async def get_stats() -> Dict[str, Any]:
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ tables = [
+ "wiki_file_state",
+ "wiki_slice_refs",
+ "wiki_pages",
+ "wiki_page_slices",
+ "wiki_sync_jobs",
+ "wiki_pending_ops",
+ "wiki_dead_letters",
+ "wiki_llm_cache",
+ "wiki_page_locks",
+ "wiki_finalize_ops",
+ ]
+ stats: Dict[str, Any] = {}
+ for table in tables:
+ await cur.execute(f"SELECT COUNT(*) FROM {table}")
+ stats[table] = (await cur.fetchone())[0]
+ if await _table_exists(cur, "wiki_slice_embeddings"):
+ await cur.execute("SELECT COUNT(*) FROM wiki_slice_embeddings")
+ stats["wiki_slice_embeddings"] = (await cur.fetchone())[0]
+ return stats
+
+
+async def list_file_states(kb_ids: Sequence[str]) -> List[Dict[str, Any]]:
+ ids = [str(x) for x in kb_ids if str(x)]
+ if not ids:
+ return []
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ """
+ SELECT kb_id, file_id, filename, file_hash, status, slice_count,
+ source_updated_at, metadata, last_synced_at
+ FROM wiki_file_state
+ WHERE kb_id = ANY(%s)
+ """,
+ (ids,),
+ )
+ rows = await cur.fetchall()
+ keys = [
+ "kb_id", "file_id", "filename", "file_hash", "status", "slice_count",
+ "source_updated_at", "metadata", "last_synced_at",
+ ]
+ return [_row_to_dict(keys, row) for row in rows]
+
+
+async def delete_wiki_by_file(file_id: str) -> Dict[str, int]:
+ file_id = str(file_id)
+ counts = {
+ "wiki_pages": 0,
+ "wiki_page_slices": 0,
+ "wiki_slice_embeddings": 0,
+ "wiki_slice_refs": 0,
+ "wiki_file_state": 0,
+ }
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "SELECT DISTINCT wiki_page_id FROM wiki_page_slices WHERE file_id = %s",
+ (file_id,),
+ )
+ page_ids = [row[0] for row in await cur.fetchall()]
+ if page_ids:
+ await cur.execute("DELETE FROM wiki_pages WHERE id = ANY(%s)", (page_ids,))
+ counts["wiki_pages"] = cur.rowcount or 0
+ counts["wiki_slice_embeddings"] = await _delete_optional_table_rows(
+ cur, "wiki_slice_embeddings", "file_id = %s", [file_id]
+ )
+ await cur.execute("DELETE FROM wiki_slice_refs WHERE file_id = %s", (file_id,))
+ counts["wiki_slice_refs"] = cur.rowcount or 0
+ await cur.execute("DELETE FROM wiki_file_state WHERE file_id = %s", (file_id,))
+ counts["wiki_file_state"] = cur.rowcount or 0
+ return counts
+
+
+async def delete_wiki_by_kb(kb_id: str) -> Dict[str, int]:
+ kb_id = str(kb_id)
+ counts = {
+ "wiki_pages": 0,
+ "wiki_slice_embeddings": 0,
+ "wiki_slice_refs": 0,
+ "wiki_file_state": 0,
+ "wiki_sync_jobs": 0,
+ "wiki_query_logs": 0,
+ }
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute("DELETE FROM wiki_pages WHERE kb_id = %s", (kb_id,))
+ counts["wiki_pages"] = cur.rowcount or 0
+ counts["wiki_slice_embeddings"] = await _delete_optional_table_rows(
+ cur, "wiki_slice_embeddings", "kb_id = %s", [kb_id]
+ )
+ await cur.execute("DELETE FROM wiki_slice_refs WHERE kb_id = %s", (kb_id,))
+ counts["wiki_slice_refs"] = cur.rowcount or 0
+ await cur.execute("DELETE FROM wiki_file_state WHERE kb_id = %s", (kb_id,))
+ counts["wiki_file_state"] = cur.rowcount or 0
+ await cur.execute("DELETE FROM wiki_sync_jobs WHERE kb_id = %s", (kb_id,))
+ counts["wiki_sync_jobs"] = cur.rowcount or 0
+ await cur.execute("DELETE FROM wiki_query_logs WHERE kb_id = %s", (kb_id,))
+ counts["wiki_query_logs"] = cur.rowcount or 0
+ return counts
+
+
+async def delete_wiki_by_time(
+ before: Optional[datetime] = None,
+ after: Optional[datetime] = None,
+ kb_id: Optional[str] = None,
+) -> Dict[str, int]:
+ if before is None and after is None:
+ raise ValueError("before or after is required")
+
+ counts = {
+ "wiki_pages": 0,
+ "wiki_slice_embeddings": 0,
+ "wiki_slice_refs": 0,
+ "wiki_file_state": 0,
+ "wiki_sync_jobs": 0,
+ "wiki_query_logs": 0,
+ }
+ where_pages, params_pages = _time_where("updated_at", before, after)
+ where_slices, params_slices = _time_where("updated_at", before, after)
+ where_files, params_files = _time_where("last_synced_at", before, after)
+ where_jobs, params_jobs = _time_where("created_at", before, after)
+ where_logs, params_logs = _time_where("created_at", before, after)
+
+ if kb_id:
+ where_pages += " AND kb_id = %s"
+ params_pages.append(str(kb_id))
+ where_slices += " AND kb_id = %s"
+ params_slices.append(str(kb_id))
+ where_files += " AND kb_id = %s"
+ params_files.append(str(kb_id))
+ where_jobs += " AND kb_id = %s"
+ params_jobs.append(str(kb_id))
+ where_logs += " AND kb_id = %s"
+ params_logs.append(str(kb_id))
+
+ async with await _connect_pool() as pool:
+ async with pool.connection() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(f"DELETE FROM wiki_pages WHERE {where_pages}", params_pages)
+ counts["wiki_pages"] = cur.rowcount or 0
+ counts["wiki_slice_embeddings"] = await _delete_optional_table_rows(
+ cur, "wiki_slice_embeddings", where_pages, params_pages
+ )
+ await cur.execute(f"DELETE FROM wiki_slice_refs WHERE {where_slices}", params_slices)
+ counts["wiki_slice_refs"] = cur.rowcount or 0
+ await cur.execute(f"DELETE FROM wiki_file_state WHERE {where_files}", params_files)
+ counts["wiki_file_state"] = cur.rowcount or 0
+ await cur.execute(f"DELETE FROM wiki_sync_jobs WHERE {where_jobs}", params_jobs)
+ counts["wiki_sync_jobs"] = cur.rowcount or 0
+ await cur.execute(f"DELETE FROM wiki_query_logs WHERE {where_logs}", params_logs)
+ counts["wiki_query_logs"] = cur.rowcount or 0
+ return counts
+
+
+def _row_to_dict(keys: Sequence[str], row: Sequence[Any]) -> Dict[str, Any]:
+ data = dict(zip(keys, row))
+ for key, value in list(data.items()):
+ if isinstance(value, datetime):
+ data[key] = value.isoformat()
+ return data
+
+
+def _admin_plan_from_row(row: Optional[Sequence[Any]]) -> Dict[str, Any]:
+ if not row:
+ return {}
+ keys = [
+ "id", "chat_id", "status", "intent", "scope", "kb_id", "kb_name",
+ "summary", "actions", "executed", "metadata", "created_at", "updated_at", "expires_at",
+ ]
+ return _row_to_dict(keys, row)
+
+
+def _time_where(
+ column: str,
+ before: Optional[datetime],
+ after: Optional[datetime],
+) -> tuple[str, List[Any]]:
+ parts = []
+ params: List[Any] = []
+ if before is not None:
+ parts.append(f"{column} < %s")
+ params.append(before)
+ if after is not None:
+ parts.append(f"{column} >= %s")
+ params.append(after)
+ return " AND ".join(parts) if parts else "1=0", params
+
+
+def _extract_search_terms(query: str) -> List[str]:
+ import re
+
+ query = (query or "").strip()
+ if not query:
+ return []
+ terms = []
+ for token in re.findall(r"[\u4e00-\u9fff]{2,12}|[A-Za-z0-9][A-Za-z0-9_\-]{1,30}", query):
+ if token not in terms:
+ terms.append(token)
+ if re.fullmatch(r"[\u4e00-\u9fff]{3,12}", token):
+ for size in (2, 3, 4, 5, 6):
+ if len(token) < size:
+ continue
+ for idx in range(0, len(token) - size + 1):
+ gram = token[idx:idx + size]
+ if gram not in terms:
+ terms.append(gram)
+ compact_phrase = _compact_search_phrase(query)
+ if compact_phrase and compact_phrase not in terms:
+ terms.insert(0, compact_phrase)
+ if query and query not in terms:
+ terms.append(query[:80])
+ return terms[:40]
+
+
+_LOW_VALUE_SEARCH_TERMS = {
+ "什么", "如何", "怎么", "哪些", "为什么", "是什么", "有哪些",
+ "介绍", "说明", "查询", "检索", "请问", "帮我", "一下",
+ "主要", "内容", "信息", "情况", "方法", "步骤", "这个",
+ "那个", "是否", "有没有", "吗", "呢", "what", "how",
+ "why", "which", "please",
+}
+
+
+def _compact_search_phrase(text: str) -> str:
+ text = re.sub(r"\s+", "", (text or "").strip().lower())
+ for word in _LOW_VALUE_SEARCH_TERMS:
+ text = text.replace(word, "")
+ return text[:80]
+
+
+def _is_low_value_search_term(term: str) -> bool:
+ term = (term or "").strip().lower()
+ if not term:
+ return True
+ if term in _LOW_VALUE_SEARCH_TERMS:
+ return True
+ if re.fullmatch(r"\d+", term):
+ return True
+ if re.fullmatch(r"[\u4e00-\u9fff]", term):
+ return True
+ return False
+
+
+def _meaningful_search_terms(query: str) -> List[str]:
+ terms = []
+ for term in _extract_search_terms(query):
+ t = (term or "").strip().lower()
+ if not t or _is_low_value_search_term(t):
+ continue
+ if t not in terms:
+ terms.append(t)
+ return terms[:24]
+
+
+def _score_page(query: str, page: Dict[str, Any]) -> float:
+ title = str(page.get("title") or "").lower()
+ slug = str(page.get("slug") or "").lower()
+ summary = str(page.get("summary") or "").lower()
+ content = str(page.get("content") or "")[:4000].lower()
+ keywords = json.dumps(page.get("keywords") or [], ensure_ascii=False).lower()
+ aliases = json.dumps(page.get("aliases") or [], ensure_ascii=False).lower()
+ score = float(page.get("score_hint") or 1.0)
+ compact_query = re.sub(r"\s+", "", query or "").lower()
+ compact_phrase = _compact_search_phrase(compact_query)
+ if compact_query:
+ if compact_query == title:
+ score += 40.0
+ elif compact_query in title:
+ score += 25.0
+ elif compact_query in aliases:
+ score += 18.0
+ elif compact_query in summary:
+ score += 8.0
+ elif compact_query in content:
+ score += 2.0
+ if compact_phrase and compact_phrase != compact_query:
+ if compact_phrase == title:
+ score += 36.0
+ elif compact_phrase in title:
+ score += 24.0
+ elif compact_phrase in aliases:
+ score += 18.0
+ elif compact_phrase in summary:
+ score += 14.0
+ elif compact_phrase in content:
+ score += 10.0
+ for term in _extract_search_terms(query):
+ t = term.lower()
+ if not t:
+ continue
+ weight = _term_weight(t)
+ if title == t:
+ score += 18.0 * weight
+ elif slug == t:
+ score += 14.0 * weight
+ elif t in title:
+ score += 9.0 * weight
+ if t in aliases:
+ score += 7.0 * weight
+ if t in keywords:
+ score += 4.0 * weight
+ if t in summary:
+ score += 2.0 * weight
+ if t in slug:
+ score += 2.0 * weight
+ if t in content:
+ score += 0.4 * weight
+ meaningful_terms = _meaningful_search_terms(query)
+ if meaningful_terms:
+ title_text = f"{title} {aliases}"
+ surface_text = f"{title} {aliases} {keywords} {summary}"
+ full_text = f"{surface_text} {content}"
+ matched_title = [t for t in meaningful_terms if t in title_text]
+ matched_surface = [t for t in meaningful_terms if t in surface_text]
+ matched_full = [t for t in meaningful_terms if t in full_text]
+ score += 18.0 * (len(matched_title) / len(meaningful_terms))
+ score += 12.0 * (len(matched_surface) / len(meaningful_terms))
+ score += 18.0 * (len(matched_full) / len(meaningful_terms))
+ longest_query_term = max(meaningful_terms, key=len)
+ if len(longest_query_term) >= 4 and longest_query_term not in full_text:
+ score *= 0.72
+ specific_terms = [
+ (term.lower(), _term_weight(term.lower()))
+ for term in _extract_search_terms(query)
+ if _term_weight(term.lower()) >= 2.0
+ ]
+ max_specific_weight = max((weight for _, weight in specific_terms), default=0.0)
+ top_specific_terms = [
+ term for term, weight in specific_terms
+ if weight >= max_specific_weight and term
+ ]
+ if (
+ str(page.get("page_type") or "") == "document_summary"
+ and top_specific_terms
+ and not any(term in title or term in aliases or term in slug for term in top_specific_terms)
+ ):
+ score *= 0.45
+ return score
+
+
+def _term_weight(term: str) -> float:
+ if not term:
+ return 0.0
+ if _is_low_value_search_term(term):
+ return 0.2
+ if re.search(r"[a-z]", term) and re.search(r"[0-9]", term):
+ return 3.0
+ if term in {"系统", "故障", "设备", "维修", "操作", "方法", "内容", "什么", "怎么", "如何", "哪些"}:
+ return 0.2
+ if re.fullmatch(r"[\u4e00-\u9fff]{2}", term):
+ return 1.2
+ if re.fullmatch(r"[\u4e00-\u9fff]+", term):
+ return min(2.5, max(0.6, len(term) / 3.0))
+ return min(2.5, max(0.8, len(term) / 6.0))
+
+
+def _score_slice(query: str, item: Dict[str, Any]) -> float:
+ filename = str(item.get("filename") or "").lower()
+ title = " ".join(str(x) for x in (item.get("title_path") or [])).lower()
+ keywords = json.dumps(item.get("keywords") or [], ensure_ascii=False).lower()
+ brief = str(item.get("brief") or "").lower()
+ content = str(item.get("content") or "").lower()
+ score = 0.0
+ for term in _extract_search_terms(query):
+ t = term.lower()
+ if not t:
+ continue
+ if t in filename:
+ score += 3.0
+ if t in title:
+ score += 2.5
+ if t in keywords:
+ score += 2.0
+ if t in brief:
+ score += 1.5
+ if t in content:
+ score += 1.0
+ return score
+
+
+async def _table_exists(cur: Any, table_name: str) -> bool:
+ try:
+ await cur.execute("SELECT to_regclass(%s)", (table_name,))
+ row = await cur.fetchone()
+ return bool(row and row[0])
+ except Exception:
+ return False
+
+
+async def _delete_optional_table_rows(
+ cur: Any,
+ table_name: str,
+ where_sql: str,
+ params: Sequence[Any],
+) -> int:
+ if not await _table_exists(cur, table_name):
+ return 0
+ await cur.execute(f"DELETE FROM {table_name} WHERE {where_sql}", list(params))
+ return cur.rowcount or 0
diff --git a/wiki_engine/htknow_client.py b/wiki_engine/htknow_client.py
new file mode 100644
index 0000000..ac9912c
--- /dev/null
+++ b/wiki_engine/htknow_client.py
@@ -0,0 +1,132 @@
+import asyncio
+from typing import Any, Dict, List, Optional
+from urllib.parse import quote
+
+import httpx
+
+from config import RAG_CONFIG, WIKI_CONFIG
+
+
+class HTKnowClient:
+ """Small async client for the 11000 knowledge service."""
+
+ def __init__(self, base_url: Optional[str] = None):
+ self.base_url = (base_url or WIKI_CONFIG["htknow_base_url"]).rstrip("/")
+
+ def _headers(self) -> Dict[str, str]:
+ return {
+ "accept": "application/json",
+ "x-user-id": _safe_header(RAG_CONFIG.get("x-user-id") or WIKI_CONFIG["default_user_id"]),
+ "x-user-name": _safe_header(RAG_CONFIG.get("x-user-name") or WIKI_CONFIG["default_user_name"]),
+ "x-role": _safe_header(RAG_CONFIG.get("x-role") or WIKI_CONFIG["default_role"]),
+ }
+
+ async def list_files(
+ self,
+ kb_id: Optional[str] = None,
+ limit: int = 20,
+ offset: int = 0,
+ ) -> List[Dict[str, Any]]:
+ url = f"{self.base_url}/api/v1/knowledge/files/"
+ # The 11000 service is more stable with its default-ish page sizes.
+ # Fetch at least 10 rows and trim locally instead of requesting tiny pages.
+ size = min(max(limit, 10), 100)
+ page = max(offset // size, 0) + 1
+ collected: List[Dict[str, Any]] = []
+ async with httpx.AsyncClient(timeout=60.0, verify=False, trust_env=False) as client:
+ while len(collected) < limit:
+ params: Dict[str, Any] = {"page": page, "size": size}
+ if kb_id:
+ params["kb_id"] = kb_id
+ payload = await _get_json_with_retry(client, url, params=params, headers=self._headers())
+ items = _extract_list(payload, candidates=("results", "data", "items", "files"))
+ if not items:
+ break
+ collected.extend(items)
+ if len(items) < params["size"]:
+ break
+ page += 1
+ return collected[:limit]
+
+ async def get_file_slices(self, file_id: str, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+ url = f"{self.base_url}/api/v1/knowledge/files/{file_id}/slices"
+ params = {}
+ if limit:
+ params["limit"] = limit
+ async with httpx.AsyncClient(timeout=120.0, verify=False, trust_env=False) as client:
+ payload = await _get_json_with_retry(client, url, params=params, headers=self._headers())
+ return _extract_list(payload, candidates=("slices", "results", "data", "items"))
+
+ async def get_file(self, file_id: str) -> Dict[str, Any]:
+ url = f"{self.base_url}/api/v1/knowledge/files/{file_id}"
+ async with httpx.AsyncClient(timeout=60.0, verify=False, trust_env=False) as client:
+ payload = await _get_json_with_retry(client, url, params={}, headers=self._headers())
+ if isinstance(payload, dict):
+ for key in ("data", "file", "item"):
+ value = payload.get(key)
+ if isinstance(value, dict):
+ return value
+ return payload
+ return {}
+
+ async def get_kb_tree(self) -> List[Dict[str, Any]]:
+ url = f"{self.base_url}/api/v1/knowledge/knowledge_base/tree"
+ async with httpx.AsyncClient(timeout=60.0, verify=False, trust_env=False) as client:
+ payload = await _get_json_with_retry(client, url, params={}, headers=self._headers())
+ if isinstance(payload, list):
+ return [x for x in payload if isinstance(x, dict)]
+ if isinstance(payload, dict):
+ data = payload.get("data") or payload.get("list") or payload
+ if isinstance(data, dict) and isinstance(data.get("children"), list):
+ return [x for x in data["children"] if isinstance(x, dict)]
+ if isinstance(data, list):
+ return [x for x in data if isinstance(x, dict)]
+ return []
+
+
+def _extract_list(payload: Any, candidates: tuple[str, ...]) -> List[Dict[str, Any]]:
+ if isinstance(payload, list):
+ return [x for x in payload if isinstance(x, dict)]
+ if not isinstance(payload, dict):
+ return []
+ for key in candidates:
+ value = payload.get(key)
+ if isinstance(value, list):
+ return [x for x in value if isinstance(x, dict)]
+ if isinstance(value, dict):
+ nested = _extract_list(value, candidates)
+ if nested:
+ return nested
+ return []
+
+
+def _safe_header(value: Any) -> str:
+ text = str(value or "")
+ try:
+ text.encode("latin-1")
+ return text
+ except UnicodeEncodeError:
+ return quote(text, safe="")
+
+
+async def _get_json_with_retry(
+ client: httpx.AsyncClient,
+ url: str,
+ params: Dict[str, Any],
+ headers: Dict[str, str],
+ retries: int = 2,
+) -> Any:
+ last_exc: Optional[Exception] = None
+ for attempt in range(retries + 1):
+ try:
+ response = await client.get(url, params=params, headers=headers)
+ response.raise_for_status()
+ return response.json()
+ except (httpx.HTTPStatusError, httpx.RequestError) as exc:
+ last_exc = exc
+ if attempt >= retries:
+ raise
+ await asyncio.sleep(0.5 * (attempt + 1))
+ if last_exc:
+ raise last_exc
+ return {}
diff --git a/wiki_engine/routes.py b/wiki_engine/routes.py
new file mode 100644
index 0000000..dc02a2e
--- /dev/null
+++ b/wiki_engine/routes.py
@@ -0,0 +1,230 @@
+from datetime import datetime
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from wiki_engine import db
+from wiki_engine import service
+
+
+router = APIRouter(prefix="/api/wiki", tags=["wiki"])
+
+
+class WikiSyncFileRequest(BaseModel):
+ kb_id: Optional[str] = None
+ kb_name: Optional[str] = None
+ use_llm: bool = False
+ background: bool = True
+
+
+class WikiSyncKbRequest(BaseModel):
+ kb_id: Optional[str] = None
+ kb_name: Optional[str] = None
+ limit: Optional[int] = None
+ use_llm: bool = False
+ background: bool = True
+
+
+class WikiSearchRequest(BaseModel):
+ query: str
+ kb_id: Optional[str] = None
+ kb_name: Optional[str] = None
+ limit: int = 8
+ include_slices: bool = True
+
+
+class WikiDeleteKbRequest(BaseModel):
+ kb_id: Optional[str] = None
+ kb_name: Optional[str] = None
+
+
+class WikiDeleteFileRequest(BaseModel):
+ file_id: str
+
+
+class WikiDeleteTimeRequest(BaseModel):
+ before: Optional[datetime] = None
+ after: Optional[datetime] = None
+ kb_id: Optional[str] = None
+ kb_name: Optional[str] = None
+
+
+class WikiDiffKbRequest(BaseModel):
+ kb_id: Optional[str] = None
+ kb_name: Optional[str] = None
+ include_children: bool = False
+ limit: int = 200
+ parsed_only: bool = True
+
+
+class WikiReconcileKbRequest(BaseModel):
+ kb_id: Optional[str] = None
+ kb_name: Optional[str] = None
+ include_children: bool = False
+ limit: int = 200
+ sync_missing: bool = True
+ rebuild_stale: bool = True
+ delete_orphan: bool = False
+ use_llm: bool = False
+ dry_run: bool = True
+
+
+@router.post("/init")
+async def init_wiki():
+ await db.init_wiki_tables()
+ return {"success": True}
+
+
+@router.post("/sync/file/{file_id}")
+async def sync_file(file_id: str, request: WikiSyncFileRequest):
+ kb_id = await _resolve_optional_kb_id(request.kb_id, request.kb_name)
+ if request.background:
+ job_id = await service.start_sync_job(
+ target_type="file",
+ target_id=file_id,
+ kb_id=kb_id,
+ use_llm=request.use_llm,
+ )
+ return {"success": True, "job_id": job_id, "status": "running", "job": await db.get_job(job_id)}
+ return await service.sync_file(file_id=file_id, kb_id=kb_id, use_llm=request.use_llm)
+
+
+@router.post("/sync/kb")
+async def sync_kb(request: WikiSyncKbRequest):
+ kb_id = await _resolve_required_kb_id(request.kb_id, request.kb_name)
+ if request.background:
+ job_id = await service.start_sync_job(
+ target_type="kb",
+ target_id=kb_id,
+ kb_id=kb_id,
+ limit=request.limit,
+ use_llm=request.use_llm,
+ )
+ return {"success": True, "job_id": job_id, "status": "running", "job": await db.get_job(job_id)}
+ return await service.sync_kb(kb_id=kb_id, limit=request.limit, use_llm=request.use_llm)
+
+
+@router.get("/sync/jobs/{job_id}")
+async def get_sync_job(job_id: str):
+ job = await db.get_job(job_id)
+ if not job:
+ raise HTTPException(status_code=404, detail="sync job not found")
+ return {"success": True, "job": job}
+
+
+@router.post("/search")
+async def search_wiki(request: WikiSearchRequest):
+ if not request.query.strip():
+ raise HTTPException(status_code=400, detail="query is empty")
+ kb_id = await _resolve_optional_kb_id(request.kb_id, request.kb_name)
+ return await service.search_wiki(
+ query=request.query,
+ kb_id=kb_id,
+ limit=request.limit,
+ include_slices=request.include_slices,
+ )
+
+
+@router.post("/evidence/search")
+async def search_evidence(request: WikiSearchRequest):
+ if not request.query.strip():
+ raise HTTPException(status_code=400, detail="query is empty")
+ kb_id = await _resolve_optional_kb_id(request.kb_id, request.kb_name)
+ return await service.evidence_search(
+ query=request.query,
+ kb_id=kb_id,
+ limit=request.limit,
+ )
+
+
+@router.get("/pages/{page_id}")
+async def get_page(page_id: int):
+ page = await service.get_page(page_id)
+ if not page:
+ raise HTTPException(status_code=404, detail="wiki page not found")
+ return {"success": True, "page": page}
+
+
+@router.get("/stats")
+async def get_stats():
+ return {"success": True, "stats": await service.get_stats()}
+
+
+@router.get("/queue/stats")
+async def get_queue_stats():
+ return {"success": True, "stats": await service.get_queue_stats()}
+
+
+@router.get("/tree")
+async def get_kb_tree():
+ return await service.get_kb_tree()
+
+
+@router.post("/diff/kb")
+async def diff_kb(request: WikiDiffKbRequest):
+ kb_id = await _resolve_required_kb_id(request.kb_id, request.kb_name)
+ return await service.diff_kb(
+ kb_id=kb_id,
+ include_children=request.include_children,
+ limit=request.limit,
+ parsed_only=request.parsed_only,
+ )
+
+
+@router.post("/reconcile/kb")
+async def reconcile_kb(request: WikiReconcileKbRequest):
+ kb_id = await _resolve_required_kb_id(request.kb_id, request.kb_name)
+ return await service.reconcile_kb(
+ kb_id=kb_id,
+ include_children=request.include_children,
+ limit=request.limit,
+ sync_missing=request.sync_missing,
+ rebuild_stale=request.rebuild_stale,
+ delete_orphan=request.delete_orphan,
+ use_llm=request.use_llm,
+ dry_run=request.dry_run,
+ )
+
+
+@router.post("/delete/by-kb")
+async def delete_by_kb(request: WikiDeleteKbRequest):
+ kb_id = await _resolve_required_kb_id(request.kb_id, request.kb_name)
+ return await service.delete_by_kb(kb_id)
+
+
+@router.post("/delete/by-file")
+async def delete_by_file(request: WikiDeleteFileRequest):
+ if not request.file_id.strip():
+ raise HTTPException(status_code=400, detail="file_id is empty")
+ return await service.delete_by_file(request.file_id)
+
+
+@router.post("/delete/by-time")
+async def delete_by_time(request: WikiDeleteTimeRequest):
+ if request.before is None and request.after is None:
+ raise HTTPException(status_code=400, detail="before or after is required")
+ kb_id = await _resolve_optional_kb_id(request.kb_id, request.kb_name)
+ return await service.delete_by_time(
+ before=request.before,
+ after=request.after,
+ kb_id=kb_id,
+ )
+
+
+async def _resolve_required_kb_id(kb_id: Optional[str], kb_name: Optional[str]) -> str:
+ resolved = await _resolve_optional_kb_id(kb_id, kb_name)
+ if not resolved:
+ raise HTTPException(status_code=400, detail="kb_name or kb_id is required")
+ return resolved
+
+
+async def _resolve_optional_kb_id(kb_id: Optional[str], kb_name: Optional[str]) -> Optional[str]:
+ if kb_id and kb_id.strip():
+ return kb_id.strip()
+ if not kb_name or not kb_name.strip():
+ return None
+ resolved = await service.resolve_kb_reference(kb_name.strip())
+ if not resolved.get("success"):
+ raise HTTPException(status_code=404, detail=resolved)
+ return str(resolved["kb_id"])
diff --git a/wiki_engine/service.py b/wiki_engine/service.py
new file mode 100644
index 0000000..6e7656f
--- /dev/null
+++ b/wiki_engine/service.py
@@ -0,0 +1,967 @@
+import asyncio
+import hashlib
+import json
+import re
+import uuid
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from config import WIKI_CONFIG
+from wiki_engine import db
+from wiki_engine.chunk_enhancer import enhance_slices
+from wiki_engine.htknow_client import HTKnowClient
+from wiki_engine.wiki_builder import build_pages_for_file
+
+
+_FILE_SLICE_TEXT_CACHE: Dict[str, Dict[str, str]] = {}
+
+
+async def sync_file(
+ file_id: str,
+ kb_id: Optional[str] = None,
+ file_info: Optional[Dict[str, Any]] = None,
+ use_llm: bool = False,
+ job_id: Optional[str] = None,
+ force: bool = False,
+) -> Dict[str, Any]:
+ client = HTKnowClient()
+ file_info = dict(file_info or {})
+ filename = str(file_info.get("filename") or file_info.get("name") or "")
+ if job_id:
+ await db.update_job(
+ job_id,
+ stage="fetch_file",
+ current_file_id=str(file_id),
+ current_filename=filename,
+ )
+ if not file_info or not (file_info.get("filename") or file_info.get("name")):
+ try:
+ file_info.update(await client.get_file(str(file_id)))
+ except Exception as exc:
+ print(f"[wiki_engine] get file detail failed for {file_id}: {exc}")
+ file_info.setdefault("id", str(file_id))
+ file_info.setdefault("file_id", str(file_id))
+ if kb_id:
+ file_info.setdefault("kb_id", str(kb_id))
+ filename = str(file_info.get("filename") or file_info.get("name") or "")
+ if job_id:
+ await db.update_job(job_id, stage="fetch_slices", current_filename=filename)
+
+ slices = await client.get_file_slices(str(file_id), limit=WIKI_CONFIG.get("max_slices_per_file"))
+ if job_id:
+ await db.update_job(job_id, stage="enhance_slices")
+ refs = enhance_slices(
+ slices,
+ file_info=file_info,
+ max_slices=int(WIKI_CONFIG.get("max_slices_per_file") or 300),
+ )
+ if not refs:
+ raise ValueError(f"no slices returned for file {file_id}; keep existing wiki data unchanged")
+
+ content_signature = _file_content_signature(refs)
+ existing_state = await db.get_file_state(str(file_id))
+ existing_page_count = await db.count_file_pages(str(file_id)) if existing_state else 0
+ if not force and _is_unchanged_file(existing_state, refs, content_signature) and existing_page_count > 0:
+ if job_id:
+ await db.update_job(job_id, stage="skipped_unchanged")
+ return {
+ "success": True,
+ "skipped": True,
+ "reason": "unchanged",
+ "file_id": str(file_id),
+ "kb_id": str(kb_id or file_info.get("kb_id") or ""),
+ "slice_count": len(refs),
+ "embedding_count": 0,
+ "embedding_error": "",
+ "page_count": existing_page_count,
+ "page_ids": [],
+ }
+
+ source_hash = str(file_info.get("hash") or "")
+ metadata = file_info.get("metadata") if isinstance(file_info.get("metadata"), dict) else {}
+ metadata = dict(metadata)
+ metadata["content_signature"] = content_signature
+ metadata["source_hash"] = source_hash
+ metadata["wiki_pipeline_version"] = "wiki-sync-v3"
+ file_info["metadata"] = metadata
+ file_info["content_signature"] = content_signature
+ if not source_hash:
+ file_info["hash"] = content_signature
+
+ await db.upsert_file_state(file_info, slice_count=len(refs))
+ if job_id:
+ await db.update_job(job_id, stage="write_slices")
+ await db.replace_file_slice_refs(str(file_id), refs)
+ if job_id:
+ await db.update_job(job_id, stage="build_pages")
+ async def progress(stage: str) -> None:
+ if job_id and WIKI_CONFIG.get("stage_progress_enabled", True):
+ await db.update_job(job_id, stage=stage)
+
+ pages = await build_pages_for_file(file_info, refs, use_llm=use_llm, progress=progress if job_id else None)
+ if job_id:
+ await db.update_job(job_id, stage="write_pages")
+ page_ids = []
+ for page in pages:
+ page_id = await _upsert_page_with_slug_lock(page, owner=job_id or f"sync:{file_id}")
+ page_ids.append(page_id)
+ await db.replace_page_slices(page_id, page.get("links", []))
+ if job_id:
+ await db.update_job(job_id, stage="file_done")
+
+ return {
+ "success": True,
+ "skipped": False,
+ "file_id": str(file_id),
+ "kb_id": str(kb_id or file_info.get("kb_id") or ""),
+ "slice_count": len(refs),
+ "embedding_count": 0,
+ "embedding_error": "",
+ "page_count": len(page_ids),
+ "page_ids": page_ids,
+ }
+
+
+async def sync_kb(
+ kb_id: str,
+ limit: Optional[int] = None,
+ use_llm: bool = False,
+ job_id: Optional[str] = None,
+ force: bool = False,
+) -> Dict[str, Any]:
+ limit = int(limit or WIKI_CONFIG.get("default_sync_limit") or 20)
+ client = HTKnowClient()
+ files = await client.list_files(kb_id=kb_id, limit=limit)
+ if job_id:
+ await db.update_job(job_id, total_files=len(files))
+
+ processed = 0
+ total_slices = 0
+ skipped = 0
+ page_count = 0
+ errors: List[Dict[str, str]] = []
+ for file_info in files:
+ file_id = str(file_info.get("id") or file_info.get("file_id") or "")
+ if not file_id:
+ continue
+ try:
+ result = await sync_file(file_id, kb_id=kb_id, file_info=file_info, use_llm=use_llm, job_id=job_id, force=force)
+ processed += 1
+ total_slices += int(result.get("slice_count") or 0)
+ skipped += 1 if result.get("skipped") else 0
+ if not result.get("skipped"):
+ page_count += int(result.get("page_count") or 0)
+ except Exception as exc:
+ errors.append({"file_id": file_id, "error": str(exc)})
+ if job_id:
+ await db.update_job(
+ job_id,
+ processed_files=processed,
+ total_slices=total_slices,
+ skipped_files=skipped,
+ page_count=page_count,
+ )
+ await asyncio.sleep(0)
+
+ if job_id and WIKI_CONFIG.get("finalize_enabled", True):
+ await enqueue_finalize_job(kb_id, job_id)
+
+ return {
+ "success": not errors,
+ "kb_id": str(kb_id),
+ "file_count": len(files),
+ "processed_files": processed,
+ "total_slices": total_slices,
+ "skipped_files": skipped,
+ "page_count": page_count,
+ "errors": errors[:20],
+ }
+
+
+async def start_sync_job(
+ target_type: str,
+ target_id: str,
+ kb_id: Optional[str] = None,
+ limit: Optional[int] = None,
+ use_llm: bool = False,
+ force: bool = False,
+) -> str:
+ job_id = str(uuid.uuid4())
+ await db.create_job(job_id, target_type=target_type, target_id=target_id, kb_id=kb_id or "")
+ if WIKI_CONFIG.get("queue_enabled", True):
+ if target_type == "file":
+ await db.set_job_totals(job_id, 1)
+ await db.enqueue_wiki_file_op(
+ job_id=job_id,
+ kb_id=kb_id or "",
+ file_id=target_id,
+ file_info={},
+ use_llm=use_llm,
+ force=force,
+ )
+ return job_id
+ if target_type == "kb":
+ client = HTKnowClient()
+ files = await client.list_files(kb_id=target_id, limit=limit or WIKI_CONFIG.get("default_sync_limit"))
+ valid_files = [
+ (str(file_info.get("id") or file_info.get("file_id") or ""), file_info)
+ for file_info in files
+ if str(file_info.get("id") or file_info.get("file_id") or "")
+ ]
+ await db.set_job_totals(job_id, len(valid_files))
+ if not valid_files:
+ await db.update_job(job_id, status="success")
+ return job_id
+ for file_id, file_info in valid_files:
+ await db.enqueue_wiki_file_op(
+ job_id=job_id,
+ kb_id=target_id,
+ file_id=file_id,
+ file_info=file_info,
+ use_llm=use_llm,
+ force=force,
+ )
+ return job_id
+
+ async def _runner() -> None:
+ try:
+ if target_type == "file":
+ await sync_file(target_id, kb_id=kb_id, use_llm=use_llm, job_id=job_id, force=force)
+ elif target_type == "kb":
+ await sync_kb(target_id, limit=limit, use_llm=use_llm, job_id=job_id, force=force)
+ else:
+ raise ValueError(f"unsupported target_type: {target_type}")
+ await db.update_job(job_id, status="success")
+ except Exception as exc:
+ await db.update_job(job_id, status="failed", error=str(exc))
+
+ asyncio.create_task(_runner())
+ return job_id
+
+
+async def get_queue_stats() -> Dict[str, Any]:
+ return await db.get_wiki_queue_stats()
+
+
+async def get_sync_job(job_id: str) -> Optional[Dict[str, Any]]:
+ return await db.get_job(job_id)
+
+
+async def enqueue_finalize_job(kb_id: str, job_id: str = "") -> Dict[str, Any]:
+ if not WIKI_CONFIG.get("finalize_enabled", True):
+ return {"success": True, "enqueued": False, "reason": "disabled"}
+ op_id = await db.enqueue_wiki_finalize_op(
+ job_id=job_id,
+ kb_id=kb_id,
+ delay_seconds=int(WIKI_CONFIG.get("finalize_debounce_seconds") or 20),
+ )
+ return {"success": True, "enqueued": bool(op_id), "op_id": op_id}
+
+
+async def finalize_kb(kb_id: str, job_id: str = "") -> Dict[str, Any]:
+ if job_id:
+ await db.update_job(job_id, stage="finalize")
+ page_count = await db.count_pages_for_kb(kb_id)
+ if job_id:
+ await db.update_job(job_id, stage="completed", page_count=page_count, current_file_id="", current_filename="")
+ return {"success": True, "kb_id": str(kb_id or ""), "page_count": page_count}
+
+
+async def search_wiki(
+ query: str,
+ kb_id: Optional[str] = None,
+ limit: int = 8,
+ include_slices: bool = True,
+) -> Dict[str, Any]:
+ page_limit = max(int(WIKI_CONFIG.get("page_search_top_k") or limit or 8), limit)
+ pages = await db.search_pages(query=query, kb_id=kb_id, limit=page_limit)
+
+ slice_map = {}
+ if include_slices and pages:
+ slice_map = await db.get_page_slices(
+ [int(p["id"]) for p in pages],
+ max_slices_per_page=int(WIKI_CONFIG.get("evidence_slices_per_page") or 6),
+ )
+ items = []
+ for rank, page in enumerate(pages, start=1):
+ page_id = int(page["id"])
+ slices = slice_map.get(page_id, [])
+ related_slice_ids = [str(s.get("slice_id")) for s in slices if s.get("slice_id")]
+ related_file_ids = list({
+ str(s.get("file_id"))
+ for s in slices
+ if s.get("file_id")
+ } | set([str(x) for x in (page.get("related_file_ids") or []) if x]))
+ items.append({
+ "source_type": "wiki",
+ "source_id": str(page_id),
+ "kb_id": str(page.get("kb_id") or ""),
+ "rank_sources": {"page": rank},
+ "slug": page.get("slug", ""),
+ "title": page.get("title", ""),
+ "page_type": page.get("page_type", ""),
+ "summary": page.get("summary", ""),
+ "content": page.get("content", ""),
+ "keywords": page.get("keywords", []),
+ "aliases": page.get("aliases", []),
+ "score": page.get("score", 0.0),
+ "related_file_ids": related_file_ids,
+ "related_slice_ids": related_slice_ids,
+ "slices": slices,
+ })
+
+ items = _fuse_wiki_items(items, limit=max(limit, 1))
+ await db.log_query(query, kb_id, len(items))
+ return {
+ "success": True,
+ "query": query,
+ "kb_id": kb_id or "",
+ "items": items,
+ "debug": {"page_hits": len(pages), "vector_hits": 0, "text_hits": 0},
+ }
+
+
+async def evidence_search(query: str, kb_id: Optional[str] = None, limit: int = 5) -> Dict[str, Any]:
+ result = await search_wiki(
+ query=query,
+ kb_id=kb_id,
+ limit=max(limit * 2, limit),
+ include_slices=True,
+ )
+ evidence_items = []
+ for item in result.get("items", []):
+ slices = await _hydrate_wiki_slices(item.get("slices", []))
+ slice_texts = []
+ for s in slices[:6]:
+ text = str(s.get("content") or s.get("brief") or "").strip()
+ if not text:
+ continue
+ filename = str(s.get("filename") or "").strip()
+ title_path = " / ".join(str(x) for x in (s.get("title_path") or []) if x)
+ prefix_parts = [x for x in (filename, title_path) if x]
+ prefix = f"{' - '.join(prefix_parts)}\n" if prefix_parts else ""
+ slice_texts.append(
+ f"{prefix}{text}\n"
+ f"(file_id={s.get('file_id') or ''}, slice_id={s.get('slice_id') or ''})"
+ )
+ deep_text = "\n\n".join(slice_texts)
+ page_text = "\n".join(x for x in [
+ str(item.get("summary") or "").strip(),
+ str(item.get("content") or "").strip(),
+ ] if x).strip()
+ text = "\n\n".join(x for x in [page_text, deep_text] if x).strip()
+ if not text:
+ continue
+ evidence_items.append({
+ "source_type": "wiki",
+ "source_id": item.get("source_id"),
+ "title": item.get("title"),
+ "text": text,
+ "score": item.get("score", 0.0),
+ "kb_id": item.get("kb_id", ""),
+ "metadata": {
+ "kb_id": item.get("kb_id", ""),
+ "page_type": item.get("page_type"),
+ "keywords": item.get("keywords", []),
+ "related_file_ids": item.get("related_file_ids", []),
+ "related_slice_ids": item.get("related_slice_ids", []),
+ "slices": slices,
+ "rank_sources": item.get("rank_sources", {}),
+ },
+ })
+ # Keep the page-search order here. The baike workflow does a later cross-source
+ # rerank, and preserving the Wiki top hit makes the displayed Wiki evidence
+ # match `/api/wiki/search`.
+ evidence_items.sort(key=lambda x: float(x.get("score") or 0.0), reverse=True)
+ return {"success": True, "query": query, "items": evidence_items[:limit], "debug": result.get("debug", {})}
+
+
+def _fuse_wiki_items(items: List[Dict[str, Any]], limit: int) -> List[Dict[str, Any]]:
+ rrf_k = max(int(WIKI_CONFIG.get("rrf_k") or 60), 1)
+ by_key: Dict[str, Dict[str, Any]] = {}
+ for item in items:
+ key = str(item.get("source_id") or item.get("title") or "")
+ if not key:
+ continue
+ fused_score = float(item.get("score") or 0.0)
+ for rank in (item.get("rank_sources") or {}).values():
+ try:
+ fused_score += 1.0 / (rrf_k + int(rank))
+ except Exception:
+ continue
+ item = dict(item)
+ item["score"] = fused_score
+ existing = by_key.get(key)
+ if existing:
+ merged_sources = dict(existing.get("rank_sources") or {})
+ for source, rank in (item.get("rank_sources") or {}).items():
+ if source not in merged_sources or int(rank) < int(merged_sources.get(source) or 999999):
+ merged_sources[source] = rank
+ item["rank_sources"] = merged_sources
+ fused_score = max(fused_score, float(existing.get("score") or 0.0))
+ for rank in merged_sources.values():
+ try:
+ fused_score += 1.0 / (rrf_k + int(rank))
+ except Exception:
+ continue
+ item["score"] = fused_score
+ if not existing or fused_score > float(existing.get("score") or 0.0):
+ by_key[key] = item
+ return sorted(by_key.values(), key=lambda x: float(x.get("score") or 0.0), reverse=True)[:limit]
+
+
+async def _rerank_wiki_evidence(query: str, items: List[Dict[str, Any]], top_k: int) -> List[Dict[str, Any]]:
+ if not query.strip() or not items:
+ return items[:top_k]
+ try:
+ from modelsAPI.model_api import OpenaiAPI
+
+ selected = items[:24]
+ documents = [
+ "\n".join([
+ str(item.get("title") or ""),
+ str(item.get("text") or ""),
+ ])[:4000]
+ for item in selected
+ ]
+ rows = await OpenaiAPI.rerank(query=query, documents=documents, top_k=top_k)
+ if not rows:
+ return items[:top_k]
+ ranked: List[Dict[str, Any]] = []
+ used = set()
+ for row in rows:
+ idx = int(row.get("index"))
+ if idx < 0 or idx >= len(selected) or idx in used:
+ continue
+ item = dict(selected[idx])
+ item["rerank_score"] = float(row.get("relevance_score") or 0.0)
+ item["score"] = max(float(item.get("score") or 0.0), item["rerank_score"])
+ ranked.append(item)
+ used.add(idx)
+ for idx, item in enumerate(selected):
+ if idx not in used and len(ranked) < top_k:
+ ranked.append(item)
+ return ranked[:top_k]
+ except Exception as exc:
+ print(f"[wiki_engine] wiki evidence rerank failed: {exc}")
+ return items[:top_k]
+
+
+async def _hydrate_wiki_slices(slices: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """Fill missing slice content from 11000 for old Wiki rows."""
+ out = [dict(s or {}) for s in slices]
+ missing_by_file: Dict[str, set[str]] = {}
+ for s in out:
+ if str(s.get("content") or "").strip():
+ continue
+ file_id = str(s.get("file_id") or "").strip()
+ slice_id = str(s.get("slice_id") or "").strip()
+ if file_id and slice_id:
+ missing_by_file.setdefault(file_id, set()).add(slice_id)
+ if not missing_by_file:
+ return out
+
+ client = HTKnowClient()
+ fetched: Dict[str, str] = {}
+ for file_id, wanted_ids in missing_by_file.items():
+ cached = _FILE_SLICE_TEXT_CACHE.get(file_id)
+ if cached is not None:
+ fetched.update({sid: cached.get(sid, "") for sid in wanted_ids})
+ continue
+ try:
+ raw_slices = await client.get_file_slices(file_id, limit=WIKI_CONFIG.get("max_slices_per_file"))
+ except Exception as exc:
+ print(f"[wiki_engine] hydrate slices failed for file {file_id}: {exc}")
+ continue
+ file_texts: Dict[str, str] = {}
+ for idx, raw in enumerate(raw_slices):
+ sid = str(raw.get("id") or raw.get("slice_id") or raw.get("uuid") or f"{file_id}:{idx}")
+ text = _raw_slice_text(raw)
+ if text:
+ file_texts[sid] = text
+ if sid in wanted_ids:
+ fetched[sid] = text
+ _FILE_SLICE_TEXT_CACHE[file_id] = file_texts
+
+ for s in out:
+ sid = str(s.get("slice_id") or "")
+ if sid in fetched and fetched[sid]:
+ s["content"] = fetched[sid]
+ return out
+
+
+def _file_content_signature(refs: List[Dict[str, Any]]) -> str:
+ rows = []
+ for ref in sorted(refs, key=lambda x: (int(x.get("ordinal") or 0), str(x.get("slice_id") or ""))):
+ rows.append({
+ "slice_id": str(ref.get("slice_id") or ""),
+ "content_hash": str(ref.get("content_hash") or ""),
+ "title_path": ref.get("title_path") or [],
+ "keywords": ref.get("keywords") or [],
+ })
+ raw = json.dumps(rows, ensure_ascii=False, sort_keys=True)
+ return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+def _is_unchanged_file(
+ state: Optional[Dict[str, Any]],
+ refs: List[Dict[str, Any]],
+ content_signature: str,
+) -> bool:
+ if not state:
+ return False
+ if int(state.get("slice_count") or 0) != len(refs):
+ return False
+ metadata = state.get("metadata") if isinstance(state.get("metadata"), dict) else {}
+ stored_signature = str(metadata.get("content_signature") or "")
+ if stored_signature and stored_signature == content_signature:
+ return True
+ return str(state.get("file_hash") or "") == content_signature
+
+
+async def _upsert_page_with_slug_lock(page: Dict[str, Any], owner: str) -> int:
+ kb_id = str(page.get("kb_id") or "")
+ slug = str(page.get("slug") or "")
+ if not kb_id or not slug:
+ return await db.upsert_wiki_page(page)
+
+ ttl = int(WIKI_CONFIG.get("slug_lock_ttl_seconds") or 600)
+ acquired = False
+ for _ in range(600):
+ acquired = await db.acquire_wiki_page_lock(kb_id, slug, owner=owner, ttl_seconds=ttl)
+ if acquired:
+ break
+ await asyncio.sleep(0.1)
+ if not acquired:
+ raise TimeoutError(f"wiki page lock timeout: {kb_id}/{slug}")
+
+ try:
+ return await db.upsert_wiki_page(page)
+ finally:
+ await db.release_wiki_page_lock(kb_id, slug, owner=owner)
+
+
+def _raw_slice_text(raw: Dict[str, Any]) -> str:
+ for key in ("content", "text", "page_content", "slice_content", "markdown"):
+ value = raw.get(key)
+ if isinstance(value, str) and value.strip():
+ return value
+ return ""
+
+
+async def get_page(page_id: int) -> Optional[Dict[str, Any]]:
+ return await db.get_page(page_id)
+
+
+async def get_stats() -> Dict[str, Any]:
+ return await db.get_stats()
+
+
+async def get_kb_tree() -> Dict[str, Any]:
+ client = HTKnowClient()
+ tree = await client.get_kb_tree()
+ return {"success": True, "tree": tree}
+
+
+async def resolve_kb_reference(reference: str) -> Dict[str, Any]:
+ """Resolve a user-facing knowledge-base name or admin id to a kb_id."""
+ ref = str(reference or "").strip()
+ if not ref:
+ return {"success": False, "error": "knowledge base name is empty", "matches": []}
+
+ client = HTKnowClient()
+ tree = await client.get_kb_tree()
+ nodes = _flatten_kb_tree(tree)
+ ref_norm = _normalize_kb_text(ref)
+
+ scored: List[Dict[str, Any]] = []
+ for node in nodes:
+ node_id = str(node.get("id") or "")
+ names = _node_names(node)
+ best_score = 0
+ best_name = names[0] if names else ""
+ if node_id and node_id == ref:
+ best_score = 10000
+ for name in names:
+ name_norm = _normalize_kb_text(name)
+ if not name_norm:
+ continue
+ score = 0
+ if name_norm == ref_norm:
+ score = 9000 + len(name_norm)
+ elif name_norm in ref_norm:
+ score = 7000 + len(name_norm)
+ elif ref_norm in name_norm:
+ score = 5000 + len(ref_norm)
+ if score > best_score:
+ best_score = score
+ best_name = name
+ if best_score:
+ scored.append({
+ "kb_id": node_id,
+ "name": best_name,
+ "path": node.get("_path", best_name),
+ "score": best_score,
+ })
+
+ scored.sort(key=lambda item: (-int(item.get("score") or 0), -len(str(item.get("path") or ""))))
+ if not scored:
+ return {
+ "success": False,
+ "error": f"未找到名为“{ref}”的知识库",
+ "matches": [],
+ "candidates": [
+ {"kb_id": str(node.get("id") or ""), "name": (_node_names(node) or [""])[0], "path": node.get("_path", "")}
+ for node in nodes[:20]
+ ],
+ }
+
+ top_score = int(scored[0].get("score") or 0)
+ top_matches = [item for item in scored if int(item.get("score") or 0) == top_score]
+ if len({item.get("kb_id") for item in top_matches}) > 1 and top_score < 10000:
+ return {
+ "success": False,
+ "error": f"知识库名称“{ref}”匹配到多个结果，请说得更完整一点",
+ "matches": top_matches[:10],
+ }
+
+ best = scored[0]
+ return {
+ "success": True,
+ "kb_id": str(best.get("kb_id") or ""),
+ "kb_name": str(best.get("name") or ""),
+ "kb_path": str(best.get("path") or ""),
+ "matches": scored[:10],
+ }
+
+
+async def diff_kb(
+ kb_id: str,
+ include_children: bool = False,
+ limit: int = 200,
+ parsed_only: bool = True,
+) -> Dict[str, Any]:
+ client = HTKnowClient()
+ kb_ids = await _resolve_kb_ids(client, kb_id, include_children=include_children)
+ ht_files: Dict[str, Dict[str, Any]] = {}
+ skipped_files: List[Dict[str, Any]] = []
+
+ for kid in kb_ids:
+ files = await client.list_files(kb_id=kid, limit=max(int(limit or 200), 1))
+ for file_info in files:
+ fid = str(file_info.get("id") or file_info.get("file_id") or "")
+ if not fid:
+ continue
+ if parsed_only and str(file_info.get("status")) != "1":
+ skipped_files.append(_compact_file(file_info))
+ continue
+ file_info["file_id"] = fid
+ file_info["kb_id"] = str(file_info.get("kb_id") or kid)
+ ht_files[fid] = file_info
+
+ wiki_states = await db.list_file_states(kb_ids)
+ wiki_by_file = {str(item.get("file_id")): item for item in wiki_states}
+
+ missing_in_wiki = []
+ stale_in_wiki = []
+ for fid, file_info in ht_files.items():
+ state = wiki_by_file.get(fid)
+ if not state:
+ missing_in_wiki.append(_compact_file(file_info))
+ elif _is_stale(file_info, state):
+ stale_in_wiki.append({
+ "file_id": fid,
+ "kb_id": str(file_info.get("kb_id") or ""),
+ "filename": str(file_info.get("filename") or ""),
+ "htknow_hash": str(file_info.get("hash") or ""),
+ "wiki_hash": str(state.get("file_hash") or ""),
+ "htknow_updated_at": str(file_info.get("updated_at") or ""),
+ "wiki_source_updated_at": str(state.get("source_updated_at") or ""),
+ "last_synced_at": state.get("last_synced_at") or "",
+ })
+
+ orphan_in_wiki = []
+ for fid, state in wiki_by_file.items():
+ if fid not in ht_files:
+ orphan_in_wiki.append({
+ "file_id": fid,
+ "kb_id": str(state.get("kb_id") or ""),
+ "filename": str(state.get("filename") or ""),
+ "last_synced_at": state.get("last_synced_at") or "",
+ })
+
+ return {
+ "success": True,
+ "kb_id": str(kb_id),
+ "include_children": include_children,
+ "kb_ids": kb_ids,
+ "summary": {
+ "htknow_files": len(ht_files),
+ "wiki_files": len(wiki_by_file),
+ "missing_in_wiki": len(missing_in_wiki),
+ "stale_in_wiki": len(stale_in_wiki),
+ "orphan_in_wiki": len(orphan_in_wiki),
+ "skipped_unparsed": len(skipped_files),
+ },
+ "missing_in_wiki": missing_in_wiki,
+ "stale_in_wiki": stale_in_wiki,
+ "orphan_in_wiki": orphan_in_wiki,
+ "skipped_unparsed": skipped_files[:50],
+ }
+
+
+async def reconcile_kb(
+ kb_id: str,
+ include_children: bool = False,
+ limit: int = 200,
+ sync_missing: bool = True,
+ rebuild_stale: bool = True,
+ delete_orphan: bool = False,
+ use_llm: bool = False,
+ dry_run: bool = True,
+) -> Dict[str, Any]:
+ diff = await diff_kb(kb_id=kb_id, include_children=include_children, limit=limit, parsed_only=True)
+ actions: List[Dict[str, Any]] = []
+ if sync_missing:
+ actions.extend({"action": "sync_file", "file_id": x["file_id"], "kb_id": x.get("kb_id", "")} for x in diff["missing_in_wiki"])
+ if rebuild_stale:
+ actions.extend({"action": "rebuild_file", "file_id": x["file_id"], "kb_id": x.get("kb_id", "")} for x in diff["stale_in_wiki"])
+ if delete_orphan:
+ actions.extend({"action": "delete_wiki_file", "file_id": x["file_id"], "kb_id": x.get("kb_id", "")} for x in diff["orphan_in_wiki"])
+
+ executed: List[Dict[str, Any]] = []
+ if not dry_run:
+ client = HTKnowClient()
+ for action in actions:
+ try:
+ if action["action"] in {"sync_file", "rebuild_file"}:
+ file_info = await client.get_file(action["file_id"])
+ if action.get("kb_id"):
+ file_info["kb_id"] = action["kb_id"]
+ result = await sync_file(
+ action["file_id"],
+ kb_id=action.get("kb_id"),
+ file_info=file_info,
+ use_llm=use_llm,
+ force=action["action"] == "rebuild_file",
+ )
+ elif action["action"] == "delete_wiki_file":
+ result = await delete_by_file(action["file_id"])
+ else:
+ result = {"success": False, "error": "unknown action"}
+ executed.append({"action": action, "result": result})
+ except Exception as exc:
+ executed.append({"action": action, "result": {"success": False, "error": str(exc)}})
+
+ return {
+ "success": True,
+ "dry_run": dry_run,
+ "kb_id": str(kb_id),
+ "include_children": include_children,
+ "summary": diff.get("summary", {}),
+ "planned_actions": actions,
+ "executed": executed,
+ }
+
+
+async def create_admin_plan(
+ chat_id: str,
+ intent: str,
+ scope: str,
+ actions: List[Dict[str, Any]],
+ kb_id: str = "",
+ kb_name: str = "",
+ summary: Optional[Dict[str, Any]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+ return await db.create_admin_plan(
+ plan_id=str(uuid.uuid4()),
+ chat_id=chat_id,
+ intent=intent,
+ scope=scope,
+ kb_id=kb_id,
+ kb_name=kb_name,
+ summary=summary or {},
+ actions=actions,
+ metadata=metadata or {},
+ )
+
+
+async def get_pending_admin_plan(chat_id: str) -> Optional[Dict[str, Any]]:
+ return await db.get_latest_pending_admin_plan(chat_id)
+
+
+async def cancel_admin_plan(plan_id: str) -> Optional[Dict[str, Any]]:
+ return await db.update_admin_plan_status(plan_id, "cancelled")
+
+
+async def execute_admin_plan(
+ plan: Dict[str, Any],
+ selected_indexes: Optional[List[int]] = None,
+ use_llm: bool = False,
+) -> Dict[str, Any]:
+ actions = plan.get("actions") or []
+ if selected_indexes:
+ selected = {idx for idx in selected_indexes if idx >= 1}
+ actions = [action for idx, action in enumerate(actions, start=1) if idx in selected]
+ executed: List[Dict[str, Any]] = []
+ client = HTKnowClient()
+ for action in actions:
+ try:
+ action_type = action.get("action")
+ if action_type in {"sync_file", "rebuild_file"}:
+ job_id = await start_sync_job(
+ target_type="file",
+ target_id=str(action.get("file_id") or ""),
+ kb_id=str(action.get("kb_id") or ""),
+ use_llm=bool(action.get("use_llm") or use_llm),
+ force=action_type == "rebuild_file",
+ )
+ job = await db.get_job(job_id)
+ result = {"success": True, "status": "running", "job_id": job_id, "job": job}
+ elif action_type == "delete_wiki_file":
+ result = await delete_by_file(str(action.get("file_id") or ""))
+ elif action_type == "delete_wiki_kb":
+ result = await delete_by_kb(str(action.get("kb_id") or plan.get("kb_id") or ""))
+ else:
+ result = {"success": False, "error": f"unknown action: {action_type}"}
+ executed.append({"action": action, "result": result})
+ except Exception as exc:
+ executed.append({"action": action, "result": {"success": False, "error": str(exc)}})
+ await asyncio.sleep(0)
+
+ failed = [item for item in executed if not item.get("result", {}).get("success")]
+ status = "failed" if failed else "executed"
+ await db.update_admin_plan_status(str(plan.get("id") or ""), status, executed=executed)
+ return {
+ "success": not failed,
+ "plan_id": plan.get("id", ""),
+ "status": status,
+ "executed_count": len(executed),
+ "failed_count": len(failed),
+ "executed": executed,
+ }
+
+
+async def delete_by_file(file_id: str) -> Dict[str, Any]:
+ counts = await db.delete_wiki_by_file(file_id)
+ return {"success": True, "scope": "file", "file_id": str(file_id), "deleted": counts}
+
+
+async def delete_by_kb(kb_id: str) -> Dict[str, Any]:
+ counts = await db.delete_wiki_by_kb(kb_id)
+ return {"success": True, "scope": "kb", "kb_id": str(kb_id), "deleted": counts}
+
+
+async def delete_by_time(
+ before: Optional[datetime] = None,
+ after: Optional[datetime] = None,
+ kb_id: Optional[str] = None,
+) -> Dict[str, Any]:
+ counts = await db.delete_wiki_by_time(before=before, after=after, kb_id=kb_id)
+ return {
+ "success": True,
+ "scope": "time",
+ "kb_id": str(kb_id or ""),
+ "before": before.isoformat() if before else "",
+ "after": after.isoformat() if after else "",
+ "deleted": counts,
+ }
+
+
+async def _resolve_kb_ids(client: HTKnowClient, kb_id: str, include_children: bool = False) -> List[str]:
+ root = str(kb_id)
+ if not include_children:
+ return [root]
+ tree = await client.get_kb_tree()
+ found = _find_kb_node(tree, root)
+ if not found:
+ return [root]
+ ids: List[str] = []
+ _collect_kb_ids(found, ids)
+ return ids or [root]
+
+
+def _find_kb_node(nodes: List[Dict[str, Any]], kb_id: str) -> Optional[Dict[str, Any]]:
+ for node in nodes:
+ if str(node.get("id")) == str(kb_id):
+ return node
+ children = node.get("children") or []
+ if isinstance(children, list):
+ found = _find_kb_node(children, kb_id)
+ if found:
+ return found
+ return None
+
+
+def _collect_kb_ids(node: Dict[str, Any], out: List[str]) -> None:
+ kid = node.get("id")
+ if kid is not None:
+ out.append(str(kid))
+ children = node.get("children") or []
+ if isinstance(children, list):
+ for child in children:
+ if isinstance(child, dict):
+ _collect_kb_ids(child, out)
+
+
+def _flatten_kb_tree(nodes: List[Dict[str, Any]], parent_path: str = "") -> List[Dict[str, Any]]:
+ flattened: List[Dict[str, Any]] = []
+ for node in nodes:
+ if not isinstance(node, dict):
+ continue
+ item = dict(node)
+ names = _node_names(item)
+ display_name = names[0] if names else str(item.get("id") or "")
+ item["_path"] = f"{parent_path}/{display_name}" if parent_path else display_name
+ flattened.append(item)
+ children = item.get("children") or []
+ if isinstance(children, list):
+ flattened.extend(_flatten_kb_tree(children, item["_path"]))
+ return flattened
+
+
+def _node_names(node: Dict[str, Any]) -> List[str]:
+ keys = (
+ "name",
+ "title",
+ "label",
+ "kb_name",
+ "knowledge_base_name",
+ "knowledgeBaseName",
+ )
+ names: List[str] = []
+ for key in keys:
+ value = node.get(key)
+ if value is not None:
+ text = str(value).strip()
+ if text and text not in names:
+ names.append(text)
+ return names
+
+
+def _normalize_kb_text(text: str) -> str:
+ return re.sub(r"[\s:：,，.。;；\"'“”‘’《》<>（）()【】\\[\\]_-]+", "", str(text or "").lower())
+
+
+def _compact_file(file_info: Dict[str, Any]) -> Dict[str, Any]:
+ return {
+ "file_id": str(file_info.get("file_id") or file_info.get("id") or ""),
+ "kb_id": str(file_info.get("kb_id") or ""),
+ "filename": str(file_info.get("filename") or file_info.get("name") or ""),
+ "hash": str(file_info.get("hash") or ""),
+ "status": str(file_info.get("status") or ""),
+ "updated_at": str(file_info.get("updated_at") or ""),
+ }
+
+
+def _is_stale(file_info: Dict[str, Any], state: Dict[str, Any]) -> bool:
+ source_hash = str(file_info.get("hash") or "")
+ wiki_hash = str(state.get("file_hash") or "")
+ if source_hash and wiki_hash and source_hash != wiki_hash:
+ return True
+ source_updated = str(file_info.get("updated_at") or "")
+ wiki_source_updated = str(state.get("source_updated_at") or "")
+ if source_updated and wiki_source_updated and source_updated != wiki_source_updated:
+ return True
+ return False
diff --git a/wiki_engine/wiki_builder.py b/wiki_engine/wiki_builder.py
new file mode 100644
index 0000000..24a93e6
--- /dev/null
+++ b/wiki_engine/wiki_builder.py
@@ -0,0 +1,1317 @@
+import asyncio
+import hashlib
+import json
+import re
+from collections import Counter, defaultdict
+from typing import Any, Awaitable, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
+
+from config import WIKI_CONFIG
+from wiki_engine import db
+
+
+MAX_CITATION_BATCH_CHARS = 12000
+DEFAULT_CITATION_CONCURRENCY = 3
+DEFAULT_REDUCE_CONCURRENCY = 3
+DEFAULT_LLM_CONCURRENCY = 4
+
+
+_LLM_SEMAPHORE: Optional[asyncio.Semaphore] = None
+_LLM_SEMAPHORE_LIMIT = 0
+
+
+WIKI_GRANULARITY_GUIDANCE = {
+ "focused": """**FOCUSED mode - aggressive pruning.**
+Extract ONLY the document's primary subjects: the handful of entities/concepts that this document is fundamentally ABOUT.
+
+INCLUDE:
+- The document's main subject(s).
+- At most 3-7 items total across entities and concepts combined.
+
+EXCLUDE:
+- Technology stacks, libraries, frameworks, and infrastructure terms mentioned in passing.
+- Generic concepts and methodologies that are only implementation details.
+- Anything that would normally get only a one-sentence description because there is not enough content.
+
+If you are unsure whether an item belongs, leave it out.""",
+ "standard": """**STANDARD mode - balanced.**
+Extract the document's main subjects PLUS entities/concepts that are substantively discussed, meaning they have a dedicated paragraph, multiple bullet points, or at least 2-3 sentences of context.
+
+INCLUDE:
+- The document's main subject(s).
+- Secondary entities/concepts that receive a concrete block of content.
+- Named methodologies, architectures, procedures, standards, systems, data structures, or techniques when the document explains how they work or how they are used.
+
+EXCLUDE:
+- One-off mentions, parenthetical references, and generic infrastructure nouns.
+- Items whose entire contribution to the document would fit in a single short sentence.
+
+Aim for a tight, curated index. When in doubt about a marginal item, prefer to exclude it.""",
+ "exhaustive": """**EXHAUSTIVE mode - maximum recall.**
+Extract every named entity and every recognizable concept, including technologies, tools, standards, methodologies, procedures, faults, equipment, database tables, fields, and domain terms, provided they are concrete and supported by the document.
+
+INCLUDE:
+- All main and secondary subjects.
+- All named technologies, systems, devices, components, databases, tables, fields, services, protocols, standards, procedures, and fault phenomena.
+- Recognizable concepts that are useful as a future question target.
+
+EXCLUDE ONLY:
+- Truly generic terms with no domain value.
+- Items that appear only inside URL paths or incidental references.
+
+Use this mode when the knowledge base functions as a technical glossary rather than a curated narrative wiki.""",
+}
+
+
+WIKI_CANDIDATE_SLUG_PROMPT = """You are a knowledge extraction system. Analyze the following document and list all significant entities AND key concepts as a lightweight candidate set. Another pass will later attach concrete supporting chunks to each item, so you do NOT need to write exhaustive per-item facts here.
+
+
+
+{content}
+
+
+
+
+{previous_slugs}
+
+
+
+Return a JSON object with two arrays: "entities" and "concepts".
+IMPORTANT: Write ALL names, descriptions, and details in {language}.
+
+If the block above is empty, contains only image references with no extracted text, or otherwise carries no substantive information, return {{"entities": [], "concepts": []}}. Do NOT invent entities or concepts from any other source.
+
+### Extraction Scope (Granularity: {granularity})
+{granularity_guidance}
+
+### Slug Continuity Rules
+If previous slugs are provided above, you MUST follow these rules:
+- If an entity or concept from the previous extraction still exists in the current document, reuse its exact slug from the previous list. Do NOT generate a new slug for the same thing.
+- If an entity or concept no longer appears in the document, do NOT include it in the output.
+- Only generate new slugs for entities/concepts that are genuinely new.
+- This ensures slug stability across document updates.
+
+### Entities
+Each entity should have:
+- "name": The entity name in {language}.
+- "slug": URL-friendly slug, format "entity/"; use romanized/pinyin form for non-Latin names when possible; reuse previous slug if applicable.
+- "aliases": names that refer to THE EXACT SAME entity only. Do NOT include parent categories, related products, generic terms, or broader concepts.
+- "description": one self-contained sentence, 15-40 words, describing what this entity is and its role in the document.
+- "details": a short 1-3 sentence fallback summary under 300 characters.
+
+Never promote trivially-mentioned names into entities.
+
+### Concepts
+Each concept should have:
+- "name": The concept name in {language}.
+- "slug": URL-friendly slug, format "concept/"; use romanized/pinyin form for non-Latin names when possible; reuse previous slug if applicable.
+- "aliases": exact synonyms, official abbreviations, or interchangeable names only. Do NOT include sub-topics, related techniques, broader categories, or implementation details.
+- "description": one self-contained sentence, 15-40 words, defining what this concept is.
+- "details": a short 1-3 sentence fallback summary under 300 characters.
+
+Skip concepts that are merely name-dropped without discussion.
+
+### Deduplication Rules
+- Specific named things belong only in "entities".
+- Abstract ideas, methodologies, procedures, standards, faults, data structures, and topics belong only in "concepts".
+- Never duplicate items across the two arrays.
+
+### JSON Formatting Rules
+- Do NOT use literal newline characters inside JSON string values. If needed, use escaped "\\n".
+
+
+Output ONLY valid JSON."""
+
+
+WIKI_CHUNK_CITATION_PROMPT = """You are a precise citation system. Your job is to scan a batch of document chunks and decide, for each candidate entity/concept below, which chunks substantively discuss it.
+
+
+IMPORTANT: Write ALL names, descriptions, and details in {language}.
+
+### Primary task
+For each candidate slug listed in , select the chunk IDs from that substantively discuss that entity/concept. "Substantively" means the chunk states at least one concrete fact, attribute, step, date, number, relationship, field, constraint, example, cause, effect, or other useful piece of information about the candidate - not a passing mention.
+
+- Only cite chunks that appear in the block below.
+- Use the "id" attribute of each element verbatim, for example "c003".
+- If a candidate is not meaningfully discussed in ANY chunk in this batch, omit it from the output.
+- A chunk CAN be cited by multiple candidates if it genuinely discusses multiple of them.
+- If a chunk is overly long or mixes unrelated topics, still cite it for every candidate it discusses.
+
+### Secondary task: new slugs
+If this batch reveals a significant entity/concept that is NOT in , you may add it under "new_slugs" so it gets incorporated. Only add genuinely new, substantively-discussed items. Do NOT rediscover items already listed in .
+
+Each new slug must include:
+- "type": "entity" or "concept"
+- "name", "slug", "aliases", "description", "details"
+- "source_chunks": list of chunk IDs in the current batch that discuss it
+
+### JSON Formatting Rules
+- Do NOT use literal newline characters inside JSON string values. If needed, use escaped "\\n".
+- Output ONLY valid JSON, no preamble.
+
+
+Output format:
+{{
+ "citations": {{
+ "entity/xxx": ["c001", "c003"],
+ "concept/yyy": ["c002"]
+ }},
+ "new_slugs": [
+ {{
+ "type": "entity",
+ "name": "Example",
+ "slug": "entity/example",
+ "aliases": [],
+ "description": "...",
+ "details": "...",
+ "source_chunks": ["c005"]
+ }}
+ ]
+}}
+
+If nothing in this batch is cite-worthy, return: {{"citations": {{}}, "new_slugs": []}}
+
+
+{candidate_slugs}
+
+
+
+{chunks_xml}
+
+
+Now apply the instructions above to the chunks and output ONLY the JSON."""
+
+
+WIKI_PAGE_MODIFY_PROMPT = """You are a wiki editor tasked with updating an existing wiki page. You must process a set of NEW information to add.
+
+### SOURCE GROUNDING & MERGE RULES (CRITICAL)
+1. No Inline Chunk IDs: Chunk aliases such as [c003] are internal processing metadata. NEVER output them in the page body or summary. Source associations are stored separately by the system.
+2. Mandatory Grounding: Every newly added factual claim, entity, numerical value, field, step, cause, condition, or relationship MUST be directly supported by the provided new source chunks.
+3. No Hallucination: Do not invent, synthesize, or infer information that is not explicitly present in the provided source chunks.
+
+
+ {page_slug}
+ {page_title}
+ {page_type}
+ {page_aliases}
+
+
+This wiki page is specifically about **{page_title}**. Every statement on the page MUST be directly about this exact page topic, not adjacent or similarly named topics.
+
+
+{existing_content}
+
+
+
+{new_content}
+
+
+The block above is assembled from VERBATIM source chunks that were already cited as directly supporting this page. Stay close to the source wording. You may lightly reorder, deduplicate, and join related sentences, but do NOT expand short statements into longer unsourced explanations.
+
+
+{available_slugs}
+
+
+
+1. The FIRST line of your output MUST be: SUMMARY: {{one sentence, 15-40 words, describing what this page is about after the update for wiki index listing}}
+2. ADD and MERGE the facts from into the page.
+3. First verify that the new information is actually about {page_title}. If a piece belongs to a different but related thing, reject that part.
+4. Prefer newer directly supported information if it contradicts old content.
+5. Do NOT over-structure. Only introduce section headings if the source itself uses that heading OR the page already has one. For a new page, a single "# {page_title}" heading plus concise paragraphs and flat bullet facts is preferred.
+6. Do NOT add rhetorical filler such as "aims to", "plays an important role", or "is designed to" unless the source literally says it.
+7. Keep [[slug|name]] wiki-link references ONLY if the slug appears in . Remove invalid wiki links. Do NOT invent new wiki-link slugs.
+8. Write in {language}.
+
+
+Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble."""
+
+
+WIKI_SUMMARY_PROMPT = """You are a wiki editor. Given the following document content, create a structured wiki summary page in Markdown format.
+
+
+
+{content}
+
+
+
+
+{extracted_slugs}
+
+
+
+1. The FIRST line of your output MUST be: SUMMARY: {{one sentence, 15-40 words, describing what this document is about for wiki index listing}}
+2. Create a concise but useful document-level summary.
+3. Include the document's main subject, scope, important procedures, standards, systems, equipment, tables, fields, faults, or operational points when present.
+4. Use wiki links only from , in the form [[slug|name]].
+5. Do NOT invent facts. Stay grounded in the document content.
+6. Write in {language}.
+7. If the content is empty or has no substantive information, output exactly: "SUMMARY: No textual content was extractable from this document." followed by a brief note.
+
+
+Output the SUMMARY line first, then the Markdown content. Do not include any other preamble."""
+
+
+async def build_pages_for_file(
+ file_info: Dict[str, Any],
+ slice_refs: List[Dict[str, Any]],
+ use_llm: bool = False,
+ progress: Optional[Callable[[str], Awaitable[None]]] = None,
+) -> List[Dict[str, Any]]:
+ if not slice_refs:
+ return []
+
+ if WIKI_CONFIG.get("llm_summary_enabled", True):
+ try:
+ pages = await _build_pages_with_weknora_pipeline(file_info, slice_refs, progress=progress)
+ if pages:
+ return pages
+ except Exception as exc:
+ print(f"[wiki_engine] WeKnora-style wiki build failed, fallback to rule pages: {exc}")
+
+ return _build_pages_by_rule(file_info, slice_refs)
+
+
+async def _build_pages_with_weknora_pipeline(
+ file_info: Dict[str, Any],
+ slice_refs: List[Dict[str, Any]],
+ progress: Optional[Callable[[str], Awaitable[None]]] = None,
+) -> List[Dict[str, Any]]:
+ filename = str(file_info.get("filename") or file_info.get("name") or "untitled")
+ kb_id = str(file_info.get("kb_id") or "")
+ file_id = str(file_info.get("file_id") or file_info.get("id") or "")
+ language = str(WIKI_CONFIG.get("language") or "Chinese")
+ granularity = _normalize_granularity(WIKI_CONFIG.get("extraction_granularity") or "standard")
+
+ await _emit_progress(progress, "load_existing_pages")
+ try:
+ existing_pages = await db.list_wiki_pages_for_kb(kb_id)
+ except Exception as exc:
+ print(f"[wiki_engine] load existing wiki pages failed: {exc}")
+ existing_pages = []
+ existing_by_slug = {str(p.get("slug") or ""): p for p in existing_pages if p.get("slug")}
+ previous_slugs = _render_previous_slugs(existing_pages)
+ document_content = _reconstruct_document_content(slice_refs)
+
+ await _emit_progress(progress, "extract_candidates")
+ extracted_entities, extracted_concepts = await _extract_candidate_slugs(
+ content=document_content,
+ previous_slugs=previous_slugs,
+ language=language,
+ granularity=granularity,
+ )
+ extracted_entities, extracted_concepts = _deduplicate_items(extracted_entities, extracted_concepts)
+
+ candidates_xml = _render_candidate_slugs(extracted_entities, extracted_concepts)
+ await _emit_progress(progress, "cite_chunks")
+ citations, new_slugs = await _classify_chunk_citations(
+ candidates_xml=candidates_xml,
+ slice_refs=slice_refs,
+ language=language,
+ )
+ extracted_entities, extracted_concepts = _merge_citations_into_items(
+ extracted_entities,
+ extracted_concepts,
+ citations,
+ new_slugs,
+ )
+ supplemental_entities, supplemental_concepts = _extract_structured_maintenance_items(slice_refs)
+ extracted_entities.extend(supplemental_entities)
+ extracted_concepts.extend(supplemental_concepts)
+ extracted_entities, extracted_concepts = _deduplicate_items(extracted_entities, extracted_concepts)
+
+ slug_items = {
+ str(item.get("slug") or ""): item
+ for item in extracted_entities + extracted_concepts
+ if item.get("slug") and item.get("name")
+ }
+ available_slugs = _render_available_slugs(slug_items, existing_pages)
+
+ pages: List[Dict[str, Any]] = []
+ await _emit_progress(progress, "build_summary")
+ summary_page = await _build_document_summary_page(
+ kb_id=kb_id,
+ file_id=file_id,
+ filename=filename,
+ refs=slice_refs,
+ slug_items=slug_items,
+ content=document_content,
+ language=language,
+ )
+ pages.append(summary_page)
+
+ max_pages = max(int(WIKI_CONFIG.get("max_pages_per_file") or 12), 1)
+ items = _rank_items_by_source_coverage(extracted_entities + extracted_concepts, slice_refs)
+ reduce_sem = asyncio.Semaphore(_wiki_reduce_concurrency())
+ reduce_total = min(len(items), max_pages - 1)
+ reduce_done = 0
+ reduce_lock = asyncio.Lock()
+
+ async def build_item_page(item: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+ nonlocal reduce_done
+ async with reduce_sem:
+ page = await _build_concept_page(
+ kb_id=kb_id,
+ file_id=file_id,
+ filename=filename,
+ item=item,
+ all_refs=slice_refs,
+ existing_page=existing_by_slug.get(str(item.get("slug") or "")),
+ available_slugs=available_slugs,
+ language=language,
+ )
+ async with reduce_lock:
+ reduce_done += 1
+ await _emit_progress(progress, f"reduce_pages {reduce_done}/{reduce_total}")
+ return page
+
+ await _emit_progress(progress, f"reduce_pages 0/{reduce_total}")
+ built_pages = await asyncio.gather(*(build_item_page(item) for item in items[: max_pages - 1]))
+ for page in built_pages:
+ if page:
+ pages.append(page)
+
+ return pages
+
+
+async def _extract_candidate_slugs(
+ content: str,
+ previous_slugs: str,
+ language: str,
+ granularity: str,
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+ prompt = WIKI_CANDIDATE_SLUG_PROMPT.format(
+ content=content,
+ previous_slugs=previous_slugs or "(none - this is a new document)",
+ language=language,
+ granularity=granularity,
+ granularity_guidance=WIKI_GRANULARITY_GUIDANCE[granularity],
+ )
+ data = await _llm_json(
+ prompt,
+ system_prompt="You are a strict knowledge extraction system. Output valid JSON only.",
+ temperature=0.1,
+ )
+ entities = [x for x in data.get("entities", []) if isinstance(x, dict)]
+ concepts = [x for x in data.get("concepts", []) if isinstance(x, dict)]
+ return _clean_items(entities, "entity"), _clean_items(concepts, "concept")
+
+
+async def _emit_progress(progress: Optional[Callable[[str], Awaitable[None]]], stage: str) -> None:
+ if not progress:
+ return
+ try:
+ await progress(stage)
+ except Exception as exc:
+ print(f"[wiki_engine] wiki progress update failed: {exc}")
+
+
+async def _classify_chunk_citations(
+ candidates_xml: str,
+ slice_refs: Sequence[Dict[str, Any]],
+ language: str,
+) -> Tuple[Dict[str, List[str]], List[Dict[str, Any]]]:
+ batches = _split_citation_batches(slice_refs)
+ if not batches or not candidates_xml.strip():
+ return {}, []
+
+ citation_sets: Dict[str, set[str]] = defaultdict(set)
+ new_slugs: List[Dict[str, Any]] = []
+ lock = asyncio.Lock()
+ sem = asyncio.Semaphore(_wiki_citation_concurrency())
+
+ async def run_batch(batch: Dict[str, Any]) -> None:
+ async with sem:
+ prompt = WIKI_CHUNK_CITATION_PROMPT.format(
+ language=language,
+ candidate_slugs=candidates_xml,
+ chunks_xml=batch["chunks_xml"],
+ )
+ try:
+ data = await _llm_json(
+ prompt,
+ system_prompt="You map wiki candidates to source chunks. Output valid JSON only.",
+ temperature=0.0,
+ )
+ except Exception as exc:
+ print(f"[wiki_engine] wiki citation batch failed: {exc}")
+ return
+
+ aliases = batch["alias_to_id"]
+ async with lock:
+ for slug, alias_list in (data.get("citations") or {}).items():
+ if not slug or not isinstance(alias_list, list):
+ continue
+ for alias in alias_list:
+ real_id = aliases.get(str(alias))
+ if real_id:
+ citation_sets[str(slug)].add(real_id)
+ for raw in data.get("new_slugs") or []:
+ if not isinstance(raw, dict):
+ continue
+ item = dict(raw)
+ item["source_chunks"] = [
+ aliases[str(alias)]
+ for alias in item.get("source_chunks") or []
+ if str(alias) in aliases
+ ]
+ if item.get("slug") and item.get("name") and item.get("source_chunks"):
+ new_slugs.append(item)
+
+ await asyncio.gather(*(run_batch(batch) for batch in batches))
+
+ chunk_order = {
+ str(ref.get("slice_id") or ""): int(ref.get("ordinal") or idx)
+ for idx, ref in enumerate(slice_refs)
+ }
+ citations = {
+ slug: sorted(ids, key=lambda sid: chunk_order.get(sid, 999999))
+ for slug, ids in citation_sets.items()
+ }
+ return citations, new_slugs
+
+
+async def _build_document_summary_page(
+ kb_id: str,
+ file_id: str,
+ filename: str,
+ refs: List[Dict[str, Any]],
+ slug_items: Dict[str, Dict[str, Any]],
+ content: str,
+ language: str,
+) -> Dict[str, Any]:
+ slug_listing = "\n".join(
+ f"- [[{slug}]] = {item.get('name') or slug}"
+ for slug, item in sorted(slug_items.items())
+ )
+ prompt = WIKI_SUMMARY_PROMPT.format(
+ content=content,
+ extracted_slugs=slug_listing or "(none)",
+ language=language,
+ )
+ summary, body = "", ""
+ try:
+ raw = await _llm_text(
+ prompt,
+ system_prompt="You are a grounded wiki editor. Do not invent facts.",
+ temperature=0.1,
+ )
+ summary, body = _split_summary_line(raw)
+ except Exception as exc:
+ print(f"[wiki_engine] wiki summary page failed: {exc}")
+
+ keywords = _keywords_from_refs(refs)
+ if not summary:
+ summary = _make_file_summary(filename, refs, keywords)
+ if not body:
+ body = _compose_topic_content(filename, refs[:12])
+
+ links = _links_for_refs(refs[:20], reason="document_summary")
+ return {
+ "kb_id": kb_id,
+ "slug": f"summary/{_slugify(file_id or filename)}",
+ "title": f"{filename} - document summary"[:300],
+ "page_type": "document_summary",
+ "summary": summary,
+ "content": body,
+ "keywords": keywords[:16],
+ "aliases": [filename],
+ "related_file_ids": [file_id] if file_id else [],
+ "source_refs": _source_refs(file_id, filename, refs[:20], reason="document_summary"),
+ "chunk_refs": _chunk_refs(refs[:20], reason="document_summary"),
+ "source_kind": "llm",
+ "score_hint": 1.0,
+ "links": links,
+ }
+
+
+async def _build_concept_page(
+ kb_id: str,
+ file_id: str,
+ filename: str,
+ item: Dict[str, Any],
+ all_refs: List[Dict[str, Any]],
+ existing_page: Optional[Dict[str, Any]],
+ available_slugs: str,
+ language: str,
+) -> Optional[Dict[str, Any]]:
+ slug = str(item.get("slug") or "").strip()
+ title = str(item.get("name") or "").strip()
+ if not slug or not title:
+ return None
+
+ source_chunk_ids = [str(x) for x in item.get("source_chunks") or [] if str(x)]
+ if not source_chunk_ids:
+ return None
+
+ refs_by_id = {str(ref.get("slice_id") or ""): ref for ref in all_refs}
+ refs = [refs_by_id[sid] for sid in source_chunk_ids if sid in refs_by_id]
+ if not refs:
+ return None
+
+ page_type = _page_type_from_slug_or_item(slug, item)
+ if existing_page:
+ title = str(existing_page.get("title") or title)
+ page_type = str(existing_page.get("page_type") or page_type)
+
+ new_content = _format_new_information(filename, refs)
+ source_kind = "llm"
+ if _should_use_structured_page_template(item):
+ summary, body = _build_structured_page_text(title=title, item=item, refs=refs)
+ source_kind = "structured"
+ else:
+ prompt = WIKI_PAGE_MODIFY_PROMPT.format(
+ page_slug=slug,
+ page_title=title,
+ page_type=page_type,
+ page_aliases=", ".join(_clean_list(item.get("aliases"), limit=12)),
+ existing_content=str((existing_page or {}).get("content") or ""),
+ new_content=new_content,
+ available_slugs=available_slugs,
+ language=language,
+ )
+
+ summary, body = "", ""
+ try:
+ raw = await _llm_text(
+ prompt,
+ system_prompt="You are a grounded wiki editor. Use only provided source chunks.",
+ temperature=0.1,
+ )
+ summary, body = _split_summary_line(raw)
+ except Exception as exc:
+ print(f"[wiki_engine] wiki page reduce failed for {slug}: {exc}")
+
+ if not summary:
+ summary = str(item.get("description") or item.get("details") or _make_topic_summary(title, refs)).strip()
+ if not body:
+ body = _compose_topic_content(title, refs)
+
+ old_source_refs = _filter_refs_by_file((existing_page or {}).get("source_refs") or [], file_id)
+ old_chunk_refs = _filter_refs_by_file((existing_page or {}).get("chunk_refs") or [], file_id)
+ old_links = _filter_refs_by_file((existing_page or {}).get("slices") or [], file_id)
+ old_related_files = [
+ str(x)
+ for x in ((existing_page or {}).get("related_file_ids") or [])
+ if str(x) and str(x) != file_id
+ ]
+
+ aliases = _dedupe_strings(
+ _clean_list((existing_page or {}).get("aliases"), limit=24)
+ + _clean_list(item.get("aliases"), limit=24)
+ )[:24]
+ keywords = _filter_page_keywords(
+ [title]
+ + aliases
+ + _clean_list((existing_page or {}).get("keywords"), limit=24)
+ + _clean_list(item.get("keywords"), limit=24)
+ + _merge_keywords(refs, lead=title)
+ )[:24]
+
+ links = old_links + _links_for_refs(refs, reason=f"wiki_citation:{slug}")
+ return {
+ "kb_id": kb_id,
+ "slug": slug[:300],
+ "title": title[:300],
+ "page_type": page_type,
+ "summary": summary,
+ "content": body,
+ "keywords": keywords,
+ "aliases": aliases,
+ "related_file_ids": _dedupe_strings(old_related_files + ([file_id] if file_id else [])),
+ "source_refs": old_source_refs + _source_refs(file_id, filename, refs, reason=f"wiki_citation:{slug}"),
+ "chunk_refs": old_chunk_refs + _chunk_refs(refs, reason=f"wiki_citation:{slug}"),
+ "source_kind": source_kind,
+ "score_hint": 1.4,
+ "links": links,
+ }
+
+
+async def _llm_json(prompt: str, system_prompt: str, temperature: float) -> Dict[str, Any]:
+ raw = await _llm_text(prompt, system_prompt=system_prompt, temperature=temperature, json_output=True)
+ cleaned = _strip_code_fence(raw)
+ try:
+ return json.loads(cleaned)
+ except json.JSONDecodeError:
+ match = re.search(r"\{.*\}", cleaned, re.S)
+ if match:
+ return json.loads(match.group(0))
+ raise
+
+
+async def _llm_text(
+ prompt: str,
+ system_prompt: str,
+ temperature: float,
+ json_output: bool = False,
+) -> str:
+ from modelsAPI.model_api import OpenaiAPI
+
+ cache_key = _llm_cache_key(prompt, system_prompt, temperature, json_output)
+ if WIKI_CONFIG.get("llm_cache_enabled", True):
+ try:
+ cached = await db.get_llm_cache(
+ cache_key,
+ ttl_seconds=int(WIKI_CONFIG.get("llm_cache_ttl_seconds") or 0),
+ )
+ if cached:
+ return cached
+ except Exception as exc:
+ print(f"[wiki_engine] wiki llm cache read failed: {exc}")
+
+ retries = _bounded_int(WIKI_CONFIG.get("llm_retry_count"), 2, 0, 5)
+ last_error: Optional[Exception] = None
+ for attempt in range(retries + 1):
+ try:
+ async with _wiki_llm_semaphore():
+ response = await OpenaiAPI.open_api_chat_without_thinking(
+ query=prompt,
+ json_output=json_output,
+ system_prompt=system_prompt,
+ temperature=temperature,
+ )
+ if WIKI_CONFIG.get("llm_cache_enabled", True) and response:
+ try:
+ await db.set_llm_cache(cache_key, response)
+ except Exception as exc:
+ print(f"[wiki_engine] wiki llm cache write failed: {exc}")
+ return response
+ except Exception as exc:
+ last_error = exc
+ if attempt >= retries:
+ break
+ await asyncio.sleep(min(5.0, 0.5 * (2 ** attempt)))
+ raise last_error or RuntimeError("wiki llm call failed")
+
+
+def _reconstruct_document_content(slice_refs: Sequence[Dict[str, Any]]) -> str:
+ max_chars = int(WIKI_CONFIG.get("full_content_max_chars") or 120000)
+ parts: List[str] = []
+ total = 0
+ ordered = sorted(slice_refs, key=lambda r: int(r.get("ordinal") or 0))
+ for ref in ordered:
+ text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "").strip()
+ if not text:
+ continue
+ title_path = " / ".join(str(x) for x in (ref.get("title_path") or []) if x)
+ prefix = f"[chunk {ref.get('slice_id') or ''}]"
+ if title_path:
+ prefix += f" {title_path}"
+ block = f"{prefix}\n{text}"
+ if total + len(block) > max_chars:
+ remain = max_chars - total
+ if remain > 500:
+ parts.append(block[:remain])
+ break
+ parts.append(block)
+ total += len(block)
+ return "\n\n".join(parts)
+
+
+def _split_citation_batches(slice_refs: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ ordered = [
+ ref for ref in sorted(slice_refs, key=lambda r: int(r.get("ordinal") or 0))
+ if str(ref.get("content") or ref.get("_text") or ref.get("brief") or "").strip()
+ ]
+ batches: List[List[Dict[str, Any]]] = []
+ current: List[Dict[str, Any]] = []
+ current_len = 0
+ for ref in ordered:
+ text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "")
+ length = len(text)
+ if current and current_len + length > MAX_CITATION_BATCH_CHARS:
+ batches.append(current)
+ current = []
+ current_len = 0
+ current.append(ref)
+ current_len += length
+ if current:
+ batches.append(current)
+
+ rendered: List[Dict[str, Any]] = []
+ for refs in batches:
+ alias_to_id: Dict[str, str] = {}
+ chunks: List[str] = []
+ for idx, ref in enumerate(refs):
+ alias = f"c{idx:03d}"
+ slice_id = str(ref.get("slice_id") or "")
+ alias_to_id[alias] = slice_id
+ text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "")
+ chunks.append(f'\n{text}\n')
+ rendered.append({"alias_to_id": alias_to_id, "chunks_xml": "\n".join(chunks)})
+ return rendered
+
+
+def _render_candidate_slugs(entities: Sequence[Dict[str, Any]], concepts: Sequence[Dict[str, Any]]) -> str:
+ lines: List[str] = []
+ for item in entities:
+ lines.append(_candidate_line(item, "entity"))
+ for item in concepts:
+ lines.append(_candidate_line(item, "concept"))
+ return "\n".join(line for line in lines if line)
+
+
+def _candidate_line(item: Dict[str, Any], kind: str) -> str:
+ slug = str(item.get("slug") or "").strip()
+ name = str(item.get("name") or "").strip()
+ if not slug or not name:
+ return ""
+ aliases = ", ".join(_clean_list(item.get("aliases"), limit=8))
+ alias_text = f', aliases="{aliases}"' if aliases else ""
+ description = str(item.get("description") or item.get("details") or "").strip()
+ return f'- slug: {slug}, type: {kind}, name: "{name}"{alias_text}, description: {description}'
+
+
+def _merge_citations_into_items(
+ entities: List[Dict[str, Any]],
+ concepts: List[Dict[str, Any]],
+ citations: Dict[str, List[str]],
+ new_slugs: List[Dict[str, Any]],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+ for item in entities + concepts:
+ item["source_chunks"] = citations.get(str(item.get("slug") or ""), [])
+
+ existing = {str(item.get("slug") or "") for item in entities + concepts if item.get("slug")}
+ merged_new: Dict[str, Dict[str, Any]] = {}
+ for raw in new_slugs:
+ item = _clean_item(raw, str(raw.get("type") or "concept"))
+ slug = str(item.get("slug") or "")
+ if not slug or slug in existing:
+ continue
+ bucket = merged_new.setdefault(slug, item)
+ bucket["source_chunks"] = _dedupe_strings(
+ _clean_list(bucket.get("source_chunks"), limit=100)
+ + _clean_list(item.get("source_chunks"), limit=100)
+ )
+
+ for item in merged_new.values():
+ if str(item.get("slug") or "").startswith("entity/"):
+ entities.append(item)
+ else:
+ concepts.append(item)
+ return entities, concepts
+
+
+def _should_use_structured_page_template(item: Dict[str, Any]) -> bool:
+ if not WIKI_CONFIG.get("structured_template_enabled", True):
+ return False
+ return int(item.get("structured_priority") or 0) >= 80
+
+
+def _build_structured_page_text(
+ title: str,
+ item: Dict[str, Any],
+ refs: Sequence[Dict[str, Any]],
+) -> Tuple[str, str]:
+ summary = str(item.get("description") or item.get("details") or "").strip()
+ if not summary:
+ summary = _make_topic_summary(title, list(refs))
+ details = str(item.get("details") or item.get("description") or "").strip()
+ keywords = _clean_list(item.get("keywords"), limit=12)
+ aliases = _clean_list(item.get("aliases"), limit=12)
+
+ lines = [f"# {title}", ""]
+ if details:
+ lines.extend([details, ""])
+ facts = _dedupe_strings([x for x in aliases + keywords if x and x != title])
+ if facts:
+ lines.append("## 关键条目")
+ for fact in facts[:10]:
+ lines.append(f"- {fact}")
+ lines.append("")
+ lines.append("## 来源摘录")
+ for ref in refs[:8]:
+ excerpt = _short_text(ref, 420)
+ if excerpt:
+ lines.append(f"- {excerpt}")
+ return summary, "\n".join(lines).strip()
+
+
+def _extract_structured_maintenance_items(
+ slice_refs: Sequence[Dict[str, Any]],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+ entities: List[Dict[str, Any]] = []
+ concepts: List[Dict[str, Any]] = []
+ seen_codes: set[str] = set()
+ row_pattern = re.compile(
+ r"| (MT[0-9O]{3}) | ([^<]{1,40}) | ([^<]{1,80}) | "
+ r"([^<]{1,120}) | ([^<]{1,40}) | ([^<]{1,40}) | ",
+ re.I,
+ )
+ for ref in slice_refs:
+ slice_id = str(ref.get("slice_id") or "")
+ if not slice_id:
+ continue
+ text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "")
+ if "维修项目" not in text or not re.search(r"MT[0-9O]{3}", text, re.I):
+ continue
+ for match in row_pattern.finditer(text):
+ code = _normalize_mt_code(match.group(1))
+ if not code or code in seen_codes:
+ continue
+ seen_codes.add(code)
+ component = _clean_maintenance_cell(match.group(3))
+ action = _clean_maintenance_cell(match.group(4))
+ interval = _clean_maintenance_cell(match.group(5))
+ level = _clean_maintenance_cell(match.group(6))
+ if not component or not action:
+ continue
+ title = component if component in action else f"{code}维修项目"
+ aliases = _dedupe_strings([code, action, f"{code}{action}", f"{component}{action}"])[:8]
+ description = "，".join(
+ part for part in [
+ f"{title}对应{code}维修项目：{action}",
+ f"维修间隔期为{interval}" if interval else "",
+ f"维修级别为{level}" if level else "",
+ ] if part
+ ) + "。"
+ entities.append({
+ "name": title,
+ "slug": f"entity/{_slugify(title)}",
+ "aliases": aliases,
+ "description": description,
+ "details": description,
+ "keywords": _dedupe_strings([component, action, code, interval, level])[:8],
+ "source_chunks": [slice_id],
+ "structured_priority": 100,
+ })
+ return entities, concepts
+
+
+def _normalize_mt_code(value: str) -> str:
+ code = re.sub(r"[^A-Za-z0-9]", "", str(value or "").upper()).replace("O", "0")
+ if re.fullmatch(r"MT\d{3}", code):
+ return code
+ return ""
+
+
+def _clean_maintenance_cell(value: str) -> str:
+ text = re.sub(r"<[^>]+>", "", str(value or ""))
+ text = re.sub(r"\s+", "", text)
+ return text.strip("：:，,。.;；、")[:80]
+
+
+def _deduplicate_items(
+ entities: List[Dict[str, Any]],
+ concepts: List[Dict[str, Any]],
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+ def dedupe(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ by_key: Dict[str, Dict[str, Any]] = {}
+ order: List[str] = []
+ out = []
+ for item in items:
+ slug = str(item.get("slug") or "").strip()
+ name = str(item.get("name") or "").strip()
+ key = slug or name
+ if not key:
+ continue
+ if key not in by_key:
+ by_key[key] = dict(item)
+ order.append(key)
+ continue
+ existing = by_key[key]
+ for field, limit in (
+ ("aliases", 24),
+ ("keywords", 24),
+ ("source_chunks", 200),
+ ):
+ existing[field] = _dedupe_strings(
+ _clean_list(existing.get(field), limit=limit)
+ + _clean_list(item.get(field), limit=limit)
+ )[:limit]
+ for field in ("description", "details"):
+ if not str(existing.get(field) or "").strip() and str(item.get(field) or "").strip():
+ existing[field] = item.get(field)
+ existing["structured_priority"] = max(
+ int(existing.get("structured_priority") or 0),
+ int(item.get("structured_priority") or 0),
+ )
+ for key in order:
+ out.append(by_key[key])
+ return out
+
+ return dedupe(entities), dedupe(concepts)
+
+
+def _rank_items_by_source_coverage(items: Sequence[Dict[str, Any]], refs: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ order = {str(ref.get("slice_id") or ""): idx for idx, ref in enumerate(refs)}
+
+ def key(item: Dict[str, Any]) -> Tuple[int, int, int, str]:
+ chunks = _clean_list(item.get("source_chunks"), limit=500)
+ first = min((order.get(sid, 999999) for sid in chunks), default=999999)
+ priority = int(item.get("structured_priority") or 0)
+ return (-priority, -len(chunks), first, str(item.get("name") or ""))
+
+ return sorted(items, key=key)
+
+
+def _render_previous_slugs(pages: Sequence[Dict[str, Any]]) -> str:
+ lines = []
+ for page in pages:
+ slug = str(page.get("slug") or "")
+ if slug.startswith(("entity/", "concept/")):
+ lines.append(f"- {slug}")
+ return "\n".join(sorted(set(lines))) or "(none - this is a new document)"
+
+
+def _render_available_slugs(
+ slug_items: Dict[str, Dict[str, Any]],
+ existing_pages: Sequence[Dict[str, Any]],
+) -> str:
+ rows: Dict[str, str] = {}
+ for page in existing_pages:
+ slug = str(page.get("slug") or "")
+ title = str(page.get("title") or "")
+ if slug:
+ rows[slug] = title or slug
+ for slug, item in slug_items.items():
+ rows[slug] = str(item.get("name") or rows.get(slug) or slug)
+ return "\n".join(f"- [[{slug}]] = {title}" for slug, title in sorted(rows.items()))
+
+
+def _format_new_information(filename: str, refs: Sequence[Dict[str, Any]]) -> str:
+ blocks = [f'']
+ for idx, ref in enumerate(refs):
+ alias = f"c{idx:03d}"
+ text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "").strip()
+ title_path = " / ".join(str(x) for x in (ref.get("title_path") or []) if x)
+ if title_path:
+ blocks.append(f'{title_path}')
+ blocks.append(f'\n{text}\n')
+ blocks.append("")
+ return "\n".join(blocks)
+
+
+def _filter_refs_by_file(items: Sequence[Any], file_id: str) -> List[Dict[str, Any]]:
+ out: List[Dict[str, Any]] = []
+ for item in items or []:
+ if not isinstance(item, dict):
+ continue
+ if file_id and str(item.get("file_id") or "") == file_id:
+ continue
+ out.append(dict(item))
+ return out
+
+
+def _clean_items(items: Sequence[Dict[str, Any]], kind: str) -> List[Dict[str, Any]]:
+ return [_clean_item(item, kind) for item in items if _clean_item(item, kind).get("name")]
+
+
+def _clean_item(item: Dict[str, Any], kind: str) -> Dict[str, Any]:
+ name = str(item.get("name") or item.get("title") or "").strip()
+ slug = str(item.get("slug") or "").strip()
+ if name and not slug:
+ slug = f"{kind}/{_slugify(name)}"
+ if slug and "/" not in slug:
+ slug = f"{kind}/{_slugify(slug)}"
+ if kind == "entity" and slug and not slug.startswith("entity/"):
+ slug = f"entity/{slug.split('/', 1)[-1]}"
+ if kind != "entity" and slug and not slug.startswith("concept/"):
+ slug = f"concept/{slug.split('/', 1)[-1]}"
+ return {
+ "name": name,
+ "slug": slug[:300],
+ "aliases": _clean_list(item.get("aliases"), limit=16),
+ "description": str(item.get("description") or item.get("summary") or "").strip(),
+ "details": str(item.get("details") or item.get("content") or "").strip(),
+ "keywords": _clean_list(item.get("keywords"), limit=16),
+ "source_chunks": _clean_list(item.get("source_chunks"), limit=100),
+ }
+
+
+def _split_summary_line(text: str) -> Tuple[str, str]:
+ lines = (text or "").strip().splitlines()
+ if not lines:
+ return "", ""
+ first = lines[0].strip()
+ if first.upper().startswith("SUMMARY:"):
+ return first.split(":", 1)[1].strip(), "\n".join(lines[1:]).strip()
+ return "", text.strip()
+
+
+def _normalize_granularity(value: Any) -> str:
+ text = str(value or "").strip().lower()
+ return text if text in WIKI_GRANULARITY_GUIDANCE else "standard"
+
+
+def _wiki_citation_concurrency() -> int:
+ return _bounded_int(WIKI_CONFIG.get("citation_concurrency"), DEFAULT_CITATION_CONCURRENCY, 1, 4)
+
+
+def _wiki_reduce_concurrency() -> int:
+ return _bounded_int(WIKI_CONFIG.get("reduce_concurrency"), DEFAULT_REDUCE_CONCURRENCY, 1, 4)
+
+
+def _wiki_llm_semaphore() -> asyncio.Semaphore:
+ global _LLM_SEMAPHORE, _LLM_SEMAPHORE_LIMIT
+ limit = _bounded_int(WIKI_CONFIG.get("llm_concurrency"), DEFAULT_LLM_CONCURRENCY, 1, 8)
+ if _LLM_SEMAPHORE is None or _LLM_SEMAPHORE_LIMIT != limit:
+ _LLM_SEMAPHORE = asyncio.Semaphore(limit)
+ _LLM_SEMAPHORE_LIMIT = limit
+ return _LLM_SEMAPHORE
+
+
+def _llm_cache_key(prompt: str, system_prompt: str, temperature: float, json_output: bool) -> str:
+ payload = {
+ "prompt": prompt,
+ "system_prompt": system_prompt,
+ "temperature": float(temperature or 0.0),
+ "json_output": bool(json_output),
+ "version": "wiki-v2",
+ }
+ raw = json.dumps(payload, ensure_ascii=False, sort_keys=True)
+ return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+def _bounded_int(value: Any, default: int, minimum: int, maximum: int) -> int:
+ try:
+ number = int(value)
+ except Exception:
+ number = default
+ return max(minimum, min(maximum, number))
+
+
+def _page_type_from_slug_or_item(slug: str, item: Dict[str, Any]) -> str:
+ if slug.startswith("entity/"):
+ return "entity"
+ return _normalize_page_type(item.get("page_type") or "concept")
+
+
+def _build_pages_by_rule(file_info: Dict[str, Any], slice_refs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ filename = str(file_info.get("filename") or file_info.get("name") or "untitled")
+ kb_id = str(file_info.get("kb_id") or "")
+ file_id = str(file_info.get("file_id") or file_info.get("id") or "")
+ max_pages = int(WIKI_CONFIG.get("max_pages_per_file") or 12)
+
+ topic_counts: Counter[str] = Counter()
+ topic_slices: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
+ for ref in slice_refs:
+ for topic in _topics_for_ref(ref):
+ topic_counts[topic] += 1
+ topic_slices[topic].append(ref)
+
+ all_keywords = [kw for kw, _ in topic_counts.most_common(16)]
+ pages = [_fallback_document_summary_page(kb_id, file_id, filename, slice_refs, all_keywords, source_kind="rule")]
+
+ for topic, _ in topic_counts.most_common(max_pages - 1):
+ refs = topic_slices[topic][:12]
+ if not refs:
+ continue
+ slug = f"concept/{_slugify(topic)}"
+ pages.append({
+ "kb_id": kb_id,
+ "slug": slug,
+ "title": topic[:300],
+ "page_type": "concept",
+ "summary": _make_topic_summary(topic, refs),
+ "content": _compose_topic_content(topic, refs),
+ "keywords": _merge_keywords(refs, lead=topic),
+ "aliases": [],
+ "related_file_ids": [file_id] if file_id else [],
+ "source_refs": _source_refs(file_id, filename, refs, reason="rule_topic"),
+ "chunk_refs": _chunk_refs(refs, reason="rule_topic"),
+ "source_kind": "rule",
+ "score_hint": 0.7,
+ "links": _links_for_refs(refs, reason=f"rule_topic:{topic}"),
+ })
+ return pages[:max_pages]
+
+
+def _fallback_document_summary_page(
+ kb_id: str,
+ file_id: str,
+ filename: str,
+ refs: List[Dict[str, Any]],
+ keywords: List[str],
+ source_kind: str,
+) -> Dict[str, Any]:
+ return {
+ "kb_id": kb_id,
+ "slug": f"summary/{_slugify(file_id or filename)}",
+ "title": f"{filename} - document summary"[:300],
+ "page_type": "document_summary",
+ "summary": _make_file_summary(filename, refs, keywords),
+ "content": _compose_topic_content(filename, refs[:12]),
+ "keywords": keywords[:16],
+ "aliases": [filename],
+ "related_file_ids": [file_id] if file_id else [],
+ "source_refs": _source_refs(file_id, filename, refs[:20], reason="document_summary"),
+ "chunk_refs": _chunk_refs(refs[:20], reason="document_summary"),
+ "source_kind": source_kind,
+ "score_hint": 1.0,
+ "links": _links_for_refs(refs[:20], reason="document_summary"),
+ }
+
+
+def _topics_for_ref(ref: Dict[str, Any]) -> List[str]:
+ topics: List[str] = []
+ for value in ref.get("title_path") or []:
+ text = str(value).strip()
+ if len(text) >= 2:
+ topics.append(text[:80])
+ for value in ref.get("keywords") or []:
+ text = str(value).strip()
+ if len(text) >= 2:
+ topics.append(text[:80])
+ return _dedupe_strings(topics)[:12]
+
+
+def _keywords_from_refs(refs: List[Dict[str, Any]]) -> List[str]:
+ counts: Counter[str] = Counter()
+ for ref in refs:
+ counts.update(str(x) for x in (ref.get("keywords") or [])[:10] if str(x).strip())
+ for title in ref.get("title_path") or []:
+ text = str(title).strip()
+ if len(text) >= 2:
+ counts[text] += 1
+ return [kw for kw, _ in counts.most_common(16)]
+
+
+def _make_file_summary(filename: str, refs: List[Dict[str, Any]], keywords: List[str]) -> str:
+ keyword_text = ", ".join(keywords[:10]) if keywords else "none"
+ return f"The file {filename} has {len(refs)} indexed source chunks. Main topics: {keyword_text}."
+
+
+def _make_topic_summary(topic: str, refs: List[Dict[str, Any]]) -> str:
+ snippets = "; ".join(_short_text(ref, 120) for ref in refs[:3] if _short_text(ref, 120))
+ return f"Topic '{topic}' is supported by {len(refs)} source chunks. Evidence excerpts: {snippets}"
+
+
+def _compose_topic_content(topic: str, refs: List[Dict[str, Any]]) -> str:
+ lines = [f"# {topic}", "", "Source evidence:"]
+ for ref in refs[:8]:
+ excerpt = _short_text(ref, 360)
+ if excerpt:
+ lines.append(f"- {excerpt}")
+ return "\n".join(lines)
+
+
+def _short_text(ref: Dict[str, Any], limit: int) -> str:
+ text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "").strip()
+ return re.sub(r"\s+", " ", text)[:limit]
+
+
+def _normalize_page_type(value: Any) -> str:
+ text = str(value or "concept").strip().lower()
+ allowed = {
+ "document_summary",
+ "entity",
+ "concept",
+ "procedure",
+ "fault",
+ "standard",
+ "operation",
+ "device",
+ "topic",
+ }
+ return text if text in allowed else "concept"
+
+
+def _merge_keywords(refs: List[Dict[str, Any]], lead: str) -> List[str]:
+ counts: Counter[str] = Counter()
+ for ref in refs:
+ counts.update(str(x) for x in (ref.get("keywords") or [])[:10] if str(x).strip())
+ out = [lead] if lead else []
+ for kw, _ in counts.most_common(20):
+ if kw not in out:
+ out.append(kw)
+ return out[:24]
+
+
+def _filter_page_keywords(values: Iterable[Any]) -> List[str]:
+ return [value for value in _dedupe_strings(values) if not _is_noise_keyword(value)][:24]
+
+
+def _is_noise_keyword(value: Any) -> bool:
+ text = str(value or "").strip().lower()
+ if not text:
+ return True
+ if text in {
+ "td", "tr", "th", "thead", "tbody", "table", "colspan", "rowspan",
+ "序号", "名称", "备注", "规格", "分钟",
+ }:
+ return True
+ if re.fullmatch(r"[<>/]+", text):
+ return True
+ return False
+
+
+def _links_for_refs(refs: Iterable[Dict[str, Any]], reason: str) -> List[Dict[str, Any]]:
+ return [
+ {
+ "file_id": str(ref.get("file_id") or ""),
+ "slice_id": str(ref.get("slice_id") or ""),
+ "weight": 1.0,
+ "reason": reason,
+ }
+ for ref in refs
+ if ref.get("slice_id")
+ ]
+
+
+def _source_refs(file_id: str, filename: str, refs: Iterable[Dict[str, Any]], reason: str) -> List[Dict[str, Any]]:
+ return [
+ {
+ "file_id": file_id or str(ref.get("file_id") or ""),
+ "filename": filename or str(ref.get("filename") or ""),
+ "slice_id": str(ref.get("slice_id") or ""),
+ "reason": reason,
+ }
+ for ref in refs
+ if ref.get("slice_id")
+ ]
+
+
+def _chunk_refs(refs: Iterable[Dict[str, Any]], reason: str) -> List[Dict[str, Any]]:
+ return [
+ {
+ "file_id": str(ref.get("file_id") or ""),
+ "slice_id": str(ref.get("slice_id") or ""),
+ "title_path": ref.get("title_path") or [],
+ "keywords": (ref.get("keywords") or [])[:10],
+ "reason": reason,
+ }
+ for ref in refs
+ if ref.get("slice_id")
+ ]
+
+
+def _clean_list(value: Any, limit: int) -> List[str]:
+ if value is None:
+ return []
+ if isinstance(value, str):
+ value = [value]
+ if not isinstance(value, (list, tuple, set)):
+ return []
+ return _dedupe_strings(str(x).strip() for x in value if str(x).strip())[:limit]
+
+
+def _dedupe_strings(values: Iterable[Any]) -> List[str]:
+ out: List[str] = []
+ seen = set()
+ for value in values:
+ text = str(value or "").strip()
+ if not text or text in seen:
+ continue
+ seen.add(text)
+ out.append(text)
+ return out
+
+
+def _slugify(text: str) -> str:
+ raw = str(text or "").strip().lower()
+ slug = re.sub(r"[\s/\\:;,.!?|]+", "-", raw)
+ slug = re.sub(r"[^0-9a-zA-Z_\-\u4e00-\u9fff]+", "", slug).strip("-")
+ if slug:
+ return slug[:160]
+ return hashlib.sha1(raw.encode("utf-8", errors="ignore")).hexdigest()[:20]
+
+
+def _strip_code_fence(text: str) -> str:
+ text = (text or "").strip()
+ if text.startswith("```"):
+ lines = text.splitlines()
+ if lines:
+ lines = lines[1:]
+ if lines and lines[-1].strip() == "```":
+ lines = lines[:-1]
+ text = "\n".join(lines).strip()
+ return text
diff --git a/wiki_engine/worker.py b/wiki_engine/worker.py
new file mode 100644
index 0000000..dcd1fee
--- /dev/null
+++ b/wiki_engine/worker.py
@@ -0,0 +1,138 @@
+import asyncio
+import json
+from typing import Any, Dict, List
+
+from config import WIKI_CONFIG
+from wiki_engine import db
+
+
+_WORKER_TASKS: List[asyncio.Task] = []
+_STARTED = False
+
+
+async def start_wiki_queue_workers() -> None:
+ """Start durable Wiki queue workers for background sync jobs."""
+ global _STARTED
+ if _STARTED or not WIKI_CONFIG.get("queue_enabled", True):
+ return
+ _STARTED = True
+ worker_count = _bounded_int(WIKI_CONFIG.get("queue_worker_concurrency"), 2, 1, 8)
+ for index in range(worker_count):
+ _WORKER_TASKS.append(asyncio.create_task(_worker_loop(index + 1)))
+ print(f"[wiki_worker] started {worker_count} worker(s)")
+
+
+async def stop_wiki_queue_workers() -> None:
+ for task in _WORKER_TASKS:
+ task.cancel()
+ if _WORKER_TASKS:
+ await asyncio.gather(*_WORKER_TASKS, return_exceptions=True)
+ _WORKER_TASKS.clear()
+
+
+async def _worker_loop(worker_id: int) -> None:
+ idle_sleep = 1.0
+ while True:
+ try:
+ ops = await db.claim_wiki_file_ops(
+ limit=1,
+ stale_seconds=_bounded_int(WIKI_CONFIG.get("queue_claim_stale_seconds"), 1800, 60, 24 * 3600),
+ per_kb_concurrency=_bounded_int(WIKI_CONFIG.get("queue_per_kb_concurrency"), 1, 1, 8),
+ )
+ if not ops:
+ finalize_ops = await db.claim_wiki_finalize_ops(limit=1)
+ if finalize_ops:
+ for finalize_op in finalize_ops:
+ await _process_finalize_op(worker_id, finalize_op)
+ continue
+ await asyncio.sleep(idle_sleep)
+ continue
+ for op in ops:
+ await _process_op(worker_id, op)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ print(f"[wiki_worker] worker {worker_id} loop failed: {exc}")
+ await asyncio.sleep(3.0)
+
+
+async def _process_op(worker_id: int, op: Dict[str, Any]) -> None:
+ payload = _decode_payload(op.get("payload"))
+ job_id = str(payload.get("job_id") or op.get("job_id") or "")
+ kb_id = str(payload.get("kb_id") or op.get("kb_id") or "")
+ file_id = str(payload.get("file_id") or op.get("file_id") or "")
+ file_info = payload.get("file_info") if isinstance(payload.get("file_info"), dict) else {}
+ use_llm = bool(payload.get("use_llm"))
+ force = bool(payload.get("force"))
+ print(f"[wiki_worker] worker {worker_id} sync_file file_id={file_id} kb_id={kb_id} job_id={job_id}")
+
+ try:
+ from wiki_engine import service
+
+ result = await service.sync_file(
+ file_id=file_id,
+ kb_id=kb_id,
+ file_info=file_info,
+ use_llm=use_llm,
+ job_id=job_id,
+ force=force,
+ )
+ await db.complete_wiki_op(int(op["id"]))
+ await db.increment_job_progress(
+ job_id,
+ processed_delta=1,
+ slice_delta=int(result.get("slice_count") or 0),
+ skipped_delta=1 if result.get("skipped") else 0,
+ page_delta=int(result.get("page_count") or 0) if not result.get("skipped") else 0,
+ stage="queued_finalize" if kb_id else "file_done",
+ )
+ if job_id and kb_id:
+ job = await db.get_job(job_id)
+ if job and int(job.get("processed_files") or 0) >= int(job.get("total_files") or 0):
+ await service.enqueue_finalize_job(kb_id=kb_id, job_id=job_id)
+ except Exception as exc:
+ terminal = await db.fail_wiki_op(
+ op,
+ error=str(exc),
+ max_retries=_bounded_int(WIKI_CONFIG.get("queue_max_retries"), 3, 1, 10),
+ )
+ if terminal and job_id:
+ await db.increment_job_progress(job_id, processed_delta=1, slice_delta=0, error=f"{file_id}: {exc}")
+ print(f"[wiki_worker] worker {worker_id} sync_file failed file_id={file_id}: {exc}")
+
+
+async def _process_finalize_op(worker_id: int, op: Dict[str, Any]) -> None:
+ payload = _decode_payload(op.get("payload"))
+ job_id = str(payload.get("job_id") or op.get("job_id") or "")
+ kb_id = str(payload.get("kb_id") or op.get("kb_id") or "")
+ print(f"[wiki_worker] worker {worker_id} finalize_kb kb_id={kb_id} job_id={job_id}")
+ try:
+ from wiki_engine import service
+
+ await service.finalize_kb(kb_id=kb_id, job_id=job_id)
+ await db.complete_wiki_finalize_op(int(op["id"]))
+ except Exception as exc:
+ await db.fail_wiki_finalize_op(int(op["id"]), str(exc))
+ if job_id:
+ await db.update_job(job_id, stage="finalize_failed", error=str(exc))
+ print(f"[wiki_worker] worker {worker_id} finalize failed kb_id={kb_id}: {exc}")
+
+
+def _decode_payload(value: Any) -> Dict[str, Any]:
+ if isinstance(value, dict):
+ return value
+ if isinstance(value, str) and value.strip():
+ try:
+ parsed = json.loads(value)
+ return parsed if isinstance(parsed, dict) else {}
+ except Exception:
+ return {}
+ return {}
+
+
+def _bounded_int(value: Any, default: int, minimum: int, maximum: int) -> int:
+ try:
+ number = int(value)
+ except Exception:
+ number = default
+ return max(minimum, min(maximum, number))
diff --git a/workflow_registry.py b/workflow_registry.py
index b1468a0..83ed507 100644
--- a/workflow_registry.py
+++ b/workflow_registry.py
@@ -72,6 +72,16 @@ WORKFLOW_CONFIG = {
"module": "workflows.workflow_statistics",
"function": "run_statistics_workflow",
"is_async": True,
+ },
+ "wiki_admin": {
+ "module": "workflows.workflow_wiki_admin",
+ "function": "run_wiki_admin_workflow",
+ "is_async": True,
+ },
+ "Wiki\u77e5\u8bc6\u5e93\u7ba1\u7406": {
+ "module": "workflows.workflow_wiki_admin",
+ "function": "run_wiki_admin_workflow",
+ "is_async": True,
}
}
diff --git a/workflows/workflow_baike.py b/workflows/workflow_baike.py
index 1b00de5..0083f27 100644
--- a/workflows/workflow_baike.py
+++ b/workflows/workflow_baike.py
@@ -1,546 +1,716 @@
"""
-工作流：普通问答Agent（使用LangGraph）
-输入：文本（转化后和原文本query）、图片描述、语音文本
-步骤：
-1. 使用RAG和GraphRAG检索相关信息
-2. 生成回答
-使用 OpenaiAPI 方法调用大模型，尽可能使用异步
+百科问答工作流。
+
+核心链路借鉴 WeKnora：
+1. 意图判断默认偏向检索
+2. 基于历史重写检索 query，并生成多个检索变体
+3. RAG / Graph / Wiki 并行召回
+4. Wiki 命中后使用关联 slice 做 deep read
+5. 多源 evidence 去重、融合排序、rerank
+6. 只基于 evidence 生成可追溯回答
"""
-import sys
-import os
-
-# 添加项目根目录到Python搜索路径
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from typing import TypedDict, Optional, Dict, Any, List
-from langgraph.graph import StateGraph, START, END
-from workflows.history_manager import init_history, DEFAULT_HISTORY_MAX_MESSAGES
-# from workflows.workflow_utils import stream_generate_and_postprocess
-from workflows.workflow_utils import dedupe_rag_results, stream_generate_and_postprocess
-
-from tools.function_tool import (
- graph_rag_search
-)
-from tools.rag_tools import rag_search
-from modelsAPI.model_api import OpenaiAPI
-from utils.function_tracker import track_function_calls, get_all_callbacks
-from utils.image_utils import extract_image_paths
-from prompts import BAIKE_PROMPTS
import asyncio
import json
-import re
+import os
import random
+import re
+import sys
+from typing import Any, Dict, List, Optional, TypedDict
+
+from langgraph.graph import END, START, StateGraph
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from modelsAPI.model_api import OpenaiAPI
+from prompts import BAIKE_PROMPTS
+from tools.function_tool import graph_rag_search
+from tools.rag_tools import rag_search
+from tools.wiki_tools import wiki_evidence_search
+from utils.function_tracker import get_all_callbacks, track_function_calls
+from utils.image_utils import extract_image_paths
+from workflows.history_manager import DEFAULT_HISTORY_MAX_MESSAGES, init_history
+from workflows.workflow_utils import dedupe_rag_results, stream_generate_and_postprocess
+
BAIKE_RAG_TOP_K = 8
BAIKE_GRAPH_TOP_K = 240
BAIKE_RESULT_LIMIT = 8
+BAIKE_WIKI_TOP_K = 8
+BAIKE_EVIDENCE_LIMIT = 10
-# =============== 定义状态 ===============
class QAState(TypedDict):
- """普通问答工作流状态"""
- extracted_text: str # 已提取的文本（统一接口，只接收文本）
+ extracted_text: str
combined_query: str
- retrieval_query: str # 用于RAG/图谱检索的重写后query
- route_flag: str # 路由标识
- history_message: str # 历史消息（JSON格式）
- history_messages: List[Dict[str, Any]] # 过滤后的历史消息列表
- need_retrieval: Optional[bool] # 是否需要检索
+ retrieval_query: str
+ retrieval_queries: List[str]
+ route_flag: str
+ history_message: str
+ history_messages: List[Dict[str, Any]]
+ need_retrieval: Optional[bool]
+ rag_search_result: Optional[List[Dict[str, Any]]]
+ graph_search_result: Optional[str]
+ wiki_search_result: Optional[List[Dict[str, Any]]]
+ evidence_items: Optional[List[Dict[str, Any]]]
+ response: str
+ actions: List[Dict[str, Any]]
+ suggestedReplies: List[Dict[str, Any]]
+ error_message: str
- # RAG检索结果
- rag_search_result: Optional[List[Dict[str, Any]]] # RAG检索结果
- graph_search_result: Optional[str] # 图谱检索结果
-
- # 最终响应
- response: str # 最终响应
- actions: List[Dict[str, Any]] # 按钮信息列表 [{"type": "...", "label": "...", "icon": "..."}]
- suggestedReplies: List[Dict[str, Any]] # 建议回复问题列表
- error_message: str # 错误信息
-
-
-# =============== 节点函数 ===============
def get_baike_history_context(state: QAState) -> Dict[str, Any]:
- """
- 知识问答步骤：获取问答类历史信息
- 问答工作流只关注历史问答记录，用于提供上下文
- """
- history_message = state.get("history_message", "") or ""
- return init_history(history_message)
+ return init_history(state.get("history_message", "") or "")
async def judge_need_retrieval(state: QAState) -> Dict[str, Any]:
- """
- 判断用户问题是否需要检索知识库
- """
original_query = (state.get("combined_query") or state.get("extracted_text") or "").strip()
history_messages = state.get("history_messages", []) or []
-
prompt = BAIKE_PROMPTS["judge_need_retrieval"].format(
- history_messages=history_messages if history_messages else '无',
- original_query=original_query
+ history_messages=history_messages if history_messages else "无",
+ original_query=original_query,
)
-
- need_retrieval = True
+
+ substantive_query = _looks_like_substantive_query(original_query)
+ need_retrieval = substantive_query
try:
result = await OpenaiAPI.open_api_chat_without_thinking(prompt, model=None)
- if result:
- result = result.strip().lower()
- if result == "false":
- need_retrieval = False
- except Exception as e:
- print(f"判断是否需要检索失败: {e}")
- need_retrieval = True
-
-
+ if not substantive_query and result and result.strip().lower() == "false":
+ need_retrieval = False
+ except Exception as exc:
+ print(f"判断是否需要检索失败: {exc}")
print(f"是否需要检索: {need_retrieval}")
return {"need_retrieval": need_retrieval}
+def _looks_like_substantive_query(query: str) -> bool:
+ text = re.sub(r"\s+", "", (query or "").strip())
+ if not text:
+ return False
+ if re.fullmatch(r"(你好|您好|hi|hello|在吗|谢谢|多谢|好的|好|嗯|哦|确认|取消|不用了)[。！!?\s]*", text, re.I):
+ return False
+ if len(text) >= 4:
+ return True
+ return bool(re.search(r"[？?]|[A-Za-z0-9]{2,}", text))
+
+
async def rewrite_query_with_history(state: QAState) -> Dict[str, Any]:
- """
- 使用历史上下文对当前问题进行query重写，生成适合RAG/图谱检索的独立查询语句。
- 只让大模型在有限的最近历史中挑选与当前问题语义相关的内容进行融合，避免简单拼接全量历史。
- """
- import json
-
- # 原始问题：优先使用 combined_query，其次使用 extracted_text
original_query = (state.get("combined_query") or state.get("extracted_text") or "").strip()
-
history_messages = state.get("history_messages", []) or []
trimmed_history = history_messages[-DEFAULT_HISTORY_MAX_MESSAGES:] if history_messages else []
-
- try:
- history_str = json.dumps(trimmed_history, ensure_ascii=False)
- except Exception:
- history_str = str(trimmed_history)
-
- # 如果没有历史或当前问题过短，就直接跳过重写
- if not original_query:
- return {"retrieval_query": original_query}
- # 在构造 prompt 前，提取上一轮用户 query（如果有）
+ history_str = _json_dumps(trimmed_history)
last_user_query = ""
- if trimmed_history:
- # 从后往前找最近一条 role == "user" 的消息
- for msg in reversed(trimmed_history):
- if msg.get("role") == "user":
- last_user_query = msg.get("content", "").strip()
- break
+ for msg in reversed(trimmed_history):
+ if msg.get("role") == "user":
+ last_user_query = str(msg.get("content") or "").strip()
+ break
+
+ if not original_query:
+ return {"retrieval_query": "", "retrieval_queries": []}
+
prompt = BAIKE_PROMPTS["rewrite_query_with_history"].format(
- last_user_query=last_user_query or '（无）',
+ last_user_query=last_user_query or "无",
history_str=history_str,
- original_query=original_query
+ original_query=original_query,
)
rewritten_query = original_query
+ query_variants: List[str] = []
try:
- result = await OpenaiAPI.open_api_chat_without_thinking(prompt, model=None)
- if result:
- candidate = result.strip().strip("“”\"'")
- # 只在模型返回的内容明显为单行短文本时替换，避免异常输出
- if candidate and len(candidate) <= 200 and "\n" not in candidate:
- rewritten_query = candidate
- except Exception as e:
- print(f"query 重写失败: {e}")
-
-
- title_options = [f"✨正在分析中，请稍等..."]
- start_event = {
- "type": "function_execution",
- "title": random.choice(title_options),
- "details": f"正在调取百科数据..."
- }
- for callback in get_all_callbacks():
- try:
- callback(start_event)
- except Exception:
- pass
+ raw = await OpenaiAPI.open_api_chat_without_thinking(prompt, model=None, json_output=True)
+ data = _parse_json_object(raw)
+ if data:
+ candidate = str(data.get("rewrite_query") or "").strip()
+ if candidate:
+ rewritten_query = candidate[:180]
+ for q in data.get("search_queries") or []:
+ if isinstance(q, str) and q.strip():
+ query_variants.append(q.strip()[:180])
+ except Exception as exc:
+ print(f"query 重写失败: {exc}")
+ query_variants = _build_search_queries(rewritten_query, original_query, query_variants)
+ _emit_event("✨正在分析中，请稍等...", "正在生成检索策略...")
print("原始检索query:", original_query)
- print("重写后检索query:", last_user_query+rewritten_query)
-
- return {"retrieval_query":last_user_query+rewritten_query}
+ print("重写后检索query:", rewritten_query)
+ print("检索query变体:", query_variants)
+ return {"retrieval_query": rewritten_query, "retrieval_queries": query_variants}
@track_function_calls
async def search_rag_and_graph(state: QAState) -> Dict[str, Any]:
- """
- 进行文本知识检索和图谱知识检索
- """
- kb_id = None
- if state.get("route_flag") == "rules":
- kb_id = '3'
-
- # 优先使用重写后的检索query，其次使用 combined_query，最后回退到原始 extracted_text
- query = state.get("retrieval_query") or state.get("combined_query") or state.get("extracted_text", "")
- if "用户问题：" in query:
- query = query.split("用户问题：", 1)[1]
-
- print("Graph 检索query", query)
- extracted_text = state.get("extracted_text", "")
- if "用户问题：" in extracted_text:
- extracted_text = extracted_text.split("用户问题：", 1)[1]
+ kb_id = "3" if state.get("route_flag") == "rules" else None
+ query = (state.get("retrieval_query") or state.get("combined_query") or state.get("extracted_text") or "").strip()
+ extracted_text = (state.get("extracted_text") or "").strip()
+ query_variants = _build_search_queries(query, extracted_text, state.get("retrieval_queries") or [])
try:
- # 异步并行调用RAG检索和图谱检索
- print(kb_id)
- print(33333333333333333333333333333333333333)
+ rag_task = _search_rag_variants(query_variants[:3], kb_id=kb_id)
+ graph_task = graph_rag_search.ainvoke({"query": query, "top_k": BAIKE_GRAPH_TOP_K})
+ wiki_task = _search_wiki_variants(query_variants)
+ rag_results, graph_results, wiki_result = await asyncio.gather(rag_task, graph_task, wiki_task)
- async def _do_rag_search():
- if kb_id:
- return await rag_search(
- query=extracted_text,
- kb_id=kb_id,
- top_k=BAIKE_RAG_TOP_K
- )
- else:
- return await rag_search(
- query=extracted_text,
- top_k=BAIKE_RAG_TOP_K
- )
-
- async def _do_graph_search():
- return await graph_rag_search.ainvoke({
- "query": query,
- "top_k": BAIKE_GRAPH_TOP_K
- })
-
- rag_result, graph_results = await asyncio.gather(
- _do_rag_search(),
- _do_graph_search()
+ wiki_items = wiki_result.get("items", []) if wiki_result.get("success") else []
+ if wiki_items:
+ _emit_event(
+ f"Wiki 命中 {len(wiki_items[:3])} 条",
+ _format_wiki_hits_preview(wiki_items[:3]),
+ )
+ wiki_kb_ids = _extract_wiki_kb_ids(wiki_items)
+ wiki_file_ids = _extract_wiki_file_ids(wiki_items)
+ rag_results = _filter_rag_results_by_wiki_scope(rag_results, wiki_kb_ids, wiki_file_ids)
+ if wiki_kb_ids and len(rag_results) < 2:
+ scoped_rag = await _search_rag_variants(query_variants[:3], kb_id=sorted(wiki_kb_ids)[0])
+ rag_results = _filter_rag_results_by_wiki_scope(
+ dedupe_rag_results([*rag_results, *scoped_rag], limit=BAIKE_RESULT_LIMIT),
+ wiki_kb_ids,
+ wiki_file_ids,
+ )
+ evidence_items = _build_evidence_items(
+ rag_results=rag_results,
+ wiki_results=wiki_items,
+ graph_results=graph_results if isinstance(graph_results, str) else "",
+ query=query,
)
-
- # 处理 RAG 结果，提取成列表格式
- rag_search_result = []
- if rag_result.get("success", False):
- source_citation = rag_result.get("sourceCitation", {})
- for resource, items in source_citation.items():
- for item in items:
- rag_search_result.append({
- "text": item.get("text", ""),
- "filename": item.get("filename", ""),
- "resource": item.get("resource", ""),
- "score": item.get("score", 0.0)
- })
-
- rag_search_result = sorted(rag_search_result,key=lambda x:float(x.get("score") or 0), reverse=True)[:BAIKE_RESULT_LIMIT]
- rag_search_result = dedupe_rag_results(rag_search_result, limit=BAIKE_RESULT_LIMIT)
+ evidence_items = await _rerank_evidence(query, evidence_items)
return {
- "rag_search_result": rag_search_result,
- "graph_search_result": graph_results if isinstance(graph_results, str) else ""
+ "rag_search_result": rag_results,
+ "graph_search_result": graph_results if isinstance(graph_results, str) else "",
+ "wiki_search_result": wiki_items,
+ "evidence_items": evidence_items,
}
- except Exception as e:
+ except Exception as exc:
return {
"rag_search_result": [],
"graph_search_result": "",
- "error_message": f"检索失败: {str(e)}"
+ "wiki_search_result": [],
+ "evidence_items": [],
+ "error_message": f"检索失败: {exc}",
}
-
async def generate_suggested_replies_baike(answer: str, question: str) -> List[Dict[str, Any]]:
- """
- 生成针对问答结果的建议回复问题
- 使用大模型生成两个相关问题
- """
- prompt = BAIKE_PROMPTS["generate_suggested_replies"].format(
- question=question,
- answer=answer[:500]
- )
+ prompt = BAIKE_PROMPTS["generate_suggested_replies"].format(question=question, answer=answer[:500])
try:
result = await OpenaiAPI.open_api_chat_without_thinking(prompt, model=None, json_output=True)
- # 尝试解析JSON
- # 提取JSON部分
- json_match = re.search(r'\[.*\]', result, re.DOTALL)
+ json_match = re.search(r"\[.*\]", result or "", re.DOTALL)
if json_match:
- json_str = json_match.group(0)
- suggested_replies = json.loads(json_str)
- # 验证格式
- if isinstance(suggested_replies, list) and len(suggested_replies) >= 2:
- # 确保每个元素都有title和content
- formatted_replies = []
- for reply in suggested_replies[:2]:
- if isinstance(reply, dict) and "title" in reply and "content" in reply:
- formatted_replies.append({
- "title": str(reply["title"]),
- "content": str(reply["content"])
- })
- if len(formatted_replies) == 2:
- return formatted_replies
-
- # 如果解析失败，返回默认问题
- return [
- {"title": "这个设备的具体工作原理是什么？", "content": "这个设备的具体工作原理是什么？"},
- {"title": "还有哪些相关的知识点？", "content": "还有哪些相关的知识点？"}
- ]
- except Exception as e:
- # 如果生成失败，返回默认问题
- return [
- {"title": "这个设备的具体工作原理是什么？", "content": "这个设备的具体工作原理是什么？"},
- {"title": "还有哪些相关的知识点？", "content": "还有哪些相关的知识点？"}
- ]
+ replies = json.loads(json_match.group(0))
+ formatted = [
+ {"title": str(x["title"]), "content": str(x["content"])}
+ for x in replies[:2]
+ if isinstance(x, dict) and x.get("title") and x.get("content")
+ ]
+ if len(formatted) == 2:
+ return formatted
+ except Exception:
+ pass
+ return [
+ {"title": "查看相关依据", "content": "这个回答对应的资料依据有哪些？"},
+ {"title": "继续展开说明", "content": "请把这个问题再展开说明一下。"},
+ ]
@track_function_calls
async def generate_answer(state: QAState) -> Dict[str, Any]:
- """
- 正在生成问答结果
- """
- extracted_text = state.get("combined_query", "")
- rag_results = state.get("rag_search_result", [])
- graph_results = state.get("graph_search_result", "")
- history_messages = state.get("history_messages", []) or []
+ question = state.get("combined_query") or state.get("extracted_text") or ""
+ evidence_items = state.get("evidence_items") or []
need_retrieval = state.get("need_retrieval", True)
- print("history_message(qa)", history_messages)
- print("是否使用检索结果:", need_retrieval)
+ history_messages = state.get("history_messages", []) or []
try:
- # 1. 处理检索结果（如果有）
- rag_ctx_parts: List[str] = []
- file_name_list = []
- all_source_text = ""
- original_image_paths = []
-
- if need_retrieval:
- # 收集所有 RAG 结果文本（只取前 4 条）
- for item in sorted(rag_results or [], key=lambda x: float(x.get("score") or 0) if isinstance(x,dict) else 0, reverse=True)[:4]:
-# for item in (rag_results or [])[:1]:
- print(item)
- print(111111111111111111111111111)
- if isinstance(item, dict):
- text = (item.get("text") or "").strip()
- file_name = (item.get("filename") or "").strip()
- if text:
- rag_ctx_parts.append(text)
- if file_name:
- file_name_list.append(file_name)
-
- # 处理图谱结果
- print("123456", graph_results)
- if graph_results and "图谱检索完成，但未返回有效数据。" in graph_results:
- graph_results = ""
- print("654321", graph_results)
-
- # 从所有结果中提取图片路径
- if graph_results:
- all_source_text = graph_results + "\n" + "\n".join(rag_ctx_parts)
- else:
- all_source_text = "\n".join(rag_ctx_parts)
- original_image_paths = extract_image_paths(all_source_text)
- print("提取到的原始图片路径:", original_image_paths)
-
- # 发送事件通知（可选）
- title_options = [f"✨已综合 {len(rag_ctx_parts)} 条检索结果", f"✨本次检索策略为综合多源信息"]
- start_event = {
- "type": "function_execution",
- "title": random.choice(title_options),
- "details": f"综合了 {len(rag_ctx_parts)} 条检索结果"
- }
- for callback in get_all_callbacks():
- try:
- callback(start_event)
- except Exception:
- pass
+ source_text = ""
+ original_image_paths: List[str] = []
+ if need_retrieval and evidence_items:
+ source_text = _format_evidence_context(evidence_items)
+ original_image_paths = extract_image_paths(source_text)
+ source_counts = _count_sources(evidence_items)
+ _emit_event(
+ f"✨已融合 {len(evidence_items)} 条证据",
+ f"RAG {source_counts.get('rag', 0)} 条，Wiki {source_counts.get('wiki', 0)} 条，图谱 {source_counts.get('graph', 0)} 条",
+ )
- query_desc = extracted_text if extracted_text else "当前未明确描述问题"
+ domain_desc = (
+ "舰船维修相关的法规、规范、行业标准及技术文件"
+ if state.get("route_flag") == "rules"
+ else "工业设备的结构原理、维护规程、故障诊断与维修操作"
+ )
- # 构建系统提示词
- if state.get("route_flag") == "rules":
- domain_desc = "舰船修理相关的法规、规范、行业标准及技术文件"
- else:
- domain_desc = "工业设备的结构原理、维护规程、故障诊断与维修操作"
-
- # ========== 第一步：专注于生成高质量内容 ==========
- # 根据是否有检索结果构建不同的提示词
- if need_retrieval and all_source_text.strip():
- content_system_prompt = BAIKE_PROMPTS["generate_answer_with_retrieval_system"].format(domain_desc=domain_desc)
- content_user_content = BAIKE_PROMPTS["generate_answer_with_retrieval_user"].format(
- extracted_text=extracted_text,
- rag_count=len(rag_ctx_parts),
- all_source_text=all_source_text
+ if need_retrieval and source_text.strip():
+ system_prompt = BAIKE_PROMPTS["generate_answer_with_retrieval_system"].format(domain_desc=domain_desc)
+ user_prompt = BAIKE_PROMPTS["generate_answer_with_retrieval_user"].format(
+ extracted_text=question,
+ rag_count=len(evidence_items),
+ all_source_text=source_text,
)
else:
- content_system_prompt = BAIKE_PROMPTS["generate_answer_without_retrieval_system"].format(domain_desc=domain_desc)
- content_user_content = BAIKE_PROMPTS["generate_answer_without_retrieval_user"].format(extracted_text=extracted_text)
+ system_prompt = BAIKE_PROMPTS["generate_answer_without_retrieval_system"].format(domain_desc=domain_desc)
+ user_prompt = BAIKE_PROMPTS["generate_answer_without_retrieval_user"].format(extracted_text=question)
- content_messages = []
+ messages = []
if history_messages:
- content_messages.extend(history_messages)
- content_messages.append({"role": "user", "content": content_user_content})
-
- title_options = ["🤖 诊断分析中", "🤖 故障预检中"]
- start_event = {
- "type": "function_execution",
- "title": random.choice(title_options),
- "details": "正在生成回答..."
- }
- for callback in get_all_callbacks():
- try:
- callback(start_event)
- except Exception:
- pass
+ messages.extend(history_messages)
+ messages.append({"role": "user", "content": user_prompt})
+ _emit_event(random.choice(["🔍 证据整理中", "🧠 正在生成回答"]), "正在基于检索资料生成回答...")
answer = await stream_generate_and_postprocess(
- system_prompt=content_system_prompt,
- messages=content_messages,
+ system_prompt=system_prompt,
+ messages=messages,
original_image_paths=original_image_paths,
- temperature=0.5,
- fallback="抱歉，无法生成回答。"
+ temperature=0.2,
+ fallback="抱歉，无法生成回答。",
)
- print("llm output", answer)
+ suggested_replies = await generate_suggested_replies_baike(answer, question)
+ return {"response": answer, "actions": [], "result_tag": "qa", "suggestedReplies": suggested_replies}
+ except Exception as exc:
+ return {"response": f"生成回答失败：{exc}", "actions": [], "suggestedReplies": []}
+
+
+async def _search_rag_variants(queries: List[str], kb_id: Optional[str] = None) -> List[Dict[str, Any]]:
+ tasks = [rag_search(query=q, kb_id=kb_id, top_k=BAIKE_RAG_TOP_K) for q in queries if q.strip()]
+ results = await asyncio.gather(*tasks, return_exceptions=True) if tasks else []
+ items: List[Dict[str, Any]] = []
+ for result in results:
+ if isinstance(result, Exception) or not result.get("success"):
+ continue
+ for resource, rows in (result.get("sourceCitation") or {}).items():
+ for row in rows:
+ item = dict(row)
+ item.setdefault("resource", resource)
+ items.append(item)
+ items = sorted(items, key=lambda x: float(x.get("score") or 0), reverse=True)
+ return dedupe_rag_results(items, limit=BAIKE_RESULT_LIMIT)
+
+
+async def _search_wiki_variants(queries: List[str]) -> Dict[str, Any]:
+ seen_keys = set()
+ merged: List[Dict[str, Any]] = []
+ used_queries = []
+ for q in queries[:8]:
+ try:
+ result = await wiki_evidence_search(query=q, limit=BAIKE_WIKI_TOP_K)
+ except Exception as exc:
+ print(f"Wiki 检索失败: {exc}")
+ continue
+ used_queries.append(q)
+ if not result.get("success"):
+ continue
+ for item in result.get("items", []):
+ key = (item.get("source_id"), item.get("title"))
+ if key in seen_keys:
+ continue
+ seen_keys.add(key)
+ item = dict(item)
+ item["used_query"] = q
+ merged.append(item)
+ merged.sort(key=lambda x: float(x.get("score") or 0), reverse=True)
+ return {"success": True, "items": merged[:BAIKE_WIKI_TOP_K], "used_queries": used_queries}
+
+
+def _format_wiki_hits_preview(items: List[Dict[str, Any]]) -> str:
+ lines: List[str] = []
+ type_names = {
+ "entity": "实体",
+ "concept": "概念",
+ "document_summary": "文档摘要",
+ "procedure": "流程",
+ "fault": "故障",
+ "operation": "操作",
+ "standard": "标准",
+ }
+ for idx, item in enumerate(items[:3], start=1):
+ metadata = item.get("metadata") or {}
+ page_type = type_names.get(str(metadata.get("page_type") or item.get("page_type") or ""), "Wiki")
+ title = str(item.get("title") or "未命名页面").strip()
+ score = item.get("score", "")
+ try:
+ score_text = f"{float(score):.2f}"
+ except Exception:
+ score_text = str(score or "")
+ slice_ids = metadata.get("related_slice_ids") or []
+ slice_text = f"；slice: {', '.join(str(x) for x in slice_ids[:3])}" if slice_ids else ""
+ lines.append(f"{idx}. [{page_type}] {title}（score {score_text}{slice_text}）")
+ return "\n".join(lines) if lines else "未检索到可展示的 Wiki 命中。"
+
+
+def _wiki_preview_text(item: Dict[str, Any]) -> str:
+ text = str(item.get("text") or item.get("summary") or "").strip()
+ if not text:
+ metadata = item.get("metadata") or {}
+ slices = metadata.get("slices") or []
+ for s in slices:
+ if isinstance(s, dict) and str(s.get("content") or s.get("brief") or "").strip():
+ text = str(s.get("content") or s.get("brief") or "").strip()
+ break
+ text = re.sub(r"\s+", " ", text)
+ return text[:120] + ("..." if len(text) > 120 else "")
+
+
+def _extract_wiki_kb_ids(items: List[Dict[str, Any]]) -> set:
+ kb_ids = set()
+ for item in items or []:
+ metadata = item.get("metadata") or {}
+ for container in (item, metadata):
+ kb_id = container.get("kb_id") or container.get("knowledge_base_id")
+ if kb_id not in (None, ""):
+ kb_ids.add(str(kb_id))
+ for slice_item in metadata.get("slices") or []:
+ if not isinstance(slice_item, dict):
+ continue
+ kb_id = slice_item.get("kb_id") or slice_item.get("knowledge_base_id")
+ if kb_id not in (None, ""):
+ kb_ids.add(str(kb_id))
+ return kb_ids
+
+
+def _extract_wiki_file_ids(items: List[Dict[str, Any]]) -> set:
+ file_ids = set()
+ for item in items or []:
+ metadata = item.get("metadata") or {}
+ for file_id in item.get("related_file_ids") or metadata.get("related_file_ids") or []:
+ if file_id not in (None, ""):
+ file_ids.add(str(file_id))
+ for slice_item in metadata.get("slices") or []:
+ if not isinstance(slice_item, dict):
+ continue
+ file_id = slice_item.get("file_id")
+ if file_id not in (None, ""):
+ file_ids.add(str(file_id))
+ return file_ids
+
+
+def _filter_rag_results_by_wiki_scope(
+ rag_results: List[Dict[str, Any]],
+ allowed_kb_ids: set,
+ allowed_file_ids: set,
+) -> List[Dict[str, Any]]:
+ if not allowed_kb_ids and not allowed_file_ids:
+ return rag_results
+ filtered = []
+ for item in rag_results or []:
+ kb_id = item.get("kb_id") or item.get("knowledge_base_id")
+ file_id = item.get("file_id")
+ if kb_id not in (None, ""):
+ if str(kb_id) in allowed_kb_ids:
+ filtered.append(item)
+ continue
+ if file_id not in (None, ""):
+ if str(file_id) in allowed_file_ids:
+ filtered.append(item)
+ continue
+ if not allowed_file_ids:
+ filtered.append(item)
+ return filtered
+
+
+def _build_evidence_items(
+ rag_results: List[Dict[str, Any]],
+ wiki_results: List[Dict[str, Any]],
+ graph_results: str,
+ query: str,
+) -> List[Dict[str, Any]]:
+ candidates: List[Dict[str, Any]] = []
+ for item in rag_results or []:
+ candidates.append({
+ "source_type": "rag",
+ "title": item.get("filename") or item.get("resource") or "RAG 检索结果",
+ "filename": item.get("filename") or item.get("resource") or "",
+ "text": item.get("text") or "",
+ "score": float(item.get("score") or 0),
+ "file_id": str(item.get("file_id") or ""),
+ "slice_id": str(item.get("id") or item.get("slice_id") or ""),
+ "metadata": item,
+ })
+
+ for item in wiki_results or []:
+ metadata = item.get("metadata") or {}
+ slices = metadata.get("slices") or []
+ file_ids = metadata.get("related_file_ids") or []
+ slice_ids = metadata.get("related_slice_ids") or []
+ candidates.append({
+ "source_type": "wiki",
+ "title": item.get("title") or "Wiki 命中",
+ "filename": _first_non_empty([s.get("filename") for s in slices if isinstance(s, dict)]),
+ "text": item.get("text") or "",
+ "score": float(item.get("score") or 0),
+ "file_id": ", ".join(str(x) for x in file_ids[:5]),
+ "slice_id": ", ".join(str(x) for x in slice_ids[:8]),
+ "metadata": metadata,
+ "used_query": item.get("used_query", ""),
+ })
+
+ graph_text = str(graph_results or "").strip()
+ if graph_text and "图谱检索完成，但未返回有效数据。" not in graph_text and graph_text != "查询无结果":
+ candidates.append({
+ "source_type": "graph",
+ "title": "图谱检索结果",
+ "filename": "",
+ "text": graph_text,
+ "score": 0.45,
+ "file_id": "",
+ "slice_id": "",
+ "metadata": {},
+ })
+
+ deduped: Dict[str, Dict[str, Any]] = {}
+ for item in candidates:
+ text = _clean_text(item.get("text") or "")
+ if not text:
+ continue
+ item["text"] = text[:5000]
+ key = item.get("slice_id") or _content_key(text)
+ local_score = _local_evidence_score(query, item)
+ item["local_score"] = local_score
+ existing = deduped.get(key)
+ if not existing or local_score > float(existing.get("local_score") or 0):
+ deduped[key] = item
+ return sorted(deduped.values(), key=lambda x: float(x.get("local_score") or 0), reverse=True)
+
+
+async def _rerank_evidence(query: str, evidence_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ if not query.strip() or not evidence_items:
+ return evidence_items[:BAIKE_EVIDENCE_LIMIT]
+ selected = evidence_items[:24]
+ documents = [
+ "\n".join([
+ str(item.get("title") or ""),
+ str(item.get("filename") or ""),
+ str(item.get("text") or ""),
+ ])[:4000]
+ for item in selected
+ ]
+ rows = await OpenaiAPI.rerank(query=query, documents=documents, top_k=BAIKE_EVIDENCE_LIMIT)
+ if not rows:
+ return evidence_items[:BAIKE_EVIDENCE_LIMIT]
+
+ ranked: List[Dict[str, Any]] = []
+ seen = set()
+ for row in rows:
+ idx = int(row.get("index") or 0)
+ if idx < 0 or idx >= len(selected) or idx in seen:
+ continue
+ seen.add(idx)
+ item = dict(selected[idx])
+ item["rerank_score"] = float(row.get("relevance_score") or 0.0)
+ ranked.append(item)
+ for idx, item in enumerate(selected):
+ if idx not in seen and len(ranked) < BAIKE_EVIDENCE_LIMIT:
+ ranked.append(item)
+ return ranked[:BAIKE_EVIDENCE_LIMIT]
+
+
+def _format_evidence_context(items: List[Dict[str, Any]]) -> str:
+ parts = []
+ label_map = {"rag": "RAG", "wiki": "Wiki", "graph": "Graph"}
+ for idx, item in enumerate(items[:BAIKE_EVIDENCE_LIMIT], start=1):
+ source = label_map.get(item.get("source_type"), item.get("source_type") or "unknown")
+ score = item.get("rerank_score", item.get("local_score", item.get("score", "")))
+ meta_lines = [
+ f"[Evidence {idx}] source={source} score={score}",
+ f"Title: {item.get('title') or ''}",
+ ]
+ if item.get("filename"):
+ meta_lines.append(f"File: {item.get('filename')}")
+ if item.get("file_id"):
+ meta_lines.append(f"File ID: {item.get('file_id')}")
+ if item.get("slice_id"):
+ meta_lines.append(f"Slice ID: {item.get('slice_id')}")
+ meta_lines.append("Content:")
+ meta_lines.append(str(item.get("text") or ""))
+ parts.append("\n".join(meta_lines))
+ return "\n\n---\n\n".join(parts)
+
+
+def _local_evidence_score(query: str, item: Dict[str, Any]) -> float:
+ base = {"wiki": 1.15, "rag": 1.0, "graph": 0.65}.get(item.get("source_type"), 0.8)
+ raw = float(item.get("score") or 0)
+ score = base + min(max(raw, 0.0), 1.0)
+ text = " ".join([
+ str(item.get("title") or ""),
+ str(item.get("filename") or ""),
+ str(item.get("text") or "")[:1500],
+ ]).lower()
+ title = str(item.get("title") or "").lower()
+ for term in _extract_search_terms(query):
+ t = term.lower()
+ if not t:
+ continue
+ if t in title:
+ score += 0.7
+ elif t in text:
+ score += 0.25
+ return score
+
+
+def _build_search_queries(main_query: str, original_query: str = "", extra_queries: Optional[List[str]] = None) -> List[str]:
+ candidates: List[str] = []
+ for text in [main_query, original_query, *(extra_queries or [])]:
+ text = _strip_query_noise(text)
+ if not text:
+ continue
+ candidates.append(text)
+ regex_terms = _extract_search_terms(text)
+ if regex_terms:
+ candidates.append(" ".join(regex_terms[:4]))
+ candidates.extend(regex_terms[:4])
+ return _dedupe_strings(candidates)[:10]
+
+
+def _extract_search_terms(query: str) -> List[str]:
+ stop = {"什么", "如何", "怎么", "哪些", "主要", "内容", "相关", "信息", "一下", "这个", "那个", "是否", "有没有"}
+ terms = []
+ for token in re.findall(r"[\u4e00-\u9fff]{2,12}|[A-Za-z0-9][A-Za-z0-9_\-]{1,30}", query or ""):
+ token = token.strip("：:，,。.;；、（）()[]【】")
+ if len(token) < 2 or token in stop or re.fullmatch(r"\d+", token):
+ continue
+ terms.append(token)
+ return _dedupe_strings(terms)
+
+
+def _strip_query_noise(text: str) -> str:
+ text = str(text or "").strip().strip('"“”')
+ if "用户问题：" in text:
+ text = text.split("用户问题：", 1)[1].strip()
+ return re.sub(r"\s+", " ", text)
+
+
+def _parse_json_object(raw: str) -> Dict[str, Any]:
+ if not raw:
+ return {}
+ try:
+ return json.loads(raw)
+ except Exception:
+ match = re.search(r"\{.*\}", raw, re.DOTALL)
+ if match:
+ try:
+ return json.loads(match.group(0))
+ except Exception:
+ return {}
+ return {}
+
+
+def _json_dumps(value: Any) -> str:
+ try:
+ return json.dumps(value, ensure_ascii=False)
+ except Exception:
+ return str(value)
+
+
+def _content_key(text: str) -> str:
+ return re.sub(r"\s+", "", text or "")[:220]
+
+
+def _clean_text(text: str) -> str:
+ return re.sub(r"\n{3,}", "\n\n", str(text or "")).strip()
+
+
+def _first_non_empty(values: List[Any]) -> str:
+ for value in values:
+ if value:
+ return str(value)
+ return ""
+
+
+def _dedupe_strings(values: List[str]) -> List[str]:
+ out = []
+ seen = set()
+ for value in values:
+ value = str(value or "").strip()
+ if not value or value in seen:
+ continue
+ seen.add(value)
+ out.append(value)
+ return out
+
+
+def _count_sources(items: List[Dict[str, Any]]) -> Dict[str, int]:
+ counts: Dict[str, int] = {}
+ for item in items:
+ key = str(item.get("source_type") or "unknown")
+ counts[key] = counts.get(key, 0) + 1
+ return counts
+
+
+def _emit_event(title: str, details: str) -> None:
+ event = {"type": "function_execution", "title": title, "details": details}
+ for callback in get_all_callbacks():
+ try:
+ callback(event)
+ except Exception:
+ pass
- # 生成建议回复问题
- suggested_replies = await generate_suggested_replies_baike(answer, extracted_text)
- return {
- "response": answer,
- "actions": [],
- "result_tag": "qa",
- "suggestedReplies": suggested_replies
- }
- except Exception as e:
- return {
- "response": f"生成回答失败：{str(e)}",
- "actions": [],
- "suggestedReplies": []
- }
def route_after_judge(state: QAState) -> str:
- """
- 判断后的路由：根据need_retrieval决定下一步
- """
- need_retrieval = state.get("need_retrieval", True)
- if need_retrieval:
- return "rewrite_query_with_history"
- else:
- return "generate_answer"
+ return "rewrite_query_with_history" if state.get("need_retrieval", True) else "generate_answer"
-# =============== 构建工作流 ===============
def create_baike_workflow():
- """创建普通问答工作流"""
workflow = StateGraph(QAState)
-
- # 添加节点
workflow.add_node("get_baike_history_context", get_baike_history_context)
workflow.add_node("judge_need_retrieval", judge_need_retrieval)
workflow.add_node("rewrite_query_with_history", rewrite_query_with_history)
workflow.add_node("search_rag_and_graph", search_rag_and_graph)
workflow.add_node("generate_answer", generate_answer)
- # 设置边
workflow.add_edge(START, "get_baike_history_context")
workflow.add_edge("get_baike_history_context", "judge_need_retrieval")
workflow.add_conditional_edges(
"judge_need_retrieval",
route_after_judge,
- {
- "rewrite_query_with_history": "rewrite_query_with_history",
- "generate_answer": "generate_answer"
- }
+ {"rewrite_query_with_history": "rewrite_query_with_history", "generate_answer": "generate_answer"},
)
workflow.add_edge("rewrite_query_with_history", "search_rag_and_graph")
workflow.add_edge("search_rag_and_graph", "generate_answer")
workflow.add_edge("generate_answer", END)
-
- # 编译工作流
return workflow.compile()
-# =============== 工作流执行函数 ===============
async def run_baike_workflow(
extracted_text: str,
combined_query: str,
history_message: str = "",
- route_flag: str = ""
+ route_flag: str = "",
) -> Dict[str, Any]:
- """
- 执行普通问答工作流（统一接口）
-
- Args:
- extracted_text: 文本描述（统一的输入参数）
- history_message: 历史消息（目前只接收，不做处理）
-
- Returns:
- 工作流执行结果
- """
- # 创建并编译工作流
app = create_baike_workflow()
-
- # 初始化状态（统一接口，只使用extracted_text和history_message）
initial_state = {
"extracted_text": extracted_text,
"combined_query": combined_query,
"retrieval_query": "",
+ "retrieval_queries": [],
"route_flag": route_flag,
"history_message": history_message,
"history_messages": [],
"need_retrieval": None,
"rag_search_result": None,
"graph_search_result": None,
+ "wiki_search_result": None,
+ "evidence_items": None,
"response": "",
"actions": [],
"suggestedReplies": [],
- "error_message": ""
+ "error_message": "",
}
-
+
try:
- # 执行工作流（使用异步方式）
final_state = await app.ainvoke(initial_state)
-
- # 检查是否有错误
if final_state.get("error_message"):
return {
"response": f"普通问答工作流执行失败: {final_state['error_message']}",
"actions": [],
"result_tag": "qa",
- "suggestedReplies": []
+ "suggestedReplies": [],
}
-
- # 统一输出格式：完全透传 generate_answer 的结果
return {
"response": final_state.get("response", ""),
"actions": final_state.get("actions", []),
"result_tag": "qa",
- "suggestedReplies": final_state.get("suggestedReplies", [])
+ "suggestedReplies": final_state.get("suggestedReplies", []),
}
-
- except Exception as e:
+ except Exception as exc:
return {
- "response": f"普通问答工作流执行失败: {str(e)}",
+ "response": f"普通问答工作流执行失败: {exc}",
"actions": [],
"result_tag": "qa",
- "suggestedReplies": []
+ "suggestedReplies": [],
}
-
-
-
-# =============== 测试 ===============
-if __name__ == "__main__":
- import asyncio
-
- # 测试用例
- test_cases = [
- "船舰主发动机的安全警告是什么",
- ]
-
- async def test():
- for i, test_text in enumerate(test_cases, 1):
- print(f"\n{'='*60}")
- print(f"测试用例 {i}")
- print(f"{'='*60}")
-
- # ✅ 修复：传入 combined_query 参数
- result = await run_baike_workflow(
- extracted_text=test_text,
- combined_query=test_text
- )
-
- print(f"\n执行结果:")
- print(f"成功：{result.get('success', False)}")
- print(f"\n响应内容:")
- print(result.get("response", "无响应"))
-
- asyncio.run(test())
-
diff --git a/workflows/workflow_utils.py b/workflows/workflow_utils.py
index a9bf6a6..5a1c3f2 100644
--- a/workflows/workflow_utils.py
+++ b/workflows/workflow_utils.py
@@ -46,35 +46,27 @@ def safe_json_extract(text: str) -> Optional[Any]:
from workflows.history_manager import parse_history, build_history_str, filter_image_urls, init_history # noqa: F401
+
import re
+
def normalize_latex_spacing(text: str) -> str:
"""
规范 Markdown 中的 LaTeX 公式。
- 规则：
- 1. 行内公式 $...$
- - 清除公式定界符内部首尾空格
- - 公式前后只要存在非空白字符，就补一个空格
- - ($T$) 会转换为 ( $T$ )
+ 处理：
+ - $...$：行内公式，清理内侧空格并补充正文间空格
+ - $$...$$：段落公式，完整保留开始和结束定界符
+ - \\(...\\)：转换为 $...$
+ - \\[...\\]：转换为 $$...$$
- 2. 段落公式 $$...$$
- - 多行公式完整保留，不会删除结尾的 $$
- - 单行 $$公式$$ 转换为标准多行段落公式
- - Markdown 表格中的 $$公式$$ 不拆行
-
- 3. 转换：
- - \\(...\\) 转换为 $...$
- - \\[...\\] 转换为 $$...$$
-
- 4. 不处理：
- - Markdown 围栏代码块
- - Markdown 行内代码
- - HTML code、pre、script、style
- - HTML 注释
- - Markdown 链接地址
- - 转义美元符号 \\$
+ 不修改：
+ - Markdown 围栏代码块
+ - Markdown 行内代码
+ - HTML code/pre/script/style
+ - HTML 注释
+ - 转义美元符号 \\$
"""
if not isinstance(text, str):
raise TypeError(
@@ -101,13 +93,9 @@ def normalize_latex_spacing(text: str) -> str:
for index, original in enumerate(protected_parts):
token = f"\uE000LATEX_PROTECTED_{index}\uE001"
content = content.replace(token, original)
-
return content
def is_escaped(content: str, position: int) -> bool:
- """
- 判断 content[position] 是否被奇数个反斜杠转义。
- """
slash_count = 0
position -= 1
@@ -118,27 +106,27 @@ def normalize_latex_spacing(text: str) -> str:
return slash_count % 2 == 1
# ================================================================
- # 1. 保护 Markdown 围栏代码块
+ # 保护 Markdown 围栏代码块
# ================================================================
lines = text.splitlines(keepends=True)
- protected_lines: list[str] = []
+ output_lines: list[str] = []
line_index = 0
while line_index < len(lines):
line = lines[line_index]
- opening_match = re.match(
+ fence_match = re.match(
r"^[ \t]{0,3}(`{3,}|~{3,})",
line,
)
- if not opening_match:
- protected_lines.append(line)
+ if not fence_match:
+ output_lines.append(line)
line_index += 1
continue
- opening_fence = opening_match.group(1)
+ opening_fence = fence_match.group(1)
fence_character = opening_fence[0]
fence_length = len(opening_fence)
@@ -160,14 +148,14 @@ def normalize_latex_spacing(text: str) -> str:
if closing_pattern.match(current_line):
break
- protected_lines.append(
+ output_lines.append(
protect("".join(block_lines))
)
- text = "".join(protected_lines)
+ text = "".join(output_lines)
# ================================================================
- # 2. 保护 HTML 代码区域和注释
+ # 保护 HTML 代码区域和注释
# ================================================================
text = re.sub(
@@ -186,7 +174,7 @@ def normalize_latex_spacing(text: str) -> str:
)
# ================================================================
- # 3. 保护 Markdown 链接地址
+ # 保护 Markdown 链接地址
# ================================================================
text = re.sub(
@@ -199,27 +187,16 @@ def normalize_latex_spacing(text: str) -> str:
text,
)
- text = re.sub(
- r"(?im)"
- r"^([ \t]{0,3}\[[^\]\n]+\]:[ \t]*)"
- r"(\S+(?:[ \t]+.*)?)$",
- lambda match: (
- match.group(1)
- + protect(match.group(2))
- ),
- text,
- )
-
# ================================================================
- # 4. 保护 Markdown 行内代码
+ # 保护 Markdown 行内代码
# ================================================================
- inline_code_result: list[str] = []
+ inline_code_output: list[str] = []
position = 0
while position < len(text):
if text[position] != "`":
- inline_code_result.append(text[position])
+ inline_code_output.append(text[position])
position += 1
continue
@@ -250,11 +227,11 @@ def normalize_latex_spacing(text: str) -> str:
else ""
)
- candidate_end = candidate + len(delimiter)
+ next_position = candidate + len(delimiter)
next_character = (
- text[candidate_end]
- if candidate_end < len(text)
+ text[next_position]
+ if next_position < len(text)
else ""
)
@@ -268,22 +245,22 @@ def normalize_latex_spacing(text: str) -> str:
search_position = candidate + 1
if closing_position < 0:
- inline_code_result.append(delimiter)
+ inline_code_output.append(delimiter)
position = delimiter_end
continue
end_position = closing_position + len(delimiter)
- inline_code_result.append(
+ inline_code_output.append(
protect(text[position:end_position])
)
position = end_position
- text = "".join(inline_code_result)
+ text = "".join(inline_code_output)
# ================================================================
- # 5. 转换其他 LaTeX 公式定界符
+ # 转换其他 LaTeX 定界符
# ================================================================
# \( x + y \) -> $x + y$
@@ -305,166 +282,78 @@ def normalize_latex_spacing(text: str) -> str:
)
# ================================================================
- # 6. 完整提取并保护 $$...$$ 段落公式
+ # 扫描公式
# ================================================================
- display_result: list[str] = []
+ opening_punctuation = set(
+ "([{（【《〈“‘"
+ )
+
+ closing_punctuation = set(
+ ")]}）】》〉”’、，。；：！？,.!?;:"
+ )
+
+ result: list[str] = []
position = 0
while position < len(text):
- if not (
+ # ------------------------------------------------------------
+ # 段落公式 $$...$$
+ # ------------------------------------------------------------
+
+ if (
text.startswith("$$", position)
and not is_escaped(text, position)
):
- display_result.append(text[position])
- position += 1
- continue
+ search_position = position + 2
+ closing_position = -1
- opening_position = position
- search_position = position + 2
- closing_position = -1
-
- while search_position < len(text):
- candidate = text.find(
- "$$",
- search_position,
- )
-
- if candidate < 0:
- break
-
- if not is_escaped(text, candidate):
- closing_position = candidate
- break
-
- search_position = candidate + 2
-
- # 未找到闭合 $$，剩余内容原样保留。
- if closing_position < 0:
- display_result.append(
- text[opening_position:]
- )
- position = len(text)
- break
-
- complete_formula = text[
- opening_position:closing_position + 2
- ]
-
- formula_body = text[
- opening_position + 2:closing_position
- ]
-
- # ------------------------------------------------------------
- # 已经是多行段落公式时，完整原样保留。
- #
- # $$
- # R = K
- # $$
- #
- # 不拆解、不重建，避免结尾 $$ 丢失。
- # ------------------------------------------------------------
-
- if "\n" in complete_formula:
- display_result.append(
- protect(complete_formula)
- )
- position = closing_position + 2
- continue
-
- # ------------------------------------------------------------
- # 判断公式是否位于 Markdown 表格内。
- # 表格中的公式不能强制拆成多行。
- # ------------------------------------------------------------
-
- line_start = text.rfind(
- "\n",
- 0,
- opening_position,
- ) + 1
-
- line_end = text.find(
- "\n",
- closing_position + 2,
- )
-
- if line_end < 0:
- line_end = len(text)
-
- current_line = text[line_start:line_end]
- clean_body = formula_body.strip()
-
- if "|" in current_line:
- display_result.append(
- protect(
- "$$" + clean_body + "$$"
+ while search_position < len(text):
+ candidate = text.find(
+ "$$",
+ search_position,
+ )
+
+ if candidate < 0:
+ break
+
+ if not is_escaped(text, candidate):
+ closing_position = candidate
+ break
+
+ search_position = candidate + 2
+
+ # 未闭合时，原样保留剩余内容。
+ if closing_position < 0:
+ result.append(text[position:])
+ break
+
+ complete_formula = text[
+ position:closing_position + 2
+ ]
+
+ formula_body = text[
+ position + 2:closing_position
+ ]
+
+ # 多行段落公式完整原样保留。
+ if "\n" in complete_formula:
+ result.append(complete_formula)
+ else:
+ # 单行段落公式只清理内侧空格，
+ # 不删除或重建结束的 $$。
+ result.append(
+ "$$" + formula_body.strip() + "$$"
)
- )
position = closing_position + 2
continue
# ------------------------------------------------------------
- # 单行段落公式转换成标准多行格式。
- #
- # $$R = K$$
- #
- # 转换为：
- #
- # $$
- # R = K
- # $$
+ # 行内公式 $...$
# ------------------------------------------------------------
- normalized_formula = (
- "$$\n"
- + clean_body
- + "\n$$"
- )
-
- before_text = "".join(display_result)
-
- # 公式前存在普通正文时，确保前面有空行。
- if before_text and not before_text.endswith("\n\n"):
- before_text = before_text.rstrip(" \t\n") + "\n\n"
-
- display_result = [before_text] if before_text else []
-
- display_result.append(
- protect(normalized_formula)
- )
-
- position = closing_position + 2
-
- # 清理公式后已有的普通空格。
- while (
- position < len(text)
- and text[position] in " \t"
- ):
- position += 1
-
- # 后面还有内容时，确保公式后有空行。
- if position < len(text):
- if text[position] == "\n":
- while (
- position < len(text)
- and text[position] == "\n"
- ):
- position += 1
-
- display_result.append("\n\n")
-
- text = "".join(display_result)
-
- # ================================================================
- # 7. 规范 $...$ 行内公式
- # ================================================================
-
- inline_result: list[str] = []
- position = 0
-
- while position < len(text):
- is_inline_opening = (
+ if (
text[position] == "$"
and not is_escaped(text, position)
and not text.startswith("$$", position)
@@ -472,127 +361,111 @@ def normalize_latex_spacing(text: str) -> str:
position > 0
and text[position - 1] == "$"
)
- )
+ ):
+ opening_position = position
+ search_position = position + 1
+ closing_position = -1
- if not is_inline_opening:
- inline_result.append(text[position])
- position += 1
- continue
+ while search_position < len(text):
+ candidate = text.find(
+ "$",
+ search_position,
+ )
- opening_position = position
+ if candidate < 0:
+ break
- # 避免把 $100 识别成行内公式开始。
- next_opening_character = (
- text[position + 1]
- if position + 1 < len(text)
- else ""
- )
+ # 行内公式不允许跨行。
+ if "\n" in text[
+ opening_position + 1:candidate
+ ]:
+ break
- if next_opening_character.isdigit():
- inline_result.append("$")
- position += 1
- continue
+ if is_escaped(text, candidate):
+ search_position = candidate + 1
+ continue
- search_position = position + 1
- closing_position = -1
+ if text.startswith("$$", candidate):
+ search_position = candidate + 2
+ continue
- while search_position < len(text):
- candidate = text.find(
- "$",
- search_position,
- )
+ next_character = (
+ text[candidate + 1]
+ if candidate + 1 < len(text)
+ else ""
+ )
- if candidate < 0:
+ # 避免把 $100 和 $200 配对成公式。
+ if next_character.isdigit():
+ search_position = candidate + 1
+ continue
+
+ closing_position = candidate
break
- # 行内公式不能跨行。
- if "\n" in text[
- opening_position + 1:candidate
- ]:
- break
-
- if is_escaped(text, candidate):
- search_position = candidate + 1
+ if closing_position < 0:
+ result.append("$")
+ position += 1
continue
- # 跳过属于 $$ 的美元符号。
- if text.startswith("$$", candidate):
- search_position = candidate + 2
+ formula_body = text[
+ opening_position + 1:closing_position
+ ].strip()
+
+ if not formula_body:
+ result.append(
+ text[
+ opening_position:
+ closing_position + 1
+ ]
+ )
+ position = closing_position + 1
continue
- closing_position = candidate
- break
-
- # 未找到闭合符时原样保留。
- if closing_position < 0:
- inline_result.append("$")
- position += 1
- continue
-
- formula_body = text[
- opening_position + 1:closing_position
- ].strip()
-
- # 空公式不处理。
- if not formula_body:
- inline_result.append(
- text[
- opening_position:closing_position + 1
- ]
+ formula_body = re.sub(
+ r"[ \t]+",
+ " ",
+ formula_body,
)
+ previous_character = ""
+
+ if result and result[-1]:
+ previous_character = result[-1][-1]
+
+ next_character = (
+ text[closing_position + 1]
+ if closing_position + 1 < len(text)
+ else ""
+ )
+
+ if (
+ previous_character
+ and not previous_character.isspace()
+ and previous_character
+ not in opening_punctuation
+ ):
+ result.append(" ")
+
+ result.append(
+ "$" + formula_body + "$"
+ )
+
+ if (
+ next_character
+ and not next_character.isspace()
+ and next_character
+ not in closing_punctuation
+ ):
+ result.append(" ")
+
position = closing_position + 1
continue
- # 折叠公式内部普通空格，不修改 LaTeX 命令。
- formula_body = re.sub(
- r"[ \t]+",
- " ",
- formula_body,
- )
+ result.append(text[position])
+ position += 1
- previous_character = ""
-
- for part in reversed(inline_result):
- if part:
- previous_character = part[-1]
- break
-
- next_character = (
- text[closing_position + 1]
- if closing_position + 1 < len(text)
- else ""
- )
-
- # 公式前只要存在非空白字符，就补空格。
- #
- # 参数$T$ -> 参数 $T$
- # ($T$) -> ( $T$
- # （$T$） -> （ $T$
- if (
- previous_character
- and not previous_character.isspace()
- ):
- inline_result.append(" ")
-
- inline_result.append(
- "$" + formula_body + "$"
- )
-
- # 公式后只要存在非空白字符，就补空格。
- #
- # $T$参数 -> $T$ 参数
- # ($T$) -> $T$ )
- # （$T$） -> $T$ ）
- if (
- next_character
- and not next_character.isspace()
- ):
- inline_result.append(" ")
-
- position = closing_position + 1
-
- return restore("".join(inline_result))
+ return restore("".join(result))
def remove_duplicate_content(text: str) -> str:
@@ -605,18 +478,13 @@ def remove_duplicate_content(text: str) -> str:
- equation、align、gather 等 LaTeX 环境
- Markdown 围栏代码块
- Markdown 缩进代码块
- - Markdown 标题、列表、引用、表格等结构行
+ - 标题、列表、引用、表格等 Markdown 结构行
这样不会删除段落公式末尾的第二个 $$。
"""
if not text or len(text) < 10:
return text
- if not isinstance(text, str):
- raise TypeError(
- f"text 必须是 str，实际类型为 {type(text).__name__}"
- )
-
text = text.replace("\r\n", "\n").replace("\r", "\n")
lines = text.split("\n")
@@ -662,9 +530,6 @@ def remove_duplicate_content(text: str) -> str:
return slash_count % 2 == 1
def count_unescaped_double_dollars(value: str) -> int:
- """
- 统计一行中未转义的 $$ 数量。
- """
count = 0
position = 0
@@ -681,16 +546,13 @@ def remove_duplicate_content(text: str) -> str:
return count
def is_markdown_structure_line(value: str) -> bool:
- """
- Markdown 结构行不参与去重，避免破坏语义。
- """
stripped = value.strip()
if not stripped:
return True
patterns = (
- # 标题
+ # Markdown 标题
r"^#{1,6}\s+",
# 无序列表
@@ -708,7 +570,7 @@ def remove_duplicate_content(text: str) -> str:
# 水平分隔线
r"^(?:-{3,}|\*{3,}|_{3,})$",
- # 表格分隔行
+ # Markdown 表格分隔行
r"^\|?\s*:?-{3,}:?"
r"(?:\s*\|\s*:?-{3,}:?)+\s*\|?$",
@@ -738,7 +600,7 @@ def remove_duplicate_content(text: str) -> str:
stripped = line.strip()
# ============================================================
- # 1. Markdown 围栏代码块
+ # Markdown 围栏代码块
# ============================================================
fence_match = re.match(
@@ -773,13 +635,13 @@ def remove_duplicate_content(text: str) -> str:
continue
- # Markdown 缩进代码块不参与去重。
+ # Markdown 缩进代码块。
if re.match(r"^(?:\t| {4})", line):
result.append(line)
continue
# ============================================================
- # 2. \[ ... \] 段落公式
+ # \[ ... \] 段落公式
# ============================================================
if in_bracket_math:
@@ -807,7 +669,7 @@ def remove_duplicate_content(text: str) -> str:
continue
# ============================================================
- # 3. LaTeX display 环境
+ # LaTeX display 环境
# ============================================================
begin_matches = re.findall(
@@ -869,7 +731,7 @@ def remove_duplicate_content(text: str) -> str:
continue
# ============================================================
- # 4. $$ ... $$ 段落公式
+ # $$ ... $$ 段落公式
# ============================================================
double_dollar_count = (
@@ -877,7 +739,7 @@ def remove_duplicate_content(text: str) -> str:
)
if in_dollar_math:
- # 公式块中的内容全部保留，包括最后一行的 $$。
+ # 公式内部和结束的 $$ 全部原样保留。
result.append(line)
if double_dollar_count % 2 == 1:
@@ -886,17 +748,17 @@ def remove_duplicate_content(text: str) -> str:
continue
if double_dollar_count > 0:
- # 含有 $$ 的行永远不参与去重。
+ # 包含 $$ 的行不参与去重。
result.append(line)
- # 奇数个 $$ 代表开启了跨行段落公式。
+ # 奇数个 $$ 表示开启了跨行公式。
if double_dollar_count % 2 == 1:
in_dollar_math = True
continue
# ============================================================
- # 5. 空行和 Markdown 结构行
+ # 空行和 Markdown 结构行
# ============================================================
if not stripped:
@@ -908,7 +770,7 @@ def remove_duplicate_content(text: str) -> str:
continue
# ============================================================
- # 6. 仅对普通正文行去重
+ # 仅对普通正文行去重
# ============================================================
normalized_line = re.sub(
@@ -931,6 +793,41 @@ def remove_duplicate_content(text: str) -> str:
# text = remove_duplicate_content(text)
# text = normalize_latex_spacing(text)
+#def normalize_latex_spacing(text: str) -> str:
+# placeholder = "\x00DOUBLE_DOLLAR\x00"
+# text = text.replace("$$", placeholder)
+# text = re.sub(r'(? str:
+# if not text or len(text) < 10:
+# return text
+#
+# lines = text.split('\n')
+# result = []
+# seen_segments = set()
+#
+# for line in lines:
+# line = line.rstrip()
+# if not line:
+# result.append(line)
+# continue
+#
+# normalized_line = line.strip().lower()
+# if normalized_line in seen_segments:
+# continue
+#
+# seen_segments.add(normalized_line)
+# result.append(line)
+#
+# cleaned_text = '\n'.join(result)
+# return cleaned_text
+
def _extract_rag_text(item: Any) -> str:
if isinstance(item, dict):
diff --git a/workflows/workflow_wiki_admin.py b/workflows/workflow_wiki_admin.py
new file mode 100644
index 0000000..76c6b4b
--- /dev/null
+++ b/workflows/workflow_wiki_admin.py
@@ -0,0 +1,493 @@
+import json
+import re
+from typing import Any, Dict, List, Optional
+
+from wiki_engine import service
+
+
+CONFIRM_WORDS = ("确认", "执行", "开始", "确定", "同意", "dry_run=false", "不是预览")
+CANCEL_WORDS = ("取消", "先不", "不要执行", "作废")
+
+
+async def run_wiki_admin_workflow(
+ extracted_text: str,
+ combined_query: str,
+ history_message: str = "",
+ route_flag: str = "",
+ chat_id: str = "",
+ message_id: str = "",
+ **_: Any,
+) -> Dict[str, Any]:
+ query = (combined_query or extracted_text or "").strip()
+ if not query:
+ return _response("请告诉我要管理哪个知识库或文件，例如：检查知识库 163舰 的 Wiki 同步情况。")
+
+ try:
+ result = await _dispatch(query=query, chat_id=chat_id)
+ return _response(_format_result(result), result=result)
+ except Exception as exc:
+ return _response(f"Wiki 管理操作失败：{exc}", error=str(exc))
+
+
+async def _dispatch(query: str, chat_id: str = "") -> Dict[str, Any]:
+ pending_plan = await service.get_pending_admin_plan(chat_id) if chat_id else None
+ confirmed = _is_confirmed(query)
+ selected_indexes = _extract_selection_indexes(query)
+ job_id = _extract_job_id(query)
+
+ if job_id and any(word in query for word in ("进度", "状态", "查看", "job", "任务", "同步")):
+ job = await service.get_sync_job(job_id)
+ if not job:
+ return {"operation": "job_progress", "data": {"success": False, "job_id": job_id, "message": "未找到这个同步任务。"}}
+ return {"operation": "job_progress", "data": {"success": True, "job": job}}
+
+ if pending_plan and _is_cancel(query):
+ cancelled = await service.cancel_admin_plan(str(pending_plan.get("id") or ""))
+ return {"operation": "plan_cancelled", "data": {"success": True, "plan": cancelled or pending_plan}}
+
+ if pending_plan and confirmed and not _contains_new_target(query):
+ data = await service.execute_admin_plan(pending_plan, selected_indexes=selected_indexes)
+ data["plan"] = pending_plan
+ return {"operation": "plan_executed", "data": data}
+
+ kb_id = _extract_kb_id(query)
+ file_id = _extract_id(query, ("文件", "file", "file_id"))
+ include_children = any(word in query for word in ("子知识库", "子库", "包含子", "包括子"))
+ use_llm = any(word.lower() in query.lower() for word in ("大模型", "llm", "模型摘要", "智能摘要"))
+ limit = _extract_limit(query) or 20
+
+ if any(word in query for word in ("统计", "状态", "概览", "数量")) and not kb_id and not file_id:
+ return {"operation": "stats", "data": {"success": True, "stats": await service.get_stats()}}
+
+ if any(word in query for word in ("检查", "对比", "差异", "巡检", "同步情况")):
+ target = await _resolve_kb_target(query, kb_id)
+ if not target.get("success"):
+ return {"operation": "need_kb_name", "data": target}
+ diff = await service.diff_kb(kb_id=target["kb_id"], include_children=include_children, limit=limit)
+ diff.update({"kb_name": target.get("kb_name", ""), "kb_path": target.get("kb_path", "")})
+ actions = _sync_actions_from_diff(diff, use_llm=use_llm)
+ plan = await _save_plan(
+ chat_id=chat_id,
+ intent="sync_after_check",
+ scope="kb",
+ target=target,
+ summary=diff.get("summary", {}),
+ actions=actions,
+ metadata={"source_operation": "diff_kb", "include_children": include_children, "limit": limit},
+ )
+ diff["plan"] = plan
+ diff["planned_actions"] = actions
+ return {"operation": "diff_kb", "data": diff}
+
+ if any(word in query for word in ("清理", "修复", "对齐", "同步当前", "reconcile")):
+ target = await _resolve_kb_target(query, kb_id)
+ if not target.get("success"):
+ return {"operation": "need_kb_name", "data": target}
+ delete_orphan = any(word in query for word in ("删除孤儿", "删除脏数据", "清理脏数据", "已删除", "不存在", "孤儿"))
+ preview = await service.reconcile_kb(
+ kb_id=target["kb_id"],
+ include_children=include_children,
+ limit=limit,
+ sync_missing=True,
+ rebuild_stale=True,
+ delete_orphan=delete_orphan,
+ use_llm=use_llm,
+ dry_run=True,
+ )
+ preview.update({"kb_name": target.get("kb_name", ""), "kb_path": target.get("kb_path", "")})
+ return await _plan_or_execute(
+ chat_id=chat_id,
+ confirmed=confirmed,
+ intent="reconcile_kb",
+ scope="kb",
+ target=target,
+ summary=preview.get("summary", {}),
+ actions=preview.get("planned_actions", []),
+ metadata={"delete_orphan": delete_orphan, "include_children": include_children, "limit": limit},
+ preview=preview,
+ )
+
+ if any(word in query for word in ("删除", "移除")):
+ if file_id:
+ action = {"action": "delete_wiki_file", "file_id": file_id}
+ return await _plan_or_execute(
+ chat_id=chat_id,
+ confirmed=confirmed,
+ intent="delete_wiki_file",
+ scope="file",
+ target={"kb_id": kb_id or "", "kb_name": "", "kb_path": ""},
+ summary={"delete_files": 1},
+ actions=[action],
+ metadata={"destructive": True},
+ preview={"success": True, "planned_actions": [action]},
+ )
+ target = await _resolve_kb_target(query, kb_id)
+ if target.get("success"):
+ action = {"action": "delete_wiki_kb", "kb_id": target["kb_id"], "kb_name": target.get("kb_name", "")}
+ return await _plan_or_execute(
+ chat_id=chat_id,
+ confirmed=confirmed,
+ intent="delete_wiki_kb",
+ scope="kb",
+ target=target,
+ summary={"delete_kb": 1},
+ actions=[action],
+ metadata={"destructive": True},
+ preview={"success": True, "planned_actions": [action], **target},
+ )
+ return {"operation": "need_target", "data": {"success": False, "message": "删除 Wiki 数据需要提供文件 id，或说清楚知识库名称。", **target}}
+
+ if any(word in query for word in ("同步", "构建", "建立", "重建", "生成")):
+ if file_id:
+ action = {"action": "sync_file", "file_id": file_id, "kb_id": kb_id or "", "use_llm": use_llm}
+ return await _plan_or_execute(
+ chat_id=chat_id,
+ confirmed=confirmed,
+ intent="sync_file",
+ scope="file",
+ target={"kb_id": kb_id or "", "kb_name": "", "kb_path": ""},
+ summary={"sync_files": 1},
+ actions=[action],
+ metadata={"use_llm": use_llm},
+ preview={"success": True, "planned_actions": [action]},
+ )
+ target = await _resolve_kb_target(query, kb_id)
+ if target.get("success"):
+ preview = await service.reconcile_kb(
+ kb_id=target["kb_id"],
+ include_children=include_children,
+ limit=limit,
+ sync_missing=True,
+ rebuild_stale=True,
+ delete_orphan=False,
+ use_llm=use_llm,
+ dry_run=True,
+ )
+ preview.update({"kb_name": target.get("kb_name", ""), "kb_path": target.get("kb_path", "")})
+ return await _plan_or_execute(
+ chat_id=chat_id,
+ confirmed=confirmed,
+ intent="sync_kb",
+ scope="kb",
+ target=target,
+ summary=preview.get("summary", {}),
+ actions=preview.get("planned_actions", []),
+ metadata={"include_children": include_children, "limit": limit, "use_llm": use_llm},
+ preview=preview,
+ )
+ return {"operation": "need_target", "data": {"success": False, "message": "同步 Wiki 需要提供文件 id，或说清楚知识库名称。", **target}}
+
+ if pending_plan:
+ return {"operation": "pending_plan", "data": {"success": True, "plan": pending_plan}}
+
+ return {
+ "operation": "help",
+ "data": {
+ "success": True,
+ "examples": [
+ "检查知识库 163舰 的 Wiki 同步情况",
+ "同步知识库 163舰 的前 50 个文件",
+ "清理知识库 163舰 中 11000 已删除但 Wiki 还存在的数据",
+ "确认执行",
+ "只执行第 1、3 个",
+ ],
+ },
+ }
+
+
+async def _plan_or_execute(
+ chat_id: str,
+ confirmed: bool,
+ intent: str,
+ scope: str,
+ target: Dict[str, Any],
+ summary: Dict[str, Any],
+ actions: List[Dict[str, Any]],
+ metadata: Dict[str, Any],
+ preview: Dict[str, Any],
+) -> Dict[str, Any]:
+ plan = await _save_plan(chat_id, intent, scope, target, summary, actions, metadata)
+ preview["plan"] = plan
+ preview["planned_actions"] = actions
+ if confirmed and actions:
+ if not plan:
+ return {"operation": "plan_preview", "data": preview}
+ executed = await service.execute_admin_plan(plan)
+ executed["plan"] = plan
+ return {"operation": "plan_executed", "data": executed}
+ return {"operation": "plan_preview", "data": preview}
+
+
+async def _save_plan(
+ chat_id: str,
+ intent: str,
+ scope: str,
+ target: Dict[str, Any],
+ summary: Dict[str, Any],
+ actions: List[Dict[str, Any]],
+ metadata: Dict[str, Any],
+) -> Dict[str, Any]:
+ if not chat_id or not actions:
+ return {}
+ return await service.create_admin_plan(
+ chat_id=chat_id,
+ intent=intent,
+ scope=scope,
+ kb_id=str(target.get("kb_id") or ""),
+ kb_name=str(target.get("kb_name") or target.get("kb_path") or ""),
+ summary=summary,
+ actions=actions,
+ metadata=metadata,
+ )
+
+
+def _sync_actions_from_diff(diff: Dict[str, Any], use_llm: bool = False) -> List[Dict[str, Any]]:
+ actions: List[Dict[str, Any]] = []
+ for item in diff.get("missing_in_wiki", []):
+ actions.append({
+ "action": "sync_file",
+ "file_id": item.get("file_id", ""),
+ "kb_id": item.get("kb_id") or diff.get("kb_id", ""),
+ "filename": item.get("filename", ""),
+ "use_llm": use_llm,
+ })
+ for item in diff.get("stale_in_wiki", []):
+ actions.append({
+ "action": "rebuild_file",
+ "file_id": item.get("file_id", ""),
+ "kb_id": item.get("kb_id") or diff.get("kb_id", ""),
+ "filename": item.get("filename", ""),
+ "use_llm": use_llm,
+ })
+ return actions
+
+
+def _format_result(result: Dict[str, Any]) -> str:
+ op = result.get("operation", "")
+ data = result.get("data", {})
+ if op == "help":
+ return "我是 Wiki 管理助手，可以检查、生成计划、同步、清理和删除 Wiki 指针数据。\n\n示例：\n" + "\n".join(f"- {x}" for x in data.get("examples", []))
+ if op in {"need_kb_id", "need_kb_name", "need_target"}:
+ matches = data.get("matches") or data.get("candidates") or []
+ if matches:
+ options = "\n".join(
+ f"- {item.get('path') or item.get('name')}（id: {item.get('kb_id')}）"
+ for item in matches[:5]
+ )
+ return f"{data.get('message') or data.get('error') or '请说清楚知识库名称。'}\n可选知识库：\n{options}"
+ return data.get("message", "请补充必要参数。")
+ if op == "stats":
+ stats = data.get("stats", {})
+ return "当前 Wiki 数据统计：\n" + "\n".join(f"- {k}: {v}" for k, v in stats.items())
+ if op == "job_progress":
+ if not data.get("success"):
+ return data.get("message") or "未找到同步任务。"
+ return _format_job_progress(data.get("job") or {})
+ if op == "diff_kb":
+ summary = data.get("summary", {})
+ target = data.get("kb_path") or data.get("kb_name") or data.get("kb_id")
+ lines = [
+ f"知识库 {target} 的 Wiki 同步检查完成。",
+ f"- 11000 可用文件数：{summary.get('htknow_files', 0)}",
+ f"- Wiki 已同步文件数：{summary.get('wiki_files', 0)}",
+ f"- 11000 有但 Wiki 缺失：{summary.get('missing_in_wiki', 0)}",
+ f"- Wiki 已过期需重建：{summary.get('stale_in_wiki', 0)}",
+ f"- Wiki 有但 11000 不存在：{summary.get('orphan_in_wiki', 0)}",
+ f"- 跳过未解析文件：{summary.get('skipped_unparsed', 0)}",
+ ]
+ lines.extend(_format_plan_tail(data.get("plan") or {}, data.get("planned_actions") or []))
+ return "\n".join(lines)
+ if op in {"plan_preview", "pending_plan"}:
+ plan = data.get("plan") or {}
+ actions = data.get("planned_actions") or plan.get("actions") or []
+ target = data.get("kb_path") or data.get("kb_name") or plan.get("kb_name") or plan.get("kb_id") or "当前目标"
+ lines = [f"已生成 {target} 的操作计划。", f"- 计划编号：{plan.get('id') or '未保存'}", f"- 待执行动作：{len(actions)}"]
+ lines.extend(_format_actions(actions))
+ if plan:
+ lines.append("确认无误后，请回复“确认执行”；也可以说“只执行第 1、3 个”或“取消”。")
+ else:
+ lines.append("当前请求没有 chat_id，计划未落库；需要前端带 chat_id 才能多轮确认执行。")
+ return "\n".join(lines)
+ if op == "plan_executed":
+ plan = data.get("plan") or {}
+ lines = [
+ f"已提交 Wiki 同步计划 {plan.get('id') or ''}。",
+ f"- 已提交任务：{data.get('executed_count', 0)}",
+ f"- 提交失败：{data.get('failed_count', 0)}",
+ ]
+ jobs = _extract_executed_jobs(data.get("executed") or [])
+ if jobs:
+ lines.append("同步正在后台执行，可继续使用系统。当前任务：")
+ for job in jobs[:8]:
+ lines.append(f"- job_id: {job.get('id') or job.get('job_id')}，进度：{_job_progress_text(job)}")
+ if len(jobs) > 8:
+ lines.append(f"- 还有 {len(jobs) - 8} 个后台任务")
+ lines.append("想看进度时，可以问：查看同步进度 ")
+ return "\n".join(lines)
+ if op == "plan_cancelled":
+ plan = data.get("plan") or {}
+ return f"已取消待执行计划 {plan.get('id') or ''}。"
+ return json.dumps(data, ensure_ascii=False, indent=2)
+
+
+def _format_plan_tail(plan: Dict[str, Any], actions: List[Dict[str, Any]]) -> List[str]:
+ if not actions:
+ return ["当前没有需要同步或重建的 Wiki 动作。"]
+ lines = [f"已为缺失/过期数据生成待确认计划：{plan.get('id') or '未保存'}", f"- 待同步/重建动作：{len(actions)}"]
+ lines.extend(_format_actions(actions))
+ if plan:
+ lines.append("如果要执行，请回复“确认执行”；如果要清理孤儿数据，请明确说“清理孤儿数据”。")
+ return lines
+
+
+def _format_actions(actions: List[Dict[str, Any]], max_items: int = 8) -> List[str]:
+ lines = []
+ for idx, action in enumerate(actions[:max_items], start=1):
+ lines.append(f" {idx}. {_action_label(action)}")
+ if len(actions) > max_items:
+ lines.append(f" ... 还有 {len(actions) - max_items} 个动作")
+ return lines
+
+
+def _action_label(action: Dict[str, Any]) -> str:
+ action_type = action.get("action", "")
+ if action_type == "sync_file":
+ return f"同步文件 {action.get('filename') or action.get('file_id')}"
+ if action_type == "rebuild_file":
+ return f"重建文件 {action.get('filename') or action.get('file_id')}"
+ if action_type == "delete_wiki_file":
+ return f"删除 Wiki 中的文件数据 {action.get('file_id')}"
+ if action_type == "delete_wiki_kb":
+ return f"删除 Wiki 中的知识库数据 {action.get('kb_name') or action.get('kb_id')}"
+ return json.dumps(action, ensure_ascii=False)
+
+
+def _extract_executed_jobs(executed: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ jobs: List[Dict[str, Any]] = []
+ for item in executed:
+ result = item.get("result") or {}
+ job = result.get("job") if isinstance(result.get("job"), dict) else {}
+ if job:
+ jobs.append(job)
+ elif result.get("job_id"):
+ jobs.append({"id": result.get("job_id"), "status": result.get("status") or "running"})
+ return jobs
+
+
+def _format_job_progress(job: Dict[str, Any]) -> str:
+ current = str(job.get("current_filename") or job.get("current_file_id") or "").strip()
+ lines = [
+ f"同步任务 {job.get('id') or ''}",
+ f"- 状态：{job.get('status') or ''}",
+ f"- 目标：{job.get('target_type') or ''} {job.get('target_id') or ''}",
+ f"- 当前阶段：{job.get('stage') or '处理中'}",
+ ]
+ if current:
+ lines.append(f"- 当前文件：{current}")
+ lines.extend([
+ f"- 进度：{_job_progress_text(job)}",
+ f"- 已处理文件：{job.get('processed_files', 0)} / {job.get('total_files', 0)}",
+ f"- 已跳过未变化文件：{job.get('skipped_files', 0)}",
+ f"- 已同步切片：{job.get('total_slices', 0)}",
+ f"- 已生成/更新页面：{job.get('page_count', 0)}",
+ f"- 更新时间：{job.get('updated_at') or ''}",
+ f"- 错误：{job.get('error') or '无'}",
+ ])
+ return "\n".join(lines)
+
+
+def _job_progress_text(job: Dict[str, Any]) -> str:
+ total = int(job.get("total_files") or 0)
+ processed = int(job.get("processed_files") or 0)
+ status = str(job.get("status") or "")
+ if total > 0:
+ percent = min(100.0, max(0.0, processed * 100.0 / total))
+ return f"{processed}/{total}（{percent:.0f}%），状态：{status}"
+ return f"等待统计文件数，状态：{status}"
+
+
+def _response(text: str, result: Optional[Dict[str, Any]] = None, error: str = "") -> Dict[str, Any]:
+ return {
+ "response": text,
+ "actions": [],
+ "result_tag": "wiki_admin",
+ "suggestedReplies": [
+ {"title": "检查同步情况", "content": "检查知识库 163舰 的 Wiki 同步情况"},
+ {"title": "确认执行", "content": "确认执行"},
+ ],
+ "route_flag": "wiki_admin",
+ "wiki_admin_result": result or {},
+ "error_message": error,
+ }
+
+
+def _is_confirmed(query: str) -> bool:
+ return any(word in query for word in CONFIRM_WORDS)
+
+
+def _is_cancel(query: str) -> bool:
+ return any(word in query for word in CANCEL_WORDS)
+
+
+def _contains_new_target(query: str) -> bool:
+ return any(word in query for word in ("知识库", "文件", "file_id", "kb_id"))
+
+
+def _extract_selection_indexes(query: str) -> Optional[List[int]]:
+ if not any(word in query for word in ("第", "只", "选择", "操作", "执行")):
+ return None
+ nums = [int(x) for x in re.findall(r"\d+", query)]
+ return nums or None
+
+
+def _extract_job_id(query: str) -> Optional[str]:
+ match = re.search(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b", query or "")
+ return match.group(0) if match else None
+
+
+def _extract_id(query: str, labels: tuple[str, ...]) -> Optional[str]:
+ for label in labels:
+ patterns = [
+ rf"{re.escape(label)}\s*[:：]?\s*([A-Za-z0-9_\-]+)",
+ rf"([A-Za-z0-9_\-]+)\s*号?{re.escape(label)}",
+ ]
+ for pattern in patterns:
+ match = re.search(pattern, query, flags=re.IGNORECASE)
+ if match:
+ return match.group(1)
+ return None
+
+
+def _extract_kb_id(query: str) -> Optional[str]:
+ patterns = [
+ r"(?:kb_id|kb|知识库\s*(?:id|ID|编号))\s*[:：=]?\s*([A-Za-z0-9_\-]+)",
+ r"知识库\s*[:：]?\s*(\d+)(?![A-Za-z0-9_\-])",
+ r"(\d+)\s*号?\s*知识库",
+ ]
+ for pattern in patterns:
+ match = re.search(pattern, query, flags=re.IGNORECASE)
+ if match:
+ return match.group(1)
+ return None
+
+
+async def _resolve_kb_target(query: str, kb_id: Optional[str]) -> Dict[str, Any]:
+ if kb_id:
+ return {"success": True, "kb_id": kb_id, "kb_name": "", "kb_path": ""}
+ resolved = await service.resolve_kb_reference(query)
+ if resolved.get("success"):
+ return resolved
+ return {
+ "success": False,
+ "message": resolved.get("error") or "请说清楚知识库名称。",
+ "matches": resolved.get("matches") or resolved.get("candidates") or [],
+ }
+
+
+def _extract_limit(query: str) -> Optional[int]:
+ patterns = [r"前\s*(\d+)\s*个", r"limit\s*[:=]\s*(\d+)", r"最多\s*(\d+)\s*个"]
+ for pattern in patterns:
+ match = re.search(pattern, query, flags=re.IGNORECASE)
+ if match:
+ return max(1, min(int(match.group(1)), 1000))
+ return None