import asyncio
import hashlib
import json
import re
from collections import Counter, defaultdict
from typing import Any, Awaitable, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
from config import WIKI_CONFIG
from wiki_engine import db
MAX_CITATION_BATCH_CHARS = 12000
DEFAULT_CITATION_CONCURRENCY = 3
DEFAULT_REDUCE_CONCURRENCY = 3
DEFAULT_LLM_CONCURRENCY = 4
_LLM_SEMAPHORE: Optional[asyncio.Semaphore] = None
_LLM_SEMAPHORE_LIMIT = 0
WIKI_GRANULARITY_GUIDANCE = {
"focused": """**FOCUSED mode - aggressive pruning.**
Extract ONLY the document's primary subjects: the handful of entities/concepts that this document is fundamentally ABOUT.
INCLUDE:
- The document's main subject(s).
- At most 3-7 items total across entities and concepts combined.
EXCLUDE:
- Technology stacks, libraries, frameworks, and infrastructure terms mentioned in passing.
- Generic concepts and methodologies that are only implementation details.
- Anything that would normally get only a one-sentence description because there is not enough content.
If you are unsure whether an item belongs, leave it out.""",
"standard": """**STANDARD mode - balanced.**
Extract the document's main subjects PLUS entities/concepts that are substantively discussed, meaning they have a dedicated paragraph, multiple bullet points, or at least 2-3 sentences of context.
INCLUDE:
- The document's main subject(s).
- Secondary entities/concepts that receive a concrete block of content.
- Named methodologies, architectures, procedures, standards, systems, data structures, or techniques when the document explains how they work or how they are used.
EXCLUDE:
- One-off mentions, parenthetical references, and generic infrastructure nouns.
- Items whose entire contribution to the document would fit in a single short sentence.
Aim for a tight, curated index. When in doubt about a marginal item, prefer to exclude it.""",
"exhaustive": """**EXHAUSTIVE mode - maximum recall.**
Extract every named entity and every recognizable concept, including technologies, tools, standards, methodologies, procedures, faults, equipment, database tables, fields, and domain terms, provided they are concrete and supported by the document.
INCLUDE:
- All main and secondary subjects.
- All named technologies, systems, devices, components, databases, tables, fields, services, protocols, standards, procedures, and fault phenomena.
- Recognizable concepts that are useful as a future question target.
EXCLUDE ONLY:
- Truly generic terms with no domain value.
- Items that appear only inside URL paths or incidental references.
Use this mode when the knowledge base functions as a technical glossary rather than a curated narrative wiki.""",
}
WIKI_CANDIDATE_SLUG_PROMPT = """You are a knowledge extraction system. Analyze the following document and list all significant entities AND key concepts as a lightweight candidate set. Another pass will later attach concrete supporting chunks to each item, so you do NOT need to write exhaustive per-item facts here.
{content}
{previous_slugs}
Return a JSON object with two arrays: "entities" and "concepts".
IMPORTANT: Write ALL names, descriptions, and details in {language}.
If the block above is empty, contains only image references with no extracted text, or otherwise carries no substantive information, return {{"entities": [], "concepts": []}}. Do NOT invent entities or concepts from any other source.
### Extraction Scope (Granularity: {granularity})
{granularity_guidance}
### Slug Continuity Rules
If previous slugs are provided above, you MUST follow these rules:
- If an entity or concept from the previous extraction still exists in the current document, reuse its exact slug from the previous list. Do NOT generate a new slug for the same thing.
- If an entity or concept no longer appears in the document, do NOT include it in the output.
- Only generate new slugs for entities/concepts that are genuinely new.
- This ensures slug stability across document updates.
### Entities
Each entity should have:
- "name": The entity name in {language}.
- "slug": URL-friendly slug, format "entity/"; use romanized/pinyin form for non-Latin names when possible; reuse previous slug if applicable.
- "aliases": names that refer to THE EXACT SAME entity only. Do NOT include parent categories, related products, generic terms, or broader concepts.
- "description": one self-contained sentence, 15-40 words, describing what this entity is and its role in the document.
- "details": a short 1-3 sentence fallback summary under 300 characters.
Never promote trivially-mentioned names into entities.
### Concepts
Each concept should have:
- "name": The concept name in {language}.
- "slug": URL-friendly slug, format "concept/"; use romanized/pinyin form for non-Latin names when possible; reuse previous slug if applicable.
- "aliases": exact synonyms, official abbreviations, or interchangeable names only. Do NOT include sub-topics, related techniques, broader categories, or implementation details.
- "description": one self-contained sentence, 15-40 words, defining what this concept is.
- "details": a short 1-3 sentence fallback summary under 300 characters.
Skip concepts that are merely name-dropped without discussion.
### Deduplication Rules
- Specific named things belong only in "entities".
- Abstract ideas, methodologies, procedures, standards, faults, data structures, and topics belong only in "concepts".
- Never duplicate items across the two arrays.
### JSON Formatting Rules
- Do NOT use literal newline characters inside JSON string values. If needed, use escaped "\\n".
Output ONLY valid JSON."""
WIKI_CHUNK_CITATION_PROMPT = """You are a precise citation system. Your job is to scan a batch of document chunks and decide, for each candidate entity/concept below, which chunks substantively discuss it.
IMPORTANT: Write ALL names, descriptions, and details in {language}.
### Primary task
For each candidate slug listed in , select the chunk IDs from that substantively discuss that entity/concept. "Substantively" means the chunk states at least one concrete fact, attribute, step, date, number, relationship, field, constraint, example, cause, effect, or other useful piece of information about the candidate - not a passing mention.
- Only cite chunks that appear in the block below.
- Use the "id" attribute of each element verbatim, for example "c003".
- If a candidate is not meaningfully discussed in ANY chunk in this batch, omit it from the output.
- A chunk CAN be cited by multiple candidates if it genuinely discusses multiple of them.
- If a chunk is overly long or mixes unrelated topics, still cite it for every candidate it discusses.
### Secondary task: new slugs
If this batch reveals a significant entity/concept that is NOT in , you may add it under "new_slugs" so it gets incorporated. Only add genuinely new, substantively-discussed items. Do NOT rediscover items already listed in .
Each new slug must include:
- "type": "entity" or "concept"
- "name", "slug", "aliases", "description", "details"
- "source_chunks": list of chunk IDs in the current batch that discuss it
### JSON Formatting Rules
- Do NOT use literal newline characters inside JSON string values. If needed, use escaped "\\n".
- Output ONLY valid JSON, no preamble.
Output format:
{{
"citations": {{
"entity/xxx": ["c001", "c003"],
"concept/yyy": ["c002"]
}},
"new_slugs": [
{{
"type": "entity",
"name": "Example",
"slug": "entity/example",
"aliases": [],
"description": "...",
"details": "...",
"source_chunks": ["c005"]
}}
]
}}
If nothing in this batch is cite-worthy, return: {{"citations": {{}}, "new_slugs": []}}
{candidate_slugs}
{chunks_xml}
Now apply the instructions above to the chunks and output ONLY the JSON."""
WIKI_PAGE_MODIFY_PROMPT = """You are a wiki editor tasked with updating an existing wiki page. You must process a set of NEW information to add.
### SOURCE GROUNDING & MERGE RULES (CRITICAL)
1. No Inline Chunk IDs: Chunk aliases such as [c003] are internal processing metadata. NEVER output them in the page body or summary. Source associations are stored separately by the system.
2. Mandatory Grounding: Every newly added factual claim, entity, numerical value, field, step, cause, condition, or relationship MUST be directly supported by the provided new source chunks.
3. No Hallucination: Do not invent, synthesize, or infer information that is not explicitly present in the provided source chunks.
{page_slug}
{page_title}
{page_type}
{page_aliases}
This wiki page is specifically about **{page_title}**. Every statement on the page MUST be directly about this exact page topic, not adjacent or similarly named topics.
{existing_content}
{new_content}
The block above is assembled from VERBATIM source chunks that were already cited as directly supporting this page. Stay close to the source wording. You may lightly reorder, deduplicate, and join related sentences, but do NOT expand short statements into longer unsourced explanations.
{available_slugs}
1. The FIRST line of your output MUST be: SUMMARY: {{one sentence, 15-40 words, describing what this page is about after the update for wiki index listing}}
2. ADD and MERGE the facts from into the page.
3. First verify that the new information is actually about {page_title}. If a piece belongs to a different but related thing, reject that part.
4. Prefer newer directly supported information if it contradicts old content.
5. Do NOT over-structure. Only introduce section headings if the source itself uses that heading OR the page already has one. For a new page, a single "# {page_title}" heading plus concise paragraphs and flat bullet facts is preferred.
6. Do NOT add rhetorical filler such as "aims to", "plays an important role", or "is designed to" unless the source literally says it.
7. Keep [[slug|name]] wiki-link references ONLY if the slug appears in . Remove invalid wiki links. Do NOT invent new wiki-link slugs.
8. Write in {language}.
Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble."""
WIKI_SUMMARY_PROMPT = """You are a wiki editor. Given the following document content, create a structured wiki summary page in Markdown format.
{content}
{extracted_slugs}
1. The FIRST line of your output MUST be: SUMMARY: {{one sentence, 15-40 words, describing what this document is about for wiki index listing}}
2. Create a concise but useful document-level summary.
3. Include the document's main subject, scope, important procedures, standards, systems, equipment, tables, fields, faults, or operational points when present.
4. Use wiki links only from , in the form [[slug|name]].
5. Do NOT invent facts. Stay grounded in the document content.
6. Write in {language}.
7. If the content is empty or has no substantive information, output exactly: "SUMMARY: No textual content was extractable from this document." followed by a brief note.
Output the SUMMARY line first, then the Markdown content. Do not include any other preamble."""
async def build_pages_for_file(
file_info: Dict[str, Any],
slice_refs: List[Dict[str, Any]],
use_llm: bool = False,
progress: Optional[Callable[[str], Awaitable[None]]] = None,
) -> List[Dict[str, Any]]:
if not slice_refs:
return []
if WIKI_CONFIG.get("llm_summary_enabled", True):
try:
pages = await _build_pages_with_weknora_pipeline(file_info, slice_refs, progress=progress)
if pages:
return pages
except Exception as exc:
print(f"[wiki_engine] WeKnora-style wiki build failed, fallback to rule pages: {exc}")
return _build_pages_by_rule(file_info, slice_refs)
async def _build_pages_with_weknora_pipeline(
file_info: Dict[str, Any],
slice_refs: List[Dict[str, Any]],
progress: Optional[Callable[[str], Awaitable[None]]] = None,
) -> List[Dict[str, Any]]:
filename = str(file_info.get("filename") or file_info.get("name") or "untitled")
kb_id = str(file_info.get("kb_id") or "")
file_id = str(file_info.get("file_id") or file_info.get("id") or "")
language = str(WIKI_CONFIG.get("language") or "Chinese")
granularity = _normalize_granularity(WIKI_CONFIG.get("extraction_granularity") or "standard")
await _emit_progress(progress, "load_existing_pages")
try:
existing_pages = await db.list_wiki_pages_for_kb(kb_id)
except Exception as exc:
print(f"[wiki_engine] load existing wiki pages failed: {exc}")
existing_pages = []
existing_by_slug = {str(p.get("slug") or ""): p for p in existing_pages if p.get("slug")}
previous_slugs = _render_previous_slugs(existing_pages)
document_content = _reconstruct_document_content(slice_refs)
await _emit_progress(progress, "extract_candidates")
extracted_entities, extracted_concepts = await _extract_candidate_slugs(
content=document_content,
previous_slugs=previous_slugs,
language=language,
granularity=granularity,
)
extracted_entities, extracted_concepts = _deduplicate_items(extracted_entities, extracted_concepts)
candidates_xml = _render_candidate_slugs(extracted_entities, extracted_concepts)
await _emit_progress(progress, "cite_chunks")
citations, new_slugs = await _classify_chunk_citations(
candidates_xml=candidates_xml,
slice_refs=slice_refs,
language=language,
)
extracted_entities, extracted_concepts = _merge_citations_into_items(
extracted_entities,
extracted_concepts,
citations,
new_slugs,
)
supplemental_entities, supplemental_concepts = _extract_structured_maintenance_items(slice_refs)
extracted_entities.extend(supplemental_entities)
extracted_concepts.extend(supplemental_concepts)
extracted_entities, extracted_concepts = _deduplicate_items(extracted_entities, extracted_concepts)
slug_items = {
str(item.get("slug") or ""): item
for item in extracted_entities + extracted_concepts
if item.get("slug") and item.get("name")
}
available_slugs = _render_available_slugs(slug_items, existing_pages)
pages: List[Dict[str, Any]] = []
await _emit_progress(progress, "build_summary")
summary_page = await _build_document_summary_page(
kb_id=kb_id,
file_id=file_id,
filename=filename,
refs=slice_refs,
slug_items=slug_items,
content=document_content,
language=language,
)
pages.append(summary_page)
max_pages = max(int(WIKI_CONFIG.get("max_pages_per_file") or 12), 1)
items = _rank_items_by_source_coverage(extracted_entities + extracted_concepts, slice_refs)
reduce_sem = asyncio.Semaphore(_wiki_reduce_concurrency())
reduce_total = min(len(items), max_pages - 1)
reduce_done = 0
reduce_lock = asyncio.Lock()
async def build_item_page(item: Dict[str, Any]) -> Optional[Dict[str, Any]]:
nonlocal reduce_done
async with reduce_sem:
page = await _build_concept_page(
kb_id=kb_id,
file_id=file_id,
filename=filename,
item=item,
all_refs=slice_refs,
existing_page=existing_by_slug.get(str(item.get("slug") or "")),
available_slugs=available_slugs,
language=language,
)
async with reduce_lock:
reduce_done += 1
await _emit_progress(progress, f"reduce_pages {reduce_done}/{reduce_total}")
return page
await _emit_progress(progress, f"reduce_pages 0/{reduce_total}")
built_pages = await asyncio.gather(*(build_item_page(item) for item in items[: max_pages - 1]))
for page in built_pages:
if page:
pages.append(page)
return pages
async def _extract_candidate_slugs(
content: str,
previous_slugs: str,
language: str,
granularity: str,
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
prompt = WIKI_CANDIDATE_SLUG_PROMPT.format(
content=content,
previous_slugs=previous_slugs or "(none - this is a new document)",
language=language,
granularity=granularity,
granularity_guidance=WIKI_GRANULARITY_GUIDANCE[granularity],
)
data = await _llm_json(
prompt,
system_prompt="You are a strict knowledge extraction system. Output valid JSON only.",
temperature=0.1,
)
entities = [x for x in data.get("entities", []) if isinstance(x, dict)]
concepts = [x for x in data.get("concepts", []) if isinstance(x, dict)]
return _clean_items(entities, "entity"), _clean_items(concepts, "concept")
async def _emit_progress(progress: Optional[Callable[[str], Awaitable[None]]], stage: str) -> None:
if not progress:
return
try:
await progress(stage)
except Exception as exc:
print(f"[wiki_engine] wiki progress update failed: {exc}")
async def _classify_chunk_citations(
candidates_xml: str,
slice_refs: Sequence[Dict[str, Any]],
language: str,
) -> Tuple[Dict[str, List[str]], List[Dict[str, Any]]]:
batches = _split_citation_batches(slice_refs)
if not batches or not candidates_xml.strip():
return {}, []
citation_sets: Dict[str, set[str]] = defaultdict(set)
new_slugs: List[Dict[str, Any]] = []
lock = asyncio.Lock()
sem = asyncio.Semaphore(_wiki_citation_concurrency())
async def run_batch(batch: Dict[str, Any]) -> None:
async with sem:
prompt = WIKI_CHUNK_CITATION_PROMPT.format(
language=language,
candidate_slugs=candidates_xml,
chunks_xml=batch["chunks_xml"],
)
try:
data = await _llm_json(
prompt,
system_prompt="You map wiki candidates to source chunks. Output valid JSON only.",
temperature=0.0,
)
except Exception as exc:
print(f"[wiki_engine] wiki citation batch failed: {exc}")
return
aliases = batch["alias_to_id"]
async with lock:
for slug, alias_list in (data.get("citations") or {}).items():
if not slug or not isinstance(alias_list, list):
continue
for alias in alias_list:
real_id = aliases.get(str(alias))
if real_id:
citation_sets[str(slug)].add(real_id)
for raw in data.get("new_slugs") or []:
if not isinstance(raw, dict):
continue
item = dict(raw)
item["source_chunks"] = [
aliases[str(alias)]
for alias in item.get("source_chunks") or []
if str(alias) in aliases
]
if item.get("slug") and item.get("name") and item.get("source_chunks"):
new_slugs.append(item)
await asyncio.gather(*(run_batch(batch) for batch in batches))
chunk_order = {
str(ref.get("slice_id") or ""): int(ref.get("ordinal") or idx)
for idx, ref in enumerate(slice_refs)
}
citations = {
slug: sorted(ids, key=lambda sid: chunk_order.get(sid, 999999))
for slug, ids in citation_sets.items()
}
return citations, new_slugs
async def _build_document_summary_page(
kb_id: str,
file_id: str,
filename: str,
refs: List[Dict[str, Any]],
slug_items: Dict[str, Dict[str, Any]],
content: str,
language: str,
) -> Dict[str, Any]:
slug_listing = "\n".join(
f"- [[{slug}]] = {item.get('name') or slug}"
for slug, item in sorted(slug_items.items())
)
prompt = WIKI_SUMMARY_PROMPT.format(
content=content,
extracted_slugs=slug_listing or "(none)",
language=language,
)
summary, body = "", ""
try:
raw = await _llm_text(
prompt,
system_prompt="You are a grounded wiki editor. Do not invent facts.",
temperature=0.1,
)
summary, body = _split_summary_line(raw)
except Exception as exc:
print(f"[wiki_engine] wiki summary page failed: {exc}")
keywords = _keywords_from_refs(refs)
if not summary:
summary = _make_file_summary(filename, refs, keywords)
if not body:
body = _compose_topic_content(filename, refs[:12])
links = _links_for_refs(refs[:20], reason="document_summary")
return {
"kb_id": kb_id,
"slug": f"summary/{_slugify(file_id or filename)}",
"title": f"{filename} - document summary"[:300],
"page_type": "document_summary",
"summary": summary,
"content": body,
"keywords": keywords[:16],
"aliases": [filename],
"related_file_ids": [file_id] if file_id else [],
"source_refs": _source_refs(file_id, filename, refs[:20], reason="document_summary"),
"chunk_refs": _chunk_refs(refs[:20], reason="document_summary"),
"source_kind": "llm",
"score_hint": 1.0,
"links": links,
}
async def _build_concept_page(
kb_id: str,
file_id: str,
filename: str,
item: Dict[str, Any],
all_refs: List[Dict[str, Any]],
existing_page: Optional[Dict[str, Any]],
available_slugs: str,
language: str,
) -> Optional[Dict[str, Any]]:
slug = str(item.get("slug") or "").strip()
title = str(item.get("name") or "").strip()
if not slug or not title:
return None
source_chunk_ids = [str(x) for x in item.get("source_chunks") or [] if str(x)]
if not source_chunk_ids:
return None
refs_by_id = {str(ref.get("slice_id") or ""): ref for ref in all_refs}
refs = [refs_by_id[sid] for sid in source_chunk_ids if sid in refs_by_id]
if not refs:
return None
page_type = _page_type_from_slug_or_item(slug, item)
if existing_page:
title = str(existing_page.get("title") or title)
page_type = str(existing_page.get("page_type") or page_type)
new_content = _format_new_information(filename, refs)
source_kind = "llm"
if _should_use_structured_page_template(item):
summary, body = _build_structured_page_text(title=title, item=item, refs=refs)
source_kind = "structured"
else:
prompt = WIKI_PAGE_MODIFY_PROMPT.format(
page_slug=slug,
page_title=title,
page_type=page_type,
page_aliases=", ".join(_clean_list(item.get("aliases"), limit=12)),
existing_content=str((existing_page or {}).get("content") or ""),
new_content=new_content,
available_slugs=available_slugs,
language=language,
)
summary, body = "", ""
try:
raw = await _llm_text(
prompt,
system_prompt="You are a grounded wiki editor. Use only provided source chunks.",
temperature=0.1,
)
summary, body = _split_summary_line(raw)
except Exception as exc:
print(f"[wiki_engine] wiki page reduce failed for {slug}: {exc}")
if not summary:
summary = str(item.get("description") or item.get("details") or _make_topic_summary(title, refs)).strip()
if not body:
body = _compose_topic_content(title, refs)
old_source_refs = _filter_refs_by_file((existing_page or {}).get("source_refs") or [], file_id)
old_chunk_refs = _filter_refs_by_file((existing_page or {}).get("chunk_refs") or [], file_id)
old_links = _filter_refs_by_file((existing_page or {}).get("slices") or [], file_id)
old_related_files = [
str(x)
for x in ((existing_page or {}).get("related_file_ids") or [])
if str(x) and str(x) != file_id
]
aliases = _dedupe_strings(
_clean_list((existing_page or {}).get("aliases"), limit=24)
+ _clean_list(item.get("aliases"), limit=24)
)[:24]
keywords = _filter_page_keywords(
[title]
+ aliases
+ _clean_list((existing_page or {}).get("keywords"), limit=24)
+ _clean_list(item.get("keywords"), limit=24)
+ _merge_keywords(refs, lead=title)
)[:24]
links = old_links + _links_for_refs(refs, reason=f"wiki_citation:{slug}")
return {
"kb_id": kb_id,
"slug": slug[:300],
"title": title[:300],
"page_type": page_type,
"summary": summary,
"content": body,
"keywords": keywords,
"aliases": aliases,
"related_file_ids": _dedupe_strings(old_related_files + ([file_id] if file_id else [])),
"source_refs": old_source_refs + _source_refs(file_id, filename, refs, reason=f"wiki_citation:{slug}"),
"chunk_refs": old_chunk_refs + _chunk_refs(refs, reason=f"wiki_citation:{slug}"),
"source_kind": source_kind,
"score_hint": 1.4,
"links": links,
}
async def _llm_json(prompt: str, system_prompt: str, temperature: float) -> Dict[str, Any]:
raw = await _llm_text(prompt, system_prompt=system_prompt, temperature=temperature, json_output=True)
cleaned = _strip_code_fence(raw)
try:
return json.loads(cleaned)
except json.JSONDecodeError:
match = re.search(r"\{.*\}", cleaned, re.S)
if match:
return json.loads(match.group(0))
raise
async def _llm_text(
prompt: str,
system_prompt: str,
temperature: float,
json_output: bool = False,
) -> str:
from modelsAPI.model_api import OpenaiAPI
cache_key = _llm_cache_key(prompt, system_prompt, temperature, json_output)
if WIKI_CONFIG.get("llm_cache_enabled", True):
try:
cached = await db.get_llm_cache(
cache_key,
ttl_seconds=int(WIKI_CONFIG.get("llm_cache_ttl_seconds") or 0),
)
if cached:
return cached
except Exception as exc:
print(f"[wiki_engine] wiki llm cache read failed: {exc}")
retries = _bounded_int(WIKI_CONFIG.get("llm_retry_count"), 2, 0, 5)
last_error: Optional[Exception] = None
for attempt in range(retries + 1):
try:
async with _wiki_llm_semaphore():
response = await OpenaiAPI.open_api_chat_without_thinking(
query=prompt,
json_output=json_output,
system_prompt=system_prompt,
temperature=temperature,
)
if WIKI_CONFIG.get("llm_cache_enabled", True) and response:
try:
await db.set_llm_cache(cache_key, response)
except Exception as exc:
print(f"[wiki_engine] wiki llm cache write failed: {exc}")
return response
except Exception as exc:
last_error = exc
if attempt >= retries:
break
await asyncio.sleep(min(5.0, 0.5 * (2 ** attempt)))
raise last_error or RuntimeError("wiki llm call failed")
def _reconstruct_document_content(slice_refs: Sequence[Dict[str, Any]]) -> str:
max_chars = int(WIKI_CONFIG.get("full_content_max_chars") or 120000)
parts: List[str] = []
total = 0
ordered = sorted(slice_refs, key=lambda r: int(r.get("ordinal") or 0))
for ref in ordered:
text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "").strip()
if not text:
continue
title_path = " / ".join(str(x) for x in (ref.get("title_path") or []) if x)
prefix = f"[chunk {ref.get('slice_id') or ''}]"
if title_path:
prefix += f" {title_path}"
block = f"{prefix}\n{text}"
if total + len(block) > max_chars:
remain = max_chars - total
if remain > 500:
parts.append(block[:remain])
break
parts.append(block)
total += len(block)
return "\n\n".join(parts)
def _split_citation_batches(slice_refs: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
ordered = [
ref for ref in sorted(slice_refs, key=lambda r: int(r.get("ordinal") or 0))
if str(ref.get("content") or ref.get("_text") or ref.get("brief") or "").strip()
]
batches: List[List[Dict[str, Any]]] = []
current: List[Dict[str, Any]] = []
current_len = 0
for ref in ordered:
text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "")
length = len(text)
if current and current_len + length > MAX_CITATION_BATCH_CHARS:
batches.append(current)
current = []
current_len = 0
current.append(ref)
current_len += length
if current:
batches.append(current)
rendered: List[Dict[str, Any]] = []
for refs in batches:
alias_to_id: Dict[str, str] = {}
chunks: List[str] = []
for idx, ref in enumerate(refs):
alias = f"c{idx:03d}"
slice_id = str(ref.get("slice_id") or "")
alias_to_id[alias] = slice_id
text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "")
chunks.append(f'\n{text}\n')
rendered.append({"alias_to_id": alias_to_id, "chunks_xml": "\n".join(chunks)})
return rendered
def _render_candidate_slugs(entities: Sequence[Dict[str, Any]], concepts: Sequence[Dict[str, Any]]) -> str:
lines: List[str] = []
for item in entities:
lines.append(_candidate_line(item, "entity"))
for item in concepts:
lines.append(_candidate_line(item, "concept"))
return "\n".join(line for line in lines if line)
def _candidate_line(item: Dict[str, Any], kind: str) -> str:
slug = str(item.get("slug") or "").strip()
name = str(item.get("name") or "").strip()
if not slug or not name:
return ""
aliases = ", ".join(_clean_list(item.get("aliases"), limit=8))
alias_text = f', aliases="{aliases}"' if aliases else ""
description = str(item.get("description") or item.get("details") or "").strip()
return f'- slug: {slug}, type: {kind}, name: "{name}"{alias_text}, description: {description}'
def _merge_citations_into_items(
entities: List[Dict[str, Any]],
concepts: List[Dict[str, Any]],
citations: Dict[str, List[str]],
new_slugs: List[Dict[str, Any]],
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
for item in entities + concepts:
item["source_chunks"] = citations.get(str(item.get("slug") or ""), [])
existing = {str(item.get("slug") or "") for item in entities + concepts if item.get("slug")}
merged_new: Dict[str, Dict[str, Any]] = {}
for raw in new_slugs:
item = _clean_item(raw, str(raw.get("type") or "concept"))
slug = str(item.get("slug") or "")
if not slug or slug in existing:
continue
bucket = merged_new.setdefault(slug, item)
bucket["source_chunks"] = _dedupe_strings(
_clean_list(bucket.get("source_chunks"), limit=100)
+ _clean_list(item.get("source_chunks"), limit=100)
)
for item in merged_new.values():
if str(item.get("slug") or "").startswith("entity/"):
entities.append(item)
else:
concepts.append(item)
return entities, concepts
def _should_use_structured_page_template(item: Dict[str, Any]) -> bool:
if not WIKI_CONFIG.get("structured_template_enabled", True):
return False
return int(item.get("structured_priority") or 0) >= 80
def _build_structured_page_text(
title: str,
item: Dict[str, Any],
refs: Sequence[Dict[str, Any]],
) -> Tuple[str, str]:
summary = str(item.get("description") or item.get("details") or "").strip()
if not summary:
summary = _make_topic_summary(title, list(refs))
details = str(item.get("details") or item.get("description") or "").strip()
keywords = _clean_list(item.get("keywords"), limit=12)
aliases = _clean_list(item.get("aliases"), limit=12)
lines = [f"# {title}", ""]
if details:
lines.extend([details, ""])
facts = _dedupe_strings([x for x in aliases + keywords if x and x != title])
if facts:
lines.append("## 关键条目")
for fact in facts[:10]:
lines.append(f"- {fact}")
lines.append("")
lines.append("## 来源摘录")
for ref in refs[:8]:
excerpt = _short_text(ref, 420)
if excerpt:
lines.append(f"- {excerpt}")
return summary, "\n".join(lines).strip()
def _extract_structured_maintenance_items(
slice_refs: Sequence[Dict[str, Any]],
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
entities: List[Dict[str, Any]] = []
concepts: List[Dict[str, Any]] = []
seen_codes: set[str] = set()
row_pattern = re.compile(
r"| (MT[0-9O]{3}) | ([^<]{1,40}) | ([^<]{1,80}) | "
r"([^<]{1,120}) | ([^<]{1,40}) | ([^<]{1,40}) | ",
re.I,
)
for ref in slice_refs:
slice_id = str(ref.get("slice_id") or "")
if not slice_id:
continue
text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "")
if "维修项目" not in text or not re.search(r"MT[0-9O]{3}", text, re.I):
continue
for match in row_pattern.finditer(text):
code = _normalize_mt_code(match.group(1))
if not code or code in seen_codes:
continue
seen_codes.add(code)
component = _clean_maintenance_cell(match.group(3))
action = _clean_maintenance_cell(match.group(4))
interval = _clean_maintenance_cell(match.group(5))
level = _clean_maintenance_cell(match.group(6))
if not component or not action:
continue
title = component if component in action else f"{code}维修项目"
aliases = _dedupe_strings([code, action, f"{code}{action}", f"{component}{action}"])[:8]
description = "，".join(
part for part in [
f"{title}对应{code}维修项目：{action}",
f"维修间隔期为{interval}" if interval else "",
f"维修级别为{level}" if level else "",
] if part
) + "。"
entities.append({
"name": title,
"slug": f"entity/{_slugify(title)}",
"aliases": aliases,
"description": description,
"details": description,
"keywords": _dedupe_strings([component, action, code, interval, level])[:8],
"source_chunks": [slice_id],
"structured_priority": 100,
})
return entities, concepts
def _normalize_mt_code(value: str) -> str:
code = re.sub(r"[^A-Za-z0-9]", "", str(value or "").upper()).replace("O", "0")
if re.fullmatch(r"MT\d{3}", code):
return code
return ""
def _clean_maintenance_cell(value: str) -> str:
text = re.sub(r"<[^>]+>", "", str(value or ""))
text = re.sub(r"\s+", "", text)
return text.strip("：:，,。.;；、")[:80]
def _deduplicate_items(
entities: List[Dict[str, Any]],
concepts: List[Dict[str, Any]],
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
def dedupe(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
by_key: Dict[str, Dict[str, Any]] = {}
order: List[str] = []
out = []
for item in items:
slug = str(item.get("slug") or "").strip()
name = str(item.get("name") or "").strip()
key = slug or name
if not key:
continue
if key not in by_key:
by_key[key] = dict(item)
order.append(key)
continue
existing = by_key[key]
for field, limit in (
("aliases", 24),
("keywords", 24),
("source_chunks", 200),
):
existing[field] = _dedupe_strings(
_clean_list(existing.get(field), limit=limit)
+ _clean_list(item.get(field), limit=limit)
)[:limit]
for field in ("description", "details"):
if not str(existing.get(field) or "").strip() and str(item.get(field) or "").strip():
existing[field] = item.get(field)
existing["structured_priority"] = max(
int(existing.get("structured_priority") or 0),
int(item.get("structured_priority") or 0),
)
for key in order:
out.append(by_key[key])
return out
return dedupe(entities), dedupe(concepts)
def _rank_items_by_source_coverage(items: Sequence[Dict[str, Any]], refs: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
order = {str(ref.get("slice_id") or ""): idx for idx, ref in enumerate(refs)}
def key(item: Dict[str, Any]) -> Tuple[int, int, int, str]:
chunks = _clean_list(item.get("source_chunks"), limit=500)
first = min((order.get(sid, 999999) for sid in chunks), default=999999)
priority = int(item.get("structured_priority") or 0)
return (-priority, -len(chunks), first, str(item.get("name") or ""))
return sorted(items, key=key)
def _render_previous_slugs(pages: Sequence[Dict[str, Any]]) -> str:
lines = []
for page in pages:
slug = str(page.get("slug") or "")
if slug.startswith(("entity/", "concept/")):
lines.append(f"- {slug}")
return "\n".join(sorted(set(lines))) or "(none - this is a new document)"
def _render_available_slugs(
slug_items: Dict[str, Dict[str, Any]],
existing_pages: Sequence[Dict[str, Any]],
) -> str:
rows: Dict[str, str] = {}
for page in existing_pages:
slug = str(page.get("slug") or "")
title = str(page.get("title") or "")
if slug:
rows[slug] = title or slug
for slug, item in slug_items.items():
rows[slug] = str(item.get("name") or rows.get(slug) or slug)
return "\n".join(f"- [[{slug}]] = {title}" for slug, title in sorted(rows.items()))
def _format_new_information(filename: str, refs: Sequence[Dict[str, Any]]) -> str:
blocks = [f'']
for idx, ref in enumerate(refs):
alias = f"c{idx:03d}"
text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "").strip()
title_path = " / ".join(str(x) for x in (ref.get("title_path") or []) if x)
if title_path:
blocks.append(f'{title_path}')
blocks.append(f'\n{text}\n')
blocks.append("")
return "\n".join(blocks)
def _filter_refs_by_file(items: Sequence[Any], file_id: str) -> List[Dict[str, Any]]:
out: List[Dict[str, Any]] = []
for item in items or []:
if not isinstance(item, dict):
continue
if file_id and str(item.get("file_id") or "") == file_id:
continue
out.append(dict(item))
return out
def _clean_items(items: Sequence[Dict[str, Any]], kind: str) -> List[Dict[str, Any]]:
return [_clean_item(item, kind) for item in items if _clean_item(item, kind).get("name")]
def _clean_item(item: Dict[str, Any], kind: str) -> Dict[str, Any]:
name = str(item.get("name") or item.get("title") or "").strip()
slug = str(item.get("slug") or "").strip()
if name and not slug:
slug = f"{kind}/{_slugify(name)}"
if slug and "/" not in slug:
slug = f"{kind}/{_slugify(slug)}"
if kind == "entity" and slug and not slug.startswith("entity/"):
slug = f"entity/{slug.split('/', 1)[-1]}"
if kind != "entity" and slug and not slug.startswith("concept/"):
slug = f"concept/{slug.split('/', 1)[-1]}"
return {
"name": name,
"slug": slug[:300],
"aliases": _clean_list(item.get("aliases"), limit=16),
"description": str(item.get("description") or item.get("summary") or "").strip(),
"details": str(item.get("details") or item.get("content") or "").strip(),
"keywords": _clean_list(item.get("keywords"), limit=16),
"source_chunks": _clean_list(item.get("source_chunks"), limit=100),
}
def _split_summary_line(text: str) -> Tuple[str, str]:
lines = (text or "").strip().splitlines()
if not lines:
return "", ""
first = lines[0].strip()
if first.upper().startswith("SUMMARY:"):
return first.split(":", 1)[1].strip(), "\n".join(lines[1:]).strip()
return "", text.strip()
def _normalize_granularity(value: Any) -> str:
text = str(value or "").strip().lower()
return text if text in WIKI_GRANULARITY_GUIDANCE else "standard"
def _wiki_citation_concurrency() -> int:
return _bounded_int(WIKI_CONFIG.get("citation_concurrency"), DEFAULT_CITATION_CONCURRENCY, 1, 4)
def _wiki_reduce_concurrency() -> int:
return _bounded_int(WIKI_CONFIG.get("reduce_concurrency"), DEFAULT_REDUCE_CONCURRENCY, 1, 4)
def _wiki_llm_semaphore() -> asyncio.Semaphore:
global _LLM_SEMAPHORE, _LLM_SEMAPHORE_LIMIT
limit = _bounded_int(WIKI_CONFIG.get("llm_concurrency"), DEFAULT_LLM_CONCURRENCY, 1, 8)
if _LLM_SEMAPHORE is None or _LLM_SEMAPHORE_LIMIT != limit:
_LLM_SEMAPHORE = asyncio.Semaphore(limit)
_LLM_SEMAPHORE_LIMIT = limit
return _LLM_SEMAPHORE
def _llm_cache_key(prompt: str, system_prompt: str, temperature: float, json_output: bool) -> str:
payload = {
"prompt": prompt,
"system_prompt": system_prompt,
"temperature": float(temperature or 0.0),
"json_output": bool(json_output),
"version": "wiki-v2",
}
raw = json.dumps(payload, ensure_ascii=False, sort_keys=True)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def _bounded_int(value: Any, default: int, minimum: int, maximum: int) -> int:
try:
number = int(value)
except Exception:
number = default
return max(minimum, min(maximum, number))
def _page_type_from_slug_or_item(slug: str, item: Dict[str, Any]) -> str:
if slug.startswith("entity/"):
return "entity"
return _normalize_page_type(item.get("page_type") or "concept")
def _build_pages_by_rule(file_info: Dict[str, Any], slice_refs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
filename = str(file_info.get("filename") or file_info.get("name") or "untitled")
kb_id = str(file_info.get("kb_id") or "")
file_id = str(file_info.get("file_id") or file_info.get("id") or "")
max_pages = int(WIKI_CONFIG.get("max_pages_per_file") or 12)
topic_counts: Counter[str] = Counter()
topic_slices: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
for ref in slice_refs:
for topic in _topics_for_ref(ref):
topic_counts[topic] += 1
topic_slices[topic].append(ref)
all_keywords = [kw for kw, _ in topic_counts.most_common(16)]
pages = [_fallback_document_summary_page(kb_id, file_id, filename, slice_refs, all_keywords, source_kind="rule")]
for topic, _ in topic_counts.most_common(max_pages - 1):
refs = topic_slices[topic][:12]
if not refs:
continue
slug = f"concept/{_slugify(topic)}"
pages.append({
"kb_id": kb_id,
"slug": slug,
"title": topic[:300],
"page_type": "concept",
"summary": _make_topic_summary(topic, refs),
"content": _compose_topic_content(topic, refs),
"keywords": _merge_keywords(refs, lead=topic),
"aliases": [],
"related_file_ids": [file_id] if file_id else [],
"source_refs": _source_refs(file_id, filename, refs, reason="rule_topic"),
"chunk_refs": _chunk_refs(refs, reason="rule_topic"),
"source_kind": "rule",
"score_hint": 0.7,
"links": _links_for_refs(refs, reason=f"rule_topic:{topic}"),
})
return pages[:max_pages]
def _fallback_document_summary_page(
kb_id: str,
file_id: str,
filename: str,
refs: List[Dict[str, Any]],
keywords: List[str],
source_kind: str,
) -> Dict[str, Any]:
return {
"kb_id": kb_id,
"slug": f"summary/{_slugify(file_id or filename)}",
"title": f"{filename} - document summary"[:300],
"page_type": "document_summary",
"summary": _make_file_summary(filename, refs, keywords),
"content": _compose_topic_content(filename, refs[:12]),
"keywords": keywords[:16],
"aliases": [filename],
"related_file_ids": [file_id] if file_id else [],
"source_refs": _source_refs(file_id, filename, refs[:20], reason="document_summary"),
"chunk_refs": _chunk_refs(refs[:20], reason="document_summary"),
"source_kind": source_kind,
"score_hint": 1.0,
"links": _links_for_refs(refs[:20], reason="document_summary"),
}
def _topics_for_ref(ref: Dict[str, Any]) -> List[str]:
topics: List[str] = []
for value in ref.get("title_path") or []:
text = str(value).strip()
if len(text) >= 2:
topics.append(text[:80])
for value in ref.get("keywords") or []:
text = str(value).strip()
if len(text) >= 2:
topics.append(text[:80])
return _dedupe_strings(topics)[:12]
def _keywords_from_refs(refs: List[Dict[str, Any]]) -> List[str]:
counts: Counter[str] = Counter()
for ref in refs:
counts.update(str(x) for x in (ref.get("keywords") or [])[:10] if str(x).strip())
for title in ref.get("title_path") or []:
text = str(title).strip()
if len(text) >= 2:
counts[text] += 1
return [kw for kw, _ in counts.most_common(16)]
def _make_file_summary(filename: str, refs: List[Dict[str, Any]], keywords: List[str]) -> str:
keyword_text = ", ".join(keywords[:10]) if keywords else "none"
return f"The file {filename} has {len(refs)} indexed source chunks. Main topics: {keyword_text}."
def _make_topic_summary(topic: str, refs: List[Dict[str, Any]]) -> str:
snippets = "; ".join(_short_text(ref, 120) for ref in refs[:3] if _short_text(ref, 120))
return f"Topic '{topic}' is supported by {len(refs)} source chunks. Evidence excerpts: {snippets}"
def _compose_topic_content(topic: str, refs: List[Dict[str, Any]]) -> str:
lines = [f"# {topic}", "", "Source evidence:"]
for ref in refs[:8]:
excerpt = _short_text(ref, 360)
if excerpt:
lines.append(f"- {excerpt}")
return "\n".join(lines)
def _short_text(ref: Dict[str, Any], limit: int) -> str:
text = str(ref.get("content") or ref.get("_text") or ref.get("brief") or "").strip()
return re.sub(r"\s+", " ", text)[:limit]
def _normalize_page_type(value: Any) -> str:
text = str(value or "concept").strip().lower()
allowed = {
"document_summary",
"entity",
"concept",
"procedure",
"fault",
"standard",
"operation",
"device",
"topic",
}
return text if text in allowed else "concept"
def _merge_keywords(refs: List[Dict[str, Any]], lead: str) -> List[str]:
counts: Counter[str] = Counter()
for ref in refs:
counts.update(str(x) for x in (ref.get("keywords") or [])[:10] if str(x).strip())
out = [lead] if lead else []
for kw, _ in counts.most_common(20):
if kw not in out:
out.append(kw)
return out[:24]
def _filter_page_keywords(values: Iterable[Any]) -> List[str]:
return [value for value in _dedupe_strings(values) if not _is_noise_keyword(value)][:24]
def _is_noise_keyword(value: Any) -> bool:
text = str(value or "").strip().lower()
if not text:
return True
if text in {
"td", "tr", "th", "thead", "tbody", "table", "colspan", "rowspan",
"序号", "名称", "备注", "规格", "分钟",
}:
return True
if re.fullmatch(r"[<>/]+", text):
return True
return False
def _links_for_refs(refs: Iterable[Dict[str, Any]], reason: str) -> List[Dict[str, Any]]:
return [
{
"file_id": str(ref.get("file_id") or ""),
"slice_id": str(ref.get("slice_id") or ""),
"weight": 1.0,
"reason": reason,
}
for ref in refs
if ref.get("slice_id")
]
def _source_refs(file_id: str, filename: str, refs: Iterable[Dict[str, Any]], reason: str) -> List[Dict[str, Any]]:
return [
{
"file_id": file_id or str(ref.get("file_id") or ""),
"filename": filename or str(ref.get("filename") or ""),
"slice_id": str(ref.get("slice_id") or ""),
"reason": reason,
}
for ref in refs
if ref.get("slice_id")
]
def _chunk_refs(refs: Iterable[Dict[str, Any]], reason: str) -> List[Dict[str, Any]]:
return [
{
"file_id": str(ref.get("file_id") or ""),
"slice_id": str(ref.get("slice_id") or ""),
"title_path": ref.get("title_path") or [],
"keywords": (ref.get("keywords") or [])[:10],
"reason": reason,
}
for ref in refs
if ref.get("slice_id")
]
def _clean_list(value: Any, limit: int) -> List[str]:
if value is None:
return []
if isinstance(value, str):
value = [value]
if not isinstance(value, (list, tuple, set)):
return []
return _dedupe_strings(str(x).strip() for x in value if str(x).strip())[:limit]
def _dedupe_strings(values: Iterable[Any]) -> List[str]:
out: List[str] = []
seen = set()
for value in values:
text = str(value or "").strip()
if not text or text in seen:
continue
seen.add(text)
out.append(text)
return out
def _slugify(text: str) -> str:
raw = str(text or "").strip().lower()
slug = re.sub(r"[\s/\\:;,.!?|]+", "-", raw)
slug = re.sub(r"[^0-9a-zA-Z_\-\u4e00-\u9fff]+", "", slug).strip("-")
if slug:
return slug[:160]
return hashlib.sha1(raw.encode("utf-8", errors="ignore")).hexdigest()[:20]
def _strip_code_fence(text: str) -> str:
text = (text or "").strip()
if text.startswith("```"):
lines = text.splitlines()
if lines:
lines = lines[1:]
if lines and lines[-1].strip() == "```":
lines = lines[:-1]
text = "\n".join(lines).strip()
return text