From fe8f0a875a856036e4c06ffa38790d8fddf3f636 Mon Sep 17 00:00:00 2001 From: Defeng Date: Fri, 24 Jul 2026 10:28:31 +0800 Subject: [PATCH] Initial project import --- .cargo/config.toml | 11 + .dockerignore | 43 + .gitea/scripts/setup-rust.sh | 247 + .gitea/workflows/ci.yml | 638 ++ .gitignore | 6 + .rustfmt.toml | 60 + CLAUDE.md | 159 + Cargo.lock | 7367 +++++++++++++++++ Cargo.toml | 96 + DEBUG_MEMORY.md | 184 + Dockerfile | 77 + Dockerfile.debug | 88 + README.md | 178 + build-docker.sh | 62 + deploy_ci.md | 24 + docker-compose.yml | 61 + docker/apt-fast | 735 ++ frontend/.gitignore | 24 + frontend/.vscode/extensions.json | 3 + frontend/README.md | 20 + frontend/index.html | 13 + frontend/package-lock.json | 2192 +++++ frontend/package.json | 21 + frontend/public/excel-viewer.html | 359 + frontend/public/pdf-highlight.html | 127 + frontend/public/vite.svg | 1 + frontend/src/App.vue | 322 + frontend/src/api.js | 734 ++ frontend/src/assets/vue.svg | 1 + .../src/components/AdvancedSearchPanel.vue | 205 + frontend/src/components/ArchiveTreeNode.vue | 78 + frontend/src/components/ArchiveViewer.vue | 260 + .../src/components/CreateKnowledgeBase.vue | 223 + frontend/src/components/EntityDetail.vue | 194 + frontend/src/components/ExportRecordPanel.vue | 128 + frontend/src/components/FileCard.vue | 554 ++ frontend/src/components/FileGraph.vue | 123 + frontend/src/components/FileSlices.vue | 270 + frontend/src/components/FileStatusSummary.vue | 202 + frontend/src/components/FileUpload.vue | 322 + .../src/components/GraphVisualization.vue | 1121 +++ frontend/src/components/KbPermissionModal.vue | 175 + .../src/components/KnowledgeBaseDetail.vue | 164 + frontend/src/components/KnowledgeBaseList.vue | 793 ++ .../src/components/KnowledgeBaseSelector.vue | 118 + frontend/src/components/KnowledgeGraph.vue | 424 + frontend/src/components/Pagination.vue | 113 + frontend/src/components/SearchBar.vue | 330 + .../components/SearchDictionaryManager.vue | 667 ++ frontend/src/components/SearchResults.vue | 166 + frontend/src/main.js | 5 + frontend/src/store.js | 10 + frontend/src/style.css | 1 + frontend/vite.config.js | 25 + src/api/common.rs | 106 + src/api/error.rs | 125 + src/api/file.rs | 3245 ++++++++ src/api/graph.rs | 377 + src/api/knowledge_base.rs | 1818 ++++ src/api/mod.rs | 254 + src/api/search.rs | 2960 +++++++ src/api/system.rs | 518 ++ src/archive.rs | 631 ++ src/config.rs | 516 ++ src/db.rs | 1061 +++ src/export.rs | 1800 ++++ src/file_content.rs | 62 + src/frontend.rs | 44 + src/graph/graph_manager.rs | 299 + src/graph/llm_extractor.rs | 315 + src/graph/mod.rs | 121 + src/image_description.rs | 129 + src/image_parse.rs | 239 + src/init.sql | 256 + src/lib.rs | 62 + src/log4rs.rs | 74 + src/main.rs | 191 + src/pdf_content.rs | 85 + src/pdf_highlight.rs | 411 + src/processor.rs | 4165 ++++++++++ src/search/advanced.rs | 427 + src/search/chinese_tokenizer.rs | 206 + src/search/embedding.rs | 209 + src/search/lancedb.rs | 1411 ++++ src/search/mod.rs | 1969 +++++ src/search/tantivy_engine.rs | 999 +++ src/slice_content.rs | 54 + stopwords.txt | 1395 ++++ test_result.md | 28 + tests/api_concurrency.rs | 247 + tests/api_integration.rs | 918 ++ tests/common/mod.rs | 293 + todo.md | 3 + 93 files changed, 48547 insertions(+) create mode 100644 .cargo/config.toml create mode 100644 .dockerignore create mode 100644 .gitea/scripts/setup-rust.sh create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .rustfmt.toml create mode 100644 CLAUDE.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 DEBUG_MEMORY.md create mode 100644 Dockerfile create mode 100644 Dockerfile.debug create mode 100644 README.md create mode 100644 build-docker.sh create mode 100644 deploy_ci.md create mode 100644 docker-compose.yml create mode 100644 docker/apt-fast create mode 100644 frontend/.gitignore create mode 100644 frontend/.vscode/extensions.json create mode 100644 frontend/README.md create mode 100644 frontend/index.html create mode 100644 frontend/package-lock.json create mode 100644 frontend/package.json create mode 100644 frontend/public/excel-viewer.html create mode 100644 frontend/public/pdf-highlight.html create mode 100644 frontend/public/vite.svg create mode 100644 frontend/src/App.vue create mode 100644 frontend/src/api.js create mode 100644 frontend/src/assets/vue.svg create mode 100644 frontend/src/components/AdvancedSearchPanel.vue create mode 100644 frontend/src/components/ArchiveTreeNode.vue create mode 100644 frontend/src/components/ArchiveViewer.vue create mode 100644 frontend/src/components/CreateKnowledgeBase.vue create mode 100644 frontend/src/components/EntityDetail.vue create mode 100644 frontend/src/components/ExportRecordPanel.vue create mode 100644 frontend/src/components/FileCard.vue create mode 100644 frontend/src/components/FileGraph.vue create mode 100644 frontend/src/components/FileSlices.vue create mode 100644 frontend/src/components/FileStatusSummary.vue create mode 100644 frontend/src/components/FileUpload.vue create mode 100644 frontend/src/components/GraphVisualization.vue create mode 100644 frontend/src/components/KbPermissionModal.vue create mode 100644 frontend/src/components/KnowledgeBaseDetail.vue create mode 100644 frontend/src/components/KnowledgeBaseList.vue create mode 100644 frontend/src/components/KnowledgeBaseSelector.vue create mode 100644 frontend/src/components/KnowledgeGraph.vue create mode 100644 frontend/src/components/Pagination.vue create mode 100644 frontend/src/components/SearchBar.vue create mode 100644 frontend/src/components/SearchDictionaryManager.vue create mode 100644 frontend/src/components/SearchResults.vue create mode 100644 frontend/src/main.js create mode 100644 frontend/src/store.js create mode 100644 frontend/src/style.css create mode 100644 frontend/vite.config.js create mode 100644 src/api/common.rs create mode 100644 src/api/error.rs create mode 100644 src/api/file.rs create mode 100644 src/api/graph.rs create mode 100644 src/api/knowledge_base.rs create mode 100644 src/api/mod.rs create mode 100644 src/api/search.rs create mode 100644 src/api/system.rs create mode 100644 src/archive.rs create mode 100644 src/config.rs create mode 100644 src/db.rs create mode 100644 src/export.rs create mode 100644 src/file_content.rs create mode 100644 src/frontend.rs create mode 100644 src/graph/graph_manager.rs create mode 100644 src/graph/llm_extractor.rs create mode 100644 src/graph/mod.rs create mode 100644 src/image_description.rs create mode 100644 src/image_parse.rs create mode 100644 src/init.sql create mode 100644 src/lib.rs create mode 100644 src/log4rs.rs create mode 100644 src/main.rs create mode 100644 src/pdf_content.rs create mode 100644 src/pdf_highlight.rs create mode 100644 src/processor.rs create mode 100644 src/search/advanced.rs create mode 100644 src/search/chinese_tokenizer.rs create mode 100644 src/search/embedding.rs create mode 100644 src/search/lancedb.rs create mode 100644 src/search/mod.rs create mode 100644 src/search/tantivy_engine.rs create mode 100644 src/slice_content.rs create mode 100644 stopwords.txt create mode 100644 test_result.md create mode 100644 tests/api_concurrency.rs create mode 100644 tests/api_integration.rs create mode 100644 tests/common/mod.rs create mode 100644 todo.md diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..3a2d786 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,11 @@ +[build] +rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup"] + +[target.x86_64-apple-darwin] +rustflags = ["-C", "link-arg=-Wl,-undefined,dynamic_lookup"] + +[target.aarch64-apple-darwin] +rustflags = ["-C", "link-arg=-Wl,-undefined,dynamic_lookup"] + +[target.aarch64-unknown-linux-gnu] +linker = "aarch64-linux-gnu-gcc" diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c84a5e3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,43 @@ +# Rust build artifacts (except the release binary we need) +target/*/incremental +target/*/build +target/*/deps +target/*/examples +target/*/.fingerprint +*.pdb + +# Git +.git +.gitignore + +# IDE +.vscode +.idea +*.swp +*.swo +*~ + +# Local data (will be mounted as volumes in production) +data/ + +# Logs +*.log + +# OS files +.DS_Store +Thumbs.db + +# Documentation +README.md +docs/ + +# CI/CD +.gitea +.github +.gitlab-ci.yml + +# Development files +clear.sh + +# Frontend is embedded into the Rust binary before docker build +frontend/ diff --git a/.gitea/scripts/setup-rust.sh b/.gitea/scripts/setup-rust.sh new file mode 100644 index 0000000..a91844e --- /dev/null +++ b/.gitea/scripts/setup-rust.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +set -euo pipefail + +REQUIRED_RUST_VERSION="${REQUIRED_RUST_VERSION:-1.88.0}" +DEFAULT_RUSTUP_INIT_URL="https://rsproxy.cn/rustup-init.sh" +DEFAULT_RUSTUP_DIST_SERVER="https://rsproxy.cn" +DEFAULT_RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" +RUSTUP_INIT_URL="${RUSTUP_INIT_URL:-${DEFAULT_RUSTUP_INIT_URL}}" +export RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-${DEFAULT_RUSTUP_DIST_SERVER}}" +export RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-${DEFAULT_RUSTUP_UPDATE_ROOT}}" + +APT_UPDATED=0 +PACMAN_SYNCED=0 + +as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "Need root privileges to install system packages." + exit 1 + fi +} + +ensure_pkg() { + if command -v pacman >/dev/null 2>&1; then + if [ "${PACMAN_SYNCED}" -ne 1 ]; then + as_root pacman -Sy --noconfirm + PACMAN_SYNCED=1 + fi + as_root pacman --noconfirm --needed "$@" + elif command -v apt-get >/dev/null 2>&1; then + if [ "${APT_UPDATED}" -ne 1 ]; then + as_root apt-get update + APT_UPDATED=1 + fi + as_root apt-get install -y --no-install-recommends "$@" + elif command -v apk >/dev/null 2>&1; then + as_root apk add --no-cache "$@" + elif command -v dnf >/dev/null 2>&1; then + as_root dnf install -y "$@" + else + echo "Unsupported package manager. Please preinstall required tools: $*" + exit 1 + fi +} + +has_pkg_config() { + command -v pkg-config >/dev/null 2>&1 || command -v pkgconf >/dev/null 2>&1 +} + +ensure_cc_toolchain() { + if command -v cc >/dev/null 2>&1 && command -v make >/dev/null 2>&1 && has_pkg_config; then + return + fi + if command -v pacman >/dev/null 2>&1; then + ensure_pkg gcc make pkgconf + elif command -v apt-get >/dev/null 2>&1; then + ensure_pkg build-essential pkg-config + elif command -v apk >/dev/null 2>&1; then + ensure_pkg build-base pkgconf + elif command -v dnf >/dev/null 2>&1; then + ensure_pkg gcc gcc-c++ make pkgconf-pkg-config + else + echo "Unsupported package manager. Please preinstall C toolchain (cc/make/pkg-config)." + exit 1 + fi +} + +resolve_protoc_include() { + local protoc_bin candidate proto_file + + protoc_bin="$(command -v protoc || true)" + for candidate in \ + "${PROTOC_INCLUDE:-}" \ + /usr/include \ + /usr/local/include \ + /usr/include/x86_64-linux-gnu \ + /usr/include/aarch64-linux-gnu \ + /usr/lib/protobuf/include \ + "$(dirname "${protoc_bin}")/../include"; do + [ -n "${candidate}" ] || continue + if [ -f "${candidate}/google/protobuf/empty.proto" ]; then + ( + cd "${candidate}" + pwd + ) + return 0 + fi + done + + if command -v find >/dev/null 2>&1; then + proto_file="$(find /usr /usr/local -type f -path '*/google/protobuf/empty.proto' 2>/dev/null | head -n 1 || true)" + if [ -n "${proto_file}" ]; then + ( + cd "$(dirname "$(dirname "$(dirname "${proto_file}")")")" + pwd + ) + return 0 + fi + fi + + return 1 +} + +ensure_protoc() { + if command -v protoc >/dev/null 2>&1 && resolve_protoc_include >/dev/null 2>&1; then + return + fi + + if command -v pacman >/dev/null 2>&1; then + ensure_pkg protobuf + elif command -v apt-get >/dev/null 2>&1; then + ensure_pkg protobuf-compiler libprotobuf-dev + elif command -v apk >/dev/null 2>&1; then + ensure_pkg protobuf protobuf-dev + elif command -v dnf >/dev/null 2>&1; then + ensure_pkg protobuf-compiler protobuf-devel + else + echo "Unsupported package manager. Please preinstall protoc." + exit 1 + fi + + if ! command -v protoc >/dev/null 2>&1; then + echo "protoc not found after installation." + exit 1 + fi + + if ! resolve_protoc_include >/dev/null 2>&1; then + echo "protoc found but standard protobuf include files are missing." + exit 1 + fi +} + +version_ge() { + [ "$(printf '%s\n%s\n' "$2" "$1" | sort -V | head -n 1)" = "$2" ] +} + +has_usable_system_rust() { + local current_rust_version + + if ! command -v cargo >/dev/null 2>&1 || ! command -v rustc >/dev/null 2>&1; then + return 1 + fi + + current_rust_version="$(rustc --version | awk '{print $2}')" + if [ -z "${current_rust_version}" ] || ! version_ge "${current_rust_version}" "${REQUIRED_RUST_VERSION}"; then + return 1 + fi + + cargo fmt --version >/dev/null 2>&1 +} + +ensure_downloader() { + if command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; then + return + fi + ensure_pkg curl +} + +install_rustup_with() { + local init_url + + init_url="$1" + if command -v curl >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -fsSL "${init_url}" \ + | sh -s -- -y --profile minimal --default-toolchain none + else + wget -qO- "${init_url}" \ + | sh -s -- -y --profile minimal --default-toolchain none + fi +} + +setup_rustup_toolchain() { + ensure_downloader + + if ! command -v rustup >/dev/null 2>&1; then + if ! install_rustup_with "${RUSTUP_INIT_URL}"; then + if [ "${RUSTUP_INIT_URL}" = "${DEFAULT_RUSTUP_INIT_URL}" ] \ + && [ "${RUSTUP_DIST_SERVER}" = "${DEFAULT_RUSTUP_DIST_SERVER}" ] \ + && [ "${RUSTUP_UPDATE_ROOT}" = "${DEFAULT_RUSTUP_UPDATE_ROOT}" ]; then + echo "Primary rustup mirror is unavailable, falling back to official endpoints." + export RUSTUP_DIST_SERVER="https://static.rust-lang.org" + export RUSTUP_UPDATE_ROOT="https://static.rust-lang.org/rustup" + install_rustup_with "https://sh.rustup.rs" + else + echo "Failed to install rustup from ${RUSTUP_INIT_URL}." + exit 1 + fi + fi + fi + + if [ -f "${HOME}/.cargo/env" ]; then + . "${HOME}/.cargo/env" + fi + export PATH="${HOME}/.cargo/bin:${PATH}" + + rustup set auto-self-update disable || true + rustup set profile minimal + + if ! rustup toolchain list | awk '{print $1}' | grep -Eq "^${REQUIRED_RUST_VERSION}(-|$)"; then + rustup toolchain install "${REQUIRED_RUST_VERSION}" --profile minimal --component rustfmt --no-self-update + fi + + rustup default "${REQUIRED_RUST_VERSION}" + rustup component add rustfmt --toolchain "${REQUIRED_RUST_VERSION}" || true +} + +ensure_cc_toolchain +ensure_protoc + +if ! has_usable_system_rust; then + setup_rustup_toolchain +fi + +if ! has_usable_system_rust; then + echo "Rust toolchain ${REQUIRED_RUST_VERSION} with rustfmt is not available after setup." + exit 1 +fi + +PROTOC_BIN="$(command -v protoc || true)" +if [ -z "${PROTOC_BIN}" ]; then + echo "protoc not found after installation." + exit 1 +fi + +PROTOC_INC="$(resolve_protoc_include || true)" +if [ -z "${PROTOC_INC}" ]; then + echo "Could not locate google/protobuf include dir for protoc." + exit 1 +fi + +if [ -n "${GITHUB_ENV:-}" ]; then + echo "PROTOC=${PROTOC_BIN}" >> "${GITHUB_ENV}" + echo "PROTOC_INCLUDE=${PROTOC_INC}" >> "${GITHUB_ENV}" +fi +export PROTOC="${PROTOC_BIN}" +export PROTOC_INCLUDE="${PROTOC_INC}" + +if [ -d "${HOME}/.cargo/bin" ] && [ -n "${GITHUB_PATH:-}" ]; then + echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" +fi + +cargo --version +rustc --version +protoc --version diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..04208b5 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,638 @@ +name: CI + +on: + push: + branches: + - master + - main + tags: + - "*" + pull_request: + workflow_dispatch: + +env: + REQUIRED_RUST_VERSION: 1.91.0 + +jobs: + backend: + name: Backend (Rust) + if: github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/') + runs-on: amd + steps: + - name: Checkout (shell) + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + SERVER_URL="${CI_REPO_BASE_URL:-${GITHUB_SERVER_URL%/}}" + PRIMARY_URL="${SERVER_URL}/${GITHUB_REPOSITORY}.git" + URLS="${PRIMARY_URL}" + case "${PRIMARY_URL}" in + https://*) URLS="${URLS} http://${PRIMARY_URL#https://}" ;; + http://*) URLS="${URLS} https://${PRIMARY_URL#http://}" ;; + esac + + [ -d .git ] || git init . + + BASIC_AUTH="" + if [ -n "${GITHUB_TOKEN:-}" ]; then + BASIC_AUTH="$(printf '%s' "${GITHUB_ACTOR}:${GITHUB_TOKEN}" | base64 | tr -d '\n')" + fi + git_fetch() { + if [ -n "${BASIC_AUTH}" ]; then + git -c "http.extraHeader=Authorization: Basic ${BASIC_AUTH}" fetch --depth 1 origin "${GITHUB_SHA}" + else + git fetch --depth 1 origin "${GITHUB_SHA}" + fi + } + + FETCH_OK=0 + for URL in ${URLS}; do + if git remote get-url origin >/dev/null 2>&1; then + git remote set-url origin "${URL}" + else + git remote add origin "${URL}" + fi + if git_fetch; then + FETCH_OK=1 + break + fi + done + + if [ "${FETCH_OK}" -ne 1 ]; then + echo "Failed to fetch ${GITHUB_SHA} from ${PRIMARY_URL} (tried http/https)." + exit 1 + fi + git checkout --force FETCH_HEAD + + - name: Setup Rust + run: bash .gitea/scripts/setup-rust.sh + + - name: Check format + run: | + set -euo pipefail + NIGHTLY_ONLY_KEYS='^(indent_style|wrap_comments|normalize_comments|format_macro_matchers|format_macro_bodies|struct_lit_single_line|fn_single_line|imports_indent|imports_layout|imports_granularity|group_imports|type_punctuation_density|space_before_colon|space_after_colon|spaces_around_ranges|binop_separator|overflow_delimited_expr|match_arm_blocks|force_multiline_blocks|brace_style|control_brace_style|trailing_semicolon|trailing_comma)\s*=' + if [ -f .rustfmt.toml ] && grep -Eq "${NIGHTLY_ONLY_KEYS}" .rustfmt.toml; then + echo "Skip rustfmt check: .rustfmt.toml includes nightly-only options." + exit 0 + fi + cargo fmt --all -- --check + + - name: Setup Node (shell) + run: | + set -euo pipefail + if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1; then + node --version + npm --version + exit 0 + fi + as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "Need root privileges to install nodejs/npm." + exit 1 + fi + } + if command -v pacman >/dev/null 2>&1; then + as_root pacman -Sy --noconfirm --needed nodejs npm + elif command -v apt-get >/dev/null 2>&1; then + as_root apt-get update + as_root apt-get install -y nodejs npm + elif command -v apk >/dev/null 2>&1; then + as_root apk add --no-cache nodejs npm + elif command -v dnf >/dev/null 2>&1; then + as_root dnf install -y nodejs npm + else + echo "Unsupported package manager. Preinstall nodejs/npm on runner." + exit 1 + fi + node --version + npm --version + + - name: Build frontend assets for RustEmbed + working-directory: frontend + run: | + npm config set registry https://registry.npmjs.org + npm config set replace-registry-host always + npm ci --cache .npm-cache --prefer-online + npm run build + + - name: Run tests + run: cargo test --workspace --all-features + + frontend: + name: Frontend (Vite) + if: github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/') + runs-on: amd + steps: + - name: Checkout (shell) + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + SERVER_URL="${CI_REPO_BASE_URL:-${GITHUB_SERVER_URL%/}}" + PRIMARY_URL="${SERVER_URL}/${GITHUB_REPOSITORY}.git" + URLS="${PRIMARY_URL}" + case "${PRIMARY_URL}" in + https://*) URLS="${URLS} http://${PRIMARY_URL#https://}" ;; + http://*) URLS="${URLS} https://${PRIMARY_URL#http://}" ;; + esac + + [ -d .git ] || git init . + + BASIC_AUTH="" + if [ -n "${GITHUB_TOKEN:-}" ]; then + BASIC_AUTH="$(printf '%s' "${GITHUB_ACTOR}:${GITHUB_TOKEN}" | base64 | tr -d '\n')" + fi + git_fetch() { + if [ -n "${BASIC_AUTH}" ]; then + git -c "http.extraHeader=Authorization: Basic ${BASIC_AUTH}" fetch --depth 1 origin "${GITHUB_SHA}" + else + git fetch --depth 1 origin "${GITHUB_SHA}" + fi + } + + FETCH_OK=0 + for URL in ${URLS}; do + if git remote get-url origin >/dev/null 2>&1; then + git remote set-url origin "${URL}" + else + git remote add origin "${URL}" + fi + if git_fetch; then + FETCH_OK=1 + break + fi + done + + if [ "${FETCH_OK}" -ne 1 ]; then + echo "Failed to fetch ${GITHUB_SHA} from ${PRIMARY_URL} (tried http/https)." + exit 1 + fi + git checkout --force FETCH_HEAD + + - name: Setup Node (shell) + run: | + set -euo pipefail + if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1; then + node --version + npm --version + exit 0 + fi + as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "Need root privileges to install nodejs/npm." + exit 1 + fi + } + if command -v pacman >/dev/null 2>&1; then + as_root pacman -Sy --noconfirm --needed nodejs npm + elif command -v apt-get >/dev/null 2>&1; then + as_root apt-get update + as_root apt-get install -y nodejs npm + elif command -v apk >/dev/null 2>&1; then + as_root apk add --no-cache nodejs npm + elif command -v dnf >/dev/null 2>&1; then + as_root dnf install -y nodejs npm + else + echo "Unsupported package manager. Preinstall nodejs/npm on runner." + exit 1 + fi + node --version + npm --version + + - name: Install dependencies + working-directory: frontend + run: | + npm config set registry https://registry.npmjs.org + npm config set replace-registry-host always + npm ci --cache .npm-cache --prefer-online + + - name: Build + working-directory: frontend + run: npm run build + + docker_release: + name: Docker Release Tar + if: startsWith(github.ref, 'refs/tags/') + runs-on: amd + permissions: + contents: write + releases: write + steps: + - name: Checkout (shell) + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + SERVER_URL="${CI_REPO_BASE_URL:-${GITHUB_SERVER_URL%/}}" + PRIMARY_URL="${SERVER_URL}/${GITHUB_REPOSITORY}.git" + URLS="${PRIMARY_URL}" + case "${PRIMARY_URL}" in + https://*) URLS="${URLS} http://${PRIMARY_URL#https://}" ;; + http://*) URLS="${URLS} https://${PRIMARY_URL#http://}" ;; + esac + + [ -d .git ] || git init . + + BASIC_AUTH="" + if [ -n "${GITHUB_TOKEN:-}" ]; then + BASIC_AUTH="$(printf '%s' "${GITHUB_ACTOR}:${GITHUB_TOKEN}" | base64 | tr -d '\n')" + fi + git_fetch() { + if [ -n "${BASIC_AUTH}" ]; then + git -c "http.extraHeader=Authorization: Basic ${BASIC_AUTH}" fetch --depth 1 origin "${GITHUB_SHA}" + else + git fetch --depth 1 origin "${GITHUB_SHA}" + fi + } + + FETCH_OK=0 + for URL in ${URLS}; do + if git remote get-url origin >/dev/null 2>&1; then + git remote set-url origin "${URL}" + else + git remote add origin "${URL}" + fi + if git_fetch; then + FETCH_OK=1 + break + fi + done + + if [ "${FETCH_OK}" -ne 1 ]; then + echo "Failed to fetch ${GITHUB_SHA} from ${PRIMARY_URL} (tried http/https)." + exit 1 + fi + git checkout --force FETCH_HEAD + + - name: Setup Rust + run: bash .gitea/scripts/setup-rust.sh + + - name: Setup Node (shell) + run: | + set -euo pipefail + if command -v node >/dev/null 2>&1 && command -v npm >/dev/null 2>&1; then + node --version + npm --version + exit 0 + fi + as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "Need root privileges to install nodejs/npm." + exit 1 + fi + } + if command -v pacman >/dev/null 2>&1; then + as_root pacman -Sy --noconfirm --needed nodejs npm + elif command -v apt-get >/dev/null 2>&1; then + as_root apt-get update + as_root apt-get install -y nodejs npm + elif command -v apk >/dev/null 2>&1; then + as_root apk add --no-cache nodejs npm + elif command -v dnf >/dev/null 2>&1; then + as_root dnf install -y nodejs npm + else + echo "Unsupported package manager. Preinstall nodejs/npm on runner." + exit 1 + fi + node --version + npm --version + + - name: Setup Docker (shell) + run: | + set -euo pipefail + as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "Need root privileges to install/start docker." + exit 1 + fi + } + if ! command -v docker >/dev/null 2>&1; then + if command -v pacman >/dev/null 2>&1; then + as_root pacman -Sy --noconfirm --needed docker + elif command -v apt-get >/dev/null 2>&1; then + as_root apt-get update + as_root apt-get install -y docker.io + elif command -v apk >/dev/null 2>&1; then + as_root apk add --no-cache docker + elif command -v dnf >/dev/null 2>&1; then + if ! as_root dnf install -y docker; then + as_root dnf install -y moby-engine + fi + else + echo "Unsupported package manager. Please preinstall docker." + exit 1 + fi + fi + + if ! docker info >/dev/null 2>&1; then + if command -v dockerd >/dev/null 2>&1; then + nohup dockerd >/tmp/dockerd.log 2>&1 & + for _ in $(seq 1 60); do + if docker info >/dev/null 2>&1; then + break + fi + sleep 1 + done + fi + fi + + if ! docker info >/dev/null 2>&1; then + echo "Docker daemon is not available. Runner must support docker (daemon or mounted /var/run/docker.sock)." + [ -f /tmp/dockerd.log ] && tail -n 80 /tmp/dockerd.log || true + exit 1 + fi + + docker version + + - name: Build Docker image by script + run: | + set -euo pipefail + TAG_NAME="${GITHUB_REF_NAME}" + SAFE_TAG="$(echo "${TAG_NAME}" | tr '/' '-')" + npm config set registry https://registry.npmjs.org + npm config set replace-registry-host always + IMAGE_TAG="${SAFE_TAG}" sh build-docker.sh + docker save "htknow:${SAFE_TAG}" -o "htknow-${SAFE_TAG}.tar" + gzip -f "htknow-${SAFE_TAG}.tar" + echo "TAG_NAME=${TAG_NAME}" >> "${GITHUB_ENV}" + echo "SAFE_TAG=${SAFE_TAG}" >> "${GITHUB_ENV}" + echo "ASSET_FILE=htknow-${SAFE_TAG}.tar.gz" >> "${GITHUB_ENV}" + + - name: Create release (if needed) and upload tar + env: + RELEASE_TOKEN: ${{ secrets.RELEASE_TOKEN }} + run: | + set -euo pipefail + as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "Need root privileges to install system packages." + exit 1 + fi + } + ensure_pkg() { + if command -v pacman >/dev/null 2>&1; then + as_root pacman -Sy --noconfirm --needed "$@" + elif command -v apt-get >/dev/null 2>&1; then + as_root apt-get update + as_root apt-get install -y "$@" + elif command -v apk >/dev/null 2>&1; then + as_root apk add --no-cache "$@" + elif command -v dnf >/dev/null 2>&1; then + as_root dnf install -y "$@" + else + echo "Unsupported package manager. Please preinstall required tools: $*" + exit 1 + fi + } + command -v curl >/dev/null 2>&1 || ensure_pkg curl + command -v jq >/dev/null 2>&1 || ensure_pkg jq + + if [ -z "${RELEASE_TOKEN:-}" ]; then + echo "secrets.RELEASE_TOKEN is required to create releases and upload assets." + exit 1 + fi + + REPO_OWNER="${GITHUB_REPOSITORY%%/*}" + REPO_NAME="${GITHUB_REPOSITORY##*/}" + API_BASE="${GITHUB_API_URL:-${GITHUB_SERVER_URL}/api/v1}" + AUTH_HEADER="Authorization: token ${RELEASE_TOKEN}" + RELEASE_URL="${API_BASE}/repos/${REPO_OWNER}/${REPO_NAME}/releases" + + LOOKUP_CODE="$(curl -sS -o release.json -w '%{http_code}' \ + -H "${AUTH_HEADER}" \ + "${RELEASE_URL}/tags/${TAG_NAME}")" + + if [ "${LOOKUP_CODE}" = "200" ]; then + echo "Release for tag ${TAG_NAME} already exists." + elif [ "${LOOKUP_CODE}" = "404" ]; then + cat > create-release.json </dev/null + echo "Deleted existing asset ${ASSET_FILE}." + fi + + curl -sS -f -X POST \ + -H "${AUTH_HEADER}" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @"${ASSET_FILE}" \ + "${RELEASE_URL}/${RELEASE_ID}/assets?name=${ASSET_FILE}" >/dev/null + + echo "Uploaded ${ASSET_FILE} to release ${TAG_NAME}." + + - name: Deploy to target server + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + DEPLOY_PASSWORD: ${{ secrets.DEPLOY_PASSWORD }} + DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }} + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_COMMAND: ${{ secrets.DEPLOY_COMMAND }} + run: | + set -euo pipefail + + if [ -z "${DEPLOY_HOST:-}" ] || [ -z "${DEPLOY_USER:-}" ]; then + echo "Deploy secrets not fully configured (DEPLOY_HOST/DEPLOY_USER). Skip deployment." + exit 0 + fi + if [ -z "${DEPLOY_PASSWORD:-}" ] && [ -z "${DEPLOY_SSH_KEY:-}" ]; then + echo "Neither DEPLOY_PASSWORD nor DEPLOY_SSH_KEY is configured. Skip deployment." + exit 0 + fi + + as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "Need root privileges to install openssh client." + exit 1 + fi + } + ensure_pkg() { + if command -v pacman >/dev/null 2>&1; then + as_root pacman -Sy --noconfirm --needed "$@" + elif command -v apt-get >/dev/null 2>&1; then + as_root apt-get update + as_root apt-get install -y "$@" + elif command -v apk >/dev/null 2>&1; then + as_root apk add --no-cache "$@" + elif command -v dnf >/dev/null 2>&1; then + as_root dnf install -y "$@" + else + echo "Unsupported package manager. Please preinstall openssh client." + exit 1 + fi + } + + ensure_ssh_client() { + if command -v ssh >/dev/null 2>&1 && command -v scp >/dev/null 2>&1 && command -v ssh-keyscan >/dev/null 2>&1; then + return + fi + if command -v pacman >/dev/null 2>&1; then + ensure_pkg openssh + elif command -v apt-get >/dev/null 2>&1; then + ensure_pkg openssh-client + elif command -v apk >/dev/null 2>&1; then + ensure_pkg openssh-client + elif command -v dnf >/dev/null 2>&1; then + ensure_pkg openssh-clients + else + echo "Unsupported package manager. Please preinstall ssh/scp." + exit 1 + fi + } + ensure_ssh_client + + PORT="${DEPLOY_PORT:-22}" + REMOTE_PATH_RAW="${DEPLOY_PATH:-/tmp/htknow-release}" + case "${REMOTE_PATH_RAW}" in + "~") REMOTE_PATH_REMOTE="\$HOME" ;; + "~/"*) REMOTE_PATH_REMOTE="\$HOME/${REMOTE_PATH_RAW#~/}" ;; + *) REMOTE_PATH_REMOTE="${REMOTE_PATH_RAW}" ;; + esac + KNOWN_HOSTS="${HOME}/.ssh/known_hosts" + + mkdir -p ~/.ssh + chmod 700 ~/.ssh + touch "${KNOWN_HOSTS}" + chmod 600 "${KNOWN_HOSTS}" + ssh-keyscan -T 10 -p "${PORT}" -H "${DEPLOY_HOST}" >> "${KNOWN_HOSTS}" 2>/dev/null || true + + run_ssh() { + ssh -p "${PORT}" \ + -o BatchMode=yes \ + -o ConnectTimeout=10 \ + -o ServerAliveInterval=10 \ + -o ServerAliveCountMax=3 \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="${KNOWN_HOSTS}" \ + -i "${HOME}/.ssh/id_deploy" \ + "$@" + } + run_scp() { + scp -P "${PORT}" \ + -o BatchMode=yes \ + -o ConnectTimeout=10 \ + -o ServerAliveInterval=10 \ + -o ServerAliveCountMax=3 \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="${KNOWN_HOSTS}" \ + -i "${HOME}/.ssh/id_deploy" \ + "$@" + } + + if [ -n "${DEPLOY_PASSWORD:-}" ]; then + command -v sshpass >/dev/null 2>&1 || ensure_pkg sshpass + export SSHPASS="${DEPLOY_PASSWORD}" + run_ssh() { + sshpass -e ssh -p "${PORT}" \ + -o PreferredAuthentications=password \ + -o PubkeyAuthentication=no \ + -o ConnectTimeout=10 \ + -o ServerAliveInterval=10 \ + -o ServerAliveCountMax=3 \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="${KNOWN_HOSTS}" \ + "$@" + } + run_scp() { + sshpass -e scp -P "${PORT}" \ + -o PreferredAuthentications=password \ + -o PubkeyAuthentication=no \ + -o ConnectTimeout=10 \ + -o ServerAliveInterval=10 \ + -o ServerAliveCountMax=3 \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="${KNOWN_HOSTS}" \ + "$@" + } + else + printf '%s\n' "${DEPLOY_SSH_KEY}" | sed 's/\r$//' > ~/.ssh/id_deploy + chmod 600 ~/.ssh/id_deploy + fi + + if ! run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "echo 'ssh connectivity ok'"; then + echo "SSH connectivity check failed. Verify DEPLOY_HOST/DEPLOY_PORT/DEPLOY_USER and DEPLOY_PASSWORD or DEPLOY_SSH_KEY." + exit 1 + fi + + if [ ! -f "${ASSET_FILE}" ]; then + echo "Local asset file not found: ${ASSET_FILE}" + ls -lah + exit 1 + fi + + run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "sh -lc 'set -e; REMOTE_PATH=\"${REMOTE_PATH_REMOTE}\"; REMOTE_PATH=\$(eval echo \"\${REMOTE_PATH}\"); mkdir -p \"\${REMOTE_PATH}\"'" + run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "sh -lc 'set -e; REMOTE_PATH=\"${REMOTE_PATH_REMOTE}\"; REMOTE_PATH=\$(eval echo \"\${REMOTE_PATH}\"); cat > \"\${REMOTE_PATH}/${ASSET_FILE}\"'" < "${ASSET_FILE}" + run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "sh -lc 'set -e; REMOTE_PATH=\"${REMOTE_PATH_REMOTE}\"; REMOTE_PATH=\$(eval echo \"\${REMOTE_PATH}\"); ls -lh \"\${REMOTE_PATH}/${ASSET_FILE}\"'" + + if [ -n "${DEPLOY_COMMAND:-}" ]; then + DEPLOY_CMD_B64="$(printf '%s' "${DEPLOY_COMMAND}" | base64 | tr -d '\n')" + run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" \ + "ASSET_FILE='${ASSET_FILE}' SAFE_TAG='${SAFE_TAG}' DEPLOY_PATH='${REMOTE_PATH_REMOTE}' DEPLOY_CMD_B64='${DEPLOY_CMD_B64}' sh -lc 'DEPLOY_PATH=\$(eval echo \"\$DEPLOY_PATH\"); printf %s \"\$DEPLOY_CMD_B64\" | base64 -d | sh'" + else + run_ssh "${DEPLOY_USER}@${DEPLOY_HOST}" "sh -lc ' + set -e + DEPLOY_PATH=\"${REMOTE_PATH_REMOTE}\" + DEPLOY_PATH=\$(eval echo \"\$DEPLOY_PATH\") + EXTRACTED_TAR_FILE=\"${ASSET_FILE%.gz}\" + gzip -dc \"\$DEPLOY_PATH/${ASSET_FILE}\" > \"\$DEPLOY_PATH/\$EXTRACTED_TAR_FILE\" + docker load -i \"\$DEPLOY_PATH/\$EXTRACTED_TAR_FILE\" + rm -f \"\$DEPLOY_PATH/\$EXTRACTED_TAR_FILE\" + docker rm -f htknow >/dev/null 2>&1 || true + docker run -d --name htknow --restart unless-stopped -p 8080:8080 -v /opt/htknow/data:/app/data \"htknow:${SAFE_TAG}\" + '" + fi + + echo "Deployed htknow:${SAFE_TAG} to ${DEPLOY_USER}@${DEPLOY_HOST}:${REMOTE_PATH_RAW}" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..42d28c3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/target +app.sqlite* +/data +*.log +.DS_Store +*.pdf diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000..ba204e7 --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,60 @@ +# .rustfmt.toml + +# 基本设置 +edition = "2024" +max_width = 120 # 每行最大字符数（建议 80~120） +hard_tabs = false # 使用空格而不是制表符 +tab_spaces = 4 # 一个缩进等于 4 个空格 +newline_style = "Unix" # 换行符使用 \n（Linux/macOS/WSL 风格） + +use_small_heuristics = "Max" # 尽量将短的 use 合并成一行 + + +# 空格与括号 +space_before_colon = false # 类型声明中冒号前不加空格: `foo: i32` +space_after_colon = true # 冒号后加空格 +spaces_around_ranges = false # 不在 range 周围加空格: `..` 而不是 ` .. ` + +# 控制结构格式 +indent_style = "Block" # 使用块式缩进（而非 Visual） +control_brace_style = "AlwaysSameLine" # if/else、fn 的 `{` 放下一行 +brace_style = "PreferSameLine" # 结构体、enum 的 `{` 放同一行（除非有 where） + +# 函数相关 +fn_params_layout = "Compressed" # 函数参数紧凑布局（少换行） +fn_single_line = false # 不强制简单函数写成单行 + +# 结构体与元组 +struct_lit_single_line = true # 简短结构体字面量放在一行: `Point { x: 1, y: 2 }` + +# use 语句的分组方式 +group_imports = "StdExternalCrate" # 可选: "StdExternalCrate", "Preserve", "One" +# 导入排序 +reorder_imports = true # 自动对 use 导入语句排序 +reorder_modules = true # 是否对模块内的 use 按照路径排序 +imports_indent = "Block" # use 导入缩进为块风格 +imports_layout = "Horizontal" # 垂直排列导入（每行一个） +# 导入同一模块的类型，应该置于同一个块内 +imports_granularity = "Crate" + +# 模式匹配 +match_arm_blocks = true # match 分支使用块时换行 +match_arm_leading_pipes = "Never" # 不在 match 分支前加 `|` +force_multiline_blocks = false # 不强制复杂块换行 + +# 表达式 +trailing_comma = "Vertical" # 垂直列表末尾加逗号 +trailing_semicolon = true # 语句结尾加分号 +wrap_comments = false # 不自动换行注释（保留手动格式） + +# 类型与泛型 +type_punctuation_density = "Compressed" # 类型标点紧凑: `Vec` 而非 `Vec< i32 >` +binop_separator = "Front" # 二元操作符换行时，操作符放下一行开头 + +# 宏 +format_macro_matchers = true # 格式化 macro_rules! 中的匹配模式 +format_macro_bodies = true # 格式化宏体内容 + +# 高级（按需启用） +normalize_comments = false # 不自动规范化注释大小写或空格 +overflow_delimited_expr = true # 允许括号表达式溢出时不换行（实验性） diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..6881d54 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,159 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +HTKnow is a Chinese knowledge base management system. The backend is Rust (axum + SQLite + Tantivy + LanceDB) and the frontend is a Vue 3 SPA built with Vite and Tailwind CSS 4. The built frontend is embedded into the Rust binary via `rust-embed`. + +## Common Commands + +### Backend + +```bash +# Run locally (requires external services; see below) +cargo run + +# Release build +cargo build --release + +# Run all tests +cargo test + +# Run a specific integration test +cargo test knowledge_base_flow + +# Run only integration tests +cargo test --test api_integration + +# Format code (uses .rustfmt.toml config) +cargo fmt + +# Build with etcd config center support +cargo build --features etcd + +# Build with jemalloc heap profiling (debug only) +cargo build --features profiling +``` + +Rust edition is 2024 and the project uses the nightly toolchain. Formatting is configured in `.rustfmt.toml` (120 char width, 4-space indentation, `StdExternalCrate` import grouping). + +### Frontend + +```bash +cd frontend +npm ci +npm run dev # Dev server proxies /api to localhost:3000 +npm run build # Outputs to frontend/dist/ (embedded in binary) +``` + +### Docker + +```bash +# Run with docker-compose (port 8080 -> 3000) +docker compose up -d + +# Build Docker image (builds frontend + Rust binary) +./build-docker.sh + +# Debug build with profiling tools +./build-docker.sh debug +``` + +## Architecture + +### Entry Point and Routing (`src/main.rs`, `src/lib.rs`, `src/api/mod.rs`) + +The axum server is started in `main.rs`. Routes are defined in `src/api/mod.rs` and organized into nested routers: + +- `/api/v1/knowledge/knowledge_base/` — KB CRUD, tree, reparse +- `/api/v1/knowledge/files/` — upload, list, download, slices, highlighted PDF +- `/api/v1/knowledge/search/` — full-text, vector, image, graph-enhanced, advanced stream, lexicon/synonym management +- `/api/v1/knowledge/graph/` — entity search, entity detail, graph stats +- `/api/v1/knowledge/system/` — heap profiling, LanceDB compact, index force merge/rebuild + +All `/api/v1/knowledge/*` routes require the `auth` middleware (`src/lib.rs`), which extracts `x-user-id` and `x-role` from headers into an `AuthUser` extension. `x-user-name` is optional and base64-decoded if valid UTF-8. + +Swagger UI is at `/docs` and the OpenAPI JSON is at `/api-docs/openapi.json`. + +### Database (`src/db.rs`, `src/init.sql`) + +SQLite with WAL mode enabled. The schema is auto-initialized on startup from `src/init.sql`. Key tables include `knowledge_base`, `file`, `file_slice`, `entity`, `relation`, `mention`, `lexicon`, and `synonym`. The pool is created via `sqlx` and passed as axum state to handlers. + +### Search (`src/search/`) + +`SearchEngine` (`src/search/mod.rs`) orchestrates three search backends: + +- **Tantivy** (`src/search/tantivy_engine.rs`) — full-text search with a custom Chinese tokenizer based on `jieba-rs`. Maintains two indexes: a default index and a "full" index. Supports lexicon-based synonym expansion. +- **LanceDB** (`src/search/lancedb.rs`) — vector search. Stores text and image embeddings in separate tables. +- **Embedding client** (`src/search/embedding.rs`) — calls external embedding services for text and image vectors. + +The search module also handles reranking via an external service and provides an advanced streaming search endpoint (`src/search/advanced.rs`). + +### Background File Processing (`src/processor.rs`) + +`FileProcessor` runs as a background task polling the database for pending files. It: + +1. Converts Office docs to PDF via an external service. +2. Parses PDFs via MinerU (or a custom parse service if configured). +3. Transcribes audio files via an external service. +4. Splits content into slices (smart or fixed overlap). +5. Stores slices in SQLite, indexes them in Tantivy, and generates embeddings for LanceDB. +6. Optionally builds a knowledge graph via LLM extraction (if `HTKNOW_BUILD_KNOWLEDGE_GRAPH=true`). + +File statuses: `0=pending`, `1=completed`, `2=processing`, `3=skipped`, `-1=failed`. + +### Knowledge Graph (`src/graph/`) + +- `graph_manager.rs` — manages entities, relations, and mentions in SQLite. +- `llm_extractor.rs` — uses an LLM API to extract entities and relations from file content. + +Graph data is stored relationally in SQLite and queried through the graph API endpoints. + +### Configuration (`src/config.rs`) + +All configuration is environment-variable based with sensible defaults. No config files. Key prefixes: + +- `HTKNOW_SERVER_*` — server settings +- `HTKNOW_MINERU_URL`, `HTKNOW_OFFICE_CONVERT_URL`, `HTKNOW_EMBEDDING_URL`, etc. — external services +- `DATABASE_URL` — SQLite connection string +- `HTKNOW_DATA_DIR`, `HTKNOW_LANCEDB_PATH`, `HTKNOW_TANTIVY_INDEX_PATH` — storage paths +- `LLM_API_URL`, `LLM_API_KEY`, `LLM_MODEL` — optional LLM for knowledge graph + +See `README.md` for the full environment variable table. + +### Frontend (`frontend/`) + +Vue 3 SPA using Vite and Tailwind CSS 4. The `vite.config.js` proxies `/api` to `http://localhost:3000` in dev mode. Built assets in `frontend/dist/` are served by the Rust binary via `rust-embed` (`src/frontend.rs`). + +## External Service Dependencies + +For full functionality, these external services must be running: + +- **MinerU** — PDF parsing (`HTKNOW_MINERU_URL`) +- **Office converter** — Office to PDF (`HTKNOW_OFFICE_CONVERT_URL`) +- **Embedding service** — text embeddings (`HTKNOW_EMBEDDING_URL`) +- **Image embedding service** — image embeddings (`HTKNOW_IMAGE_EMBEDDING_URL`) +- **Rerank service** — result reranking (`HTKNOW_RERANK_URL`) +- **Audio transcription** — audio to text (`HTKNOW_AUDIO_TRANSCRIPTION_URL`) + +Default values in `docker-compose.yml` and `src/config.rs` point to internal network IPs. + +## Testing + +Integration tests live in `tests/api_integration.rs`. They set up an isolated test environment with a temp directory and a fresh SQLite database, then exercise the full axum router. + +The `APP` router is initialized once via `tokio::sync::OnceCell` and reused across tests. Environment variables are set via `unsafe { std::env::set_var(...) }` during test setup. + +## Memory Profiling (Debug Builds) + +Debug builds use jemalloc as the global allocator. With the `profiling` feature, heap profiling is enabled and accessible via: + +- `GET /api/v1/knowledge/system/heap` — returns a heap profile +- `GET /api/v1/knowledge/system/heap/pdf` — returns a PDF visualization + +## Notes + +- LanceDB auto-compact runs on a cron schedule (`HTKNOW_LANCEDB_COMPACT_CRON`, default `0 0 3 * * *`). Compaction holds a lock to prevent concurrent writes. +- If Tantivy indexes become corrupted (e.g., from an unclean shutdown), the app will panic on startup. Recovery: back up and remove the `tantivy_index` and `tantivy_full_index` directories, then trigger a rebuild via the system endpoints. +- The `cargo build --features etcd` flag enables etcd as a config center, but the etcd integration is optional and not enabled by default. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..fbe2e8d --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,7367 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arc-swap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d03449bb8ca2cc2ef70869af31463d1ae5ccc8fa3e334b307203fbf815207e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "arrow" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da746f4180004e3ce7b83c977daf6394d768332349d3d913998b10a120b790a" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex 0.12.1", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap 2.12.1", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" +dependencies = [ + "bitflags", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-compression" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ec5f6c2f8bc326c994cb9e241cc257ddaba9afa8555a43cffbb5dd86efaa37" +dependencies = [ + "compression-codecs", + "compression-core", + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async_cell" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447ab28afbb345f5408b120702a44e5529ebf90b1796ec76e9528df8e288e6c2" +dependencies = [ + "loom", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core 0.5.5", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "multer", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +dependencies = [ + "serde_core", +] + +[[package]] +name = "bitpacking" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" +dependencies = [ + "crunchy", +] + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq 0.4.2", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bon" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "calamine" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138646b9af2c5d7f1804ea4bf93afc597737d2bd4f7341d67c48b03316976eb1" +dependencies = [ + "byteorder", + "codepage", + "encoding_rs", + "log", + "quick-xml", + "serde", + "zip 2.4.2", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.2.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cedarwood" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90" +dependencies = [ + "smallvec", +] + +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "comfy-table" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "compression-codecs" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f7ac3e5b97fdce45e8922fb05cae2c37f7bbd63d30dd94821dacfd8f3f2bf2" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "croner" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c344b0690c1ad1c7176fe18eb173e0c927008fdaaa256e40dfd43ddd149c0843" +dependencies = [ + "chrono", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7541353e77dc7262b71ca27be07d8393661737e3a73b5d1b1c6f7d814c64fa2a" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bytes", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "regex", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion-catalog" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9997731f90fa5398ef831ad0e69600f92c861b79c0d38bd1a29b6f0e3a0ce4c8" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b30a3dd50dec860c9559275c8d97d9de602e611237a6ecfbda0b3b63b872352" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools 0.14.0", + "log", + "object_store", +] + +[[package]] +name = "datafusion-common" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d551054acec0398ca604512310b77ce05c46f66e54b54d48200a686e385cca4e" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc", + "chrono", + "half", + "hashbrown 0.16.1", + "indexmap 2.12.1", + "libc", + "log", + "object_store", + "paste", + "sqlparser", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567d40e285f5b79f8737b576605721cd6c1133b5d2b00bdbd5d9838d90d0812f" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d2668f51b3b30befae2207472569e37807fdedd1d14da58acc6f8ca6257eae" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store", + "rand 0.9.2", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e02e1b3e3a8ec55f1f62de4252b0407c8567363d056078769a197e24fc834a0f" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b559d7bf87d4f900f847baba8509634f838d9718695389e903604cdcccdb01f3" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250e2d7591ba8b638f063854650faa40bca4e8bd4059b2ece8836f6388d02db4" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9496cb0db222dbb9a3735760ceca7fc56f35e1d5502c38d0caa77a81e9c1f6a" + +[[package]] +name = "datafusion-execution" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc45d23c516ed8d3637751e44e09e21b45b3f58b473c802dddd1f1ad4fe435ff" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "dashmap", + "datafusion-common", + "datafusion-expr", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63dd30526d2db4fda6440806a41e4676334a94bc0596cc9cc2a0efed20ef2c44" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap 2.12.1", + "itertools 0.14.0", + "paste", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b486b5f6255d40976b88bb83813b0d035a8333e0ec39864824e78068cf42fa6" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap 2.12.1", + "itertools 0.14.0", + "paste", +] + +[[package]] +name = "datafusion-functions" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07356c94118d881130dd0ffbff127540407d969c8978736e324edcd6c41cd48f" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "hex", + "itertools 0.14.0", + "log", + "md-5", + "num-traits", + "rand 0.9.2", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b644f9cf696df9233ce6958b9807666d78563b56f923267474dd6c07795f1f8f" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1de2deaaabe8923ce9ea9f29c47bbb4ee14f67ea2fe1ab5398d9bbebcf86e56" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552f8d92e4331ee91d23c02d12bb6acf32cbfd5215117e01c0fb63cd4b15af1a" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "itertools 0.14.0", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-table" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970fd0cdd3df8802b9a9975ff600998289ba9d46682a4f7285cba4820c9ada78" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b4c21a7c8a986a1866c0a87ab756d0bbf7b5f41f306009fa2d9af79c52ed31" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1210ad73b8b3211aeaf4a42bef9bd7a2b7fce3ec119a478831f18c6ff7f7b93" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaa566a963013a38681ad82a727a654bc7feb19632426aea8c3412d415d200c5" +dependencies = [ + "datafusion-doc", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "datafusion-optimizer" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff9aa82b240252a88dee118372f9b9757c545ab9e53c0736bebab2e7da0ef1f2" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap 2.12.1", + "itertools 0.14.0", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d48022b8af9988c1d852644f9e8b5584c490659769a550c5e8d39457a1da0a5" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.16.1", + "indexmap 2.12.1", + "itertools 0.14.0", + "parking_lot", + "paste", + "petgraph 0.8.3", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae7a8abc0b4fe624000972a9b145b30b7f1b680bffaa950ea53f78d9b21c27c3" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147253ca3e6b9d59c162de64c02800973018660e13340dd1886dd038d17ac429" +dependencies = [ + "ahash", + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.16.1", + "indexmap 2.12.1", + "itertools 0.14.0", + "parking_lot", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "689156bb2282107b6239db8d7ef44b4dab10a9b33d3491a0c74acac5e4fedd72" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-plan" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68253dc0ee5330aa558b2549c9b0da5af9fc17d753ae73022939014ad616fc28" +dependencies = [ + "ahash", + "arrow", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.16.1", + "indexmap 2.12.1", + "itertools 0.14.0", + "log", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-pruning" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcad240a54d0b1d3e8f668398900260a53122d522b2102ab57218590decacd6" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", +] + +[[package]] +name = "datafusion-session" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58e83a68bb67007a8fcbf005c44cefe441270c7ee7f6dee10c0e0109b556f6d" +dependencies = [ + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-sql" +version = "52.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be53e9eb55db0fbb8980bb6d87f2435b0524acf4c718ed54a57cabbb299b2ab3" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "indexmap 2.12.1", + "log", + "regex", + "sqlparser", +] + +[[package]] +name = "deepsize" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c" +dependencies = [ + "deepsize_derive", +] + +[[package]] +name = "deepsize_derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "deflate64" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.117", + "unicode-xid", +] + +[[package]] +name = "destructure_traitobject" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c877555693c14d2f84191cfd3ad8582790fc52b5e2274b40b59cf5f5cea25c7" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcd-client" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0452bcc559431b16f472b7ab86e2f9ccd5f3c2da3795afbd6b773665e047fe" +dependencies = [ + "http", + "prost 0.13.5", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tower 0.4.13", + "tower-service", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "ethnum" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fast-float2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +dependencies = [ + "crc32fast", + "libz-rs-sys", + "miniz_oxide", +] + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix 0.38.44", + "windows-sys 0.52.0", +] + +[[package]] +name = "fsst" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195cc7f87e84bd695586137de99605e7e9579b26ec5e01b82960ddb4d0922f2" +dependencies = [ + "arrow-array", + "rand 0.9.2", +] + +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" +dependencies = [ + "utf8-ranges", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generator" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f04ae4152da20c76fe800fa48659201d5cf627c5149ca0b707b69d7eef6cf9" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.12.1", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "htknow" +version = "0.1.0" +dependencies = [ + "aho-corasick", + "anyhow", + "arrow-array", + "arrow-schema", + "axum 0.8.8", + "base64", + "bincode", + "bytes", + "bzip2 0.4.4", + "calamine", + "chrono", + "encoding_rs", + "etcd-client", + "flate2", + "futures", + "hex", + "http-body-util", + "jieba-rs", + "lancedb", + "lazy_static", + "log", + "log4rs", + "lopdf", + "mime_guess", + "once_cell", + "petgraph 0.6.5", + "regex", + "reqwest", + "rust-embed", + "serde", + "serde_json", + "serde_with", + "sha2", + "sqlx", + "sysinfo", + "tantivy", + "tar", + "tempfile", + "thiserror 1.0.69", + "tikv-jemalloc-ctl", + "tikv-jemallocator", + "tokio", + "tokio-cron-scheduler", + "tokio-stream", + "tokio-util", + "tower 0.5.2", + "utoipa", + "utoipa-swagger-ui", + "uuid", + "xz2", + "zip 2.4.2", +] + +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.1", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperloglogplus" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "621debdf94dcac33e50475fdd76d34d5ea9c0362a834b9db08c3024696c1fbe3" +dependencies = [ + "serde", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010" + +[[package]] +name = "jieba-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f0c1347cd3ac8d7c6e3a2dc33ac496d365cf09fc0831aa61111e1a6738983e" +dependencies = [ + "cedarwood", + "fxhash", + "hashbrown 0.14.5", + "lazy_static", + "phf 0.11.3", + "phf_codegen", + "regex", +] + +[[package]] +name = "jiff" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys 0.61.2", +] + +[[package]] +name = "jiff-static" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "jsonb" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953" +dependencies = [ + "byteorder", + "ethnum", + "fast-float2", + "itoa", + "jiff", + "nom 8.0.0", + "num-traits", + "ordered-float 5.3.0", + "rand 0.9.2", + "serde", + "serde_json", + "zmij 1.0.21", +] + +[[package]] +name = "lance" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efe6c3ddd79cdfd2b7e1c23cafae52806906bc40fbd97de9e8cf2f8c7a75fc04" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "async_cell", + "byteorder", + "bytes", + "chrono", + "crossbeam-skiplist", + "dashmap", + "datafusion", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-plan", + "deepsize", + "either", + "futures", + "half", + "humantime", + "itertools 0.13.0", + "lance-arrow", + "lance-core", + "lance-datafusion", + "lance-encoding", + "lance-file", + "lance-index", + "lance-io", + "lance-linalg", + "lance-namespace", + "lance-table", + "log", + "moka", + "object_store", + "permutation", + "pin-project", + "prost 0.14.3", + "prost-types 0.14.3", + "rand 0.9.2", + "roaring", + "semver", + "serde", + "serde_json", + "snafu 0.9.0", + "tantivy", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "lance-arrow" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d9f5d95bdda2a2b790f1fb8028b5b6dcf661abeb3133a8bca0f3d24b054af87" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "bytes", + "futures", + "getrandom 0.2.16", + "half", + "jsonb", + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "lance-bitpacking" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f827d6ab9f8f337a9509d5ad66a12f3314db8713868260521c344ef6135eb4e4" +dependencies = [ + "arrayref", + "paste", + "seq-macro", +] + +[[package]] +name = "lance-core" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1e25df6a79bf72ee6bcde0851f19b1cd36c5848c1b7db83340882d3c9fdecb" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "async-trait", + "byteorder", + "bytes", + "chrono", + "datafusion-common", + "datafusion-sql", + "deepsize", + "futures", + "itertools 0.13.0", + "lance-arrow", + "libc", + "log", + "mock_instant", + "moka", + "num_cpus", + "object_store", + "pin-project", + "prost 0.14.3", + "rand 0.9.2", + "roaring", + "serde_json", + "snafu 0.9.0", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "lance-datafusion" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93146de8ae720cb90edef81c2f2d0a1b065fc2f23ecff2419546f389b0fa70a4" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-trait", + "chrono", + "datafusion", + "datafusion-common", + "datafusion-functions", + "datafusion-physical-expr", + "futures", + "jsonb", + "lance-arrow", + "lance-core", + "lance-datagen", + "log", + "pin-project", + "prost 0.14.3", + "prost-build 0.14.3", + "snafu 0.9.0", + "tokio", + "tracing", +] + +[[package]] +name = "lance-datagen" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccec8ce4d8e0a87a99c431dab2364398029f2ffb649c1a693c60c79e05ed30dd" +dependencies = [ + "arrow", + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "futures", + "half", + "hex", + "rand 0.9.2", + "rand_distr 0.5.1", + "rand_xoshiro", + "random_word", +] + +[[package]] +name = "lance-encoding" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1aec0bbbac6bce829bc10f1ba066258126100596c375fb71908ecf11c2c2a5" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "bytemuck", + "byteorder", + "bytes", + "fsst", + "futures", + "hex", + "hyperloglogplus", + "itertools 0.13.0", + "lance-arrow", + "lance-bitpacking", + "lance-core", + "log", + "lz4", + "num-traits", + "prost 0.14.3", + "prost-build 0.14.3", + "prost-types 0.14.3", + "rand 0.9.2", + "snafu 0.9.0", + "strum", + "tokio", + "tracing", + "xxhash-rust", + "zstd", +] + +[[package]] +name = "lance-file" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14a8c548804f5b17486dc2d3282356ed1957095a852780283bc401fdd69e9075" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "byteorder", + "bytes", + "datafusion-common", + "deepsize", + "futures", + "lance-arrow", + "lance-core", + "lance-encoding", + "lance-io", + "log", + "num-traits", + "object_store", + "prost 0.14.3", + "prost-build 0.14.3", + "prost-types 0.14.3", + "snafu 0.9.0", + "tokio", + "tracing", +] + +[[package]] +name = "lance-index" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da212f0090ea59f79ac3686660f596520c167fe1cb5f408900cf71d215f0e03" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-channel", + "async-recursion", + "async-trait", + "bitpacking", + "bitvec", + "bytes", + "chrono", + "crossbeam-queue", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-sql", + "deepsize", + "dirs", + "fst", + "futures", + "half", + "itertools 0.13.0", + "jsonb", + "lance-arrow", + "lance-core", + "lance-datafusion", + "lance-datagen", + "lance-encoding", + "lance-file", + "lance-io", + "lance-linalg", + "lance-table", + "libm", + "log", + "ndarray", + "num-traits", + "object_store", + "prost 0.14.3", + "prost-build 0.14.3", + "prost-types 0.14.3", + "rand 0.9.2", + "rand_distr 0.5.1", + "rangemap", + "rayon", + "roaring", + "serde", + "serde_json", + "smallvec", + "snafu 0.9.0", + "tantivy", + "tempfile", + "tokio", + "tracing", + "twox-hash", + "uuid", +] + +[[package]] +name = "lance-io" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d958eb4b56f03bbe0f5f85eb2b4e9657882812297b6f711f201ffc995f259f" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "byteorder", + "bytes", + "chrono", + "deepsize", + "futures", + "http", + "lance-arrow", + "lance-core", + "lance-namespace", + "log", + "object_store", + "path_abs", + "pin-project", + "prost 0.14.3", + "rand 0.9.2", + "serde", + "snafu 0.9.0", + "tempfile", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "lance-linalg" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0285b70da35def7ed95e150fae1d5308089554e1290470403ed3c50cb235bc5e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "cc", + "deepsize", + "half", + "lance-arrow", + "lance-core", + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "lance-namespace" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f78e2a828b654e062a495462c6e3eb4fcf0e7e907d761b8f217fc09ccd3ceac" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "lance-core", + "lance-namespace-reqwest-client", + "serde", + "snafu 0.9.0", +] + +[[package]] +name = "lance-namespace-impls" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2392314f3da38f00d166295e44244208a65ccfc256e274fa8631849fc3f4d94" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "async-trait", + "bytes", + "chrono", + "futures", + "lance", + "lance-core", + "lance-index", + "lance-io", + "lance-namespace", + "lance-table", + "log", + "object_store", + "rand 0.9.2", + "serde_json", + "snafu 0.9.0", + "tokio", + "url", +] + +[[package]] +name = "lance-namespace-reqwest-client" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2e48de899e2931afb67fcddd0a08e439bf5d8b6ea2a2ed9cb8f4df669bd5cc" +dependencies = [ + "reqwest", + "serde", + "serde_json", + "serde_repr", + "url", +] + +[[package]] +name = "lance-table" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df9c4adca3eb2074b3850432a9fb34248a3d90c3d6427d158b13ff9355664ee" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ipc", + "arrow-schema", + "async-trait", + "byteorder", + "bytes", + "chrono", + "deepsize", + "futures", + "lance-arrow", + "lance-core", + "lance-file", + "lance-io", + "log", + "object_store", + "prost 0.14.3", + "prost-build 0.14.3", + "prost-types 0.14.3", + "rand 0.9.2", + "rangemap", + "roaring", + "semver", + "serde", + "serde_json", + "snafu 0.9.0", + "tokio", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "lance-testing" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ed7119bdd6983718387b4ac44af873a165262ca94f181b104cd6f97912eb3bf" +dependencies = [ + "arrow-array", + "arrow-schema", + "lance-arrow", + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "lancedb" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce0f4d7f739dc30608fe8b202cbb40986c2937e1a5a189f98fb06d7b8543156a" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "chrono", + "datafusion", + "datafusion-catalog", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-sql", + "futures", + "half", + "lance", + "lance-arrow", + "lance-core", + "lance-datafusion", + "lance-datagen", + "lance-encoding", + "lance-file", + "lance-index", + "lance-io", + "lance-linalg", + "lance-namespace", + "lance-namespace-impls", + "lance-table", + "lance-testing", + "lazy_static", + "log", + "moka", + "num-traits", + "object_store", + "pin-project", + "rand 0.9.2", + "regex", + "semver", + "serde", + "serde_json", + "serde_with", + "snafu 0.8.9", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.178" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libredox" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50" +dependencies = [ + "bitflags", + "libc", + "redox_syscall 0.6.0", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-rs-sys" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" +dependencies = [ + "zlib-rs", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +dependencies = [ + "serde_core", +] + +[[package]] +name = "log-mdc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a94d21414c1f4a51209ad204c1776a3d0765002c76c6abcb602a6f09f1e881c7" + +[[package]] +name = "log4rs" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e947bb896e702c711fccc2bf02ab2abb6072910693818d1d6b07ee2b9dfd86c" +dependencies = [ + "anyhow", + "arc-swap", + "chrono", + "derive_more", + "fnv", + "humantime", + "libc", + "log", + "log-mdc", + "mock_instant", + "parking_lot", + "rand 0.9.2", + "serde", + "serde-value", + "serde_json", + "serde_yaml", + "thiserror 2.0.17", + "thread-id", + "typemap-ors", + "unicode-segmentation", + "winapi", +] + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lopdf" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59fa2559e99ba0f26a12458aabc754432c805bbb8cba516c427825a997af1fb7" +dependencies = [ + "aes", + "bitflags", + "cbc", + "ecb", + "encoding_rs", + "flate2", + "indexmap 2.12.1", + "itoa", + "log", + "md-5", + "nom 8.0.0", + "nom_locate", + "rand 0.9.2", + "rangemap", + "sha2", + "stringprep", + "thiserror 2.0.17", + "weezl", +] + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" + +[[package]] +name = "lz4_flex" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "num_cpus", + "once_cell", + "rawpointer", + "thread-tree", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "memmap2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +dependencies = [ + "libc", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mock_instant" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6" + +[[package]] +name = "moka" +version = "0.12.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom_locate" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" +dependencies = [ + "bytecount", + "memchr", + "nom 8.0.0", +] + +[[package]] +name = "ntapi" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object_store" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "http", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror 2.0.17", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "oneshot" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce411919553d3f9fa53a0880544cda985a112117a0444d5ff1e870a893d6ea" + +[[package]] +name = "openssl-probe" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "path_abs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3" +dependencies = [ + "serde", + "serde_derive", + "std_prelude", + "stfu8", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "permutation" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df202b0b0f5b8e389955afd5f27b007b00fb948162953f1db9c70d2c7e3157d7" + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap 2.12.1", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", + "indexmap 2.12.1", +] + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset 0.5.7", + "hashbrown 0.15.5", + "indexmap 2.12.1", + "serde", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.5", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive 0.14.3", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph 0.7.1", + "prettyplease", + "prost 0.13.5", + "prost-types 0.13.5", + "regex", + "syn 2.0.117", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "petgraph 0.8.3", + "prettyplease", + "prost 0.14.3", + "prost-types 0.14.3", + "regex", + "syn 2.0.117", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost 0.13.5", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost 0.14.3", +] + +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.6.1", + "thiserror 2.0.17", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.17", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.1", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.3", +] + +[[package]] +name = "random_word" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47a395bdb55442b883c89062d6bcff25dc90fa5f8369af81e0ac6d49d78cf81" +dependencies = [ + "ahash", + "brotli", + "paste", + "rand 0.9.2", + "unicase", +] + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec96166dafa0886eb81fe1c0a388bece180fbef2135f97c1e2cf8302e74b43b5" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 2.0.17", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower 0.5.2", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rsa" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-embed" +version = "8.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947d7f3fad52b283d261c4c99a084937e2fe492248cb9a68a8435a861b8798ca" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa2c8c9e8711e10f9c4fd2d64317ef13feaab820a4c51541f1a8c8e2e851ab2" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.117", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b161f275cb337fe0a44d924a5f4df0ed69c2c39519858f931ce61c779d3475" +dependencies = [ + "sha2", + "walkdir", +] + +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62049b2877bf12821e8f9ad256ee38fdc31db7387ec2d3b3f403024de2034aea" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float 2.10.1", + "serde", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af14725505314343e673e9ecb7cd7e8a36aa9791eb936235a3567cc31447ae4" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij 0.1.7", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.12.1", + "schemars 0.9.0", + "schemars 1.2.0", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.12.1", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +dependencies = [ + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" +dependencies = [ + "serde", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive 0.8.9", +] + +[[package]] +name = "snafu" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +dependencies = [ + "snafu-derive 0.9.0", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "snafu-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlparser" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap 2.12.1", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.17", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.117", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.117", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.17", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.17", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.17", + "tracing", + "url", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "std_prelude" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe" + +[[package]] +name = "stfu8" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "sysinfo" +version = "0.29.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tantivy" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a966cb0e76e311f09cf18507c9af192f15d34886ee43d7ba7c7e3803660c43" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "hyperloglogplus", + "itertools 0.14.0", + "levenshtein_automata", + "log", + "lru", + "lz4_flex 0.11.5", + "measure_time", + "memmap2", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 2.0.17", + "time", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1adc286a39e089ae9938935cd488d7d34f14502544a36607effd2239ff0e2494" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6300428e0c104c4f7db6f95b466a6f5c1b9aece094ec57cdd365337908dc7344" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools 0.14.0", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b6ea6090ce03dc72c27d0619e77185d26cc3b20775966c346c6d4f7e99d7f" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e810cdeeebca57fc3f7bfec5f85fdbea9031b2ac9b990eb5ff49b371d52bbe6a" +dependencies = [ + "nom 7.1.3", + "serde", + "serde_json", +] + +[[package]] +name = "tantivy-sstable" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709f22c08a4c90e1b36711c1c6cad5ae21b20b093e535b69b18783dd2cb99416" +dependencies = [ + "futures-util", + "itertools 0.14.0", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bcdebb267671311d1e8891fd9d1301803fdb8ad21ba22e0a30d0cab49ba59c1" +dependencies = [ + "murmurhash32", + "rand_distr 0.4.3", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa942fcee81e213e09715bbce8734ae2180070b97b33839a795ba1de201547d" +dependencies = [ + "serde", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix 1.1.3", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl 2.0.17", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "thread-id" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99043e46c5a15af379c06add30d9c93a6c0e8849de00d244c4a2c417da128d80" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "thread-tree" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbd370cb847953a25954d9f63e14824a36113f8c72eecf6eccef5dc4b45d630" +dependencies = [ + "crossbeam-channel", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.1", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-cron-scheduler" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a5597b569b4712cf78aa0c9ae29742461b7bda1e49c2a5fdad1d79bf022f8f0" +dependencies = [ + "chrono", + "croner", + "num-derive", + "num-traits", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.7.9", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.5", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.13.5", + "prost-types 0.13.5", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "iri-string", + "pin-project-lite", + "tokio", + "tokio-util", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.2", +] + +[[package]] +name = "typemap-ors" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68c24b707f02dd18f1e4ccceb9d49f2058c2fb86384ef9972592904d7a28867" +dependencies = [ + "unsafe-any-ors", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsafe-any-ors" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a303d30665362d9680d7d91d78b23f5f899504d4f08b3c4cf08d055d87c0ad" +dependencies = [ + "destructure_traitobject", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utoipa" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fcc29c80c21c31608227e0912b2d7fddba57ad76b606890627ba8ee7964e993" +dependencies = [ + "indexmap 2.12.1", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d79d08d92ab8af4c5e8a6da20c47ae3f61a0f1dabc1997cdf2d082b757ca08b" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn 2.0.117", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "9.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d047458f1b5b65237c2f6dc6db136945667f40a7668627b3490b9513a3d43a55" +dependencies = [ + "axum 0.8.8", + "base64", + "mime_guess", + "regex", + "rust-embed", + "serde", + "serde_json", + "url", + "utoipa", + "utoipa-swagger-ui-vendored", + "zip 3.0.0", +] + +[[package]] +name = "utoipa-swagger-ui-vendored" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d" + +[[package]] +name = "uuid" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.4", +] + +[[package]] +name = "webpki-roots" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.3", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.5.2", + "constant_time_eq 0.3.1", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.4", + "hmac", + "indexmap 2.12.1", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.17", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zip" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12598812502ed0105f607f941c386f43d441e00148fce9dec3ca5ffb0bde9308" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.12.1", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + +[[package]] +name = "zmij" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e404bcd8afdaf006e529269d3e85a743f9480c3cef60034d77860d02964f3ba" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f990c57 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,96 @@ +[package] +name = "htknow" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.100" +axum = { version = "0.8.8", features = ["multipart"] } +chrono = "0.4" +tokio = { version = "1", features = ["full"] } +tokio-stream = "0.1" +tokio-util = { version = "0.7", features = ["io"] } +tokio-cron-scheduler = "0.13" +sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "tls-rustls", "sqlite", "chrono", "derive"] } +lazy_static = "1.4" +log = "0.4" +log4rs = "1.3.0" + +lancedb = "0.27.2" +arrow-array = "57" +arrow-schema = "57" +futures = "0.3" +bytes = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "http2", "stream"] } + +once_cell = "1" + +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +# Swagger/OpenAPI 文档 +utoipa = { version = "5.3", features = ["axum_extras", "chrono"] } +utoipa-swagger-ui = { version = "9.0.2", features = ["axum", "vendored"] } + +tantivy = "0.24.2" +jieba-rs = "0.6" + +sha2 = "0.10" +hex = "0.4" + +rust-embed = "8" +mime_guess = "2" +uuid = "1.19.0" +base64 = "0.22.1" + +# 知识图谱相关依赖 +petgraph = "0.6" # 图数据结构 +bincode = "1.3" # 序列化图结构 +aho-corasick = "1.0" # 高效字符串匹配（词典） +regex = "1" # 正则表达式 + +# PDF 操作 +lopdf = { version = "0.36", default-features = false } +tempfile = "3" + +# Excel 解析 +calamine = "0.26" + +# 压缩文件解压 +zip = { version = "2", default-features = false, features = ["aes-crypto", "bzip2", "deflate", "deflate64", "lzma", "time", "zstd"] } +tar = "0.4" +flate2 = "1" +bzip2 = "0.4" +xz2 = "0.1" + +# 错误处理 +thiserror = "1" + +# 编码处理（ZIP 中文文件名 GBK/GB18030 解码） +encoding_rs = "0.8" + +# etcd 配置中心（可选） +etcd-client = { version = "0.14", optional = true } +serde_with = "3.16.1" +sysinfo = "0.29" +# jemalloc allocator +# Note: profiling is only needed for debug/analysis, disabled in release for better performance +tikv-jemallocator = { version = "0.6" } +tikv-jemalloc-ctl = { version = "0.6", optional = true, features = ["use_std"] } + +[features] +default = [] +etcd = ["etcd-client"] +# Enable jemalloc profiling for debug/analysis builds only +profiling = ["tikv-jemalloc-ctl"] + +[dev-dependencies] +http-body-util = "0.1" +tower = "0.5" + +[profile.release] +opt-level = "s" +lto = "thin" +codegen-units = 1 +panic = "abort" +strip = "symbols" diff --git a/DEBUG_MEMORY.md b/DEBUG_MEMORY.md new file mode 100644 index 0000000..81f1993 --- /dev/null +++ b/DEBUG_MEMORY.md @@ -0,0 +1,184 @@ +# Debug 镜像内存分析指南 + +## 构建 Debug 镜像 + +Debug 镜像包含 jemalloc 内存分析工具: +- `libjemalloc-dev` - jemalloc 开发库 +- `perl` - jeprof 脚本依赖 +- `graphviz` - 生成可视化图表 +- `binutils` - 符号解析工具 + +### 构建命令 + +```bash +# 构建 debug 镜像 +./build-docker.sh debug + +# 构建 release 镜像 (不包含分析工具) +./build-docker.sh +``` + +## 导出 Heap Dump + +### 方法 1: 直接获取 PDF 报告 (最简单 🚀) + +```bash +# 一键生成并下载 PDF 报告 +curl -H "x-user-id: 1" \ + -H "x-user-name: admin" \ + -H "x-role: admin" \ + -o heap_analysis.pdf \ + http://localhost:8080/api/v1/knowledge/system/heap/pdf + +# 然后直接打开查看 +open heap_analysis.pdf # macOS +xdg-open heap_analysis.pdf # Linux +``` + +### 方法 2: 通过 API 导出原始 heap 文件 + +```bash +# 导出当前内存快照 +curl -H "x-user-id: 1" \ + -H "x-user-name: admin" \ + -H "x-role: admin" \ + -o htknow.heap.$(date +%s).heap \ + http://localhost:8080/api/v1/knowledge/system/heap + +# 查看堆分析状态 +curl -H "x-user-id: 1" \ + -H "x-user-name: admin" \ + -H "x-role: admin" \ + http://localhost:8080/api/v1/knowledge/system/heap/status | jq + +# 查看内存占用 +curl -H "x-user-id: 1" \ + -H "x-user-name: admin" \ + -H "x-role: admin" \ + http://localhost:8080/api/v1/knowledge/system/memory | jq +``` + +### 方法 3: 进入容器手动导出 + +```bash +# 进入容器 +docker exec -it htknow bash + +# 安装 jeprof (如果需要在容器内分析) +apt update && apt install -y google-perftools + +# 使用 jeprof 生成报告 +jeprof --show_bytes --text /app/htknow /path/to/heap.file +``` + +## 分析 Heap Dump + +### 手动分析 + +```bash +# 安装 gperftools (macOS) +brew install gperftools + +# 安装 gperftools (Linux) +apt-get install google-perftools + +# 生成 PDF 报告 +jeprof --show_bytes --pdf target/debug/htknow htknow.heap.*.heap > report.pdf + +# 生成文本报告 +jeprof --show_bytes --text target/debug/htknow htknow.heap.*.heap > report.txt + +# 查看前 10 个内存分配点 +jeprof --show_bytes --text target/debug/htknow htknow.heap.*.heap | head -20 +``` + +## 采样率配置 + +当前配置在 `src/main.rs:6`: + +```rust +pub static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:10\0"; +``` + +| lg_prof_sample | 采样间隔 | 适用场景 | 性能开销 | +|----------------|----------|----------|----------| +| 0 | 每次分配 | 详细分析 | 非常高 | +| **10** | 1 KB | **推荐** | 中等 | +| 15 | 32 KB | 轻量监控 | 较低 | +| 19 | 512 KB | 只看大块 | 很低 | + +## 内存优化 + +如果发现内存占用过高,可以调整以下配置 (在 `docker-compose.yml`): + +### 1. 降低 Tantivy 缓存 + +```yaml +- HTKNOW_TANTIVY_MEMORY_MB=25 # 默认 50 +``` + +### 2. 减少数据库连接 + +```yaml +- HTKNOW_DB_MAX_CONNECTIONS=5 # 默认 10 +``` + +### 3. 调整搜索结果数量 + +```yaml +- HTKNOW_SEARCH_LIMIT=5 # 默认 10 +``` + +## 容器内分析工具 + +进入容器后可用的工具: + +```bash +# 查看进程内存 +htop + +# 查看系统内存 +free -h + +# 查看磁盘占用 +df -h + +# 查看目录大小 +du -sh /app/data/* + +# 实时监控容器 +docker stats htknow +``` + +## 常见问题 + +### Q: 为什么 heap dump 文件很小? + +A: 采样率太高 (lg_prof_sample:19),降低到 10 可获得更详细的数据。 + +### Q: 容器内存 251MB 但 heap 只有 1.7MB? + +A: 可能原因: +1. 采样率过高,未记录小内存分配 +2. 向量数据库 (LanceDB) 占用大量内存 +3. 全文索引 (Tantivy) 缓存 +4. Tokio 运行时和其他系统内存 + +### Q: Debug 镜像可以用于生产吗? + +A: 不推荐。Debug 镜像: +- 包含额外的分析工具 (~100MB) +- 使用 debug 二进制 (更大,更慢) +- 启用了详细日志 (RUST_LOG=debug) +- 启用了内存分析 (性能开销) + +生产环境请使用 release 镜像: +```bash +./build-docker.sh +``` + +## 参考链接 + +- [jemalloc Documentation](http://jemalloc.net/) +- [jeprof Manual](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling) +- [Rust jemalloc](https://github.com/tikv/jemallocator) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d89b5b1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,77 @@ +# Runtime image for htknow +# Binary should be compiled outside the container using: cargo build --release + +FROM docker.1ms.run/debian:bookworm-20260406-slim + +ARG DEBIAN_FRONTEND=noninteractive +ARG APT_MIRROR=http://mirrors.aliyun.com + +COPY docker/apt-fast /usr/local/sbin/apt-fast + +# Switch Debian APT to Aliyun mirror and bootstrap vendored apt-fast. +RUN set -eux; \ + for sources in /etc/apt/sources.list.d/debian.sources /etc/apt/sources.list; do \ + if [ -f "${sources}" ]; then \ + sed -i \ + -e "s|http://deb.debian.org/debian|${APT_MIRROR}/debian|g" \ + -e "s|https://deb.debian.org/debian|${APT_MIRROR}/debian|g" \ + -e "s|http://deb.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \ + -e "s|https://deb.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \ + -e "s|http://security.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \ + -e "s|https://security.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \ + "${sources}"; \ + fi; \ + done; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates aria2; \ + chmod +x /usr/local/sbin/apt-fast; \ + printf '%s\n' \ + '_APTMGR=apt-get' \ + 'DOWNLOADBEFORE=true' \ + '_MAXNUM=8' \ + '_MAXCONPERSRV=8' \ + '_SPLITCON=8' \ + "MIRRORS=(" \ + " '${APT_MIRROR}/debian,http://mirrors.tuna.tsinghua.edu.cn/debian,http://mirrors.ustc.edu.cn/debian'" \ + " '${APT_MIRROR}/debian-security,http://mirrors.tuna.tsinghua.edu.cn/debian-security,http://mirrors.ustc.edu.cn/debian-security'" \ + ')' \ + > /etc/apt-fast.conf; \ + rm -rf /var/lib/apt/lists/* + +# Install runtime dependencies +RUN apt-fast update && apt-fast install -y \ + # Timezone data + tzdata \ + # sqlite3 + sqlite3 \ + # Image normalization for formats MinerU does not accept directly (ICO / AVIF / SVG) + imagemagick \ + libheif1 \ + librsvg2-bin \ + && rm -rf /var/lib/apt/lists/* + +# Configure timezone +ENV TZ=Asia/Shanghai +RUN ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone + +# Create application directory +WORKDIR /app + +# Create data directories +RUN mkdir -p /app/data/images /app/data/temp /app/data/db + +# Copy the compiled binary from host +# Make sure to build with: cargo build --release +COPY target/release/htknow /app/htknow + +# Set permissions +RUN chmod +x /app/htknow + +# Expose the application port (adjust based on your app's configuration) +EXPOSE 8080 + +# Set environment variables +ENV RUST_LOG=info + +# Run the application +CMD ["/app/htknow"] diff --git a/Dockerfile.debug b/Dockerfile.debug new file mode 100644 index 0000000..e29549c --- /dev/null +++ b/Dockerfile.debug @@ -0,0 +1,88 @@ +# Debug image for htknow with jemalloc profiling tools +# Binary should be compiled outside the container using: cargo build --profile dev + +FROM docker.hjkl01.cn/debian:bookworm-slim + +ARG DEBIAN_FRONTEND=noninteractive +ARG APT_MIRROR=http://mirrors.aliyun.com + +COPY docker/apt-fast /usr/local/sbin/apt-fast + +# Switch Debian APT to Aliyun mirror and bootstrap vendored apt-fast. +RUN set -eux; \ + for sources in /etc/apt/sources.list.d/debian.sources /etc/apt/sources.list; do \ + if [ -f "${sources}" ]; then \ + sed -i \ + -e "s|http://deb.debian.org/debian|${APT_MIRROR}/debian|g" \ + -e "s|https://deb.debian.org/debian|${APT_MIRROR}/debian|g" \ + -e "s|http://deb.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \ + -e "s|https://deb.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \ + -e "s|http://security.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \ + -e "s|https://security.debian.org/debian-security|${APT_MIRROR}/debian-security|g" \ + "${sources}"; \ + fi; \ + done; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates aria2; \ + chmod +x /usr/local/sbin/apt-fast; \ + printf '%s\n' \ + '_APTMGR=apt-get' \ + 'DOWNLOADBEFORE=true' \ + '_MAXNUM=8' \ + '_MAXCONPERSRV=8' \ + '_SPLITCON=8' \ + "MIRRORS=(" \ + " '${APT_MIRROR}/debian,http://mirrors.tuna.tsinghua.edu.cn/debian,http://mirrors.ustc.edu.cn/debian'" \ + " '${APT_MIRROR}/debian-security,http://mirrors.tuna.tsinghua.edu.cn/debian-security,http://mirrors.ustc.edu.cn/debian-security'" \ + ')' \ + > /etc/apt-fast.conf; \ + rm -rf /var/lib/apt/lists/* + +# Install runtime dependencies + debug tools +RUN apt-fast update && apt-fast install -y \ + # Timezone data + tzdata \ + # sqlite3 + sqlite3 \ + # Image normalization for formats MinerU does not accept directly (ICO / AVIF / SVG) + imagemagick \ + libheif1 \ + librsvg2-bin \ + # Jemalloc profiling tools + libjemalloc-dev \ + google-perftools \ + perl \ + graphviz \ + binutils \ + # Optional: useful debug utilities + curl \ + htop \ + # Clean up + && rm -rf /var/lib/apt/lists/* + +# Configure timezone +ENV TZ=Asia/Shanghai +RUN ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone + +# Create application directory +WORKDIR /app + +# Create data directories +RUN mkdir -p /app/data/images /app/data/temp /app/data/db + +# Copy the compiled binary from host (debug build) +# Make sure to build with: cargo build --profile dev +COPY target/debug/htknow /app/htknow + +# Set permissions +RUN chmod +x /app/htknow + +# Expose the application port (adjust based on your app's configuration) +EXPOSE 8080 + +# Set environment variables for debug +ENV RUST_LOG=debug +ENV RUST_BACKTRACE=1 + +# Run the application +CMD ["/app/htknow"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..1cf35b9 --- /dev/null +++ b/README.md @@ -0,0 +1,178 @@ +# HTKnow + +HTKnow 知识库管理系统，提供文档上传、检索与知识图谱能力，内置前端界面与 OpenAPI 文档。 + +## 功能概览 +- 知识库管理、文件上传与解析 +- 全文/向量/图谱增强搜索 +- 知识图谱查询与可视化 +- 内置前端界面与 Swagger API 文档 + +## 快速开始 + +### 本地运行 +1. 准备外部服务：MinerU、Embedding、图片 Embedding、Rerank（见“配置”）。 +2. 启动服务： +```shell +cargo run +# 或 +cargo build --release +./target/release/htknow +``` + +### Docker Compose +```shell +docker compose up -d +``` +`docker-compose.yml` 默认将 `8080 -> 3000`。 + +### 构建 Docker 镜像 +```shell +./build-docker.sh +``` +脚本会先编译二进制，再构建镜像并给出运行示例。 + +### 访问入口 +- 前端界面: `http://localhost:3000/` +- API 文档: `http://localhost:3000/docs` +- OpenAPI JSON: `http://localhost:3000/api-docs/openapi.json` + +> 使用 docker-compose 时，请将端口替换为 `8080`。 + +## API 认证 +`/api/v1/knowledge/*` 需要请求头： +- `x-user-id`（必填） +- `x-role`（必填） +- `x-user-name`（可选） + +示例： +```shell +curl -H 'x-user-id: 1' -H 'x-role: admin' -H 'x-user-name: testuser' \ + http://localhost:3000/api/v1/knowledge/knowledge_base/ +``` + +## 启动 mineru +```shell +docker run -d --name mineru-api --restart unless-stopped --ipc host -p 10001:10001 -e MINERU_MODEL_SOURCE=local --ulimit memlock=-1 --ulimit stack=67108864 --gpus all alexsuntop/mineru:latest mineru-api --host 0.0.0.0 --port 10001 +``` + +## 配置 +支持通过环境变量覆盖配置，未设置时使用默认值。 + +### 服务器 +| 环境变量 | 默认值 | 说明 | +| --- | --- | --- | +| `HTKNOW_SERVER_HOST` | `0.0.0.0` | 监听地址 | +| `HTKNOW_SERVER_PORT` | `3000` | 监听端口 | +| `HTKNOW_SERVER_UPLOAD_LIMIT_MB` | `500` | 上传大小限制（MB） | +| `HTKNOW_SERVER_PROCESS_INTERVAL_SECS` | `10` | 文件处理间隔（秒） | +| `HTKNOW_SERVER_PROCESS_CONCURRENCY` | `1` | 后台文件处理并发数 | +| `HTKNOW_PARSE_ENABLED` | `true` | 是否启动后台文件解析（false 时仅即时解析生效） | +| `HTKNOW_REUSE_DUPLICATE_FILES` | `true` | 是否复用重复文件的已解析结果 | +| `HTKNOW_BUILD_KNOWLEDGE_GRAPH` | `false` | 文件解析完成后是否构建知识图谱（依赖 LLM 配置） | +| `HTKNOW_LANCEDB_COMPACT_CRON` | `0 0 3 * * *` | LanceDB 自动压缩 cron 表达式（本地时区，off/disabled/0 禁用） | + +### 外部服务 +默认值为示例地址，请按实际部署环境调整。 +| 环境变量 | 默认值 | 说明 | +| --- | --- | --- | +| `HTKNOW_MINERU_URL` | `http://192.168.0.46:10001/file_parse` | MinerU PDF 解析 | +| `HTKNOW_REQUEST_TIMEOUT_SECS` | `600` | 外部接口请求超时（秒），适用于文件解析相关接口 | +| `HTKNOW_MINERU_MAX_PAGES` | `50` | MinerU 单次解析 PDF 最大页数（0 表示不限制） | +| `HTKNOW_OFFICE_CONVERT_URL` | `http://192.168.0.46:8003/convert` | Office 文档转 PDF 服务，使用 multipart `file` 字段并自动追加 `target_format=pdf` | +| `HTKNOW_CUSTOM_PARSE_URL` | 空 | 自定义解析服务地址（配置后仅 Word/PPT/PDF 解析走该服务，需返回已切片数据） | +| `HTKNOW_CUSTOM_PARSE_REUSE_URL` | 空 | 自定义解析复用服务地址（仅输入 pdf_contents，不包含图片） | +| `HTKNOW_AUDIO_TRANSCRIPTION_URL` | `http://192.168.0.46:59805/api/v1/audio/transcriptions` | 音频转写服务 | +| `HTKNOW_AUDIO_TRANSCRIPTION_KEY` | 空 | 音频转写服务 API Key | +| `HTKNOW_EMBEDDING_URL` | `http://222.190.139.186:59700/v1/embeddings` | 文本向量服务 | +| `HTKNOW_IMAGE_EMBEDDING_URL` | `http://192.168.0.46:59802/v1/embeddings/file` | 图片向量服务 | +| `HTKNOW_RERANK_URL` | `http://222.190.139.186:59600/v1/rerank` | Rerank 服务 | + +### AI 模型 +| 环境变量 | 默认值 | 说明 | +| --- | --- | --- | +| `HTKNOW_EMBEDDING_MODEL` | `bge-m3` | Embedding 模型 | +| `HTKNOW_EMBEDDING_DIM` | `1024` | Embedding 维度 | +| `HTKNOW_IMAGE_EMBEDDING_DIM` | `2048` | 图片 Embedding 维度 | +| `HTKNOW_EMBEDDING_BATCH_SIZE` | `8` | Embedding 批量请求批次大小 | +| `HTKNOW_RERANK_MODEL` | `bge-rerank` | Rerank 模型 | +| `HTKNOW_RERANK_THRESHOLD` | `0.1` | Rerank 阈值 | + +### 数据库 +| 环境变量 | 默认值 | 说明 | +| --- | --- | --- | +| `DATABASE_URL` | `sqlite://data/app.sqlite` | 数据库连接 | +| `HTKNOW_DB_MAX_CONNECTIONS` | `16` | 最大连接数 | +| `HTKNOW_DB_MIN_CONNECTIONS` | `2` | 最小空闲连接数 | +| `HTKNOW_DB_BUSY_TIMEOUT_MS` | `5000` | busy_timeout（毫秒） | +| `HTKNOW_DB_INIT_DEFAULT_KBS` | `true` | 是否初始化默认知识库 | + +### 存储路径 +| 环境变量 | 默认值 | 说明 | +| --- | --- | --- | +| `HTKNOW_DATA_DIR` | `data` | 数据目录 | +| `HTKNOW_LANCEDB_PATH` | `data/lancedb_data` | LanceDB 路径 | +| `HTKNOW_TEMP_PATH` | `data/temp` | 临时目录 | +| `HTKNOW_IMAGES_PATH` | `data/images` | 图片目录 | +| `HTKNOW_PDF_PATH` | `data/pdfs` | PDF 目录 | +| `HTKNOW_FILES_PATH` | `data/files` | 文件目录 | +| `HTKNOW_ARCHIVES_PATH` | `data/archives` | 压缩文件解压目录 | +| `HTKNOW_CONTENTS_PATH` | `data/contents` | 文件解析后完整文本目录 | + +### 搜索 +| 环境变量 | 默认值 | 说明 | +| --- | --- | --- | +| `HTKNOW_SEARCH_LIMIT` | `10` | 搜索结果限制 | +| `HTKNOW_TANTIVY_INDEX_PATH` | `data/tantivy_index` | Tantivy 索引路径 | +| `HTKNOW_TANTIVY_FULL_INDEX_PATH` | `data/tantivy_full_index` | Tantivy 全文索引 | +| `HTKNOW_TANTIVY_MEMORY_MB` | `50` | Tantivy 内存（MB） | +| `HTKNOW_SEARCH_TANTIVY_REBUILD_BATCH_SIZE` | `100` | Tantivy 索引重建批次大小 | +| `HTKNOW_SEARCH_LANCEDB_REBUILD_BATCH_SIZE` | `100` | LanceDB 从 SQLite 重建批次大小 | +| `HTKNOW_SEARCH_EMBEDDING_TIMEOUT_SECS` | `30` | embedding / 图片 embedding 请求超时（秒） | +| `HTKNOW_SEARCH_RERANK_TIMEOUT_SECS` | `20` | rerank 请求超时（秒） | +| `HTKNOW_SEARCH_SYNONYM_ENABLED` | `true` | 是否启用同义词查询扩展 | +| `HTKNOW_SEARCH_SYNONYM_BOOST` | `0.7` | 同义词权重因子（与行权重相乘） | +| `HTKNOW_SEARCH_MAX_SYNONYMS_PER_TERM` | `5` | 每个词最多扩展同义词数 | +| `HTKNOW_SEARCH_MAX_TOTAL_SYNONYMS` | `30` | 单次查询最多扩展同义词总数 | +| `HTKNOW_HIGHLIGHT_PAGE_MIN_POSITIONS` | `20` | 高亮页码选择阈值：首选页位置数少于该值时优先使用第二页 | + +### 切片 +| 环境变量 | 默认值 | 说明 | +| --- | --- | --- | +| `HTKNOW_SMART_SLICE_MAX_CHARS` | `8000` | 智能切片最大字数 | +| `HTKNOW_FIXED_SLICE_OVERLAP_CHARS` | `100` | 固定切片重叠字数 | + +### LLM（可选） +| 环境变量 | 默认值 | 说明 | +| --- | --- | --- | +| `LLM_API_URL` | 空 | LLM API 地址 | +| `LLM_API_KEY` | 空 | LLM API Key | +| `LLM_MODEL` | `gpt-3.5-turbo` | LLM 模型 | + +## 启用 etcd 配置 +`cargo build --features etcd` + +## 日志级别（RUST_LOG） +默认日志级别为 `info`，可通过环境变量覆盖： +```shell +RUST_LOG=debug ./htknow +RUST_LOG=warn,htknow::search=debug ./htknow +``` + +## 问题处理 +1. 全文索引 tantivy 异常 +可能是异常停止导致的，报错 +thread 'main' (1) panicked at src/search/mod.rs:903:29:failed to create tantivy index reader: Failed to open file for read: 'FileDoestiotExist("/app/data/tantivy index/eafseaef4f2340... +run with 'RuST_BAcKTRAcE=l environment variable to display a backtrace + +**方案一**：在 `meta.json` 中去掉报错的索引，注意 `meta.json` 中记录的名称有 `-` + +**方案二**： + +先去掉原本的索引使应用正常启动 +```shell +mv tantivy_index tantivy_index_bak0423 +mv tantivy_full_index tantivy_full_index_bak0423 +docker start htknow +``` +然后在词典可以重建全文检索的索引 diff --git a/build-docker.sh b/build-docker.sh new file mode 100644 index 0000000..c97d770 --- /dev/null +++ b/build-docker.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -e + +# Parse command line arguments +BUILD_MODE="release" +PROFILING_FLAG="" +CUSTOM_IMAGE_TAG="${IMAGE_TAG:-}" +if [ "${1:-}" = "debug" ] || [ "${1:-}" = "--debug" ]; then + BUILD_MODE="debug" + PROFILING_FLAG="--features profiling" +fi + +# Build frontend +cd frontend +npm install +npm run build +cd .. + +if [ "$BUILD_MODE" = "debug" ]; then + echo "Building htknow for debug with profiling..." + cargo build ${PROFILING_FLAG} + if [ -n "$CUSTOM_IMAGE_TAG" ]; then + IMAGE_TAG="${CUSTOM_IMAGE_TAG}-debug" + else + IMAGE_TAG="$(date +%Y%m%d%H%M)-debug" + fi + DOCKERFILE="Dockerfile.debug" + echo "Using Dockerfile.debug with jemalloc profiling tools" +else + echo "Building htknow for release (profiling disabled)..." + cargo build --release + if [ -n "$CUSTOM_IMAGE_TAG" ]; then + IMAGE_TAG="${CUSTOM_IMAGE_TAG}" + else + IMAGE_TAG="$(date +%Y%m%d%H%M)" + fi + DOCKERFILE="Dockerfile" +fi + +# Build Docker image +echo "Building Docker image with ${DOCKERFILE}..." +docker build -f "${DOCKERFILE}" -t "htknow:${IMAGE_TAG}" -t htknow:latest . + +echo "Build complete!" +echo "" +echo "Image tag: htknow:${IMAGE_TAG}" +echo "Latest tag: htknow:latest" +echo "BUILT_IMAGE=htknow:${IMAGE_TAG}" +echo "BUILT_LATEST_IMAGE=htknow:latest" +echo "Build mode: ${BUILD_MODE}" +echo "" +echo "To run the container:" +echo " docker-compose up -d" +echo "" +echo "Or manually:" +echo " docker run -d -p 8080:8080 -v \$(pwd)/data:/app/data --name htknow htknow:${IMAGE_TAG}" +echo "" +echo "Usage:" +echo " ./build-docker.sh # Build in release mode (default)" +echo " ./build-docker.sh debug # Build in debug mode" +echo " IMAGE_TAG=v1.2.3 ./build-docker.sh # Use custom image tag" diff --git a/deploy_ci.md b/deploy_ci.md new file mode 100644 index 0000000..af033ad --- /dev/null +++ b/deploy_ci.md @@ -0,0 +1,24 @@ +## 配置 gitea 密钥 +`DEPLOY_HOST` SSH HOST +`DEPLOY_PORT` SSH PORT +`DEPLOY_USER` SSH USER +`DEPLOY_PASSWORD` SSH PASSWORD +`DEPLOY_PATH` 部署路径 +`DEPLOY_COMMAND` 部署命令 +```shell +set -e + +TAR_FILE="$DEPLOY_PATH/$ASSET_FILE" +COMPOSE_FILE="$DEPLOY_PATH/docker-compose.yaml" # 或写绝对路径 + +EXTRACTED_TAR="${TAR_FILE%.gz}" +gzip -dc "$TAR_FILE" > "$EXTRACTED_TAR" +docker load -i "$EXTRACTED_TAR" +rm -f "$EXTRACTED_TAR" + +if docker compose version >/dev/null 2>&1; then + IMAGE_TAG="$SAFE_TAG" docker compose -f "$COMPOSE_FILE" up -d --remove-orphans +else + IMAGE_TAG="$SAFE_TAG" docker-compose -f "$COMPOSE_FILE" up -d --remove-orphans +fi +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f2107d5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,61 @@ +version: "3.8" + +services: + htknow: + build: + context: . + dockerfile: Dockerfile + container_name: htknow + ports: + - "8080:3000" + volumes: + # Mount data directory for persistent storage + - ./data:/app/data + environment: + - RUST_LOG=info + # 服务器配置 + - HTKNOW_SERVER_HOST=0.0.0.0 + - HTKNOW_SERVER_PORT=3000 + - HTKNOW_SERVER_UPLOAD_LIMIT_MB=500 + - HTKNOW_SERVER_PROCESS_INTERVAL_SECS=10 + - HTKNOW_BUILD_KNOWLEDGE_GRAPH=false + - "HTKNOW_LANCEDB_COMPACT_CRON=0 0 3 * * *" + # 外部服务地址 + - HTKNOW_MINERU_URL=http://192.168.0.46:10001/file_parse + - HTKNOW_REQUEST_TIMEOUT_SECS=600 + - HTKNOW_MINERU_MAX_PAGES=0 + - HTKNOW_OFFICE_CONVERT_URL=http://192.168.0.46:8003/convert + - HTKNOW_EMBEDDING_URL=http://192.168.0.46:9700/v1/embeddings + - HTKNOW_RERANK_URL=http://192.168.0.46:9600/v1/rerank + # AI 模型配置 + - HTKNOW_EMBEDDING_MODEL=bge-m3 + - HTKNOW_EMBEDDING_DIM=1024 + - HTKNOW_EMBEDDING_BATCH_SIZE=8 + - HTKNOW_RERANK_MODEL=bge-rerank + - HTKNOW_RERANK_THRESHOLD=0.1 + # 数据库配置 + - DATABASE_URL=sqlite://data/app.sqlite + - HTKNOW_DB_MAX_CONNECTIONS=10 + - HTKNOW_DB_BUSY_TIMEOUT_MS=5000 + # 存储路径配置 + - HTKNOW_DATA_DIR=data + - HTKNOW_LANCEDB_PATH=data/lancedb_data + - HTKNOW_TEMP_PATH=data/temp + - HTKNOW_IMAGES_PATH=data/images + # 搜索配置 + - HTKNOW_SEARCH_LIMIT=10 + - HTKNOW_TANTIVY_INDEX_PATH=data/tantivy_index + - HTKNOW_TANTIVY_MEMORY_MB=50 + # LLM 配置（可选，用于知识图谱构建） + # - LLM_API_URL=https://api.openai.com/v1/chat/completions + # - LLM_API_KEY=your-api-key + # - LLM_MODEL=gpt-3.5-turbo + restart: unless-stopped + # Uncomment if you need to connect to other services + # networks: + # - htknow-network + +# Uncomment if you need a network +# networks: +# htknow-network: +# driver: bridge diff --git a/docker/apt-fast b/docker/apt-fast new file mode 100644 index 0000000..e473f66 --- /dev/null +++ b/docker/apt-fast @@ -0,0 +1,735 @@ +#!/bin/bash +# +# apt-fast v1.10.0 +# Use this just like aptitude or apt-get for faster package downloading. +# +# Copyright: 2008-2012 Matt Parnell, http://www.mattparnell.com +# Improvements, maintenance, revisions - 2012, 2017-2019 Dominique Lasserre +# +# You may distribute this file under the terms of the GNU General +# Public License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# + +shopt -s nullglob + +[ -n "$DEBUG" ] && set -xv + +# Print colored messages. +# Usage: msg "message text" "message type" "optional: err" +# Message types are 'normal', 'hint' or 'warning'. Warnings and messages with a +# third argument are piped to stderr. +msg(){ + msg_options=() + case "$2" in + normal) beginColor="$cGreen";; + hint) beginColor="$cBlue";; + warning) beginColor="$cRed";; + question) beginColor="$cRed"; msg_options=(-n);; + *) beginColor= ;; + esac + + if [ -z "$3" ] && [ "$2" != "warning" ]; then + echo -e "${msg_options[@]}" "${aptfast_prefix}${beginColor}$1${endColor}" + else + echo -e "${msg_options[@]}" "${aptfast_prefix}${beginColor}$1${endColor}" >&2 + fi +} + +# Search for known options and decide if root privileges are needed. +root=1 # default value: we need root privileges +option= +for argument in "$@"; do + case "$argument" in + upgrade | full-upgrade | install | dist-upgrade | build-dep) + option="install" + ;; + clean | autoclean) + option="clean" + ;; + download) + option="download" + root=0 + ;; + source) + option="source" + root=0 + ;; + changelog | show) + root=0 + ;; + esac +done + +# To handle priority of options correctly (environment over config file vars) +# we need to preserve all interesting env variables. As this wouldn't be +# difficult enough we have to preserve complete env vars (especially if value +# ist set (even empty) or not) when changing context (sudo)... +# Set a 'random' string to all unset variables. +TMP_RANDOM="13979853562951413" +TMP_LCK_FILE="${LCK_FILE-${TMP_RANDOM}}" +TMP_DOWNLOADBEFORE="${DOWNLOADBEFORE-${TMP_RANDOM}}" +TMP__APTMGR="${_APTMGR-${TMP_RANDOM}}" +TMP_APTCACHE="${APTCACHE-${TMP_RANDOM}}" +TMP_DLDIR="${DLDIR-${TMP_RANDOM}}" +TMP_DLLIST="${DLLIST-${TMP_RANDOM}}" +TMP__MAXNUM="${MAXNUM-${TMP_RANDOM}}" +TMP__MAXCONPERSRV="${MAXCONPERSRV-${TMP_RANDOM}}" +TMP__SPLITCON="${SPLITCON-${TMP_RANDOM}}" +TMP__MINSPLITSZ=${MINSPLITSZ-${TMP_RANDOM}} +TMP__PIECEALGO=${PIECEALGO-${TMP_RANDOM}} +TMP_aptfast_prefix="${aptfast_prefix-${TMP_RANDOM}}" +TMP_APT_FAST_TIMEOUT="${APT_FAST_TIMEOUT-${TMP_RANDOM}}" +TMP_APT_FAST_APT_AUTH="${APT_FAST_APT_AUTH-${TMP_RANDOM}}" +TMP_VERBOSE_OUTPUT="${VERBOSE_OUTPUT-${TMP_RANDOM}}" +TMP_ftp_proxy="${ftp_proxy-${TMP_RANDOM}}" +TMP_http_proxy="${http_proxy-${TMP_RANDOM}}" +TMP_https_proxy="${https_proxy-${TMP_RANDOM}}" + +# Check for proper privileges. +# Call explicitly with environment variables to get them into root conext. +if [ "$root" = 1 ] && [ "$UID" != 0 ]; then + exec sudo DEBUG="$DEBUG" \ + LCK_FILE="$TMP_LCK_FILE" \ + DOWNLOADBEFORE="$TMP_DOWNLOADBEFORE" \ + _APTMGR="$TMP__APTMGR" \ + APTCACHE="$TMP_APTCACHE" \ + DLDIR="$TMP_DLDIR" \ + DLLIST="$TMP_DLLIST" \ + _MAXNUM="$TMP__MAXNUM" \ + _MAXCONPERSRV="$TMP__MAXCONPERSRV" \ + _SPLITCON="$TMP__SPLITCON" \ + _MINSPLITSZ="$TMP__MINSPLITSZ" \ + _PIECEALGO="$TMP__PIECEALGO" \ + aptfast_prefix="$TMP_aptfast_prefix" \ + APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT" \ + APT_FAST_APT_AUTH="$TMP_APT_FAST_APT_AUTH" \ + VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT" \ + ftp_proxy="$TMP_ftp_proxy" \ + http_proxy="$TMP_http_proxy" \ + https_proxy="$TMP_https_proxy" \ + "$0" "$@" +fi + +# Define lockfile. +# Use /tmp as directory because everybody (not only root) has to have write +# permissions. +# We need lock for non-root commands too, because we only have one download +# list file. +LCK_FILE="/tmp/apt-fast" +LCK_FD=99 + +# Set default package manager, APT cache, temporary download dir, +# temporary download list file, and maximal parallel downloads +_APTMGR='apt-get' +eval "$(apt-config shell APTCACHE Dir::Cache::archives/d)" +# Check if APT config option Dir::Cache::archives::apt-fast-partial is set. +eval "$(apt-config shell apt_fast_partial Dir::Cache::archives::apt-fast-partial/d)" +if [ -z "$apt_fast_partial" ]; then + DLDIR="$(realpath "${APTCACHE}/../apt-fast")" +else + DLDIR="${apt_fast_partial}" +fi + +# Check for apt auth files +eval "$(apt-config shell NETRC Dir::Etc::netrc/f)" +eval "$(apt-config shell NETRCDIR Dir::Etc::netrcparts/d)" +APTAUTHFILES=() +if [ -f "$NETRC" ]; then + APTAUTHFILES=("$NETRC") +fi +APTAUTHFILES+=("$NETRCDIR"*) + +DLLIST="/tmp/apt-fast.list" +_MAXNUM=5 +_MAXCONPERSRV=10 +_SPLITCON=8 +_MINSPLITSZ="1M" +_PIECEALGO="default" +MIRRORS=() + +# Prefix in front of apt-fast output: +aptfast_prefix= +# aptfast_prefix="$(date '+%b %_d %T.%N') apt-fast: " + +# Set color variables. +cGreen='\e[0;32m' +cRed='\e[0;31m' +cBlue='\e[0;34m' +endColor='\e[0m' + +# Set timout value for apt-fast download confirmation dialog. +# Value is in seconds. +APT_FAST_TIMEOUT=60 + +# Ask for download confirmation if unset +DOWNLOADBEFORE= + +# Enable APT authentication support +APT_FAST_APT_AUTH=1 + +# Formatted package list in download confirmation if unset +VERBOSE_OUTPUT= + +# Download command. +_DOWNLOADER='aria2c --no-conf -c -j ${_MAXNUM} -x ${_MAXCONPERSRV} -s ${_SPLITCON} -i ${DLLIST} --min-split-size=${_MINSPLITSZ} --stream-piece-selector=${_PIECEALGO} --connect-timeout=600 --timeout=600 -m0 --header "Accept: */*"' + +# Load config file. +for Prefix in /usr/local/etc /etc; do + CONFFILE=${Prefix}/apt-fast.conf + if [ -e "$CONFFILE" ]; then + source "$CONFFILE" + break + fi +done + +# no proxy as default +ftp_proxy= +http_proxy= +https_proxy= + +# Now overwrite with preserved values if values were set before (compare with +# 'random' string). +[ "$TMP_LCK_FILE" = "$TMP_RANDOM" ] || LCK_FILE="$TMP_LCK_FILE" +[ "$TMP_DOWNLOADBEFORE" = "$TMP_RANDOM" ] || DOWNLOADBEFORE="$TMP_DOWNLOADBEFORE" +[ "$TMP__APTMGR" = "$TMP_RANDOM" ] || _APTMGR="$TMP__APTMGR" +[ "$TMP_APTCACHE" = "$TMP_RANDOM" ] || APTCACHE="$TMP_APTCACHE" +[ "$TMP_DLDIR" = "$TMP_RANDOM" ] || DLDIR="$TMP_DLDIR" +[ "$TMP_DLLIST" = "$TMP_RANDOM" ] || DLLIST="$TMP_DLLIST" +[ "$TMP__MAXNUM" = "$TMP_RANDOM" ] || _MAXNUM="$TMP__MAXNUM" +[ "$TMP__MAXCONPERSRV" = "$TMP_RANDOM" ] || _MAXCONPERSRV="$TMP__MAXCONPERSRV" +[ "$TMP__SPLITCON" = "$TMP_RANDOM" ] || _SPLITCON="$TMP__SPLITCON" +[ "$TMP__MINSPLITSZ" = "$TMP_RANDOM" ] || _MINSPLITSZ="$TMP__MINSPLITSZ" +[ "$TMP__PIECEALGO" = "$TMP_RANDOM" ] || _PIECEALGO="$TMP__PIECEALGO" +[ "$TMP_aptfast_prefix" = "$TMP_RANDOM" ] || aptfast_prefix="$TMP_aptfast_prefix" +[ "$TMP_APT_FAST_TIMEOUT" = "$TMP_RANDOM" ] || APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT" +[ "$TMP_APT_FAST_APT_AUTH" = "$TMP_RANDOM" ] || APT_FAST_APT_AUTH="$TMP_APT_FAST_APT_AUTH" +[ "$TMP_VERBOSE_OUTPUT" = "$TMP_RANDOM" ] || VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT" +[ "$TMP_ftp_proxy" = "$TMP_RANDOM" ] || ftp_proxy="$TMP_ftp_proxy" +[ "$TMP_http_proxy" = "$TMP_RANDOM" ] || http_proxy="$TMP_http_proxy" +[ "$TMP_https_proxy" = "$TMP_RANDOM" ] || https_proxy="$TMP_https_proxy" + + +# Disable colors if not executed in terminal. +if [ ! -t 1 ]; then + cGreen= + cRed= + cBlue= + endColor= + #FIXME: Time not updated. + [ -z "$aptfast_prefix" ] && aptfast_prefix="[apt-fast $(date +"%T")]" +fi + + +msg_already_running() +{ + msg "apt-fast already running!" "warning" + msg "Verify that all apt-fast processes are finished then remove $LCK_FILE.lock and try again." "hint" +} + +# Check if a lock file exists. +if [ -f "$LCK_FILE.lock" ]; then + msg_already_running + exit 1 +fi + + +# create the lock file and lock it, die on failure +_create_lock() +{ + eval "exec $LCK_FD>\"$LCK_FILE.lock\"" + flock -n $LCK_FD || { msg_already_running; exit 1; } + + trap "cleanup_aptfast; exit_cleanup_state" EXIT + trap "cleanup_aptfast; exit 1" INT TERM +} + +# unlock and remove the lock file +_remove_lock() +{ + flock -u "$LCK_FD" 2>/dev/null + rm -f "$LCK_FILE.lock" +} + +# Move download file away so missing permissions won't stop usage. +CLEANUP_STATE=0 +cleanup_dllist() +{ + if [ -f "$DLLIST" ] + then + if ! mv -- "$DLLIST{,.old}" 2>/dev/null + then + if ! rm -f -- "$DLLIST" 2>/dev/null + then + msg "Could not clean up download list file." "warning" + CLEANUP_STATE=1 + fi + fi + fi +} + +cleanup_aptfast() +{ + local last_exit_code=$? + [ "$CLEANUP_STATE" -eq 0 ] && CLEANUP_STATE=$last_exit_code + cleanup_dllist + _remove_lock +} + +exit_cleanup_state() +{ + exit $CLEANUP_STATE +} + +# decode url string +# translates %xx but must not convert '+' in spaces +urldecode() +{ + printf '%b' "${1//%/\\x}" +} + +# Check if mirrors are available. And if so add all mirrors to download list. +get_mirrors(){ + # Check all mirror lists. + for mirrorstr in "${MIRRORS[@]}"; do + # Build mirrors array from comma separated string. + IFS=", " read -r -a mirrors <<< "$mirrorstr" + # Check for all mirrors if URI of $1 is from mirror. If so add all other + # mirrors to (resmirror) list and break all loops. + for mirror in "${mirrors[@]}"; do + # Real expension. + if [[ "$1" == "$mirror"* ]]; then + filepath="${1#"${mirror}"}" + # Build list for aria download list. + list="${mirrors[*]}" + echo -e "${list// /${filepath}\\t}$filepath\n" + return 0 + fi + done + done + # No other mirrors found. + echo "$1" +} + +AUTH_INFO_PARSED=() +# Parse apt authentication files. +# Undefined behavior on whitespaces in host, username or password. +prepare_auth(){ + if [ "$APT_FAST_APT_AUTH" -eq 0 ]; then + return + fi + for auth_file in "${APTAUTHFILES[@]}"; do + # auth files have netrc syntax, possible multiline entries starting with "machine" + auth_info="$(tr '\n' ' ' < "$auth_file" | sed 's/\(\\)/\n\1/g' | sed '1d')" + while IFS= read -r auth; do + machine="$(echo "$auth" | sed 's/.*\[ \t]\+\([^ \t]\+\).*/\1/')" + login="$(echo "$auth" | sed 's/.*\[ \t]\+\([^ \t]\+\).*/\1/')" + password="$(echo "$auth" | sed 's/.*\[ \t]\+\([^ \t]\+\).*/\1/')" + # if machine does not have protocol, try https:// + if ! [[ "$machine" =~ ^.*:// ]]; then + machine="https://$machine" + fi + if [ -z "$machine" ] || [ -z "$login" ] || [ -z "$password" ]; then + msg "Could not parse apt authentication (skipping): $auth ($auth_file)" "warning" + continue + fi + # use space separated string to convert back to array later + AUTH_INFO_PARSED+=("$machine $login $password") + done <<< "$auth_info" + done +} + +# Gets URI as parameter and tries to add basic http credentials. Will fail on +# credentials that contain characters that need URL-encoding. +get_auth(){ + if [ "$APT_FAST_APT_AUTH" -eq 0 ]; then + echo "$1" + return + fi + for auth_info in "${AUTH_INFO_PARSED[@]}"; do + # convert to array, don't escape variable here + auth_info_arr=($auth_info) + machine="${auth_info_arr[0]}" + # takes first match + if [[ "$1" == "$machine"* ]]; then + login="${auth_info_arr[1]}" + password="${auth_info_arr[2]}" + uri="$(echo "$1" | sed "s|^\([^:]\+://\)|\1$login:$password@|")" + echo "$uri" + return + fi + done + echo "$1" +} + +# Globals to save package name, version, size and overall size. +DOWNLOAD_DISPLAY= +DOWNLOAD_SIZE=0 +# Get the package URLs. +get_uris(){ + if [ ! -d "$(dirname "$DLLIST")" ] + then + if ! mkdir -p -- "$(dirname "$DLLIST")" + then + msg "Could not create download file directory." "warning" + CLEANUP_STATE=1 + exit + fi + elif [ -f "$DLLIST" ]; then + if ! rm -f -- "$DLLIST" 2>/dev/null && ! touch -- "$DLLIST" 2>/dev/null + then + msg "Unable to write to download file. Try restarting with root permissions or run 'apt-fast clean' first." "warning" + CLEANUP_STATE=1 + exit + fi + fi + + # Add header to overwrite file. + echo "# apt-fast mirror list: $(date)" > "$DLLIST" + # NOTE: "aptitude" doesn't have this functionality + # so we use "${_APTMGR}" to get package URI's + case "$(basename "${_APTMGR}")" in + 'apt'|'apt-get') uri_mgr="${_APTMGR}";; + *) uri_mgr='apt-get';; + esac + uris_full="$("$uri_mgr" "${APT_SCRIPT_WARNING[@]}" -y --print-uris "$@")" + CLEANUP_STATE="$?" + if [ "$CLEANUP_STATE" -ne 0 ] + then + msg "Package manager quit with exit code." "warning" + exit + fi + prepare_auth + while read -r pkg_uri_info + do + [ -z "$pkg_uri_info" ] && continue + ## --print-uris format is: + # 'fileurl' filename filesize checksum_hint:filechecksum + uri="$(get_auth "$(echo "$pkg_uri_info" | cut -d' ' -f1 | tr -d "'")")" + filename="$(echo "$pkg_uri_info" | cut -d' ' -f2)" + filesize="$(echo "$pkg_uri_info" | cut -d' ' -f3)" + checksum_string="$(echo "$pkg_uri_info" | cut -d' ' -f4)" + hash_algo="$(echo "$checksum_string" | cut -d':' -f1)" + checksum="$(echo "$checksum_string" | cut -d':' -f2)" + + filename_decoded="$(urldecode "$filename")" + DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY}$(echo "$filename_decoded" | cut -d'_' -f1)" + DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY} $(echo "$filename_decoded" | cut -d'_' -f2)" + DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY} $(echo "$filesize" | numfmt --to=iec-i --suffix=B)\n" + DOWNLOAD_SIZE=$((DOWNLOAD_SIZE + filesize)) + + ## whole uri comes encoded (urlencoded). Filename must NOT be decoded because + # plain aptitude do not decode it when download and install it. Therefore, we + # will have ugly named packages at /var/cache/apt/archives but is the standard + # behavior. + # But package version must be decoded, otherways package=version calls will + # not work. + + if [ -n "$HASH_SUPPORTED" ]; then + case "$hash_algo" in + SHA512) [ -z "$SHA512_SUPPORTED" ] && hash_algo= || hash_algo=sha-512 ;; + SHA256) [ -z "$SHA256_SUPPORTED" ] && hash_algo= || hash_algo=sha-256 ;; + SHA1) [ -z "$SHA1_SUPPORTED" ] && hash_algo= || hash_algo=sha-1 ;; + MD5Sum) [ -z "$MD5sum_SUPPORTED" ] && hash_algo= || hash_algo=md5 ;; + *) hash_algo= + esac + + # Using apt-cache show package=version to ensure recover single and + # correct package version. + # Warning: assuming that package naming uses '_' as field separator. + # Therefore, this code expects package-name_version_arch.deb Otherways + # below code will fail resoundingly + if [ -z "$hash_algo" ]; then + pkg_name="$(echo "$filename" | cut -d'_' -f1)" + pkg_version="$(echo "$filename" | cut -d'_' -f2)" + pkg_version="$(urldecode "$pkg_version")" + package_info="$(apt-cache show "$pkg_name=$pkg_version")" + + patch_checksum= + if [ -n "$SHA512_SUPPORTED" ]; then + patch_checksum="$(echo "$package_info" | grep SHA512 | head -n 1)" + [ -n "$patch_checksum" ] && hash_algo="sha-512" + fi + if [ -z "$patch_checksum" ] && [ -n "$SHA256_SUPPORTED" ]; then + patch_checksum="$(echo "$package_info" | grep SHA256 | head -n 1)" + [ -n "$patch_checksum" ] && hash_algo="sha-256" + fi + if [ -z "$patch_checksum" ] && [ -n "$SHA1_SUPPORTED" ]; then + patch_checksum="$(echo "$package_info" | grep SHA1 | head -n 1)" + [ -n "$patch_checksum" ] && hash_algo="sha-1" + fi + if [ -z "$patch_checksum" ] && [ -n "$MD5sum_SUPPORTED" ]; then + patch_checksum="$(echo "$package_info" | grep MD5sum | head -n 1)" + [ -n "$patch_checksum" ] && hash_algo="md5" + fi + + if [ -n "$patch_checksum" ]; then + checksum="$(echo "$patch_checksum" | cut -d' ' -f2)" + else + msg "Couldn't get supported checksum for $pkg_name ($pkg_version)." "warning" + REMOVE_WORKING_MESSAGE= + fi + fi + else + hash_algo= + fi + + { + get_mirrors "$uri" + #echo " dir=$DLDIR" + if [ -n "$hash_algo" ]; then + echo " checksum=$hash_algo=$checksum" + fi + echo " out=$filename" + } >> "$DLLIST" + done <<<"$(echo "$uris_full" | grep -E "^'(http(s|)|(s|)ftp)://")" + + #cat "$DLLIST" + #LCK_RM + #exit +} + +display_downloadfile(){ + if [ -n "$VERBOSE_OUTPUT" ]; then + cat "$DLLIST" + else + DISPLAY_SORT_OPTIONS=(-k 1,1) + # Sort output after package download size (decreasing): + #DISPLAY_SORT_OPTIONS=(-k 3,3 -hr) + while read -r line; do + [ -z "$line" ] && continue + pkg="$(echo "$line" | cut -d' ' -f1)" + ver="$(echo "$line" | cut -d' ' -f2)" + size="$(echo "$line" | cut -d' ' -f3)" + printf '%s%-40s %-20s %10s\n' "$aptfast_prefix" "$pkg" "$ver" "$size" + done <<<"$(echo -e "$DOWNLOAD_DISPLAY" | sort "${DISPLAY_SORT_OPTIONS[@]}")" + fi + msg "Download size: $(echo "$DOWNLOAD_SIZE" | numfmt --to=iec-i --suffix=B)" "normal" +} + +# Create and insert a PID number to lockfile. +_create_lock + +# Make sure aria2c (in general first parameter from _DOWNLOADER) is available. +CMD="$(echo "$_DOWNLOADER" | sed 's/^\s*\([^ ]\+\).*$/\1/')" +if [ ! "$(command -v "$CMD")" ]; then + msg "Command not found: $CMD" "normal" "err" + msg "You must configure $CONFFILE to use aria2c or another supported download manager" "normal" "err" + CLEANUP_STATE=1 + exit +fi + +# Make sure package manager is available. +if [ ! "$(command -v "$_APTMGR")" ]; then + msg "\`$_APTMGR\` command not available." "warning" + msg "You must configure $CONFFILE to use either apt-get or aptitude." "normal" "err" + CLEANUP_STATE=1 + exit +fi + +# Disable script warning if apt is used. +APT_SCRIPT_WARNING=() +if [ "$(basename "${_APTMGR}")" == 'apt' ]; then + APT_SCRIPT_WARNING=(-o "Apt::Cmd::Disable-Script-Warning=true") +fi + +# Set supported hash algorithms by aria2c (and also by Debian repository). +SHA512_SUPPORTED= +SHA256_SUPPORTED= +SHA1_SUPPORTED= +MD5sum_SUPPORTED= +HASH_SUPPORTED= +if [ "$CMD" == "aria2c" ]; then + for supported_hash in $(LC_ALL=C aria2c -v | sed '/^Hash Algorithms:/!d; s/\(^Hash Algorithms: \|,\)\+//g'); do + case "$supported_hash" in + sha-512) SHA512_SUPPORTED=y; HASH_SUPPORTED=y ;; + sha-256) SHA256_SUPPORTED=y; HASH_SUPPORTED=y ;; + sha-1) SHA1_SUPPORTED=y; HASH_SUPPORTED=y ;; + md5) MD5sum_SUPPORTED=y; HASH_SUPPORTED=y ;; + esac + done + if [ -z "$HASH_SUPPORTED" ]; then + msg "Couldn't find supported checksum algorithm from aria2c. Checksums disabled." "warning" + fi +fi + +# Check if "assume yes" switch is enabled and if yes enable $DOWNLOADBEFORE. +# Also check if "download only" switch is enabled. +#TODO: Get real value over APT items APT::Get::Assume-Yes and +# APT::Get::Assume-No . +# Respectively Aptitude::CmdLine::Download-Only and APT::Get::Download-Only. +DOWNLOAD_ONLY= +while true; do + while getopts ":dy-:" optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + yes | assume-yes) DOWNLOADBEFORE=true ;; + assume-no) DOWNLOADBEFORE= ;; + download-only) DOWNLOAD_ONLY=true ;; + esac + ;; + y) + DOWNLOADBEFORE=true + ;; + d) + DOWNLOAD_ONLY=true + ;; + *) + ;; + esac + done + ((OPTIND++)) + [ $OPTIND -gt $# ] && break +done + +# Configure proxies. Use apt values over environment variables. +# Note: If proxy setting is not set, there is no apt-config output. +# Therefore variable doesn't get overriden, which is intended. +# Export the variables to make them available in subshells (aka the +# downloader command). +eval "$(apt-config shell ftp_proxy Acquire::ftp::proxy)" +export ftp_proxy +eval "$(apt-config shell http_proxy Acquire::http::proxy)" +export http_proxy +eval "$(apt-config shell https_proxy Acquire::https::proxy)" +export https_proxy + +# aria2 has no socks support (see https://github.com/aria2/aria2/issues/153) +if echo "$http_proxy" | grep -q "^socks5h://" || echo "$https_proxy" | grep -q "^socks5h://"; then + msg "Socks proxy detected. Falling back to ${_APTMGR}" "hint" + "${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@" + exit +fi + +# Run actions. +if [ "$option" == "install" ]; then + msg + msg "Working... this may take a while." "normal" + REMOVE_WORKING_MESSAGE=y + + get_uris "$@" + + [ -t 1 ] && [ -n "$REMOVE_WORKING_MESSAGE" ] && tput cuu 1 && tput el && tput cuu 1 + # Test /tmp/apt-fast.list file exists and not just the apt-fast comment line. + # Then download all files from the list. + if [ -f "$DLLIST" ] && [ "$(wc -l "$DLLIST" | cut -d' ' -f1)" -gt 1 ] && [ ! "$DOWNLOADBEFORE" ]; then + display_downloadfile + msg + msg "Do you want to download the packages? [Y/n] " "question" + + while ((!updsys)); do + read -r -sn1 -t "$APT_FAST_TIMEOUT" answer || { msg; msg "Timed out." "warning"; CLEANUP_STATE=1; exit; } + case "$answer" in + [JjYy]) result=1; updsys=1 ;; + [Nn]) result=0; updsys=1 ;; + "") result=1; updsys=1 ;; + *) updsys=0 ;; + esac + done + else + result=1 + fi + + if ((DOWNLOAD_SIZE)); then + msg + # Continue if answer was right or DOWNLOADBEFORE is enabled. + if ((result)); then + if [ -s "$DLLIST" ]; then + # Test if apt-fast directory is present where we put packages. + if [ ! -d "$DLDIR" ]; then + mkdir -p -- "$DLDIR" + fi + + cd "$DLDIR" &>/dev/null || { msg; msg "Not able to change into download directory." "warning"; CLEANUP_STATE=1; exit; } + + eval "${_DOWNLOADER}" # execute downloadhelper command + if [ "$(find "$DLDIR" -printf . | wc -c)" -gt 1 ]; then + + # Delete incomplete/corrupted downloaded files, if any: Not recursive, as we don't expect any dirs to exist within $DLDIR. + + # When Aria2c downloads a file and detects it is corrupted, its filename won't be renamed back to its actual name, + # preserving .aria2 file extension, which also indicates when a file hasn't been completely downloaded. + for x in *.aria2; do + rm -f "$x" "${x%.aria2}" + done + + # Move all packages to the apt install directory by force to ensure + # already existing debs which may be incomplete are replaced + find . -type f \( -name '*.deb' -o -name '*.ddeb' \) -execdir mv -ft "$APTCACHE" {} \+ + fi + cd - &>/dev/null || msg "Failed to change back directory" "warning" + fi + else + CLEANUP_STATE=1 + exit + fi + else + [ -t 1 ] && tput el + fi + + # different problem resolving for aptitude + if [ -z "$DOWNLOAD_ONLY" ] || [ "$(basename "${_APTMGR}")" == 'aptitude' ]; then + "${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@" + fi + + +elif [ "$option" == "clean" ]; then + "${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@" && { + if [ -d "$DLDIR" ]; then + find "$DLDIR" -maxdepth 1 -type f -delete + CLEANUP_STATE="$?" + [ -f "$DLLIST" ] && rm -f -- "$DLLIST"* || true + fi + } + +elif [ "$option" == "download" ]; then + msg + msg "Working... this may take a while." "normal" + REMOVE_WORKING_MESSAGE=y + + get_uris "$@" + + [ -t 1 ] && [ -n "$REMOVE_WORKING_MESSAGE" ] && tput cuu 1 && tput el && tput cuu 1 + + if [ -f "$DLLIST" ] && [ "$(wc -l "$DLLIST" | cut -d' ' -f1)" -gt 1 ]; then + display_downloadfile + eval "${_DOWNLOADER}" + fi + + # different problem resolving for aptitude + if [ "$(basename "${_APTMGR}")" == 'aptitude' ]; then + "${_APTMGR}" "$@" + fi + +elif [ "$option" == "source" ]; then + msg + msg "Working... this may take a while." "normal" + REMOVE_WORKING_MESSAGE=y + + get_uris "$@" + + [ -t 1 ] && [ -n "$REMOVE_WORKING_MESSAGE" ] && tput cuu 1 && tput el && tput cuu 1 + + if [ -f "$DLLIST" ] && [ "$(wc -l "$DLLIST" | cut -d' ' -f1)" -gt 1 ]; then + display_downloadfile + eval "${_DOWNLOADER}" + fi + # We use APT manager here to provide more verbose output. This method is + # slightly slower then extractiong packages manually after download but also + # more hardened (e.g. some options like --compile are available). + "${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@" + # Uncomment following snippet to extract source directly and comment + # both lines before. + #while read srcfile; do + # # extract only .dsc files + # echo "$srcfile" | grep -q '\.dsc$' || continue + # dpkg-source -x "$(basename "$srcfile")" + #done < "$DLLIST" + +# Execute package manager directly if unknown options are passed. +else + "${_APTMGR}" "${APT_SCRIPT_WARNING[@]}" "$@" +fi + +# After error or all done remove our lockfile (done with EXIT trap) diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..a547bf3 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/frontend/.vscode/extensions.json b/frontend/.vscode/extensions.json new file mode 100644 index 0000000..a7cea0b --- /dev/null +++ b/frontend/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["Vue.volar"] +} diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000..30ed5e2 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,20 @@ +# Frontend + +前端使用 `Vue 3 + Vite + Tailwind CSS 4`，构建产物输出到 `frontend/dist`，由后端通过 `rust-embed` 内嵌并提供静态资源服务。 + +## 开发 + +```bash +npm ci +npm run dev +``` + +默认开发代理会把 `/api` 转发到 `http://localhost:3000`。 + +## 构建 + +```bash +npm run build +``` + +构建完成后，后端重新编译时会把 `dist/` 打包进二进制。 diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..de17e1a --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,13 @@ + + + + + + + 知识库管理系统 + + +
+ + + diff --git a/frontend/package-lock.json b/frontend/package-lock.json new file mode 100644 index 0000000..fda5d1c --- /dev/null +++ b/frontend/package-lock.json @@ -0,0 +1,2192 @@ +{ + "name": "frontend", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "frontend", + "version": "0.0.0", + "dependencies": { + "pdfjs-dist": "^4.2.67", + "vue": "^3.5.24" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.1.18", + "@vitejs/plugin-vue": "^6.0.1", + "tailwindcss": "^4.1.18", + "vite": "^7.2.4" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmmirror.com/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmmirror.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.5", + "resolved": "https://registry.npmmirror.com/@babel/parser/-/parser-7.28.5.tgz", + "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.5" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.5", + "resolved": "https://registry.npmmirror.com/@babel/types/-/types-7.28.5.tgz", + "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", + "integrity": "sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/android-arm/-/android-arm-0.27.2.tgz", + "integrity": "sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/android-arm64/-/android-arm64-0.27.2.tgz", + "integrity": "sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/android-x64/-/android-x64-0.27.2.tgz", + "integrity": "sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/darwin-arm64/-/darwin-arm64-0.27.2.tgz", + "integrity": "sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/darwin-x64/-/darwin-x64-0.27.2.tgz", + "integrity": "sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.2.tgz", + "integrity": "sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/freebsd-x64/-/freebsd-x64-0.27.2.tgz", + "integrity": "sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-arm/-/linux-arm-0.27.2.tgz", + "integrity": "sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-arm64/-/linux-arm64-0.27.2.tgz", + "integrity": "sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-ia32/-/linux-ia32-0.27.2.tgz", + "integrity": "sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-loong64/-/linux-loong64-0.27.2.tgz", + "integrity": "sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-mips64el/-/linux-mips64el-0.27.2.tgz", + "integrity": "sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-ppc64/-/linux-ppc64-0.27.2.tgz", + "integrity": "sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-riscv64/-/linux-riscv64-0.27.2.tgz", + "integrity": "sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-s390x/-/linux-s390x-0.27.2.tgz", + "integrity": "sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/linux-x64/-/linux-x64-0.27.2.tgz", + "integrity": "sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.2.tgz", + "integrity": "sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/netbsd-x64/-/netbsd-x64-0.27.2.tgz", + "integrity": "sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.2.tgz", + "integrity": "sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/openbsd-x64/-/openbsd-x64-0.27.2.tgz", + "integrity": "sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.2.tgz", + "integrity": "sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/sunos-x64/-/sunos-x64-0.27.2.tgz", + "integrity": "sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/win32-arm64/-/win32-arm64-0.27.2.tgz", + "integrity": "sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/win32-ia32/-/win32-ia32-0.27.2.tgz", + "integrity": "sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/@esbuild/win32-x64/-/win32-x64-0.27.2.tgz", + "integrity": "sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmmirror.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmmirror.com/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmmirror.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmmirror.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas/-/canvas-0.1.88.tgz", + "integrity": "sha512-/p08f93LEbsL5mDZFQ3DBxcPv/I4QG9EDYRRq1WNlCOXVfAHBTHMSVMwxlqG/AtnSfUr9+vgfN7MKiyDo0+Weg==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.88", + "@napi-rs/canvas-darwin-arm64": "0.1.88", + "@napi-rs/canvas-darwin-x64": "0.1.88", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.88", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.88", + "@napi-rs/canvas-linux-arm64-musl": "0.1.88", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.88", + "@napi-rs/canvas-linux-x64-gnu": "0.1.88", + "@napi-rs/canvas-linux-x64-musl": "0.1.88", + "@napi-rs/canvas-win32-arm64-msvc": "0.1.88", + "@napi-rs/canvas-win32-x64-msvc": "0.1.88" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.88.tgz", + "integrity": "sha512-KEaClPnZuVxJ8smUWjV1wWFkByBO/D+vy4lN+Dm5DFH514oqwukxKGeck9xcKJhaWJGjfruGmYGiwRe//+/zQQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.88.tgz", + "integrity": "sha512-Xgywz0dDxOKSgx3eZnK85WgGMmGrQEW7ZLA/E7raZdlEE+xXCozobgqz2ZvYigpB6DJFYkqnwHjqCOTSDGlFdg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.88.tgz", + "integrity": "sha512-Yz4wSCIQOUgNucgk+8NFtQxQxZV5NO8VKRl9ePKE6XoNyNVC8JDqtvhh3b3TPqKK8W5p2EQpAr1rjjm0mfBxdg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.88.tgz", + "integrity": "sha512-9gQM2SlTo76hYhxHi2XxWTAqpTOb+JtxMPEIr+H5nAhHhyEtNmTSDRtz93SP7mGd2G3Ojf2oF5tP9OdgtgXyKg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.88.tgz", + "integrity": "sha512-7qgaOBMXuVRk9Fzztzr3BchQKXDxGbY+nwsovD3I/Sx81e+sX0ReEDYHTItNb0Je4NHbAl7D0MKyd4SvUc04sg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.88.tgz", + "integrity": "sha512-kYyNrUsHLkoGHBc77u4Unh067GrfiCUMbGHC2+OTxbeWfZkPt2o32UOQkhnSswKd9Fko/wSqqGkY956bIUzruA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.88.tgz", + "integrity": "sha512-HVuH7QgzB0yavYdNZDRyAsn/ejoXB0hn8twwFnOqUbCCdkV+REna7RXjSR7+PdfW0qMQ2YYWsLvVBT5iL/mGpw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.88.tgz", + "integrity": "sha512-hvcvKIcPEQrvvJtJnwD35B3qk6umFJ8dFIr8bSymfrSMem0EQsfn1ztys8ETIFndTwdNWJKWluvxztA41ivsEw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.88.tgz", + "integrity": "sha512-eSMpGYY2xnZSQ6UxYJ6plDboxq4KeJ4zT5HaVkUnbObNN6DlbJe0Mclh3wifAmquXfrlgTZt6zhHsUgz++AK6g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.88.tgz", + "integrity": "sha512-qcIFfEgHrchyYqRrxsCeTQgpJZ/GqHiqPcU/Fvw/ARVlQeDX1VyFH+X+0gCR2tca6UJrq96vnW+5o7buCq+erA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.88", + "resolved": "https://registry.npmmirror.com/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.88.tgz", + "integrity": "sha512-ROVqbfS4QyZxYkqmaIBBpbz/BQvAR+05FXM5PAtTYVc0uyY8Y4BHJSMdGAaMf6TdIVRsQsiq+FG/dH9XhvWCFQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.53", + "resolved": "https://registry.npmmirror.com/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.53.tgz", + "integrity": "sha512-vENRlFU4YbrwVqNDZ7fLvy+JR1CRkyr01jhSiDpE1u6py3OMzQfztQU2jxykW3ALNxO4kSlqIDeYyD0Y9RcQeQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.54.0.tgz", + "integrity": "sha512-OywsdRHrFvCdvsewAInDKCNyR3laPA2mc9bRYJ6LBp5IyvF3fvXbbNR0bSzHlZVFtn6E0xw2oZlyjg4rKCVcng==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.54.0.tgz", + "integrity": "sha512-Skx39Uv+u7H224Af+bDgNinitlmHyQX1K/atIA32JP3JQw6hVODX5tkbi2zof/E69M1qH2UoN3Xdxgs90mmNYw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.54.0.tgz", + "integrity": "sha512-k43D4qta/+6Fq+nCDhhv9yP2HdeKeP56QrUUTW7E6PhZP1US6NDqpJj4MY0jBHlJivVJD5P8NxrjuobZBJTCRw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.54.0.tgz", + "integrity": "sha512-cOo7biqwkpawslEfox5Vs8/qj83M/aZCSSNIWpVzfU2CYHa2G3P1UN5WF01RdTHSgCkri7XOlTdtk17BezlV3A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.54.0.tgz", + "integrity": "sha512-miSvuFkmvFbgJ1BevMa4CPCFt5MPGw094knM64W9I0giUIMMmRYcGW/JWZDriaw/k1kOBtsWh1z6nIFV1vPNtA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.54.0.tgz", + "integrity": "sha512-KGXIs55+b/ZfZsq9aR026tmr/+7tq6VG6MsnrvF4H8VhwflTIuYh+LFUlIsRdQSgrgmtM3fVATzEAj4hBQlaqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.54.0.tgz", + "integrity": "sha512-EHMUcDwhtdRGlXZsGSIuXSYwD5kOT9NVnx9sqzYiwAc91wfYOE1g1djOEDseZJKKqtHAHGwnGPQu3kytmfaXLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.54.0.tgz", + "integrity": "sha512-+pBrqEjaakN2ySv5RVrj/qLytYhPKEUwk+e3SFU5jTLHIcAtqh2rLrd/OkbNuHJpsBgxsD8ccJt5ga/SeG0JmA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.54.0.tgz", + "integrity": "sha512-NSqc7rE9wuUaRBsBp5ckQ5CVz5aIRKCwsoa6WMF7G01sX3/qHUw/z4pv+D+ahL1EIKy6Enpcnz1RY8pf7bjwng==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.54.0.tgz", + "integrity": "sha512-gr5vDbg3Bakga5kbdpqx81m2n9IX8M6gIMlQQIXiLTNeQW6CucvuInJ91EuCJ/JYvc+rcLLsDFcfAD1K7fMofg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.54.0.tgz", + "integrity": "sha512-gsrtB1NA3ZYj2vq0Rzkylo9ylCtW/PhpLEivlgWe0bpgtX5+9j9EZa0wtZiCjgu6zmSeZWyI/e2YRX1URozpIw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.54.0.tgz", + "integrity": "sha512-y3qNOfTBStmFNq+t4s7Tmc9hW2ENtPg8FeUD/VShI7rKxNW7O4fFeaYbMsd3tpFlIg1Q8IapFgy7Q9i2BqeBvA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.54.0.tgz", + "integrity": "sha512-89sepv7h2lIVPsFma8iwmccN7Yjjtgz0Rj/Ou6fEqg3HDhpCa+Et+YSufy27i6b0Wav69Qv4WBNl3Rs6pwhebQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.54.0.tgz", + "integrity": "sha512-ZcU77ieh0M2Q8Ur7D5X7KvK+UxbXeDHwiOt/CPSBTI1fBmeDMivW0dPkdqkT4rOgDjrDDBUed9x4EgraIKoR2A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.54.0.tgz", + "integrity": "sha512-2AdWy5RdDF5+4YfG/YesGDDtbyJlC9LHmL6rZw6FurBJ5n4vFGupsOBGfwMRjBYH7qRQowT8D/U4LoSvVwOhSQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.54.0.tgz", + "integrity": "sha512-WGt5J8Ij/rvyqpFexxk3ffKqqbLf9AqrTBbWDk7ApGUzaIs6V+s2s84kAxklFwmMF/vBNGrVdYgbblCOFFezMQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.54.0.tgz", + "integrity": "sha512-JzQmb38ATzHjxlPHuTH6tE7ojnMKM2kYNzt44LO/jJi8BpceEC8QuXYA908n8r3CNuG/B3BV8VR3Hi1rYtmPiw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.54.0.tgz", + "integrity": "sha512-huT3fd0iC7jigGh7n3q/+lfPcXxBi+om/Rs3yiFxjvSxbSB6aohDFXbWvlspaqjeOh+hx7DDHS+5Es5qRkWkZg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.54.0.tgz", + "integrity": "sha512-c2V0W1bsKIKfbLMBu/WGBz6Yci8nJ/ZJdheE0EwB73N3MvHYKiKGs3mVilX4Gs70eGeDaMqEob25Tw2Gb9Nqyw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.54.0.tgz", + "integrity": "sha512-woEHgqQqDCkAzrDhvDipnSirm5vxUXtSKDYTVpZG3nUdW/VVB5VdCYA2iReSj/u3yCZzXID4kuKG7OynPnB3WQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.54.0.tgz", + "integrity": "sha512-dzAc53LOuFvHwbCEOS0rPbXp6SIhAf2txMP5p6mGyOXXw5mWY8NGGbPMPrs4P1WItkfApDathBj/NzMLUZ9rtQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.54.0.tgz", + "integrity": "sha512-hYT5d3YNdSh3mbCU1gwQyPgQd3T2ne0A3KG8KSBdav5TiBg6eInVmV+TeR5uHufiIgSFg0XsOWGW5/RhNcSvPg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@tailwindcss/node": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/node/-/node-4.1.18.tgz", + "integrity": "sha512-DoR7U1P7iYhw16qJ49fgXUlry1t4CpXeErJHnQ44JgTSKMaZUdf17cfn5mHchfJ4KRBZRFA/Coo+MUF5+gOaCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.4", + "enhanced-resolve": "^5.18.3", + "jiti": "^2.6.1", + "lightningcss": "1.30.2", + "magic-string": "^0.30.21", + "source-map-js": "^1.2.1", + "tailwindcss": "4.1.18" + } + }, + "node_modules/@tailwindcss/oxide": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide/-/oxide-4.1.18.tgz", + "integrity": "sha512-EgCR5tTS5bUSKQgzeMClT6iCY3ToqE1y+ZB0AKldj809QXk1Y+3jB0upOYZrn9aGIzPtUsP7sX4QQ4XtjBB95A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@tailwindcss/oxide-android-arm64": "4.1.18", + "@tailwindcss/oxide-darwin-arm64": "4.1.18", + "@tailwindcss/oxide-darwin-x64": "4.1.18", + "@tailwindcss/oxide-freebsd-x64": "4.1.18", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.1.18", + "@tailwindcss/oxide-linux-arm64-gnu": "4.1.18", + "@tailwindcss/oxide-linux-arm64-musl": "4.1.18", + "@tailwindcss/oxide-linux-x64-gnu": "4.1.18", + "@tailwindcss/oxide-linux-x64-musl": "4.1.18", + "@tailwindcss/oxide-wasm32-wasi": "4.1.18", + "@tailwindcss/oxide-win32-arm64-msvc": "4.1.18", + "@tailwindcss/oxide-win32-x64-msvc": "4.1.18" + } + }, + "node_modules/@tailwindcss/oxide-android-arm64": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.1.18.tgz", + "integrity": "sha512-dJHz7+Ugr9U/diKJA0W6N/6/cjI+ZTAoxPf9Iz9BFRF2GzEX8IvXxFIi/dZBloVJX/MZGvRuFA9rqwdiIEZQ0Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-darwin-arm64": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.1.18.tgz", + "integrity": "sha512-Gc2q4Qhs660bhjyBSKgq6BYvwDz4G+BuyJ5H1xfhmDR3D8HnHCmT/BSkvSL0vQLy/nkMLY20PQ2OoYMO15Jd0A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-darwin-x64": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.1.18.tgz", + "integrity": "sha512-FL5oxr2xQsFrc3X9o1fjHKBYBMD1QZNyc1Xzw/h5Qu4XnEBi3dZn96HcHm41c/euGV+GRiXFfh2hUCyKi/e+yw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-freebsd-x64": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.1.18.tgz", + "integrity": "sha512-Fj+RHgu5bDodmV1dM9yAxlfJwkkWvLiRjbhuO2LEtwtlYlBgiAT4x/j5wQr1tC3SANAgD+0YcmWVrj8R9trVMA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.1.18.tgz", + "integrity": "sha512-Fp+Wzk/Ws4dZn+LV2Nqx3IilnhH51YZoRaYHQsVq3RQvEl+71VGKFpkfHrLM/Li+kt5c0DJe/bHXK1eHgDmdiA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.1.18.tgz", + "integrity": "sha512-S0n3jboLysNbh55Vrt7pk9wgpyTTPD0fdQeh7wQfMqLPM/Hrxi+dVsLsPrycQjGKEQk85Kgbx+6+QnYNiHalnw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.1.18.tgz", + "integrity": "sha512-1px92582HkPQlaaCkdRcio71p8bc8i/ap5807tPRDK/uw953cauQBT8c5tVGkOwrHMfc2Yh6UuxaH4vtTjGvHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-gnu": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.1.18.tgz", + "integrity": "sha512-v3gyT0ivkfBLoZGF9LyHmts0Isc8jHZyVcbzio6Wpzifg/+5ZJpDiRiUhDLkcr7f/r38SWNe7ucxmGW3j3Kb/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-musl": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.1.18.tgz", + "integrity": "sha512-bhJ2y2OQNlcRwwgOAGMY0xTFStt4/wyU6pvI6LSuZpRgKQwxTec0/3Scu91O8ir7qCR3AuepQKLU/kX99FouqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.1.18.tgz", + "integrity": "sha512-LffYTvPjODiP6PT16oNeUQJzNVyJl1cjIebq/rWWBF+3eDst5JGEFSc5cWxyRCJ0Mxl+KyIkqRxk1XPEs9x8TA==", + "bundleDependencies": [ + "@napi-rs/wasm-runtime", + "@emnapi/core", + "@emnapi/runtime", + "@tybys/wasm-util", + "@emnapi/wasi-threads", + "tslib" + ], + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1", + "@emnapi/wasi-threads": "^1.1.0", + "@napi-rs/wasm-runtime": "^1.1.0", + "@tybys/wasm-util": "^0.10.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.18.tgz", + "integrity": "sha512-HjSA7mr9HmC8fu6bdsZvZ+dhjyGCLdotjVOgLA2vEqxEBZaQo9YTX4kwgEvPCpRh8o4uWc4J/wEoFzhEmjvPbA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/oxide-win32-x64-msvc": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.1.18.tgz", + "integrity": "sha512-bJWbyYpUlqamC8dpR7pfjA0I7vdF6t5VpUGMWRkXVE3AXgIZjYUYAK7II1GNaxR8J1SSrSrppRar8G++JekE3Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tailwindcss/vite": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/@tailwindcss/vite/-/vite-4.1.18.tgz", + "integrity": "sha512-jVA+/UpKL1vRLg6Hkao5jldawNmRo7mQYrZtNHMIVpLfLhDml5nMRUo/8MwoX2vNXvnaXNNMedrMfMugAVX1nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tailwindcss/node": "4.1.18", + "@tailwindcss/oxide": "4.1.18", + "tailwindcss": "4.1.18" + }, + "peerDependencies": { + "vite": "^5.2.0 || ^6 || ^7" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmmirror.com/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vitejs/plugin-vue": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/@vitejs/plugin-vue/-/plugin-vue-6.0.3.tgz", + "integrity": "sha512-TlGPkLFLVOY3T7fZrwdvKpjprR3s4fxRln0ORDo1VQ7HHyxJwTlrjKU3kpVWTlaAjIEuCTokmjkZnr8Tpc925w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "1.0.0-beta.53" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0", + "vue": "^3.2.25" + } + }, + "node_modules/@vue/compiler-core": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/@vue/compiler-core/-/compiler-core-3.5.26.tgz", + "integrity": "sha512-vXyI5GMfuoBCnv5ucIT7jhHKl55Y477yxP6fc4eUswjP8FG3FFVFd41eNDArR+Uk3QKn2Z85NavjaxLxOC19/w==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.5", + "@vue/shared": "3.5.26", + "entities": "^7.0.0", + "estree-walker": "^2.0.2", + "source-map-js": "^1.2.1" + } + }, + "node_modules/@vue/compiler-dom": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/@vue/compiler-dom/-/compiler-dom-3.5.26.tgz", + "integrity": "sha512-y1Tcd3eXs834QjswshSilCBnKGeQjQXB6PqFn/1nxcQw4pmG42G8lwz+FZPAZAby6gZeHSt/8LMPfZ4Rb+Bd/A==", + "license": "MIT", + "dependencies": { + "@vue/compiler-core": "3.5.26", + "@vue/shared": "3.5.26" + } + }, + "node_modules/@vue/compiler-sfc": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/@vue/compiler-sfc/-/compiler-sfc-3.5.26.tgz", + "integrity": "sha512-egp69qDTSEZcf4bGOSsprUr4xI73wfrY5oRs6GSgXFTiHrWj4Y3X5Ydtip9QMqiCMCPVwLglB9GBxXtTadJ3mA==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.5", + "@vue/compiler-core": "3.5.26", + "@vue/compiler-dom": "3.5.26", + "@vue/compiler-ssr": "3.5.26", + "@vue/shared": "3.5.26", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.21", + "postcss": "^8.5.6", + "source-map-js": "^1.2.1" + } + }, + "node_modules/@vue/compiler-ssr": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/@vue/compiler-ssr/-/compiler-ssr-3.5.26.tgz", + "integrity": "sha512-lZT9/Y0nSIRUPVvapFJEVDbEXruZh2IYHMk2zTtEgJSlP5gVOqeWXH54xDKAaFS4rTnDeDBQUYDtxKyoW9FwDw==", + "license": "MIT", + "dependencies": { + "@vue/compiler-dom": "3.5.26", + "@vue/shared": "3.5.26" + } + }, + "node_modules/@vue/reactivity": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/@vue/reactivity/-/reactivity-3.5.26.tgz", + "integrity": "sha512-9EnYB1/DIiUYYnzlnUBgwU32NNvLp/nhxLXeWRhHUEeWNTn1ECxX8aGO7RTXeX6PPcxe3LLuNBFoJbV4QZ+CFQ==", + "license": "MIT", + "dependencies": { + "@vue/shared": "3.5.26" + } + }, + "node_modules/@vue/runtime-core": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/@vue/runtime-core/-/runtime-core-3.5.26.tgz", + "integrity": "sha512-xJWM9KH1kd201w5DvMDOwDHYhrdPTrAatn56oB/LRG4plEQeZRQLw0Bpwih9KYoqmzaxF0OKSn6swzYi84e1/Q==", + "license": "MIT", + "dependencies": { + "@vue/reactivity": "3.5.26", + "@vue/shared": "3.5.26" + } + }, + "node_modules/@vue/runtime-dom": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/@vue/runtime-dom/-/runtime-dom-3.5.26.tgz", + "integrity": "sha512-XLLd/+4sPC2ZkN/6+V4O4gjJu6kSDbHAChvsyWgm1oGbdSO3efvGYnm25yCjtFm/K7rrSDvSfPDgN1pHgS4VNQ==", + "license": "MIT", + "dependencies": { + "@vue/reactivity": "3.5.26", + "@vue/runtime-core": "3.5.26", + "@vue/shared": "3.5.26", + "csstype": "^3.2.3" + } + }, + "node_modules/@vue/server-renderer": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/@vue/server-renderer/-/server-renderer-3.5.26.tgz", + "integrity": "sha512-TYKLXmrwWKSodyVuO1WAubucd+1XlLg4set0YoV+Hu8Lo79mp/YMwWV5mC5FgtsDxX3qo1ONrxFaTP1OQgy1uA==", + "license": "MIT", + "dependencies": { + "@vue/compiler-ssr": "3.5.26", + "@vue/shared": "3.5.26" + }, + "peerDependencies": { + "vue": "3.5.26" + } + }, + "node_modules/@vue/shared": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/@vue/shared/-/shared-3.5.26.tgz", + "integrity": "sha512-7Z6/y3uFI5PRoKeorTOSXKcDj0MSasfNNltcslbFrPpcw6aXRUALq4IfJlaTRspiWIUOEZbrpM+iQGmCOiWe4A==", + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmmirror.com/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/enhanced-resolve": { + "version": "5.18.4", + "resolved": "https://registry.npmmirror.com/enhanced-resolve/-/enhanced-resolve-5.18.4.tgz", + "integrity": "sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/entities/-/entities-7.0.0.tgz", + "integrity": "sha512-FDWG5cmEYf2Z00IkYRhbFrwIwvdFKH07uV8dvNy0omp/Qb1xcyCWp2UDtcwJF4QZZvk0sLudP6/hAu42TaqVhQ==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/esbuild": { + "version": "0.27.2", + "resolved": "https://registry.npmmirror.com/esbuild/-/esbuild-0.27.2.tgz", + "integrity": "sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.2", + "@esbuild/android-arm": "0.27.2", + "@esbuild/android-arm64": "0.27.2", + "@esbuild/android-x64": "0.27.2", + "@esbuild/darwin-arm64": "0.27.2", + "@esbuild/darwin-x64": "0.27.2", + "@esbuild/freebsd-arm64": "0.27.2", + "@esbuild/freebsd-x64": "0.27.2", + "@esbuild/linux-arm": "0.27.2", + "@esbuild/linux-arm64": "0.27.2", + "@esbuild/linux-ia32": "0.27.2", + "@esbuild/linux-loong64": "0.27.2", + "@esbuild/linux-mips64el": "0.27.2", + "@esbuild/linux-ppc64": "0.27.2", + "@esbuild/linux-riscv64": "0.27.2", + "@esbuild/linux-s390x": "0.27.2", + "@esbuild/linux-x64": "0.27.2", + "@esbuild/netbsd-arm64": "0.27.2", + "@esbuild/netbsd-x64": "0.27.2", + "@esbuild/openbsd-arm64": "0.27.2", + "@esbuild/openbsd-x64": "0.27.2", + "@esbuild/openharmony-arm64": "0.27.2", + "@esbuild/sunos-x64": "0.27.2", + "@esbuild/win32-arm64": "0.27.2", + "@esbuild/win32-ia32": "0.27.2", + "@esbuild/win32-x64": "0.27.2" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmmirror.com/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmmirror.com/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.6.1", + "resolved": "https://registry.npmmirror.com/jiti/-/jiti-2.6.1.tgz", + "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/lightningcss": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss/-/lightningcss-1.30.2.tgz", + "integrity": "sha512-utfs7Pr5uJyyvDETitgsaqSyjCb2qNRAtuqUeWIAKztsOYdcACf2KtARYXg2pSvhkt+9NfoaNY7fxjl6nuMjIQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.30.2", + "lightningcss-darwin-arm64": "1.30.2", + "lightningcss-darwin-x64": "1.30.2", + "lightningcss-freebsd-x64": "1.30.2", + "lightningcss-linux-arm-gnueabihf": "1.30.2", + "lightningcss-linux-arm64-gnu": "1.30.2", + "lightningcss-linux-arm64-musl": "1.30.2", + "lightningcss-linux-x64-gnu": "1.30.2", + "lightningcss-linux-x64-musl": "1.30.2", + "lightningcss-win32-arm64-msvc": "1.30.2", + "lightningcss-win32-x64-msvc": "1.30.2" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-android-arm64/-/lightningcss-android-arm64-1.30.2.tgz", + "integrity": "sha512-BH9sEdOCahSgmkVhBLeU7Hc9DWeZ1Eb6wNS6Da8igvUwAe0sqROHddIlvU06q3WyXVEOYDZ6ykBZQnjTbmo4+A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.30.2.tgz", + "integrity": "sha512-ylTcDJBN3Hp21TdhRT5zBOIi73P6/W0qwvlFEk22fkdXchtNTOU4Qc37SkzV+EKYxLouZ6M4LG9NfZ1qkhhBWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.30.2.tgz", + "integrity": "sha512-oBZgKchomuDYxr7ilwLcyms6BCyLn0z8J0+ZZmfpjwg9fRVZIR5/GMXd7r9RH94iDhld3UmSjBM6nXWM2TfZTQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.30.2.tgz", + "integrity": "sha512-c2bH6xTrf4BDpK8MoGG4Bd6zAMZDAXS569UxCAGcA7IKbHNMlhGQ89eRmvpIUGfKWNVdbhSbkQaWhEoMGmGslA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.30.2.tgz", + "integrity": "sha512-eVdpxh4wYcm0PofJIZVuYuLiqBIakQ9uFZmipf6LF/HRj5Bgm0eb3qL/mr1smyXIS1twwOxNWndd8z0E374hiA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.30.2.tgz", + "integrity": "sha512-UK65WJAbwIJbiBFXpxrbTNArtfuznvxAJw4Q2ZGlU8kPeDIWEX1dg3rn2veBVUylA2Ezg89ktszWbaQnxD/e3A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.30.2.tgz", + "integrity": "sha512-5Vh9dGeblpTxWHpOx8iauV02popZDsCYMPIgiuw97OJ5uaDsL86cnqSFs5LZkG3ghHoX5isLgWzMs+eD1YzrnA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.30.2.tgz", + "integrity": "sha512-Cfd46gdmj1vQ+lR6VRTTadNHu6ALuw2pKR9lYq4FnhvgBc4zWY1EtZcAc6EffShbb1MFrIPfLDXD6Xprbnni4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.30.2.tgz", + "integrity": "sha512-XJaLUUFXb6/QG2lGIW6aIk6jKdtjtcffUT0NKvIqhSBY3hh9Ch+1LCeH80dR9q9LBjG3ewbDjnumefsLsP6aiA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.30.2.tgz", + "integrity": "sha512-FZn+vaj7zLv//D/192WFFVA0RgHawIcHqLX9xuWiQt7P0PtdFEVaxgF9rjM/IRYHQXNnk61/H/gb2Ei+kUQ4xQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.30.2", + "resolved": "https://registry.npmmirror.com/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.30.2.tgz", + "integrity": "sha512-5g1yc73p+iAkid5phb4oVFMB45417DkRevRbt/El/gKXJk4jid+vPFF/AXbxn05Aky8PapwzZrdJShv5C0avjw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmmirror.com/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmmirror.com/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/pdfjs-dist": { + "version": "4.10.38", + "resolved": "https://registry.npmmirror.com/pdfjs-dist/-/pdfjs-dist-4.10.38.tgz", + "integrity": "sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.65" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rollup": { + "version": "4.54.0", + "resolved": "https://registry.npmmirror.com/rollup/-/rollup-4.54.0.tgz", + "integrity": "sha512-3nk8Y3a9Ea8szgKhinMlGMhGMw89mqule3KWczxhIzqudyHdCIOHw8WJlj/r329fACjKLEh13ZSk7oE22kyeIw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.54.0", + "@rollup/rollup-android-arm64": "4.54.0", + "@rollup/rollup-darwin-arm64": "4.54.0", + "@rollup/rollup-darwin-x64": "4.54.0", + "@rollup/rollup-freebsd-arm64": "4.54.0", + "@rollup/rollup-freebsd-x64": "4.54.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.54.0", + "@rollup/rollup-linux-arm-musleabihf": "4.54.0", + "@rollup/rollup-linux-arm64-gnu": "4.54.0", + "@rollup/rollup-linux-arm64-musl": "4.54.0", + "@rollup/rollup-linux-loong64-gnu": "4.54.0", + "@rollup/rollup-linux-ppc64-gnu": "4.54.0", + "@rollup/rollup-linux-riscv64-gnu": "4.54.0", + "@rollup/rollup-linux-riscv64-musl": "4.54.0", + "@rollup/rollup-linux-s390x-gnu": "4.54.0", + "@rollup/rollup-linux-x64-gnu": "4.54.0", + "@rollup/rollup-linux-x64-musl": "4.54.0", + "@rollup/rollup-openharmony-arm64": "4.54.0", + "@rollup/rollup-win32-arm64-msvc": "4.54.0", + "@rollup/rollup-win32-ia32-msvc": "4.54.0", + "@rollup/rollup-win32-x64-gnu": "4.54.0", + "@rollup/rollup-win32-x64-msvc": "4.54.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tailwindcss": { + "version": "4.1.18", + "resolved": "https://registry.npmmirror.com/tailwindcss/-/tailwindcss-4.1.18.tgz", + "integrity": "sha512-4+Z+0yiYyEtUVCScyfHCxOYP06L5Ne+JiHhY2IjR2KWMIWhJOYZKLSGZaP5HkZ8+bY0cxfzwDE5uOmzFXyIwxw==", + "dev": true, + "license": "MIT" + }, + "node_modules/tapable": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/tapable/-/tapable-2.3.0.tgz", + "integrity": "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmmirror.com/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/vite": { + "version": "7.3.0", + "resolved": "https://registry.npmmirror.com/vite/-/vite-7.3.0.tgz", + "integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vue": { + "version": "3.5.26", + "resolved": "https://registry.npmmirror.com/vue/-/vue-3.5.26.tgz", + "integrity": "sha512-SJ/NTccVyAoNUJmkM9KUqPcYlY+u8OVL1X5EW9RIs3ch5H2uERxyyIUI4MRxVCSOiEcupX9xNGde1tL9ZKpimA==", + "license": "MIT", + "dependencies": { + "@vue/compiler-dom": "3.5.26", + "@vue/compiler-sfc": "3.5.26", + "@vue/runtime-dom": "3.5.26", + "@vue/server-renderer": "3.5.26", + "@vue/shared": "3.5.26" + }, + "peerDependencies": { + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + } + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..f00894f --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,21 @@ +{ + "name": "frontend", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "pdfjs-dist": "^4.2.67", + "vue": "^3.5.24" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.1.18", + "@vitejs/plugin-vue": "^6.0.1", + "tailwindcss": "^4.1.18", + "vite": "^7.2.4" + } +} diff --git a/frontend/public/excel-viewer.html b/frontend/public/excel-viewer.html new file mode 100644 index 0000000..2378e70 --- /dev/null +++ b/frontend/public/excel-viewer.html @@ -0,0 +1,359 @@ + + + + + + Excel 查看器 + + + +
+
+ + 加载中... +
+
+ + + + diff --git a/frontend/public/pdf-highlight.html b/frontend/public/pdf-highlight.html new file mode 100644 index 0000000..fbd136b --- /dev/null +++ b/frontend/public/pdf-highlight.html @@ -0,0 +1,127 @@ + + + + + + PDF Highlight Viewer + + + +
+
Loading PDF...
+
+ + + + diff --git a/frontend/public/vite.svg b/frontend/public/vite.svg new file mode 100644 index 0000000..e7b8dfb --- /dev/null +++ b/frontend/public/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/App.vue b/frontend/src/App.vue new file mode 100644 index 0000000..a82baba --- /dev/null +++ b/frontend/src/App.vue @@ -0,0 +1,322 @@ + + + diff --git a/frontend/src/api.js b/frontend/src/api.js new file mode 100644 index 0000000..f5c405b --- /dev/null +++ b/frontend/src/api.js @@ -0,0 +1,734 @@ +const API_BASE = '/api/v1/knowledge' + +// 用户认证信息（实际应用中应该从登录获取） +const USER_ID = 'user1' +const USER_NAME = '42tr' +const ROLE = 'admin' + +const getHeaders = (contentType = true) => { + const headers = { + 'x-user-id': USER_ID, + 'x-user-name': USER_NAME, + 'x-role': ROLE, + } + if (contentType) { + headers['Content-Type'] = 'application/json' + } + return headers +} + +const readErrorMessage = async (response, fallback) => { + try { + const payload = await response.json() + if (payload?.error) return payload.error + if (payload?.message) return payload.message + } catch (_) { + // ignore json parse error + } + return fallback +} + +export const api = { + // 搜索 + async search(query, kbId = null, fileId = null, options = {}) { + const { advanced = false } = options + let url = `${API_BASE}/search/?query=${encodeURIComponent(query)}` + if (kbId) { + url += `&kb_id=${kbId}` + } + if (fileId) { + url += `&file_id=${fileId}` + } + if (advanced) { + url += '&advanced=true' + } + const response = await fetch(url, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('搜索失败') + const data = await response.json() + return data.results || [] + }, + + advancedSearchStream(params, handlers = {}) { + const { + query, + kbId = null, + fileId = null, + maxSubQueries = 3, + perQueryLimit = 10, + contextChars = 2000, + debug = false, + } = params + + const searchParams = new URLSearchParams() + searchParams.set('query', query) + searchParams.set('max_sub_queries', maxSubQueries) + searchParams.set('per_query_limit', perQueryLimit) + searchParams.set('context_chars', contextChars) + if (debug) searchParams.set('debug', 'true') + if (kbId) searchParams.set('kb_id', kbId) + if (fileId) searchParams.set('file_id', fileId) + + const controller = new AbortController() + const url = `${API_BASE}/search/advanced/stream?${searchParams.toString()}` + const headers = getHeaders(false) + const decoder = new TextDecoder('utf-8') + + const parseEvent = (chunk) => { + const lines = chunk.split('\n') + let eventType = 'message' + let dataLines = [] + for (const line of lines) { + const trimmed = line.trim() + if (trimmed.startsWith('event:')) { + eventType = trimmed.slice(6).trim() + } else if (trimmed.startsWith('data:')) { + dataLines.push(trimmed.slice(5).trim()) + } + } + const dataStr = dataLines.join('\n') + let parsedData = null + if (dataStr) { + try { + parsedData = JSON.parse(dataStr) + } catch (err) { + parsedData = dataStr + } + } + return { eventType, data: parsedData } + } + + const streamPromise = (async () => { + try { + const response = await fetch(url, { + headers, + signal: controller.signal, + }) + if (!response.ok) { + throw new Error('高级搜索连接失败') + } + const reader = response.body.getReader() + let buffer = '' + while (true) { + const { value, done } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + let boundary + while ((boundary = buffer.indexOf('\n\n')) !== -1) { + const rawEvent = buffer.slice(0, boundary).trim() + buffer = buffer.slice(boundary + 2) + if (!rawEvent) continue + const { eventType, data } = parseEvent(rawEvent) + switch (eventType) { + case 'status': + handlers.onStatus?.(data) + break + case 'plan': + handlers.onPlan?.(data) + break + case 'step': + handlers.onStep?.(data) + break + case 'candidate': + handlers.onCandidate?.(data) + break + case 'filtered': + handlers.onFiltered?.(data) + break + case 'result': + handlers.onResult?.(data) + break + case 'error': + handlers.onErrorEvent?.(data) + break + case 'done': + handlers.onDone?.() + break + default: + handlers.onMessage?.({ eventType, data }) + } + } + } + handlers.onComplete?.() + } catch (err) { + if (controller.signal.aborted) { + handlers.onAbort?.() + } else { + handlers.onError?.(err) + } + } finally { + handlers.onFinally?.() + } + })() + + return { + cancel: () => controller.abort(), + finished: streamPromise, + } + }, + async searchFull(query, kbId = null, fileId = null) { + let url = `${API_BASE}/search/full?query=${encodeURIComponent(query)}` + if (kbId) { + url += `&kb_id=${kbId}` + } + if (fileId) { + url += `&file_id=${fileId}` + } + const response = await fetch(url, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('全文搜索失败') + const data = await response.json() + return data.results || [] + }, + async searchImage(file, text = '', kbId = null, fileId = null) { + const formData = new FormData() + formData.append('file', file) + formData.append('text', text) + let url = `${API_BASE}/search/image` + const params = [] + if (kbId) { + params.push(`kb_id=${kbId}`) + } + if (fileId) { + params.push(`file_id=${fileId}`) + } + if (params.length > 0) { + url += `?${params.join('&')}` + } + const response = await fetch(url, { + method: 'POST', + headers: getHeaders(false), + body: formData, + }) + if (!response.ok) throw new Error('图片搜索失败') + const data = await response.json() + return data.results || [] + }, + + // 词表 + async listLexicons(params = {}) { + const searchParams = new URLSearchParams() + if (params.q) searchParams.set('q', params.q) + if (params.enabled === true) searchParams.set('enabled', 'true') + if (params.enabled === false) searchParams.set('enabled', 'false') + if (params.limit) searchParams.set('limit', String(params.limit)) + if (params.offset) searchParams.set('offset', String(params.offset)) + const query = searchParams.toString() + const url = `${API_BASE}/search/lexicons${query ? `?${query}` : ''}` + const response = await fetch(url, { headers: getHeaders() }) + if (!response.ok) throw new Error(await readErrorMessage(response, '获取词表失败')) + return response.json() + }, + + async createLexicon(data) { + const response = await fetch(`${API_BASE}/search/lexicons`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify(data), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '创建词条失败')) + return response.json() + }, + + async updateLexicon(id, data) { + const response = await fetch(`${API_BASE}/search/lexicons/${id}`, { + method: 'PUT', + headers: getHeaders(), + body: JSON.stringify(data), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '更新词条失败')) + return response.json() + }, + + async deleteLexicon(id) { + const response = await fetch(`${API_BASE}/search/lexicons/${id}`, { + method: 'DELETE', + headers: getHeaders(), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '删除词条失败')) + return response.json() + }, + + async toggleLexiconEnabled(id, enabled) { + const response = await fetch(`${API_BASE}/search/lexicons/${id}/enabled`, { + method: 'PUT', + headers: getHeaders(), + body: JSON.stringify({ enabled }), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '更新词条状态失败')) + return response.json() + }, + + async reloadLexicon() { + const response = await fetch(`${API_BASE}/search/lexicons/reload`, { + method: 'POST', + headers: getHeaders(), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '重载词表失败')) + return response.json() + }, + + async publishLexicon() { + const response = await fetch(`${API_BASE}/search/lexicons/publish`, { + method: 'POST', + headers: getHeaders(), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '发布词表失败')) + return response.json() + }, + + // 同义词 + async listSynonyms(params = {}) { + const searchParams = new URLSearchParams() + if (params.q) searchParams.set('q', params.q) + if (params.enabled === true) searchParams.set('enabled', 'true') + if (params.enabled === false) searchParams.set('enabled', 'false') + if (params.limit) searchParams.set('limit', String(params.limit)) + if (params.offset) searchParams.set('offset', String(params.offset)) + const query = searchParams.toString() + const url = `${API_BASE}/search/synonyms${query ? `?${query}` : ''}` + const response = await fetch(url, { headers: getHeaders() }) + if (!response.ok) throw new Error(await readErrorMessage(response, '获取同义词失败')) + return response.json() + }, + + async createSynonym(data) { + const response = await fetch(`${API_BASE}/search/synonyms`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify(data), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '创建同义词失败')) + return response.json() + }, + + async updateSynonym(id, data) { + const response = await fetch(`${API_BASE}/search/synonyms/${id}`, { + method: 'PUT', + headers: getHeaders(), + body: JSON.stringify(data), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '更新同义词失败')) + return response.json() + }, + + async deleteSynonym(id) { + const response = await fetch(`${API_BASE}/search/synonyms/${id}`, { + method: 'DELETE', + headers: getHeaders(), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '删除同义词失败')) + return response.json() + }, + + async toggleSynonymEnabled(id, enabled) { + const response = await fetch(`${API_BASE}/search/synonyms/${id}/enabled`, { + method: 'PUT', + headers: getHeaders(), + body: JSON.stringify({ enabled }), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '更新同义词状态失败')) + return response.json() + }, + + // 系统 + async getIndexRebuildStatus() { + const response = await fetch(`${API_BASE}/system/index/rebuild/status`, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '获取重建状态失败')) + return response.json() + }, + + // 知识库 + async getKnowledgeBases(parentId = null) { + let url = `${API_BASE}/knowledge_base/`; + const params = new URLSearchParams(); + // A null parentId fetches top-level KBs by default on the backend. + if (parentId) { + params.append('parent_id', parentId); + } + const queryString = params.toString(); + if (queryString) { + url += `?${queryString}`; + } + const response = await fetch(url, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('获取知识库列表失败') + return response.json() + }, + + async getKnowledgeBase(id) { + const response = await fetch(`${API_BASE}/knowledge_base/${id}`, { + headers: getHeaders(), + }); + if (!response.ok) throw new Error('获取知识库详情失败'); + return response.json(); + }, + + async getKnowledgeBaseFiles(id, params = {}) { + const searchParams = new URLSearchParams() + if (params.page) searchParams.set('page', String(params.page)) + if (params.size) searchParams.set('size', String(params.size)) + if (params.filename) searchParams.set('filename', params.filename) + if (params.tag) searchParams.set('tag', params.tag) + const query = searchParams.toString() + const response = await fetch( + `${API_BASE}/knowledge_base/${id}/files${query ? `?${query}` : ''}`, + { headers: getHeaders() } + ) + if (!response.ok) throw new Error(await readErrorMessage(response, '获取知识库文件失败')) + return response.json() + }, + + async createKnowledgeBase(data) { // data may include { name, description, parent_id } + const response = await fetch(`${API_BASE}/knowledge_base/`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify(data), + }) + if (!response.ok) throw new Error('创建知识库失败') + return response.json() + }, + + async updateKnowledgeBase(id, data) { // data may include { name, description, parent_id, is_public } + const response = await fetch(`${API_BASE}/knowledge_base/${id}`, { + method: 'PUT', + headers: getHeaders(), + body: JSON.stringify(data), + }) + if (!response.ok) throw new Error('更新知识库失败') + return response.json() + }, + + async deleteKnowledgeBase(id) { + const response = await fetch(`${API_BASE}/knowledge_base/${id}`, { + method: 'DELETE', + headers: getHeaders(), + }) + if (!response.ok) throw new Error('删除知识库失败') + return + }, + + async reparseKnowledgeBases() { + const response = await fetch(`${API_BASE}/knowledge_base/reparse`, { + method: 'POST', + headers: getHeaders(), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '重新解析失败')) + return response.json() + }, + + async reparseKnowledgeBase(id) { + const response = await fetch(`${API_BASE}/knowledge_base/${id}/reparse`, { + method: 'POST', + headers: getHeaders(), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '重新解析指定知识库失败')) + return response.json() + }, + + async exportKnowledgeBases(kbIds, includeChildren = false) { + const response = await fetch(`${API_BASE}/knowledge_base/export`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify({ kb_ids: kbIds, include_children: includeChildren }), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '导出知识库失败')) + return response.json() + }, + + // 知识库权限管理 + async getKbPermissions(kbId) { + const response = await fetch(`${API_BASE}/knowledge_base/${kbId}/permissions`, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('获取权限列表失败') + return response.json() + }, + + async addKbPermission(kbId, userId, permission) { + const response = await fetch(`${API_BASE}/knowledge_base/${kbId}/permissions`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify({ user_id: userId, permission }), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '添加权限失败')) + return response.json() + }, + + async removeKbPermission(kbId, userId) { + const response = await fetch(`${API_BASE}/knowledge_base/${kbId}/permissions/${userId}`, { + method: 'DELETE', + headers: getHeaders(), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '删除权限失败')) + return response.json() + }, + + // 文件 + async uploadFiles(knowledgeBaseId, files, tags = [], isPublic = false, sliceType = 'smart') { + const formData = new FormData() + if (knowledgeBaseId) { + formData.append('kb_id', knowledgeBaseId) + } + if (tags.length > 0) { + formData.append('tags', JSON.stringify(tags)) + } + formData.append('is_public', isPublic ? 'true' : 'false') + formData.append('slice_type', sliceType) + for (const file of files) { + formData.append('file', file) + } + + const response = await fetch(`${API_BASE}/files/`, { + method: 'POST', + headers: getHeaders(false), + body: formData, + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '上传文件失败')) + return response.json() + }, + + async deleteFile(id) { + const response = await fetch(`${API_BASE}/files/${id}`, { + method: 'DELETE', + headers: getHeaders(), + }) + if (!response.ok) throw new Error('删除文件失败') + return + }, + + async batchDeleteFiles(ids, options = {}) { + const { + strict = false, + allowProcessing = false, + } = options + + const response = await fetch(`${API_BASE}/files/batch-delete`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify({ + ids, + strict, + allow_processing: allowProcessing, + }), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '批量删除文件失败')) + return response.json() + }, + + async getFiles(kbId, tag = null, pagination = {}) { + let url = `${API_BASE}/files/` + const params = [] + if (kbId === null) { + // 明确传递null时，获取未分配知识库的文件 + params.push('kb_id=null') + } else if (kbId !== undefined) { + // 传递具体的知识库ID + params.push(`kb_id=${kbId}`) + } + if (tag) { + params.push(`tag=${encodeURIComponent(tag)}`) + } + if (pagination.page) params.push(`page=${pagination.page}`) + if (pagination.size) params.push(`size=${pagination.size}`) + if (params.length > 0) { + url += `?${params.join('&')}` + } + const response = await fetch(url, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('获取文件列表失败') + return response.json() + }, + + async getFileStats(options = {}) { + const { + kbId = undefined, + includeDescendants = true, + includeUnassigned = true, + } = options + const params = new URLSearchParams() + if (kbId === null) { + params.append('kb_id', 'null') + } else if (kbId !== undefined) { + params.append('kb_id', kbId) + } + if (!includeDescendants) { + params.append('include_descendants', 'false') + } + if (!includeUnassigned) { + params.append('include_unassigned', 'false') + } + const query = params.toString() + const response = await fetch( + `${API_BASE}/files/stats${query ? `?${query}` : ''}`, + { headers: getHeaders() } + ) + if (!response.ok) throw new Error('获取文件状态统计失败') + return response.json() + }, + + async reparseFailedFiles(options = {}) { + const { + kbId = undefined, + includeDescendants = true, + includeUnassigned = true, + unassignedOnly = false, + } = options + + const payload = { + include_descendants: includeDescendants, + include_unassigned: includeUnassigned, + unassigned_only: unassignedOnly, + } + if (kbId !== undefined) { + payload.kb_id = kbId ?? null + } + + const response = await fetch(`${API_BASE}/files/reparse-failed`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify(payload), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '重新解析失败文件失败')) + return response.json() + }, + + async getFile(id) { + const response = await fetch(`${API_BASE}/files/${id}`, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('获取文件失败') + return response.json() + }, + + async downloadFile(id) { + const response = await fetch(`${API_BASE}/files/${id}/download`, { + headers: getHeaders(false), + }) + if (!response.ok) throw new Error('下载文件失败') + return response.blob() + }, + + async updateFile(id, data) { + const response = await fetch(`${API_BASE}/files/${id}`, { + method: 'PUT', + headers: getHeaders(), + body: JSON.stringify(data), + }) + if (!response.ok) throw new Error('更新文件失败') + return response.json() + }, + + async moveFile(id, targetKbId) { + const response = await fetch(`${API_BASE}/files/${id}/move`, { + method: 'PUT', + headers: getHeaders(), + body: JSON.stringify({ + target_kb_id: targetKbId ?? null, + }), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '移动文件失败')) + return response.json() + }, + + async getFileSlices(fileId) { + const response = await fetch(`${API_BASE}/files/${fileId}/slices`, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('获取切片失败') + return response.json() + }, + + async updateSlices(fileId, slices) { + const response = await fetch(`${API_BASE}/files/${fileId}/slices`, { + method: 'PUT', + headers: getHeaders(), + body: JSON.stringify({ slices }), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '更新切片失败')) + return response.json() + }, + + // 压缩文件 + async getArchiveEntries(id) { + const response = await fetch(`${API_BASE}/files/${id}/archive-entries`, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('获取压缩文件列表失败') + return response.json() + }, + + async extractArchive(id, password = null) { + const response = await fetch(`${API_BASE}/files/${id}/archive-extract`, { + method: 'POST', + headers: getHeaders(), + body: JSON.stringify({ password }), + }) + if (!response.ok) throw new Error(await readErrorMessage(response, '解压失败')) + return response.json() + }, + + async downloadArchiveEntry(id, path) { + const encodedPath = encodeURIComponent(path) + const response = await fetch(`${API_BASE}/files/${id}/archive-download?path=${encodedPath}`, { + headers: getHeaders(false), + }) + if (!response.ok) throw new Error('下载文件失败') + return response.blob() + }, + + // 知识图谱 + async searchEntities(query = null, entityType = null, kbId = null, limit = 100, fileId = null) { + let url = `${API_BASE}/graph/entities?limit=${limit}` + if (query) { + url += `&q=${encodeURIComponent(query)}` + } + if (entityType) { + url += `&entity_type=${encodeURIComponent(entityType)}` + } + if (kbId) { + url += `&kb_id=${kbId}` + } + if (fileId) { + url += `&file_id=${fileId}` + } + const response = await fetch(url, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('搜索实体失败') + return response.json() + }, + + async getEntity(id) { + const response = await fetch(`${API_BASE}/graph/entities/${id}`, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('获取实体详情失败') + return response.json() + }, + + async getGraphStats(kbId = null, fileId = null) { + let url = `${API_BASE}/graph/stats` + const params = [] + if (kbId) { + params.push(`kb_id=${kbId}`) + } + if (fileId) { + params.push(`file_id=${fileId}`) + } + if (params.length > 0) { + url += '?' + params.join('&') + } + const response = await fetch(url, { + headers: getHeaders(), + }) + if (!response.ok) throw new Error('获取图谱统计失败') + return response.json() + }, +} diff --git a/frontend/src/assets/vue.svg b/frontend/src/assets/vue.svg new file mode 100644 index 0000000..770e9d3 --- /dev/null +++ b/frontend/src/assets/vue.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/components/AdvancedSearchPanel.vue b/frontend/src/components/AdvancedSearchPanel.vue new file mode 100644 index 0000000..d8c3778 --- /dev/null +++ b/frontend/src/components/AdvancedSearchPanel.vue @@ -0,0 +1,205 @@ + + + diff --git a/frontend/src/components/ArchiveTreeNode.vue b/frontend/src/components/ArchiveTreeNode.vue new file mode 100644 index 0000000..9bb4ea7 --- /dev/null +++ b/frontend/src/components/ArchiveTreeNode.vue @@ -0,0 +1,78 @@ + + + diff --git a/frontend/src/components/ArchiveViewer.vue b/frontend/src/components/ArchiveViewer.vue new file mode 100644 index 0000000..35cd4ce --- /dev/null +++ b/frontend/src/components/ArchiveViewer.vue @@ -0,0 +1,260 @@ + + + diff --git a/frontend/src/components/CreateKnowledgeBase.vue b/frontend/src/components/CreateKnowledgeBase.vue new file mode 100644 index 0000000..97d1a9f --- /dev/null +++ b/frontend/src/components/CreateKnowledgeBase.vue @@ -0,0 +1,223 @@ + + + diff --git a/frontend/src/components/EntityDetail.vue b/frontend/src/components/EntityDetail.vue new file mode 100644 index 0000000..64f3c46 --- /dev/null +++ b/frontend/src/components/EntityDetail.vue @@ -0,0 +1,194 @@ + + + diff --git a/frontend/src/components/ExportRecordPanel.vue b/frontend/src/components/ExportRecordPanel.vue new file mode 100644 index 0000000..555b8a7 --- /dev/null +++ b/frontend/src/components/ExportRecordPanel.vue @@ -0,0 +1,128 @@ + + + diff --git a/frontend/src/components/FileCard.vue b/frontend/src/components/FileCard.vue new file mode 100644 index 0000000..e1e57ed --- /dev/null +++ b/frontend/src/components/FileCard.vue @@ -0,0 +1,554 @@ + + + diff --git a/frontend/src/components/FileGraph.vue b/frontend/src/components/FileGraph.vue new file mode 100644 index 0000000..76afc70 --- /dev/null +++ b/frontend/src/components/FileGraph.vue @@ -0,0 +1,123 @@ + + + diff --git a/frontend/src/components/FileSlices.vue b/frontend/src/components/FileSlices.vue new file mode 100644 index 0000000..5902693 --- /dev/null +++ b/frontend/src/components/FileSlices.vue @@ -0,0 +1,270 @@ + + + diff --git a/frontend/src/components/FileStatusSummary.vue b/frontend/src/components/FileStatusSummary.vue new file mode 100644 index 0000000..9b8adf0 --- /dev/null +++ b/frontend/src/components/FileStatusSummary.vue @@ -0,0 +1,202 @@ + + + diff --git a/frontend/src/components/FileUpload.vue b/frontend/src/components/FileUpload.vue new file mode 100644 index 0000000..59b9c61 --- /dev/null +++ b/frontend/src/components/FileUpload.vue @@ -0,0 +1,322 @@ + + + diff --git a/frontend/src/components/GraphVisualization.vue b/frontend/src/components/GraphVisualization.vue new file mode 100644 index 0000000..ce623e6 --- /dev/null +++ b/frontend/src/components/GraphVisualization.vue @@ -0,0 +1,1121 @@ + + + diff --git a/frontend/src/components/KbPermissionModal.vue b/frontend/src/components/KbPermissionModal.vue new file mode 100644 index 0000000..3430903 --- /dev/null +++ b/frontend/src/components/KbPermissionModal.vue @@ -0,0 +1,175 @@ + + + diff --git a/frontend/src/components/KnowledgeBaseDetail.vue b/frontend/src/components/KnowledgeBaseDetail.vue new file mode 100644 index 0000000..b8e80ce --- /dev/null +++ b/frontend/src/components/KnowledgeBaseDetail.vue @@ -0,0 +1,164 @@ + + + diff --git a/frontend/src/components/KnowledgeBaseList.vue b/frontend/src/components/KnowledgeBaseList.vue new file mode 100644 index 0000000..6b08b36 --- /dev/null +++ b/frontend/src/components/KnowledgeBaseList.vue @@ -0,0 +1,793 @@ + + + diff --git a/frontend/src/components/KnowledgeBaseSelector.vue b/frontend/src/components/KnowledgeBaseSelector.vue new file mode 100644 index 0000000..c8be0f6 --- /dev/null +++ b/frontend/src/components/KnowledgeBaseSelector.vue @@ -0,0 +1,118 @@ + + + diff --git a/frontend/src/components/KnowledgeGraph.vue b/frontend/src/components/KnowledgeGraph.vue new file mode 100644 index 0000000..1a0ce04 --- /dev/null +++ b/frontend/src/components/KnowledgeGraph.vue @@ -0,0 +1,424 @@ + + + diff --git a/frontend/src/components/Pagination.vue b/frontend/src/components/Pagination.vue new file mode 100644 index 0000000..94b44c7 --- /dev/null +++ b/frontend/src/components/Pagination.vue @@ -0,0 +1,113 @@ + + + diff --git a/frontend/src/components/SearchBar.vue b/frontend/src/components/SearchBar.vue new file mode 100644 index 0000000..5db715d --- /dev/null +++ b/frontend/src/components/SearchBar.vue @@ -0,0 +1,330 @@ + + + diff --git a/frontend/src/components/SearchDictionaryManager.vue b/frontend/src/components/SearchDictionaryManager.vue new file mode 100644 index 0000000..f8c5dfc --- /dev/null +++ b/frontend/src/components/SearchDictionaryManager.vue @@ -0,0 +1,667 @@ + + + diff --git a/frontend/src/components/SearchResults.vue b/frontend/src/components/SearchResults.vue new file mode 100644 index 0000000..75c3c28 --- /dev/null +++ b/frontend/src/components/SearchResults.vue @@ -0,0 +1,166 @@ + + + + + diff --git a/frontend/src/main.js b/frontend/src/main.js new file mode 100644 index 0000000..2425c0f --- /dev/null +++ b/frontend/src/main.js @@ -0,0 +1,5 @@ +import { createApp } from 'vue' +import './style.css' +import App from './App.vue' + +createApp(App).mount('#app') diff --git a/frontend/src/store.js b/frontend/src/store.js new file mode 100644 index 0000000..0724763 --- /dev/null +++ b/frontend/src/store.js @@ -0,0 +1,10 @@ +import { ref } from 'vue' + +// This is a simple reactive store for cross-component state. +// For larger applications, consider using Pinia. + +export const currentKb = ref({ id: null, name: '所有知识库' }); + +export const setCurrentKb = (kb) => { + currentKb.value = kb; +}; diff --git a/frontend/src/style.css b/frontend/src/style.css new file mode 100644 index 0000000..f1d8c73 --- /dev/null +++ b/frontend/src/style.css @@ -0,0 +1 @@ +@import "tailwindcss"; diff --git a/frontend/vite.config.js b/frontend/vite.config.js new file mode 100644 index 0000000..754f8d7 --- /dev/null +++ b/frontend/vite.config.js @@ -0,0 +1,25 @@ +import { defineConfig } from 'vite' +import vue from '@vitejs/plugin-vue' +import tailwindcss from '@tailwindcss/vite' + +export default defineConfig({ + plugins: [vue(), tailwindcss()], + optimizeDeps: { + include: ['pdfjs-dist'], + }, + resolve: { + dedupe: ['pdfjs-dist'], + }, + build: { + outDir: 'dist', + emptyOutDir: true, + }, + server: { + proxy: { + '/api': { + target: 'http://localhost:3000', + changeOrigin: true, + }, + }, + }, +}) diff --git a/src/api/common.rs b/src/api/common.rs new file mode 100644 index 0000000..03c6a92 --- /dev/null +++ b/src/api/common.rs @@ -0,0 +1,106 @@ +use sqlx::{QueryBuilder, Sqlite, SqlitePool}; + +use crate::{ + AuthUser, api::{ + error::{ApiError, ApiResult}, knowledge_base + } +}; + +/// 要求当前用户为 admin，否则返回 BadRequest。 +pub fn ensure_admin(auth_user: &AuthUser) -> ApiResult<()> { + if auth_user.is_admin() { Ok(()) } else { Err(ApiError::BadRequest("admin role required".to_string())) } +} + +/// 校验用户能否访问指定知识库（owner / public / 显式授权）。不存在或无权限返回 NotFound。 +pub async fn ensure_kb_accessible(pool: &SqlitePool, kb_id: i64, user_id: &str, is_admin: bool) -> ApiResult<()> { + let perm = knowledge_base::get_kb_permission(pool, kb_id, user_id, is_admin).await; + if perm.is_none() { + return Err(ApiError::NotFound("Knowledge base not found or permission denied".to_string())); + } + Ok(()) +} + +/// 校验用户是否为指定知识库的 editor / admin。无权限返回 Forbidden。 +pub async fn ensure_kb_editor_or_admin(pool: &SqlitePool, kb_id: i64, auth_user: &AuthUser) -> ApiResult<()> { + let perm = knowledge_base::get_kb_permission(pool, kb_id, &auth_user.user_id, auth_user.is_admin()).await; + if !knowledge_base::meets_requirement(perm.as_deref(), "editor") { + return Err(ApiError::Forbidden("Permission denied. Requires editor or admin.".to_string())); + } + Ok(()) +} + +/// 追加知识库访问过滤条件（不含别名）。 +/// 生成 `AND (user_id = ? OR is_public = 1 OR id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ?))` +pub fn push_kb_access_filter<'a>(qb: &mut QueryBuilder<'a, Sqlite>, user_id: &'a str) { + qb.push(" AND (user_id = "); + qb.push_bind(user_id); + qb.push(" OR is_public = 1 OR id IN (SELECT kb_id FROM kb_permissions WHERE user_id = "); + qb.push_bind(user_id); + qb.push(")"); + qb.push(")"); +} + +/// 追加知识库访问过滤条件（带 `kb.` 别名）。 +/// 生成 `WHERE (kb.user_id = ? OR kb.is_public = 1 OR kb.id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ?))` +pub fn push_kb_access_filter_where<'a>(qb: &mut QueryBuilder<'a, Sqlite>, user_id: &'a str) { + qb.push(" WHERE (kb.user_id = "); + qb.push_bind(user_id); + qb.push(" OR kb.is_public = 1 OR kb.id IN (SELECT kb_id FROM kb_permissions WHERE user_id = "); + qb.push_bind(user_id); + qb.push(")"); + qb.push(")"); +} + +/// 追加文件访问过滤条件。 +/// 生成 `(f.user_id = ? OR f.is_public = 1)`；调用方需自行在前面加 `AND `。 +pub fn push_file_access_filter<'a>(qb: &mut QueryBuilder<'a, Sqlite>, user_id: &'a str, alias: Option<&'a str>) { + let col_prefix = alias.map(|a| format!("{}.", a)).unwrap_or_default(); + qb.push("("); + qb.push(format!("{}user_id", col_prefix)); + qb.push(" = "); + qb.push_bind(user_id); + qb.push(format!(" OR {}is_public = 1", col_prefix)); + qb.push(")"); +} + +/// 收集某个知识库的所有后代知识库 ID（包含自身）。 +/// 若 `exclude_storage` 为 true，则结果中过滤掉 `kb_type = 'storage'` 的节点。 +pub async fn collect_kb_descendant_ids( + pool: &SqlitePool, root_kb_id: i64, exclude_storage: bool, +) -> Result, sqlx::Error> { + let rows: Vec<(i64,)> = if exclude_storage { + sqlx::query_as( + r#" + WITH RECURSIVE descendants AS ( + SELECT id, kb_type FROM knowledge_bases WHERE id = ? + UNION ALL + SELECT kb.id, kb.kb_type + FROM knowledge_bases kb + INNER JOIN descendants d ON kb.parent_id = d.id + ) + SELECT id FROM descendants WHERE kb_type != ?; + "#, + ) + .bind(root_kb_id) + .bind(knowledge_base::KB_TYPE_STORAGE) + .fetch_all(pool) + .await? + } else { + sqlx::query_as( + r#" + WITH RECURSIVE descendants AS ( + SELECT id FROM knowledge_bases WHERE id = ? + UNION ALL + SELECT kb.id + FROM knowledge_bases kb + INNER JOIN descendants d ON kb.parent_id = d.id + ) + SELECT id FROM descendants; + "#, + ) + .bind(root_kb_id) + .fetch_all(pool) + .await? + }; + Ok(rows.into_iter().map(|(id,)| id).collect()) +} diff --git a/src/api/error.rs b/src/api/error.rs new file mode 100644 index 0000000..5dc2682 --- /dev/null +++ b/src/api/error.rs @@ -0,0 +1,125 @@ +use std::{fmt, num::ParseIntError, result}; + +use axum::{ + Json, http::StatusCode, response::{IntoResponse, Response} +}; +use serde::Serialize; +use sqlx; + +/// Unified API error type for handlers to return. +/// +/// Example: +/// - `Result` is aliased as `ApiResult` below. +/// - Convert database errors with `ApiError::from(sqlx::Error)`. +#[derive(Debug)] +pub enum ApiError { + BadRequest(String), + // Unauthorized(String), + Forbidden(String), + NotFound(String), + Database(sqlx::Error), + System(std::io::Error), + Internal(String), +} + +impl fmt::Display for ApiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ApiError::BadRequest(msg) => write!(f, "BadRequest: {}", msg), + // ApiError::Unauthorized(msg) => write!(f, "Unauthorized: {}", msg), + ApiError::Forbidden(msg) => write!(f, "Forbidden: {}", msg), + ApiError::NotFound(msg) => write!(f, "NotFound: {}", msg), + ApiError::Database(e) => write!(f, "Database error: {}", e), + ApiError::System(e) => write!(f, "System error: {}", e), + ApiError::Internal(msg) => write!(f, "Internal error: {}", msg), + } + } +} + +impl std::error::Error for ApiError {} + +impl From for ApiError { + fn from(e: sqlx::Error) -> Self { + ApiError::Database(e) + } +} + +impl From for ApiError { + fn from(e: std::io::Error) -> Self { + ApiError::System(e) + } +} + +impl From for ApiError { + fn from(e: axum::extract::multipart::MultipartError) -> Self { + ApiError::Internal(format!("Multipart error: {}", e)) + } +} + +impl From for ApiError { + fn from(e: ParseIntError) -> Self { + ApiError::BadRequest(format!("Int error: {}", e)) + } +} + +impl From for ApiError { + fn from(e: anyhow::Error) -> Self { + ApiError::Internal(format!("Internal error: {}", e)) + } +} + +impl From for ApiError { + fn from(e: serde_json::Error) -> Self { + ApiError::BadRequest(format!("JSON error: {}", e)) + } +} + +/// Standardized JSON body returned for errors. +#[derive(Serialize)] +struct ErrorBody { + code: u16, + error: String, + // optional: you can add `details: Option` if you want more info +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + // Map each error variant to an HTTP status code and message. + let (status, message) = match &self { + ApiError::BadRequest(msg) => (StatusCode::BAD_REQUEST, msg.clone()), + // ApiError::Unauthorized(msg) => (StatusCode::UNAUTHORIZED, msg.clone()), + ApiError::Forbidden(msg) => (StatusCode::FORBIDDEN, msg.clone()), + ApiError::NotFound(msg) => (StatusCode::NOT_FOUND, msg.clone()), + ApiError::Database(e) => { + // Log detailed error server-side but avoid leaking internals to clients. + log::error!("Database error: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, "Internal server error".into()) + } + ApiError::System(e) => { + // Log detailed error server-side but avoid leaking internals to clients. + log::error!("System error: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, "Internal server error".into()) + } + ApiError::Internal(msg) => { + log::error!("Internal error: {}", msg); + (StatusCode::INTERNAL_SERVER_ERROR, msg.clone()) + } + }; + + let body = ErrorBody { code: status.as_u16(), error: message }; + + (status, Json(body)).into_response() + } +} + +/// Convenience alias for handlers' return type. +pub type ApiResult = result::Result; + +impl ApiError { + /// Helper to create a generic internal error (also logs). + pub fn internal>(msg: M) -> Self { + let msg = msg.into(); + log::error!("Internal error created: {}", msg); + ApiError::Internal(msg) + } +} diff --git a/src/api/file.rs b/src/api/file.rs new file mode 100644 index 0000000..7166e79 --- /dev/null +++ b/src/api/file.rs @@ -0,0 +1,3245 @@ +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + io::{Seek, Write}, + path::Component, + sync::{Arc, OnceLock}, + time::Instant, +}; + +use anyhow::Result as AnyResult; +use axum::{ + Extension, + body::Body, + extract::{Multipart, Path, Query, State}, + http::{StatusCode, header}, + response::Json, +}; +use bytes::Bytes; +use log::{debug, error, info, warn}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use sqlx::{QueryBuilder, Row, Sqlite, SqlitePool}; +use tempfile::NamedTempFile; +use tokio::{ + fs, + io::AsyncWriteExt as _, + spawn, + sync::{OwnedSemaphorePermit, Semaphore}, +}; +use utoipa::{IntoParams, ToSchema}; + +use crate::{ + AuthUser, + api::{ + common, + error::{ApiError, ApiResult}, + }, + archive::{self, ArchiveEntry, ExtractResult}, + config, pdf_highlight, processor, + search::SearchEngine, +}; + +/// 以流式方式打开文件，返回 (字节数, Body)。 +/// +/// 避免将整个文件读入内存（大文件下载会按 chunk 流式发送），同时返回文件大小用于 Content-Length。 +async fn open_file_stream(path: &std::path::Path) -> Result<(u64, Body), ApiError> { + let file = fs::File::open(path).await?; + let len = file.metadata().await?.len(); + let stream = tokio_util::io::ReaderStream::new(file); + Ok((len, Body::from_stream(stream))) +} + +/// 将已位于内存中的字节写入临时文件并流式返回，避免把大 body 直接交给 axum 造成额外拷贝。 +/// +/// `NamedTempFile::into_file()` 会立即删除磁盘上的路径，只保留文件描述符；当返回的 `Body` +/// 消费完毕、文件描述符关闭后，磁盘空间会自动释放。 +async fn body_from_bytes(bytes: Vec) -> Result<(u64, Body), ApiError> { + let (std_file, len) = tokio::task::spawn_blocking(move || { + let mut temp = NamedTempFile::new()?; + temp.write_all(&bytes)?; + temp.flush()?; + let mut file = temp.into_file(); + file.seek(std::io::SeekFrom::Start(0))?; + let len = file.metadata()?.len(); + anyhow::Ok((file, len)) + }) + .await + .map_err(|e| ApiError::Internal(format!("temp file task panicked: {}", e)))? + .map_err(|e| ApiError::Internal(format!("failed to write temp file: {}", e)))?; + + let file = fs::File::from_std(std_file); + let stream = tokio_util::io::ReaderStream::new(file); + Ok((len, Body::from_stream(stream))) +} + +/// 将已拥有所有权的 `std::fs::File`（通常来自 `NamedTempFile::into_file()`）转换为流式 Body。 +/// +/// 调用前需确保文件偏移在起始位置；返回 `(字节数, Body)`。 +fn stream_from_std_file(mut file: std::fs::File) -> Result<(u64, Body), ApiError> { + use std::io::{Seek, SeekFrom}; + file.seek(SeekFrom::Start(0))?; + let len = file.metadata()?.len(); + let file = fs::File::from_std(file); + let stream = tokio_util::io::ReaderStream::new(file); + Ok((len, Body::from_stream(stream))) +} + +/// `files` 表查询列清单，其中 `content` 以 NULL 占位。 +/// +/// 列表类查询不需要全文 content，用此清单避免把可能很大的文本从 SQLite/文件系统读进内存。 +pub(crate) const FILE_COLS_NO_CONTENT: &str = "id, user_id, user_name, hash, filename, path, size, NULL as content, \ + tags, status, log, slice_type, kb_id, is_public, meta, summary, created_at, updated_at, artifact_id"; + +/// Excel 单 sheet 数据 +#[derive(Debug, Serialize, ToSchema)] +pub struct ExcelSheetData { + pub name: String, + pub headers: Vec, + pub rows: Vec>, + pub truncated: bool, +} + +/// Excel 文件数据 +#[derive(Debug, Serialize, ToSchema)] +pub struct ExcelData { + pub filename: String, + pub sheets: Vec, +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct ExcelDataQuery { + pub sheet_name: Option, + pub row_num: Option, +} + +/// 解压压缩文件请求 +#[derive(Debug, Deserialize, ToSchema)] +pub struct ArchiveExtractReq { + /// 解压密码（加密压缩文件时需要） + pub password: Option, +} + +/// 下载压缩包内文件请求参数 +#[derive(Debug, Deserialize, IntoParams)] +pub struct ArchiveDownloadQuery { + /// 压缩包内文件路径 + pub path: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug, sqlx::FromRow, ToSchema)] +pub struct File { + pub id: i64, + pub user_id: String, + pub user_name: String, + pub hash: String, + pub filename: String, + pub path: String, + pub size: i64, + /// 文件解析后的完整文本。 + /// + /// 该字段已从 SQLite 迁移到 `contents/` 目录下的独立文件。`#[sqlx(default)]` + /// 保证在新 schema（无 `content` 列）下 `SELECT *` 仍能正常映射。 + #[sqlx(default)] + pub content: Option, + pub tags: String, + pub status: i32, + pub log: String, + pub slice_type: String, + pub kb_id: Option, + pub is_public: bool, + pub meta: Option, + pub summary: Option, + pub created_at: i64, + pub updated_at: i64, + /// 共享解析产物内部引用，不作为对外 API 字段返回。 + #[serde(skip)] + #[sqlx(default)] + pub artifact_id: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, Default, ToSchema)] +pub struct FileStatusBreakdown { + /// 所有满足条件的文件总数 + pub total: i64, + /// status = 0，待处理 + pub pending: i64, + /// status = 2，处理中 + pub processing: i64, + /// status = 1，已完成 + pub completed: i64, + /// status = 3，不解析/跳过 + pub skipped: i64, + /// status = -1，处理失败 + pub failed: i64, + /// 其他未知状态 + pub unknown: i64, + /// 当前正在处理的文件（按更新时间倒序，最多10条） + pub processing_files: Vec, + /// 最近处理失败的文件（按更新时间倒序，最多10条） + pub failed_files: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug, sqlx::FromRow, ToSchema)] +pub struct FileStatusPreview { + pub id: i64, + pub filename: String, + pub kb_id: Option, + pub kb_name: Option, + pub kb_path: Option, + pub updated_at: i64, +} + +impl FileStatusBreakdown { + fn add(&mut self, status: i32, count: i64) { + self.total += count; + match status { + 0 => self.pending += count, + 1 => self.completed += count, + 2 => self.processing += count, + 3 => self.skipped += count, + -1 => self.failed += count, + _ => self.unknown += count, + } + } +} + +#[derive(Clone, Copy)] +enum FileStatsScope { + Global { include_unassigned: bool }, + KnowledgeBase { kb_id: i64, include_descendants: bool }, + UnassignedOnly, +} + +fn ensure_file_readable(file: &File, auth_user: &AuthUser) -> ApiResult<()> { + if auth_user.is_admin() || file.is_public || file.user_id == auth_user.user_id { + Ok(()) + } else { + Err(ApiError::NotFound("File not found or permission denied".to_string())) + } +} + +async fn ensure_file_writable(pool: &SqlitePool, file: &File, auth_user: &AuthUser) -> ApiResult<()> { + let is_admin = auth_user.is_admin(); + match file.kb_id { + Some(kb_id) => { + let perm = super::knowledge_base::get_kb_permission(pool, kb_id, &auth_user.user_id, is_admin).await; + if !super::knowledge_base::meets_requirement(perm.as_deref(), "editor") { + return Err(ApiError::Forbidden("Permission denied. Requires editor or admin.".to_string())); + } + } + None => { + if !is_admin && file.user_id != auth_user.user_id { + return Err(ApiError::Forbidden("Permission denied.".to_string())); + } + } + } + Ok(()) +} + +const PARSE_ASSETS_META_KEY: &str = "_htknow_parse_assets"; +const CUSTOM_IMAGES_META_KEY: &str = "custom_images"; +const CUSTOM_IMAGES_META_VERSION: i64 = 1; + +async fn query_file_status_breakdown( + pool: &SqlitePool, scope: FileStatsScope, user_id: &str, is_admin: bool, +) -> AnyResult { + let mut qb = QueryBuilder::::new(""); + + if let FileStatsScope::KnowledgeBase { kb_id, include_descendants: true } = scope { + qb.push("WITH RECURSIVE descendants AS (SELECT id FROM knowledge_bases WHERE id = "); + qb.push_bind(kb_id); + if !is_admin { + qb.push(" AND (user_id = ").push_bind(user_id).push(" OR is_public = 1)"); + } + qb.push(" UNION ALL SELECT kb.id FROM knowledge_bases kb JOIN descendants d ON kb.parent_id = d.id"); + if !is_admin { + qb.push(" WHERE kb.user_id = ").push_bind(user_id).push(" OR kb.is_public = 1"); + } + qb.push(") "); + } + + qb.push("SELECT COALESCE(f.status, -99) AS status, COUNT(*) AS cnt FROM files f WHERE 1=1"); + + match scope { + FileStatsScope::Global { include_unassigned } => { + if !include_unassigned { + qb.push(" AND f.kb_id IS NOT NULL"); + } + } + FileStatsScope::KnowledgeBase { kb_id, include_descendants } => { + if include_descendants { + qb.push(" AND f.kb_id IN (SELECT id FROM descendants)"); + } else { + qb.push(" AND f.kb_id = ").push_bind(kb_id); + } + } + FileStatsScope::UnassignedOnly => { + qb.push(" AND f.kb_id IS NULL"); + } + } + + if !is_admin { + qb.push(" AND (f.user_id = ").push_bind(user_id).push(" OR f.is_public = 1)"); + } + + qb.push(" GROUP BY f.status"); + + let rows = qb.build().fetch_all(pool).await?; + let mut breakdown = FileStatusBreakdown::default(); + for row in rows { + let status: i32 = row.get("status"); + let cnt: i64 = row.get("cnt"); + breakdown.add(status, cnt); + } + breakdown.processing_files = fetch_status_files_for_scope(pool, scope, user_id, is_admin, 2).await?; + breakdown.failed_files = fetch_status_files_for_scope(pool, scope, user_id, is_admin, -1).await?; + Ok(breakdown) +} + +async fn fetch_status_files_for_scope( + pool: &SqlitePool, scope: FileStatsScope, user_id: &str, is_admin: bool, status: i32, +) -> AnyResult> { + let mut qb = QueryBuilder::::new( + "WITH RECURSIVE kb_paths(id, path) AS ( \ + SELECT id, name FROM knowledge_bases WHERE parent_id IS NULL \ + UNION ALL \ + SELECT kb.id, kb_paths.path || ' / ' || kb.name \ + FROM knowledge_bases kb JOIN kb_paths ON kb.parent_id = kb_paths.id \ + ) \ + SELECT f.id, f.filename, f.kb_id, kb.name AS kb_name, kb_paths.path AS kb_path, f.updated_at \ + FROM files f \ + LEFT JOIN knowledge_bases kb ON kb.id = f.kb_id \ + LEFT JOIN kb_paths ON kb_paths.id = f.kb_id \ + WHERE f.status = ", + ); + qb.push_bind(status); + + match scope { + FileStatsScope::Global { include_unassigned } => { + if !include_unassigned { + qb.push(" AND f.kb_id IS NOT NULL"); + } + } + FileStatsScope::KnowledgeBase { kb_id, include_descendants } => { + if include_descendants { + qb.push(" AND f.kb_id IN (WITH RECURSIVE descendants AS (SELECT id FROM knowledge_bases WHERE id = "); + qb.push_bind(kb_id); + if !is_admin { + qb.push(" AND (user_id = ").push_bind(user_id).push(" OR is_public = 1)"); + } + qb.push( + " UNION ALL SELECT kb.id FROM knowledge_bases kb JOIN descendants d ON kb.parent_id = d.id \ + ) SELECT id FROM descendants)", + ); + } else { + qb.push(" AND f.kb_id = ").push_bind(kb_id); + } + } + FileStatsScope::UnassignedOnly => { + qb.push(" AND f.kb_id IS NULL"); + } + } + + if !is_admin { + qb.push(" AND (f.user_id = ").push_bind(user_id).push(" OR f.is_public = 1)"); + } + + qb.push(" ORDER BY f.updated_at DESC LIMIT 10"); + + let rows = qb.build_query_as::().fetch_all(pool).await?; + Ok(rows) +} + +pub async fn get_file_status_breakdown_for_kb( + pool: &SqlitePool, kb_id: i64, include_descendants: bool, user_id: &str, is_admin: bool, +) -> AnyResult { + query_file_status_breakdown(pool, FileStatsScope::KnowledgeBase { kb_id, include_descendants }, user_id, is_admin) + .await +} + +pub async fn get_file_status_breakdown_for_all( + pool: &SqlitePool, include_unassigned: bool, user_id: &str, is_admin: bool, +) -> AnyResult { + query_file_status_breakdown(pool, FileStatsScope::Global { include_unassigned }, user_id, is_admin).await +} + +pub async fn get_file_status_breakdown_for_unassigned( + pool: &SqlitePool, user_id: &str, is_admin: bool, +) -> AnyResult { + query_file_status_breakdown(pool, FileStatsScope::UnassignedOnly, user_id, is_admin).await +} + +static BACKGROUND_REUSE_SEMAPHORE: OnceLock> = OnceLock::new(); + +fn background_reuse_semaphore() -> Arc { + BACKGROUND_REUSE_SEMAPHORE + .get_or_init(|| { + let cfg = config::get(); + let limit = cfg.server.process_concurrency.max(1).min(cfg.database.max_connections as usize).max(1); + Arc::new(Semaphore::new(limit)) + }) + .clone() +} + +async fn acquire_background_reuse_permit(semaphore: Arc, file_id: i64) -> Option { + match semaphore.acquire_owned().await { + Ok(permit) => Some(permit), + Err(e) => { + warn!("Background reuse semaphore closed for file {}: {}", file_id, e); + None + } + } +} + +fn map_search_engine_error(err: anyhow::Error) -> ApiError { + let msg = err.to_string(); + if msg.contains("LockBusy") || msg.contains("Failed to acquire index lock") { + return ApiError::BadRequest("Search index is busy, please retry shortly.".to_string()); + } + ApiError::Internal(format!("Internal error: {}", msg)) +} + +/// 上传文件（支持单个或多个文件） +/// +/// form-data 参数： +/// - file: 文件内容（可多次出现） +/// - slice_type: 分片类型 +/// - kb_id: 知识库 ID +/// - tags: JSON 数组字符串，例如 ["tag1","tag2"] +/// - is_public: true/false 或 1/0 +/// - meta: 元数据字符串 +/// - immediate_parse: true/1 时后台解析 +/// - sync: true/1 时等待解析完成并返回最新的文件记录 +#[utoipa::path( + post, + path = "/api/v1/knowledge/files/", + operation_id = "file_upload", + tag = "file", + request_body( + content_type = "multipart/form-data", + description = "form-data: file, slice_type, kb_id, tags(JSON array string), is_public, meta, immediate_parse, sync" + ), + responses( + (status = 200, description = "文件上传成功", body = Vec), + (status = 400, description = "请求参数错误"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn upload( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, mut multipart: Multipart, +) -> ApiResult>> { + debug!("Starting file upload for user: {}", auth_user.user_id); + let cfg = config::get(); + let dir = &cfg.storage.files_path; + tokio::fs::create_dir_all(dir).await?; + let reuse_duplicates = cfg.server.reuse_duplicate_files; + + let mut files_data: Vec<(String, String, String, i64)> = Vec::new(); + let mut slice_type = String::new(); + let mut kb_id = None; + let mut is_public = 0i32; + let mut tags: Vec = Vec::new(); + let mut meta: Option = None; + let mut immediate_parse = false; + let mut sync = false; + + loop { + match multipart.next_field().await { + Ok(Some(mut field)) => { + let field_name = field.name().map(|s| s.to_string()); + debug!("Processing field: {:?}", field_name); + + match field_name.as_deref() { + Some("file") => { + let filename = field.file_name().unwrap_or("unknown").to_string(); + debug!("Uploading file: {}", filename); + let tempname = uuid::Uuid::new_v4().to_string(); + let filepath = format!("{}/{}", dir, tempname); + let mut file = tokio::fs::File::create(filepath.clone()).await?; + + // 把 SHA-256 计算卸载到阻塞线程，避免在异步循环里做 CPU 密集哈希。 + let (hash_tx, hash_rx) = std::sync::mpsc::channel::(); + let hash_handle = tokio::task::spawn_blocking(move || { + let mut hasher = Sha256::new(); + while let Ok(chunk) = hash_rx.recv() { + hasher.update(&chunk); + } + hex::encode(hasher.finalize()) + }); + + let mut size: i64 = 0; + while let Some(chunk) = field.chunk().await? { + size += chunk.len() as i64; + file.write_all(&chunk).await?; + if hash_tx.send(chunk).is_err() { + break; + } + } + drop(hash_tx); + let hash = + hash_handle.await.map_err(|e| ApiError::Internal(format!("hash task panicked: {}", e)))?; + + debug!("File saved to: {}", filepath); + files_data.push((hash, filename, filepath, size)); + } + Some("slice_type") => { + slice_type = field.text().await?; + debug!("Slice type: {}", slice_type); + } + Some("kb_id") => { + let kb_id_text = field.text().await?; + debug!("KB ID text: {}", kb_id_text); + kb_id = Some(kb_id_text.parse::()?); + } + Some("tags") => { + let tags_text = field.text().await?; + debug!("Tags text: {}", tags_text); + if !tags_text.is_empty() { + tags = serde_json::from_str(&tags_text).unwrap_or_default(); + } + } + Some("is_public") => { + let is_public_text = field.text().await?; + debug!("Is public text: {}", is_public_text); + is_public = match is_public_text.as_str() { + "true" | "1" => 1, + "false" | "0" => 0, + _ => is_public_text.parse::().map_or(0, |b| if b { 1 } else { 0 }), + }; + } + Some("meta") => { + let meta_text = field.text().await?; + debug!("Meta text: {}", meta_text); + if !meta_text.is_empty() { + meta = Some(meta_text); + } + } + Some("immediate_parse") => { + // 立刻开始解析 + let parse_text = field.text().await?; + debug!("Immediate parse text: {}", parse_text); + immediate_parse = parse_text == "true" || parse_text == "1"; + } + Some("sync") => { + // 等待解析结果 + let sync_text = field.text().await?; + debug!("Sync parse text: {}", sync_text); + sync = sync_text == "true" || sync_text == "1"; + } + _ => { + debug!("Skipping unknown field: {:?}", field_name); + } + } + } + Ok(None) => { + debug!("No more fields"); + break; + } + Err(e) => { + log::error!("Error reading multipart field: {}", e); + return Err(ApiError::Internal(format!("Multipart error: {}", e))); + } + } + } + + debug!("File upload completed. Files count: {}", files_data.len()); + if files_data.is_empty() { + return Err(ApiError::BadRequest("file is required".to_string())); + } + + let mut kb_type: Option = None; + if let Some(kb_id_value) = kb_id { + let kb: Option<(String, Option)> = + sqlx::query_as("SELECT user_id, kb_type FROM knowledge_bases WHERE id = ?") + .bind(kb_id_value) + .fetch_optional(&pool) + .await?; + let (_, kb_type_value) = kb.ok_or_else(|| ApiError::NotFound("Knowledge base not found".to_string()))?; + kb_type = kb_type_value; + + // Check KB permission: need editor or admin to upload + let perm = + super::knowledge_base::get_kb_permission(&pool, kb_id_value, &auth_user.user_id, auth_user.is_admin()) + .await; + if !super::knowledge_base::meets_requirement(perm.as_deref(), "editor") { + return Err(ApiError::Forbidden( + "Permission denied. Requires editor or admin to upload files.".to_string(), + )); + } + } + + let is_storage_kb = matches!(kb_type.as_deref(), Some("storage")); + let tags_json = serde_json::to_string(&tags)?; + let status = if is_storage_kb { 3 } else { 0 }; + let log_message = if is_storage_kb { "Storage mode: not parsed" } else { "" }; + + let mut uploaded_files: Vec = Vec::new(); + let mut uploaded_file_ids: Vec = Vec::new(); + let mut parse_file_ids: Vec = Vec::new(); + + for (hash, filename, filepath, size) in files_data { + let sql = "INSERT INTO files (user_id, user_name, hash, filename, path, size, slice_type, kb_id, is_public, tags, status, log, meta) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + let id = sqlx::query(sql) + .bind(&auth_user.user_id) + .bind(&auth_user.user_name) + .bind(hash) + .bind(filename) + .bind(filepath) + .bind(size) + .bind(&slice_type) + .bind(kb_id) + .bind(is_public) + .bind(&tags_json) + .bind(status) + .bind(log_message) + .bind(meta.clone()) + .execute(&pool) + .await? + .last_insert_rowid(); + + let mut file: File = sqlx::query_as(&format!("SELECT {FILE_COLS_NO_CONTENT} FROM files WHERE id = ?")) + .bind(id) + .fetch_one(&pool) + .await?; + if file.status == 3 { + uploaded_files.push(file); + uploaded_file_ids.push(id); + continue; + } + let mut reused_now = false; + + if reuse_duplicates { + if sync { + match processor::try_reuse_file_with_file(pool.clone(), search_engine.clone(), file.clone()).await { + Ok(true) => { + reused_now = true; + file = sqlx::query_as(&format!("SELECT {FILE_COLS_NO_CONTENT} FROM files WHERE id = ?")) + .bind(id) + .fetch_one(&pool) + .await?; + } + Ok(false) => {} + Err(e) => { + warn!("Immediate reuse failed for file {}: {}", id, e); + } + } + } else if !immediate_parse { + let pool_clone = pool.clone(); + let search_engine_clone = search_engine.clone(); + let file_clone = file.clone(); + let file_id = id; + let semaphore = background_reuse_semaphore(); + spawn(async move { + let Some(_permit) = acquire_background_reuse_permit(semaphore, file_id).await else { + return; + }; + + let reuse_result = processor::try_reuse_file_with_file( + pool_clone.clone(), + search_engine_clone.clone(), + file_clone, + ) + .await; + + match reuse_result { + Ok(true) => {} + Ok(false) => { + let reuse_failed = + match sqlx::query_scalar::<_, String>("SELECT log FROM files WHERE id = ?") + .bind(file_id) + .fetch_optional(&pool_clone) + .await + { + Ok(Some(log)) => log.starts_with("Reuse failed:"), + Ok(None) => false, + Err(e) => { + warn!("Failed to read reuse log for file {}: {}", file_id, e); + false + } + }; + if reuse_failed { + warn!("Background reuse failed for file {}, falling back to normal parsing", file_id); + if let Err(parse_err) = + processor::process_file_immediate(pool_clone, search_engine_clone, file_id).await + { + log::error!( + "Fallback parse failed for file {} after reuse failure: {}", + file_id, + parse_err + ); + } + } + } + Err(e) => { + log::error!("Background reuse failed for file {}: {}", file_id, e); + if let Err(parse_err) = + processor::process_file_immediate(pool_clone, search_engine_clone, file_id).await + { + log::error!( + "Fallback parse failed for file {} after reuse error: {}", + file_id, + parse_err + ); + } + } + } + }); + } + } + + if (immediate_parse || sync) && !reused_now { + parse_file_ids.push(id); + } + + uploaded_files.push(file); + uploaded_file_ids.push(id); + } + + if immediate_parse || sync { + if sync { + let reuse_already_tried = reuse_duplicates; + let concurrency = config::get().server.process_concurrency.max(1); + let semaphore = Arc::new(Semaphore::new(concurrency)); + let mut handles = Vec::with_capacity(parse_file_ids.len()); + for file_id in parse_file_ids { + let pool_c = pool.clone(); + let se_c = search_engine.clone(); + let sem_c = semaphore.clone(); + let reuse_tried = reuse_already_tried; + handles.push(spawn(async move { + let _permit = sem_c.acquire().await; + if reuse_tried { + processor::process_file_immediate_skip_reuse(pool_c, se_c, file_id).await + } else { + processor::process_file_immediate(pool_c, se_c, file_id).await + } + })); + } + for handle in handles { + handle.await.map_err(|e| ApiError::internal(format!("parse join error: {}", e)))??; + } + uploaded_files.clear(); + if !uploaded_file_ids.is_empty() { + let mut qb = QueryBuilder::::new("SELECT * FROM files WHERE id IN ("); + let mut sep = qb.separated(", "); + for id in &uploaded_file_ids { + sep.push_bind(*id); + } + qb.push(")"); + let rows: Vec = qb.build_query_as().fetch_all(&pool).await?; + let order: HashMap = uploaded_file_ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + let mut sorted = rows; + sorted.sort_by_key(|f| order.get(&f.id).copied().unwrap_or(usize::MAX)); + uploaded_files = sorted; + } + } else { + let pool = pool.clone(); + let search_engine = search_engine.clone(); + spawn(async move { + for file_id in parse_file_ids { + if let Err(e) = + processor::process_file_immediate(pool.clone(), search_engine.clone(), file_id).await + { + log::error!("Failed to parse file {}: {}", file_id, e); + } + } + }); + } + } + + Ok(Json(uploaded_files)) +} + +/// 获取文件详情 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/{id}", + operation_id = "file_get", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID") + ), + responses( + (status = 200, description = "成功返回文件详情", body = File), + (status = 404, description = "文件不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn get( + State(pool): State, Path(id): Path, Extension(auth_user): Extension, +) -> ApiResult> { + let query = "SELECT * FROM files WHERE id = ?"; + let mut file: File = sqlx::query_as(query).bind(id).fetch_one(&pool).await?; + + ensure_file_readable(&file, &auth_user)?; + + file.content = crate::file_content::read(id).await?; + Ok(Json(file)) +} + +#[derive(Deserialize, ToSchema)] +pub struct UpdateFileReq { + pub slice_type: Option, + pub filename: Option, + pub tags: Option>, + pub is_public: Option, + pub meta: Option, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct MoveFileReq { + /// 目标知识库 ID。传 null 表示移出知识库（未分配）。 + pub target_kb_id: Option, +} + +const MAX_BATCH_DELETE_IDS: usize = 200; +const SQLITE_DELETE_CHUNK_SIZE: usize = 900; + +#[derive(Debug, Deserialize, ToSchema)] +pub struct BatchDeleteFilesReq { + /// 待删除文件 ID 列表（会自动去重） + pub ids: Vec, + /// 严格模式：只要有任意文件不可删，则整批拒绝 + #[serde(default)] + pub strict: bool, + /// 是否允许删除处理中(status=2)的文件，默认 false + #[serde(default)] + pub allow_processing: bool, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct BatchDeleteSkippedItem { + pub id: i64, + pub reason: String, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct BatchDeleteCleanupFailedItem { + pub id: i64, + /// 清理阶段：file/pdf/search + pub stage: String, + pub error: String, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct BatchDeleteFilesResp { + pub requested: i64, + pub accepted: i64, + pub deleted: i64, + pub deleted_ids: Vec, + pub skipped: Vec, + pub cleanup_failed: Vec, +} + +const fn default_true_flag() -> bool { + true +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct ReparseFailedFilesReq { + /// 指定知识库 ID；不传表示全局范围 + pub kb_id: Option, + /// 指定知识库时是否包含子知识库，默认 true + #[serde(default = "default_true_flag")] + pub include_descendants: bool, + /// 全局范围时是否包含未分配文件，默认 true + #[serde(default = "default_true_flag")] + pub include_unassigned: bool, + /// 是否只处理未分配知识库中的失败文件，默认 false + #[serde(default)] + pub unassigned_only: bool, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ReparseFailedFilesResp { + pub file_count: i64, +} + +/// 更新文件 +#[utoipa::path( + put, + path = "/api/v1/knowledge/files/{id}", + operation_id = "file_update", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID") + ), + request_body = UpdateFileReq, + responses( + (status = 200, description = "成功更新文件", body = File), + (status = 400, description = "请求参数错误"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn update( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, Path(id): Path, Json(req): Json, +) -> ApiResult> { + let file: File = sqlx::query_as::<_, File>("SELECT * FROM files WHERE id = ?") + .bind(id) + .fetch_optional(&pool) + .await? + .ok_or_else(|| ApiError::NotFound("File not found".to_string()))?; + ensure_file_writable(&pool, &file, &auth_user).await?; + + let update_is_public = req.is_public.is_some(); + let mut has_updates = false; + let mut reset_parse_data = false; + let mut image_paths_to_remove = Vec::new(); + debug!("update_is_public: {}", update_is_public); + + let mut qb = QueryBuilder::::new("UPDATE files SET "); + let mut separated = qb.separated(", "); + + if let Some(slice_type) = req.slice_type.as_deref() { + let kb_type: Option = + sqlx::query_scalar("SELECT kb_type FROM knowledge_bases WHERE id = (SELECT kb_id FROM files WHERE id = ?)") + .bind(id) + .fetch_optional(&pool) + .await?; + if matches!(kb_type.as_deref(), Some("storage")) { + return Err(ApiError::BadRequest("Storage knowledge base files do not support parsing.".to_string())); + } + + image_paths_to_remove = collect_image_paths_for_files(&pool, &[id]).await?; + reset_parse_data = true; + + separated.push("slice_type = ").push_bind_unseparated(slice_type); + separated.push("status = ").push_bind_unseparated(0); + separated.push("log = ").push_bind_unseparated(""); + has_updates = true; + } + + if let Some(filename) = req.filename.as_deref() { + separated.push("filename = ").push_bind_unseparated(filename); + has_updates = true; + } + + if let Some(tags) = req.tags.as_ref() { + let tags_json = serde_json::to_string(tags)?; + debug!("tags_json: {}", tags_json); + separated.push("tags = ").push_bind_unseparated(tags_json); + has_updates = true; + } + + if let Some(is_public) = req.is_public { + let is_public = if is_public { 1 } else { 0 }; + separated.push("is_public = ").push_bind_unseparated(is_public); + has_updates = true; + } + + if let Some(meta) = req.meta { + let meta_json = serde_json::to_string(&meta)?; + debug!("meta_json: {}", meta_json); + separated.push("meta = ").push_bind_unseparated(meta_json); + has_updates = true; + } + + if !has_updates { + return Err(ApiError::BadRequest("No fields to update".to_string())); + } + + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64; + separated.push("updated_at = ").push_bind_unseparated(now); + qb.push(" WHERE id = "); + qb.push_bind(id); + if reset_parse_data { + let mut tx = pool.begin().await?; + clear_file_parse_rows_in_tx(&mut tx, id).await?; + qb.build().execute(&mut *tx).await?; + tx.commit().await?; + + if let Err(e) = search_engine.delete(Some(id), None).await { + warn!("Failed to delete search index for updated file {}: {}", id, e); + } + remove_image_files(image_paths_to_remove).await; + remove_converted_pdfs(&[id]).await; + if let Err(e) = crate::file_content::delete(id).await { + warn!("Failed to delete content file for updated file {}: {}", id, e); + } + if let Err(e) = crate::pdf_content::delete(id).await { + warn!("Failed to delete PDF content file for updated file {}: {}", id, e); + } + if let Err(e) = crate::slice_content::delete(id).await { + warn!("Failed to delete slice content file for updated file {}: {}", id, e); + } + } else { + qb.build().execute(&pool).await?; + } + + let file = sqlx::query_as("SELECT * FROM files WHERE id = ?").bind(id).fetch_one(&pool).await?; + Ok(Json(file)) +} + +/// 移动文件到另一个知识库 +#[utoipa::path( + put, + path = "/api/v1/knowledge/files/{id}/move", + operation_id = "file_move", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID") + ), + request_body = MoveFileReq, + responses( + (status = 200, description = "成功移动文件", body = File), + (status = 400, description = "请求参数错误"), + (status = 401, description = "未授权"), + (status = 404, description = "文件或知识库不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn move_to_kb( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, Path(id): Path, Json(req): Json, +) -> ApiResult> { + if let Some(target_kb_id) = req.target_kb_id + && target_kb_id <= 0 + { + return Err(ApiError::BadRequest("Invalid target_kb_id".to_string())); + } + + let file: File = sqlx::query_as::<_, File>("SELECT * FROM files WHERE id = ?") + .bind(id) + .fetch_optional(&pool) + .await? + .ok_or_else(|| ApiError::NotFound("File not found".to_string()))?; + + ensure_file_writable(&pool, &file, &auth_user).await?; + + if file.status == 2 { + return Err(ApiError::BadRequest("File is processing, cannot move now".to_string())); + } + + if file.kb_id == req.target_kb_id { + return Ok(Json(file)); + } + + // Check target KB permission + let target_kb_type: Option = match req.target_kb_id { + Some(target_kb_id) => { + common::ensure_kb_editor_or_admin(&pool, target_kb_id, &auth_user).await?; + let kb_type: Option = sqlx::query_scalar("SELECT kb_type FROM knowledge_bases WHERE id = ?") + .bind(target_kb_id) + .fetch_optional(&pool) + .await?; + Some(kb_type.ok_or_else(|| ApiError::NotFound("Knowledge base not found".to_string()))?) + } + None => None, + }; + + let (next_status, next_log) = + if matches!(target_kb_type.as_deref(), Some("storage")) { (3, "Storage mode: not parsed") } else { (0, "") }; + + let image_paths = collect_image_paths_for_files(&pool, &[id]).await?; + + let mut tx = pool.begin().await?; + let update_result = sqlx::query( + "UPDATE files SET kb_id = ?, status = ?, log = ?, updated_at = strftime('%s','now') WHERE id = ? AND status != 2 AND updated_at = ?", + ) + .bind(req.target_kb_id) + .bind(next_status) + .bind(next_log) + .bind(id) + .bind(file.updated_at) + .execute(&mut *tx) + .await?; + if update_result.rows_affected() == 0 { + return Err(ApiError::BadRequest("File state changed, please retry".to_string())); + } + clear_file_parse_rows_in_tx(&mut tx, id).await?; + tx.commit().await?; + + if let Err(e) = search_engine.delete(Some(id), None).await { + warn!("Failed to delete search index for moved file {}: {}", id, e); + } + + remove_image_files(image_paths).await; + let cfg = config::get(); + let pdf_path = std::path::Path::new(&cfg.storage.pdf_path).join(format!("{}.pdf", id)); + if let Err(e) = fs::remove_file(&pdf_path).await + && !matches!(e.kind(), std::io::ErrorKind::NotFound) + { + warn!("Failed to delete converted pdf {} after file move: {}", pdf_path.display(), e); + } + if let Err(e) = crate::file_content::delete(id).await { + warn!("Failed to delete content file after file move for file {}: {}", id, e); + } + if let Err(e) = crate::pdf_content::delete(id).await { + warn!("Failed to delete PDF content file after file move for file {}: {}", id, e); + } + if let Err(e) = crate::slice_content::delete(id).await { + warn!("Failed to delete slice content file after file move for file {}: {}", id, e); + } + + let moved: File = sqlx::query_as("SELECT * FROM files WHERE id = ?").bind(id).fetch_one(&pool).await?; + Ok(Json(moved)) +} + +/// 批量删除文件 +#[utoipa::path( + post, + path = "/api/v1/knowledge/files/batch-delete", + operation_id = "file_batch_delete", + tag = "file", + request_body = BatchDeleteFilesReq, + responses( + (status = 200, description = "批量删除完成（可能部分成功）", body = BatchDeleteFilesResp), + (status = 400, description = "请求参数错误"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn batch_delete( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, Json(req): Json, +) -> ApiResult> { + let result = + execute_batch_delete(&pool, &search_engine, &auth_user, req.ids, req.strict, req.allow_processing).await?; + Ok(Json(result)) +} + +/// 重新解析失败文件 +#[utoipa::path( + post, + path = "/api/v1/knowledge/files/reparse-failed", + operation_id = "file_reparse_failed", + tag = "file", + request_body = ReparseFailedFilesReq, + responses( + (status = 200, description = "已提交失败文件重新解析", body = ReparseFailedFilesResp), + (status = 400, description = "请求参数错误"), + (status = 401, description = "未授权"), + (status = 404, description = "知识库不存在或无权限") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn reparse_failed( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, Json(req): Json, +) -> ApiResult> { + let result = execute_reparse_failed(&pool, &search_engine, &auth_user, req).await?; + Ok(Json(result)) +} + +/// 删除文件 +#[utoipa::path( + delete, + path = "/api/v1/knowledge/files/{id}", + operation_id = "file_delete", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID") + ), + responses( + (status = 200, description = "成功删除文件"), + (status = 401, description = "未授权"), + (status = 404, description = "文件不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn delete( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, Path(id): Path, +) -> ApiResult<()> { + let result = execute_batch_delete(&pool, &search_engine, &auth_user, vec![id], false, true).await?; + if result.deleted == 0 { + return Err(ApiError::NotFound("File not found or permission denied".to_string())); + } + Ok(()) +} + +fn normalize_batch_delete_ids(ids: Vec) -> ApiResult> { + if ids.is_empty() { + return Err(ApiError::BadRequest("ids is required".to_string())); + } + if ids.len() > MAX_BATCH_DELETE_IDS { + return Err(ApiError::BadRequest(format!("Too many ids: max {}", MAX_BATCH_DELETE_IDS))); + } + + let mut seen = HashSet::new(); + let mut normalized = Vec::with_capacity(ids.len()); + for id in ids { + if id <= 0 { + return Err(ApiError::BadRequest(format!("Invalid file id: {}", id))); + } + if seen.insert(id) { + normalized.push(id); + } + } + Ok(normalized) +} + +async fn query_deletable_files(pool: &SqlitePool, ids: &[i64], auth_user: &AuthUser) -> Result, sqlx::Error> { + let is_admin = auth_user.is_admin(); + // Fetch all files by id first + let mut qb = QueryBuilder::::new(&format!("SELECT {} FROM files WHERE id IN (", FILE_COLS_NO_CONTENT)); + crate::db::push_i64_list(&mut qb, ids); + qb.push(")"); + let all_files: Vec = qb.build_query_as::().fetch_all(pool).await?; + + if is_admin { + return Ok(all_files); + } + + // Filter: keep files where user has editor permission on the KB, or owns unassigned files. + // Batch-resolve KB permissions to avoid an N+1 query per file. + let kb_ids: Vec = all_files.iter().filter_map(|f| f.kb_id).collect(); + let perms = super::knowledge_base::get_kb_permissions_batch(pool, &kb_ids, &auth_user.user_id, false).await; + + let mut deletable = Vec::new(); + for file in all_files { + match file.kb_id { + Some(kb_id) => { + let perm = perms.get(&kb_id).map(String::as_str); + if super::knowledge_base::meets_requirement(perm, "editor") { + deletable.push(file); + } + } + None => { + if file.user_id == auth_user.user_id { + deletable.push(file); + } + } + } + } + Ok(deletable) +} + +async fn clear_file_parse_rows_in_tx(tx: &mut sqlx::Transaction<'_, Sqlite>, file_id: i64) -> Result<(), sqlx::Error> { + clear_file_parse_rows_for_ids_in_tx(tx, &[file_id]).await +} + +async fn clear_file_parse_rows_for_ids_in_tx( + tx: &mut sqlx::Transaction<'_, Sqlite>, file_ids: &[i64], +) -> Result<(), sqlx::Error> { + if file_ids.is_empty() { + return Ok(()); + } + + for chunk in file_ids.chunks(SQLITE_DELETE_CHUNK_SIZE) { + let mut mentions_qb = QueryBuilder::::new( + "DELETE FROM entity_mentions WHERE slice_id IN (SELECT id FROM slices WHERE file_id IN (", + ); + crate::db::push_i64_list(&mut mentions_qb, chunk); + mentions_qb.push("))"); + mentions_qb.build().execute(&mut **tx).await?; + } + + for chunk in file_ids.chunks(SQLITE_DELETE_CHUNK_SIZE) { + let mut positions_qb = QueryBuilder::::new( + "DELETE FROM slice_positions WHERE slice_id IN (SELECT id FROM slices WHERE file_id IN (", + ); + crate::db::push_i64_list(&mut positions_qb, chunk); + positions_qb.push("))"); + positions_qb.build().execute(&mut **tx).await?; + } + + for chunk in file_ids.chunks(SQLITE_DELETE_CHUNK_SIZE) { + let mut slices_qb = QueryBuilder::::new("DELETE FROM slices WHERE file_id IN ("); + crate::db::push_i64_list(&mut slices_qb, chunk); + slices_qb.push(")"); + slices_qb.build().execute(&mut **tx).await?; + } + + for chunk in file_ids.chunks(SQLITE_DELETE_CHUNK_SIZE) { + let mut files_qb = QueryBuilder::::new("UPDATE files SET summary = NULL WHERE id IN ("); + crate::db::push_i64_list(&mut files_qb, chunk); + files_qb.push(")"); + files_qb.build().execute(&mut **tx).await?; + } + + Ok(()) +} + +pub(crate) async fn delete_file_rows_in_tx( + tx: &mut sqlx::Transaction<'_, Sqlite>, file_ids: &[i64], +) -> Result<(), sqlx::Error> { + if !file_ids.is_empty() { + // 先取得 SQLite 写锁，避免 deferred transaction 在并发读后升级写锁时返回 SQLITE_BUSY。 + let mut lock_qb = QueryBuilder::::new("UPDATE files SET updated_at = updated_at WHERE id IN ("); + crate::db::push_i64_list(&mut lock_qb, file_ids); + lock_qb.push(")"); + lock_qb.build().execute(&mut **tx).await?; + + let mut refs_qb = QueryBuilder::::new( + "SELECT COUNT(*) FROM parse_artifacts pa JOIN files ref ON ref.artifact_id = pa.id \ + WHERE pa.source_file_id IN (", + ); + crate::db::push_i64_list(&mut refs_qb, file_ids); + refs_qb.push(") AND ref.id NOT IN ("); + crate::db::push_i64_list(&mut refs_qb, file_ids); + refs_qb.push(")"); + let external_refs: i64 = refs_qb.build_query_scalar().fetch_one(&mut **tx).await?; + if external_refs > 0 { + return Err(sqlx::Error::Protocol( + "cannot delete a parse artifact source while other files still reference it".to_string(), + )); + } + } + clear_file_parse_rows_for_ids_in_tx(tx, file_ids).await?; + + for chunk in file_ids.chunks(SQLITE_DELETE_CHUNK_SIZE) { + let mut artifacts_qb = QueryBuilder::::new("DELETE FROM parse_artifacts WHERE source_file_id IN ("); + crate::db::push_i64_list(&mut artifacts_qb, chunk); + artifacts_qb.push(")"); + artifacts_qb.build().execute(&mut **tx).await?; + } + + for chunk in file_ids.chunks(SQLITE_DELETE_CHUNK_SIZE) { + let mut files_qb = QueryBuilder::::new("DELETE FROM files WHERE id IN ("); + crate::db::push_i64_list(&mut files_qb, chunk); + files_qb.push(")"); + files_qb.build().execute(&mut **tx).await?; + } + + Ok(()) +} + +/// 每批文件在独立事务里删除的批大小。SQLite 写串行，单个超大事务会长时间持有写锁、 +/// 阻塞其它写入；分批提交把锁持有时间限制在每批范围内。 +const FILE_DELETE_TX_BATCH_SIZE: usize = 500; + +/// 分批删除文件相关行，每批独立提交。 +/// +/// 与 [`delete_file_rows_in_tx`] 的区别：后者把全部删除塞进调用方的单个事务（适合少量文件、 +/// 需要与其它操作原子提交的场景）；本函数用于删除可能极多的文件（如删除整个知识库）， +/// 牺牲整体原子性换取更短的写锁持有时间。失败时已提交的批次不会回滚，重试可继续。 +pub(crate) async fn delete_file_rows_batched(pool: &SqlitePool, file_ids: &[i64]) -> Result<(), sqlx::Error> { + for batch in file_ids.chunks(FILE_DELETE_TX_BATCH_SIZE) { + let mut tx = pool.begin().await?; + delete_file_rows_in_tx(&mut tx, batch).await?; + tx.commit().await?; + } + Ok(()) +} + +async fn query_failed_file_ids_for_reparse( + pool: &SqlitePool, auth_user: &AuthUser, req: &ReparseFailedFilesReq, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + + if req.unassigned_only { + let mut qb = QueryBuilder::::new("SELECT f.id FROM files f WHERE f.status = -1 AND f.kb_id IS NULL"); + if !is_admin { + qb.push(" AND f.user_id = ").push_bind(&auth_user.user_id); + } + qb.push(" ORDER BY f.updated_at DESC"); + let ids = qb.build_query_scalar::().fetch_all(pool).await?; + return Ok(ids); + } + + if let Some(kb_id) = req.kb_id { + common::ensure_kb_editor_or_admin(pool, kb_id, auth_user).await?; + + if req.include_descendants { + let mut qb = QueryBuilder::::new( + "WITH RECURSIVE descendants AS (SELECT id, kb_type FROM knowledge_bases WHERE id = ", + ); + qb.push_bind(kb_id); + qb.push( + " UNION ALL SELECT kb.id, kb.kb_type FROM knowledge_bases kb JOIN descendants d ON kb.parent_id = d.id)", + ); + qb.push(" SELECT id, kb_type FROM descendants WHERE kb_type != "); + qb.push_bind("storage"); + let rows: Vec<(i64, String)> = qb.build_query_as().fetch_all(pool).await?; + let desc_kb_ids: Vec = rows.iter().map(|(id, _)| *id).collect(); + let perms = + super::knowledge_base::get_kb_permissions_batch(pool, &desc_kb_ids, &auth_user.user_id, is_admin).await; + let mut allowed_kb_ids: Vec = Vec::new(); + for (desc_kb_id, _) in rows { + let perm = perms.get(&desc_kb_id).map(String::as_str); + if super::knowledge_base::meets_requirement(perm, "editor") { + allowed_kb_ids.push(desc_kb_id); + } + } + if allowed_kb_ids.is_empty() { + return Ok(Vec::new()); + } + let mut qb2 = QueryBuilder::::new("SELECT f.id FROM files f WHERE f.status = -1 AND f.kb_id IN ("); + let mut sep = qb2.separated(", "); + for id in &allowed_kb_ids { + sep.push_bind(id); + } + qb2.push(") ORDER BY f.updated_at DESC"); + let ids = qb2.build_query_scalar::().fetch_all(pool).await?; + return Ok(ids); + } + + let mut qb = QueryBuilder::::new( + "SELECT f.id FROM files f \ + JOIN knowledge_bases kb ON kb.id = f.kb_id \ + WHERE f.status = -1 AND f.kb_id = ", + ); + qb.push_bind(kb_id); + qb.push(" AND kb.kb_type != ").push_bind("storage"); + if !is_admin { + qb.push(" AND f.user_id = ").push_bind(&auth_user.user_id); + } + qb.push(" ORDER BY f.updated_at DESC"); + let ids = qb.build_query_scalar::().fetch_all(pool).await?; + return Ok(ids); + } + + // Global reparse-failed: only user's own files + files in KBs where user has editor permission + let mut qb = QueryBuilder::::new( + "SELECT f.id, f.kb_id FROM files f \ + LEFT JOIN knowledge_bases kb ON kb.id = f.kb_id \ + WHERE f.status = -1 AND (f.kb_id IS NULL OR kb.kb_type != ", + ); + qb.push_bind("storage"); + qb.push(")"); + if !req.include_unassigned { + qb.push(" AND f.kb_id IS NOT NULL"); + } + qb.push(" ORDER BY f.updated_at DESC"); + let rows: Vec<(i64, Option)> = qb.build_query_as().fetch_all(pool).await?; + + // Batch-resolve permissions for all assigned KBs to avoid an N+1 query per file. + let assigned_kb_ids: Vec = rows.iter().filter_map(|(_, kb)| *kb).collect(); + let perms = + super::knowledge_base::get_kb_permissions_batch(pool, &assigned_kb_ids, &auth_user.user_id, is_admin).await; + + let mut allowed_ids = Vec::new(); + for (file_id, file_kb_id) in rows { + match file_kb_id { + None => { + // unassigned file: only own files + if is_admin { + allowed_ids.push(file_id); + } + } + Some(kid) => { + let perm = perms.get(&kid).map(String::as_str); + if super::knowledge_base::meets_requirement(perm, "editor") { + allowed_ids.push(file_id); + } + } + } + } + Ok(allowed_ids) +} + +async fn remove_converted_pdfs(file_ids: &[i64]) { + if file_ids.is_empty() { + return; + } + + let cfg = config::get(); + let pdf_root = std::path::Path::new(&cfg.storage.pdf_path); + for file_id in file_ids { + let pdf_path = pdf_root.join(format!("{}.pdf", file_id)); + if let Err(e) = fs::remove_file(&pdf_path).await + && !matches!(e.kind(), std::io::ErrorKind::NotFound) + { + warn!("Failed to delete converted pdf {}: {}", pdf_path.display(), e); + } + } +} + +async fn execute_reparse_failed( + pool: &SqlitePool, search_engine: &SearchEngine, auth_user: &AuthUser, req: ReparseFailedFilesReq, +) -> ApiResult { + let file_ids = query_failed_file_ids_for_reparse(pool, auth_user, &req).await?; + if file_ids.is_empty() { + return Ok(ReparseFailedFilesResp { file_count: 0 }); + } + + let image_paths = collect_image_paths_for_files(pool, &file_ids).await?; + search_engine.delete_batch(Some(&file_ids), None).await.map_err(map_search_engine_error)?; + + let mut tx = pool.begin().await?; + clear_file_parse_rows_for_ids_in_tx(&mut tx, &file_ids).await?; + + let mut qb = QueryBuilder::::new( + "UPDATE files SET status = 0, parse_run_id = NULL, log = '', summary = NULL, updated_at = strftime('%s','now') WHERE status = -1 AND id IN (", + ); + crate::db::push_i64_list(&mut qb, &file_ids); + qb.push(")"); + let updated = qb.build().execute(&mut *tx).await?.rows_affected() as i64; + tx.commit().await?; + + remove_image_files(image_paths).await; + remove_converted_pdfs(&file_ids).await; + for file_id in &file_ids { + if let Err(e) = crate::file_content::delete(*file_id).await { + warn!("Failed to delete content file for reparse-failed file {}: {}", file_id, e); + } + if let Err(e) = crate::pdf_content::delete(*file_id).await { + warn!("Failed to delete PDF content file for reparse-failed file {}: {}", file_id, e); + } + if let Err(e) = crate::slice_content::delete(*file_id).await { + warn!("Failed to delete slice content file for reparse-failed file {}: {}", file_id, e); + } + } + + Ok(ReparseFailedFilesResp { file_count: updated }) +} + +pub(crate) async fn cleanup_deleted_files( + search_engine: &SearchEngine, files: &[File], image_paths: Vec, +) -> Vec { + let cfg = config::get(); + + // 各文件的磁盘清理彼此独立，并发执行；搜索索引删除合并为一次批量提交。 + let fs_cleanups = files.iter().map(|file| { + let cfg = cfg.clone(); + async move { + let mut failed = Vec::new(); + + if let Err(e) = fs::remove_file(&file.path).await + && !matches!(e.kind(), std::io::ErrorKind::NotFound) + { + warn!("Failed to delete file {}: {}", file.path, e); + failed.push(BatchDeleteCleanupFailedItem { + id: file.id, + stage: "file".to_string(), + error: e.to_string(), + }); + } + + let pdf_path = std::path::Path::new(&cfg.storage.pdf_path).join(format!("{}.pdf", file.id)); + if let Err(e) = fs::remove_file(&pdf_path).await + && !matches!(e.kind(), std::io::ErrorKind::NotFound) + { + warn!("Failed to delete converted pdf {}: {}", pdf_path.display(), e); + failed.push(BatchDeleteCleanupFailedItem { + id: file.id, + stage: "pdf".to_string(), + error: e.to_string(), + }); + } + + // 清理解析后的完整文本文件 + let content_path = std::path::Path::new(&cfg.storage.contents_path).join(format!("{}.txt", file.id)); + if let Err(e) = fs::remove_file(&content_path).await + && !matches!(e.kind(), std::io::ErrorKind::NotFound) + { + warn!("Failed to delete content file {}: {}", content_path.display(), e); + failed.push(BatchDeleteCleanupFailedItem { + id: file.id, + stage: "content".to_string(), + error: e.to_string(), + }); + } + + let pdf_content_path = std::path::Path::new(&cfg.storage.contents_path).join(format!("{}.json", file.id)); + if let Err(e) = fs::remove_file(&pdf_content_path).await + && !matches!(e.kind(), std::io::ErrorKind::NotFound) + { + warn!("Failed to delete PDF content file {}: {}", pdf_content_path.display(), e); + failed.push(BatchDeleteCleanupFailedItem { + id: file.id, + stage: "pdf_content".to_string(), + error: e.to_string(), + }); + } + + let slice_content_path = + std::path::Path::new(&cfg.storage.slice_contents_path).join(format!("{}.json", file.id)); + if let Err(e) = fs::remove_file(&slice_content_path).await + && !matches!(e.kind(), std::io::ErrorKind::NotFound) + { + warn!("Failed to delete slice content file {}: {}", slice_content_path.display(), e); + failed.push(BatchDeleteCleanupFailedItem { + id: file.id, + stage: "slice_content".to_string(), + error: e.to_string(), + }); + } + + // 清理压缩文件解压目录 + let archive_dir = std::path::Path::new(&cfg.storage.archives_path).join(file.id.to_string()); + if archive_dir.exists() + && let Err(e) = tokio::fs::remove_dir_all(&archive_dir).await + { + warn!("Failed to delete archive dir {}: {}", archive_dir.display(), e); + failed.push(BatchDeleteCleanupFailedItem { + id: file.id, + stage: "archive".to_string(), + error: e.to_string(), + }); + } + + failed + } + }); + + let (fs_results, _) = tokio::join!(futures::future::join_all(fs_cleanups), remove_image_files(image_paths)); + let mut cleanup_failed: Vec = fs_results.into_iter().flatten().collect(); + + // 一次性删除所有文件的搜索索引，避免逐个文件触发 commit/锁竞争。 + let file_ids: Vec = files.iter().map(|f| f.id).collect(); + if let Err(e) = search_engine.delete_batch(Some(&file_ids), None).await { + warn!("Failed to delete search index for files {:?}: {}", file_ids, e); + for file in files { + cleanup_failed.push(BatchDeleteCleanupFailedItem { + id: file.id, + stage: "search".to_string(), + error: e.to_string(), + }); + } + } + + cleanup_failed +} + +async fn execute_batch_delete( + pool: &SqlitePool, search_engine: &SearchEngine, auth_user: &AuthUser, ids: Vec, strict: bool, + allow_processing: bool, +) -> ApiResult { + let overall_start = Instant::now(); + let ids = normalize_batch_delete_ids(ids)?; + let requested = ids.len() as i64; + + let step_start = Instant::now(); + let deletable_files = query_deletable_files(pool, &ids, auth_user).await?; + debug!( + "file_batch_delete query_deletable count={} requested={} {}ms", + deletable_files.len(), + ids.len(), + step_start.elapsed().as_millis() + ); + + let mut file_map: HashMap = deletable_files.into_iter().map(|file| (file.id, file)).collect(); + let mut files_to_delete = Vec::new(); + let mut skipped = Vec::new(); + for id in &ids { + let Some(file) = file_map.remove(id) else { + skipped.push(BatchDeleteSkippedItem { id: *id, reason: "not_found_or_no_permission".to_string() }); + continue; + }; + if !allow_processing && file.status == 2 { + skipped.push(BatchDeleteSkippedItem { id: *id, reason: "processing".to_string() }); + continue; + } + files_to_delete.push(file); + } + + if strict && !skipped.is_empty() { + let ids = skipped.iter().map(|item| item.id.to_string()).collect::>().join(", "); + return Err(ApiError::BadRequest(format!("Batch delete precheck failed for ids: {}", ids))); + } + + let file_ids: Vec = files_to_delete.iter().map(|file| file.id).collect(); + let accepted = file_ids.len() as i64; + if file_ids.is_empty() { + return Ok(BatchDeleteFilesResp { + requested, + accepted, + deleted: 0, + deleted_ids: Vec::new(), + skipped, + cleanup_failed: Vec::new(), + }); + } + + let step_start = Instant::now(); + let image_paths = collect_image_paths_for_files(pool, &file_ids).await?; + debug!("file_batch_delete collect_image_paths ids={} {}ms", file_ids.len(), step_start.elapsed().as_millis()); + + let step_start = Instant::now(); + // Delete in bounded transactions so a large request cannot monopolize SQLite's writer lock. + delete_file_rows_batched(pool, &file_ids).await?; + debug!("file_batch_delete delete_rows ids={} {}ms", file_ids.len(), step_start.elapsed().as_millis()); + + let step_start = Instant::now(); + let cleanup_failed = cleanup_deleted_files(search_engine, &files_to_delete, image_paths).await; + debug!( + "file_batch_delete cleanup ids={} failed={} {}ms", + file_ids.len(), + cleanup_failed.len(), + step_start.elapsed().as_millis() + ); + + debug!( + "file_batch_delete total requested={} accepted={} deleted={} {}ms", + requested, + accepted, + file_ids.len(), + overall_start.elapsed().as_millis() + ); + + Ok(BatchDeleteFilesResp { + requested, + accepted, + deleted: file_ids.len() as i64, + deleted_ids: file_ids, + skipped, + cleanup_failed, + }) +} + +#[derive(Default)] +struct FileImagePathState { + has_pdf_contents: bool, + meta_checked: bool, + paths: Vec, +} + +pub(crate) async fn collect_image_raw_paths_for_files(pool: &SqlitePool, file_ids: &[i64]) -> AnyResult> { + if file_ids.is_empty() { + return Ok(Vec::new()); + } + + let mut states: HashMap = + file_ids.iter().copied().map(|file_id| (file_id, FileImagePathState::default())).collect(); + + for file_id in file_ids { + let rows = crate::pdf_content::read(*file_id).await?; + if !rows.is_empty() + && let Some(state) = states.get_mut(file_id) + { + state.has_pdf_contents = true; + for row in rows { + if let Some(path) = row.img_path { + let trimmed = path.trim(); + if !trimmed.is_empty() { + state.paths.push(trimmed.to_string()); + } + } + } + } + } + + let meta_candidate_ids: Vec = + file_ids.iter().copied().filter(|id| !states.get(id).is_some_and(|state| state.has_pdf_contents)).collect(); + for chunk in meta_candidate_ids.chunks(SQLITE_DELETE_CHUNK_SIZE) { + let mut qb = QueryBuilder::::new("SELECT id, meta FROM files WHERE id IN ("); + crate::db::push_i64_list(&mut qb, chunk); + qb.push(")"); + let rows: Vec<(i64, Option)> = qb.build_query_as().fetch_all(pool).await?; + for (file_id, meta) in rows { + if let Some(paths) = extract_custom_image_paths_from_meta(meta.as_deref()) + && let Some(state) = states.get_mut(&file_id) + { + state.meta_checked = true; + state.paths.extend(paths); + } + } + } + + let regex_candidate_ids: Vec = file_ids + .iter() + .copied() + .filter(|id| states.get(id).is_some_and(|state| !state.has_pdf_contents && !state.meta_checked)) + .collect(); + let mut regex_paths = extract_slice_image_paths_by_file(pool, ®ex_candidate_ids).await?; + for file_id in regex_candidate_ids { + if let Some(paths) = regex_paths.remove(&file_id) + && let Some(state) = states.get_mut(&file_id) + { + state.paths.extend(paths); + } + } + + let mut raw_paths = Vec::new(); + let mut seen = HashSet::new(); + for file_id in file_ids { + let Some(state) = states.get(file_id) else { continue }; + for path in &state.paths { + let trimmed = path.trim(); + if !trimmed.is_empty() && seen.insert(trimmed.to_string()) { + raw_paths.push(trimmed.to_string()); + } + } + } + Ok(raw_paths) +} + +pub(crate) async fn collect_image_paths_for_files(pool: &SqlitePool, file_ids: &[i64]) -> AnyResult> { + let raw_paths = collect_image_raw_paths_for_files(pool, file_ids).await?; + let mut resolved = Vec::new(); + for raw in raw_paths { + if let Some(path) = resolve_image_storage_path(&raw) { + resolved.push(path); + } + } + Ok(resolved) +} + +pub(crate) async fn update_file_custom_image_meta( + pool: &SqlitePool, file_id: i64, image_paths: &[String], source: &str, +) -> AnyResult<()> { + let current_meta: Option = + sqlx::query_scalar("SELECT meta FROM files WHERE id = ?").bind(file_id).fetch_optional(pool).await?; + let merged_meta = merge_custom_image_meta(current_meta.as_deref(), image_paths, source); + sqlx::query("UPDATE files SET meta = ? WHERE id = ?").bind(merged_meta).bind(file_id).execute(pool).await?; + Ok(()) +} + +pub(crate) async fn backfill_missing_image_meta_for_files( + pool: &SqlitePool, file_ids: &[i64], source: &str, +) -> AnyResult { + if file_ids.is_empty() { + return Ok(0); + } + + let mut candidate_ids = Vec::new(); + for chunk in file_ids.chunks(SQLITE_DELETE_CHUNK_SIZE) { + let mut qb = QueryBuilder::::new("SELECT f.id, f.meta FROM files f WHERE f.id IN ("); + crate::db::push_i64_list(&mut qb, chunk); + qb.push(")"); + let rows: Vec<(i64, Option)> = qb.build_query_as().fetch_all(pool).await?; + for (file_id, meta) in rows { + if crate::pdf_content::read(file_id).await?.is_empty() + && extract_custom_image_paths_from_meta(meta.as_deref()).is_none() + { + candidate_ids.push(file_id); + } + } + } + + if candidate_ids.is_empty() { + return Ok(0); + } + + let mut extracted = extract_slice_image_paths_by_file(pool, &candidate_ids).await?; + let mut updated = 0usize; + for file_id in candidate_ids { + let paths = extracted.remove(&file_id).unwrap_or_default(); + update_file_custom_image_meta(pool, file_id, &paths, source).await?; + updated += 1; + } + Ok(updated) +} + +fn extract_custom_image_paths_from_meta(meta: Option<&str>) -> Option> { + let raw = meta?.trim(); + if raw.is_empty() { + return None; + } + let value: serde_json::Value = serde_json::from_str(raw).ok()?; + let custom_images = value.get(PARSE_ASSETS_META_KEY)?.get(CUSTOM_IMAGES_META_KEY)?; + if !custom_images.get("checked").and_then(|v| v.as_bool()).unwrap_or(false) { + return None; + } + + let mut paths = Vec::new(); + if let Some(raw_paths) = custom_images.get("paths").and_then(|v| v.as_array()) { + for path in raw_paths { + if let Some(path) = path.as_str() { + let trimmed = path.trim(); + if !trimmed.is_empty() { + paths.push(trimmed.to_string()); + } + } + } + } + Some(paths) +} + +fn merge_custom_image_meta(meta: Option<&str>, image_paths: &[String], source: &str) -> String { + let mut root = parse_meta_object(meta); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or_default(); + + let mut unique_paths = Vec::new(); + let mut seen = HashSet::new(); + for path in image_paths { + let trimmed = path.trim(); + if !trimmed.is_empty() && seen.insert(trimmed.to_string()) { + unique_paths.push(trimmed.to_string()); + } + } + + let parse_assets = root.entry(PARSE_ASSETS_META_KEY.to_string()).or_insert_with(|| serde_json::json!({})); + if !parse_assets.is_object() { + *parse_assets = serde_json::json!({}); + } + let parse_assets_obj = parse_assets.as_object_mut().expect("parse assets meta is object"); + parse_assets_obj.insert("version".to_string(), serde_json::json!(CUSTOM_IMAGES_META_VERSION)); + parse_assets_obj.insert( + CUSTOM_IMAGES_META_KEY.to_string(), + serde_json::json!({ + "source": source, + "paths": unique_paths, + "checked": true, + "updated_at": now + }), + ); + + serde_json::Value::Object(root).to_string() +} + +fn parse_meta_object(meta: Option<&str>) -> serde_json::Map { + let Some(raw) = meta.map(str::trim).filter(|raw| !raw.is_empty()) else { + return serde_json::Map::new(); + }; + + match serde_json::from_str::(raw) { + Ok(serde_json::Value::Object(map)) => map, + Ok(value) => { + let mut map = serde_json::Map::new(); + map.insert("_legacy_meta".to_string(), value); + map + } + Err(_) => { + let mut map = serde_json::Map::new(); + map.insert("_legacy_meta".to_string(), serde_json::Value::String(raw.to_string())); + map + } + } +} + +async fn extract_slice_image_paths_by_file( + pool: &SqlitePool, file_ids: &[i64], +) -> Result>, sqlx::Error> { + let mut result: HashMap> = HashMap::new(); + let mut seen: HashMap> = HashMap::new(); + if file_ids.is_empty() { + return Ok(result); + } + + for chunk in file_ids.chunks(SQLITE_DELETE_CHUNK_SIZE) { + let mut qb = QueryBuilder::::new("SELECT DISTINCT file_id FROM slices WHERE file_id IN ("); + crate::db::push_i64_list(&mut qb, chunk); + qb.push(")"); + let rows: Vec = qb.build_query_scalar().fetch_all(pool).await?; + for file_id in rows { + let contents = + crate::slice_content::read_all(file_id).await.map_err(|e| sqlx::Error::Protocol(e.to_string()))?; + let file_seen = seen.entry(file_id).or_default(); + for content in contents.values() { + for path in extract_image_paths_from_text(content) { + if file_seen.insert(path.clone()) { + result.entry(file_id).or_default().push(path); + } + } + } + } + } + + Ok(result) +} + +fn extract_image_paths_from_text(content: &str) -> HashSet { + static IMAGE_REF_RE: OnceLock = OnceLock::new(); + let re = IMAGE_REF_RE.get_or_init(|| { + regex::Regex::new(r"/api/v1/knowledge/files/([^\s'\)>]+)").expect("valid image reference regex") + }); + + let mut paths = HashSet::new(); + for cap in re.captures_iter(content) { + let Some(matched) = cap.get(1) else { continue }; + let path = matched.as_str().trim(); + if !path.is_empty() { + paths.insert(path.to_string()); + } + } + paths +} + +fn is_safe_relative_path(path: &str) -> bool { + let path = std::path::Path::new(path); + if path.is_absolute() { + return false; + } + for component in path.components() { + match component { + Component::ParentDir | Component::RootDir | Component::Prefix(_) => return false, + _ => {} + } + } + true +} + +pub(crate) fn resolve_image_storage_path(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + if !is_safe_relative_path(trimmed) { + log::warn!("Skipping unsafe image path: {}", trimmed); + return None; + } + + let cfg = config::get(); + let images_root = std::path::Path::new(&cfg.storage.images_path); + + // 图片实际存储路径为 images_path/trimmed + // （与 save_mineru_images 的 "{images_path}/{img_name}" 一致） + let resolved = images_root.join(trimmed); + + Some(resolved.to_string_lossy().to_string()) +} + +pub(crate) async fn find_reusable_parsed_file( + pool: &SqlitePool, hash: &str, slice_type: &str, exclude_file_id: Option, +) -> Result, sqlx::Error> { + let base_sql = "SELECT * FROM files WHERE hash = ? AND slice_type = ? AND status = 1"; + let sql = if exclude_file_id.is_some() { + format!("{} AND id != ? ORDER BY updated_at DESC LIMIT 1", base_sql) + } else { + format!("{} ORDER BY updated_at DESC LIMIT 1", base_sql) + }; + + let mut query = sqlx::query_as::<_, File>(&sql).bind(hash).bind(slice_type); + if let Some(file_id) = exclude_file_id { + query = query.bind(file_id); + } + + query.fetch_optional(pool).await +} + +pub(crate) async fn effective_parse_file_id(pool: &SqlitePool, file_id: i64) -> Result { + Ok(sqlx::query_scalar( + "SELECT COALESCE(pa.source_file_id, f.id) + FROM files f LEFT JOIN parse_artifacts pa ON pa.id = f.artifact_id + WHERE f.id = ?", + ) + .bind(file_id) + .fetch_optional(pool) + .await? + .unwrap_or(file_id)) +} + +pub(crate) async fn remove_image_files(image_paths: Vec) { + info!("Removing image files: {:?}", image_paths); + if image_paths.is_empty() { + return; + } + + let cfg = config::get(); + let images_root = std::path::Path::new(&cfg.storage.images_path); + let data_root = images_root.parent(); + let mut seen = HashSet::new(); + for img_path in image_paths { + let trimmed = img_path.trim(); + if trimmed.is_empty() || !seen.insert(trimmed.to_string()) { + continue; + } + + let candidate = std::path::Path::new(trimmed); + let full_path = if candidate.is_absolute() + || candidate.starts_with(images_root) + || data_root.is_some_and(|root| candidate.starts_with(root)) + { + candidate.to_path_buf() + } else { + match resolve_image_storage_path(trimmed) { + Some(path) => std::path::PathBuf::from(path), + None => continue, + } + }; + let allowed = full_path.starts_with(images_root) || data_root.is_some_and(|root| full_path.starts_with(root)); + if !allowed { + log::warn!("Skipping unsafe image path: {}", full_path.display()); + continue; + } + if let Err(e) = fs::remove_file(&full_path).await { + if matches!(e.kind(), std::io::ErrorKind::NotFound) { + if !candidate.is_absolute() + && (trimmed.contains('/') || trimmed.contains('\\')) + && let Some(file_name) = candidate.file_name() + { + let fallback_path = images_root.join(file_name); + if let Err(e) = fs::remove_file(&fallback_path).await + && !matches!(e.kind(), std::io::ErrorKind::NotFound) + { + log::warn!("Failed to delete image {}: {}", fallback_path.display(), e); + } + } + } else { + log::warn!("Failed to delete image {}: {}", full_path.display(), e); + } + } + } +} + +#[derive(Deserialize, IntoParams)] +pub struct ListQuery { + pub kb_id: Option, + pub tag: Option, + /// 页码，从1开始 + pub page: Option, + /// 每页条数 + pub size: Option, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct FileListResponse { + pub total: i64, + pub items: Vec, +} + +#[derive(Deserialize, IntoParams)] +pub struct FileStatsQuery { + /// 指定知识库 ID，统计该知识库以及（可选）子知识库 + pub kb_id: Option, + /// 是否包含子知识库文件，默认 true + pub include_descendants: Option, + /// 全局统计时是否包含未分配知识库的文件，默认 true + pub include_unassigned: Option, +} + +/// 获取文件列表 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/", + operation_id = "file_list", + tag = "file", + params(ListQuery), + responses( + (status = 200, description = "成功返回文件列表", body = FileListResponse), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn list( + State(pool): State, Extension(auth_user): Extension, Query(query): Query, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + let user_id = auth_user.user_id.clone(); + let size = query.size.unwrap_or(10).clamp(1, 200); + let page = query.page.unwrap_or(1).max(1); + let limit = size; + let offset = (page - 1) * size; + + let kb_filter = match query.kb_id.as_deref() { + Some("null") | Some("unassigned") => Some(None), + Some(kb_id_str) => { + let kb_id = kb_id_str.parse::().map_err(|_| ApiError::internal("Invalid kb_id format"))?; + common::ensure_kb_accessible(&pool, kb_id, &user_id, is_admin).await?; + Some(Some(kb_id)) + } + None => None, + }; + + let tag = query.tag.as_deref().filter(|s| !s.is_empty()); + + if let Some(tag) = tag { + let mut qb = QueryBuilder::::new(&format!("SELECT {FILE_COLS_NO_CONTENT} FROM files f WHERE 1=1")); + match kb_filter { + Some(None) => { + qb.push(" AND f.kb_id IS NULL"); + } + Some(Some(kb_id)) => { + qb.push(" AND f.kb_id = ").push_bind(kb_id); + } + None => {} + } + if !is_admin { + qb.push(" AND "); + common::push_file_access_filter(&mut qb, &user_id, Some("f")); + } + qb.push(" ORDER BY f.created_at DESC"); + let mut files: Vec = qb.build_query_as().fetch_all(&pool).await?; + files.retain(|file: &File| { + if let Ok(tags) = serde_json::from_str::>(&file.tags) { + tags.contains(&tag.to_string()) + } else { + false + } + }); + let total = files.len() as i64; + let items = files + .into_iter() + .skip(offset as usize) + .take(limit as usize) + .map(|mut file| { + file.content = None; + file + }) + .collect(); + return Ok(Json(FileListResponse { total, items })); + } + + let mut count_qb = QueryBuilder::::new("SELECT COUNT(*) FROM files f WHERE 1=1"); + match kb_filter { + Some(None) => { + count_qb.push(" AND f.kb_id IS NULL"); + } + Some(Some(kb_id)) => { + count_qb.push(" AND f.kb_id = ").push_bind(kb_id); + } + None => {} + } + if !is_admin { + count_qb.push(" AND "); + common::push_file_access_filter(&mut count_qb, &user_id, Some("f")); + } + let total: i64 = count_qb.build_query_scalar().fetch_one(&pool).await?; + + let mut list_qb = QueryBuilder::::new(&format!("SELECT {FILE_COLS_NO_CONTENT} FROM files f WHERE 1=1")); + match kb_filter { + Some(None) => { + list_qb.push(" AND f.kb_id IS NULL"); + } + Some(Some(kb_id)) => { + list_qb.push(" AND f.kb_id = ").push_bind(kb_id); + } + None => {} + } + if !is_admin { + list_qb.push(" AND "); + common::push_file_access_filter(&mut list_qb, &user_id, Some("f")); + } + list_qb.push(" ORDER BY f.created_at DESC LIMIT ").push_bind(limit); + list_qb.push(" OFFSET ").push_bind(offset); + let mut items: Vec = list_qb.build_query_as().fetch_all(&pool).await?; + + for file in &mut items { + file.content = None; + } + + Ok(Json(FileListResponse { total, items })) +} + +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/stats", + operation_id = "file_stats", + tag = "file", + params(FileStatsQuery), + responses( + (status = 200, description = "成功返回文件状态统计", body = FileStatusBreakdown), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn stats( + State(pool): State, Extension(auth_user): Extension, Query(query): Query, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + let include_descendants = query.include_descendants.unwrap_or(true); + let include_unassigned = query.include_unassigned.unwrap_or(true); + + let breakdown = match query.kb_id.as_deref() { + Some("null") | Some("unassigned") => { + get_file_status_breakdown_for_unassigned(&pool, &auth_user.user_id, is_admin).await? + } + Some(kb_id_str) => { + let kb_id = + kb_id_str.parse::().map_err(|_| ApiError::BadRequest("Invalid kb_id format".to_string()))?; + common::ensure_kb_accessible(&pool, kb_id, &auth_user.user_id, is_admin).await?; + get_file_status_breakdown_for_kb(&pool, kb_id, include_descendants, &auth_user.user_id, is_admin).await? + } + None => get_file_status_breakdown_for_all(&pool, include_unassigned, &auth_user.user_id, is_admin).await?, + }; + + Ok(Json(breakdown)) +} + +#[derive(Debug, Clone, Serialize, sqlx::FromRow, ToSchema)] +pub struct Slice { + pub id: i64, + pub file_id: i64, + pub content: String, + pub created_at: i64, + pub updated_at: i64, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct UpdateSliceItem { + pub id: i64, + pub content: String, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct UpdateSlicesReq { + pub slices: Vec, +} + +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct SlicePosition { + pub page_idx: i32, + pub bbox: [i32; 4], + pub sheet_name: Option, + pub row_num: Option, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct SliceHighlight { + pub positions: Vec, +} + +/// 切片推荐高亮页码 +#[derive(Debug, Serialize, ToSchema)] +pub struct SliceHighlightPage { + /// 推荐高亮页码（0-based，与 slice_positions.page_idx 保持一致） + pub page_idx: i32, +} + +#[derive(Debug, sqlx::FromRow)] +struct SlicePositionRow { + page_idx: i32, + x1: i32, + y1: i32, + x2: i32, + y2: i32, + sheet_name: Option, + row_num: Option, +} + +/// 获取文件的所有切片 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/{id}/slices", + operation_id = "file_get_slices", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID") + ), + responses( + (status = 200, description = "成功返回切片列表", body = Vec), + (status = 404, description = "文件不存在") + ) +)] +pub async fn get_slices(State(pool): State, Path(id): Path) -> ApiResult>> { + let source_id = effective_parse_file_id(&pool, id).await?; + let rows: Vec<(i64, i64, i64)> = + sqlx::query_as("SELECT id, created_at, updated_at FROM slices WHERE file_id = ? ORDER BY id") + .bind(source_id) + .fetch_all(&pool) + .await?; + let contents = crate::slice_content::read_all(source_id).await?; + let slices = rows + .into_iter() + .map(|(slice_id, created_at, updated_at)| Slice { + id: slice_id, + file_id: id, + content: contents.get(&slice_id).cloned().unwrap_or_default(), + created_at, + updated_at, + }) + .collect(); + Ok(Json(slices)) +} + +/// 批量更新文件切片内容 +#[utoipa::path( + put, + path = "/api/v1/knowledge/files/{id}/slices", + operation_id = "file_update_slices", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID") + ), + request_body = UpdateSlicesReq, + responses( + (status = 200, description = "成功更新切片", body = Vec), + (status = 400, description = "请求参数错误"), + (status = 403, description = "无权限"), + (status = 404, description = "文件或切片不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn update_slices( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, Path(id): Path, Json(req): Json, +) -> ApiResult>> { + if req.slices.is_empty() { + return Err(ApiError::BadRequest("No slices to update".to_string())); + } + + // 1. 查询文件信息并校验权限 + let file: File = sqlx::query_as(&format!("SELECT {} FROM files WHERE id = ?", FILE_COLS_NO_CONTENT)) + .bind(id) + .fetch_optional(&pool) + .await? + .ok_or_else(|| ApiError::NotFound("File not found".to_string()))?; + + ensure_file_writable(&pool, &file, &auth_user).await?; + + // 校验知识库类型非 storage + if let Some(kb_id) = file.kb_id { + let kb_type: Option = sqlx::query_scalar("SELECT kb_type FROM knowledge_bases WHERE id = ?") + .bind(kb_id) + .fetch_optional(&pool) + .await?; + if matches!(kb_type.as_deref(), Some("storage")) { + return Err(ApiError::BadRequest("Storage knowledge base files do not support slice editing.".to_string())); + } + } + + // 2. 校验文件状态 + if file.status != 1 { + return Err(ApiError::BadRequest("File is not processed".to_string())); + } + if file.artifact_id.is_some() && effective_parse_file_id(&pool, id).await? != id { + return Err(ApiError::BadRequest( + "Shared parsed slices are read-only; detach the artifact before editing".to_string(), + )); + } + + // 3. 校验 content 非空 + for item in &req.slices { + if item.content.trim().is_empty() { + return Err(ApiError::BadRequest(format!("Slice {} content cannot be empty", item.id))); + } + } + + // 4. 校验 slice id 存在且属于该文件 + let requested_ids: Vec = req.slices.iter().map(|s| s.id).collect(); + let existing_ids_ordered: Vec = + sqlx::query_scalar("SELECT id FROM slices WHERE file_id = ? ORDER BY id").bind(id).fetch_all(&pool).await?; + let existing_contents = crate::slice_content::read_all(id).await?; + let existing_slices: Vec<(i64, String)> = existing_ids_ordered + .iter() + .map(|slice_id| (*slice_id, existing_contents.get(slice_id).cloned().unwrap_or_default())) + .collect(); + let existing_ids: std::collections::HashSet = existing_slices.iter().map(|(id, _)| *id).collect(); + + for item in &req.slices { + if !existing_ids.contains(&item.id) { + return Err(ApiError::NotFound(format!("Slice {} not found", item.id))); + } + } + + // 5. 构建 id -> new content 映射 + let mut content_map: std::collections::HashMap = + req.slices.into_iter().map(|s| (s.id, s.content)).collect(); + + // 6. 正文包使用原子 rename 更新，数据库只更新时间戳。 + let mut tx = pool.begin().await?; + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64; + + let changed: Vec<(i64, String)> = requested_ids + .iter() + .map(|slice_id| (*slice_id, content_map.get(slice_id).cloned().unwrap_or_default())) + .collect(); + crate::slice_content::upsert_many(id, &changed).await?; + if !requested_ids.is_empty() { + let mut qb = QueryBuilder::::new("UPDATE slices SET updated_at = "); + qb.push_bind(now).push(" WHERE file_id = ").push_bind(id).push(" AND id IN ("); + crate::db::push_i64_list(&mut qb, &requested_ids); + qb.push(")"); + qb.build().execute(&mut *tx).await?; + } + + // 重新拼接完整内容：使用最新切片内容 + let full_content = existing_slices + .iter() + .map(|(slice_id, content)| content_map.get(slice_id).cloned().unwrap_or_else(|| content.clone())) + .collect::>() + .join("\n\n"); + + sqlx::query("UPDATE files SET updated_at = ? WHERE id = ?").bind(now).bind(id).execute(&mut *tx).await?; + tx.commit().await?; + crate::file_content::write(id, &full_content).await?; + + // 7. 同步搜索索引（事务外，失败仅记录 warning） + let updates: Vec<(i64, String)> = requested_ids + .iter() + .map(|slice_id| (*slice_id, content_map.remove(slice_id).unwrap_or_default())) + .filter(|(_, content)| !content.is_empty()) + .collect(); + + let (slice_index_result, full_index_result) = tokio::join!( + search_engine.update_slices(id, file.kb_id, updates), + search_engine.update_full_index_for_file(id, file.kb_id, file.filename.clone(), full_content) + ); + if let Err(e) = slice_index_result { + warn!("Failed to update slice search index for file {}: {}", id, e); + } + if let Err(e) = full_index_result { + warn!("Failed to update full search index for file {}: {}", id, e); + } + + // 8. 返回更新后的切片列表 + get_slices(State(pool), Path(id)).await +} + +/// 获取单个切片的高亮位置信息 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/{id}/slices/{slice_id}/highlight", + operation_id = "file_get_slice_highlight", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID"), + ("slice_id" = i64, Path, description = "切片 ID") + ), + responses( + (status = 200, description = "成功返回切片高亮位置", body = SliceHighlight), + (status = 400, description = "切片不属于该文件"), + (status = 404, description = "文件或切片不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn get_slice_highlight( + State(pool): State, Path((id, slice_id)): Path<(i64, i64)>, Extension(auth_user): Extension, +) -> ApiResult> { + let file: File = sqlx::query_as(&format!("SELECT {} FROM files WHERE id = ?", FILE_COLS_NO_CONTENT)) + .bind(id) + .fetch_one(&pool) + .await?; + + ensure_file_readable(&file, &auth_user)?; + + let parse_file_id = effective_parse_file_id(&pool, id).await?; + let slice: Option<(i64,)> = + sqlx::query_as("SELECT file_id FROM slices WHERE id = ?").bind(slice_id).fetch_optional(&pool).await?; + + match slice { + Some((file_id,)) if file_id == parse_file_id => {} + Some(_) => return Err(ApiError::BadRequest("Slice does not belong to the file".to_string())), + None => return Err(ApiError::NotFound("Slice not found".to_string())), + } + + let rows: Vec = sqlx::query_as( + "SELECT page_idx, x1, y1, x2, y2, sheet_name, row_num FROM slice_positions WHERE slice_id = ? ORDER BY page_idx, id", + ) + .bind(slice_id) + .fetch_all(&pool) + .await?; + + let positions = rows + .into_iter() + .map(|row| SlicePosition { + page_idx: row.page_idx, + bbox: [row.x1, row.y1, row.x2, row.y2], + sheet_name: row.sheet_name, + row_num: row.row_num, + }) + .collect(); + + Ok(Json(SliceHighlight { positions })) +} + +/// 获取切片推荐高亮页码 +/// +/// 优先返回首页；如果第一页的内容字数（按 pdf_content 中对应位置的内容文本长度计算） +/// 少于配置阈值且存在第二页，则返回第二页。若 pdf_content 不存在或找不到匹配位置， +/// 则退化为按位置数量计数。 +/// 返回的 `page_idx` 为 0-based，前端展示时需自行 +1。 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/{id}/slices/{slice_id}/highlight-page", + operation_id = "file_get_slice_highlight_page", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID"), + ("slice_id" = i64, Path, description = "切片 ID") + ), + responses( + (status = 200, description = "成功返回推荐高亮页码", body = SliceHighlightPage), + (status = 400, description = "切片不属于该文件"), + (status = 404, description = "文件或切片不存在，或切片没有高亮位置") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn get_slice_highlight_page( + State(pool): State, Path((id, slice_id)): Path<(i64, i64)>, Extension(auth_user): Extension, +) -> ApiResult> { + let file: File = sqlx::query_as(&format!("SELECT {} FROM files WHERE id = ?", FILE_COLS_NO_CONTENT)) + .bind(id) + .fetch_one(&pool) + .await?; + + ensure_file_readable(&file, &auth_user)?; + + let parse_file_id = effective_parse_file_id(&pool, id).await?; + let slice: Option<(i64,)> = + sqlx::query_as("SELECT file_id FROM slices WHERE id = ?").bind(slice_id).fetch_optional(&pool).await?; + + match slice { + Some((file_id,)) if file_id == parse_file_id => {} + Some(_) => return Err(ApiError::BadRequest("Slice does not belong to the file".to_string())), + None => return Err(ApiError::NotFound("Slice not found".to_string())), + } + + let rows: Vec<(i32, i32, i32, i32, i32)> = + sqlx::query_as("SELECT page_idx, x1, y1, x2, y2 FROM slice_positions WHERE slice_id = ? ORDER BY page_idx, id") + .bind(slice_id) + .fetch_all(&pool) + .await?; + + if rows.is_empty() { + return Err(ApiError::NotFound("Slice has no highlight positions".to_string())); + } + + // 按 page_idx + bbox 去 pdf_content 里匹配内容长度；匹配不到时退化为 1（保持原“位置数量”语义） + let pdf_contents = crate::pdf_content::read(parse_file_id).await.unwrap_or_default(); + let mut content_len_by_bbox: HashMap<(i32, [i32; 4]), i64> = HashMap::new(); + for row in &pdf_contents { + let Some(raw_bbox) = &row.bbox else { continue }; + let Ok(bbox) = serde_json::from_str::>(raw_bbox) else { continue }; + if bbox.len() != 4 { + continue; + } + let mut len = 0i64; + if let Some(text) = &row.text { + len += text.chars().count() as i64; + } + if let Some(body) = &row.table_body { + len += body.chars().count() as i64; + } + let key = (row.page_idx, [bbox[0], bbox[1], bbox[2], bbox[3]]); + content_len_by_bbox.insert(key, len); + } + + let mut page_counts: BTreeMap = BTreeMap::new(); + for (page_idx, x1, y1, x2, y2) in rows { + let bbox = [x1, y1, x2, y2]; + let len = content_len_by_bbox.get(&(page_idx, bbox)).copied().unwrap_or(1); + *page_counts.entry(page_idx).or_insert(0) += len; + } + + let threshold = config::get().search.highlight_page_min_chars as i64; + let mut iter = page_counts.iter(); + let first = iter.next().expect("page_counts not empty"); + let page_idx = + if *first.1 < threshold && page_counts.len() > 1 { *iter.next().expect("has second page").0 } else { *first.0 }; + + Ok(Json(SliceHighlightPage { page_idx })) +} + +/// 获取存储目录中的图片 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/images/{filename}", + operation_id = "file_get_image_by_filename", + tag = "file", + params( + ("filename" = String, Path, description = "图片文件名") + ), + responses( + (status = 200, description = "成功返回图片文件", content_type = "image/*"), + (status = 400, description = "请求参数错误"), + (status = 404, description = "图片不存在") + ) +)] +pub async fn get_image_by_filename( + Path(filename): Path, +) -> Result<(StatusCode, [(header::HeaderName, String); 2], Body), ApiError> { + let mut components = std::path::Path::new(&filename).components(); + match components.next() { + Some(Component::Normal(_)) if components.next().is_none() => {} + _ => return Err(ApiError::BadRequest("Invalid filename".to_string())), + } + + let cfg = config::get(); + let image_path = std::path::Path::new(&cfg.storage.images_path).join(&filename); + if !fs::try_exists(&image_path).await? { + return Err(ApiError::NotFound("Image not found".to_string())); + } + + let (len, body) = open_file_stream(&image_path).await?; + let mime_type = mime_guess::from_path(&filename).first_or_octet_stream().to_string(); + + Ok((StatusCode::OK, [(header::CONTENT_TYPE, mime_type), (header::CONTENT_LENGTH, len.to_string())], body)) +} + +/// 下载文件 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/{id}/download", + operation_id = "file_download", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID") + ), + responses( + (status = 200, description = "成功下载文件", content_type = "application/octet-stream"), + (status = 404, description = "文件不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn download( + State(pool): State, Path(id): Path, Extension(auth_user): Extension, +) -> Result<(StatusCode, [(header::HeaderName, String); 3], Body), ApiError> { + let file: File = sqlx::query_as(&format!("SELECT {} FROM files WHERE id = ?", FILE_COLS_NO_CONTENT)) + .bind(id) + .fetch_one(&pool) + .await?; + + ensure_file_readable(&file, &auth_user)?; + let (len, body) = open_file_stream(std::path::Path::new(&file.path)).await?; + let mime_type = mime_guess::from_path(&file.filename).first_or_octet_stream().to_string(); + let content_disposition = format!("attachment; filename=\"{}\"", file.filename); + + Ok(( + StatusCode::OK, + [ + (header::CONTENT_TYPE, mime_type), + (header::CONTENT_DISPOSITION, content_disposition), + (header::CONTENT_LENGTH, len.to_string()), + ], + body, + )) +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct HighlightQuery { + /// 切片 ID，不传则只返回原 PDF 不做高亮 + pub slice_id: Option, +} + +/// 获取带高亮标注的 PDF +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/{id}/highlighted-pdf", + operation_id = "file_get_highlighted_pdf", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID"), + HighlightQuery, + ), + responses( + (status = 200, description = "成功返回带高亮的 PDF", content_type = "application/pdf"), + (status = 400, description = "请求参数错误"), + (status = 404, description = "文件不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn get_highlighted_pdf( + State(pool): State, Path(id): Path, Query(params): Query, + Extension(auth_user): Extension, +) -> Result<(StatusCode, [(header::HeaderName, String); 3], Body), ApiError> { + let file: File = sqlx::query_as(&format!("SELECT {} FROM files WHERE id = ?", FILE_COLS_NO_CONTENT)) + .bind(id) + .fetch_one(&pool) + .await?; + + ensure_file_readable(&file, &auth_user)?; + + let parse_file_id = effective_parse_file_id(&pool, id).await?; + + let positions: Vec = if let Some(slice_id) = params.slice_id { + // 校验切片属于该文件 + let slice: Option<(i64,)> = + sqlx::query_as("SELECT file_id FROM slices WHERE id = ?").bind(slice_id).fetch_optional(&pool).await?; + match slice { + Some((file_id,)) if file_id == parse_file_id => {} + Some(_) => return Err(ApiError::BadRequest("Slice does not belong to the file".to_string())), + None => return Err(ApiError::NotFound("Slice not found".to_string())), + } + + // 从数据库查询 slice 的 positions + let rows: Vec = sqlx::query_as( + "SELECT page_idx, x1, y1, x2, y2, sheet_name, row_num FROM slice_positions WHERE slice_id = ? ORDER BY page_idx, id", + ) + .bind(slice_id) + .fetch_all(&pool) + .await?; + + rows.iter() + .map(|row| pdf_highlight::HighlightPosition { + page_idx: row.page_idx, + bbox: [row.x1, row.y1, row.x2, row.y2], + }) + .collect() + } else { + Vec::new() + }; + + let coord_bounds_by_page = if positions.is_empty() { + None + } else { + let rows = crate::pdf_content::read(parse_file_id).await?; + + let mut bounds: HashMap = HashMap::new(); + for row in rows { + if let Some(raw_bbox) = row.bbox + && let Ok(bbox) = serde_json::from_str::>(&raw_bbox) + && bbox.len() == 4 + { + let x1 = bbox[0]; + let y1 = bbox[1]; + let x2 = bbox[2]; + let y2 = bbox[3]; + let min_x = x1.min(x2); + let min_y = y1.min(y2); + let max_x = x1.max(x2); + let max_y = y1.max(y2); + + bounds + .entry(row.page_idx) + .and_modify(|b| { + b.min_x = b.min_x.min(min_x); + b.min_y = b.min_y.min(min_y); + b.max_x = b.max_x.max(max_x); + b.max_y = b.max_y.max(max_y); + }) + .or_insert(pdf_highlight::PageCoordBounds { min_x, min_y, max_x, max_y }); + } + } + + bounds.retain(|_, b| { + b.min_x.is_finite() + && b.min_y.is_finite() + && b.max_x.is_finite() + && b.max_y.is_finite() + && b.max_x > b.min_x + && b.max_y > b.min_y + }); + + if bounds.is_empty() { None } else { Some(bounds) } + }; + + // 确定 PDF 文件路径 + let filename_lower = file.filename.to_lowercase(); + let pdf_path = if filename_lower.ends_with(".doc") + || filename_lower.ends_with(".docx") + || filename_lower.ends_with(".ppt") + || filename_lower.ends_with(".pptx") + || filename_lower.ends_with(".xls") + || filename_lower.ends_with(".xlsx") + { + let cfg = config::get(); + let path = std::path::Path::new(&cfg.storage.pdf_path).join(format!("{}.pdf", parse_file_id)); + if !tokio::fs::try_exists(&path).await? { + return Err(ApiError::NotFound("Converted PDF not found".to_string())); + } + path + } else if filename_lower.ends_with(".pdf") { + std::path::PathBuf::from(&file.path) + } else { + return Err(ApiError::BadRequest("File is not a PDF, Word, PowerPoint, or Excel document".to_string())); + }; + + let content_disposition = format!("inline; filename=\"highlighted_{}.pdf\"", file.id); + + // 无高亮时直接流式返回原 PDF + if positions.is_empty() { + let (len, body) = open_file_stream(&pdf_path).await?; + return Ok(( + StatusCode::OK, + [ + (header::CONTENT_TYPE, "application/pdf".to_string()), + (header::CONTENT_DISPOSITION, content_disposition), + (header::CONTENT_LENGTH, len.to_string()), + ], + body, + )); + } + + // 有高亮时，在阻塞线程中生成高亮 PDF 并写入临时文件，然后流式返回 + let highlighted_pdf = tokio::task::spawn_blocking(move || { + let pdf_bytes = std::fs::read(&pdf_path)?; + pdf_highlight::add_highlights_to_pdf_with_bounds(&pdf_bytes, &positions, coord_bounds_by_page.as_ref()) + }) + .await + .map_err(|e| ApiError::Internal(format!("PDF highlight task panicked: {}", e)))? + .map_err(|e| ApiError::Internal(format!("Failed to add highlights: {}", e)))?; + + let (len, body) = body_from_bytes(highlighted_pdf).await?; + Ok(( + StatusCode::OK, + [ + (header::CONTENT_TYPE, "application/pdf".to_string()), + (header::CONTENT_DISPOSITION, content_disposition), + (header::CONTENT_LENGTH, len.to_string()), + ], + body, + )) +} + +const EXCEL_MAX_ROWS_PER_SHEET: usize = 10000; + +/// 获取 Excel 文件的结构化数据 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/{id}/excel-data", + operation_id = "file_get_excel_data", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID"), + ("sheet_name" = Option, Query, description = "指定 sheet 名称"), + ("row_num" = Option, Query, description = "指定 Excel 行号（1-based，包含表头行）"), + ), + responses( + (status = 200, description = "成功返回 Excel 数据", body = ExcelData), + (status = 400, description = "文件不是 Excel"), + (status = 404, description = "文件不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn excel_data( + State(pool): State, Path(id): Path, Extension(auth_user): Extension, + Query(query): Query, +) -> ApiResult> { + let file: File = sqlx::query_as(&format!("SELECT {} FROM files WHERE id = ?", FILE_COLS_NO_CONTENT)) + .bind(id) + .fetch_one(&pool) + .await?; + + ensure_file_readable(&file, &auth_user)?; + + let filename_lower = file.filename.to_lowercase(); + let is_excel = filename_lower.ends_with(".xls") || filename_lower.ends_with(".xlsx"); + if !is_excel { + return Err(ApiError::BadRequest("File is not an Excel document".to_string())); + } + + let filter_sheet = query.sheet_name.clone(); + let filter_row = query.row_num; + + let data = tokio::task::spawn_blocking(move || { + use calamine::{Reader, open_workbook_auto}; + + let path = std::path::Path::new(&file.path); + let mut workbook: calamine::Sheets> = + open_workbook_auto(path).map_err(|e| ApiError::Internal(format!("Failed to open Excel: {}", e)))?; + + let mut sheets = Vec::new(); + let sheet_names = workbook.sheet_names().to_vec(); + + for sheet_name in &sheet_names { + if let Some(ref target) = filter_sheet + && sheet_name != target + { + continue; + } + + let range = match workbook.worksheet_range(sheet_name) { + Ok(r) => r, + Err(_) => continue, + }; + + let all_rows: Vec<_> = range.rows().collect(); + if all_rows.is_empty() { + continue; + } + + let headers: Vec = all_rows[0] + .iter() + .enumerate() + .map(|(idx, cell)| { + let s = cell.to_string().trim().to_string(); + if s.is_empty() { format!("列{}", idx + 1) } else { s } + }) + .collect(); + + let data_rows = &all_rows[1..]; + let truncated = data_rows.len() > EXCEL_MAX_ROWS_PER_SHEET; + let limit = data_rows.len().min(EXCEL_MAX_ROWS_PER_SHEET); + + let rows: Vec> = if let Some(row_num) = filter_row { + if row_num == 0 || row_num > all_rows.len() { + return Err(ApiError::BadRequest(format!( + "Row number {} out of range for sheet '{}', valid range: 1-{}", + row_num, + sheet_name, + all_rows.len() + ))); + } + vec![all_rows[row_num - 1].iter().map(|cell| cell.to_string()).collect()] + } else { + data_rows[..limit].iter().map(|row| row.iter().map(|cell| cell.to_string()).collect()).collect() + }; + + sheets.push(ExcelSheetData { + name: sheet_name.clone(), + headers, + rows, + truncated: truncated && filter_row.is_none(), + }); + } + + Ok::<_, ApiError>(ExcelData { filename: file.filename, sheets }) + }) + .await + .map_err(|e| ApiError::Internal(format!("Excel parsing task failed: {}", e)))??; + + Ok(Json(data)) +} + +// ============================================================================ +// 压缩文件相关 API +// ============================================================================ + +/// 获取压缩文件内容列表 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/{id}/archive-entries", + operation_id = "file_archive_entries", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID") + ), + responses( + (status = 200, description = "成功返回压缩文件列表", body = Vec), + (status = 400, description = "不是压缩文件"), + (status = 404, description = "文件不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn archive_entries( + State(pool): State, Path(id): Path, Extension(auth_user): Extension, +) -> ApiResult>> { + let file: File = sqlx::query_as(&format!("SELECT {} FROM files WHERE id = ?", FILE_COLS_NO_CONTENT)) + .bind(id) + .fetch_one(&pool) + .await?; + + ensure_file_readable(&file, &auth_user)?; + + if !archive::is_archive_file(&file.filename) { + return Err(ApiError::BadRequest("File is not an archive".to_string())); + } + + let entries: Vec = sqlx::query_as( + "SELECT id, file_id, entry_path, size, is_directory FROM archive_entries WHERE file_id = ? ORDER BY entry_path", + ) + .bind(id) + .fetch_all(&pool) + .await?; + + Ok(Json(entries)) +} + +/// 解压压缩文件 +#[utoipa::path( + post, + path = "/api/v1/knowledge/files/{id}/archive-extract", + operation_id = "file_archive_extract", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID") + ), + request_body = ArchiveExtractReq, + responses( + (status = 200, description = "解压成功", body = ExtractResult), + (status = 400, description = "请求参数错误"), + (status = 404, description = "文件不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn archive_extract( + State(pool): State, Path(id): Path, Extension(auth_user): Extension, + Json(req): Json, +) -> ApiResult> { + let file: File = sqlx::query_as(&format!("SELECT {} FROM files WHERE id = ?", FILE_COLS_NO_CONTENT)) + .bind(id) + .fetch_one(&pool) + .await?; + + ensure_file_readable(&file, &auth_user)?; + + if !archive::is_archive_file(&file.filename) { + return Err(ApiError::BadRequest("File is not an archive".to_string())); + } + + let cfg = config::get(); + let dest_dir = std::path::Path::new(&cfg.storage.archives_path).join(id.to_string()); + + // 清理已有解压内容 + if let Err(e) = archive::cleanup_archive_dir(&cfg.storage.archives_path, id) { + warn!("Failed to cleanup existing archive dir for file {}: {}", id, e); + } + + // 清理数据库中已有记录 + sqlx::query("DELETE FROM archive_entries WHERE file_id = ?").bind(id).execute(&pool).await?; + + // 执行解压 + let password = req.password.clone(); + let file_path = file.path.clone(); + let filename = file.filename.clone(); + info!( + "archive_extract: starting extraction for file_id={}, path={:?}, filename={:?}, password_provided={}", + id, + file_path, + filename, + password.is_some() + ); + let entries = match tokio::task::spawn_blocking(move || { + archive::extract_archive(&file_path, dest_dir.to_string_lossy().as_ref(), &filename, password.as_deref(), id) + }) + .await + { + Ok(Ok(entries)) => { + info!("archive_extract: extraction succeeded for file_id={}, entries={}", id, entries.len()); + entries + } + Ok(Err(archive::ArchiveError::PasswordRequired)) => { + info!("archive_extract: password required for file_id={}", id); + return Ok(Json(ExtractResult { entries: vec![], needs_password: true })); + } + Ok(Err(e)) => { + warn!("archive_extract: extraction failed for file_id={}: {}", id, e); + return Err(ApiError::BadRequest(e.to_string())); + } + Err(e) => { + error!("archive_extract: spawn_blocking failed for file_id={}: {}", id, e); + return Err(ApiError::Internal(format!("Archive extraction task failed: {}", e))); + } + }; + + // 写入数据库：批量 INSERT 的多个 chunk 与末尾的 meta 更新合并进单个事务， + // 把多次提交合成一次 fsync，并保证 entries 与 meta 的原子性。 + let meta = serde_json::json!({ + "archive": { + "extracted": true, + "needs_password": false, + "entry_count": entries.len() + } + }); + + let mut tx = pool.begin().await?; + if !entries.is_empty() { + let binds_per_row = 4_usize; + let max_vars = 999_usize; + let batch_size = std::cmp::max(1, max_vars / binds_per_row); + for chunk in entries.chunks(batch_size) { + let mut chunk_qb = + QueryBuilder::::new("INSERT INTO archive_entries (file_id, entry_path, size, is_directory) "); + chunk_qb.push_values(chunk.iter(), |mut b, entry| { + b.push_bind(entry.file_id) + .push_bind(&entry.entry_path) + .push_bind(entry.size) + .push_bind(entry.is_directory); + }); + chunk_qb.build().execute(&mut *tx).await?; + } + } + sqlx::query("UPDATE files SET meta = ?, updated_at = strftime('%s','now') WHERE id = ?") + .bind(meta.to_string()) + .bind(id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(ExtractResult { entries, needs_password: false })) +} + +/// 下载压缩包内的单个文件 +#[utoipa::path( + get, + path = "/api/v1/knowledge/files/{id}/archive-download", + operation_id = "file_archive_download", + tag = "file", + params( + ("id" = i64, Path, description = "文件 ID"), + ArchiveDownloadQuery, + ), + responses( + (status = 200, description = "成功返回文件内容", content_type = "application/octet-stream"), + (status = 400, description = "请求参数错误"), + (status = 404, description = "文件不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn archive_download( + State(pool): State, Path(id): Path, Query(query): Query, + Extension(auth_user): Extension, +) -> Result<(StatusCode, [(header::HeaderName, String); 3], Body), ApiError> { + let file: File = sqlx::query_as(&format!("SELECT {} FROM files WHERE id = ?", FILE_COLS_NO_CONTENT)) + .bind(id) + .fetch_one(&pool) + .await?; + + ensure_file_readable(&file, &auth_user)?; + + if !archive::is_archive_file(&file.filename) { + return Err(ApiError::BadRequest("File is not an archive".to_string())); + } + + // 安全检查：防止目录遍历 + let entry_path = query.path.trim(); + if entry_path.is_empty() || entry_path.contains("..") || entry_path.starts_with('/') || entry_path.starts_with('\\') + { + return Err(ApiError::BadRequest("Invalid entry path".to_string())); + } + + let cfg = config::get(); + let mime_type = mime_guess::from_path(entry_path).first_or_octet_stream().to_string(); + let filename = std::path::Path::new(entry_path).file_name().and_then(|n| n.to_str()).unwrap_or(entry_path); + let content_disposition = format!("attachment; filename=\"{}\"", filename); + + // 尝试从解压目录读取 + if let Some(resolved) = archive::resolve_archive_entry_path(&cfg.storage.archives_path, id, entry_path) + && resolved.exists() + && resolved.is_file() + { + let (len, body) = open_file_stream(&resolved).await?; + return Ok(( + StatusCode::OK, + [ + (header::CONTENT_TYPE, mime_type), + (header::CONTENT_DISPOSITION, content_disposition), + (header::CONTENT_LENGTH, len.to_string()), + ], + body, + )); + } + + // 如果解压目录没有，尝试直接从压缩包读取（ZIP/TAR 支持） + let src_path = file.path.clone(); + let filename = file.filename.clone(); + let entry_path_owned = entry_path.to_string(); + let temp = match tokio::task::spawn_blocking(move || { + archive::read_archive_entry(&src_path, &filename, &entry_path_owned, None) + }) + .await + { + Ok(Ok(temp)) => temp, + Ok(Err(archive::ArchiveError::PasswordRequired)) => { + return Err(ApiError::BadRequest("Archive is password protected".to_string())); + } + Ok(Err(e)) => return Err(ApiError::NotFound(e.to_string())), + Err(e) => return Err(ApiError::Internal(format!("Archive read task failed: {}", e))), + }; + + let (len, body) = stream_from_std_file(temp.into_file())?; + Ok(( + StatusCode::OK, + [ + (header::CONTENT_TYPE, mime_type), + (header::CONTENT_DISPOSITION, content_disposition), + (header::CONTENT_LENGTH, len.to_string()), + ], + body, + )) +} diff --git a/src/api/graph.rs b/src/api/graph.rs new file mode 100644 index 0000000..64a8527 --- /dev/null +++ b/src/api/graph.rs @@ -0,0 +1,377 @@ +use std::collections::HashMap; + +use axum::{ + Extension, extract::{Path, Query, State}, response::Json +}; +use serde::{Deserialize, Serialize}; +use sqlx::SqlitePool; +use utoipa::{IntoParams, ToSchema}; + +use crate::{AuthUser, api::error::ApiError}; + +// Type aliases for complex SQL row types +#[allow(clippy::type_complexity)] +type EntityRow = (i64, String, String, Option, Option, Option, i64); +#[allow(clippy::type_complexity)] +type NeighborRow = (i64, String, String, Option, Option, Option, i64, String); +type MentionRow = (i64, String, i64, String); + +/// 实体信息 +#[derive(Debug, Serialize, ToSchema)] +pub struct EntityInfo { + pub id: i64, + pub name: String, + pub entity_type: String, + pub properties: HashMap, + pub file_id: Option, + pub kb_id: Option, + pub created_at: i64, +} + +/// 实体搜索参数 +#[derive(Debug, Deserialize, IntoParams)] +pub struct EntitySearchParams { + pub q: Option, // 搜索关键词 + pub entity_type: Option, // 实体类型筛选 + pub kb_id: Option, // 知识库ID筛选 + pub file_id: Option, // 文件ID筛选 + pub limit: Option, // 返回数量限制 +} + +/// 实体查询 +#[utoipa::path( + get, + path = "/api/v1/knowledge/graph/entities", + operation_id = "graph_search_entities", + tag = "graph", + params(EntitySearchParams), + responses( + (status = 200, description = "成功返回实体列表", body = Vec), + (status = 400, description = "请求参数错误") + ) +)] +pub async fn search_entities( + Query(params): Query, State(pool): State, Extension(auth_user): Extension, +) -> Result>, ApiError> { + let is_admin = auth_user.is_admin(); + + // Check KB permission if kb_id is specified + if let Some(kb_id) = params.kb_id { + let perm = crate::api::knowledge_base::get_kb_permission(&pool, kb_id, &auth_user.user_id, is_admin).await; + if !crate::api::knowledge_base::meets_requirement(perm.as_deref(), "viewer") { + return Err(ApiError::Forbidden("Permission denied.".to_string())); + } + } + + let mut sql = + "SELECT id, name, entity_type, properties, file_id, kb_id, created_at FROM graph_nodes WHERE 1=1".to_string(); + + // 构建查询条件 + if params.q.is_some() { + sql.push_str(" AND name LIKE '%' || ? || '%'"); + } + if params.entity_type.is_some() { + sql.push_str(" AND entity_type = ?"); + } + if params.kb_id.is_some() { + sql.push_str(" AND kb_id = ?"); + } + if params.file_id.is_some() { + sql.push_str(" AND file_id = ?"); + } + + sql.push_str(" ORDER BY created_at DESC"); + + if let Some(limit) = params.limit { + sql.push_str(&format!(" LIMIT {}", limit)); + } else { + sql.push_str(" LIMIT 100"); + } + + // 执行查询 + let mut query = sqlx::query_as::<_, (i64, String, String, Option, Option, Option, i64)>(&sql); + + if let Some(ref q) = params.q { + query = query.bind(q); + } + if let Some(ref entity_type) = params.entity_type { + query = query.bind(entity_type); + } + if let Some(kb_id) = params.kb_id { + query = query.bind(kb_id); + } + if let Some(file_id) = params.file_id { + query = query.bind(file_id); + } + + let rows = query.fetch_all(&pool).await?; + + // Preload accessible KB ids for filtering + let allowed_kb_ids: std::collections::HashSet = if is_admin { + std::collections::HashSet::new() + } else { + crate::api::knowledge_base::get_user_viewable_kb_ids(&pool, &auth_user.user_id, false) + .await + .into_iter() + .collect() + }; + + let entities: Vec = rows + .into_iter() + .filter_map(|(id, name, entity_type, properties_json, file_id, kb_id, created_at)| { + // Filter by KB permission + if let Some(kid) = kb_id + && !is_admin + && !allowed_kb_ids.contains(&kid) + { + return None; + } + let properties: HashMap = if let Some(ref json) = properties_json { + serde_json::from_str(json).unwrap_or_default() + } else { + HashMap::new() + }; + + Some(EntityInfo { id, name, entity_type, properties, file_id, kb_id, created_at }) + }) + .collect(); + + Ok(Json(entities)) +} + +/// 实体详情（包含邻居和提及） +#[derive(Debug, Serialize, ToSchema)] +pub struct EntityDetail { + pub entity: EntityInfo, + pub neighbors: Vec, + pub mentions: Vec, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct NeighborInfo { + pub entity: EntityInfo, + pub relation_type: String, + pub direction: String, // "outgoing" or "incoming" +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct MentionInfo { + pub slice_id: i64, + pub context: String, + pub file_id: i64, + pub filename: String, +} + +/// 获取实体详情 +#[utoipa::path( + get, + path = "/api/v1/knowledge/graph/entities/{id}", + operation_id = "graph_get_entity", + tag = "graph", + params( + ("id" = i64, Path, description = "实体 ID") + ), + responses( + (status = 200, description = "成功返回实体详情", body = EntityDetail), + (status = 404, description = "实体不存在") + ) +)] +pub async fn get_entity( + Path(id): Path, State(pool): State, Extension(auth_user): Extension, +) -> Result, ApiError> { + let is_admin = auth_user.is_admin(); + // 查询实体基本信息 + let entity_sql = + "SELECT id, name, entity_type, properties, file_id, kb_id, created_at FROM graph_nodes WHERE id = ?"; + let entity_row: Option = sqlx::query_as(entity_sql).bind(id).fetch_optional(&pool).await?; + + let entity_row = entity_row.ok_or_else(|| ApiError::Internal("Entity not found".to_string()))?; + + // Check permission on the entity's KB + if let Some(kb_id) = entity_row.5 { + let perm = crate::api::knowledge_base::get_kb_permission(&pool, kb_id, &auth_user.user_id, is_admin).await; + if !crate::api::knowledge_base::meets_requirement(perm.as_deref(), "viewer") { + return Err(ApiError::Forbidden("Permission denied.".to_string())); + } + } + + let properties: HashMap = + if let Some(ref json) = entity_row.3 { serde_json::from_str(json).unwrap_or_default() } else { HashMap::new() }; + + let entity = EntityInfo { + id: entity_row.0, + name: entity_row.1, + entity_type: entity_row.2, + properties, + file_id: entity_row.4, + kb_id: entity_row.5, + created_at: entity_row.6, + }; + + // 出边、入边、提及三条查询相互独立，并发执行 + let outgoing_sql = "SELECT n.id, n.name, n.entity_type, n.properties, n.file_id, n.kb_id, n.created_at, e.relation_type \ + FROM graph_edges e \ + INNER JOIN graph_nodes n ON e.target_node_id = n.id \ + WHERE e.source_node_id = ? LIMIT 50"; + let incoming_sql = "SELECT n.id, n.name, n.entity_type, n.properties, n.file_id, n.kb_id, n.created_at, e.relation_type \ + FROM graph_edges e \ + INNER JOIN graph_nodes n ON e.source_node_id = n.id \ + WHERE e.target_node_id = ? LIMIT 50"; + let mentions_sql = "SELECT m.slice_id, m.context, s.file_id, f.filename \ + FROM entity_mentions m \ + INNER JOIN slices s ON m.slice_id = s.id \ + INNER JOIN files f ON s.file_id = f.id \ + WHERE m.node_id = ? LIMIT 20"; + + let (outgoing_rows, incoming_rows, mentions_rows): (Vec, Vec, Vec) = tokio::try_join!( + sqlx::query_as(outgoing_sql).bind(id).fetch_all(&pool), + sqlx::query_as(incoming_sql).bind(id).fetch_all(&pool), + sqlx::query_as(mentions_sql).bind(id).fetch_all(&pool), + )?; + + let mut neighbors = Vec::new(); + + for row in outgoing_rows { + let properties: HashMap = + if let Some(ref json) = row.3 { serde_json::from_str(json).unwrap_or_default() } else { HashMap::new() }; + + neighbors.push(NeighborInfo { + entity: EntityInfo { + id: row.0, + name: row.1, + entity_type: row.2, + properties, + file_id: row.4, + kb_id: row.5, + created_at: row.6, + }, + relation_type: row.7, + direction: "outgoing".to_string(), + }); + } + + for row in incoming_rows { + let properties: HashMap = + if let Some(ref json) = row.3 { serde_json::from_str(json).unwrap_or_default() } else { HashMap::new() }; + + neighbors.push(NeighborInfo { + entity: EntityInfo { + id: row.0, + name: row.1, + entity_type: row.2, + properties, + file_id: row.4, + kb_id: row.5, + created_at: row.6, + }, + relation_type: row.7, + direction: "incoming".to_string(), + }); + } + + let mentions: Vec = mentions_rows + .into_iter() + .map(|(slice_id, context, file_id, filename)| MentionInfo { slice_id, context, file_id, filename }) + .collect(); + + Ok(Json(EntityDetail { entity, neighbors, mentions })) +} + +/// 图统计信息 +#[derive(Debug, Serialize, ToSchema)] +pub struct GraphStats { + pub node_count: i64, + pub edge_count: i64, + pub entity_types: HashMap, + pub relation_types: HashMap, +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct StatsParams { + pub kb_id: Option, + pub file_id: Option, +} + +/// 获取图统计信息 +#[utoipa::path( + get, + path = "/api/v1/knowledge/graph/stats", + operation_id = "graph_get_graph_stats", + tag = "graph", + params(StatsParams), + responses( + (status = 200, description = "成功返回图统计信息", body = GraphStats), + (status = 400, description = "请求参数错误") + ) +)] +pub async fn get_graph_stats( + Query(params): Query, State(pool): State, Extension(auth_user): Extension, +) -> Result, ApiError> { + let is_admin = auth_user.is_admin(); + + // Check KB permission if kb_id is specified + if let Some(kb_id) = params.kb_id { + let perm = crate::api::knowledge_base::get_kb_permission(&pool, kb_id, &auth_user.user_id, is_admin).await; + if !crate::api::knowledge_base::meets_requirement(perm.as_deref(), "viewer") { + return Err(ApiError::Forbidden("Permission denied.".to_string())); + } + } + + // 构建 WHERE 子句 + let where_clause = if let Some(file_id) = params.file_id { + format!("WHERE file_id = {}", file_id) + } else if let Some(kb_id) = params.kb_id { + format!("WHERE kb_id = {}", kb_id) + } else { + "".to_string() + }; + + // 节点数量 + let node_count_sql = format!("SELECT COUNT(*) FROM graph_nodes {}", where_clause); + let node_count: (i64,) = sqlx::query_as(&node_count_sql).fetch_one(&pool).await?; + + // 边数量 + let edge_count_sql = if let Some(file_id) = params.file_id { + format!("SELECT COUNT(*) FROM graph_edges WHERE file_id = {}", file_id) + } else if let Some(kb_id) = params.kb_id { + format!( + "SELECT COUNT(*) FROM graph_edges e \ + INNER JOIN graph_nodes n ON e.source_node_id = n.id \ + WHERE n.kb_id = {}", + kb_id + ) + } else { + "SELECT COUNT(*) FROM graph_edges".to_string() + }; + + let edge_count: (i64,) = sqlx::query_as(&edge_count_sql).fetch_one(&pool).await?; + + // 实体类型分布 + let entity_types_sql = + format!("SELECT entity_type, COUNT(*) FROM graph_nodes {} GROUP BY entity_type", where_clause); + let entity_types_rows: Vec<(String, i64)> = sqlx::query_as(&entity_types_sql).fetch_all(&pool).await?; + let entity_types: HashMap = entity_types_rows.into_iter().collect(); + + // 关系类型分布 + let relation_types_sql = if let Some(file_id) = params.file_id { + format!( + "SELECT relation_type, COUNT(*) FROM graph_edges \ + WHERE file_id = {} GROUP BY relation_type", + file_id + ) + } else if let Some(kb_id) = params.kb_id { + format!( + "SELECT e.relation_type, COUNT(*) FROM graph_edges e \ + INNER JOIN graph_nodes n ON e.source_node_id = n.id \ + WHERE n.kb_id = {} GROUP BY e.relation_type", + kb_id + ) + } else { + "SELECT relation_type, COUNT(*) FROM graph_edges GROUP BY relation_type".to_string() + }; + + let relation_types_rows: Vec<(String, i64)> = sqlx::query_as(&relation_types_sql).fetch_all(&pool).await?; + let relation_types: HashMap = relation_types_rows.into_iter().collect(); + + Ok(Json(GraphStats { node_count: node_count.0, edge_count: edge_count.0, entity_types, relation_types })) +} diff --git a/src/api/knowledge_base.rs b/src/api/knowledge_base.rs new file mode 100644 index 0000000..506f7b2 --- /dev/null +++ b/src/api/knowledge_base.rs @@ -0,0 +1,1818 @@ +use std::collections::HashMap; + +use axum::{ + Extension, + extract::{Path, Query, State}, + response::Json, +}; +use log::warn; +use serde::{Deserialize, Serialize}; +use sqlx::{QueryBuilder, Row, Sqlite, SqlitePool}; +use utoipa::{IntoParams, ToSchema}; + +use super::file::{self, FileStatusBreakdown}; +use crate::{ + AuthUser, + api::{ + common, + error::{ApiError, ApiResult}, + }, + search::SearchEngine, +}; + +pub(crate) const KB_TYPE_ANALYSIS: &str = "analysis"; +pub(crate) const KB_TYPE_STORAGE: &str = "storage"; + +fn normalize_kb_type(kb_type: Option) -> Result { + let raw = kb_type.unwrap_or_else(|| KB_TYPE_ANALYSIS.to_string()); + let normalized = raw.trim().to_lowercase(); + match normalized.as_str() { + KB_TYPE_ANALYSIS | KB_TYPE_STORAGE => Ok(normalized), + _ => Err(ApiError::BadRequest("Invalid kb_type. Use 'analysis' or 'storage'.".to_string())), + } +} + +fn normalize_parse_priority(parse_priority: Option) -> Result { + let value = parse_priority.unwrap_or(50); + if !(0..=100).contains(&value) { + return Err(ApiError::BadRequest("Invalid parse_priority. Use integer in [0, 100].".to_string())); + } + Ok(value) +} + +// --------------------------------------------------------------------------- +// KB permission helpers +// --------------------------------------------------------------------------- + +/// Get the highest permission level a user has on a knowledge base. +/// Priority: global admin > owner > explicit permission > is_public. +/// Returns None if the user has no access at all. +pub async fn get_kb_permission(pool: &SqlitePool, kb_id: i64, user_id: &str, is_admin: bool) -> Option { + if is_admin { + return Some("admin".to_string()); + } + + // 1. Check if owner + let owner: Option = sqlx::query_scalar("SELECT user_id FROM knowledge_bases WHERE id = ?") + .bind(kb_id) + .fetch_optional(pool) + .await + .ok() + .flatten(); + if owner.as_deref() == Some(user_id) { + return Some("admin".to_string()); + } + + // 2. Check explicit permission + let explicit: Option = + sqlx::query_scalar("SELECT permission FROM kb_permissions WHERE kb_id = ? AND user_id = ?") + .bind(kb_id) + .bind(user_id) + .fetch_optional(pool) + .await + .ok() + .flatten(); + if let Some(perm) = explicit { + return Some(perm); + } + + // 3. Check if public + let is_public: Option = sqlx::query_scalar("SELECT is_public FROM knowledge_bases WHERE id = ?") + .bind(kb_id) + .fetch_optional(pool) + .await + .ok() + .flatten(); + if is_public == Some(1) { + return Some("viewer".to_string()); + } + + None +} + +/// Batch version of [`get_kb_permission`]: resolve permissions for many KBs in a fixed +/// number of queries (2 instead of 3*N), avoiding the N+1 pattern in bulk operations. +/// +/// Returns a map kb_id -> highest permission. KBs the user cannot access are absent from the map. +pub async fn get_kb_permissions_batch( + pool: &SqlitePool, kb_ids: &[i64], user_id: &str, is_admin: bool, +) -> HashMap { + let mut result = HashMap::new(); + if kb_ids.is_empty() { + return result; + } + + if is_admin { + for &id in kb_ids { + result.insert(id, "admin".to_string()); + } + return result; + } + + // Dedupe ids to keep the IN lists small. + let unique_ids: Vec = { + let set: std::collections::HashSet = kb_ids.iter().copied().collect(); + set.into_iter().collect() + }; + + // Query 1: owner + is_public for each KB. + let mut kb_qb = QueryBuilder::::new("SELECT id, user_id, is_public FROM knowledge_bases WHERE id IN ("); + { + let mut sep = kb_qb.separated(", "); + for id in &unique_ids { + sep.push_bind(*id); + } + } + kb_qb.push(")"); + let kb_rows: Vec<(i64, String, i64)> = kb_qb.build_query_as().fetch_all(pool).await.unwrap_or_default(); + + // Query 2: explicit permissions for this user. + let mut perm_qb = QueryBuilder::::new("SELECT kb_id, permission FROM kb_permissions WHERE user_id = "); + perm_qb.push_bind(user_id); + perm_qb.push(" AND kb_id IN ("); + { + let mut sep = perm_qb.separated(", "); + for id in &unique_ids { + sep.push_bind(*id); + } + } + perm_qb.push(")"); + let perm_rows: Vec<(i64, String)> = perm_qb.build_query_as().fetch_all(pool).await.unwrap_or_default(); + let explicit: HashMap = perm_rows.into_iter().collect(); + + for (id, owner, is_public) in kb_rows { + // Priority: owner > explicit permission > is_public. + let perm = if owner == user_id { + Some("admin".to_string()) + } else if let Some(p) = explicit.get(&id) { + Some(p.clone()) + } else if is_public == 1 { + Some("viewer".to_string()) + } else { + None + }; + if let Some(perm) = perm { + result.insert(id, perm); + } + } + + result +} + +/// Numeric level for comparison: admin=3, editor=2, viewer=1, none=0. +pub fn perm_level(perm: &str) -> i32 { + match perm { + "admin" => 3, + "editor" => 2, + "viewer" => 1, + _ => 0, + } +} + +/// Check whether `actual` permission meets at least `required` permission. +pub fn meets_requirement(actual: Option<&str>, required: &str) -> bool { + let actual_level = actual.map(perm_level).unwrap_or(0); + let required_level = perm_level(required); + actual_level >= required_level +} + +/// Get all KB ids that the user has at least viewer access to. +pub async fn get_user_viewable_kb_ids(pool: &SqlitePool, user_id: &str, is_admin: bool) -> Vec { + if is_admin { + return sqlx::query_scalar("SELECT id FROM knowledge_bases").fetch_all(pool).await.unwrap_or_default(); + } + sqlx::query_scalar( + "SELECT id FROM knowledge_bases WHERE user_id = ?1 OR is_public = 1 \ + UNION \ + SELECT kb_id FROM kb_permissions WHERE user_id = ?1", + ) + .bind(user_id) + .fetch_all(pool) + .await + .unwrap_or_default() +} + +// --------------------------------------------------------------------------- +// Data models +// --------------------------------------------------------------------------- + +#[derive(Serialize, Deserialize, Clone, Debug, sqlx::FromRow, ToSchema)] +pub struct Knowledge { + pub id: i64, + pub user_id: String, + pub user_name: String, + pub name: String, + pub description: String, + pub kb_type: String, + pub parent_id: Option, + pub is_public: bool, + pub parse_priority: i64, + #[serde(default)] + #[sqlx(default)] + pub current_user_permission: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] +pub struct KnowledgeResponse { + pub id: i64, + pub user_id: String, + pub user_name: String, + pub name: String, + pub description: String, + pub kb_type: String, + pub parent_id: Option, + pub is_public: bool, + pub parse_priority: i64, + pub file_count: i64, + pub children_kb_count: i64, + pub current_user_permission: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] +pub struct KnowledgeDetailResponse { + pub id: i64, + pub user_id: String, + pub user_name: String, + pub name: String, + pub description: String, + pub kb_type: String, + pub parent_id: Option, + pub is_public: bool, + pub parse_priority: i64, + pub file_count: i64, + pub children_kb_count: i64, + pub children_kbs: Vec, + pub path: Vec, // For breadcrumbs + pub status_breakdown: FileStatusBreakdown, + pub current_user_permission: String, +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct KnowledgeBaseFilesQuery { + /// 页码，从1开始 + pub page: Option, + /// 每页条数 + pub size: Option, + /// 文件名模糊搜索（%filename%） + pub filename: Option, + /// 根据标签筛选 + pub tag: Option, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct KnowledgeBaseFilesResponse { + pub total: i64, + pub items: Vec, +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct KnowledgeBaseTagsQuery { + /// 是否包含当前用户可访问的子知识库，默认 false + pub include_descendants: Option, +} + +#[derive(Debug, Serialize, sqlx::FromRow, ToSchema)] +pub struct TagFileCount { + pub tag: String, + pub file_count: i64, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct KnowledgeBaseTagsResponse { + pub kb_id: i64, + pub tags: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug, sqlx::FromRow, ToSchema)] +pub struct FileWithoutContent { + pub id: i64, + pub user_id: String, + pub user_name: String, + pub hash: String, + pub filename: String, + pub path: String, + pub size: i64, + pub tags: String, + pub status: i32, + pub log: String, + pub slice_type: String, + pub kb_id: Option, + pub is_public: bool, + pub meta: Option, + pub summary: Option, + pub created_at: i64, + pub updated_at: i64, +} + +#[derive(Serialize, Deserialize, Clone, Debug, sqlx::FromRow, ToSchema)] +pub struct KnowledgeTreeFile { + pub id: i64, + pub size: i64, + pub filename: String, + pub meta: Option, + pub kb_id: Option, + pub is_public: bool, +} + +#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] +pub struct KnowledgeTreeNode { + pub id: i64, + pub name: String, + pub description: String, + pub is_public: bool, + pub kb_type: String, + pub files: Vec, + #[schema(no_recursion)] + pub children: Vec, +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct ListQuery { + /// 页码，从1开始 + pub page: Option, + /// 每页条数 + pub size: Option, + // /// 关键词搜索信息（在 name + description 中搜索） + // pub keyword: Option, + /// 模糊搜索 name 字段 + pub name: Option, + /// 知识库 ID（精确匹配） + pub id: Option, + /// 根据父知识库ID筛选，若不传则获取顶级知识库 + pub parent_id: Option, +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct TreeQuery { + /// 知识库 ID（可选），传入则返回该知识库的子树，不传则返回完整树 + pub kb_id: Option, +} + +/// 获取知识库列表 +#[utoipa::path( + get, + path = "/api/v1/knowledge/knowledge_base/", + operation_id = "knowledge_base_list", + tag = "knowledge_base", + params(ListQuery), + responses( + (status = 200, description = "成功返回知识库列表", body = Vec), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn list( + State(pool): State, Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult>> { + let is_admin = auth_user.is_admin(); + // Determine pagination: default size 10, default page 1 + let size = params.size.unwrap_or(10).max(1); + let page = params.page.unwrap_or(1).max(1); + let limit = size; + let offset = (page - 1) * size; + + // Start building the query + let mut qb = QueryBuilder::::new( + "SELECT id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority FROM knowledge_bases WHERE 1=1 ", + ); + if !is_admin { + let user_id = auth_user.user_id.clone(); + qb.push(" AND (user_id = ") + .push_bind(user_id.clone()) + .push(" OR is_public = 1 OR id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ") + .push_bind(user_id) + .push("))"); + } + + // Filter by parent_id + if let Some(parent_id) = params.parent_id { + qb.push(" AND parent_id = ").push_bind(parent_id); + } else { + qb.push(" AND parent_id IS NULL"); + } + + // If `id` provided, try to parse as integer id and filter by id + if let Some(id_str) = params.id.as_deref() { + qb.push(" AND id = ").push_bind(id_str); + } + + // name fuzzy search (only name column) + if let Some(name) = ¶ms.name { + qb.push("AND name LIKE ").push_bind(format!("%{}%", name)); + } + + // ordering and pagination + qb.push(" ORDER BY id"); + qb.push(" LIMIT ").push_bind(limit); + qb.push(" OFFSET ").push_bind(offset); + + // Execute + let query = qb.build_query_as::(); + let knowledges = query.fetch_all(&pool).await?; + let knowledge_ids: Vec = knowledges.iter().map(|kb| kb.id).collect(); + + // Get file counts and children counts in parallel + let (file_counts_res, children_counts_res) = tokio::join!( + get_file_counts(&pool, &knowledge_ids, &auth_user.user_id, is_admin), + get_children_kb_counts(&pool, &knowledge_ids, &auth_user.user_id, is_admin) + ); + let file_counts = file_counts_res?; + let children_counts = children_counts_res?; + + let knowledge_responses = knowledges + .into_iter() + .map(|kb| KnowledgeResponse { + id: kb.id, + user_id: kb.user_id.clone(), + user_name: kb.user_name.clone(), + name: kb.name.clone(), + description: kb.description.clone(), + kb_type: kb.kb_type.clone(), + parent_id: kb.parent_id, + is_public: kb.is_public, + parse_priority: kb.parse_priority, + file_count: *file_counts.get(&kb.id).unwrap_or(&0), + children_kb_count: *children_counts.get(&kb.id).unwrap_or(&0), + current_user_permission: if kb.user_id == auth_user.user_id || is_admin { + "admin".to_string() + } else if kb.is_public { + "viewer".to_string() + } else { + // fallback - should not happen since query already filters + "viewer".to_string() + }, + }) + .collect(); + + Ok(Json(knowledge_responses)) +} + +async fn get_children_kb_counts( + pool: &SqlitePool, knowledge_ids: &[i64], user_id: &str, is_admin: bool, +) -> anyhow::Result> { + if knowledge_ids.is_empty() { + return Ok(HashMap::new()); + } + + let mut qb = QueryBuilder::::new( + "WITH RECURSIVE descendants(root_id, kb_id) AS (SELECT id AS root_id, id AS kb_id FROM knowledge_bases WHERE id IN (", + ); + let mut separated = qb.separated(", "); + for id in knowledge_ids { + separated.push_bind(id); + } + qb.push(")"); + if !is_admin { + common::push_kb_access_filter(&mut qb, user_id); + } + qb.push(" UNION ALL SELECT d.root_id, kb.id FROM knowledge_bases kb "); + qb.push("JOIN descendants d ON kb.parent_id = d.kb_id"); + if !is_admin { + common::push_kb_access_filter_where(&mut qb, user_id); + } + qb.push(") "); + qb.push("SELECT root_id, COUNT(*) - 1 AS cnt FROM descendants GROUP BY root_id"); + + let rows = qb.build().fetch_all(pool).await?; + + let children_counts = rows + .into_iter() + .filter_map(|row| { + let root_id: Option = row.get("root_id"); + let cnt: i64 = row.get("cnt"); + root_id.map(|id| (id, cnt)) + }) + .collect(); + + Ok(children_counts) +} + +async fn get_file_counts( + pool: &SqlitePool, knowledge_ids: &[i64], user_id: &str, is_admin: bool, +) -> anyhow::Result> { + if knowledge_ids.is_empty() { + return Ok(HashMap::new()); + } + + let mut qb = QueryBuilder::::new( + "WITH RECURSIVE descendants(root_id, kb_id) AS (SELECT id AS root_id, id AS kb_id FROM knowledge_bases WHERE id IN (", + ); + let mut separated = qb.separated(", "); + for id in knowledge_ids { + separated.push_bind(id); + } + qb.push(")"); + if !is_admin { + common::push_kb_access_filter(&mut qb, user_id); + } + qb.push(" UNION ALL SELECT d.root_id, kb.id FROM knowledge_bases kb "); + qb.push("JOIN descendants d ON kb.parent_id = d.kb_id"); + if !is_admin { + common::push_kb_access_filter_where(&mut qb, user_id); + } + qb.push(") "); + qb.push("SELECT d.root_id, COUNT(f.id) AS cnt FROM descendants d "); + qb.push("LEFT JOIN files f ON f.kb_id = d.kb_id"); + if !is_admin { + qb.push(" AND (f.user_id = "); + qb.push_bind(user_id); + qb.push(" OR f.is_public = 1)"); + } + qb.push(" GROUP BY d.root_id"); + + let rows = qb.build().fetch_all(pool).await?; + + let file_counts = rows + .into_iter() + .filter_map(|row| { + let root_id: Option = row.get("root_id"); + let cnt: i64 = row.get("cnt"); + root_id.map(|id| (id, cnt)) + }) + .collect(); + + Ok(file_counts) +} + +#[derive(Deserialize, ToSchema)] +pub struct KnowledgeCreateReq { + pub name: String, + pub description: String, + pub kb_type: Option, + pub parent_id: Option, + pub is_public: Option, + pub parse_priority: Option, +} + +/// 创建知识库 +#[utoipa::path( + post, + path = "/api/v1/knowledge/knowledge_base/", + operation_id = "knowledge_base_create", + tag = "knowledge_base", + request_body = KnowledgeCreateReq, + responses( + (status = 200, description = "成功创建知识库", body = Knowledge), + (status = 400, description = "请求参数错误"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn create( + State(pool): State, Extension(auth_user): Extension, + Json(knowledge): Json, +) -> ApiResult> { + let is_public = if knowledge.is_public.unwrap_or(false) { 1 } else { 0 }; + let kb_type = normalize_kb_type(knowledge.kb_type)?; + let parse_priority = normalize_parse_priority(knowledge.parse_priority)?; + let query = "INSERT INTO knowledge_bases (user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + let id = sqlx::query(query) + .bind(auth_user.user_id) + .bind(auth_user.user_name) + .bind(knowledge.name) + .bind(knowledge.description.clone()) + .bind(kb_type) + .bind(knowledge.parent_id) + .bind(is_public) + .bind(parse_priority) + .execute(&pool) + .await? + .last_insert_rowid(); + let mut kb: Knowledge = sqlx::query_as( + "SELECT id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority FROM knowledge_bases WHERE id = ?", + ) + .bind(id) + .fetch_one(&pool) + .await?; + kb.current_user_permission = "admin".to_string(); + Ok(Json(kb)) +} + +#[derive(Deserialize, ToSchema)] +pub struct KnowledgeUpdateReq { + pub name: Option, + pub description: Option, + pub kb_type: Option, + #[serde(default, with = "::serde_with::rust::double_option")] + pub parent_id: Option>, + pub is_public: Option, + pub parse_priority: Option, +} + +/// 更新知识库 +#[utoipa::path( + put, + path = "/api/v1/knowledge/knowledge_base/{id}", + operation_id = "knowledge_base_update", + tag = "knowledge_base", + params( + ("id" = i64, Path, description = "知识库 ID") + ), + request_body = KnowledgeUpdateReq, + responses( + (status = 200, description = "成功更新知识库", body = Knowledge), + (status = 400, description = "请求参数错误"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn update( + Path(id): Path, State(pool): State, Extension(auth_user): Extension, + Json(knowledge): Json, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + let user_perm = get_kb_permission(&pool, id, &auth_user.user_id, is_admin).await; + let perm_str = user_perm.as_deref().unwrap_or(""); + + if !meets_requirement(user_perm.as_deref(), "editor") { + return Err(ApiError::Forbidden("Permission denied. Requires editor or admin.".to_string())); + } + + // Prevent moving a knowledge base into itself. + if let Some(Some(parent_id)) = knowledge.parent_id + && parent_id == id + { + return Err(crate::api::error::ApiError::BadRequest("Cannot move a knowledge base into itself.".to_string())); + } + // A full descendant check would be needed for production to prevent moving a KB into its own child. + // This requires a recursive query and is omitted for this iteration. + + // Only admin-level can change sensitive fields: is_public, parent_id, kb_type + let is_kb_admin = meets_requirement(Some(perm_str), "admin"); + if let Some(ref _kb_type) = knowledge.kb_type + && !is_kb_admin + { + return Err(ApiError::Forbidden("Only admin can change kb_type.".to_string())); + } + if knowledge.parent_id.is_some() && !is_kb_admin { + return Err(ApiError::Forbidden("Only admin can change parent_id.".to_string())); + } + if knowledge.is_public.is_some() && !is_kb_admin { + return Err(ApiError::Forbidden("Only admin can change visibility.".to_string())); + } + + let mut qb = QueryBuilder::::new("UPDATE knowledge_bases SET "); + let mut has_update = false; + + if let Some(name) = knowledge.name { + if has_update { + qb.push(", "); + } + qb.push("name = "); + qb.push_bind(name); + has_update = true; + } + if let Some(description) = knowledge.description { + if has_update { + qb.push(", "); + } + qb.push("description = "); + qb.push_bind(description); + has_update = true; + } + if let Some(kb_type) = knowledge.kb_type { + let kb_type = normalize_kb_type(Some(kb_type))?; + if has_update { + qb.push(", "); + } + qb.push("kb_type = "); + qb.push_bind(kb_type); + has_update = true; + } + // With double_option, this correctly distinguishes "not present" from "present and null" + if let Some(parent_id) = knowledge.parent_id { + if has_update { + qb.push(", "); + } + qb.push("parent_id = "); + qb.push_bind(parent_id); // This binds Option which sqlx handles (None becomes NULL) + has_update = true; + } + if let Some(is_public) = knowledge.is_public { + if has_update { + qb.push(", "); + } + qb.push("is_public = "); + qb.push_bind(if is_public { 1 } else { 0 }); + has_update = true; + } + if let Some(parse_priority) = knowledge.parse_priority { + let parse_priority = normalize_parse_priority(Some(parse_priority))?; + if has_update { + qb.push(", "); + } + qb.push("parse_priority = "); + qb.push_bind(parse_priority); + has_update = true; + } + + if !has_update { + let mut kb: Knowledge = sqlx::query_as( + "SELECT id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority FROM knowledge_bases WHERE id = ?", + ) + .bind(id) + .fetch_one(&pool) + .await?; + kb.current_user_permission = "admin".to_string(); + return Ok(Json(kb)); + } + + qb.push(" WHERE id = "); + qb.push_bind(id); + + let result = qb.build().execute(&pool).await?; + + if result.rows_affected() == 0 { + return Err(crate::api::error::ApiError::NotFound( + "Knowledge base not found or permission denied.".to_string(), + )); + } + + let mut kb: Knowledge = sqlx::query_as( + "SELECT id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority FROM knowledge_bases WHERE id = ?", + ) + .bind(id) + .fetch_one(&pool) + .await?; + kb.current_user_permission = "admin".to_string(); + Ok(Json(kb)) +} + +/// 获取知识库详情 +#[utoipa::path( + get, + path = "/api/v1/knowledge/knowledge_base/{id}", + operation_id = "knowledge_base_get", + tag = "knowledge_base", + params( + ("id" = i64, Path, description = "知识库 ID") + ), + responses( + (status = 200, description = "成功返回知识库详情", body = KnowledgeDetailResponse), + (status = 401, description = "未授权"), + (status = 404, description = "知识库不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn get( + State(pool): State, Path(id): Path, Extension(auth_user): Extension, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + let user_id = auth_user.user_id.clone(); + + // 0. Check permission + let user_perm = get_kb_permission(&pool, id, &user_id, is_admin).await; + if user_perm.is_none() { + return Err(ApiError::NotFound("Knowledge base not found or permission denied.".to_string())); + } + let current_user_permission = user_perm.clone().unwrap(); + + // 1. Fetch the main knowledge base (already permission-checked above) + let main_kb: Knowledge = sqlx::query_as( + "SELECT id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority FROM knowledge_bases WHERE id = ?", + ) + .bind(id) + .fetch_one(&pool) + .await?; + + // 2. Fetch children KBs + let children_future = async { + let mut qb = QueryBuilder::::new( + "SELECT id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority \ + FROM knowledge_bases WHERE parent_id = ", + ); + qb.push_bind(id); + if !is_admin { + let uid = user_id.clone(); + qb.push(" AND (user_id = ") + .push_bind(uid.clone()) + .push(" OR is_public = 1 OR id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ") + .push_bind(uid) + .push("))"); + } + qb.push(" ORDER BY name"); + qb.build_query_as::().fetch_all(&pool).await + }; + + let children_kbs: Vec = children_future.await?; + let mut count_ids = Vec::with_capacity(children_kbs.len() + 1); + count_ids.push(id); + count_ids.extend(children_kbs.iter().map(|kb| kb.id)); + let (file_counts_res, children_counts_res) = tokio::join!( + get_file_counts(&pool, &count_ids, &auth_user.user_id, is_admin), + get_children_kb_counts(&pool, &count_ids, &auth_user.user_id, is_admin) + ); + let file_counts = file_counts_res?; + let children_counts = children_counts_res?; + let file_count = *file_counts.get(&id).unwrap_or(&0); + let children_kb_count = *children_counts.get(&id).unwrap_or(&0); + let status_breakdown = + file::get_file_status_breakdown_for_kb(&pool, id, true, &auth_user.user_id, is_admin).await?; + + // Compute permission for each child KB in a fixed number of queries (avoid N+1). + let child_ids: Vec = children_kbs.iter().map(|kb| kb.id).collect(); + let child_perms = get_kb_permissions_batch(&pool, &child_ids, &user_id, is_admin).await; + + let children_kbs: Vec = children_kbs + .into_iter() + .map(|kb| KnowledgeResponse { + id: kb.id, + user_id: kb.user_id, + user_name: kb.user_name, + name: kb.name, + description: kb.description, + kb_type: kb.kb_type, + parent_id: kb.parent_id, + is_public: kb.is_public, + parse_priority: kb.parse_priority, + file_count: *file_counts.get(&kb.id).unwrap_or(&0), + children_kb_count: *children_counts.get(&kb.id).unwrap_or(&0), + current_user_permission: child_perms.get(&kb.id).cloned().unwrap_or_else(|| "viewer".to_string()), + }) + .collect(); + + // 3. Fetch the breadcrumb path in a single recursive CTE query (root -> parent), + // avoiding one round-trip per ancestor level. + let path: Vec = sqlx::query_as( + "WITH RECURSIVE ancestors(id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority, depth) AS ( \ + SELECT id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority, 0 \ + FROM knowledge_bases WHERE id = ? \ + UNION ALL \ + SELECT k.id, k.user_id, k.user_name, k.name, k.description, k.kb_type, k.parent_id, k.is_public, k.parse_priority, a.depth + 1 \ + FROM knowledge_bases k INNER JOIN ancestors a ON k.id = a.parent_id \ + ) \ + SELECT id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority \ + FROM ancestors ORDER BY depth DESC", + ) + .bind(main_kb.parent_id) + .fetch_all(&pool) + .await?; + + // 4. Construct the response + let response = KnowledgeDetailResponse { + id: main_kb.id, + user_id: main_kb.user_id, + user_name: main_kb.user_name, + name: main_kb.name, + description: main_kb.description, + kb_type: main_kb.kb_type, + parent_id: main_kb.parent_id, + is_public: main_kb.is_public, + parse_priority: main_kb.parse_priority, + file_count, + children_kb_count, + children_kbs, + path, + status_breakdown, + current_user_permission, + }; + + Ok(Json(response)) +} + +/// 获取知识库文件列表（分页） +#[utoipa::path( + get, + path = "/api/v1/knowledge/knowledge_base/{id}/files", + operation_id = "knowledge_base_files", + tag = "knowledge_base", + params( + ("id" = i64, Path, description = "知识库 ID"), + KnowledgeBaseFilesQuery + ), + responses( + (status = 200, description = "成功返回知识库文件列表", body = KnowledgeBaseFilesResponse), + (status = 401, description = "未授权"), + (status = 404, description = "知识库不存在或无权限") + ), + security(("x-user-id" = []), ("x-role" = [])) +)] +pub async fn get_files( + State(pool): State, Path(id): Path, Query(params): Query, + Extension(auth_user): Extension, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + let user_id = auth_user.user_id.clone(); + + let user_perm = get_kb_permission(&pool, id, &user_id, is_admin).await; + if user_perm.is_none() { + return Err(ApiError::NotFound("Knowledge base not found or permission denied.".to_string())); + } + + let size = params.size.unwrap_or(10).clamp(1, 200); + let page = params.page.unwrap_or(1).max(1); + let limit = size; + let offset = (page - 1) * size; + + let filename = params.filename.as_deref().filter(|s| !s.is_empty()); + let tag = params.tag.as_deref().filter(|s| !s.is_empty()); + + let push_filters = |qb: &mut QueryBuilder| { + if !is_admin { + qb.push(" AND (user_id = ").push_bind(user_id.clone()).push(" OR is_public = 1)"); + } + if let Some(name) = filename { + qb.push(" AND filename LIKE ").push_bind(format!("%{}%", name)); + } + }; + + if let Some(tag) = tag { + let mut qb = QueryBuilder::::new( + "SELECT id, user_id, user_name, hash, filename, path, size, tags, status, log, slice_type, kb_id, is_public, meta, summary, created_at, updated_at FROM files WHERE kb_id = ", + ); + qb.push_bind(id); + push_filters(&mut qb); + qb.push(" ORDER BY updated_at DESC"); + let rows: Vec = qb.build_query_as().fetch_all(&pool).await?; + let filtered: Vec = rows + .into_iter() + .filter(|f| { + serde_json::from_str::>(&f.tags) + .map(|tags| tags.contains(&tag.to_string())) + .unwrap_or(false) + }) + .collect(); + let total = filtered.len() as i64; + let items = filtered.into_iter().skip(offset as usize).take(limit as usize).collect(); + return Ok(Json(KnowledgeBaseFilesResponse { total, items })); + } + + let mut count_qb = QueryBuilder::::new("SELECT COUNT(*) FROM files WHERE kb_id = "); + count_qb.push_bind(id); + push_filters(&mut count_qb); + let total: i64 = count_qb.build_query_scalar().fetch_one(&pool).await?; + + let mut list_qb = QueryBuilder::::new( + "SELECT id, user_id, user_name, hash, filename, path, size, tags, status, log, slice_type, kb_id, is_public, meta, summary, created_at, updated_at FROM files WHERE kb_id = ", + ); + list_qb.push_bind(id); + push_filters(&mut list_qb); + list_qb.push(" ORDER BY updated_at DESC LIMIT ").push_bind(limit); + list_qb.push(" OFFSET ").push_bind(offset); + let items: Vec = list_qb.build_query_as().fetch_all(&pool).await?; + + Ok(Json(KnowledgeBaseFilesResponse { total, items })) +} + +/// 获取知识库文件标签及其文件数量 +#[utoipa::path( + get, + path = "/api/v1/knowledge/knowledge_base/{id}/tags", + operation_id = "knowledge_base_tags", + tag = "knowledge_base", + params( + ("id" = i64, Path, description = "知识库 ID"), + KnowledgeBaseTagsQuery + ), + responses( + (status = 200, description = "成功返回知识库标签统计", body = KnowledgeBaseTagsResponse), + (status = 401, description = "未授权"), + (status = 404, description = "知识库不存在或无权限") + ), + security(("x-user-id" = []), ("x-role" = [])) +)] +pub async fn get_tags( + State(pool): State, Path(id): Path, Query(params): Query, + Extension(auth_user): Extension, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + let user_id = auth_user.user_id.clone(); + if is_admin { + let exists = sqlx::query_scalar::<_, i64>("SELECT id FROM knowledge_bases WHERE id = ?") + .bind(id) + .fetch_optional(&pool) + .await?; + if exists.is_none() { + return Err(ApiError::NotFound("Knowledge base not found or permission denied.".to_string())); + } + } + common::ensure_kb_accessible(&pool, id, &user_id, is_admin).await?; + + let include_descendants = params.include_descendants.unwrap_or(false); + let mut qb = QueryBuilder::::new(""); + if include_descendants { + qb.push("WITH RECURSIVE descendants AS (SELECT id FROM knowledge_bases WHERE id = ") + .push_bind(id) + .push(" UNION ALL SELECT kb.id FROM knowledge_bases kb JOIN descendants d ON kb.parent_id = d.id"); + if !is_admin { + qb.push(" WHERE kb.user_id = ") + .push_bind(user_id.clone()) + .push(" OR kb.is_public = 1 OR kb.id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ") + .push_bind(user_id.clone()) + .push(")"); + } + qb.push(") "); + } + + qb.push( + "SELECT CAST(tag.value AS TEXT) AS tag, COUNT(DISTINCT f.id) AS file_count \ + FROM files f \ + JOIN json_each( \ + CASE WHEN json_valid(f.tags) \ + THEN CASE WHEN json_type(f.tags) = 'array' THEN f.tags ELSE '[]' END \ + ELSE '[]' END \ + ) AS tag \ + WHERE tag.type = 'text' AND trim(CAST(tag.value AS TEXT)) <> ''", + ); + if include_descendants { + qb.push(" AND f.kb_id IN (SELECT id FROM descendants)"); + } else { + qb.push(" AND f.kb_id = ").push_bind(id); + } + if !is_admin { + qb.push(" AND "); + common::push_file_access_filter(&mut qb, &user_id, Some("f")); + } + qb.push(" GROUP BY tag.value ORDER BY file_count DESC, tag COLLATE NOCASE ASC"); + + let tags = qb.build_query_as::().fetch_all(&pool).await?; + Ok(Json(KnowledgeBaseTagsResponse { kb_id: id, tags })) +} + +/// 删除知识库 +#[utoipa::path( + delete, + path = "/api/v1/knowledge/knowledge_base/{id}", + operation_id = "knowledge_base_delete", + tag = "knowledge_base", + params( + ("id" = i64, Path, description = "知识库 ID") + ), + responses( + (status = 200, description = "成功删除知识库"), + (status = 401, description = "未授权"), + (status = 404, description = "知识库不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn delete( + State(pool): State, Extension(search_engine): Extension, Path(id): Path, + Extension(auth_user): Extension, +) -> ApiResult<()> { + let is_admin = auth_user.is_admin(); + + // Check permission: only admin can delete + let user_perm = get_kb_permission(&pool, id, &auth_user.user_id, is_admin).await; + if !meets_requirement(user_perm.as_deref(), "admin") { + return Err(ApiError::Forbidden("Permission denied. Admin role required.".to_string())); + } + + let all_kb_ids: Vec = sqlx::query_scalar( + r#" + WITH RECURSIVE kb_hierarchy AS ( + SELECT id FROM knowledge_bases WHERE id = ? + UNION ALL + SELECT kb.id FROM knowledge_bases kb + INNER JOIN kb_hierarchy kh ON kb.parent_id = kh.id + ) + SELECT id FROM kb_hierarchy; + "#, + ) + .bind(id) + .fetch_all(&pool) + .await?; + + if all_kb_ids.is_empty() { + return Err(crate::api::error::ApiError::NotFound( + "Knowledge base not found or permission denied.".to_string(), + )); + } + + let mut files_qb = + QueryBuilder::new(format!("SELECT {} FROM files WHERE kb_id IN (", super::file::FILE_COLS_NO_CONTENT)); + let mut files_separated = files_qb.separated(", "); + for kb_id in &all_kb_ids { + files_separated.push_bind(kb_id); + } + files_qb.push(")"); + let files: Vec = files_qb.build_query_as().fetch_all(&pool).await?; + + let file_ids: Vec = files.iter().map(|f| f.id).collect(); + let image_paths = super::file::collect_image_paths_for_files(&pool, &file_ids).await?; + + // 先分批删除文件相关行（每批独立提交）。删除整个知识库时文件数可能极多， + // 单个事务级联删除会长时间持有 SQLite 写锁、阻塞其它写入；分批提交把锁持有时间限制在每批内。 + super::file::delete_file_rows_batched(&pool, &file_ids).await?; + + // 文件已删除，再在一个短事务里删除知识库本身（含子库由外键级联处理，与原行为一致）。 + let mut tx = pool.begin().await?; + let result = sqlx::query("DELETE FROM knowledge_bases WHERE id = ?").bind(id).execute(&mut *tx).await?; + + if result.rows_affected() == 0 { + return Err(crate::api::error::ApiError::NotFound( + "Knowledge base not found or permission denied.".to_string(), + )); + } + + tx.commit().await?; + + let cleanup_failed = super::file::cleanup_deleted_files(&search_engine, &files, image_paths).await; + for failure in cleanup_failed { + log::warn!( + "Knowledge base delete cleanup failed for file {} at {}: {}", + failure.id, + failure.stage, + failure.error + ); + } + + for kb_id in &all_kb_ids { + if let Err(e) = search_engine.delete(None, Some(*kb_id)).await { + log::warn!("Failed to delete search index for knowledge base {}: {}", kb_id, e); + } + } + + Ok(()) +} + +#[derive(Serialize, ToSchema)] +pub struct ReparseKnowledgeBaseResponse { + pub kb_count: i64, + pub file_count: i64, +} + +async fn query_file_ids_for_kbs( + pool: &SqlitePool, kb_ids: &[i64], user_id_filter: Option<&str>, +) -> Result, sqlx::Error> { + if kb_ids.is_empty() { + return Ok(Vec::new()); + } + + let mut qb = QueryBuilder::::new("SELECT id FROM files WHERE kb_id IN ("); + crate::db::push_i64_list(&mut qb, kb_ids); + qb.push(")"); + if let Some(user_id) = user_id_filter { + qb.push(" AND user_id = ").push_bind(user_id); + } + qb.build_query_scalar().fetch_all(pool).await +} + +async fn reset_reparse_scope( + pool: &SqlitePool, search_engine: &SearchEngine, analysis_kb_ids: &[i64], unassigned_file_ids: &[i64], + file_ids: &[i64], clear_unassigned_graph: bool, +) -> ApiResult<()> { + // 不能在解析 worker 写入切片/索引时清空同一文件，否则旧批次的失败清理可能误删新数据。 + // 调用方可在当前解析完成后安全重试。 + if !file_ids.is_empty() { + let mut processing_qb = QueryBuilder::::new("SELECT COUNT(*) FROM files WHERE status = 2 AND id IN ("); + crate::db::push_i64_list(&mut processing_qb, file_ids); + processing_qb.push(")"); + let processing_count: i64 = processing_qb.build_query_scalar().fetch_one(pool).await?; + if processing_count > 0 { + return Err(ApiError::BadRequest(format!( + "{} file(s) are currently being parsed; retry reparse after they finish", + processing_count + ))); + } + + let mut refs_qb = QueryBuilder::::new( + "SELECT COUNT(*) FROM parse_artifacts pa JOIN files ref ON ref.artifact_id = pa.id \ + WHERE pa.source_file_id IN (", + ); + crate::db::push_i64_list(&mut refs_qb, file_ids); + refs_qb.push(") AND ref.id NOT IN ("); + crate::db::push_i64_list(&mut refs_qb, file_ids); + refs_qb.push(")"); + let external_refs: i64 = refs_qb.build_query_scalar().fetch_one(pool).await?; + if external_refs > 0 { + return Err(ApiError::BadRequest( + "Cannot reparse an artifact source while files outside this scope still reference it".to_string(), + )); + } + } + + // 清理搜索索引 + search_engine.delete_batch(None, Some(analysis_kb_ids)).await?; + search_engine.delete_batch(Some(unassigned_file_ids), None).await?; + + // 清理知识图谱数据（节点会级联删除边和提及） + if !analysis_kb_ids.is_empty() { + let mut del_nodes_qb = QueryBuilder::::new("DELETE FROM graph_nodes WHERE kb_id IN ("); + crate::db::push_i64_list(&mut del_nodes_qb, analysis_kb_ids); + del_nodes_qb.push(")"); + del_nodes_qb.build().execute(pool).await?; + + let mut del_snapshots_qb = QueryBuilder::::new("DELETE FROM graph_snapshots WHERE kb_id IN ("); + crate::db::push_i64_list(&mut del_snapshots_qb, analysis_kb_ids); + del_snapshots_qb.push(")"); + del_snapshots_qb.build().execute(pool).await?; + } + if clear_unassigned_graph { + sqlx::query("DELETE FROM graph_nodes WHERE kb_id IS NULL").execute(pool).await?; + sqlx::query("DELETE FROM graph_snapshots WHERE kb_id IS NULL").execute(pool).await?; + } + + if !file_ids.is_empty() { + let mut del_slices_qb = QueryBuilder::::new("DELETE FROM slices WHERE file_id IN ("); + crate::db::push_i64_list(&mut del_slices_qb, file_ids); + del_slices_qb.push(")"); + del_slices_qb.build().execute(pool).await?; + for file_id in file_ids { + if let Err(e) = crate::slice_content::delete(*file_id).await { + warn!("Failed to delete slice content file for knowledge base reparse file {}: {}", file_id, e); + } + } + + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64; + let mut update_qb = QueryBuilder::::new( + "UPDATE files SET status = 0, parse_run_id = NULL, log = '', summary = NULL, updated_at = ", + ); + update_qb.push_bind(now); + update_qb.push(" WHERE id IN ("); + crate::db::push_i64_list(&mut update_qb, file_ids); + update_qb.push(")"); + update_qb.build().execute(pool).await?; + } + + Ok(()) +} + +/// 重新解析所有知识库 +#[utoipa::path( + post, + path = "/api/v1/knowledge/knowledge_base/reparse", + operation_id = "knowledge_base_reparse", + tag = "knowledge_base", + responses( + (status = 200, description = "已提交重新解析", body = ReparseKnowledgeBaseResponse), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn reparse( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, +) -> ApiResult> { + // Get analysis-type KBs owned by or explicitly editable by the user + let mut analysis_kb_ids: Vec = + sqlx::query_scalar("SELECT id FROM knowledge_bases WHERE user_id = ? AND kb_type != ?") + .bind(auth_user.user_id.clone()) + .bind(KB_TYPE_STORAGE) + .fetch_all(&pool) + .await?; + + // Also include KBs where user has editor/admin via kb_permissions + let perm_kb_ids: Vec = + sqlx::query_scalar("SELECT kb_id FROM kb_permissions WHERE user_id = ? AND permission IN ('editor', 'admin')") + .bind(&auth_user.user_id) + .fetch_all(&pool) + .await?; + + for kb_id in perm_kb_ids { + if !analysis_kb_ids.contains(&kb_id) { + let is_analysis: Option = + sqlx::query_scalar("SELECT 1 FROM knowledge_bases WHERE id = ? AND kb_type != ?") + .bind(kb_id) + .bind(KB_TYPE_STORAGE) + .fetch_optional(&pool) + .await?; + if is_analysis.is_some() { + analysis_kb_ids.push(kb_id); + } + } + } + + let unassigned_file_ids: Vec = sqlx::query_scalar("SELECT id FROM files WHERE user_id = ? AND kb_id IS NULL") + .bind(auth_user.user_id.clone()) + .fetch_all(&pool) + .await?; + + let kb_file_ids = query_file_ids_for_kbs(&pool, &analysis_kb_ids, Some(&auth_user.user_id)).await?; + + let mut file_ids = kb_file_ids.clone(); + file_ids.extend(unassigned_file_ids.iter().copied()); + if analysis_kb_ids.is_empty() && unassigned_file_ids.is_empty() { + return Ok(Json(ReparseKnowledgeBaseResponse { kb_count: 0, file_count: 0 })); + } + + reset_reparse_scope(&pool, &search_engine, &analysis_kb_ids, &unassigned_file_ids, &file_ids, true).await?; + + Ok(Json(ReparseKnowledgeBaseResponse { kb_count: analysis_kb_ids.len() as i64, file_count: file_ids.len() as i64 })) +} + +/// 重新解析指定知识库（包含子知识库） +#[utoipa::path( + post, + path = "/api/v1/knowledge/knowledge_base/{id}/reparse", + operation_id = "knowledge_base_reparse_by_id", + tag = "knowledge_base", + params( + ("id" = i64, Path, description = "知识库 ID") + ), + responses( + (status = 200, description = "已提交指定知识库重新解析", body = ReparseKnowledgeBaseResponse), + (status = 404, description = "知识库不存在或无权限"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn reparse_by_id( + State(pool): State, Extension(search_engine): Extension, Path(id): Path, + Extension(auth_user): Extension, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + let user_perm = get_kb_permission(&pool, id, &auth_user.user_id, is_admin).await; + if !meets_requirement(user_perm.as_deref(), "editor") { + return Err(ApiError::Forbidden("Permission denied. Requires editor or admin.".to_string())); + } + + let analysis_kb_ids: Vec = common::collect_kb_descendant_ids(&pool, id, true).await?; + + let file_ids = query_file_ids_for_kbs(&pool, &analysis_kb_ids, None).await?; + if analysis_kb_ids.is_empty() { + return Ok(Json(ReparseKnowledgeBaseResponse { kb_count: analysis_kb_ids.len() as i64, file_count: 0 })); + } + + reset_reparse_scope(&pool, &search_engine, &analysis_kb_ids, &[], &file_ids, false).await?; + Ok(Json(ReparseKnowledgeBaseResponse { kb_count: analysis_kb_ids.len() as i64, file_count: file_ids.len() as i64 })) +} + +async fn load_tree_knowledges( + pool: &SqlitePool, root_kb_id: Option, user_id: &str, is_admin: bool, +) -> anyhow::Result> { + let access_clause = "(user_id = ? OR is_public = 1 OR id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ?))"; + let access_where = + "WHERE kb.user_id = ? OR kb.is_public = 1 OR kb.id IN (SELECT kb_id FROM kb_permissions WHERE user_id = ?)"; + + let rows = match (root_kb_id, is_admin) { + (Some(kb_id), true) => { + sqlx::query_as( + r#" + WITH RECURSIVE tree AS ( + SELECT id, name, description, kb_type, parent_id, is_public + FROM knowledge_bases + WHERE id = ? + UNION ALL + SELECT kb.id, kb.name, kb.description, kb.kb_type, kb.parent_id, kb.is_public + FROM knowledge_bases kb + INNER JOIN tree t ON kb.parent_id = t.id + ) + SELECT id, name, description, kb_type, parent_id, is_public + FROM tree + ORDER BY name + "#, + ) + .bind(kb_id) + .fetch_all(pool) + .await? + } + (Some(kb_id), false) => { + sqlx::query_as( + r#" + WITH RECURSIVE tree AS ( + SELECT id, name, description, kb_type, parent_id, is_public + FROM knowledge_bases + WHERE id = ? AND #ACCESS# + UNION ALL + SELECT kb.id, kb.name, kb.description, kb.kb_type, kb.parent_id, kb.is_public + FROM knowledge_bases kb + INNER JOIN tree t ON kb.parent_id = t.id + #ACCESS_WHERE# + ) + SELECT id, name, description, kb_type, parent_id, is_public + FROM tree + ORDER BY name + "# + .replace("#ACCESS#", access_clause) + .replace("#ACCESS_WHERE#", access_where) + .as_str(), + ) + .bind(kb_id) + .bind(user_id) + .bind(user_id) + .bind(user_id) + .bind(user_id) + .fetch_all(pool) + .await? + } + (None, true) => { + sqlx::query_as( + r#" + WITH RECURSIVE tree AS ( + SELECT id, name, description, kb_type, parent_id, is_public + FROM knowledge_bases + WHERE parent_id IS NULL + UNION ALL + SELECT kb.id, kb.name, kb.description, kb.kb_type, kb.parent_id, kb.is_public + FROM knowledge_bases kb + INNER JOIN tree t ON kb.parent_id = t.id + ) + SELECT id, name, description, kb_type, parent_id, is_public + FROM tree + ORDER BY name + "#, + ) + .fetch_all(pool) + .await? + } + (None, false) => { + sqlx::query_as( + r#" + WITH RECURSIVE tree AS ( + SELECT id, name, description, kb_type, parent_id, is_public + FROM knowledge_bases + WHERE parent_id IS NULL AND #ACCESS# + UNION ALL + SELECT kb.id, kb.name, kb.description, kb.kb_type, kb.parent_id, kb.is_public + FROM knowledge_bases kb + INNER JOIN tree t ON kb.parent_id = t.id + #ACCESS_WHERE# + ) + SELECT id, name, description, kb_type, parent_id, is_public + FROM tree + ORDER BY name + "# + .replace("#ACCESS#", access_clause) + .replace("#ACCESS_WHERE#", access_where) + .as_str(), + ) + .bind(user_id) + .bind(user_id) + .bind(user_id) + .bind(user_id) + .fetch_all(pool) + .await? + } + }; + Ok(rows) +} + +#[derive(Clone, Debug, sqlx::FromRow)] +struct TreeKnowledge { + id: i64, + name: String, + description: String, + kb_type: String, + parent_id: Option, + is_public: bool, +} + +async fn load_tree_files_by_kb( + pool: &SqlitePool, kb_ids: &[i64], user_id: &str, is_admin: bool, +) -> anyhow::Result>> { + if kb_ids.is_empty() { + return Ok(HashMap::new()); + } + + let mut qb = + QueryBuilder::::new("SELECT id, size, filename, meta, kb_id, is_public FROM files WHERE kb_id IN ("); + crate::db::push_i64_list(&mut qb, kb_ids); + qb.push(")"); + if !is_admin { + qb.push(" AND (user_id = ").push_bind(user_id).push(" OR is_public = 1)"); + } + qb.push(" ORDER BY kb_id, filename"); + + let files: Vec = qb.build_query_as().fetch_all(pool).await?; + let mut files_by_kb = HashMap::>::new(); + for file in files { + if let Some(kb_id) = file.kb_id { + files_by_kb.entry(kb_id).or_default().push(file); + } + } + Ok(files_by_kb) +} + +fn build_tree_node( + kb_id: i64, knowledges: &HashMap, children_map: &HashMap, Vec>, + files_by_kb: &mut HashMap>, +) -> Option { + let kb = knowledges.get(&kb_id)?; + let child_ids = children_map.get(&Some(kb_id)).cloned().unwrap_or_default(); + let mut children = Vec::with_capacity(child_ids.len()); + for child_id in child_ids { + if let Some(child) = build_tree_node(child_id, knowledges, children_map, files_by_kb) { + children.push(child); + } + } + + Some(KnowledgeTreeNode { + id: kb.id, + name: kb.name.clone(), + description: kb.description.clone(), + is_public: kb.is_public, + kb_type: kb.kb_type.clone(), + files: files_by_kb.remove(&kb_id).unwrap_or_default(), + children, + }) +} + +fn assemble_tree( + knowledges: Vec, mut files_by_kb: HashMap>, root_kb_id: Option, +) -> Vec { + if knowledges.is_empty() { + return Vec::new(); + } + + let mut knowledge_map = HashMap::::with_capacity(knowledges.len()); + let mut children_map = HashMap::, Vec>::new(); + for kb in knowledges { + children_map.entry(kb.parent_id).or_default().push(kb.id); + knowledge_map.insert(kb.id, kb); + } + + for child_ids in children_map.values_mut() { + child_ids.sort_by(|left_id, right_id| { + let left_name = knowledge_map.get(left_id).map(|kb| kb.name.as_str()).unwrap_or_default(); + let right_name = knowledge_map.get(right_id).map(|kb| kb.name.as_str()).unwrap_or_default(); + left_name.cmp(right_name) + }); + } + + let root_ids = match root_kb_id { + Some(root_id) => { + if knowledge_map.contains_key(&root_id) { + vec![root_id] + } else { + Vec::new() + } + } + None => children_map.get(&None).cloned().unwrap_or_default(), + }; + + let mut tree = Vec::with_capacity(root_ids.len()); + for root_id in root_ids { + if let Some(node) = build_tree_node(root_id, &knowledge_map, &children_map, &mut files_by_kb) { + tree.push(node); + } + } + tree +} + +/// 获取知识库树结构 +#[utoipa::path( + get, + path = "/api/v1/knowledge/knowledge_base/tree", + operation_id = "knowledge_base_tree", + tag = "knowledge_base", + params(TreeQuery), + responses( + (status = 200, description = "成功返回知识库树结构", body = Vec), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn tree( + State(pool): State, Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult>> { + let is_admin = auth_user.is_admin(); + let knowledges = load_tree_knowledges(&pool, params.kb_id, &auth_user.user_id, is_admin).await?; + let kb_ids: Vec = knowledges.iter().map(|kb| kb.id).collect(); + let files_by_kb = load_tree_files_by_kb(&pool, &kb_ids, &auth_user.user_id, is_admin).await?; + let tree = assemble_tree(knowledges, files_by_kb, params.kb_id); + Ok(Json(tree)) +} + +#[derive(Serialize, ToSchema)] +pub struct ExportKbResponse { + pub export_path: String, + pub manifest: crate::export::ExportManifest, +} + +#[derive(Deserialize, ToSchema)] +pub struct BatchExportKbRequest { + pub kb_ids: Vec, + #[serde(default)] + pub include_children: bool, +} + +/// 批量导出多个知识库 +#[utoipa::path( + post, + path = "/api/v1/knowledge/knowledge_base/export", + operation_id = "knowledge_base_batch_export", + tag = "knowledge_base", + request_body = BatchExportKbRequest, + responses( + (status = 200, description = "成功导出知识库", body = ExportKbResponse), + (status = 400, description = "请求参数错误"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn batch_export_kb( + State(pool): State, Extension(auth_user): Extension, Json(req): Json, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + + if req.kb_ids.is_empty() { + return Err(ApiError::BadRequest("kb_ids cannot be empty".to_string())); + } + + // Verify all knowledge bases exist and user has access (viewer or above) + let mut allowed_ids = Vec::new(); + for kb_id in &req.kb_ids { + let perm = get_kb_permission(&pool, *kb_id, &auth_user.user_id, is_admin).await; + if perm.is_some() { + allowed_ids.push(*kb_id); + } + } + if allowed_ids.is_empty() { + return Err(ApiError::NotFound("No knowledge bases found or permission denied.".to_string())); + } + + if allowed_ids.len() != req.kb_ids.len() { + let missing: Vec = req.kb_ids.iter().filter(|id| !allowed_ids.contains(id)).copied().collect(); + warn!("User {} tried to export inaccessible KBs: {:?}", auth_user.user_id, missing); + } + + let export_path = crate::export::export_knowledge_bases(&pool, &allowed_ids, req.include_children) + .await + .map_err(|e| ApiError::Internal(format!("Export failed: {}", e)))?; + + // Read manifest + let manifest_path = std::path::Path::new(&export_path).join("manifest.json"); + let manifest_json = tokio::fs::read_to_string(&manifest_path) + .await + .map_err(|e| ApiError::Internal(format!("Failed to read manifest: {}", e)))?; + let manifest: crate::export::ExportManifest = serde_json::from_str(&manifest_json) + .map_err(|e| ApiError::Internal(format!("Failed to parse manifest: {}", e)))?; + + Ok(Json(ExportKbResponse { export_path, manifest })) +} + +// --------------------------------------------------------------------------- +// KB Permission management endpoints +// --------------------------------------------------------------------------- + +#[derive(Serialize, Clone, Debug, sqlx::FromRow, ToSchema)] +pub struct KbPermissionItem { + pub user_id: String, + pub permission: String, + pub created_at: i64, +} + +#[derive(Deserialize, Clone, Debug, ToSchema)] +pub struct KbPermissionCreateReq { + pub user_id: String, + pub permission: String, +} + +/// 获取知识库权限列表 +#[utoipa::path( + get, + path = "/api/v1/knowledge/knowledge_base/{id}/permissions", + operation_id = "kb_permission_list", + tag = "knowledge_base", + params( + ("id" = i64, Path, description = "知识库 ID") + ), + responses( + (status = 200, description = "成功返回权限列表", body = Vec), + (status = 403, description = "无权限"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn list_permissions( + Path(id): Path, State(pool): State, Extension(auth_user): Extension, +) -> ApiResult>> { + let is_admin = auth_user.is_admin(); + let user_perm = get_kb_permission(&pool, id, &auth_user.user_id, is_admin).await; + if !meets_requirement(user_perm.as_deref(), "admin") { + return Err(ApiError::Forbidden("Permission denied. Admin role required.".to_string())); + } + + let rows: Vec = sqlx::query_as( + "SELECT user_id, permission, created_at FROM kb_permissions WHERE kb_id = ? ORDER BY created_at", + ) + .bind(id) + .fetch_all(&pool) + .await?; + + Ok(Json(rows)) +} + +/// 添加或更新知识库权限 +#[utoipa::path( + post, + path = "/api/v1/knowledge/knowledge_base/{id}/permissions", + operation_id = "kb_permission_add", + tag = "knowledge_base", + params( + ("id" = i64, Path, description = "知识库 ID") + ), + request_body = KbPermissionCreateReq, + responses( + (status = 200, description = "成功添加/更新权限", body = KbPermissionItem), + (status = 400, description = "请求参数错误"), + (status = 403, description = "无权限"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn add_permission( + Path(id): Path, State(pool): State, Extension(auth_user): Extension, + Json(req): Json, +) -> ApiResult> { + let is_admin = auth_user.is_admin(); + let user_perm = get_kb_permission(&pool, id, &auth_user.user_id, is_admin).await; + if !meets_requirement(user_perm.as_deref(), "admin") { + return Err(ApiError::Forbidden("Permission denied. Admin role required.".to_string())); + } + + // Validate permission value + let perm = req.permission.trim().to_lowercase(); + if !matches!(perm.as_str(), "viewer" | "editor" | "admin") { + return Err(ApiError::BadRequest("Invalid permission. Use 'viewer', 'editor', or 'admin'.".to_string())); + } + + // Prevent adding permission for the owner (owner already has admin implicitly) + let owner: Option = + sqlx::query_scalar("SELECT user_id FROM knowledge_bases WHERE id = ?").bind(id).fetch_optional(&pool).await?; + if owner.as_deref() == Some(&req.user_id) { + return Err(ApiError::BadRequest("Cannot set permission for the owner.".to_string())); + } + + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64; + + sqlx::query( + "INSERT INTO kb_permissions (kb_id, user_id, permission, created_at, updated_at) VALUES (?, ?, ?, ?, ?) \ + ON CONFLICT(kb_id, user_id) DO UPDATE SET permission = excluded.permission, updated_at = excluded.updated_at", + ) + .bind(id) + .bind(&req.user_id) + .bind(&perm) + .bind(now) + .bind(now) + .execute(&pool) + .await?; + + Ok(Json(KbPermissionItem { user_id: req.user_id, permission: perm, created_at: now })) +} + +/// 删除知识库权限 +#[utoipa::path( + delete, + path = "/api/v1/knowledge/knowledge_base/{id}/permissions/{user_id}", + operation_id = "kb_permission_remove", + tag = "knowledge_base", + params( + ("id" = i64, Path, description = "知识库 ID"), + ("user_id" = String, Path, description = "用户 ID") + ), + responses( + (status = 200, description = "成功删除权限"), + (status = 403, description = "无权限"), + (status = 401, description = "未授权") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn remove_permission( + Path((id, target_user_id)): Path<(i64, String)>, State(pool): State, + Extension(auth_user): Extension, +) -> ApiResult<()> { + let is_admin = auth_user.is_admin(); + let user_perm = get_kb_permission(&pool, id, &auth_user.user_id, is_admin).await; + if !meets_requirement(user_perm.as_deref(), "admin") { + return Err(ApiError::Forbidden("Permission denied. Admin role required.".to_string())); + } + + let result = sqlx::query("DELETE FROM kb_permissions WHERE kb_id = ? AND user_id = ?") + .bind(id) + .bind(target_user_id) + .execute(&pool) + .await?; + + if result.rows_affected() == 0 { + return Err(ApiError::NotFound("Permission not found.".to_string())); + } + + Ok(()) +} diff --git a/src/api/mod.rs b/src/api/mod.rs new file mode 100644 index 0000000..af35573 --- /dev/null +++ b/src/api/mod.rs @@ -0,0 +1,254 @@ +use axum::{ + Extension, Router, + routing::{delete, get, post, put}, +}; +use sqlx::SqlitePool; +use utoipa::OpenApi; + +mod common; +mod error; +mod file; +mod graph; +mod knowledge_base; +mod search; +mod system; +// 重新导出 File 类型供其他模块使用 +pub use file::File; +pub(crate) use file::{ + FILE_COLS_NO_CONTENT, backfill_missing_image_meta_for_files, collect_image_paths_for_files, + collect_image_raw_paths_for_files, effective_parse_file_id, find_reusable_parsed_file, remove_image_files, + resolve_image_storage_path, update_file_custom_image_meta, +}; + +use crate::search::SearchEngine; + +/// OpenAPI 文档定义 +#[derive(OpenApi)] +#[openapi( + paths( + // Knowledge Base + knowledge_base::list, + knowledge_base::create, + knowledge_base::get, + knowledge_base::get_files, + knowledge_base::get_tags, + knowledge_base::update, + knowledge_base::reparse, + knowledge_base::reparse_by_id, + knowledge_base::delete, + knowledge_base::tree, + knowledge_base::batch_export_kb, + knowledge_base::list_permissions, + knowledge_base::add_permission, + knowledge_base::remove_permission, + // File + file::upload, + file::list, + file::stats, + file::get, + file::update, + file::move_to_kb, + file::batch_delete, + file::reparse_failed, + file::delete, + file::get_slices, + file::update_slices, + file::get_image_by_filename, + file::download, + file::get_highlighted_pdf, + file::get_slice_highlight, + file::get_slice_highlight_page, + file::excel_data, + file::archive_entries, + file::archive_extract, + file::archive_download, + // Search + search::search, + search::search_full, + search::search_summary, + search::search_with_graph, + search::search_image, + search::search_image_by_text, + search::advanced_search_stream, + search::list_lexicons, + search::create_lexicon, + search::update_lexicon, + search::delete_lexicon, + search::toggle_lexicon_enabled, + search::reload_lexicon, + search::publish_lexicon, + search::list_synonyms, + search::create_synonym, + search::update_synonym, + search::delete_synonym, + search::toggle_synonym_enabled, + // Graph + graph::search_entities, + graph::get_entity, + graph::get_graph_stats, + // System + system::heap_profile, + system::heap_profile_pdf, + system::lancedb_compact, + system::index_force_merge, + system::index_rebuild_status, + ), + components( + schemas( + knowledge_base::Knowledge, + knowledge_base::KnowledgeResponse, + knowledge_base::KnowledgeDetailResponse, + knowledge_base::KnowledgeBaseFilesResponse, + knowledge_base::KnowledgeBaseTagsResponse, + knowledge_base::TagFileCount, + knowledge_base::KnowledgeTreeFile, + knowledge_base::KnowledgeTreeNode, + knowledge_base::KnowledgeCreateReq, + knowledge_base::KnowledgeUpdateReq, + knowledge_base::ReparseKnowledgeBaseResponse, + knowledge_base::ExportKbResponse, + knowledge_base::BatchExportKbRequest, + knowledge_base::KbPermissionItem, + knowledge_base::KbPermissionCreateReq, + crate::export::ExportManifest, + file::File, + file::FileListResponse, + file::UpdateFileReq, + file::MoveFileReq, + file::BatchDeleteFilesReq, + file::BatchDeleteFilesResp, + file::BatchDeleteSkippedItem, + file::BatchDeleteCleanupFailedItem, + file::ReparseFailedFilesReq, + file::ReparseFailedFilesResp, + file::Slice, + file::UpdateSliceItem, + file::UpdateSlicesReq, + file::SlicePosition, + file::SliceHighlight, + file::SliceHighlightPage, + file::FileStatusBreakdown, + file::ExcelData, + file::ExcelSheetData, + crate::archive::ArchiveEntry, + crate::archive::ExtractResult, + search::SearchResult, + search::SearchResultItem, + search::FullSearchResult, + search::FullSearchResultItem, + search::SummarySearchResult, + search::SummarySearchResultItem, + search::FileInfo, + search::KbInfo, + search::LexiconItem, + search::LexiconListResponse, + search::CreateLexiconReq, + search::UpdateLexiconReq, + search::ToggleLexiconReq, + search::DeleteLexiconResponse, + search::ReloadLexiconResponse, + search::PublishLexiconResponse, + search::SynonymItem, + search::SynonymListResponse, + search::CreateSynonymReq, + search::UpdateSynonymReq, + search::ToggleSynonymReq, + search::DeleteSynonymResponse, + graph::EntityInfo, + graph::EntityDetail, + graph::NeighborInfo, + graph::MentionInfo, + graph::GraphStats, + system::MemoryUsage, + system::HeapProfileStatus, + system::LanceDbCompactStats, + system::TantivyForceMergeIndexStats, + system::TantivyForceMergeResponse, + system::IndexRebuildStatus, + ) + ), + tags( + (name = "knowledge_base", description = "知识库管理接口"), + (name = "file", description = "文件管理接口"), + (name = "search", description = "搜索接口"), + (name = "graph", description = "知识图谱接口"), + (name = "system", description = "系统监控接口") + ), + info( + title = "HTKnow API", + version = "0.1.0", + description = "HTKnow 知识库管理系统 API 文档", + ), +)] +pub struct ApiDoc; + +/// 获取 OpenAPI 文档 +pub fn openapi() -> utoipa::openapi::OpenApi { + ApiDoc::openapi() +} + +pub fn app(pool: SqlitePool, search_engine: SearchEngine) -> Router { + let knowledge_router = Router::new() + .route("/", get(knowledge_base::list).post(knowledge_base::create)) + .route("/reparse", post(knowledge_base::reparse)) + .route("/{id}/reparse", post(knowledge_base::reparse_by_id)) + .route("/{id}/files", get(knowledge_base::get_files)) + .route("/{id}/tags", get(knowledge_base::get_tags)) + .route("/export", post(knowledge_base::batch_export_kb)) + .route("/tree", get(knowledge_base::tree)) + .route("/{id}", get(knowledge_base::get).put(knowledge_base::update).delete(knowledge_base::delete)) + .route("/{id}/permissions", get(knowledge_base::list_permissions).post(knowledge_base::add_permission)) + .route("/{id}/permissions/{user_id}", delete(knowledge_base::remove_permission)); + let file_router = Router::new() + .route("/", get(file::list).post(file::upload)) + .route("/batch-delete", post(file::batch_delete)) + .route("/reparse-failed", post(file::reparse_failed)) + .route("/stats", get(file::stats)) + .route("/images/{filename}", get(file::get_image_by_filename)) + .route("/{id}", get(file::get).put(file::update).delete(file::delete)) + .route("/{id}/move", put(file::move_to_kb)) + .route("/{id}/slices", get(file::get_slices).put(file::update_slices)) + .route("/{id}/slices/{slice_id}/highlight", get(file::get_slice_highlight)) + .route("/{id}/slices/{slice_id}/highlight-page", get(file::get_slice_highlight_page)) + .route("/{id}/download", get(file::download)) + .route("/{id}/highlighted-pdf", get(file::get_highlighted_pdf)) + .route("/{id}/excel-data", get(file::excel_data)) + .route("/{id}/archive-entries", get(file::archive_entries)) + .route("/{id}/archive-extract", post(file::archive_extract)) + .route("/{id}/archive-download", get(file::archive_download)); + let search_router = Router::new() + .route("/", get(search::search)) + .route("/full", get(search::search_full)) + .route("/summary", get(search::search_summary)) + .route("/graph", get(search::search_with_graph)) + .route("/image", post(search::search_image)) + .route("/image-by-text", get(search::search_image_by_text)) + .route("/lexicons", get(search::list_lexicons).post(search::create_lexicon)) + .route("/lexicons/reload", post(search::reload_lexicon)) + .route("/lexicons/publish", post(search::publish_lexicon)) + .route("/lexicons/{id}", put(search::update_lexicon).delete(search::delete_lexicon)) + .route("/lexicons/{id}/enabled", put(search::toggle_lexicon_enabled)) + .route("/synonyms", get(search::list_synonyms).post(search::create_synonym)) + .route("/synonyms/{id}", put(search::update_synonym).delete(search::delete_synonym)) + .route("/synonyms/{id}/enabled", put(search::toggle_synonym_enabled)) + .route("/advanced/stream", get(search::advanced_search_stream)); + let graph_router = Router::new() + .route("/entities", get(graph::search_entities)) + .route("/entities/{id}", get(graph::get_entity)) + .route("/stats", get(graph::get_graph_stats)); + let system_router = Router::new() + .route("/heap", get(system::heap_profile)) + .route("/heap/pdf", get(system::heap_profile_pdf)) + .route("/lancedb/compact", post(system::lancedb_compact)) + .route("/index/force-merge", post(system::index_force_merge)) + .route("/index/rebuild/status", get(system::index_rebuild_status)); + + Router::new() + .nest("/knowledge_base/", knowledge_router) + .nest("/files/", file_router) + .nest("/search/", search_router) + .nest("/graph/", graph_router) + .nest("/system/", system_router) + .with_state(pool) + .layer(Extension(search_engine)) +} diff --git a/src/api/search.rs b/src/api/search.rs new file mode 100644 index 0000000..7a617de --- /dev/null +++ b/src/api/search.rs @@ -0,0 +1,2960 @@ +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + convert::Infallible, + time::Instant, +}; + +use anyhow::anyhow; +use axum::{ + Extension, + extract::{Multipart, Path, Query, State}, + response::{ + Json, + sse::{Event, KeepAlive, KeepAliveStream, Sse}, + }, +}; +use chrono::Utc; +use log::{error, info, warn}; +use once_cell::sync::Lazy; +use regex::Regex; +use serde::{Deserialize, Serialize, de}; +use serde_json::{Value, json}; +use sqlx::{QueryBuilder, Sqlite, SqlitePool}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use utoipa::{IntoParams, ToSchema}; + +use super::File; +use crate::{ + AuthUser, + api::{ + common, + error::{ApiError, ApiResult}, + }, + processor, + search::{ + RebuildProgress, SearchEngine, SearchResultItem as EngineSearchResultItem, + advanced::{ + ChunkRefiner, ChunkSegment, LlmClient, PlanAction, PlanStep, QueryPlanner, RefineOutcome, RelevanceJudge, + assemble_context_chunk, + }, + }, +}; + +#[derive(Debug, Deserialize, IntoParams)] +pub struct SearchQuery { + /// 搜索关键词 + pub query: String, + /// 文件 ID（可选，逗号分隔多个，如 1,2） + #[param(value_type = String, example = "1,2")] + #[serde(default, deserialize_with = "deserialize_id_list")] + pub file_id: Option>, + /// 知识库 ID（可选，逗号分隔多个，如 1,2） + #[param(value_type = String, example = "1,2")] + #[serde(default, deserialize_with = "deserialize_id_list")] + pub kb_id: Option>, + /// 是否启用高级流程（仅切片搜索接口生效） + #[serde(default)] + pub advanced: bool, +} + +/// 全文搜索查询参数 +#[derive(Debug, Deserialize, IntoParams)] +pub struct FullSearchQuery { + /// 搜索关键词（当 filename 传入时可不填） + #[serde(default)] + pub query: String, + /// 文件 ID（可选，逗号分隔多个，如 1,2） + #[param(value_type = String, example = "1,2")] + #[serde(default, deserialize_with = "deserialize_id_list")] + pub file_id: Option>, + /// 知识库 ID（可选，逗号分隔多个，如 1,2） + #[param(value_type = String, example = "1,2")] + #[serde(default, deserialize_with = "deserialize_id_list")] + pub kb_id: Option>, + /// 按文件名称全匹配过滤 + pub filename: Option, +} + +fn default_max_sub_queries() -> usize { + 3 +} + +fn default_per_query_limit() -> usize { + 10 +} + +fn default_context_chars() -> usize { + 2000 +} + +const ADVANCED_JUDGE_EARLY_STOP_SCORE: f32 = 0.8; + +/// 高级搜索参数 +#[derive(Debug, Deserialize, IntoParams, Clone)] +pub struct AdvancedSearchQuery { + /// 搜索关键词 + pub query: String, + /// 文件 ID（可选，逗号分隔多个，如 1,2） + #[param(value_type = String, example = "1,2")] + #[serde(default, deserialize_with = "deserialize_id_list")] + pub file_id: Option>, + /// 知识库 ID（可选，逗号分隔多个，如 1,2） + #[param(value_type = String, example = "1,2")] + #[serde(default, deserialize_with = "deserialize_id_list")] + pub kb_id: Option>, + /// 最大执行步骤数量 + #[serde(default = "default_max_sub_queries")] + #[param(example = 3)] + pub max_sub_queries: usize, + /// 每步处理的候选文档数量 + #[serde(default = "default_per_query_limit")] + #[param(example = 10)] + pub per_query_limit: usize, + /// 组装上下文时，单侧字符数上限 + #[serde(default = "default_context_chars")] + #[param(example = 2000)] + pub context_chars: usize, + /// 是否输出调试事件 + #[serde(default)] + pub debug: bool, +} + +fn deserialize_id_list<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let raw = Option::::deserialize(deserializer)?; + let Some(raw) = raw else { + return Ok(None); + }; + let raw = raw.trim(); + if raw.is_empty() { + return Ok(None); + } + + let mut ids = Vec::new(); + for part in raw.split(',') { + let part = part.trim(); + if part.is_empty() { + continue; + } + let id = part.parse::().map_err(de::Error::custom)?; + ids.push(id); + } + + if ids.is_empty() { Ok(None) } else { Ok(Some(ids)) } +} + +/// 文件信息 +#[derive(Debug, Clone, Serialize, sqlx::FromRow, ToSchema)] +pub struct FileInfo { + pub id: i64, + pub filename: String, + pub kb_id: Option, + pub is_public: bool, + pub user_id: String, + pub created_at: i64, +} + +/// 知识库信息 +#[derive(Debug, Clone, Serialize, sqlx::FromRow, ToSchema)] +pub struct KbInfo { + pub id: i64, + pub name: String, + pub is_public: bool, + pub user_id: String, +} + +/// 单个搜索结果项 +#[derive(Debug, Serialize, ToSchema)] +pub struct SearchResultItem { + /// 切片 ID + pub id: i64, + /// 文件 ID + pub file_id: i64, + /// 切片内容 + pub content: String, + /// 搜索得分 + pub score: f32, + /// 文件信息 + pub file: Option, + /// 知识库信息 + pub kb: Option, + /// 图片名称（仅图片相关搜索结果） + #[serde(skip_serializing_if = "Option::is_none")] + pub image_filename: Option, + /// 图片文本化描述（仅图片相关搜索结果） + #[serde(skip_serializing_if = "Option::is_none")] + pub image_content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub jpg_filename: Option, + #[serde(rename = "descprition", skip_serializing_if = "Option::is_none")] + pub descprition: Option, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct SearchResult { + pub results: Vec, +} + +/// 全文搜索结果项 +#[derive(Debug, Serialize, ToSchema)] +pub struct FullSearchResultItem { + /// 命中片段（HTML，包含高亮） + pub snippet: String, + /// 搜索得分 + pub score: f32, + /// 文件信息 + pub file: Option, + /// 知识库信息 + pub kb: Option, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct FullSearchResult { + pub results: Vec, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct SummarySearchResultItem { + /// 文件摘要 + pub summary: String, + /// 搜索得分 + pub score: f32, + /// 文件信息 + pub file: Option, + /// 知识库信息 + pub kb: Option, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct SummarySearchResult { + pub results: Vec, +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct ImageSearchQuery { + /// 文件 ID（可选，逗号分隔多个，如 1,2） + #[param(value_type = String, example = "1,2")] + #[serde(default, deserialize_with = "deserialize_id_list")] + pub file_id: Option>, + /// 知识库 ID（可选，逗号分隔多个，如 1,2） + #[param(value_type = String, example = "1,2")] + #[serde(default, deserialize_with = "deserialize_id_list")] + pub kb_id: Option>, +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct LexiconListQuery { + /// 关键字（匹配 term） + pub q: Option, + /// 启用状态过滤 + pub enabled: Option, + /// 返回数量上限 + #[serde(default = "default_synonym_limit")] + pub limit: i64, + /// 偏移量 + #[serde(default)] + pub offset: i64, +} + +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct LexiconItem { + pub id: i64, + pub term: String, + pub freq: Option, + pub tag: Option, + pub enabled: bool, + pub created_at: i64, + pub updated_at: i64, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct LexiconListResponse { + pub total: i64, + pub items: Vec, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct DeleteLexiconResponse { + pub id: i64, + pub deleted: bool, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ReloadLexiconResponse { + pub loaded: usize, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct PublishLexiconResponse { + pub job_id: i64, + pub status: String, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct CreateLexiconReq { + pub term: String, + pub freq: Option, + pub tag: Option, + #[serde(default = "default_synonym_enabled")] + pub enabled: bool, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct UpdateLexiconReq { + pub term: Option, + pub freq: Option, + pub tag: Option, + pub enabled: Option, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct ToggleLexiconReq { + pub enabled: bool, +} + +#[derive(Debug, sqlx::FromRow)] +struct LexiconItemRow { + id: i64, + term: String, + freq: Option, + tag: Option, + enabled: i64, + created_at: i64, + updated_at: i64, +} + +#[derive(Debug, Deserialize, IntoParams)] +pub struct SynonymListQuery { + /// 关键字（匹配 term 或 synonym） + pub q: Option, + /// 启用状态过滤 + pub enabled: Option, + /// 返回数量上限 + #[serde(default = "default_synonym_limit")] + pub limit: i64, + /// 偏移量 + #[serde(default)] + pub offset: i64, +} + +fn default_synonym_limit() -> i64 { + 50 +} + +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct SynonymItem { + pub id: i64, + pub term: String, + pub synonym: String, + pub weight: f32, + pub bidirectional: bool, + pub enabled: bool, + pub created_at: i64, + pub updated_at: i64, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct SynonymListResponse { + pub total: i64, + pub items: Vec, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct DeleteSynonymResponse { + pub id: i64, + pub deleted: bool, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct CreateSynonymReq { + pub term: String, + pub synonym: String, + #[serde(default = "default_synonym_weight")] + pub weight: f32, + #[serde(default = "default_synonym_bidirectional")] + pub bidirectional: bool, + #[serde(default = "default_synonym_enabled")] + pub enabled: bool, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct UpdateSynonymReq { + pub term: Option, + pub synonym: Option, + pub weight: Option, + pub bidirectional: Option, + pub enabled: Option, +} + +#[derive(Debug, Deserialize, ToSchema)] +pub struct ToggleSynonymReq { + pub enabled: bool, +} + +fn default_synonym_weight() -> f32 { + 1.0 +} + +fn default_synonym_bidirectional() -> bool { + true +} + +fn default_synonym_enabled() -> bool { + true +} + +#[derive(Debug, sqlx::FromRow)] +struct SynonymItemRow { + id: i64, + term: String, + synonym: String, + weight: f32, + bidirectional: i64, + enabled: i64, + created_at: i64, + updated_at: i64, +} + +/// 搜索内容 +#[utoipa::path( + get, + path = "/api/v1/knowledge/search/", + operation_id = "search_query", + tag = "search", + params(SearchQuery), + responses( + (status = 200, description = "搜索成功", body = SearchResult), + (status = 400, description = "请求参数错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn search( + State(pool): State, Extension(search_engine): Extension, + Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult> { + let total_started = Instant::now(); + let (_is_admin, user_id, kb_ids_to_search) = + resolve_scope_for_user(&pool, &auth_user, params.kb_id.as_ref()).await?; + if no_accessible_kb_scope(kb_ids_to_search.as_deref()) { + info!( + "search request completed: user_id={}, query=\"{}\", advanced={}, file_filter={}, kb_scope={}, raw_results=0, final_results=0, elapsed_ms={}", + user_id, + preview_for_log(¶ms.query, 120), + params.advanced, + format_id_filter(params.file_id.as_deref()), + summarize_kb_scope(kb_ids_to_search.as_deref()), + total_started.elapsed().as_millis() + ); + return Ok(Json(SearchResult { results: vec![] })); + } + + if params.advanced { + let request_id = uuid::Uuid::new_v4().to_string(); + let advanced_params = AdvancedSearchQuery { + query: params.query.clone(), + file_id: params.file_id.clone(), + kb_id: params.kb_id.clone(), + max_sub_queries: default_max_sub_queries(), + per_query_limit: default_per_query_limit(), + context_chars: default_context_chars(), + debug: false, + }; + info!( + "search advanced option enabled: request_id={}, user_id={}, query=\"{}\", file_filter={}, kb_scope={}", + request_id, + user_id, + preview_for_log(¶ms.query, 120), + format_id_filter(params.file_id.as_deref()), + summarize_kb_scope(kb_ids_to_search.as_deref()) + ); + let advanced_results = run_advanced_slice_search_non_stream( + &pool, + &search_engine, + &auth_user, + &advanced_params, + kb_ids_to_search.clone(), + &request_id, + ) + .await + .map_err(|e| ApiError::internal(format!("Advanced search failed: {}", e)))?; + return Ok(Json(SearchResult { results: advanced_results })); + } + + let raw_results = search_engine + .search(¶ms.query, params.file_id.as_ref(), kb_ids_to_search.as_ref()) + .await + .map_err(|e| crate::api::error::ApiError::internal(format!("Search failed: {}", e)))?; + + let assemble_started = Instant::now(); + let results = build_slice_results_from_raw(&pool, raw_results, &auth_user, true).await?; + info!( + "search request completed: user_id={}, query=\"{}\", advanced=false, file_filter={}, kb_scope={}, final_results={}, assemble_elapsed_ms={}, elapsed_ms={}", + user_id, + preview_for_log(¶ms.query, 120), + format_id_filter(params.file_id.as_deref()), + summarize_kb_scope(kb_ids_to_search.as_deref()), + results.len(), + assemble_started.elapsed().as_millis(), + total_started.elapsed().as_millis() + ); + + Ok(Json(SearchResult { results })) +} + +/// 高级搜索（SSE 流式返回） +#[utoipa::path( + get, + path = "/api/v1/knowledge/search/advanced/stream", + operation_id = "advanced_search_stream", + tag = "search", + params(AdvancedSearchQuery), + responses( + (status = 200, description = "SSE 流式搜索结果") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn advanced_search_stream( + State(pool): State, Extension(search_engine): Extension, + Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult>>>> { + let (is_admin, user_id, kb_ids_to_search) = + resolve_scope_for_user(&pool, &auth_user, params.kb_id.as_ref()).await?; + let request_id = uuid::Uuid::new_v4().to_string(); + + info!( + "advanced_search_stream request received: request_id={}, user_id={}, is_admin={}, query=\"{}\", file_filter={}, kb_filter={}, max_sub_queries={}, per_query_limit={}, context_chars={}, debug={}", + request_id, + user_id, + is_admin, + preview_for_log(¶ms.query, 120), + format_id_filter(params.file_id.as_deref()), + format_id_filter(params.kb_id.as_deref()), + params.max_sub_queries, + params.per_query_limit, + params.context_chars, + params.debug + ); + + let kb_resolve_started = Instant::now(); + info!( + "advanced_search_stream kb scope resolved: request_id={}, scope={}, elapsed_ms={}", + request_id, + summarize_kb_scope(kb_ids_to_search.as_deref()), + kb_resolve_started.elapsed().as_millis() + ); + + let (tx, rx) = mpsc::channel(32); + let pool_clone = pool.clone(); + let search_engine_clone = search_engine.clone(); + let auth_user_clone = auth_user.clone(); + let params_clone = params.clone(); + let kb_ids_clone = kb_ids_to_search.clone(); + let request_id_for_task = request_id.clone(); + + tokio::spawn(async move { + if let Err(err) = run_advanced_search_flow( + pool_clone, + search_engine_clone, + auth_user_clone, + params_clone, + kb_ids_clone, + tx, + request_id_for_task.clone(), + ) + .await + { + error!("advanced search stream failed: request_id={}, error={}", request_id_for_task, err); + } + }); + + info!("advanced_search_stream task spawned: request_id={}, channel_capacity={}", request_id, 32); + + Ok(Sse::new(ReceiverStream::new(rx)).keep_alive(KeepAlive::new())) +} + +async fn run_advanced_search_flow( + pool: SqlitePool, search_engine: SearchEngine, auth_user: AuthUser, params: AdvancedSearchQuery, + kb_ids: Option>, tx: mpsc::Sender>, request_id: String, +) -> anyhow::Result<()> { + let flow_started = Instant::now(); + let llm_client = LlmClient::new(); + let llm_enabled = llm_client.is_enabled(); + + info!( + "advanced_search_flow started: request_id={}, user_id={}, kb_scope={}, llm_enabled={}, query=\"{}\"", + request_id, + auth_user.user_id, + summarize_kb_scope(kb_ids.as_deref()), + llm_enabled, + preview_for_log(¶ms.query, 120) + ); + + let planner = QueryPlanner::new(llm_client.clone()); + let judge = RelevanceJudge::new(llm_client.clone()); + let chunk_refiner = ChunkRefiner::new(llm_client); + + let outcome = run_advanced_search_logic( + pool, + search_engine, + auth_user, + params, + kb_ids, + planner, + judge, + chunk_refiner, + &tx, + &request_id, + ) + .await; + + match outcome { + Ok(_) => { + let _ = send_status_event(&tx, "完成", "高级搜索已完成").await; + let _ = send_done_event(&tx).await; + info!( + "advanced_search_flow completed: request_id={}, elapsed_ms={}", + request_id, + flow_started.elapsed().as_millis() + ); + Ok(()) + } + Err(err) => { + let _ = send_error_event(&tx, &err.to_string()).await; + let _ = send_done_event(&tx).await; + info!( + "advanced_search_flow finished with error: request_id={}, elapsed_ms={}", + request_id, + flow_started.elapsed().as_millis() + ); + Err(err) + } + } +} + +#[allow(clippy::too_many_arguments)] +async fn run_advanced_search_logic( + pool: SqlitePool, search_engine: SearchEngine, auth_user: AuthUser, params: AdvancedSearchQuery, + kb_ids: Option>, planner: QueryPlanner, judge: RelevanceJudge, chunk_refiner: ChunkRefiner, + tx: &mpsc::Sender>, request_id: &str, +) -> anyhow::Result<()> { + let logic_started = Instant::now(); + let slice_limit = params.per_query_limit.max(1); + let context_chars = params.context_chars.max(1); + let user_id = auth_user.user_id.clone(); + let is_admin = auth_user.is_admin(); + + info!( + "advanced_search_logic started: request_id={}, user_id={}, is_admin={}, file_filter={}, kb_scope={}, slice_limit={}, context_chars={}, debug={}", + request_id, + user_id, + is_admin, + format_id_filter(params.file_id.as_deref()), + summarize_kb_scope(kb_ids.as_deref()), + slice_limit, + context_chars, + params.debug + ); + + if no_accessible_kb_scope(kb_ids.as_deref()) { + info!("advanced_search_logic exits early: request_id={}, reason=no_accessible_kb", request_id); + send_status_event(tx, "权限校验", "无可访问的知识库，直接结束").await?; + return Ok(()); + } + + let _selected = run_advanced_plan_steps( + &pool, + &search_engine, + ¶ms, + kb_ids.as_ref(), + &planner, + &judge, + &chunk_refiner, + slice_limit, + context_chars, + &user_id, + is_admin, + Some(tx), + request_id, + "advanced_search_logic", + ) + .await?; + + info!( + "advanced_search_logic completed: request_id={}, elapsed_ms={}", + request_id, + logic_started.elapsed().as_millis() + ); + + Ok(()) +} + +#[derive(Debug, Serialize)] +struct AdvancedResultPayload { + step_action: PlanAction, + #[serde(skip_serializing_if = "Option::is_none")] + file: Option, + #[serde(skip_serializing_if = "Option::is_none")] + kb: Option, + slice_ids: Vec, + score: f32, + judge_score: f32, + judge_reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + refine_reason: Option, + content: String, +} + +#[derive(Clone)] +struct SliceCandidate { + file: FileInfo, + kb: Option, + slice: EngineSearchResultItem, +} + +struct SelectedAnswerCandidate { + file: FileInfo, + kb: Option, + base_slice_id: i64, + base_score: f32, + judge_score: f32, + judge_reason: String, + context_segments: Vec, + context_content: String, +} + +struct AdvancedSelectedSliceResult { + base_slice_id: i64, + file: FileInfo, + kb: Option, + score: f32, + content: String, +} + +#[derive(Default)] +struct PageContentStepStats { + inspected_slices: usize, + context_error_count: usize, + judge_rejected_count: usize, + refine_error_count: usize, + empty_slice_group_count: usize, + empty_content_count: usize, +} + +struct FinalizedAnswerCandidate { + file: FileInfo, + kb: Option, + base_slice_id: i64, + base_score: f32, + judge_score: f32, + judge_reason: String, + final_segments: Vec, + final_content: String, + refine_reason: Option, +} + +struct PageContentCoreOutcome { + selected: Option, + stats: PageContentStepStats, +} + +#[derive(Clone, Copy, Debug)] +enum EventType { + Plan, + Step, + Status, + Error, + Done, + Result, + Candidate, + Filtered, +} + +impl EventType { + fn name(&self) -> &'static str { + match self { + EventType::Plan => "plan", + EventType::Step => "step", + EventType::Status => "status", + EventType::Error => "error", + EventType::Done => "done", + EventType::Result => "result", + EventType::Candidate => "candidate", + EventType::Filtered => "filtered", + } + } +} + +async fn send_event( + tx: &mpsc::Sender>, event_type: EventType, payload: &T, +) -> anyhow::Result<()> { + let evt = Event::default() + .event(event_type.name()) + .json_data(payload) + .map_err(|e| anyhow!("SSE JSON encode failed: {}", e))?; + tx.send(Ok(evt)).await.map_err(|_| anyhow!("SSE client disconnected")) +} + +async fn maybe_send_event( + tx: Option<&mpsc::Sender>>, event_type: EventType, payload: &T, +) -> anyhow::Result<()> { + if let Some(tx) = tx { + send_event(tx, event_type, payload).await?; + } + Ok(()) +} + +async fn maybe_send_plan_event( + tx: Option<&mpsc::Sender>>, steps: &[PlanStep], +) -> anyhow::Result<()> { + maybe_send_event(tx, EventType::Plan, &json!({ "steps": steps })).await +} + +async fn maybe_send_step_event( + tx: Option<&mpsc::Sender>>, step: &PlanStep, status: &str, details: Option, +) -> anyhow::Result<()> { + let mut payload = json!({ + "action": step.action, + "comment": step.comment, + "status": status, + }); + if let Value::Object(ref mut map) = payload + && let Some(details) = details + { + map.insert("details".to_string(), details); + } + maybe_send_event(tx, EventType::Step, &payload).await +} + +async fn send_status_event( + tx: &mpsc::Sender>, phase: &str, message: &str, +) -> anyhow::Result<()> { + send_event(tx, EventType::Status, &json!({ "phase": phase, "message": message })).await +} + +async fn maybe_send_status_event( + tx: Option<&mpsc::Sender>>, phase: &str, message: &str, +) -> anyhow::Result<()> { + maybe_send_event(tx, EventType::Status, &json!({ "phase": phase, "message": message })).await +} + +async fn send_error_event(tx: &mpsc::Sender>, message: &str) -> anyhow::Result<()> { + send_event(tx, EventType::Error, &json!({ "message": message })).await +} + +async fn send_done_event(tx: &mpsc::Sender>) -> anyhow::Result<()> { + send_event(tx, EventType::Done, &json!({})).await +} + +fn has_visibility_permission( + file: Option<(bool, &str)>, kb: Option<(bool, &str, i64)>, user_id: &str, is_admin: bool, + allowed_kb_ids: Option<&HashSet>, +) -> bool { + if is_admin { + return true; + } + // If a KB is associated, permission is determined at the KB level. + if let Some((kb_is_public, kb_owner_id, kb_id)) = kb { + if kb_is_public || kb_owner_id == user_id { + return true; + } + if let Some(allowed) = allowed_kb_ids { + return allowed.contains(&kb_id); + } + return false; + } + // Unassigned file: check file-level ownership/public flag. + if let Some((is_public, owner_id)) = file { + return is_public || owner_id == user_id; + } + true +} + +fn has_permission( + file: Option<&FileInfo>, kb: Option<&KbInfo>, user_id: &str, is_admin: bool, allowed_kb_ids: Option<&HashSet>, +) -> bool { + has_visibility_permission( + file.map(|f| (f.is_public, f.user_id.as_str())), + kb.map(|k| (k.is_public, k.user_id.as_str(), k.id)), + user_id, + is_admin, + allowed_kb_ids, + ) +} + +fn no_accessible_kb_scope(kb_ids: Option<&[i64]>) -> bool { + matches!(kb_ids, Some(ids) if ids.is_empty()) +} + +fn file_matches_kb_scope(file: &File, kb_ids: Option<&[i64]>) -> bool { + kb_ids.is_none_or(|ids| file.kb_id.is_some_and(|kb_id| ids.contains(&kb_id))) +} + +async fn resolve_scope_for_user( + pool: &SqlitePool, auth_user: &AuthUser, kb_filter: Option<&Vec>, +) -> ApiResult<(bool, String, Option>)> { + let is_admin = auth_user.is_admin(); + let user_id = auth_user.user_id.clone(); + let kb_ids = resolve_kb_ids_to_search(pool, &user_id, is_admin, kb_filter).await?; + Ok((is_admin, user_id, kb_ids)) +} + +async fn build_slice_results_from_raw( + pool: &SqlitePool, raw_results: Vec, auth_user: &AuthUser, dedupe_by_content: bool, +) -> ApiResult> { + if raw_results.is_empty() { + return Ok(Vec::new()); + } + + let is_admin = auth_user.is_admin(); + let user_id = auth_user.user_id.clone(); + + let file_ids: Vec = raw_results.iter().map(|r| r.file_id).collect(); + let kb_ids: Vec = raw_results.iter().filter_map(|r| r.kb_id).collect(); + + // 文件和知识库查询相互独立，并发执行 + let kb_fut = async { if !kb_ids.is_empty() { get_kbs_by_ids(pool, &kb_ids).await } else { Ok(HashMap::new()) } }; + let (file_map, kb_map) = tokio::try_join!(get_files_by_ids(pool, &file_ids), kb_fut)?; + + let allowed_kb_ids: HashSet = if is_admin { + HashSet::new() + } else { + crate::api::knowledge_base::get_user_viewable_kb_ids(pool, &user_id, false).await.into_iter().collect() + }; + let allowed_ref = if is_admin { None } else { Some(&allowed_kb_ids) }; + + let mut seen_contents: HashSet = HashSet::new(); + let mut results = Vec::new(); + for r in raw_results { + let file = file_map.get(&r.file_id).cloned(); + let kb = r.kb_id.and_then(|kb_id| kb_map.get(&kb_id).cloned()); + if !has_permission(file.as_ref(), kb.as_ref(), &user_id, is_admin, allowed_ref) { + continue; + } + if dedupe_by_content && !seen_contents.insert(r.content.clone()) { + continue; + } + results.push(SearchResultItem { + id: r.id, + file_id: r.file_id, + content: r.content, + score: r.score, + file, + kb, + image_filename: None, + image_content: None, + jpg_filename: None, + descprition: None, + }); + } + Ok(results) +} + +/// 匹配切片内容中的图片引用，例如 `![img.png](/api/v1/knowledge/files/img.png)`。 +static IMAGE_REF_RE: Lazy = + Lazy::new(|| Regex::new(r"!\[.*?\]\(/api/v1/knowledge/files/([^)]+)\)").expect("image ref regex is valid")); + +fn extract_image_filename_from_content(content: &str) -> Option { + IMAGE_REF_RE.captures(content)?.get(1).map(|m| m.as_str().to_string()) +} + +/// 为图片相关搜索结果补充图片名称和 split_image 返回的 image_content（描述）。 +async fn enrich_image_search_results( + pool: &SqlitePool, mut results: Vec, +) -> ApiResult> { + let image_file_ids: HashSet = results + .iter() + .filter(|r| extract_image_filename_from_content(&r.content).is_some()) + .map(|r| r.file_id) + .collect(); + if image_file_ids.is_empty() { + return Ok(results); + } + + let descriptions = + crate::image_description::list_by_file_ids(pool, &image_file_ids.into_iter().collect::>()) + .await + .map_err(|e| ApiError::internal(format!("Failed to load image descriptions: {}", e)))?; + + for result in &mut results { + let Some(filename) = extract_image_filename_from_content(&result.content) else { continue }; + let original_content = result.content.clone(); + result.image_filename = Some(filename.clone()); + result.descprition = Some(original_content); + let (description, jpg_filename) = descriptions + .get(&(result.file_id, filename.clone())) + .cloned() + .unwrap_or_else(|| (String::new(), None)); + if !description.is_empty() { + result.image_content = Some(description); + } + let jpg_filename = jpg_filename.unwrap_or(filename); + result.jpg_filename = Some(jpg_filename.clone()); + let source_filename = result.file.as_ref().map(|file| file.filename.as_str()).unwrap_or_default(); + let kb_id = result + .file + .as_ref() + .and_then(|file| file.kb_id) + .map(|id| id.to_string()) + .unwrap_or_else(|| "null".to_string()); + result.content = format!("{}/{}/{}", source_filename, kb_id, jpg_filename); + } + + Ok(results) +} + +#[allow(clippy::too_many_arguments)] +async fn collect_relevant_slices( + pool: &SqlitePool, search_engine: &SearchEngine, query: &str, file_filter: Option<&Vec>, + kb_filter: Option<&Vec>, slice_limit: usize, user_id: &str, is_admin: bool, request_id: &str, +) -> anyhow::Result> { + let collect_started = Instant::now(); + info!( + "collect_relevant_slices started: request_id={}, query=\"{}\", file_filter={}, kb_filter={}, slice_limit={}, user_id={}, is_admin={}", + request_id, + preview_for_log(query, 120), + format_id_filter(file_filter.map(Vec::as_slice)), + format_id_filter(kb_filter.map(Vec::as_slice)), + slice_limit, + user_id, + is_admin + ); + + let mut raw_results = + search_engine.search(query, file_filter, kb_filter).await.map_err(|e| anyhow!("Search failed: {}", e))?; + let raw_count = raw_results.len(); + + if raw_results.is_empty() { + info!( + "collect_relevant_slices no results: request_id={}, elapsed_ms={}", + request_id, + collect_started.elapsed().as_millis() + ); + return Ok(Vec::new()); + } + + let file_ids: Vec = raw_results.iter().map(|r| r.file_id).collect(); + let kb_ids: Vec = raw_results.iter().filter_map(|r| r.kb_id).collect(); + let file_map = get_files_by_ids(pool, &file_ids).await?; + let kb_map = if !kb_ids.is_empty() { get_kbs_by_ids(pool, &kb_ids).await? } else { HashMap::new() }; + + let allowed_kb_ids: HashSet = if is_admin { + HashSet::new() + } else { + crate::api::knowledge_base::get_user_viewable_kb_ids(pool, user_id, false).await.into_iter().collect() + }; + let allowed_ref = if is_admin { None } else { Some(&allowed_kb_ids) }; + + raw_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal)); + + let mut selected_slices = Vec::new(); + let mut missing_file_count = 0usize; + let mut permission_denied_count = 0usize; + let mut duplicate_slice_count = 0usize; + let mut seen_slice_ids: HashSet = HashSet::new(); + + for slice in raw_results { + if selected_slices.len() >= slice_limit { + break; + } + if !seen_slice_ids.insert(slice.id) { + duplicate_slice_count += 1; + continue; + } + let file_id = slice.file_id; + let Some(file) = file_map.get(&file_id).cloned() else { + missing_file_count += 1; + continue; + }; + let kb = slice + .kb_id + .and_then(|kid| kb_map.get(&kid).cloned()) + .or_else(|| file.kb_id.and_then(|kid| kb_map.get(&kid).cloned())); + if !has_permission(Some(&file), kb.as_ref(), user_id, is_admin, allowed_ref) { + permission_denied_count += 1; + continue; + } + selected_slices.push(SliceCandidate { file, kb, slice }); + } + + info!( + "collect_relevant_slices completed: request_id={}, raw_slices={}, selected_slices={}, missing_file={}, permission_denied={}, duplicate_slice={}, elapsed_ms={}", + request_id, + raw_count, + selected_slices.len(), + missing_file_count, + permission_denied_count, + duplicate_slice_count, + collect_started.elapsed().as_millis() + ); + + Ok(selected_slices) +} + +fn summarize_slice_candidates(candidates: &[SliceCandidate]) -> Vec { + candidates + .iter() + .map(|candidate| { + let preview = preview_text(&candidate.slice.content, 160); + json!({ + "slice_id": candidate.slice.id, + "file_id": candidate.slice.file_id, + "file": candidate.file.clone(), + "kb": candidate.kb.clone(), + "score": candidate.slice.score, + "preview": preview, + }) + }) + .collect() +} + +#[allow(clippy::too_many_arguments)] +async fn run_advanced_plan_steps( + pool: &SqlitePool, search_engine: &SearchEngine, params: &AdvancedSearchQuery, kb_ids: Option<&Vec>, + planner: &QueryPlanner, judge: &RelevanceJudge, chunk_refiner: &ChunkRefiner, slice_limit: usize, + context_chars: usize, user_id: &str, is_admin: bool, tx: Option<&mpsc::Sender>>, + request_id: &str, log_prefix: &str, +) -> anyhow::Result> { + let planning_started = Instant::now(); + maybe_send_status_event(tx, "初始化", "生成执行计划").await?; + let steps = planner.plan(¶ms.query, params.max_sub_queries.max(1)).await; + let step_actions = steps.iter().map(|step| format!("{:?}", step.action)).collect::>().join(" -> "); + info!( + "{} plan generated: request_id={}, step_count={}, actions=\"{}\", elapsed_ms={}", + log_prefix, + request_id, + steps.len(), + step_actions, + planning_started.elapsed().as_millis() + ); + + if steps.is_empty() { + info!("{} exits early: request_id={}, reason=empty_plan", log_prefix, request_id); + maybe_send_status_event(tx, "计划", "未生成有效计划").await?; + return Ok(None); + } + maybe_send_plan_event(tx, &steps).await?; + + let mut slice_candidates: Vec = Vec::new(); + let mut selected: Option = None; + + for (step_idx, step) in steps.into_iter().enumerate() { + let step_started = Instant::now(); + let step_action = step.action.clone(); + info!( + "{} step started: request_id={}, step_index={}, action={:?}, comment=\"{}\"", + log_prefix, + request_id, + step_idx + 1, + step_action, + preview_for_log(&step.comment, 120) + ); + maybe_send_step_event(tx, &step, "started", None).await?; + + match step_action { + PlanAction::RecentDocuments => { + slice_candidates = collect_relevant_slices( + pool, + search_engine, + ¶ms.query, + params.file_id.as_ref(), + kb_ids, + slice_limit, + user_id, + is_admin, + request_id, + ) + .await?; + + info!( + "{} step completed: request_id={}, step_index={}, action=RecentDocuments, slice_candidates={}, elapsed_ms={}", + log_prefix, + request_id, + step_idx + 1, + slice_candidates.len(), + step_started.elapsed().as_millis() + ); + let summary = summarize_slice_candidates(&slice_candidates); + maybe_send_step_event( + tx, + &step, + "completed", + Some(json!({ "count": slice_candidates.len(), "slices": summary })), + ) + .await?; + } + PlanAction::DocumentStructure => { + info!( + "{} step skipped: request_id={}, step_index={}, action=DocumentStructure, reason=flow_simplified", + log_prefix, + request_id, + step_idx + 1 + ); + maybe_send_step_event(tx, &step, "skipped", Some(json!({ "reason": "当前流程不执行文档结构分析" }))) + .await?; + } + PlanAction::PageContent => { + if slice_candidates.is_empty() { + info!( + "{} step refilling slices: request_id={}, step_index={}, action=PageContent", + log_prefix, + request_id, + step_idx + 1 + ); + slice_candidates = collect_relevant_slices( + pool, + search_engine, + ¶ms.query, + params.file_id.as_ref(), + kb_ids, + slice_limit, + user_id, + is_admin, + request_id, + ) + .await?; + } + + if slice_candidates.is_empty() { + info!( + "{} step skipped: request_id={}, step_index={}, action=PageContent, reason=no_slices", + log_prefix, + request_id, + step_idx + 1 + ); + maybe_send_step_event(tx, &step, "skipped", Some(json!({ "reason": "未找到相关切片" }))).await?; + maybe_send_status_event(tx, "结果", "未找到可用于回答问题的内容").await?; + continue; + } + + let outcome = execute_page_content_step_core( + pool, + &slice_candidates, + slice_limit, + context_chars, + judge, + chunk_refiner, + ¶ms.query, + request_id, + if tx.is_some() { "execute_page_content_step" } else { "execute_page_content_step_non_stream" }, + tx, + tx.is_some() && params.debug, + tx.is_some(), + ) + .await?; + + let mut emitted = 0usize; + if let Some(finalized) = outcome.selected { + let slice_ids = finalized.final_segments.iter().map(|seg| seg.slice_id).collect::>(); + let payload = AdvancedResultPayload { + step_action: PlanAction::PageContent, + file: Some(finalized.file.clone()), + kb: finalized.kb.clone(), + slice_ids: slice_ids.clone(), + score: finalized.base_score, + judge_score: finalized.judge_score, + judge_reason: finalized.judge_reason.clone(), + refine_reason: finalized.refine_reason.clone(), + content: finalized.final_content.clone(), + }; + maybe_send_event(tx, EventType::Result, &payload).await?; + if tx.is_some() { + info!( + "execute_page_content_step emitted result after refine: request_id={}, file_id={}, kb_id={:?}, base_slice_id={}, slice_count={}, score={:.4}, judge_score={:.4}, refine_reason=\"{}\"", + request_id, + finalized.file.id, + finalized.kb.as_ref().map(|kb| kb.id), + finalized.base_slice_id, + payload.slice_ids.len(), + finalized.base_score, + finalized.judge_score, + preview_for_log(finalized.refine_reason.as_deref().unwrap_or("none"), 120) + ); + } + + selected = Some(AdvancedSelectedSliceResult { + base_slice_id: finalized.base_slice_id, + file: finalized.file, + kb: finalized.kb, + score: finalized.base_score, + content: finalized.final_content, + }); + emitted = 1; + } + + info!( + "{} step completed: request_id={}, step_index={}, action=PageContent, emitted={}, inspected_slices={}, judge_rejected={}, context_error={}, refine_error={}, empty_slice_group={}, empty_content={}, elapsed_ms={}", + log_prefix, + request_id, + step_idx + 1, + emitted, + outcome.stats.inspected_slices, + outcome.stats.judge_rejected_count, + outcome.stats.context_error_count, + outcome.stats.refine_error_count, + outcome.stats.empty_slice_group_count, + outcome.stats.empty_content_count, + step_started.elapsed().as_millis() + ); + + if emitted > 0 { + maybe_send_status_event(tx, "结果", "已找到可用于回答问题的内容，停止继续检索").await?; + } else { + maybe_send_status_event(tx, "结果", "未找到可用于回答问题的内容").await?; + } + maybe_send_step_event( + tx, + &step, + "completed", + Some(json!({ "emitted": emitted, "found": emitted > 0 })), + ) + .await?; + + if emitted > 0 { + break; + } + } + } + } + + Ok(selected) +} + +async fn run_advanced_slice_search_non_stream( + pool: &SqlitePool, search_engine: &SearchEngine, auth_user: &AuthUser, params: &AdvancedSearchQuery, + kb_ids: Option>, request_id: &str, +) -> anyhow::Result> { + if no_accessible_kb_scope(kb_ids.as_deref()) { + return Ok(Vec::new()); + } + + let llm_client = LlmClient::new(); + let planner = QueryPlanner::new(llm_client.clone()); + let judge = RelevanceJudge::new(llm_client.clone()); + let chunk_refiner = ChunkRefiner::new(llm_client); + + let slice_limit = params.per_query_limit.max(1); + let context_chars = params.context_chars.max(1); + let user_id = auth_user.user_id.clone(); + let is_admin = auth_user.is_admin(); + + let selected = run_advanced_plan_steps( + pool, + search_engine, + params, + kb_ids.as_ref(), + &planner, + &judge, + &chunk_refiner, + slice_limit, + context_chars, + &user_id, + is_admin, + None, + request_id, + "advanced_slice_search_non_stream", + ) + .await?; + + let Some(selected) = selected else { + return Ok(Vec::new()); + }; + + Ok(vec![SearchResultItem { + id: selected.base_slice_id, + file_id: selected.file.id, + content: selected.content, + score: selected.score, + file: Some(selected.file), + kb: selected.kb, + image_filename: None, + image_content: None, + jpg_filename: None, + descprition: None, + }]) +} + +#[allow(clippy::too_many_arguments)] +async fn execute_page_content_step_core( + pool: &SqlitePool, candidates: &[SliceCandidate], max_slices: usize, context_chars: usize, judge: &RelevanceJudge, + chunk_refiner: &ChunkRefiner, query: &str, request_id: &str, log_prefix: &str, + tx: Option<&mpsc::Sender>>, debug_events: bool, verbose_logs: bool, +) -> anyhow::Result { + let mut stats = PageContentStepStats::default(); + let max_slices = max_slices.max(1); + let mut selected_candidate: Option = None; + let mut best_candidate: Option = None; + + for candidate in candidates.iter().take(max_slices) { + stats.inspected_slices += 1; + let base_slice = &candidate.slice; + if verbose_logs { + info!( + "{} evaluating candidate before_context: request_id={}, file_id={}, kb_id={:?}, base_slice_id={}, base_score={:.4}, preview=\"{}\"", + log_prefix, + request_id, + candidate.file.id, + candidate.kb.as_ref().map(|kb| kb.id), + base_slice.id, + base_slice.score, + preview_for_log(&base_slice.content, 120) + ); + } + + let context_chunk = match assemble_context_chunk(pool, base_slice, context_chars).await { + Ok(chunk) => chunk, + Err(err) => { + stats.context_error_count += 1; + info!( + "{} context assemble failed: request_id={}, file_id={}, base_slice_id={}, error={}", + log_prefix, request_id, candidate.file.id, base_slice.id, err + ); + if debug_events && let Some(tx) = tx { + let _ = send_event( + tx, + EventType::Candidate, + &json!({ + "step_action": PlanAction::PageContent, + "file": candidate.file.clone(), + "kb": candidate.kb.clone(), + "error": format!("上下文拼接失败: {}", err), + }), + ) + .await; + } + continue; + } + }; + + if debug_events && let Some(tx) = tx { + let preview = preview_text(&context_chunk.content, 160); + let _ = send_event( + tx, + EventType::Candidate, + &json!({ + "step_action": PlanAction::PageContent, + "file": candidate.file.clone(), + "kb": candidate.kb.clone(), + "score": base_slice.score, + "slice_ids": context_chunk.slice_ids.clone(), + "preview": preview, + "stage": "before_refine", + }), + ) + .await; + } + + let judge_outcome = judge.judge(query, &context_chunk.content).await; + if !judge_outcome.is_relevant { + stats.judge_rejected_count += 1; + if verbose_logs { + info!( + "{} candidate rejected: request_id={}, file_id={}, kb_id={:?}, base_slice_id={}, judge_score={:.4}, reason=\"{}\"", + log_prefix, + request_id, + candidate.file.id, + candidate.kb.as_ref().map(|kb| kb.id), + base_slice.id, + judge_outcome.score, + preview_for_log(&judge_outcome.reason, 120) + ); + } + if debug_events && let Some(tx) = tx { + let _ = send_event( + tx, + EventType::Filtered, + &json!({ + "step_action": PlanAction::PageContent, + "file": candidate.file.clone(), + "kb": candidate.kb.clone(), + "reason": judge_outcome.reason, + "score": judge_outcome.score, + }), + ) + .await; + } + continue; + } + + let current_candidate = SelectedAnswerCandidate { + file: candidate.file.clone(), + kb: candidate.kb.clone(), + base_slice_id: base_slice.id, + base_score: base_slice.score, + judge_score: judge_outcome.score, + judge_reason: judge_outcome.reason.clone(), + context_segments: context_chunk.segments, + context_content: context_chunk.content, + }; + + if current_candidate.judge_score > ADVANCED_JUDGE_EARLY_STOP_SCORE { + if verbose_logs { + info!( + "{} selected candidate for early stop: request_id={}, file_id={}, kb_id={:?}, base_slice_id={}, score={:.4}, judge_score={:.4}", + log_prefix, + request_id, + candidate.file.id, + candidate.kb.as_ref().map(|kb| kb.id), + base_slice.id, + base_slice.score, + judge_outcome.score + ); + } + selected_candidate = Some(current_candidate); + break; + } + + let should_replace_best = best_candidate.as_ref().is_none_or(|best| { + current_candidate.judge_score > best.judge_score + || ((current_candidate.judge_score - best.judge_score).abs() < f32::EPSILON + && current_candidate.base_score > best.base_score) + }); + if should_replace_best { + if verbose_logs { + info!( + "{} updated best candidate: request_id={}, file_id={}, kb_id={:?}, base_slice_id={}, score={:.4}, judge_score={:.4}", + log_prefix, + request_id, + candidate.file.id, + candidate.kb.as_ref().map(|kb| kb.id), + base_slice.id, + base_slice.score, + judge_outcome.score + ); + } + best_candidate = Some(current_candidate); + } + } + + if selected_candidate.is_none() { + selected_candidate = best_candidate; + } + + let Some(selected) = selected_candidate else { + return Ok(PageContentCoreOutcome { selected: None, stats }); + }; + + if verbose_logs { + info!( + "{} refining selected candidate: request_id={}, file_id={}, kb_id={:?}, base_slice_id={}, judge_score={:.4}", + log_prefix, + request_id, + selected.file.id, + selected.kb.as_ref().map(|kb| kb.id), + selected.base_slice_id, + selected.judge_score + ); + } + + let refine_outcome = match chunk_refiner.refine(query, &selected.context_segments).await { + Ok(outcome) => outcome, + Err(err) => { + stats.refine_error_count += 1; + info!( + "{} refine failed: request_id={}, file_id={}, base_slice_id={}, error={}", + log_prefix, request_id, selected.file.id, selected.base_slice_id, err + ); + RefineOutcome { + segments: selected.context_segments.clone(), + reason: Some("切片筛选失败，保留全部上下文".to_string()), + } + } + }; + + let refine_reason = refine_outcome.reason.clone(); + let mut final_segments = refine_outcome.segments; + if final_segments.is_empty() { + stats.empty_slice_group_count += 1; + final_segments = selected.context_segments.clone(); + } + let mut final_content = final_segments.iter().map(|seg| seg.text.as_str()).collect::>().join("\n"); + if final_content.trim().is_empty() { + stats.empty_content_count += 1; + final_content = selected.context_content.clone(); + } + if final_content.trim().is_empty() { + if verbose_logs { + info!( + "{} selected candidate has empty content after refine: request_id={}, file_id={}, base_slice_id={}", + log_prefix, request_id, selected.file.id, selected.base_slice_id + ); + } + return Ok(PageContentCoreOutcome { selected: None, stats }); + } + + Ok(PageContentCoreOutcome { + selected: Some(FinalizedAnswerCandidate { + base_slice_id: selected.base_slice_id, + file: selected.file, + kb: selected.kb, + base_score: selected.base_score, + judge_score: selected.judge_score, + judge_reason: selected.judge_reason, + final_segments, + final_content, + refine_reason, + }), + stats, + }) +} + +fn format_id_filter(ids: Option<&[i64]>) -> String { + const MAX_IDS: usize = 12; + match ids { + None => "all".to_string(), + Some([]) => "[]".to_string(), + Some(list) if list.len() <= MAX_IDS => format!("{:?}", list), + Some(list) => { + let head = list.iter().take(MAX_IDS).map(|id| id.to_string()).collect::>().join(","); + format!("[{}...](total={})", head, list.len()) + } + } +} + +fn summarize_kb_scope(kb_ids: Option<&[i64]>) -> String { + match kb_ids { + None => "all_accessible_kbs".to_string(), + Some([]) => "no_accessible_kbs".to_string(), + Some(ids) => format!("{} kbs {}", ids.len(), format_id_filter(Some(ids))), + } +} + +fn preview_for_log(text: &str, max_chars: usize) -> String { + preview_text(&text.replace('\n', " "), max_chars) +} + +fn preview_text(text: &str, max_chars: usize) -> String { + let trimmed = text.trim(); + if trimmed.is_empty() { + return String::new(); + } + let mut buf = String::new(); + for (idx, ch) in trimmed.chars().enumerate() { + if idx >= max_chars { + buf.push_str("..."); + break; + } + buf.push(ch); + } + buf +} + +/// 全文搜索（仅 Tantivy 全文索引） +#[utoipa::path( + get, + path = "/api/v1/knowledge/search/full", + operation_id = "search_full", + tag = "search", + params(FullSearchQuery), + responses( + (status = 200, description = "全文搜索成功", body = FullSearchResult), + (status = 400, description = "请求参数错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn search_full( + State(pool): State, Extension(search_engine): Extension, + Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult> { + let (is_admin, user_id, kb_ids_to_search) = + resolve_scope_for_user(&pool, &auth_user, params.kb_id.as_ref()).await?; + if no_accessible_kb_scope(kb_ids_to_search.as_deref()) { + return Ok(Json(FullSearchResult { results: vec![] })); + } + + let allowed_kb_ids: HashSet = if is_admin { + HashSet::new() + } else { + crate::api::knowledge_base::get_user_viewable_kb_ids(&pool, &user_id, false).await.into_iter().collect() + }; + let allowed_ref = if is_admin { None } else { Some(&allowed_kb_ids) }; + + let file_ids = if let Some(filename) = params.filename.as_ref().filter(|f| !f.is_empty()) { + let matched_ids = + search_file_ids_by_name(&pool, filename, kb_ids_to_search.as_ref(), &user_id, is_admin).await?; + if matched_ids.is_empty() { + return Ok(Json(FullSearchResult { results: vec![] })); + } + match params.file_id { + Some(ref explicit_ids) if !explicit_ids.is_empty() => { + let explicit_set: HashSet = explicit_ids.iter().copied().collect(); + let intersected: Vec = matched_ids.into_iter().filter(|id| explicit_set.contains(id)).collect(); + if intersected.is_empty() { + return Ok(Json(FullSearchResult { results: vec![] })); + } + Some(intersected) + } + _ => Some(matched_ids), + } + } else { + params.file_id.clone() + }; + + let results = if params.query.trim().is_empty() { + if file_ids.is_none() { + return Ok(Json(FullSearchResult { results: vec![] })); + } + let ids = file_ids.unwrap_or_default(); + if ids.is_empty() { + return Ok(Json(FullSearchResult { results: vec![] })); + } + let file_map = get_full_files_by_ids(&pool, &ids).await?; + let kb_ids_in_files: Vec = file_map.values().filter_map(|f| f.kb_id).collect(); + let kb_map = + if !kb_ids_in_files.is_empty() { get_kbs_by_ids(&pool, &kb_ids_in_files).await? } else { HashMap::new() }; + file_map + .values() + .filter_map(|f| { + if !has_visibility_permission( + Some((f.is_public, f.user_id.as_str())), + f.kb_id.and_then(|kid| kb_map.get(&kid)).map(|k| (k.is_public, k.user_id.as_str(), k.id)), + &user_id, + is_admin, + allowed_ref, + ) { + return None; + } + Some(FullSearchResultItem { + snippet: String::new(), + score: 0.0, + file: Some(f.clone()), + kb: f.kb_id.and_then(|kid| kb_map.get(&kid).cloned()), + }) + }) + .collect::>() + } else { + let raw_results = search_engine + .search_full(¶ms.query, file_ids.as_ref(), kb_ids_to_search.as_ref()) + .await + .map_err(|e| crate::api::error::ApiError::internal(format!("Full search failed: {}", e)))?; + + if raw_results.is_empty() { + return Ok(Json(FullSearchResult { results: vec![] })); + } + + let file_ids: Vec = raw_results.iter().map(|r| r.file_id).collect(); + let kb_ids: Vec = raw_results.iter().filter_map(|r| r.kb_id).collect(); + let file_map = get_full_files_by_ids(&pool, &file_ids).await?; + let kb_map = if !kb_ids.is_empty() { get_kbs_by_ids(&pool, &kb_ids).await? } else { HashMap::new() }; + + raw_results + .into_iter() + .filter_map(|r| { + let file = file_map.get(&r.file_id).cloned(); + let kb = r.kb_id.and_then(|kb_id| kb_map.get(&kb_id).cloned()); + if has_visibility_permission( + file.as_ref().map(|f| (f.is_public, f.user_id.as_str())), + kb.as_ref().map(|k| (k.is_public, k.user_id.as_str(), k.id)), + &user_id, + is_admin, + allowed_ref, + ) { + Some(FullSearchResultItem { snippet: r.snippet, score: r.score, file, kb }) + } else { + None + } + }) + .collect() + }; + + Ok(Json(FullSearchResult { results })) +} + +/// 文件摘要语义搜索 +#[utoipa::path( + get, + path = "/api/v1/knowledge/search/summary", + operation_id = "search_summary", + tag = "search", + params(FullSearchQuery), + responses( + (status = 200, description = "文件摘要搜索成功", body = SummarySearchResult), + (status = 400, description = "请求参数错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn search_summary( + State(pool): State, Extension(search_engine): Extension, + Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult> { + let (is_admin, user_id, kb_ids_to_search) = + resolve_scope_for_user(&pool, &auth_user, params.kb_id.as_ref()).await?; + if no_accessible_kb_scope(kb_ids_to_search.as_deref()) { + return Ok(Json(SummarySearchResult { results: vec![] })); + } + + let allowed_kb_ids: HashSet = if is_admin { + HashSet::new() + } else { + crate::api::knowledge_base::get_user_viewable_kb_ids(&pool, &user_id, false).await.into_iter().collect() + }; + let allowed_ref = if is_admin { None } else { Some(&allowed_kb_ids) }; + + let file_ids = if let Some(filename) = params.filename.as_ref().filter(|f| !f.is_empty()) { + let matched_ids = + search_file_ids_by_name(&pool, filename, kb_ids_to_search.as_ref(), &user_id, is_admin).await?; + if matched_ids.is_empty() { + return Ok(Json(SummarySearchResult { results: vec![] })); + } + match params.file_id { + Some(ref explicit_ids) if !explicit_ids.is_empty() => { + let explicit_set: HashSet = explicit_ids.iter().copied().collect(); + let intersected: Vec = matched_ids.into_iter().filter(|id| explicit_set.contains(id)).collect(); + if intersected.is_empty() { + return Ok(Json(SummarySearchResult { results: vec![] })); + } + Some(intersected) + } + _ => Some(matched_ids), + } + } else { + params.file_id.clone() + }; + + let results = if params.query.trim().is_empty() { + if file_ids.is_none() { + return Ok(Json(SummarySearchResult { results: vec![] })); + } + let ids = file_ids.unwrap_or_default(); + if ids.is_empty() { + return Ok(Json(SummarySearchResult { results: vec![] })); + } + let file_map = get_full_files_by_ids(&pool, &ids).await?; + let kb_ids_in_files: Vec = file_map.values().filter_map(|f| f.kb_id).collect(); + let kb_map = + if !kb_ids_in_files.is_empty() { get_kbs_by_ids(&pool, &kb_ids_in_files).await? } else { HashMap::new() }; + file_map + .values() + .filter_map(|f| { + let summary = f.summary.as_deref().unwrap_or("").trim(); + if summary.is_empty() + || !file_matches_kb_scope(f, kb_ids_to_search.as_deref()) + || !has_visibility_permission( + Some((f.is_public, f.user_id.as_str())), + f.kb_id.and_then(|kid| kb_map.get(&kid)).map(|k| (k.is_public, k.user_id.as_str(), k.id)), + &user_id, + is_admin, + allowed_ref, + ) + { + return None; + } + Some(SummarySearchResultItem { + summary: summary.to_string(), + score: 0.0, + file: Some(f.clone()), + kb: f.kb_id.and_then(|kid| kb_map.get(&kid).cloned()), + }) + }) + .collect::>() + } else { + let raw_results = search_engine + .search_summary(¶ms.query, file_ids.as_ref(), kb_ids_to_search.as_ref()) + .await + .map_err(|e| crate::api::error::ApiError::internal(format!("Summary search failed: {}", e)))?; + + if raw_results.is_empty() { + return Ok(Json(SummarySearchResult { results: vec![] })); + } + + let file_ids: Vec = raw_results.iter().map(|r| r.file_id).collect(); + let file_map = get_full_files_by_ids(&pool, &file_ids).await?; + let kb_ids: Vec = file_map.values().filter_map(|f| f.kb_id).collect(); + let kb_map = if !kb_ids.is_empty() { get_kbs_by_ids(&pool, &kb_ids).await? } else { HashMap::new() }; + + raw_results + .into_iter() + .filter_map(|r| { + let Some(file) = file_map.get(&r.file_id).cloned() else { + warn!("Summary search skipped orphan vector result for missing file {}", r.file_id); + return None; + }; + let kb = file.kb_id.and_then(|kb_id| kb_map.get(&kb_id).cloned()); + if file_matches_kb_scope(&file, kb_ids_to_search.as_deref()) + && has_visibility_permission( + Some((file.is_public, file.user_id.as_str())), + kb.as_ref().map(|k| (k.is_public, k.user_id.as_str(), k.id)), + &user_id, + is_admin, + allowed_ref, + ) + { + Some(SummarySearchResultItem { summary: r.summary, score: r.score, file: Some(file), kb }) + } else { + None + } + }) + .collect() + }; + + Ok(Json(SummarySearchResult { results })) +} + +/// 使用知识图谱增强的搜索 +#[utoipa::path( + get, + path = "/api/v1/knowledge/search/graph", + operation_id = "search_with_graph", + tag = "search", + params(SearchQuery), + responses( + (status = 200, description = "图谱搜索成功", body = SearchResult), + (status = 400, description = "请求参数错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn search_with_graph( + State(pool): State, Extension(search_engine): Extension, + Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult> { + let (_is_admin, _user_id, kb_ids_to_search) = + resolve_scope_for_user(&pool, &auth_user, params.kb_id.as_ref()).await?; + if no_accessible_kb_scope(kb_ids_to_search.as_deref()) { + return Ok(Json(SearchResult { results: vec![] })); + } + + let raw_results = search_engine + .search_with_graph_expansion(¶ms.query, params.file_id.as_ref(), kb_ids_to_search.as_ref()) + .await + .map_err(|e| crate::api::error::ApiError::internal(format!("Graph search failed: {}", e)))?; + + let results = build_slice_results_from_raw(&pool, raw_results, &auth_user, false).await?; + + Ok(Json(SearchResult { results })) +} + +/// 以图搜图 +#[utoipa::path( + post, + path = "/api/v1/knowledge/search/image", + operation_id = "search_image", + tag = "search", + params(ImageSearchQuery), + request_body(content_type = "multipart/form-data"), + responses( + (status = 200, description = "图片搜索成功", body = SearchResult), + (status = 400, description = "请求参数错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn search_image( + State(pool): State, Extension(search_engine): Extension, + Query(params): Query, Extension(auth_user): Extension, mut multipart: Multipart, +) -> ApiResult> { + let mut file_bytes: Option> = None; + let mut file_name = "image".to_string(); + let mut content_type: Option = None; + let mut text: Option = None; + + loop { + match multipart.next_field().await { + Ok(Some(field)) => { + let name = field.name().unwrap_or_default().to_string(); + match name.as_str() { + "file" => { + file_name = field.file_name().unwrap_or("image").to_string(); + content_type = field.content_type().map(|ct| ct.to_string()); + file_bytes = Some(field.bytes().await?.to_vec()); + } + "text" => { + let value = field.text().await?; + if !value.trim().is_empty() { + text = Some(value); + } + } + _ => {} + } + } + Ok(None) => break, + Err(e) => { + return Err(ApiError::Internal(format!("Multipart error: {}", e))); + } + } + } + + let Some(file_bytes) = file_bytes else { + return Err(ApiError::BadRequest("file is required".to_string())); + }; + + let (_is_admin, _user_id, kb_ids_to_search) = + resolve_scope_for_user(&pool, &auth_user, params.kb_id.as_ref()).await?; + if no_accessible_kb_scope(kb_ids_to_search.as_deref()) { + return Ok(Json(SearchResult { results: vec![] })); + } + + if !crate::search::embedding::image_embedding_enabled() { + return Err(ApiError::BadRequest("Image embedding service is not configured".to_string())); + } + + let image_embedding = crate::search::embedding::get_image_embedding_from_bytes( + &file_name, + content_type.as_deref(), + file_bytes, + text.as_deref(), + ) + .await + .map_err(|e| ApiError::Internal(format!("Image embedding failed: {}", e)))?; + + let raw_results = search_engine + .search_image(image_embedding, params.file_id.as_ref(), kb_ids_to_search.as_ref()) + .await + .map_err(|e| ApiError::internal(format!("Image search failed: {}", e)))?; + + let results = build_slice_results_from_raw(&pool, raw_results, &auth_user, false).await?; + + Ok(Json(SearchResult { results })) +} + +/// 基于图片描述的文本检索 +#[utoipa::path( + get, + path = "/api/v1/knowledge/search/image-by-text", + operation_id = "search_image_by_text", + tag = "search", + params(SearchQuery), + responses( + (status = 200, description = "图片文本化检索成功", body = SearchResult), + (status = 400, description = "请求参数错误"), + (status = 500, description = "服务器内部错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn search_image_by_text( + State(pool): State, Extension(search_engine): Extension, + Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult> { + let query = params.query.trim(); + if query.is_empty() { + return Err(ApiError::BadRequest("query is required".to_string())); + } + + let (_is_admin, _user_id, kb_ids_to_search) = + resolve_scope_for_user(&pool, &auth_user, params.kb_id.as_ref()).await?; + if no_accessible_kb_scope(kb_ids_to_search.as_deref()) { + return Ok(Json(SearchResult { results: vec![] })); + } + + let raw_results = search_engine + .search_image_by_text(query, params.file_id.as_ref(), kb_ids_to_search.as_ref()) + .await + .map_err(|e| ApiError::internal(format!("Image-by-text search failed: {}", e)))?; + + let results = build_slice_results_from_raw(&pool, raw_results, &auth_user, false).await?; + let results = enrich_image_search_results(&pool, results).await?; + + Ok(Json(SearchResult { results })) +} + +/// 词表列表（管理员） +#[utoipa::path( + get, + path = "/api/v1/knowledge/search/lexicons", + operation_id = "search_lexicon_list", + tag = "search", + params(LexiconListQuery), + responses( + (status = 200, description = "词表列表查询成功", body = LexiconListResponse), + (status = 400, description = "请求参数错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn list_lexicons( + State(pool): State, Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let q = params.q.as_deref().map(str::trim).filter(|s| !s.is_empty()).map(str::to_string); + let limit = params.limit.clamp(1, 200); + let offset = params.offset.max(0); + + let mut count_qb = QueryBuilder::::new("SELECT COUNT(*) FROM search_lexicon WHERE 1 = 1"); + if let Some(enabled) = params.enabled { + count_qb.push(" AND enabled = "); + count_qb.push_bind(if enabled { 1_i64 } else { 0_i64 }); + } + if let Some(keyword) = &q { + let pattern = format!("%{}%", keyword); + count_qb.push(" AND term LIKE "); + count_qb.push_bind(pattern); + } + let total: i64 = count_qb.build_query_scalar().fetch_one(&pool).await?; + + let mut list_qb = QueryBuilder::::new( + "SELECT id, term, freq, tag, enabled, created_at, updated_at FROM search_lexicon WHERE 1 = 1", + ); + if let Some(enabled) = params.enabled { + list_qb.push(" AND enabled = "); + list_qb.push_bind(if enabled { 1_i64 } else { 0_i64 }); + } + if let Some(keyword) = &q { + let pattern = format!("%{}%", keyword); + list_qb.push(" AND term LIKE "); + list_qb.push_bind(pattern); + } + list_qb.push(" ORDER BY updated_at DESC, id DESC LIMIT "); + list_qb.push_bind(limit); + list_qb.push(" OFFSET "); + list_qb.push_bind(offset); + + let rows: Vec = list_qb.build_query_as().fetch_all(&pool).await?; + let items = rows.into_iter().map(lexicon_row_to_item).collect(); + Ok(Json(LexiconListResponse { total, items })) +} + +/// 新增词表词条（管理员） +#[utoipa::path( + post, + path = "/api/v1/knowledge/search/lexicons", + operation_id = "search_lexicon_create", + tag = "search", + request_body = CreateLexiconReq, + responses( + (status = 200, description = "词条新增成功", body = LexiconItem), + (status = 400, description = "请求参数错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn create_lexicon( + State(pool): State, Extension(auth_user): Extension, Json(req): Json, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let term = normalize_lexicon_term(req.term.as_str())?; + let freq = normalize_lexicon_freq(req.freq)?; + let tag = normalize_lexicon_tag(req.tag.as_deref()); + + let insert_result = sqlx::query("INSERT INTO search_lexicon (term, freq, tag, enabled) VALUES (?, ?, ?, ?)") + .bind(term.as_str()) + .bind(freq) + .bind(tag.as_deref()) + .bind(if req.enabled { 1_i64 } else { 0_i64 }) + .execute(&pool) + .await; + + let result = match insert_result { + Ok(result) => result, + Err(sqlx::Error::Database(db_err)) if db_err.message().contains("UNIQUE") => { + return Err(ApiError::BadRequest("lexicon term already exists".to_string())); + } + Err(e) => return Err(e.into()), + }; + + let id = result.last_insert_rowid(); + let row = fetch_lexicon_row_by_id(&pool, id) + .await? + .ok_or_else(|| ApiError::internal("created lexicon term not found"))?; + Ok(Json(lexicon_row_to_item(row))) +} + +/// 更新词表词条（管理员） +#[utoipa::path( + put, + path = "/api/v1/knowledge/search/lexicons/{id}", + operation_id = "search_lexicon_update", + tag = "search", + params( + ("id" = i64, Path, description = "词条 ID") + ), + request_body = UpdateLexiconReq, + responses( + (status = 200, description = "词条更新成功", body = LexiconItem), + (status = 400, description = "请求参数错误"), + (status = 404, description = "词条不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn update_lexicon( + State(pool): State, Path(id): Path, Extension(auth_user): Extension, + Json(req): Json, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + if req.term.is_none() && req.freq.is_none() && req.tag.is_none() && req.enabled.is_none() { + return Err(ApiError::BadRequest("no fields to update".to_string())); + } + + let current = fetch_lexicon_row_by_id(&pool, id) + .await? + .ok_or_else(|| ApiError::NotFound("lexicon term not found".to_string()))?; + let term = match req.term.as_deref() { + Some(value) => normalize_lexicon_term(value)?, + None => current.term.clone(), + }; + let freq = match req.freq { + Some(value) => normalize_lexicon_freq(Some(value))?, + None => current.freq, + }; + let tag = match req.tag.as_deref() { + Some(value) => normalize_lexicon_tag(Some(value)), + None => current.tag.clone(), + }; + let enabled = req.enabled.unwrap_or(current.enabled != 0); + + let update_result = sqlx::query( + "UPDATE search_lexicon SET term = ?, freq = ?, tag = ?, enabled = ?, updated_at = strftime('%s','now') WHERE id = ?", + ) + .bind(term.as_str()) + .bind(freq) + .bind(tag.as_deref()) + .bind(if enabled { 1_i64 } else { 0_i64 }) + .bind(id) + .execute(&pool) + .await; + + match update_result { + Ok(result) if result.rows_affected() == 0 => { + return Err(ApiError::NotFound("lexicon term not found".to_string())); + } + Err(sqlx::Error::Database(db_err)) if db_err.message().contains("UNIQUE") => { + return Err(ApiError::BadRequest("lexicon term already exists".to_string())); + } + Err(e) => return Err(e.into()), + _ => {} + } + + let row = fetch_lexicon_row_by_id(&pool, id) + .await? + .ok_or_else(|| ApiError::NotFound("lexicon term not found".to_string()))?; + Ok(Json(lexicon_row_to_item(row))) +} + +/// 删除词表词条（管理员） +#[utoipa::path( + delete, + path = "/api/v1/knowledge/search/lexicons/{id}", + operation_id = "search_lexicon_delete", + tag = "search", + params( + ("id" = i64, Path, description = "词条 ID") + ), + responses( + (status = 200, description = "词条删除成功", body = DeleteLexiconResponse), + (status = 404, description = "词条不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn delete_lexicon( + State(pool): State, Path(id): Path, Extension(auth_user): Extension, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let result = sqlx::query("DELETE FROM search_lexicon WHERE id = ?").bind(id).execute(&pool).await?; + if result.rows_affected() == 0 { + return Err(ApiError::NotFound("lexicon term not found".to_string())); + } + + Ok(Json(DeleteLexiconResponse { id, deleted: true })) +} + +/// 启用/停用词表词条（管理员） +#[utoipa::path( + put, + path = "/api/v1/knowledge/search/lexicons/{id}/enabled", + operation_id = "search_lexicon_toggle_enabled", + tag = "search", + params( + ("id" = i64, Path, description = "词条 ID") + ), + request_body = ToggleLexiconReq, + responses( + (status = 200, description = "词条状态更新成功", body = LexiconItem), + (status = 404, description = "词条不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn toggle_lexicon_enabled( + State(pool): State, Path(id): Path, Extension(auth_user): Extension, + Json(req): Json, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let result = sqlx::query("UPDATE search_lexicon SET enabled = ?, updated_at = strftime('%s','now') WHERE id = ?") + .bind(if req.enabled { 1_i64 } else { 0_i64 }) + .bind(id) + .execute(&pool) + .await?; + if result.rows_affected() == 0 { + return Err(ApiError::NotFound("lexicon term not found".to_string())); + } + + let row = fetch_lexicon_row_by_id(&pool, id) + .await? + .ok_or_else(|| ApiError::NotFound("lexicon term not found".to_string()))?; + Ok(Json(lexicon_row_to_item(row))) +} + +/// 重新加载词表到分词器（管理员） +#[utoipa::path( + post, + path = "/api/v1/knowledge/search/lexicons/reload", + operation_id = "search_lexicon_reload", + tag = "search", + responses( + (status = 200, description = "词表重载成功", body = ReloadLexiconResponse) + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn reload_lexicon( + Extension(search_engine): Extension, Extension(auth_user): Extension, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let loaded = search_engine + .reload_lexicon() + .await + .map_err(|e| ApiError::internal(format!("reload lexicon failed: {}", e)))?; + Ok(Json(ReloadLexiconResponse { loaded })) +} + +/// 发布词表并触发索引重建（管理员） +#[utoipa::path( + post, + path = "/api/v1/knowledge/search/lexicons/publish", + operation_id = "search_lexicon_publish", + tag = "search", + responses( + (status = 200, description = "发布成功，返回重建任务信息", body = PublishLexiconResponse), + (status = 400, description = "已有重建任务在运行或权限不足") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn publish_lexicon( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let running_job_id: Option = + sqlx::query_scalar("SELECT id FROM index_rebuild_jobs WHERE status = 'running' ORDER BY id DESC LIMIT 1") + .fetch_optional(&pool) + .await?; + if let Some(job_id) = running_job_id { + return Err(ApiError::BadRequest(format!("index rebuild job {} is already running", job_id))); + } + + let now = Utc::now().timestamp(); + let result = sqlx::query( + "INSERT INTO index_rebuild_jobs (status, phase, total_docs, processed_docs, started_at, updated_at, finished_at, error) \ + VALUES ('running', 'queued', 0, 0, ?, ?, NULL, NULL)", + ) + .bind(now) + .bind(now) + .execute(&pool) + .await?; + let job_id = result.last_insert_rowid(); + + processor::set_parse_paused(true); + + let pool_clone = pool.clone(); + let search_engine_clone = search_engine.clone(); + tokio::spawn(async move { + if let Err(err) = run_lexicon_publish_job(pool_clone, search_engine_clone, job_id).await { + warn!("Lexicon publish job {} failed: {}", job_id, err); + } + }); + + Ok(Json(PublishLexiconResponse { job_id, status: "running".to_string() })) +} + +/// 同义词列表（管理员） +#[utoipa::path( + get, + path = "/api/v1/knowledge/search/synonyms", + operation_id = "search_synonym_list", + tag = "search", + params(SynonymListQuery), + responses( + (status = 200, description = "同义词列表查询成功", body = SynonymListResponse), + (status = 400, description = "请求参数错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn list_synonyms( + State(pool): State, Query(params): Query, Extension(auth_user): Extension, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let q = params.q.as_deref().map(str::trim).filter(|s| !s.is_empty()).map(str::to_string); + let limit = params.limit.clamp(1, 200); + let offset = params.offset.max(0); + + let mut count_qb = QueryBuilder::::new("SELECT COUNT(*) FROM search_synonyms WHERE 1 = 1"); + if let Some(enabled) = params.enabled { + count_qb.push(" AND enabled = "); + count_qb.push_bind(if enabled { 1_i64 } else { 0_i64 }); + } + if let Some(keyword) = &q { + let pattern = format!("%{}%", keyword); + count_qb.push(" AND (term LIKE "); + count_qb.push_bind(pattern.clone()); + count_qb.push(" OR synonym LIKE "); + count_qb.push_bind(pattern); + count_qb.push(")"); + } + let total: i64 = count_qb.build_query_scalar().fetch_one(&pool).await?; + + let mut list_qb = QueryBuilder::::new( + "SELECT id, term, synonym, weight, bidirectional, enabled, created_at, updated_at \ + FROM search_synonyms WHERE 1 = 1", + ); + if let Some(enabled) = params.enabled { + list_qb.push(" AND enabled = "); + list_qb.push_bind(if enabled { 1_i64 } else { 0_i64 }); + } + if let Some(keyword) = &q { + let pattern = format!("%{}%", keyword); + list_qb.push(" AND (term LIKE "); + list_qb.push_bind(pattern.clone()); + list_qb.push(" OR synonym LIKE "); + list_qb.push_bind(pattern); + list_qb.push(")"); + } + list_qb.push(" ORDER BY updated_at DESC, id DESC LIMIT "); + list_qb.push_bind(limit); + list_qb.push(" OFFSET "); + list_qb.push_bind(offset); + + let rows: Vec = list_qb.build_query_as().fetch_all(&pool).await?; + let items = rows.into_iter().map(synonym_row_to_item).collect(); + Ok(Json(SynonymListResponse { total, items })) +} + +/// 新增同义词（管理员） +#[utoipa::path( + post, + path = "/api/v1/knowledge/search/synonyms", + operation_id = "search_synonym_create", + tag = "search", + request_body = CreateSynonymReq, + responses( + (status = 200, description = "同义词新增成功", body = SynonymItem), + (status = 400, description = "请求参数错误") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn create_synonym( + State(pool): State, Extension(search_engine): Extension, + Extension(auth_user): Extension, Json(req): Json, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let term = req.term.trim(); + let synonym = req.synonym.trim(); + if term.is_empty() || synonym.is_empty() { + return Err(ApiError::BadRequest("term and synonym are required".to_string())); + } + if term == synonym { + return Err(ApiError::BadRequest("term and synonym must be different".to_string())); + } + if req.weight <= 0.0 { + return Err(ApiError::BadRequest("weight must be > 0".to_string())); + } + + let insert_result = sqlx::query( + "INSERT INTO search_synonyms (term, synonym, weight, bidirectional, enabled) VALUES (?, ?, ?, ?, ?)", + ) + .bind(term) + .bind(synonym) + .bind(req.weight) + .bind(if req.bidirectional { 1_i64 } else { 0_i64 }) + .bind(if req.enabled { 1_i64 } else { 0_i64 }) + .execute(&pool) + .await; + + let result = match insert_result { + Ok(result) => result, + Err(sqlx::Error::Database(db_err)) if db_err.message().contains("UNIQUE") => { + return Err(ApiError::BadRequest("synonym pair already exists".to_string())); + } + Err(e) => return Err(e.into()), + }; + + let id = result.last_insert_rowid(); + let row = + fetch_synonym_row_by_id(&pool, id).await?.ok_or_else(|| ApiError::internal("created synonym not found"))?; + search_engine.invalidate_synonym_cache().await; + Ok(Json(synonym_row_to_item(row))) +} + +/// 更新同义词（管理员） +#[utoipa::path( + put, + path = "/api/v1/knowledge/search/synonyms/{id}", + operation_id = "search_synonym_update", + tag = "search", + params( + ("id" = i64, Path, description = "同义词 ID") + ), + request_body = UpdateSynonymReq, + responses( + (status = 200, description = "同义词更新成功", body = SynonymItem), + (status = 400, description = "请求参数错误"), + (status = 404, description = "同义词不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn update_synonym( + State(pool): State, Path(id): Path, Extension(search_engine): Extension, + Extension(auth_user): Extension, Json(req): Json, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + if req.term.is_none() + && req.synonym.is_none() + && req.weight.is_none() + && req.bidirectional.is_none() + && req.enabled.is_none() + { + return Err(ApiError::BadRequest("no fields to update".to_string())); + } + + let current = + fetch_synonym_row_by_id(&pool, id).await?.ok_or_else(|| ApiError::NotFound("synonym not found".to_string()))?; + let term = req.term.as_deref().map(str::trim).unwrap_or(current.term.as_str()); + let synonym = req.synonym.as_deref().map(str::trim).unwrap_or(current.synonym.as_str()); + if term.is_empty() || synonym.is_empty() { + return Err(ApiError::BadRequest("term and synonym cannot be empty".to_string())); + } + if term == synonym { + return Err(ApiError::BadRequest("term and synonym must be different".to_string())); + } + let weight = req.weight.unwrap_or(current.weight); + if weight <= 0.0 { + return Err(ApiError::BadRequest("weight must be > 0".to_string())); + } + let bidirectional = req.bidirectional.unwrap_or(current.bidirectional != 0); + let enabled = req.enabled.unwrap_or(current.enabled != 0); + + let update_result = sqlx::query( + "UPDATE search_synonyms SET term = ?, synonym = ?, weight = ?, bidirectional = ?, enabled = ?, updated_at = strftime('%s','now') WHERE id = ?", + ) + .bind(term) + .bind(synonym) + .bind(weight) + .bind(if bidirectional { 1_i64 } else { 0_i64 }) + .bind(if enabled { 1_i64 } else { 0_i64 }) + .bind(id) + .execute(&pool) + .await; + + match update_result { + Ok(result) if result.rows_affected() == 0 => { + return Err(ApiError::NotFound("synonym not found".to_string())); + } + Err(sqlx::Error::Database(db_err)) if db_err.message().contains("UNIQUE") => { + return Err(ApiError::BadRequest("synonym pair already exists".to_string())); + } + Err(e) => return Err(e.into()), + _ => {} + } + + let row = + fetch_synonym_row_by_id(&pool, id).await?.ok_or_else(|| ApiError::NotFound("synonym not found".to_string()))?; + search_engine.invalidate_synonym_cache().await; + Ok(Json(synonym_row_to_item(row))) +} + +/// 删除同义词（管理员） +#[utoipa::path( + delete, + path = "/api/v1/knowledge/search/synonyms/{id}", + operation_id = "search_synonym_delete", + tag = "search", + params( + ("id" = i64, Path, description = "同义词 ID") + ), + responses( + (status = 200, description = "同义词删除成功", body = DeleteSynonymResponse), + (status = 404, description = "同义词不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn delete_synonym( + State(pool): State, Path(id): Path, Extension(search_engine): Extension, + Extension(auth_user): Extension, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let result = sqlx::query("DELETE FROM search_synonyms WHERE id = ?").bind(id).execute(&pool).await?; + if result.rows_affected() == 0 { + return Err(ApiError::NotFound("synonym not found".to_string())); + } + + search_engine.invalidate_synonym_cache().await; + Ok(Json(DeleteSynonymResponse { id, deleted: true })) +} + +/// 启用/停用同义词（管理员） +#[utoipa::path( + put, + path = "/api/v1/knowledge/search/synonyms/{id}/enabled", + operation_id = "search_synonym_toggle_enabled", + tag = "search", + params( + ("id" = i64, Path, description = "同义词 ID") + ), + request_body = ToggleSynonymReq, + responses( + (status = 200, description = "同义词状态更新成功", body = SynonymItem), + (status = 404, description = "同义词不存在") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn toggle_synonym_enabled( + State(pool): State, Path(id): Path, Extension(search_engine): Extension, + Extension(auth_user): Extension, Json(req): Json, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let result = sqlx::query("UPDATE search_synonyms SET enabled = ?, updated_at = strftime('%s','now') WHERE id = ?") + .bind(if req.enabled { 1_i64 } else { 0_i64 }) + .bind(id) + .execute(&pool) + .await?; + if result.rows_affected() == 0 { + return Err(ApiError::NotFound("synonym not found".to_string())); + } + + let row = + fetch_synonym_row_by_id(&pool, id).await?.ok_or_else(|| ApiError::NotFound("synonym not found".to_string()))?; + search_engine.invalidate_synonym_cache().await; + Ok(Json(synonym_row_to_item(row))) +} + +async fn run_lexicon_publish_job(pool: SqlitePool, search_engine: SearchEngine, job_id: i64) -> anyhow::Result<()> { + let run_result: anyhow::Result<()> = async { + update_rebuild_job_progress(&pool, job_id, "reload_lexicon", 0, 0).await?; + let loaded = search_engine.reload_lexicon().await?; + info!("Search lexicon published for job {}: {} words", job_id, loaded); + + search_engine + .rebuild_tantivy_indexes(&format!("job-{}", job_id), |progress: RebuildProgress| { + let pool = pool.clone(); + async move { + if let Err(e) = update_rebuild_job_progress( + &pool, + job_id, + progress.phase.as_str(), + progress.total_docs, + progress.processed_docs, + ) + .await + { + warn!("Failed to update index rebuild progress for job {}: {}", job_id, e); + } + } + }) + .await?; + + mark_rebuild_job_completed(&pool, job_id).await?; + Ok(()) + } + .await; + + match run_result { + Ok(()) => {} + Err(err) => { + if let Err(update_err) = mark_rebuild_job_failed(&pool, job_id, &err.to_string()).await { + warn!("Failed to mark rebuild job {} as failed: {}", job_id, update_err); + } + processor::set_parse_paused(false); + return Err(err); + } + } + + processor::set_parse_paused(false); + Ok(()) +} + +async fn update_rebuild_job_progress( + pool: &SqlitePool, job_id: i64, phase: &str, total_docs: i64, processed_docs: i64, +) -> anyhow::Result<()> { + sqlx::query( + "UPDATE index_rebuild_jobs \ + SET status = 'running', phase = ?, total_docs = ?, processed_docs = ?, updated_at = strftime('%s','now'), finished_at = NULL, error = NULL \ + WHERE id = ?", + ) + .bind(phase) + .bind(total_docs.max(0)) + .bind(processed_docs.max(0)) + .bind(job_id) + .execute(pool) + .await?; + Ok(()) +} + +async fn mark_rebuild_job_completed(pool: &SqlitePool, job_id: i64) -> anyhow::Result<()> { + sqlx::query( + "UPDATE index_rebuild_jobs \ + SET status = 'completed', phase = 'completed', processed_docs = total_docs, updated_at = strftime('%s','now'), finished_at = strftime('%s','now'), error = NULL \ + WHERE id = ?", + ) + .bind(job_id) + .execute(pool) + .await?; + Ok(()) +} + +async fn mark_rebuild_job_failed(pool: &SqlitePool, job_id: i64, error: &str) -> anyhow::Result<()> { + sqlx::query( + "UPDATE index_rebuild_jobs \ + SET status = 'failed', phase = 'failed', updated_at = strftime('%s','now'), finished_at = strftime('%s','now'), error = ? \ + WHERE id = ?", + ) + .bind(error) + .bind(job_id) + .execute(pool) + .await?; + Ok(()) +} + +fn normalize_lexicon_term(input: &str) -> ApiResult { + let term = input.trim(); + if term.is_empty() { + return Err(ApiError::BadRequest("term cannot be empty".to_string())); + } + Ok(term.to_string()) +} + +fn normalize_lexicon_freq(freq: Option) -> ApiResult> { + match freq { + Some(value) if value <= 0 => Err(ApiError::BadRequest("freq must be > 0".to_string())), + Some(value) => Ok(Some(value)), + None => Ok(None), + } +} + +fn normalize_lexicon_tag(tag: Option<&str>) -> Option { + tag.and_then(|value| { + let trimmed = value.trim(); + if trimmed.is_empty() { None } else { Some(trimmed.to_string()) } + }) +} + +async fn fetch_lexicon_row_by_id(pool: &SqlitePool, id: i64) -> Result, sqlx::Error> { + sqlx::query_as::<_, LexiconItemRow>( + "SELECT id, term, freq, tag, enabled, created_at, updated_at FROM search_lexicon WHERE id = ?", + ) + .bind(id) + .fetch_optional(pool) + .await +} + +fn lexicon_row_to_item(row: LexiconItemRow) -> LexiconItem { + LexiconItem { + id: row.id, + term: row.term, + freq: row.freq, + tag: row.tag, + enabled: row.enabled != 0, + created_at: row.created_at, + updated_at: row.updated_at, + } +} + +async fn fetch_synonym_row_by_id(pool: &SqlitePool, id: i64) -> Result, sqlx::Error> { + sqlx::query_as::<_, SynonymItemRow>( + "SELECT id, term, synonym, weight, bidirectional, enabled, created_at, updated_at FROM search_synonyms WHERE id = ?", + ) + .bind(id) + .fetch_optional(pool) + .await +} + +fn synonym_row_to_item(row: SynonymItemRow) -> SynonymItem { + SynonymItem { + id: row.id, + term: row.term, + synonym: row.synonym, + weight: row.weight, + bidirectional: row.bidirectional != 0, + enabled: row.enabled != 0, + created_at: row.created_at, + updated_at: row.updated_at, + } +} + +async fn search_file_ids_by_name( + pool: &SqlitePool, filename: &str, kb_ids: Option<&Vec>, user_id: &str, is_admin: bool, +) -> Result, sqlx::Error> { + let mut qb = QueryBuilder::::new("SELECT id FROM files WHERE filename = "); + qb.push_bind(filename); + if !is_admin { + qb.push(" AND (user_id = "); + qb.push_bind(user_id); + qb.push(" OR is_public = 1 OR kb_id IN (SELECT kb_id FROM kb_permissions WHERE user_id = "); + qb.push_bind(user_id); + qb.push("))"); + } + if let Some(ids) = kb_ids + && !ids.is_empty() + { + qb.push(" AND kb_id IN ("); + let mut separated = qb.separated(", "); + for id in ids { + separated.push_bind(id); + } + qb.push(")"); + } + qb.push(" AND status = 1"); + let ids: Vec = qb.build_query_scalar().fetch_all(pool).await?; + Ok(ids) +} + +async fn get_files_by_ids(pool: &SqlitePool, file_ids: &[i64]) -> Result, sqlx::Error> { + if file_ids.is_empty() { + return Ok(HashMap::new()); + } + + let placeholders: String = file_ids.iter().map(|_| "?").collect::>().join(","); + let query = + format!("SELECT id, filename, kb_id, is_public, user_id, created_at FROM files WHERE id IN ({})", placeholders); + + let mut q = sqlx::query_as::<_, FileInfo>(&query); + for id in file_ids { + q = q.bind(id); + } + + let files: Vec = q.fetch_all(pool).await?; + Ok(files.into_iter().map(|f| (f.id, f)).collect()) +} + +async fn resolve_kb_ids_to_search( + pool: &SqlitePool, user_id: &str, is_admin: bool, root_kb_ids: Option<&Vec>, +) -> ApiResult>> { + let Some(root_kb_ids) = root_kb_ids else { + return Ok(None); + }; + if root_kb_ids.is_empty() { + return Ok(Some(Vec::new())); + } + + let mut qb = QueryBuilder::::new("WITH RECURSIVE kb_hierarchy AS ("); + qb.push("SELECT id FROM knowledge_bases WHERE id IN ("); + let mut separated = qb.separated(", "); + for kb_id in root_kb_ids { + separated.push_bind(kb_id); + } + qb.push(")"); + if !is_admin { + qb.push(" AND (user_id = "); + qb.push_bind(user_id); + qb.push(" OR is_public = 1)"); + } + qb.push(" UNION ALL "); + qb.push("SELECT kb.id FROM knowledge_bases kb "); + qb.push("INNER JOIN kb_hierarchy kh ON kb.parent_id = kh.id"); + if !is_admin { + qb.push(" WHERE kb.user_id = "); + qb.push_bind(user_id); + qb.push(" OR kb.is_public = 1"); + } + qb.push(") SELECT DISTINCT id FROM kb_hierarchy"); + + let descendant_ids: Vec = qb.build_query_scalar().fetch_all(pool).await?; + if descendant_ids.is_empty() { + return Ok(Some(Vec::new())); + } + + Ok(Some(descendant_ids)) +} + +async fn get_full_files_by_ids(pool: &SqlitePool, file_ids: &[i64]) -> Result, sqlx::Error> { + if file_ids.is_empty() { + return Ok(HashMap::new()); + } + + let placeholders: String = file_ids.iter().map(|_| "?").collect::>().join(","); + let query = format!("SELECT * FROM files WHERE id IN ({})", placeholders); + + let mut q = sqlx::query_as::<_, File>(&query); + for id in file_ids { + q = q.bind(id); + } + + let mut files: Vec = q.fetch_all(pool).await?; + for file in &mut files { + match crate::file_content::read(file.id).await { + Ok(content) => file.content = content, + Err(e) => warn!("Failed to read content file for file {}: {}", file.id, e), + } + } + Ok(files.into_iter().map(|f| (f.id, f)).collect()) +} + +async fn get_kbs_by_ids(pool: &SqlitePool, kb_ids: &[i64]) -> Result, sqlx::Error> { + if kb_ids.is_empty() { + return Ok(HashMap::new()); + } + + let placeholders: String = kb_ids.iter().map(|_| "?").collect::>().join(","); + let query = format!("SELECT id, name, is_public, user_id FROM knowledge_bases WHERE id IN ({})", placeholders); + + let mut q = sqlx::query_as::<_, KbInfo>(&query); + for id in kb_ids { + q = q.bind(id); + } + + let kbs: Vec = q.fetch_all(pool).await?; + Ok(kbs.into_iter().map(|k| (k.id, k)).collect()) +} diff --git a/src/api/system.rs b/src/api/system.rs new file mode 100644 index 0000000..2d47735 --- /dev/null +++ b/src/api/system.rs @@ -0,0 +1,518 @@ +use axum::{Extension, Json, extract::State, response::Response}; +use chrono::Utc; +use serde::Serialize; +use sqlx::SqlitePool; +use utoipa::ToSchema; + +use crate::{ + AuthUser, api::{ + common, error::{ApiError, ApiResult} + }, search::{SearchEngine, tantivy_engine::ForceMergeStats} +}; + +/// 进程内存占用 +#[derive(Debug, Serialize, ToSchema)] +pub struct MemoryUsage { + /// 进程 ID + pub pid: u32, + /// 常驻内存（RSS），单位：字节 + pub rss_bytes: u64, + /// 虚拟内存，单位：字节 + pub virtual_bytes: u64, +} + +/// 堆分析状态 +#[derive(Debug, Serialize, ToSchema)] +pub struct HeapProfileStatus { + /// jemalloc 版本 + pub jemalloc_version: Option, + /// 是否启用 heap profiling（opt.prof） + pub prof_enabled: Option, + /// 是否开启采样（prof.active） + pub prof_active: Option, + /// jemalloc 构建时默认配置 + pub malloc_conf: Option, + /// 运行时环境变量 MALLOC_CONF + pub env_malloc_conf: Option, + /// 状态读取失败的提示 + pub warnings: Vec, +} + +/// LanceDB compact 统计信息 +#[derive(Debug, Serialize, ToSchema)] +pub struct LanceDbCompactStats { + /// 删除行数（本次 compact 清理） + pub deleted_rows: u64, + /// compact 前被标记删除的行数 + pub deleted_rows_before: u64, + /// compact 后被标记删除的行数 + pub deleted_rows_after: u64, + /// compact 前总行数 + pub total_rows_before: u64, + /// compact 后总行数 + pub total_rows_after: u64, + /// compact 前磁盘大小（字节） + pub size_before_bytes: u64, + /// compact 后磁盘大小（字节） + pub size_after_bytes: u64, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct IndexRebuildStatus { + /// 任务 ID（无任务时为 null） + pub job_id: Option, + /// 任务状态：idle/running/completed/failed + pub status: String, + /// 当前阶段描述 + pub phase: String, + /// 总文档数 + pub total_docs: i64, + /// 已处理文档数 + pub processed_docs: i64, + /// 进度百分比（0-100） + pub progress_pct: f32, + /// 已运行秒数 + pub elapsed_secs: Option, + /// 预计剩余秒数（ETA） + pub eta_secs: Option, + /// 任务开始时间（秒级时间戳） + pub started_at: Option, + /// 最近更新时间（秒级时间戳） + pub updated_at: Option, + /// 任务结束时间（秒级时间戳） + pub finished_at: Option, + /// 失败时错误信息 + pub error: Option, +} + +/// 单个 Tantivy 索引 force merge 统计信息 +#[derive(Debug, Serialize, ToSchema)] +pub struct TantivyForceMergeIndexStats { + /// 索引名称：index/full_index + pub index: String, + /// merge 前 segment 数 + pub before_segments: usize, + /// merge 后 segment 数 + pub after_segments: usize, + /// merge 前存活文档数 + pub before_docs: u64, + /// merge 后存活文档数 + pub after_docs: u64, + /// merge 前已删除文档数 + pub before_deleted_docs: u64, + /// merge 后已删除文档数 + pub after_deleted_docs: u64, + /// GC 删除的不再被索引引用的文件数 + pub gc_deleted_files: usize, + /// GC 删除失败的文件数 + pub gc_failed_files: usize, + /// 是否因为 segment 少于 2 个而跳过 + pub skipped: bool, + /// 单个索引耗时（毫秒） + pub duration_ms: u128, +} + +/// Tantivy force merge 响应 +#[derive(Debug, Serialize, ToSchema)] +pub struct TantivyForceMergeResponse { + /// 普通切片索引统计 + pub index: TantivyForceMergeIndexStats, + /// 全文索引统计 + pub full_index: TantivyForceMergeIndexStats, + /// 总耗时（毫秒） + pub total_duration_ms: u128, +} + +#[derive(Debug, sqlx::FromRow)] +struct IndexRebuildStatusRow { + id: i64, + status: String, + phase: String, + total_docs: i64, + processed_docs: i64, + started_at: Option, + updated_at: Option, + finished_at: Option, + error: Option, +} + +/// 生成当前进程堆内存快照（jemalloc heap profile） +#[utoipa::path( + get, + path = "/api/v1/knowledge/system/heap", + operation_id = "system_heap_profile", + tag = "system", + responses( + (status = 200, description = "成功返回堆分析快照", body = Vec, content_type = "application/octet-stream"), + (status = 400, description = "未启用堆分析"), + (status = 500, description = "生成堆分析快照失败") + ) +)] +pub async fn heap_profile() -> ApiResult { + heap_profile_impl().await +} + +/// 生成堆内存分析 PDF 报告（使用 jeprof） +#[utoipa::path( + get, + path = "/api/v1/knowledge/system/heap/pdf", + operation_id = "system_heap_profile_pdf", + tag = "system", + responses( + (status = 200, description = "成功返回堆分析 PDF 报告", body = Vec, content_type = "application/pdf"), + (status = 400, description = "未启用堆分析或 jeprof 不可用"), + (status = 500, description = "生成堆分析 PDF 失败") + ) +)] +pub async fn heap_profile_pdf() -> ApiResult { + heap_profile_pdf_impl().await +} + +/// 主动触发 LanceDB compact +#[utoipa::path( + post, + path = "/api/v1/knowledge/system/lancedb/compact", + operation_id = "system_lancedb_compact", + tag = "system", + responses( + (status = 200, description = "LanceDB compact 成功", body = LanceDbCompactStats), + (status = 500, description = "LanceDB compact 失败") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn lancedb_compact( + Extension(search_engine): Extension, +) -> ApiResult> { + let stats = search_engine + .compact_lancedb() + .await + .map_err(|e| ApiError::internal(format!("LanceDB compact failed: {}", e)))?; + + Ok(Json(LanceDbCompactStats { + deleted_rows: stats.deleted_rows, + deleted_rows_before: stats.deleted_rows_before, + deleted_rows_after: stats.deleted_rows_after, + total_rows_before: stats.total_rows_before, + total_rows_after: stats.total_rows_after, + size_before_bytes: stats.size_before_bytes, + size_after_bytes: stats.size_after_bytes, + })) +} + +/// 强制合并 Tantivy segment，减少索引碎片 +#[utoipa::path( + post, + path = "/api/v1/knowledge/system/index/force-merge", + operation_id = "system_index_force_merge", + tag = "system", + responses( + (status = 200, description = "Tantivy force merge 成功", body = TantivyForceMergeResponse), + (status = 400, description = "权限不足"), + (status = 500, description = "Tantivy force merge 失败") + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn index_force_merge( + Extension(search_engine): Extension, Extension(auth_user): Extension, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let start = std::time::Instant::now(); + let (index_stats, full_index_stats) = search_engine + .force_merge_tantivy_indexes() + .await + .map_err(|e| ApiError::internal(format!("Tantivy force merge failed: {}", e)))?; + + Ok(Json(TantivyForceMergeResponse { + index: force_merge_index_stats("index", index_stats), + full_index: force_merge_index_stats("full_index", full_index_stats), + total_duration_ms: start.elapsed().as_millis(), + })) +} + +/// 查询索引重建进度与 ETA +#[utoipa::path( + get, + path = "/api/v1/knowledge/system/index/rebuild/status", + operation_id = "system_index_rebuild_status", + tag = "system", + responses( + (status = 200, description = "成功返回索引重建状态", body = IndexRebuildStatus) + ), + security( + ("x-user-id" = []), + ("x-role" = []) + ) +)] +pub async fn index_rebuild_status( + State(pool): State, Extension(auth_user): Extension, +) -> ApiResult> { + common::ensure_admin(&auth_user)?; + + let row: Option = sqlx::query_as( + "SELECT id, status, phase, total_docs, processed_docs, started_at, updated_at, finished_at, error + FROM index_rebuild_jobs + ORDER BY CASE WHEN status = 'running' THEN 0 ELSE 1 END, id DESC + LIMIT 1", + ) + .fetch_optional(&pool) + .await?; + + let Some(row) = row else { + return Ok(Json(IndexRebuildStatus { + job_id: None, + status: "idle".to_string(), + phase: String::new(), + total_docs: 0, + processed_docs: 0, + progress_pct: 0.0, + elapsed_secs: None, + eta_secs: None, + started_at: None, + updated_at: None, + finished_at: None, + error: None, + })); + }; + + let total_docs = row.total_docs.max(0); + let processed_docs = row.processed_docs.clamp(0, total_docs.max(0)); + let progress_pct = if total_docs > 0 { + ((processed_docs as f64 / total_docs as f64) * 100.0).clamp(0.0, 100.0) as f32 + } else if row.status == "completed" { + 100.0 + } else { + 0.0 + }; + + let now = Utc::now().timestamp(); + let elapsed_secs = match row.started_at { + Some(start) => match row.status.as_str() { + "running" => Some((now - start).max(0)), + _ => row.finished_at.map(|finish| (finish - start).max(0)).or(Some((now - start).max(0))), + }, + None => None, + }; + let eta_secs = if row.status == "running" && total_docs > processed_docs { + match elapsed_secs { + Some(elapsed) if elapsed > 0 && processed_docs > 0 => { + let speed = processed_docs as f64 / elapsed as f64; + if speed > 0.0 { + let remaining = (total_docs - processed_docs) as f64; + Some((remaining / speed).ceil() as i64) + } else { + None + } + } + _ => None, + } + } else if row.status == "completed" { + Some(0) + } else { + None + }; + + Ok(Json(IndexRebuildStatus { + job_id: Some(row.id), + status: row.status, + phase: row.phase, + total_docs, + processed_docs, + progress_pct, + elapsed_secs, + eta_secs, + started_at: row.started_at, + updated_at: row.updated_at, + finished_at: row.finished_at, + error: row.error, + })) +} + +fn force_merge_index_stats(index: &str, stats: ForceMergeStats) -> TantivyForceMergeIndexStats { + TantivyForceMergeIndexStats { + index: index.to_string(), + before_segments: stats.before_segments, + after_segments: stats.after_segments, + before_docs: stats.before_docs, + after_docs: stats.after_docs, + before_deleted_docs: stats.before_deleted_docs, + after_deleted_docs: stats.after_deleted_docs, + gc_deleted_files: stats.gc_deleted_files, + gc_failed_files: stats.gc_failed_files, + skipped: stats.skipped, + duration_ms: stats.duration_ms, + } +} + +#[cfg(feature = "profiling")] +async fn heap_profile_impl() -> ApiResult { + use std::{ + ffi::CString, io::Seek, time::{SystemTime, UNIX_EPOCH} + }; + + use axum::{ + body::Body, http::{HeaderValue, header} + }; + use tempfile::NamedTempFile; + use tikv_jemalloc_ctl::raw; + use tokio_util::io::ReaderStream; + + let pid = std::process::id(); + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| ApiError::internal(format!("SystemTime error: {}", e)))? + .as_secs(); + let filename = format!("htknow.heap.{}.{}.heap", pid, ts); + + // 将堆 dump 写入临时文件后立即删除路径，只保留 fd 用于流式返回。 + let temp = NamedTempFile::new().map_err(|e| ApiError::internal(format!("create temp file failed: {}", e)))?; + let path = temp.path().to_path_buf(); + let path_str = path.to_string_lossy().into_owned(); + let c_path = + CString::new(path_str.clone()).map_err(|_| ApiError::internal("Failed to create C string for path"))?; + + let ptr = c_path.as_ptr(); + unsafe { + raw::write(b"prof.dump\0", ptr).map_err(|e| ApiError::internal(format!("jemalloc heap dump failed: {}", e)))?; + } + + let mut std_file = temp.into_file(); + std_file + .seek(std::io::SeekFrom::Start(0)) + .map_err(|e| ApiError::internal(format!("seek temp file failed: {}", e)))?; + let file = tokio::fs::File::from_std(std_file); + let len = file.metadata().await.map_err(|e| ApiError::internal(format!("metadata failed: {}", e)))?.len(); + let stream = ReaderStream::new(file); + + let mut response = Response::new(Body::from_stream(stream)); + let headers = response.headers_mut(); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); + headers.insert( + header::CONTENT_LENGTH, + HeaderValue::from_str(&len.to_string()) + .map_err(|e| ApiError::internal(format!("Invalid header value: {}", e)))?, + ); + let disposition = format!("attachment; filename=\"{}\"", filename); + headers.insert( + header::CONTENT_DISPOSITION, + HeaderValue::from_str(&disposition).map_err(|e| ApiError::internal(format!("Invalid header value: {}", e)))?, + ); + + Ok(response) +} + +#[cfg(not(feature = "profiling"))] +async fn heap_profile_impl() -> ApiResult { + Err(ApiError::BadRequest("Heap profiling is only available with 'profiling' feature enabled.".to_string())) +} + +#[cfg(feature = "profiling")] +async fn heap_profile_pdf_impl() -> ApiResult { + use std::{ + ffi::CString, process::Stdio, time::{SystemTime, UNIX_EPOCH} + }; + + use axum::{ + body::Body, http::{HeaderValue, header} + }; + use tempfile::NamedTempFile; + use tikv_jemalloc_ctl::raw; + use tokio::{io::AsyncSeekExt, process::Command}; + use tokio_util::io::ReaderStream; + + let pid = std::process::id(); + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| ApiError::internal(format!("SystemTime error: {}", e)))? + .as_secs(); + + // 生成 heap dump 文件（路径需要保持到 jeprof 执行完毕） + let heap_temp = NamedTempFile::new().map_err(|e| ApiError::internal(format!("create heap temp failed: {}", e)))?; + let heap_path = heap_temp.path().to_path_buf(); + let heap_path_str = heap_path.to_string_lossy().into_owned(); + let c_path = CString::new(heap_path_str.clone()) + .map_err(|_| ApiError::internal("Failed to create C string for heap path"))?; + + log::info!("Generating heap dump: {}", heap_path_str); + let ptr = c_path.as_ptr(); + unsafe { + raw::write(b"prof.dump\0", ptr).map_err(|e| ApiError::internal(format!("jemalloc heap dump failed: {}", e)))?; + } + + // 生成 PDF 文件（用 NamedTempFile 承接 jeprof 标准输出） + let pdf_temp = NamedTempFile::new().map_err(|e| ApiError::internal(format!("create pdf temp failed: {}", e)))?; + let pdf_std = pdf_temp.into_file(); + let mut pdf_file = tokio::fs::File::from_std(pdf_std); + + // 获取当前二进制文件路径 + let binary_path = + std::env::current_exe().map_err(|e| ApiError::internal(format!("Failed to get binary path: {}", e)))?; + let binary_path_str = binary_path.to_string_lossy(); + + log::info!("Running jeprof: binary={}, heap={}, output=pdf_stream", binary_path_str, heap_path_str); + + let mut child = Command::new("jeprof") + .arg("--show_bytes") + .arg("--pdf") + .arg(binary_path_str.as_ref()) + .arg(&heap_path_str) + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| ApiError::internal(format!("Failed to execute jeprof (make sure jeprof is installed): {}", e)))?; + + let mut stdout = child.stdout.take().ok_or_else(|| ApiError::internal("jeprof stdout unavailable"))?; + tokio::io::copy(&mut stdout, &mut pdf_file) + .await + .map_err(|e| ApiError::internal(format!("Failed to copy jeprof output: {}", e)))?; + + let status = child.wait().await.map_err(|e| ApiError::internal(format!("jeprof process wait failed: {}", e)))?; + + // jeprof 已完成，heap dump 临时文件可以清理 + drop(heap_temp); + + if !status.success() { + let code = status.code().unwrap_or(-1); + log::error!("jeprof failed with exit code: {}", code); + return Err(ApiError::internal(format!("jeprof execution failed with exit code: {}", code))); + } + + pdf_file + .seek(std::io::SeekFrom::Start(0)) + .await + .map_err(|e| ApiError::internal(format!("seek pdf temp failed: {}", e)))?; + let len = pdf_file.metadata().await.map_err(|e| ApiError::internal(format!("pdf metadata failed: {}", e)))?.len(); + if len == 0 { + return Err(ApiError::internal("jeprof generated empty PDF")); + } + + let pdf_filename = format!("htknow.heap.{}.{}.pdf", pid, ts); + log::info!("Successfully generated heap profile PDF: {} bytes", len); + + let stream = ReaderStream::new(pdf_file); + let mut response = Response::new(Body::from_stream(stream)); + let headers = response.headers_mut(); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/pdf")); + headers.insert( + header::CONTENT_LENGTH, + HeaderValue::from_str(&len.to_string()) + .map_err(|e| ApiError::internal(format!("Invalid header value: {}", e)))?, + ); + let disposition = format!("attachment; filename=\"{}\"", pdf_filename); + headers.insert( + header::CONTENT_DISPOSITION, + HeaderValue::from_str(&disposition).map_err(|e| ApiError::internal(format!("Invalid header value: {}", e)))?, + ); + + Ok(response) +} + +#[cfg(not(feature = "profiling"))] +async fn heap_profile_pdf_impl() -> ApiResult { + Err(ApiError::BadRequest("Heap profiling is only available with 'profiling' feature enabled.".to_string())) +} diff --git a/src/archive.rs b/src/archive.rs new file mode 100644 index 0000000..9948a68 --- /dev/null +++ b/src/archive.rs @@ -0,0 +1,631 @@ +//! 压缩文件解压模块 +//! +//! 支持 ZIP、TAR（及其 GZ/BZ2/XZ 变体）格式的解压。 +//! 7Z 和 RAR 格式暂不支持。 + +use std::{ + io::{Read, Write}, path::Path +}; + +use encoding_rs::GB18030; +use serde::Serialize; +use tempfile::NamedTempFile; +use utoipa::ToSchema; + +/// 智能解码文件名：先尝试 UTF-8，发现乱码时回退到 GB18030（兼容 GBK/GB2312） +fn decode_filename(raw: &[u8]) -> String { + // 先尝试 UTF-8 + if let Ok(utf8) = std::str::from_utf8(raw) { + // 如果 UTF-8 解码成功且不包含替换字符，直接使用 + if !utf8.contains('\u{FFFD}') { + return utf8.to_string(); + } + } + + // 尝试 GB18030（兼容 GBK/GB2312） + let (decoded, _, had_errors) = GB18030.decode(raw); + if !had_errors { + return decoded.to_string(); + } + + // 回退：UTF-8 lossy + String::from_utf8_lossy(raw).to_string() +} + +/// 判断原始文件名是否表示目录（以 / 或 \ 结尾） +fn is_dir_from_raw(raw: &[u8]) -> bool { + raw.ends_with(b"/") || raw.ends_with(b"\\") +} + +/// 压缩包内单个文件条目 +#[derive(Debug, Clone, Serialize, sqlx::FromRow, ToSchema)] +pub struct ArchiveEntry { + pub id: Option, + pub file_id: i64, + pub entry_path: String, + pub size: Option, + pub is_directory: bool, +} + +/// 解压结果 +#[derive(Debug, Serialize, ToSchema)] +pub struct ExtractResult { + pub entries: Vec, + pub needs_password: bool, +} + +/// 压缩文件处理错误 +#[derive(Debug, thiserror::Error)] +pub enum ArchiveError { + #[error("密码错误或需要密码")] + PasswordRequired, + #[error("无效的密码")] + InvalidPassword, + #[error("不支持的压缩格式: {0}")] + UnsupportedFormat(String), + #[error("IO 错误: {0}")] + Io(#[from] std::io::Error), + #[error("ZIP 错误: {0}")] + Zip(String), + #[error("解压大小超过限制: 最大 {max_mb}MB, 实际约 {actual_mb}MB")] + SizeLimitExceeded { max_mb: u64, actual_mb: u64 }, + #[error("文件数量超过限制: 最大 {max}, 实际 {actual}")] + FileCountExceeded { max: usize, actual: usize }, + #[error("{0}")] + Other(String), +} + +/// 解压安全限制 +pub struct ExtractLimits { + /// 最大解压后总大小（字节），默认 1GB + pub max_total_size: u64, + /// 最大文件数量，默认 10000 + pub max_file_count: usize, + /// 单个文件最大大小（字节），默认 100MB + pub max_file_size: u64, +} + +impl Default for ExtractLimits { + fn default() -> Self { + Self { + max_total_size: 1_073_741_824, // 1GB + max_file_count: 10_000, + max_file_size: 104_857_600, // 100MB + } + } +} + +/// 支持的压缩格式 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ArchiveFormat { + Zip, + SevenZ, + Tar, + TarGz, + Tgz, + TarBz2, + TarXz, + Unknown, +} + +impl ArchiveFormat { + /// 根据文件名识别压缩格式（大小写不敏感） + pub fn from_filename(filename: &str) -> Self { + let lower = filename.to_lowercase(); + if lower.ends_with(".zip") { + ArchiveFormat::Zip + } else if lower.ends_with(".7z") { + ArchiveFormat::SevenZ + } else if lower.ends_with(".tar.gz") { + ArchiveFormat::TarGz + } else if lower.ends_with(".tgz") { + ArchiveFormat::Tgz + } else if lower.ends_with(".tar.bz2") { + ArchiveFormat::TarBz2 + } else if lower.ends_with(".tar.xz") { + ArchiveFormat::TarXz + } else if lower.ends_with(".tar") { + ArchiveFormat::Tar + } else { + ArchiveFormat::Unknown + } + } + + /// 是否为已识别的压缩格式（含暂不解压的 7Z） + pub fn is_archive(self) -> bool { + !matches!(self, ArchiveFormat::Unknown) + } + + /// 当前是否可直接解压 + pub fn is_supported(self) -> bool { + matches!(self, ArchiveFormat::Zip) || self.is_tar_variant() + } + + /// 是否为 tar 变体 + pub fn is_tar_variant(self) -> bool { + matches!( + self, + ArchiveFormat::Tar + | ArchiveFormat::TarGz + | ArchiveFormat::Tgz + | ArchiveFormat::TarBz2 + | ArchiveFormat::TarXz + ) + } + + /// 格式描述文本 + pub fn desc(self) -> &'static str { + match self { + ArchiveFormat::Zip => "ZIP", + ArchiveFormat::SevenZ => "7Z", + ArchiveFormat::Tar => "TAR", + ArchiveFormat::TarGz | ArchiveFormat::Tgz => "TAR.GZ", + ArchiveFormat::TarBz2 => "TAR.BZ2", + ArchiveFormat::TarXz => "TAR.XZ", + ArchiveFormat::Unknown => "未知", + } + } +} + +/// 判断文件是否为支持的压缩格式 +pub fn is_archive_file(filename: &str) -> bool { + ArchiveFormat::from_filename(filename).is_archive() +} + +/// 将压缩文件解压到指定目录 +/// +/// `filename` 为原始文件名（含扩展名），用于判断压缩格式。 +/// 返回解压后的文件列表 +pub fn extract_archive( + src_path: &str, dest_dir: &str, filename: &str, password: Option<&str>, file_id: i64, +) -> Result, ArchiveError> { + let limits = ExtractLimits::default(); + extract_archive_with_limits(src_path, dest_dir, filename, password, file_id, &limits) +} + +/// 带限制的解压 +pub fn extract_archive_with_limits( + src_path: &str, dest_dir: &str, filename: &str, password: Option<&str>, file_id: i64, limits: &ExtractLimits, +) -> Result, ArchiveError> { + let format = ArchiveFormat::from_filename(filename); + if !format.is_archive() { + return Err(ArchiveError::UnsupportedFormat(format.desc().to_string())); + } + + // 创建目标目录 + std::fs::create_dir_all(dest_dir)?; + + match format { + ArchiveFormat::Zip => extract_zip(src_path, dest_dir, password, file_id, limits), + ArchiveFormat::SevenZ => Err(ArchiveError::UnsupportedFormat("7Z 格式暂不支持".to_string())), + _ if format.is_tar_variant() => extract_tar(src_path, dest_dir, file_id, limits), + _ => Err(ArchiveError::UnsupportedFormat(format.desc().to_string())), + } +} + +/// 读取压缩包内单个文件内容到临时文件 +/// +/// `filename` 为原始文件名（含扩展名），用于判断压缩格式。 +/// 返回的 `NamedTempFile` 需要由调用方在合适时机消费/关闭。 +pub fn read_archive_entry( + src_path: &str, filename: &str, entry_path: &str, password: Option<&str>, +) -> Result { + let format = ArchiveFormat::from_filename(filename); + match format { + ArchiveFormat::Zip => read_zip_entry(src_path, entry_path, password), + ArchiveFormat::SevenZ => Err(ArchiveError::UnsupportedFormat("7Z 格式暂不支持".to_string())), + _ if format.is_tar_variant() => read_tar_entry(src_path, entry_path), + _ => Err(ArchiveError::UnsupportedFormat(format.desc().to_string())), + } +} + +// ============================================================================ +// ZIP 解压 +// ============================================================================ + +fn extract_zip( + src_path: &str, dest_dir: &str, password: Option<&str>, file_id: i64, limits: &ExtractLimits, +) -> Result, ArchiveError> { + let file = std::fs::File::open(src_path)?; + let mut archive = match zip::ZipArchive::new(file) { + Ok(a) => a, + Err(e) => return Err(ArchiveError::Zip(e.to_string())), + }; + + let mut entries = Vec::new(); + let mut total_size: u64 = 0; + let password_bytes = password.map(|p| p.as_bytes()); + + log::info!("ZIP archive has {} entries, file_id={}", archive.len(), file_id); + + for i in 0..archive.len() { + // 先检查文件是否加密 + let is_encrypted = { + let file = archive.by_index(i).map_err(|e| ArchiveError::Zip(e.to_string()))?; + file.encrypted() + }; + + let mut file_entry = if is_encrypted { + if let Some(pw) = password_bytes { + archive.by_index_decrypt(i, pw).map_err(|e| ArchiveError::Zip(e.to_string()))? + } else { + return Err(ArchiveError::PasswordRequired); + } + } else { + archive.by_index(i).map_err(|e| ArchiveError::Zip(e.to_string()))? + }; + + let raw_bytes = file_entry.name_raw(); + let raw_name = decode_filename(raw_bytes); + // 统一路径分隔符为 /，并处理连续分隔符和首尾分隔符 + let name: String = + raw_name.replace('\\', "/").split('/').filter(|s| !s.is_empty()).collect::>().join("/"); + if name.is_empty() { + log::info!( + "ZIP entry {}: raw={:?} utf8={:?} -> skipped (empty after normalization)", + i, + raw_bytes, + raw_name + ); + continue; + } + let is_dir = is_dir_from_raw(raw_bytes); + let name = if is_dir && name.ends_with('/') { name.trim_end_matches('/').to_string() } else { name }; + let size = file_entry.size(); + + log::info!( + "ZIP entry {}: raw={:?} decoded={:?} -> normalized={}, is_dir={}, size={}", + i, + raw_bytes, + raw_name, + name, + is_dir, + size + ); + + // 安全检查：路径遍历 + if name.contains("..") { + log::warn!("Skipping ZIP entry with path traversal: {}", name); + continue; + } + + // 大小限制检查 + if !is_dir { + if size > limits.max_file_size { + return Err(ArchiveError::SizeLimitExceeded { + max_mb: limits.max_file_size / 1_048_576, + actual_mb: size / 1_048_576, + }); + } + total_size += size; + if total_size > limits.max_total_size { + return Err(ArchiveError::SizeLimitExceeded { + max_mb: limits.max_total_size / 1_048_576, + actual_mb: total_size / 1_048_576, + }); + } + } + + // 数量限制检查 + if entries.len() >= limits.max_file_count { + return Err(ArchiveError::FileCountExceeded { max: limits.max_file_count, actual: entries.len() }); + } + + let out_path = Path::new(dest_dir).join(&name); + + if is_dir { + std::fs::create_dir_all(&out_path)?; + } else { + if let Some(parent) = out_path.parent() { + std::fs::create_dir_all(parent)?; + } + let mut out_file = std::fs::File::create(&out_path)?; + std::io::copy(&mut file_entry, &mut out_file)?; + } + + entries.push(ArchiveEntry { + id: None, + file_id, + entry_path: name, + size: Some(size as i64), + is_directory: is_dir, + }); + } + + log::info!("ZIP extraction completed: {} entries, file_id={}", entries.len(), file_id); + Ok(entries) +} + +fn read_zip_entry(src_path: &str, entry_path: &str, password: Option<&str>) -> Result { + let file = std::fs::File::open(src_path)?; + let mut archive = match zip::ZipArchive::new(file) { + Ok(a) => a, + Err(e) => return Err(ArchiveError::Zip(e.to_string())), + }; + + let password_bytes = password.map(|p| p.as_bytes()); + let normalized_target = entry_path.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string(); + + // 遍历所有条目，用解码后的文件名匹配（处理 GBK 编码中文文件名） + for i in 0..archive.len() { + let file_ref = archive.by_index(i).map_err(|e| ArchiveError::Zip(e.to_string()))?; + let decoded_name = decode_filename(file_ref.name_raw()); + let normalized_name = decoded_name.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string(); + + if normalized_name == normalized_target { + let is_encrypted = file_ref.encrypted(); + drop(file_ref); + + let mut file_entry = if is_encrypted { + if let Some(pw) = password_bytes { + archive.by_index_decrypt(i, pw).map_err(|e| ArchiveError::Zip(e.to_string()))? + } else { + return Err(ArchiveError::PasswordRequired); + } + } else { + archive.by_index(i).map_err(|e| ArchiveError::Zip(e.to_string()))? + }; + + let mut temp = NamedTempFile::new()?; + std::io::copy(&mut file_entry, &mut temp)?; + temp.flush()?; + return Ok(temp); + } + } + + Err(ArchiveError::Other(format!("文件不存在: {}", entry_path))) +} + +// ============================================================================ +// TAR 解压（含 GZ/BZ2/XZ 变体） +// ============================================================================ + +fn open_tar_reader(src_path: &str) -> Result, ArchiveError> { + let file = std::fs::File::open(src_path)?; + let lower = src_path.to_lowercase(); + + if lower.ends_with(".tar") { + Ok(Box::new(file)) + } else if lower.ends_with(".tar.gz") || lower.ends_with(".tgz") { + Ok(Box::new(flate2::read::GzDecoder::new(file))) + } else if lower.ends_with(".tar.bz2") { + Ok(Box::new(bzip2::read::BzDecoder::new(file))) + } else if lower.ends_with(".tar.xz") { + Ok(Box::new(xz2::read::XzDecoder::new(file))) + } else { + Err(ArchiveError::UnsupportedFormat("TAR 变体".to_string())) + } +} + +fn extract_tar( + src_path: &str, dest_dir: &str, file_id: i64, limits: &ExtractLimits, +) -> Result, ArchiveError> { + let reader = open_tar_reader(src_path)?; + let mut archive = tar::Archive::new(reader); + + let mut entries = Vec::new(); + let mut total_size: u64 = 0; + + for entry_result in archive.entries()? { + let mut entry = entry_result?; + let raw_path = entry.path_bytes(); + let decoded = decode_filename(&raw_path); + let name = decoded.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string(); + if name.is_empty() { + continue; + } + let is_dir = entry.header().entry_type().is_dir() || decoded.ends_with('/') || decoded.ends_with('\\'); + let name = if is_dir && name.ends_with('/') { name.trim_end_matches('/').to_string() } else { name }; + let size = entry.size(); + + // 安全检查 + if name.contains("..") { + log::warn!("Skipping TAR entry with path traversal: {}", name); + continue; + } + + // 大小限制 + if !is_dir { + if size > limits.max_file_size { + return Err(ArchiveError::SizeLimitExceeded { + max_mb: limits.max_file_size / 1_048_576, + actual_mb: size / 1_048_576, + }); + } + total_size += size; + if total_size > limits.max_total_size { + return Err(ArchiveError::SizeLimitExceeded { + max_mb: limits.max_total_size / 1_048_576, + actual_mb: total_size / 1_048_576, + }); + } + } + + if entries.len() >= limits.max_file_count { + return Err(ArchiveError::FileCountExceeded { max: limits.max_file_count, actual: entries.len() }); + } + + let out_path = Path::new(dest_dir).join(&name); + + if is_dir { + std::fs::create_dir_all(&out_path)?; + } else { + if let Some(parent) = out_path.parent() { + std::fs::create_dir_all(parent)?; + } + let mut out_file = std::fs::File::create(&out_path)?; + std::io::copy(&mut entry, &mut out_file)?; + } + + entries.push(ArchiveEntry { + id: None, + file_id, + entry_path: name, + size: Some(size as i64), + is_directory: is_dir, + }); + } + + Ok(entries) +} + +fn read_tar_entry(src_path: &str, entry_path: &str) -> Result { + let reader = open_tar_reader(src_path)?; + let mut archive = tar::Archive::new(reader); + + let normalized_target = entry_path.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string(); + + for entry_result in archive.entries()? { + let mut entry = entry_result?; + let raw_path = entry.path_bytes(); + let decoded = decode_filename(&raw_path); + let name = decoded.replace('\\', "/").trim_start_matches('/').trim_end_matches('/').to_string(); + + if name == normalized_target { + let mut temp = NamedTempFile::new()?; + std::io::copy(&mut entry, &mut temp)?; + temp.flush()?; + return Ok(temp); + } + } + + Err(ArchiveError::Other(format!("文件不存在: {}", entry_path))) +} + +// ============================================================================ +// 辅助函数 +// ============================================================================ + +/// 清理压缩文件解压目录 +pub fn cleanup_archive_dir(archives_path: &str, file_id: i64) -> std::io::Result<()> { + let dir = Path::new(archives_path).join(file_id.to_string()); + if dir.exists() { + std::fs::remove_dir_all(&dir)?; + } + Ok(()) +} + +/// 获取压缩文件条目的本地文件系统路径 +pub fn resolve_archive_entry_path(archives_path: &str, file_id: i64, entry_path: &str) -> Option { + let base = Path::new(archives_path).join(file_id.to_string()); + let resolved = base.join(entry_path); + + // 安全检查：确保解析后的路径在 base 目录下 + let canonical_base = base.canonicalize().unwrap_or(base.clone()); + let canonical_resolved = match resolved.canonicalize() { + Ok(p) => p, + Err(_) => resolved.clone(), + }; + + if !canonical_resolved.starts_with(&canonical_base) { + return None; + } + + Some(resolved) +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use super::*; + + #[test] + fn test_is_archive_file() { + assert!(is_archive_file("test.zip")); + assert!(is_archive_file("test.ZIP")); + assert!(is_archive_file("test.tar.gz")); + assert!(is_archive_file("test.tgz")); + assert!(is_archive_file("test.tar.bz2")); + assert!(is_archive_file("test.tar.xz")); + assert!(is_archive_file("test.7z")); + assert!(!is_archive_file("test.pdf")); + assert!(!is_archive_file("test.txt")); + } + + #[test] + fn test_extract_zip_with_backslash_paths() { + // 创建测试 ZIP（Windows 风格路径） + let zip_path = "/tmp/test_htknow_zip.zip"; + let extract_dir = "/tmp/test_htknow_extract"; + + // 清理 + let _ = std::fs::remove_file(zip_path); + let _ = std::fs::remove_dir_all(extract_dir); + + { + let file = std::fs::File::create(zip_path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default(); + zip.start_file("folder\\file1.txt", options).unwrap(); + zip.write_all(b"hello world 1").unwrap(); + zip.start_file("folder\\sub\\file2.txt", options).unwrap(); + zip.write_all(b"hello world 2").unwrap(); + zip.start_file("root.txt", options).unwrap(); + zip.write_all(b"hello root").unwrap(); + zip.finish().unwrap(); + } + + let entries = extract_archive(zip_path, extract_dir, "test.zip", None, 1).unwrap(); + + // 应该解压出 3 个文件条目（目录条目会自动从路径推断） + assert!( + entries.len() >= 3, + "Expected at least 3 entries, got {}: {:?}", + entries.len(), + entries.iter().map(|e| &e.entry_path).collect::>() + ); + + // 检查路径统一使用了 / + let paths: Vec = entries.iter().map(|e| e.entry_path.clone()).collect(); + assert!(paths.iter().any(|p| p == "folder/file1.txt"), "Missing folder/file1.txt in {:?}", paths); + assert!(paths.iter().any(|p| p == "folder/sub/file2.txt"), "Missing folder/sub/file2.txt in {:?}", paths); + assert!(paths.iter().any(|p| p == "root.txt"), "Missing root.txt in {:?}", paths); + + // 没有路径应该包含反斜杠 + for p in &paths { + assert!(!p.contains('\\'), "Path should not contain backslash: {}", p); + } + + // 检查文件内容 + let content1 = std::fs::read_to_string(format!("{}/folder/file1.txt", extract_dir)).unwrap(); + assert_eq!(content1, "hello world 1"); + + let content2 = std::fs::read_to_string(format!("{}/folder/sub/file2.txt", extract_dir)).unwrap(); + assert_eq!(content2, "hello world 2"); + + // 清理 + let _ = std::fs::remove_file(zip_path); + let _ = std::fs::remove_dir_all(extract_dir); + } + + #[test] + fn test_extract_zip_with_forward_slash_paths() { + let zip_path = "/tmp/test_htknow_zip2.zip"; + let extract_dir = "/tmp/test_htknow_extract2"; + + let _ = std::fs::remove_file(zip_path); + let _ = std::fs::remove_dir_all(extract_dir); + + { + let file = std::fs::File::create(zip_path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default(); + zip.start_file("docs/report.pdf", options).unwrap(); + zip.write_all(b"fake pdf").unwrap(); + zip.start_file("data/info.txt", options).unwrap(); + zip.write_all(b"info").unwrap(); + zip.finish().unwrap(); + } + + let entries = extract_archive(zip_path, extract_dir, "test.zip", None, 2).unwrap(); + assert!(entries.len() >= 2, "Expected at least 2 entries, got {}", entries.len()); + + let paths: Vec = entries.iter().map(|e| e.entry_path.clone()).collect(); + assert!(paths.iter().any(|p| p == "docs/report.pdf")); + assert!(paths.iter().any(|p| p == "data/info.txt")); + + let _ = std::fs::remove_file(zip_path); + let _ = std::fs::remove_dir_all(extract_dir); + } +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..bdfb89b --- /dev/null +++ b/src/config.rs @@ -0,0 +1,516 @@ +//! 配置管理模块 +//! +//! 支持从环境变量读取配置，预留 etcd 支持接口，并支持实时更新。 + +use std::sync::{Arc, RwLock}; + +use once_cell::sync::Lazy; + +/// 全局配置单例 +/// +/// 内层存 `Arc`，`get()` 只克隆 Arc（廉价），避免每次深拷贝整个配置结构体。 +/// 保留外层 `RwLock` 以支持未来的配置热更新（替换内层 Arc 即可）。 +static CONFIG: Lazy>> = Lazy::new(|| RwLock::new(Arc::new(AppConfig::from_env()))); + +/// 获取当前配置的只读快照（克隆 Arc，零深拷贝） +pub fn get() -> Arc { + CONFIG.read().unwrap().clone() +} + +// 重新加载配置（用于热更新） +// pub fn reload() { +// let new_config = AppConfig::from_env(); +// let mut config = CONFIG.write().unwrap(); +// *config = new_config; +// info!("Configuration reloaded"); +// } + +// 使用新配置更新（用于 etcd watch 等场景） +// pub fn update(new_config: AppConfig) { +// let mut config = CONFIG.write().unwrap(); +// *config = new_config; +// info!("Configuration updated"); +// } + +/// 应用配置 +#[derive(Debug, Clone)] +pub struct AppConfig { + pub server: ServerConfig, + pub services: ServicesConfig, + pub ai: AIConfig, + pub database: DatabaseConfig, + pub storage: StorageConfig, + pub search: SearchConfig, + pub slice: SliceConfig, + pub llm: LLMConfig, +} + +/// 服务器配置 +#[derive(Debug, Clone)] +pub struct ServerConfig { + /// 监听地址 + pub host: String, + /// 监听端口 + pub port: u16, + /// 上传文件大小限制（MB） + pub upload_limit_mb: usize, + /// 文件处理间隔（秒） + pub process_interval_secs: u64, + /// 文件处理并发数 + pub process_concurrency: usize, + /// LanceDB 自动压缩 cron 表达式（off/disabled/0 表示禁用） + pub lancedb_compact_cron: String, + /// 是否启用后台文件解析 + pub parse_enabled: bool, + /// 是否启用重复文件复用 + pub reuse_duplicate_files: bool, + /// 文件解析后是否构建知识图谱 + pub build_knowledge_graph: bool, +} + +/// 外部服务配置 +#[derive(Debug, Clone)] +pub struct ServicesConfig { + /// MinerU PDF 解析服务地址 + pub mineru_url: String, + /// 外部接口请求超时（秒） + pub request_timeout_secs: u64, + /// MinerU 单次解析 PDF 最大页数（0 表示不限制） + pub mineru_max_pages: usize, + /// Office 文档转 PDF 服务地址 + pub office_convert_url: String, + /// 自定义解析服务地址（配置后仅 Word/PPT/PDF 走该服务） + pub custom_parse_url: Option, + /// 自定义解析复用服务地址（仅输入 pdf_contents） + pub custom_parse_reuse_url: Option, + /// 音频转写服务地址 + pub audio_transcription_url: String, + /// 音频转写服务 API Key（可选） + pub audio_transcription_key: Option, + /// Embedding 服务地址 + pub embedding_url: String, + /// 图片 Embedding 服务地址（可选，未配置时不进行图片 embedding） + pub image_embedding_url: Option, + /// Rerank 服务地址 + pub rerank_url: String, + /// 图片文本化服务地址（可选） + pub image_parse_url: Option, + /// 图片文本化服务超时（秒） + pub image_parse_timeout_secs: u64, + /// 图片文本化并发数 + pub image_parse_concurrency: usize, +} + +/// AI 模型配置 +#[derive(Debug, Clone)] +pub struct AIConfig { + /// Embedding 模型名称 + pub embedding_model: String, + /// Embedding 向量维度 + pub embedding_dim: i32, + /// 图片 Embedding 向量维度 + pub image_embedding_dim: i32, + /// Rerank 模型名称 + pub rerank_model: String, + /// Rerank 分数阈值 + pub rerank_threshold: f32, + /// Embedding 批量请求批次大小 + pub embedding_batch_size: usize, +} + +/// 数据库配置 +#[derive(Debug, Clone)] +pub struct DatabaseConfig { + /// 数据库连接 URL + pub url: String, + /// 最大连接数 + pub max_connections: u32, + /// 最小空闲连接数（保持热连接，避免负载波动时反复建连） + pub min_connections: u32, + /// 忙超时（毫秒） + pub busy_timeout_ms: u64, + /// 是否初始化默认知识库 + pub init_default_kbs: bool, +} + +/// 存储路径配置 +#[derive(Debug, Clone)] +pub struct StorageConfig { + /// LanceDB 存储路径 + pub lancedb_path: String, + /// 临时目录路径 + pub temp_path: String, + /// 图片保存路径 + pub images_path: String, + /// 转换后的 PDF 保存路径 + pub pdf_path: String, + /// 上传文件保存路径 + pub files_path: String, + /// 压缩文件解压后保存路径 + pub archives_path: String, + /// 文件解析后完整文本的保存路径 + pub contents_path: String, + /// 文件切片正文（按源文件聚合）的保存路径 + pub slice_contents_path: String, +} + +/// 搜索配置 +#[derive(Debug, Clone)] +pub struct SearchConfig { + /// 搜索结果限制数 + pub limit: usize, + /// Tantivy 索引路径 + pub tantivy_index_path: String, + /// Tantivy 全文索引路径 + pub tantivy_full_index_path: String, + /// Tantivy 索引内存（MB） + pub tantivy_memory_mb: usize, + /// Tantivy 重建批次大小 + pub tantivy_rebuild_batch_size: usize, + /// LanceDB 从 SQLite 重建时的批次大小 + pub lancedb_rebuild_batch_size: usize, + /// 文本 embedding 请求超时（秒） + pub embedding_timeout_secs: u64, + /// rerank 请求超时（秒） + pub rerank_timeout_secs: u64, + /// 是否启用同义词扩展 + pub synonym_enabled: bool, + /// 同义词默认权重因子 + pub synonym_boost: f32, + /// 每个词最多扩展的同义词数 + pub max_synonyms_per_term: usize, + /// 单次查询最多扩展的同义词总数 + pub max_total_synonyms: usize, + /// 高亮页码选择阈值：首选页的内容字数（按 pdf_content 中文本长度计算）少于该值时，优先使用第二页（如果存在） + pub highlight_page_min_chars: usize, +} + +/// 切片配置 +#[derive(Debug, Clone)] +pub struct SliceConfig { + /// 智能切片每个切片的最大字数 + pub smart_slice_max_chars: usize, + /// 固定长度切片的重叠字数 + pub fixed_slice_overlap_chars: usize, +} + +/// LLM 配置 +#[derive(Debug, Clone)] +pub struct LLMConfig { + /// LLM API URL + pub api_url: Option, + /// LLM API Key + pub api_key: Option, + /// LLM 模型名称 + pub model: String, +} + +impl AppConfig { + /// 从环境变量加载配置 + pub fn from_env() -> Self { + Self { + server: ServerConfig::from_env(), + services: ServicesConfig::from_env(), + ai: AIConfig::from_env(), + database: DatabaseConfig::from_env(), + storage: StorageConfig::from_env(), + search: SearchConfig::from_env(), + slice: SliceConfig::from_env(), + llm: LLMConfig::from_env(), + } + } +} + +impl ServerConfig { + fn from_env() -> Self { + Self { + host: env_or("HTKNOW_SERVER_HOST", "0.0.0.0"), + port: env_or_parse("HTKNOW_SERVER_PORT", 3000), + upload_limit_mb: env_or_parse("HTKNOW_SERVER_UPLOAD_LIMIT_MB", 500), + process_interval_secs: env_or_parse("HTKNOW_SERVER_PROCESS_INTERVAL_SECS", 10), + process_concurrency: env_or_parse("HTKNOW_SERVER_PROCESS_CONCURRENCY", 1), + lancedb_compact_cron: env_or("HTKNOW_LANCEDB_COMPACT_CRON", "0 0 3 * * *"), + parse_enabled: env_or_parse("HTKNOW_PARSE_ENABLED", true), + reuse_duplicate_files: env_or_parse("HTKNOW_REUSE_DUPLICATE_FILES", true), + build_knowledge_graph: env_or_parse("HTKNOW_BUILD_KNOWLEDGE_GRAPH", false), + } + } +} + +impl ServicesConfig { + fn from_env() -> Self { + Self { + mineru_url: env_or("HTKNOW_MINERU_URL", "http://192.168.0.46:10001/file_parse"), + request_timeout_secs: env_or_parse("HTKNOW_REQUEST_TIMEOUT_SECS", 600), + mineru_max_pages: env_or_parse("HTKNOW_MINERU_MAX_PAGES", 50), + office_convert_url: env_or("HTKNOW_OFFICE_CONVERT_URL", "http://192.168.0.46:8003/convert"), + custom_parse_url: env_optional("HTKNOW_CUSTOM_PARSE_URL"), + custom_parse_reuse_url: env_optional("HTKNOW_CUSTOM_PARSE_REUSE_URL"), + audio_transcription_url: env_or( + "HTKNOW_AUDIO_TRANSCRIPTION_URL", + "http://192.168.0.46:59805/api/v1/audio/transcriptions", + ), + audio_transcription_key: std::env::var("HTKNOW_AUDIO_TRANSCRIPTION_KEY").ok(), + embedding_url: env_or("HTKNOW_EMBEDDING_URL", "http://222.190.139.186:59700/v1/embeddings"), + image_embedding_url: env_optional("HTKNOW_IMAGE_EMBEDDING_URL"), + rerank_url: env_or("HTKNOW_RERANK_URL", "http://222.190.139.186:59600/v1/rerank"), + image_parse_url: env_optional("HTKNOW_IMAGE_PARSE_URL"), + image_parse_timeout_secs: env_or_parse("HTKNOW_IMAGE_PARSE_TIMEOUT_SECS", 120), + image_parse_concurrency: env_or_parse("HTKNOW_IMAGE_PARSE_CONCURRENCY", 5), + } + } +} + +impl AIConfig { + fn from_env() -> Self { + let embedding_dim = env_or_parse("HTKNOW_EMBEDDING_DIM", 1024); + Self { + embedding_model: env_or("HTKNOW_EMBEDDING_MODEL", "bge-m3"), + embedding_dim, + image_embedding_dim: env_or_parse("HTKNOW_IMAGE_EMBEDDING_DIM", 2048), + rerank_model: env_or("HTKNOW_RERANK_MODEL", "bge-rerank"), + rerank_threshold: env_or_parse("HTKNOW_RERANK_THRESHOLD", 0.1), + embedding_batch_size: env_or_parse("HTKNOW_EMBEDDING_BATCH_SIZE", 8), + } + } +} + +impl DatabaseConfig { + fn from_env() -> Self { + Self { + url: env_or("DATABASE_URL", "sqlite://data/app.sqlite"), + // 文件型 SQLite 写操作串行，连接数过高只会加剧锁竞争；WAL 下读可并发，16 足够。 + max_connections: env_or_parse("HTKNOW_DB_MAX_CONNECTIONS", 16), + min_connections: env_or_parse("HTKNOW_DB_MIN_CONNECTIONS", 2), + busy_timeout_ms: env_or_parse("HTKNOW_DB_BUSY_TIMEOUT_MS", 5000), + init_default_kbs: env_or_parse("HTKNOW_DB_INIT_DEFAULT_KBS", true), + } + } +} + +impl StorageConfig { + fn from_env() -> Self { + let data_dir = env_or("HTKNOW_DATA_DIR", "data"); + Self { + lancedb_path: env_or("HTKNOW_LANCEDB_PATH", &format!("{}/lancedb_data", data_dir)), + temp_path: env_or("HTKNOW_TEMP_PATH", &format!("{}/temp", data_dir)), + images_path: env_or("HTKNOW_IMAGES_PATH", &format!("{}/images", data_dir)), + pdf_path: env_or("HTKNOW_PDF_PATH", &format!("{}/pdfs", data_dir)), + files_path: env_or("HTKNOW_FILES_PATH", &format!("{}/files", data_dir)), + archives_path: env_or("HTKNOW_ARCHIVES_PATH", &format!("{}/archives", data_dir)), + contents_path: env_or("HTKNOW_CONTENTS_PATH", &format!("{}/contents", data_dir)), + slice_contents_path: env_or("HTKNOW_SLICE_CONTENTS_PATH", &format!("{}/slice_contents", data_dir)), + } + } +} + +impl SearchConfig { + fn from_env() -> Self { + let data_dir = env_or("HTKNOW_DATA_DIR", "data"); + Self { + limit: env_or_parse("HTKNOW_SEARCH_LIMIT", 10), + tantivy_index_path: env_or("HTKNOW_TANTIVY_INDEX_PATH", &format!("{}/tantivy_index", data_dir)), + tantivy_full_index_path: env_or( + "HTKNOW_TANTIVY_FULL_INDEX_PATH", + &format!("{}/tantivy_full_index", data_dir), + ), + tantivy_memory_mb: env_or_parse("HTKNOW_TANTIVY_MEMORY_MB", 50), + tantivy_rebuild_batch_size: env_or_parse("HTKNOW_SEARCH_TANTIVY_REBUILD_BATCH_SIZE", 100), + lancedb_rebuild_batch_size: env_or_parse("HTKNOW_SEARCH_LANCEDB_REBUILD_BATCH_SIZE", 1000), + embedding_timeout_secs: env_or_parse("HTKNOW_SEARCH_EMBEDDING_TIMEOUT_SECS", 30), + rerank_timeout_secs: env_or_parse("HTKNOW_SEARCH_RERANK_TIMEOUT_SECS", 20), + synonym_enabled: env_or_parse("HTKNOW_SEARCH_SYNONYM_ENABLED", true), + synonym_boost: env_or_parse("HTKNOW_SEARCH_SYNONYM_BOOST", 0.7), + max_synonyms_per_term: env_or_parse("HTKNOW_SEARCH_MAX_SYNONYMS_PER_TERM", 5), + max_total_synonyms: env_or_parse("HTKNOW_SEARCH_MAX_TOTAL_SYNONYMS", 30), + highlight_page_min_chars: std::env::var("HTKNOW_HIGHLIGHT_PAGE_MIN_CHARS") + .ok() + .and_then(|v| v.parse().ok()) + .or_else(|| std::env::var("HTKNOW_HIGHLIGHT_PAGE_MIN_POSITIONS").ok().and_then(|v| v.parse().ok())) + .unwrap_or(20), + } + } +} + +impl SliceConfig { + fn from_env() -> Self { + Self { + smart_slice_max_chars: env_or_parse("HTKNOW_SMART_SLICE_MAX_CHARS", 8000), + fixed_slice_overlap_chars: env_or_parse("HTKNOW_FIXED_SLICE_OVERLAP_CHARS", 100), + } + } +} + +impl LLMConfig { + fn from_env() -> Self { + Self { + api_url: std::env::var("LLM_API_URL").ok(), + api_key: std::env::var("LLM_API_KEY").ok(), + model: env_or("LLM_MODEL", "gpt-3.5-turbo"), + } + } + + /// 检查 LLM 是否已启用 + pub fn is_enabled(&self) -> bool { + self.api_url.is_some() + } +} + +/// 从环境变量读取字符串，如果不存在则返回默认值 +fn env_or(key: &str, default: &str) -> String { + std::env::var(key).unwrap_or_else(|_| default.to_string()) +} + +/// 从环境变量读取字符串，空字符串视为未配置 +fn env_optional(key: &str) -> Option { + std::env::var(key).ok().and_then(|value| { + let trimmed = value.trim(); + if trimmed.is_empty() { None } else { Some(trimmed.to_string()) } + }) +} + +/// 从环境变量读取并解析值，如果不存在或解析失败则返回默认值 +fn env_or_parse(key: &str, default: T) -> T { + std::env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) +} + +// ============================================================================ +// etcd 支持（可选功能） +// ============================================================================ + +/// 配置加载器 trait +#[allow(dead_code)] +pub trait ConfigLoader: Send+Sync { + /// 加载配置 + fn load(&self) -> anyhow::Result; +} + +/// 环境变量配置加载器 +#[allow(dead_code)] +pub struct EnvConfigLoader; + +impl ConfigLoader for EnvConfigLoader { + fn load(&self) -> anyhow::Result { + Ok(AppConfig::from_env()) + } +} + +/// etcd 配置加载器（预留接口） +#[cfg(feature = "etcd")] +#[allow(dead_code)] +pub struct EtcdConfigLoader { + pub endpoints: Vec, + pub prefix: String, +} + +#[cfg(feature = "etcd")] +#[allow(dead_code)] +impl EtcdConfigLoader { + pub fn new(endpoints: Vec, prefix: String) -> Self { + Self { endpoints, prefix } + } + + /// 启动 watch 监听配置变化 + pub async fn watch(&self, _callback: F) -> anyhow::Result<()> + where + F: Fn(AppConfig)+Send+Sync+'static, { + // TODO: 实现 etcd watch 逻辑 + // 1. 连接 etcd + // 2. 监听 prefix 下的 key 变化 + // 3. 当配置变化时调用 callback + unimplemented!("etcd watch not implemented yet") + } +} + +#[cfg(feature = "etcd")] +impl ConfigLoader for EtcdConfigLoader { + fn load(&self) -> anyhow::Result { + // TODO: 实现从 etcd 读取配置 + // 1. 连接 etcd + // 2. 读取 prefix 下的所有 key + // 3. 解析并构建 AppConfig + // 4. 如果 etcd 中没有配置，回退到环境变量 + unimplemented!("etcd config loading not implemented yet") + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use once_cell::sync::Lazy; + + use super::*; + + static ENV_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); + + #[test] + fn test_default_config() { + let _guard = ENV_LOCK.lock().unwrap(); + unsafe { + std::env::remove_var("HTKNOW_PARSE_ENABLED"); + std::env::remove_var("HTKNOW_BUILD_KNOWLEDGE_GRAPH"); + std::env::remove_var("HTKNOW_REQUEST_TIMEOUT_SECS"); + } + let config = AppConfig::from_env(); + + // 验证默认值 + assert_eq!(config.server.host, "0.0.0.0"); + assert_eq!(config.server.port, 3000); + assert_eq!(config.server.upload_limit_mb, 500); + assert_eq!(config.ai.embedding_model, "bge-m3"); + assert_eq!(config.ai.embedding_dim, 1024); + assert_eq!(config.ai.image_embedding_dim, 2048); + assert_eq!(config.services.request_timeout_secs, 600); + assert_eq!(config.search.embedding_timeout_secs, 30); + assert_eq!(config.search.rerank_timeout_secs, 20); + assert!(config.server.parse_enabled); + assert!(!config.server.build_knowledge_graph); + assert_eq!(config.search.highlight_page_min_chars, 20); + } + + #[test] + fn test_get_config() { + let config = get(); + assert!(!config.server.host.is_empty()); + } + + #[test] + fn test_parse_enabled_env_override() { + let _guard = ENV_LOCK.lock().unwrap(); + unsafe { + std::env::set_var("HTKNOW_PARSE_ENABLED", "false"); + } + let config = AppConfig::from_env(); + assert!(!config.server.parse_enabled); + unsafe { + std::env::remove_var("HTKNOW_PARSE_ENABLED"); + } + } + + #[test] + fn test_build_knowledge_graph_env_override() { + let _guard = ENV_LOCK.lock().unwrap(); + unsafe { + std::env::set_var("HTKNOW_BUILD_KNOWLEDGE_GRAPH", "true"); + } + let config = AppConfig::from_env(); + assert!(config.server.build_knowledge_graph); + unsafe { + std::env::remove_var("HTKNOW_BUILD_KNOWLEDGE_GRAPH"); + } + } + + #[test] + fn test_request_timeout_env_override() { + let _guard = ENV_LOCK.lock().unwrap(); + unsafe { + std::env::set_var("HTKNOW_REQUEST_TIMEOUT_SECS", "120"); + } + let config = AppConfig::from_env(); + assert_eq!(config.services.request_timeout_secs, 120); + unsafe { + std::env::remove_var("HTKNOW_REQUEST_TIMEOUT_SECS"); + } + } +} diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..e042087 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,1061 @@ +use std::{ + collections::HashSet, + fs::OpenOptions, + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::Context; +use log::info; +use sqlx::{ + QueryBuilder, Row, Sqlite, SqlitePool, + sqlite::{SqliteConnectOptions, SqlitePoolOptions, SqliteSynchronous}, +}; + +use crate::config; + +/// 初始化数据库连接池并自动创建表 +/// +/// 在连接之前，如果 DATABASE_URL 指向一个文件型的 SQLite 数据库（例如 `sqlite://path/to/db.sqlite` +/// 或者直接 `./data/db.sqlite`），会确保父目录存在并且数据库文件被创建（如果不存在）。 +pub async fn init() -> anyhow::Result { + // 从配置读取数据库设置 + let cfg = config::get(); + let database_url = &cfg.database.url; + + // 尝试从 URL 中解析出文件路径并提前创建目录/文件 + if let Some(db_path) = sqlite_path_from_url(database_url) { + ensure_db_file(&db_path).with_context(|| format!("failed to ensure sqlite db file: {}", db_path.display()))?; + } else { + info!("Detected non-file SQLite URL or in-memory DB; skipping file creation"); + } + + anyhow::ensure!( + cfg.database.busy_timeout_ms <= i32::MAX as u64, + "HTKNOW_DB_BUSY_TIMEOUT_MS is too large for sqlite busy_timeout: {}", + cfg.database.busy_timeout_ms + ); + + let is_file_db = sqlite_path_from_url(database_url).is_some(); + let mut connect_options = database_url + .parse::() + .with_context(|| format!("failed to parse sqlite database url: {}", database_url))? + .create_if_missing(is_file_db) + .foreign_keys(true) + .busy_timeout(Duration::from_millis(cfg.database.busy_timeout_ms)); + + // 文件型 SQLite 的写性能调优：WAL 下 NORMAL 同步级别足够安全且明显更快； + // 增大页缓存与 mmap、临时表落内存，减少磁盘 IO。内存库无需这些设置。 + if is_file_db { + connect_options = connect_options + .synchronous(SqliteSynchronous::Normal) + .pragma("cache_size", "-65536") // 64MB 页缓存（负值表示 KB） + .pragma("mmap_size", "268435456") // 256MB 内存映射 + .pragma("temp_store", "MEMORY"); + } + + info!("Connecting to SQLite database..."); + + let pool = SqlitePoolOptions::new() + .max_connections(cfg.database.max_connections) + .min_connections(cfg.database.min_connections) + .connect_with(connect_options) + .await?; + + info!("SQLite database connected successfully"); + + // WAL 是数据库文件级设置；foreign_keys 和 busy_timeout 已通过连接选项应用到每条连接。 + if is_file_db { + let journal_mode: String = sqlx::query_scalar("PRAGMA journal_mode = WAL;").fetch_one(&pool).await?; + anyhow::ensure!( + journal_mode.eq_ignore_ascii_case("wal"), + "failed to enable sqlite WAL journal mode, actual mode: {}", + journal_mode + ); + } + + // 自动创建表 + create_tables(&pool).await?; + run_schema_migrations(&pool).await?; + ensure_file_content_externalized(&pool).await?; + ensure_slice_content_externalized(&pool).await?; + ensure_kb_type_column(&pool).await?; + ensure_parse_priority_column(&pool).await?; + ensure_file_parse_priority_column(&pool).await?; + ensure_user_name_columns(&pool).await?; + ensure_file_size_column(&pool).await?; + ensure_pdf_contents_externalized(&pool).await?; + ensure_slice_positions_excel_columns(&pool).await?; + ensure_image_description_support(&pool).await?; + create_indexes(&pool).await?; + ensure_orphan_slices_cleaned(&pool).await?; + if cfg.database.init_default_kbs { + ensure_default_knowledge_bases(&pool).await?; + } else { + info!("Skipping default knowledge base initialization"); + } + + info!("Database initialized successfully"); + + Ok(pool) +} + +/// 对已有数据库执行只增不减的版本化迁移。 +/// +/// `init.sql` 只负责全新数据库；已有表不会因为 `CREATE TABLE IF NOT EXISTS` +/// 自动增加列，因此升级必须在这里显式处理。 +async fn run_schema_migrations(pool: &SqlitePool) -> anyhow::Result<()> { + sqlx::query( + "CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) + )", + ) + .execute(pool) + .await?; + + const VERSION: i64 = 1; + let mut tx = pool.begin().await?; + // 先在同一事务中抢占版本号。多实例同时升级时，只有一个实例执行 DDL； + // DDL 失败会连同版本记录一起回滚。 + let claimed = sqlx::query("INSERT OR IGNORE INTO schema_migrations(version, name) VALUES (?, ?)") + .bind(VERSION) + .bind("parse_run_id_and_slice_ordinal") + .execute(&mut *tx) + .await? + .rows_affected(); + if claimed == 0 { + tx.rollback().await?; + run_parse_artifact_migration(pool).await?; + run_image_description_migration(pool).await?; + run_file_summary_migration(pool).await?; + run_image_description_jpg_filename_migration(pool).await?; + return Ok(()); + } + + let file_columns = sqlx::query("PRAGMA table_info(files)").fetch_all(&mut *tx).await?; + if !file_columns.iter().any(|row| row.get::("name") == "parse_run_id") { + sqlx::query("ALTER TABLE files ADD COLUMN parse_run_id TEXT DEFAULT NULL").execute(&mut *tx).await?; + } + + let slice_columns = sqlx::query("PRAGMA table_info(slices)").fetch_all(&mut *tx).await?; + if !slice_columns.iter().any(|row| row.get::("name") == "parse_run_id") { + sqlx::query("ALTER TABLE slices ADD COLUMN parse_run_id TEXT DEFAULT NULL").execute(&mut *tx).await?; + } + if !slice_columns.iter().any(|row| row.get::("name") == "ordinal") { + sqlx::query("ALTER TABLE slices ADD COLUMN ordinal INTEGER DEFAULT NULL").execute(&mut *tx).await?; + } + + sqlx::query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_slices_file_run_ordinal + ON slices(file_id, parse_run_id, ordinal) + WHERE parse_run_id IS NOT NULL AND ordinal IS NOT NULL", + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + info!("Applied schema migration {}: parse_run_id_and_slice_ordinal", VERSION); + + run_parse_artifact_migration(pool).await?; + run_image_description_migration(pool).await?; + run_file_summary_migration(pool).await?; + run_image_description_jpg_filename_migration(pool).await?; + Ok(()) +} + +async fn run_parse_artifact_migration(pool: &SqlitePool) -> anyhow::Result<()> { + const VERSION: i64 = 2; + let mut tx = pool.begin().await?; + let claimed = sqlx::query("INSERT OR IGNORE INTO schema_migrations(version, name) VALUES (?, ?)") + .bind(VERSION) + .bind("shared_parse_artifacts") + .execute(&mut *tx) + .await? + .rows_affected(); + if claimed == 0 { + tx.rollback().await?; + return Ok(()); + } + let columns = sqlx::query("PRAGMA table_info(files)").fetch_all(&mut *tx).await?; + if !columns.iter().any(|row| row.get::("name") == "artifact_id") { + sqlx::query("ALTER TABLE files ADD COLUMN artifact_id INTEGER DEFAULT NULL").execute(&mut *tx).await?; + } + sqlx::query( + "CREATE TABLE IF NOT EXISTS parse_artifacts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + artifact_key TEXT NOT NULL UNIQUE, + content_hash TEXT NOT NULL, + slice_type TEXT NOT NULL, + parser_version TEXT NOT NULL, + config_hash TEXT NOT NULL, + source_file_id INTEGER NOT NULL, + full_content TEXT DEFAULT NULL, + summary TEXT DEFAULT NULL, + created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), + updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) + )", + ) + .execute(&mut *tx) + .await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_files_artifact_id ON files(artifact_id)").execute(&mut *tx).await?; + tx.commit().await?; + info!("Applied schema migration {}: shared_parse_artifacts", VERSION); + Ok(()) +} + +async fn run_image_description_migration(pool: &SqlitePool) -> anyhow::Result<()> { + const VERSION: i64 = 3; + let mut tx = pool.begin().await?; + let claimed = sqlx::query("INSERT OR IGNORE INTO schema_migrations(version, name) VALUES (?, ?)") + .bind(VERSION) + .bind("image_descriptions_and_slice_is_image") + .execute(&mut *tx) + .await? + .rows_affected(); + if claimed == 0 { + tx.rollback().await?; + return Ok(()); + } + let slice_columns = sqlx::query("PRAGMA table_info(slices)").fetch_all(&mut *tx).await?; + if !slice_columns.iter().any(|row| row.get::("name") == "is_image") { + sqlx::query("ALTER TABLE slices ADD COLUMN is_image INTEGER NOT NULL DEFAULT 0").execute(&mut *tx).await?; + } + sqlx::query( + "CREATE TABLE IF NOT EXISTS image_descriptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL, + image_filename TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + raw_response TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')), + UNIQUE(file_id, image_filename) + )", + ) + .execute(&mut *tx) + .await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_image_descriptions_file_id ON image_descriptions(file_id)") + .execute(&mut *tx) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_image_descriptions_file_filename ON image_descriptions(file_id, image_filename)", + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + info!("Applied schema migration {}: image_descriptions_and_slice_is_image", VERSION); + Ok(()) +} + +async fn run_image_description_jpg_filename_migration(pool: &SqlitePool) -> anyhow::Result<()> { + const VERSION: i64 = 5; + let mut tx = pool.begin().await?; + let claimed = sqlx::query("INSERT OR IGNORE INTO schema_migrations(version, name) VALUES (?, ?)") + .bind(VERSION) + .bind("image_description_jpg_filename") + .execute(&mut *tx) + .await? + .rows_affected(); + if claimed == 0 { + tx.rollback().await?; + return Ok(()); + } + let columns = sqlx::query("PRAGMA table_info(image_descriptions)").fetch_all(&mut *tx).await?; + if !columns.iter().any(|row| row.get::("name") == "jpg_filename") { + sqlx::query("ALTER TABLE image_descriptions ADD COLUMN jpg_filename TEXT DEFAULT NULL") + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + info!("Applied schema migration {}: image_description_jpg_filename", VERSION); + Ok(()) +} +async fn run_file_summary_migration(pool: &SqlitePool) -> anyhow::Result<()> { + const VERSION: i64 = 4; + let mut tx = pool.begin().await?; + let claimed = sqlx::query("INSERT OR IGNORE INTO schema_migrations(version, name) VALUES (?, ?)") + .bind(VERSION) + .bind("file_summaries") + .execute(&mut *tx) + .await? + .rows_affected(); + if claimed == 0 { + tx.rollback().await?; + return Ok(()); + } + + let file_columns = sqlx::query("PRAGMA table_info(files)").fetch_all(&mut *tx).await?; + if !file_columns.iter().any(|row| row.get::("name") == "summary") { + sqlx::query("ALTER TABLE files ADD COLUMN summary TEXT DEFAULT NULL").execute(&mut *tx).await?; + } + + let artifact_columns = sqlx::query("PRAGMA table_info(parse_artifacts)").fetch_all(&mut *tx).await?; + if !artifact_columns.iter().any(|row| row.get::("name") == "summary") { + sqlx::query("ALTER TABLE parse_artifacts ADD COLUMN summary TEXT DEFAULT NULL").execute(&mut *tx).await?; + } + + tx.commit().await?; + info!("Applied schema migration {}: file_summaries", VERSION); + Ok(()) +} + +const DEFAULT_KNOWLEDGE_BASES: &[(i64, &str, &str, &str, i32)] = &[ + ( + 1, + "个人空间", + "您的私有工作区。管理个人草稿、收藏的文档以及发布的方案报告。支持跨设备同步与离线访问。", + "analysis", + 1, + ), + (2, "技术资料", "技术手册、船舰资料。", "analysis", 1), + (3, "法规标准", "舰船修理法规、行业标准。", "analysis", 1), + (4, "通用知识", "基本原理、结构组成等。", "analysis", 1), + (5, "故障知识", "典型故障现象、原因分析等。", "analysis", 1), + (6, "图库", "设备结构与故障样本。", "analysis", 1), + (7, "VDR数据仓", "全船航行与运行数据回放。", "storage", 1), +]; + +/// 自动创建所有需要的表 +async fn create_tables(pool: &SqlitePool) -> anyhow::Result<()> { + info!("Creating tables if not exists..."); + + let init_sql = include_str!("init.sql"); + for (idx, sql) in init_sql.split(";").enumerate() { + let sql = sql.trim(); + if sql.is_empty() || is_index_statement(sql) { + continue; + } + sqlx::query(sql) + .execute(pool) + .await + .with_context(|| format!("failed to execute init.sql statement {}", idx + 1))?; + } + + info!("Tables created successfully"); + + Ok(()) +} + +async fn create_indexes(pool: &SqlitePool) -> anyhow::Result<()> { + info!("Creating indexes if not exists..."); + + let init_sql = include_str!("init.sql"); + for (idx, sql) in init_sql.split(";").enumerate() { + let sql = sql.trim(); + if !is_index_statement(sql) { + continue; + } + sqlx::query(sql) + .execute(pool) + .await + .with_context(|| format!("failed to execute init.sql index statement {}", idx + 1))?; + } + + info!("Indexes created successfully"); + Ok(()) +} + +fn is_index_statement(sql: &str) -> bool { + let statement = + sql.lines().map(str::trim).find(|line| !line.is_empty() && !line.starts_with("--")).unwrap_or_default(); + statement.starts_with("CREATE INDEX") || statement.starts_with("CREATE UNIQUE INDEX") +} + +async fn ensure_default_knowledge_bases(pool: &SqlitePool) -> anyhow::Result<()> { + let default_ids: Vec = DEFAULT_KNOWLEDGE_BASES.iter().map(|(id, ..)| *id).collect(); + let placeholders = vec!["?"; default_ids.len()].join(", "); + let sql = format!("SELECT id FROM knowledge_bases WHERE id IN ({})", placeholders); + let mut query = sqlx::query_scalar(&sql); + for id in &default_ids { + query = query.bind(id); + } + let existing_ids: Vec = query.fetch_all(pool).await?; + let existing_set: HashSet = existing_ids.into_iter().collect(); + let mut inserted = 0; + + for &(id, name, description, kb_type, is_public) in DEFAULT_KNOWLEDGE_BASES { + if existing_set.contains(&id) { + continue; + } + sqlx::query( + "INSERT INTO knowledge_bases \ + (id, user_id, user_name, name, description, kb_type, parent_id, is_public) \ + VALUES (?, ?, ?, ?, ?, ?, NULL, ?)", + ) + .bind(id) + .bind("") + .bind("") + .bind(name) + .bind(description) + .bind(kb_type) + .bind(is_public) + .execute(pool) + .await?; + inserted += 1; + } + + if inserted > 0 { + info!("Inserted {} default knowledge bases", inserted); + } + + Ok(()) +} + +async fn ensure_kb_type_column(pool: &SqlitePool) -> anyhow::Result<()> { + let columns = sqlx::query("PRAGMA table_info(knowledge_bases);").fetch_all(pool).await?; + let has_kb_type = columns.iter().any(|row| row.get::("name") == "kb_type"); + + if !has_kb_type { + sqlx::query("ALTER TABLE knowledge_bases ADD COLUMN kb_type TEXT NOT NULL DEFAULT 'analysis'") + .execute(pool) + .await?; + } + + Ok(()) +} + +async fn ensure_parse_priority_column(pool: &SqlitePool) -> anyhow::Result<()> { + let columns = sqlx::query("PRAGMA table_info(knowledge_bases);").fetch_all(pool).await?; + let has_parse_priority = columns.iter().any(|row| row.get::("name") == "parse_priority"); + + if !has_parse_priority { + sqlx::query("ALTER TABLE knowledge_bases ADD COLUMN parse_priority INTEGER NOT NULL DEFAULT 50") + .execute(pool) + .await?; + } else { + sqlx::query("UPDATE knowledge_bases SET parse_priority = 50 WHERE parse_priority IS NULL") + .execute(pool) + .await?; + } + + Ok(()) +} + +async fn ensure_file_parse_priority_column(pool: &SqlitePool) -> anyhow::Result<()> { + let columns = sqlx::query("PRAGMA table_info(files);").fetch_all(pool).await?; + let has_parse_priority = columns.iter().any(|row| row.get::("name") == "parse_priority"); + + if !has_parse_priority { + sqlx::query("ALTER TABLE files ADD COLUMN parse_priority INTEGER NOT NULL DEFAULT 50").execute(pool).await?; + } + + sqlx::query( + "UPDATE files + SET parse_priority = COALESCE( + (SELECT kb.parse_priority FROM knowledge_bases kb WHERE kb.id = files.kb_id), + 50 + ) + WHERE status = 0 + AND parse_priority != COALESCE( + (SELECT kb.parse_priority FROM knowledge_bases kb WHERE kb.id = files.kb_id), + 50 + )", + ) + .execute(pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_files_pending_status_priority_created_at_id + ON files(status, parse_priority DESC, created_at ASC, id ASC) + WHERE status = 0", + ) + .execute(pool) + .await?; + + ensure_file_parse_priority_triggers(pool).await?; + + Ok(()) +} + +async fn ensure_file_parse_priority_triggers(pool: &SqlitePool) -> anyhow::Result<()> { + sqlx::query( + "CREATE TRIGGER IF NOT EXISTS trg_files_parse_priority_after_insert + AFTER INSERT ON files + BEGIN + UPDATE files + SET parse_priority = COALESCE( + (SELECT kb.parse_priority FROM knowledge_bases kb WHERE kb.id = NEW.kb_id), + 50 + ) + WHERE id = NEW.id + AND parse_priority != COALESCE( + (SELECT kb.parse_priority FROM knowledge_bases kb WHERE kb.id = NEW.kb_id), + 50 + ); + END", + ) + .execute(pool) + .await?; + + sqlx::query( + "CREATE TRIGGER IF NOT EXISTS trg_files_parse_priority_after_kb_change + AFTER UPDATE OF kb_id ON files + BEGIN + UPDATE files + SET parse_priority = COALESCE( + (SELECT kb.parse_priority FROM knowledge_bases kb WHERE kb.id = NEW.kb_id), + 50 + ) + WHERE id = NEW.id + AND parse_priority != COALESCE( + (SELECT kb.parse_priority FROM knowledge_bases kb WHERE kb.id = NEW.kb_id), + 50 + ); + END", + ) + .execute(pool) + .await?; + + sqlx::query( + "CREATE TRIGGER IF NOT EXISTS trg_files_parse_priority_after_pending + AFTER UPDATE OF status ON files + WHEN NEW.status = 0 + BEGIN + UPDATE files + SET parse_priority = COALESCE( + (SELECT kb.parse_priority FROM knowledge_bases kb WHERE kb.id = NEW.kb_id), + 50 + ) + WHERE id = NEW.id + AND parse_priority != COALESCE( + (SELECT kb.parse_priority FROM knowledge_bases kb WHERE kb.id = NEW.kb_id), + 50 + ); + END", + ) + .execute(pool) + .await?; + + sqlx::query( + "CREATE TRIGGER IF NOT EXISTS trg_kbs_parse_priority_update_pending_files + AFTER UPDATE OF parse_priority ON knowledge_bases + BEGIN + UPDATE files + SET parse_priority = NEW.parse_priority + WHERE kb_id = NEW.id + AND status = 0 + AND parse_priority != NEW.parse_priority; + END", + ) + .execute(pool) + .await?; + + Ok(()) +} + +async fn ensure_user_name_columns(pool: &SqlitePool) -> anyhow::Result<()> { + ensure_user_name_column(pool, "files").await?; + ensure_user_name_column(pool, "knowledge_bases").await?; + Ok(()) +} + +async fn ensure_file_size_column(pool: &SqlitePool) -> anyhow::Result<()> { + let columns = sqlx::query("PRAGMA table_info(files);").fetch_all(pool).await?; + let has_size = columns.iter().any(|row| row.get::("name") == "size"); + + if !has_size { + sqlx::query("ALTER TABLE files ADD COLUMN size INTEGER NOT NULL DEFAULT 0").execute(pool).await?; + } + + Ok(()) +} + +/// 若 `files` 表仍存在旧版的 `content` 列，则把内容迁移到 `contents/` 目录并删除该列。 +async fn ensure_file_content_externalized(pool: &SqlitePool) -> anyhow::Result<()> { + let columns = sqlx::query("PRAGMA table_info(files);").fetch_all(pool).await?; + let has_content = columns.iter().any(|row| row.get::("name") == "content"); + + if !has_content { + return Ok(()); + } + + info!("Detected legacy `files.content` column, migrating to filesystem..."); + + let contents_path = &config::get().storage.contents_path; + let contents_dir = std::path::Path::new(contents_path); + tokio::fs::create_dir_all(contents_dir) + .await + .with_context(|| format!("failed to create contents directory {}", contents_dir.display()))?; + + const BATCH: i64 = 100; + let mut offset = 0i64; + let mut migrated = 0usize; + loop { + let rows: Vec<(i64, Option)> = + sqlx::query_as("SELECT id, content FROM files WHERE content IS NOT NULL ORDER BY id LIMIT ? OFFSET ?") + .bind(BATCH) + .bind(offset) + .fetch_all(pool) + .await?; + + if rows.is_empty() { + break; + } + + for (id, content) in rows { + if let Some(content) = content { + crate::file_content::write(id, &content).await?; + migrated += 1; + } + } + + offset += BATCH; + } + + sqlx::query("ALTER TABLE files DROP COLUMN content").execute(pool).await?; + + info!("`files.content` migration completed, {} files migrated", migrated); + Ok(()) +} + +/// 将旧版 `slices.content` 聚合迁移到每个源文件一个 JSON 内容包后删除大文本列。 +async fn ensure_slice_content_externalized(pool: &SqlitePool) -> anyhow::Result<()> { + let columns = sqlx::query("PRAGMA table_info(slices)").fetch_all(pool).await?; + if !columns.iter().any(|row| row.get::("name") == "content") { + return Ok(()); + } + info!("Detected legacy `slices.content` column, migrating to slice content files..."); + const BATCH: i64 = 100; + let mut last_file_id = 0_i64; + loop { + let file_ids: Vec = + sqlx::query_scalar("SELECT DISTINCT file_id FROM slices WHERE file_id > ? ORDER BY file_id LIMIT ?") + .bind(last_file_id) + .bind(BATCH) + .fetch_all(pool) + .await?; + if file_ids.is_empty() { + break; + } + for file_id in &file_ids { + let rows: Vec<(i64, String)> = sqlx::query_as("SELECT id, content FROM slices WHERE file_id = ?") + .bind(file_id) + .fetch_all(pool) + .await?; + crate::slice_content::write_all(*file_id, &rows.into_iter().collect()).await?; + } + last_file_id = *file_ids.last().unwrap(); + } + sqlx::query("ALTER TABLE slices DROP COLUMN content").execute(pool).await?; + info!("`slices.content` migration completed"); + Ok(()) +} + +async fn ensure_user_name_column(pool: &SqlitePool, table: &str) -> anyhow::Result<()> { + let pragma_sql = format!("PRAGMA table_info({});", table); + let columns = sqlx::query(&pragma_sql).fetch_all(pool).await?; + let has_user_name = columns.iter().any(|row| row.get::("name") == "user_name"); + + if !has_user_name { + let alter_sql = format!("ALTER TABLE {} ADD COLUMN user_name TEXT NOT NULL DEFAULT ''", table); + sqlx::query(&alter_sql).execute(pool).await?; + } else { + let backfill_sql = format!("UPDATE {} SET user_name = '' WHERE user_name IS NULL", table); + sqlx::query(&backfill_sql).execute(pool).await?; + } + + Ok(()) +} + +async fn ensure_pdf_contents_externalized(pool: &SqlitePool) -> anyhow::Result<()> { + let table_exists: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pdf_contents'") + .fetch_one(pool) + .await?; + if table_exists == 0 { + return Ok(()); + } + + let columns = sqlx::query("PRAGMA table_info(pdf_contents);").fetch_all(pool).await?; + let has_bbox = columns.iter().any(|row| row.get::("name") == "bbox"); + if !has_bbox { + sqlx::query("ALTER TABLE pdf_contents ADD COLUMN bbox TEXT DEFAULT NULL").execute(pool).await?; + } + + info!("Detected legacy `pdf_contents` table, migrating to JSON files..."); + let file_ids: Vec = + sqlx::query_scalar("SELECT DISTINCT file_id FROM pdf_contents ORDER BY file_id").fetch_all(pool).await?; + for file_id in &file_ids { + let rows = sqlx::query( + "SELECT page_idx, bbox, text, text_level, img_path, table_body FROM pdf_contents WHERE file_id = ? ORDER BY page_idx, id", + ) + .bind(file_id) + .fetch_all(pool) + .await?; + let contents = rows + .into_iter() + .map(|row| crate::pdf_content::PdfContent { + page_idx: row.get("page_idx"), + bbox: row.get("bbox"), + text: row.get("text"), + text_level: row.get("text_level"), + img_path: row.get("img_path"), + table_body: row.get("table_body"), + }) + .collect::>(); + crate::pdf_content::write(*file_id, &contents).await?; + } + sqlx::query("DROP TABLE pdf_contents").execute(pool).await?; + info!("`pdf_contents` migration completed, {} files migrated", file_ids.len()); + + Ok(()) +} + +async fn ensure_slice_positions_excel_columns(pool: &SqlitePool) -> anyhow::Result<()> { + let columns = sqlx::query("PRAGMA table_info(slice_positions);").fetch_all(pool).await?; + + let has_sheet_name = columns.iter().any(|row| row.get::("name") == "sheet_name"); + let has_row_num = columns.iter().any(|row| row.get::("name") == "row_num"); + + if !has_sheet_name { + sqlx::query("ALTER TABLE slice_positions ADD COLUMN sheet_name TEXT DEFAULT NULL").execute(pool).await?; + } + if !has_row_num { + sqlx::query("ALTER TABLE slice_positions ADD COLUMN row_num INTEGER DEFAULT NULL").execute(pool).await?; + } + + Ok(()) +} + +/// 兜底修复：确保图片文本化相关的表/列存在。 +/// +/// 用于处理早期版本迁移标记已写入但列未成功添加的异常场景。 +async fn ensure_image_description_support(pool: &SqlitePool) -> anyhow::Result<()> { + let columns = sqlx::query("PRAGMA table_info(slices);").fetch_all(pool).await?; + if !columns.iter().any(|row| row.get::("name") == "is_image") { + sqlx::query("ALTER TABLE slices ADD COLUMN is_image INTEGER NOT NULL DEFAULT 0").execute(pool).await?; + info!("Repaired slices table: added is_image column"); + } + + sqlx::query( + "CREATE TABLE IF NOT EXISTS image_descriptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL, + image_filename TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + raw_response TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')), + UNIQUE(file_id, image_filename) + )", + ) + .execute(pool) + .await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_image_descriptions_file_id ON image_descriptions(file_id)") + .execute(pool) + .await?; + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_image_descriptions_file_filename ON image_descriptions(file_id, image_filename)", + ) + .execute(pool) + .await?; + Ok(()) +} + +/// 清理 slices 表中 file_id 已不在 files 表的孤儿切片。 +/// +/// slices 表没有外键约束到 files，历史删除可能遗留孤儿行；这会导致 Tantivy/LanceDB +/// 与 SQLite 的计数口径不一致，从而每次启动都触发重建。该 helper 在启动时兜底清理。 +async fn ensure_orphan_slices_cleaned(pool: &SqlitePool) -> anyhow::Result<()> { + const BATCH: usize = 900; + + let orphan_file_ids: Vec = sqlx::query_scalar( + "SELECT DISTINCT s.file_id FROM slices s LEFT JOIN files f ON f.id = s.file_id WHERE f.id IS NULL", + ) + .fetch_all(pool) + .await?; + if orphan_file_ids.is_empty() { + return Ok(()); + } + + info!("Found {} orphan file ids with slices to clean up", orphan_file_ids.len()); + + // 分批收集孤儿 slice id 并清理关联数据。 + let mut total_slices_removed = 0usize; + for chunk in orphan_file_ids.chunks(BATCH) { + let slice_ids: Vec = { + let mut qb = QueryBuilder::::new( + "SELECT s.id FROM slices s LEFT JOIN files f ON f.id = s.file_id WHERE f.id IS NULL AND s.file_id IN (", + ); + push_i64_list(&mut qb, chunk); + qb.push(")"); + qb.build_query_scalar().fetch_all(pool).await? + }; + + if slice_ids.is_empty() { + continue; + } + + for slice_chunk in slice_ids.chunks(BATCH) { + let mut mentions_qb = QueryBuilder::::new("DELETE FROM entity_mentions WHERE slice_id IN ("); + push_i64_list(&mut mentions_qb, slice_chunk); + mentions_qb.push(")"); + mentions_qb.build().execute(pool).await?; + + let mut positions_qb = QueryBuilder::::new("DELETE FROM slice_positions WHERE slice_id IN ("); + push_i64_list(&mut positions_qb, slice_chunk); + positions_qb.push(")"); + positions_qb.build().execute(pool).await?; + + let mut slices_qb = QueryBuilder::::new("DELETE FROM slices WHERE id IN ("); + push_i64_list(&mut slices_qb, slice_chunk); + slices_qb.push(")"); + slices_qb.build().execute(pool).await?; + } + + total_slices_removed += slice_ids.len(); + } + + info!("Cleaned up {} orphan slices for {} orphan files", total_slices_removed, orphan_file_ids.len()); + Ok(()) +} +/// 支持的形式（常见）： +/// - sqlite://path/to/db.sqlite +/// - sqlite:////absolute/path/to/db.sqlite +/// - file:relative/or/absolute/path.db +/// - 直接给文件路径：./data/db.sqlite 或 /abs/path/db.sqlite +/// +/// 返回 None 的情况包括内存数据库（包含 "memory" 的 URL）或不能识别为文件路径的 URL。 +fn sqlite_path_from_url(url: &str) -> Option { + let s = url.trim(); + + if is_sqlite_memory_url(s) { + return None; + } + + // sqlite://path + if let Some(rest) = s.strip_prefix("sqlite://") { + // 有可能是 sqlite:///absolute/path（多一个斜杠），也可能是相对路径 sqlite://./db.sqlite + // 对于前者，rest 以 / 开头，PathBuf 能正确处理 + let path = strip_uri_suffix(rest).to_string(); + if path.is_empty() { + return None; + } + return Some(PathBuf::from(path)); + } + + // sqlite:... 例如 sqlite:./db.sqlite 或 sqlite::memory: + if let Some(rest) = s.strip_prefix("sqlite:") { + // 如果是 ::memory: 或 :memory: 等，上面已通过 contains("memory") 处理 + let path = strip_uri_suffix(rest.trim_start_matches("//")).to_string(); + if path.is_empty() { + return None; + } + return Some(PathBuf::from(path)); + } + + // file: URI + if let Some(rest) = s.strip_prefix("file:") { + if is_sqlite_memory_url(rest) { + return None; + } + // file: would be followed by a path + let path = strip_uri_suffix(rest.trim_start_matches("//")).to_string(); + if path.is_empty() { + return None; + } + return Some(PathBuf::from(path)); + } + + // 如果字符串看起来像普通的文件路径（不含 scheme://），则直接返回 PathBuf + // 简单判断：如果包含 path separator 或以 '.' 开头，或以 '/' 开头，视为文件路径 + if s.starts_with('.') || s.starts_with('/') || s.contains(std::path::MAIN_SEPARATOR) { + return Some(PathBuf::from(s)); + } + + // 其它情况（例如带有其他 scheme），不处理 + None +} + +fn is_sqlite_memory_url(url: &str) -> bool { + url == ":memory:" || url.contains(":memory:") || url.contains("mode=memory") +} + +fn strip_uri_suffix(value: &str) -> &str { + let query_idx = value.find('?').unwrap_or(value.len()); + let fragment_idx = value.find('#').unwrap_or(value.len()); + &value[..query_idx.min(fragment_idx)] +} + +/// 确保数据库文件以及其父目录存在。如果父目录不存在会创建，数据库文件不存在会创建空文件。 +fn ensure_db_file(path: &Path) -> anyhow::Result<()> { + if let Some(parent) = path.parent() + && !parent.exists() + { + std::fs::create_dir_all(parent) + .with_context(|| format!("failed to create parent directory {}", parent.display()))?; + info!("Created parent directory for sqlite DB: {}", parent.display()); + } + + // 创建空文件（若已存在则不修改） + if !path.exists() { + OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(path) + .with_context(|| format!("failed to create sqlite db file {}", path.display()))?; + info!("Created sqlite DB file: {}", path.display()); + } else { + info!("Sqlite DB file already exists: {}", path.display()); + } + + Ok(()) +} + +/// 将 i64 ID 列表追加到 QueryBuilder 的 `IN (...)` 子句中。 +/// 调用方需自行在前后添加括号。 +pub fn push_i64_list(qb: &mut QueryBuilder<'_, Sqlite>, ids: &[i64]) { + let mut separated = qb.separated(", "); + for id in ids { + separated.push_bind(*id); + } +} + +/// 批量 INSERT 并返回自增 id。 +/// +/// - `prefix_sql`: 形如 `INSERT INTO table (col1, col2) `（含末尾空格） +/// - `bind`: 为每一行绑定列值，例如 `|qb, row| { qb.push_bind(row.0).push_bind(row.1); }` +/// - `binds_per_row`: 每行绑定的变量数，用于按 SQLite 999 变量上限分块 +pub async fn batch_insert_with_returning( + tx: &mut sqlx::Transaction<'_, Sqlite>, prefix_sql: &str, rows: &[T], binds_per_row: usize, mut bind: BindFn, +) -> anyhow::Result> +where + BindFn: FnMut(&mut QueryBuilder<'_, Sqlite>, &T), +{ + if rows.is_empty() { + return Ok(Vec::new()); + } + const MAX_VARS: usize = 999; + let batch_size = std::cmp::max(1, MAX_VARS / binds_per_row); + let mut ids = Vec::with_capacity(rows.len()); + for chunk in rows.chunks(batch_size) { + let mut qb = QueryBuilder::::new(prefix_sql); + qb.push("VALUES "); + for (i, row) in chunk.iter().enumerate() { + if i > 0 { + qb.push(", "); + } + qb.push("("); + bind(&mut qb, row); + qb.push(")"); + } + qb.push(" RETURNING id"); + let inserted: Vec<(i64,)> = qb.build_query_as().fetch_all(&mut **tx).await?; + anyhow::ensure!( + inserted.len() == chunk.len(), + "inserted row count mismatch: expected {}, got {}", + chunk.len(), + inserted.len() + ); + ids.extend(inserted.into_iter().map(|(id,)| id)); + } + Ok(ids) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn legacy_schema_adds_columns_before_creating_indexes() -> anyhow::Result<()> { + let pool = SqlitePoolOptions::new().max_connections(1).connect("sqlite::memory:").await?; + sqlx::query( + "CREATE TABLE files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT, + user_name TEXT, + hash TEXT NOT NULL, + filename TEXT NOT NULL, + path TEXT NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + tags TEXT NOT NULL DEFAULT '', + status INTEGER NOT NULL DEFAULT 0, + log TEXT DEFAULT '', + slice_type TEXT DEFAULT '', + kb_id INTEGER DEFAULT NULL, + parse_priority INTEGER NOT NULL DEFAULT 50, + is_public INTEGER NOT NULL DEFAULT 0, + meta TEXT DEFAULT NULL, + summary TEXT DEFAULT NULL, + created_at INTEGER, + updated_at INTEGER + )", + ) + .execute(&pool) + .await?; + sqlx::query( + "CREATE TABLE slices ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL, + content TEXT NOT NULL, + created_at INTEGER, + updated_at INTEGER + )", + ) + .execute(&pool) + .await?; + + create_tables(&pool).await?; + run_schema_migrations(&pool).await?; + create_indexes(&pool).await?; + + let columns = sqlx::query("PRAGMA table_info(files)").fetch_all(&pool).await?; + assert!(columns.iter().any(|row| row.get::("name") == "artifact_id")); + let artifact_index: Option = sqlx::query_scalar( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_files_artifact_id'", + ) + .fetch_optional(&pool) + .await?; + assert_eq!(artifact_index.as_deref(), Some("idx_files_artifact_id")); + Ok(()) + } + + #[tokio::test] + async fn parse_run_migration_upgrades_legacy_schema_idempotently() -> anyhow::Result<()> { + let pool = SqlitePoolOptions::new().max_connections(1).connect("sqlite::memory:").await?; + sqlx::query("CREATE TABLE files (id INTEGER PRIMARY KEY, status INTEGER NOT NULL)").execute(&pool).await?; + sqlx::query( + "CREATE TABLE slices ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL, + content TEXT NOT NULL, + created_at INTEGER, + updated_at INTEGER + )", + ) + .execute(&pool) + .await?; + + run_schema_migrations(&pool).await?; + run_schema_migrations(&pool).await?; + + let file_columns = sqlx::query("PRAGMA table_info(files)").fetch_all(&pool).await?; + assert!(file_columns.iter().any(|row| row.get::("name") == "parse_run_id")); + let slice_columns = sqlx::query("PRAGMA table_info(slices)").fetch_all(&pool).await?; + assert!(slice_columns.iter().any(|row| row.get::("name") == "parse_run_id")); + assert!(slice_columns.iter().any(|row| row.get::("name") == "ordinal")); + + sqlx::query("INSERT INTO slices(file_id, content, parse_run_id, ordinal) VALUES (1, 'a', 'run', 0)") + .execute(&pool) + .await?; + let duplicate = + sqlx::query("INSERT INTO slices(file_id, content, parse_run_id, ordinal) VALUES (1, 'b', 'run', 0)") + .execute(&pool) + .await; + assert!(duplicate.is_err()); + let migration_count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM schema_migrations WHERE version = 1").fetch_one(&pool).await?; + assert_eq!(migration_count, 1); + let artifact_migration_count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM schema_migrations WHERE version = 2").fetch_one(&pool).await?; + assert_eq!(artifact_migration_count, 1); + let artifact_column = sqlx::query("PRAGMA table_info(files)").fetch_all(&pool).await?; + assert!(artifact_column.iter().any(|row| row.get::("name") == "artifact_id")); + assert!(artifact_column.iter().any(|row| row.get::("name") == "summary")); + Ok(()) + } +} diff --git a/src/export.rs b/src/export.rs new file mode 100644 index 0000000..4916b0e --- /dev/null +++ b/src/export.rs @@ -0,0 +1,1800 @@ +use std::{collections::HashMap, path::Path, sync::Arc}; + +use anyhow::Context; +use arrow_array::{ + ArrayRef, BooleanArray, Int64Array, RecordBatch, StringArray, + builder::{FixedSizeListBuilder, Float32Builder}, +}; +use arrow_schema::{DataType, Schema as ArrowSchema}; +use futures::stream::StreamExt; +use log::{debug, info, warn}; +use serde::{Deserialize, Serialize}; +use sqlx::{Encode, QueryBuilder, Row, Sqlite, SqlitePool, Type, sqlite::SqlitePoolOptions}; +use tantivy::{ + TantivyDocument, Term, + collector::TopDocs, + doc, + query::{BooleanQuery, Occur, Query, TermQuery}, + schema::{IndexRecordOption, Value as _}, +}; +use utoipa::ToSchema; + +use crate::{ + api::{backfill_missing_image_meta_for_files, collect_image_raw_paths_for_files}, + config, + search::tantivy_engine, +}; + +const EXPORT_MANIFEST_FILENAME: &str = "manifest.json"; +const EXPORT_DB_FILENAME: &str = "app.sqlite"; + +// Type aliases for complex SQL row types used in bulk export operations +#[allow(clippy::type_complexity)] +type KbRow = (i64, String, String, String, String, String, Option, i32, i32, i64, i64); +#[allow(clippy::type_complexity)] +type FileRow = ( + i64, + String, + String, + String, + String, + String, + i64, + String, + i32, + String, + String, + Option, + i32, + i32, + Option, + Option, + i64, + i64, +); +#[allow(clippy::type_complexity)] +type EntityRow = (i64, String, String, Option, Option>, Option, Option, i32, i64, i64); +#[allow(clippy::type_complexity)] +type RelationRow = (i64, i64, i64, String, Option, Option, Option, i64); +#[allow(clippy::type_complexity)] +type MentionRow = (i64, i64, i64, Option, Option, Option, i64); +#[allow(clippy::type_complexity)] +type GraphSnapshotRow = (i64, Option, Vec, Option, Option, Option, i64); + +/// Export manifest metadata +#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] +pub struct ExportManifest { + pub version: String, + pub export_type: String, + pub kb_ids: Vec, + pub kb_names: Vec, + pub exported_at: String, + pub file_count: usize, + pub slice_count: usize, + pub node_count: usize, + pub edge_count: usize, + pub mention_count: usize, + pub snapshot_count: usize, + pub tantivy_doc_count: usize, + pub tantivy_full_doc_count: usize, + pub lancedb_row_count: usize, +} + +impl Default for ExportManifest { + fn default() -> Self { + Self { + version: "1.0".to_string(), + export_type: "knowledge_base".to_string(), + kb_ids: Vec::new(), + kb_names: Vec::new(), + exported_at: String::new(), + file_count: 0, + slice_count: 0, + node_count: 0, + edge_count: 0, + mention_count: 0, + snapshot_count: 0, + tantivy_doc_count: 0, + tantivy_full_doc_count: 0, + lancedb_row_count: 0, + } + } +} + +/// Export multiple knowledge bases (and optionally their children) to a self-contained directory. +/// +/// The exported directory can be used as a standalone `HTKNOW_DATA_DIR`: +/// ```bash +/// HTKNOW_DATA_DIR=/path/to/export_dir cargo run +/// ``` +pub async fn export_knowledge_bases( + pool: &SqlitePool, src_kb_ids: &[i64], include_children: bool, +) -> anyhow::Result { + let total_start = std::time::Instant::now(); + let cfg = config::get(); + + if src_kb_ids.is_empty() { + anyhow::bail!("No knowledge base IDs provided for export"); + } + + // 1. Collect all KB IDs (including children if requested) + let step_start = std::time::Instant::now(); + let target_kb_ids = collect_kb_ids(pool, src_kb_ids, include_children).await?; + if target_kb_ids.is_empty() { + anyhow::bail!("No knowledge bases found for export"); + } + info!("Exporting knowledge bases: {:?}", target_kb_ids); + + // Collect ancestor KB IDs to preserve hierarchy + let ancestor_kb_ids = collect_ancestor_kb_ids(pool, &target_kb_ids).await?; + if !ancestor_kb_ids.is_empty() { + info!("Including ancestor knowledge bases: {:?}", ancestor_kb_ids); + } + + // Merge target and ancestor IDs for knowledge_bases table export + let mut all_kb_ids = target_kb_ids.clone(); + all_kb_ids.extend(&ancestor_kb_ids); + all_kb_ids.sort_unstable(); + all_kb_ids.dedup(); + + // Get KB names for manifest (only target KBs) + let kb_names = fetch_kb_names(pool, &target_kb_ids).await?; + info!("[step] Collect KB IDs and names: {}ms", step_start.elapsed().as_millis()); + + // 2. Create export directory + let step_start = std::time::Instant::now(); + let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S"); + let dir_name = if src_kb_ids.len() == 1 { + format!("kb_{}_{}", src_kb_ids[0], timestamp) + } else { + let ids_str = src_kb_ids.iter().map(|id| id.to_string()).collect::>().join("_"); + format!("kb_batch_{}_{}", ids_str, timestamp) + }; + let export_dir = + Path::new(&cfg.storage.files_path).parent().unwrap_or(Path::new("data")).join("exports").join(dir_name); + tokio::fs::create_dir_all(&export_dir).await?; + info!("Export directory: {}", export_dir.display()); + + let export_dir_str = export_dir.to_string_lossy().to_string(); + + // Create subdirectories + let files_dir = export_dir.join("files"); + let pdfs_dir = export_dir.join("pdfs"); + let images_dir = export_dir.join("images"); + let contents_dir = export_dir.join("contents"); + let slice_contents_dir = export_dir.join("slice_contents"); + let tantivy_dir = export_dir.join("tantivy_index"); + let tantivy_full_dir = export_dir.join("tantivy_full_index"); + let lancedb_dir = export_dir.join("lancedb_data"); + + tokio::fs::create_dir_all(&files_dir).await?; + tokio::fs::create_dir_all(&pdfs_dir).await?; + tokio::fs::create_dir_all(&images_dir).await?; + tokio::fs::create_dir_all(&contents_dir).await?; + tokio::fs::create_dir_all(&slice_contents_dir).await?; + tokio::fs::create_dir_all(&tantivy_dir).await?; + tokio::fs::create_dir_all(&tantivy_full_dir).await?; + tokio::fs::create_dir_all(&lancedb_dir).await?; + info!("[step] Create directories: {}ms", step_start.elapsed().as_millis()); + + // 3. Export SQLite data + let step_start = std::time::Instant::now(); + let db_path = export_dir.join(EXPORT_DB_FILENAME); + let export_pool = create_export_db_pool(&db_path).await?; + init_export_schema(&export_pool).await?; + + // Disable foreign keys for faster parallel inserts (will re-enable before close) + sqlx::query("PRAGMA foreign_keys = OFF").execute(&export_pool).await?; + + let export_file_ids = collect_file_ids_for_kbs(pool, &target_kb_ids).await?; + let meta_backfilled = backfill_missing_image_meta_for_files(pool, &export_file_ids, "export_regex").await?; + if meta_backfilled > 0 { + info!("Backfilled image meta for {} files before export", meta_backfilled); + } + + let file_ids = + export_sqlite_data(pool, &export_pool, &target_kb_ids, &all_kb_ids, Some(&slice_contents_dir)).await?; + info!("Exported {} files to SQLite in {}ms", file_ids.len(), step_start.elapsed().as_millis()); + + // Re-enable foreign keys and close export pool before writing manifest + sqlx::query("PRAGMA foreign_keys = ON").execute(&export_pool).await?; + export_pool.close().await; + + // 4. Parallel: copy files, export Tantivy, export LanceDB, and count stats + let step_start = std::time::Instant::now(); + + let file_ids_for_copy = file_ids.clone(); + let file_ids_for_count = file_ids.clone(); + let target_kb_ids_for_tantivy = target_kb_ids.clone(); + let target_kb_ids_for_tantivy_full = target_kb_ids.clone(); + let target_kb_ids_for_lancedb = target_kb_ids.clone(); + let target_kb_ids_for_stats = target_kb_ids.clone(); + let pool_clone = pool.clone(); + + // 提前取出各 future 需要的路径（owned），避免 async move 各自去移动同一个 Arc 的字段 + let tantivy_index_path = cfg.search.tantivy_index_path.clone(); + let tantivy_full_index_path = cfg.search.tantivy_full_index_path.clone(); + let lancedb_path = cfg.storage.lancedb_path.clone(); + + let copy_files_future = async move { + let s = std::time::Instant::now(); + let result = + copy_files(&file_ids_for_copy, &files_dir, &pdfs_dir, &images_dir, &contents_dir, &pool_clone).await; + info!("[step] Copy files: {}ms", s.elapsed().as_millis()); + result + }; + + let tantivy_future = async move { + let s = std::time::Instant::now(); + let count = + export_tantivy_index(&tantivy_index_path, &tantivy_dir.to_string_lossy(), &target_kb_ids_for_tantivy) + .await + .unwrap_or_else(|e| { + warn!("Failed to export Tantivy slice index: {}", e); + 0 + }); + info!("[step] Export Tantivy slice index: {}ms", s.elapsed().as_millis()); + count + }; + + let tantivy_full_future = async move { + let s = std::time::Instant::now(); + let count = export_tantivy_index( + &tantivy_full_index_path, + &tantivy_full_dir.to_string_lossy(), + &target_kb_ids_for_tantivy_full, + ) + .await + .unwrap_or_else(|e| { + warn!("Failed to export Tantivy full index: {}", e); + 0 + }); + info!("[step] Export Tantivy full index: {}ms", s.elapsed().as_millis()); + count + }; + + let lancedb_future = async move { + let s = std::time::Instant::now(); + let count = export_lancedb(&lancedb_path, &lancedb_dir.to_string_lossy(), &target_kb_ids_for_lancedb) + .await + .unwrap_or_else(|e| { + warn!("Failed to export LanceDB: {}", e); + 0 + }); + info!("[step] Export LanceDB: {}ms", s.elapsed().as_millis()); + count + }; + + let stats_future = async move { + let s = std::time::Instant::now(); + let slice_count = count_slices(pool, &file_ids_for_count).await.unwrap_or(0); + let (node_count, edge_count, mention_count, snapshot_count) = + count_graph_data(pool, &target_kb_ids_for_stats).await.unwrap_or((0, 0, 0, 0)); + info!("[step] Count stats: {}ms", s.elapsed().as_millis()); + (slice_count, node_count, edge_count, mention_count, snapshot_count) + }; + + let (copy_result, tantivy_doc_count, tantivy_full_doc_count, lancedb_row_count, stats) = + tokio::join!(copy_files_future, tantivy_future, tantivy_full_future, lancedb_future, stats_future,); + + copy_result?; + let (slice_count, node_count, edge_count, mention_count, snapshot_count) = stats; + info!("[step] Parallel export (files + tantivy + lancedb + stats) total: {}ms", step_start.elapsed().as_millis()); + + // 5. Write manifest + let step_start = std::time::Instant::now(); + let file_count = file_ids.len(); + + let manifest = ExportManifest { + version: "1.0".to_string(), + export_type: "knowledge_base".to_string(), + kb_ids: target_kb_ids.clone(), + kb_names, + exported_at: chrono::Utc::now().to_rfc3339(), + file_count, + slice_count, + node_count, + edge_count, + mention_count, + snapshot_count, + tantivy_doc_count, + tantivy_full_doc_count, + lancedb_row_count, + }; + + let manifest_path = export_dir.join(EXPORT_MANIFEST_FILENAME); + let manifest_json = serde_json::to_string_pretty(&manifest)?; + tokio::fs::write(&manifest_path, manifest_json).await?; + info!("[step] Write manifest: {}ms", step_start.elapsed().as_millis()); + + info!( + "Export completed in {}ms: {} files, {} slices, {} tantivy docs, {} lancedb rows to {}", + total_start.elapsed().as_millis(), + file_count, + slice_count, + tantivy_doc_count, + lancedb_row_count, + export_dir.display() + ); + + Ok(export_dir_str) +} + +async fn collect_kb_ids(pool: &SqlitePool, root_kb_ids: &[i64], include_children: bool) -> anyhow::Result> { + if root_kb_ids.is_empty() { + return Ok(Vec::new()); + } + + if include_children { + let mut all_ids: Vec = Vec::new(); + for root_id in root_kb_ids { + let ids: Vec = sqlx::query_scalar( + r#" + WITH RECURSIVE descendants AS ( + SELECT id FROM knowledge_bases WHERE id = ? + UNION ALL + SELECT kb.id FROM knowledge_bases kb + INNER JOIN descendants d ON kb.parent_id = d.id + ) + SELECT id FROM descendants ORDER BY id + "#, + ) + .bind(root_id) + .fetch_all(pool) + .await?; + all_ids.extend(ids); + } + all_ids.sort_unstable(); + all_ids.dedup(); + Ok(all_ids) + } else { + let mut ids = root_kb_ids.to_vec(); + ids.sort_unstable(); + ids.dedup(); + Ok(ids) + } +} + +/// Collect all ancestor KB IDs for the given KB IDs to preserve hierarchy. +async fn collect_ancestor_kb_ids(pool: &SqlitePool, kb_ids: &[i64]) -> anyhow::Result> { + if kb_ids.is_empty() { + return Ok(Vec::new()); + } + + let mut ancestors = Vec::new(); + let mut seen: std::collections::HashSet = kb_ids.iter().copied().collect(); + let mut current_batch: Vec = kb_ids.to_vec(); + + while !current_batch.is_empty() { + let mut next_batch = Vec::new(); + for chunk in current_batch.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT DISTINCT parent_id FROM knowledge_bases WHERE parent_id IS NOT NULL AND id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let parent_ids: Vec> = qb.build_query_scalar().fetch_all(pool).await?; + for pid in parent_ids.into_iter().flatten() { + if seen.insert(pid) { + ancestors.push(pid); + next_batch.push(pid); + } + } + } + current_batch = next_batch; + } + + ancestors.sort_unstable(); + Ok(ancestors) +} + +async fn fetch_kb_names(pool: &SqlitePool, kb_ids: &[i64]) -> anyhow::Result> { + if kb_ids.is_empty() { + return Ok(Vec::new()); + } + let mut names = Vec::new(); + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new("SELECT name FROM knowledge_bases WHERE id IN ("); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(") ORDER BY id"); + let chunk_names: Vec = qb.build_query_scalar().fetch_all(pool).await?; + names.extend(chunk_names); + } + Ok(names) +} + +async fn collect_file_ids_for_kbs(pool: &SqlitePool, kb_ids: &[i64]) -> anyhow::Result> { + if kb_ids.is_empty() { + return Ok(Vec::new()); + } + + let mut file_ids = Vec::new(); + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new("SELECT id FROM files WHERE kb_id IN ("); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let ids: Vec = qb.build_query_scalar().fetch_all(pool).await?; + file_ids.extend(ids); + } + file_ids.sort_unstable(); + file_ids.dedup(); + Ok(file_ids) +} + +async fn create_export_db_pool(db_path: &Path) -> anyhow::Result { + let db_url = format!("sqlite://{}", db_path.display()); + let connect_options = db_url + .parse::()? + .create_if_missing(true) + .foreign_keys(true) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) + .synchronous(sqlx::sqlite::SqliteSynchronous::Normal); + // Single connection: SQLite write is serial anyway; multiple connections just create lock contention. + let pool = SqlitePoolOptions::new().max_connections(1).connect_with(connect_options).await?; + Ok(pool) +} + +async fn init_export_schema(pool: &SqlitePool) -> anyhow::Result<()> { + let init_sql = include_str!("init.sql"); + for (idx, sql) in init_sql.split(';').enumerate() { + let sql = sql.trim(); + if sql.is_empty() { + continue; + } + sqlx::query(sql).execute(pool).await.with_context(|| format!("init.sql statement {}", idx + 1))?; + } + Ok(()) +} + +/// Export SQLite data. Returns the list of exported file IDs. +/// `target_kb_ids` are used for data export; `all_kb_ids` includes ancestors for hierarchy preservation. +async fn export_sqlite_data( + src_pool: &SqlitePool, dst_pool: &SqlitePool, target_kb_ids: &[i64], all_kb_ids: &[i64], + slice_contents_dir: Option<&Path>, +) -> anyhow::Result> { + let step_start = std::time::Instant::now(); + + // Export knowledge_bases (including ancestors to preserve hierarchy) + do_export_knowledge_bases(src_pool, dst_pool, all_kb_ids).await.context("export knowledge bases")?; + info!(" [sqlite] knowledge_bases: {}ms", step_start.elapsed().as_millis()); + + // Export files (with path rewritten to relative) + let s = std::time::Instant::now(); + let file_ids = export_files(src_pool, dst_pool, target_kb_ids).await.context("export files")?; + info!(" [sqlite] files ({} ids): {}ms", file_ids.len(), s.elapsed().as_millis()); + + if file_ids.is_empty() { + // No files, export graph data by kb_id only + let s = std::time::Instant::now(); + export_graph_nodes_by_kb_id(src_pool, dst_pool, target_kb_ids).await?; + info!(" [sqlite] graph_nodes (by kb_id only): {}ms", s.elapsed().as_millis()); + + let s = std::time::Instant::now(); + export_graph_edges(src_pool, dst_pool).await?; + info!(" [sqlite] graph_edges: {}ms", s.elapsed().as_millis()); + + let s = std::time::Instant::now(); + export_entity_mentions(src_pool, dst_pool).await?; + info!(" [sqlite] entity_mentions: {}ms", s.elapsed().as_millis()); + + let s = std::time::Instant::now(); + export_graph_snapshots(src_pool, dst_pool, target_kb_ids).await?; + info!(" [sqlite] graph_snapshots: {}ms", s.elapsed().as_millis()); + + return Ok(file_ids); + } + + // 共享 artifact 导出必须先物化切片并生成 ID 映射，后续位置与 mention 再消费映射。 + export_slices(src_pool, dst_pool, &file_ids, slice_contents_dir).await.context("export materialized slices")?; + export_slice_positions(src_pool, dst_pool).await.context("export materialized slice positions")?; + export_graph_nodes(src_pool, dst_pool, target_kb_ids, &file_ids).await.context("export graph nodes")?; + export_graph_edges(src_pool, dst_pool).await.context("export graph edges")?; + export_entity_mentions(src_pool, dst_pool).await.context("export remapped entity mentions")?; + export_graph_snapshots(src_pool, dst_pool, target_kb_ids).await.context("export graph snapshots")?; + + info!(" [sqlite] all tables total: {}ms", step_start.elapsed().as_millis()); + Ok(file_ids) +} + +async fn do_export_knowledge_bases(src_pool: &SqlitePool, dst_pool: &SqlitePool, kb_ids: &[i64]) -> anyhow::Result<()> { + if kb_ids.is_empty() { + return Ok(()); + } + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT id, user_id, user_name, name, description, kb_type, parent_id, is_public, parse_priority, created_at, updated_at \ + FROM knowledge_bases WHERE id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let debug_sql = qb.sql().to_string(); + let rows = qb.build().fetch_all(src_pool).await.with_context(|| debug_sql)?; + if rows.is_empty() { + continue; + } + + let values: Vec = rows + .iter() + .map(|row| { + ( + row.get::("id"), + row.get::("user_id"), + row.get::("user_name"), + row.get::("name"), + row.get::("description"), + row.get::("kb_type"), + row.get::, _>("parent_id"), + row.get::("is_public"), + row.get::("parse_priority"), + row.get::("created_at"), + row.get::("updated_at"), + ) + }) + .collect(); + + batch_insert_rows( + dst_pool, + "knowledge_bases", + &[ + "id", + "user_id", + "user_name", + "name", + "description", + "kb_type", + "parent_id", + "is_public", + "parse_priority", + "created_at", + "updated_at", + ], + &values, + |b, (id, uid, uname, name, desc, kb_type, pid, is_pub, prio, cat, uat)| { + b.push_bind(id) + .push_bind(uid) + .push_bind(uname) + .push_bind(name) + .push_bind(desc) + .push_bind(kb_type) + .push_bind(pid) + .push_bind(is_pub) + .push_bind(prio) + .push_bind(cat) + .push_bind(uat); + }, + false, + ) + .await?; + } + Ok(()) +} + +async fn export_files(src_pool: &SqlitePool, dst_pool: &SqlitePool, kb_ids: &[i64]) -> anyhow::Result> { + if kb_ids.is_empty() { + return Ok(Vec::new()); + } + let cfg = config::get(); + let files_path_prefix = format!("{}/", cfg.storage.files_path); + + let mut file_ids = Vec::new(); + + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT id, user_id, user_name, hash, filename, path, size, tags, status, log, slice_type, kb_id, parse_priority, is_public, meta, summary, created_at, updated_at FROM files WHERE kb_id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + + let rows = qb.build().fetch_all(src_pool).await?; + if rows.is_empty() { + continue; + } + + let values: Vec = rows + .iter() + .map(|row| { + let id: i64 = row.get("id"); + let path: String = row.get("path"); + let relative_path = if path.starts_with(&files_path_prefix) { + format!("data/files/{}", &path[files_path_prefix.len()..]) + } else if let Some(filename) = Path::new(&path).file_name() { + format!("data/files/{}", filename.to_string_lossy()) + } else { + path.clone() + }; + file_ids.push(id); + ( + id, + row.get::("user_id"), + row.get::("user_name"), + row.get::("hash"), + row.get::("filename"), + relative_path, + row.get::("size"), + row.get::("tags"), + row.get::("status"), + row.get::("log"), + row.get::("slice_type"), + row.get::, _>("kb_id"), + row.get::("parse_priority"), + row.get::("is_public"), + row.get::, _>("meta"), + row.get::, _>("summary"), + row.get::("created_at"), + row.get::("updated_at"), + ) + }) + .collect(); + + batch_insert_rows( + dst_pool, + "files", + &[ + "id", + "user_id", + "user_name", + "hash", + "filename", + "path", + "size", + "tags", + "status", + "log", + "slice_type", + "kb_id", + "parse_priority", + "is_public", + "meta", + "summary", + "created_at", + "updated_at", + ], + &values, + |b, + ( + id, + uid, + uname, + hash, + filename, + path, + size, + tags, + status, + log, + stype, + kb_id, + prio, + is_pub, + meta, + summary, + cat, + uat, + )| { + b.push_bind(id) + .push_bind(uid) + .push_bind(uname) + .push_bind(hash) + .push_bind(filename) + .push_bind(path) + .push_bind(size) + .push_bind(tags) + .push_bind(status) + .push_bind(log) + .push_bind(stype) + .push_bind(kb_id) + .push_bind(prio) + .push_bind(is_pub) + .push_bind(meta) + .push_bind(summary) + .push_bind(cat) + .push_bind(uat); + }, + false, + ) + .await?; + } + + Ok(file_ids) +} + +const SQLITE_BATCH_SIZE: usize = 900; +/// Max rows per INSERT to stay under SQLite's 999 variable limit. +/// files table has 18 columns => 999/18 = 55; use 50 to be safe. +const INSERT_BATCH_SIZE: usize = 50; + +struct RowBinder<'qb, 'args> { + qb: &'qb mut QueryBuilder<'args, Sqlite>, + first: bool, +} + +impl<'qb, 'args> RowBinder<'qb, 'args> { + fn push_bind(&mut self, value: T) -> &mut Self + where + T: 'args + Send + Encode<'args, Sqlite> + Type, + { + if !self.first { + self.qb.push(", "); + } + self.first = false; + self.qb.push_bind(value); + self + } +} + +async fn batch_insert_rows( + dst_pool: &SqlitePool, table: &str, columns: &[&str], rows: &[Src], bind_row: impl Fn(&mut RowBinder<'_, '_>, Src), + insert_or_ignore: bool, +) -> Result<(), sqlx::Error> +where + Src: Clone, +{ + if rows.is_empty() { + return Ok(()); + } + let col_list = columns.join(", "); + let prefix = if insert_or_ignore { + format!("INSERT OR IGNORE INTO {table} ({col_list}) VALUES ") + } else { + format!("INSERT INTO {table} ({col_list}) VALUES ") + }; + for chunk in rows.chunks(INSERT_BATCH_SIZE) { + let mut qb = QueryBuilder::::new(&prefix); + for (i, row) in chunk.iter().enumerate() { + if i > 0 { + qb.push(", "); + } + qb.push("("); + bind_row(&mut RowBinder { qb: &mut qb, first: true }, row.clone()); + qb.push(")"); + } + let debug_sql = qb.sql().to_string(); + qb.build().execute(dst_pool).await.map_err(|err| sqlx::Error::Protocol(format!("{debug_sql}: {err}")))?; + } + Ok(()) +} + +async fn export_slices( + src_pool: &SqlitePool, dst_pool: &SqlitePool, file_ids: &[i64], slice_contents_dir: Option<&Path>, +) -> anyhow::Result<()> { + if file_ids.is_empty() { + return Ok(()); + } + sqlx::query( + "CREATE TEMP TABLE IF NOT EXISTS export_slice_id_map ( + target_file_id INTEGER NOT NULL, + source_slice_id INTEGER NOT NULL, + export_slice_id INTEGER NOT NULL, + PRIMARY KEY(target_file_id, source_slice_id) + )", + ) + .execute(dst_pool) + .await?; + for chunk in file_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT f.id AS target_file_id, s.id AS source_slice_id, s.file_id AS source_file_id, s.created_at, s.updated_at \ + FROM files f LEFT JOIN parse_artifacts pa ON pa.id = f.artifact_id \ + JOIN slices s ON s.file_id = COALESCE(pa.source_file_id, f.id) WHERE f.id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let rows = qb.build().fetch_all(src_pool).await?; + if rows.is_empty() { + continue; + } + + let mut exported_contents: std::collections::HashMap> = + std::collections::HashMap::new(); + let mut source_contents = std::collections::HashMap::new(); + // Read external slice content before opening the destination write transaction. + for row in &rows { + let source_file_id: i64 = row.get("source_file_id"); + if let std::collections::hash_map::Entry::Vacant(e) = source_contents.entry(source_file_id) { + e.insert(crate::slice_content::read_all(source_file_id).await?); + } + } + + let mut tx = dst_pool.begin().await?; + for row in rows { + let target_file_id: i64 = row.get("target_file_id"); + let source_slice_id: i64 = row.get("source_slice_id"); + let source_file_id: i64 = row.get("source_file_id"); + let export_slice_id: i64 = + sqlx::query_scalar("INSERT INTO slices(file_id, created_at, updated_at) VALUES (?, ?, ?) RETURNING id") + .bind(target_file_id) + .bind(row.get::("created_at")) + .bind(row.get::("updated_at")) + .fetch_one(&mut *tx) + .await?; + let content = + source_contents.get(&source_file_id).and_then(|v| v.get(&source_slice_id)).cloned().unwrap_or_default(); + exported_contents.entry(target_file_id).or_default().push((export_slice_id, content)); + sqlx::query( + "INSERT INTO export_slice_id_map(target_file_id, source_slice_id, export_slice_id) VALUES (?, ?, ?)", + ) + .bind(target_file_id) + .bind(source_slice_id) + .bind(export_slice_id) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + if let Some(dir) = slice_contents_dir { + for (file_id, contents) in exported_contents { + let content_map: std::collections::HashMap = contents.into_iter().collect(); + tokio::fs::write(dir.join(format!("{}.json", file_id)), serde_json::to_vec(&content_map)?).await?; + } + } + } + Ok(()) +} + +async fn export_slice_positions(src_pool: &SqlitePool, dst_pool: &SqlitePool) -> anyhow::Result<()> { + let mappings: Vec<(i64, i64)> = + sqlx::query_as("SELECT source_slice_id, export_slice_id FROM export_slice_id_map").fetch_all(dst_pool).await?; + for (source_slice_id, export_slice_id) in mappings { + let rows = sqlx::query( + "SELECT page_idx, x1, y1, x2, y2, sheet_name, row_num, created_at \ + FROM slice_positions WHERE slice_id = ? ORDER BY id", + ) + .bind(source_slice_id) + .fetch_all(src_pool) + .await?; + for row in rows { + sqlx::query( + "INSERT INTO slice_positions(slice_id, page_idx, x1, y1, x2, y2, sheet_name, row_num, created_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(export_slice_id) + .bind(row.get::("page_idx")) + .bind(row.get::("x1")) + .bind(row.get::("y1")) + .bind(row.get::("x2")) + .bind(row.get::("y2")) + .bind(row.get::, _>("sheet_name")) + .bind(row.get::, _>("row_num")) + .bind(row.get::("created_at")) + .execute(dst_pool) + .await?; + } + } + Ok(()) +} + +async fn export_graph_nodes( + src_pool: &SqlitePool, dst_pool: &SqlitePool, kb_ids: &[i64], file_ids: &[i64], +) -> anyhow::Result<()> { + // First export by kb_id (this is the primary source) + export_graph_nodes_by_kb_id(src_pool, dst_pool, kb_ids).await?; + + // Then export by file_id with INSERT OR IGNORE to avoid duplicates + if !file_ids.is_empty() { + for chunk in file_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT id, name, entity_type, properties, embedding, file_id, kb_id, is_public, created_at, updated_at \ + FROM graph_nodes WHERE file_id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let rows = qb.build().fetch_all(src_pool).await?; + if rows.is_empty() { + continue; + } + + let values: Vec = rows + .iter() + .map(|row| { + ( + row.get::("id"), + row.get::("name"), + row.get::("entity_type"), + row.get::, _>("properties"), + row.get::>, _>("embedding"), + row.get::, _>("file_id"), + row.get::, _>("kb_id"), + row.get::("is_public"), + row.get::("created_at"), + row.get::("updated_at"), + ) + }) + .collect(); + + batch_insert_rows( + dst_pool, + "graph_nodes", + &[ + "id", "name", "entity_type", "properties", "embedding", "file_id", "kb_id", "is_public", + "created_at", "updated_at", + ], + &values, + |b, (id, name, entity_type, properties, embedding, file_id, kb_id, is_public, created_at, updated_at)| { + b.push_bind(id) + .push_bind(name) + .push_bind(entity_type) + .push_bind(properties) + .push_bind(embedding) + .push_bind(file_id) + .push_bind(kb_id) + .push_bind(is_public) + .push_bind(created_at) + .push_bind(updated_at); + }, + true, + ) + .await?; + } + } + Ok(()) +} + +async fn export_graph_nodes_by_kb_id( + src_pool: &SqlitePool, dst_pool: &SqlitePool, kb_ids: &[i64], +) -> anyhow::Result<()> { + if kb_ids.is_empty() { + return Ok(()); + } + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT id, name, entity_type, properties, embedding, file_id, kb_id, is_public, created_at, updated_at \ + FROM graph_nodes WHERE kb_id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let rows = qb.build().fetch_all(src_pool).await?; + if rows.is_empty() { + continue; + } + + let values: Vec = rows + .iter() + .map(|row| { + ( + row.get::("id"), + row.get::("name"), + row.get::("entity_type"), + row.get::, _>("properties"), + row.get::>, _>("embedding"), + row.get::, _>("file_id"), + row.get::, _>("kb_id"), + row.get::("is_public"), + row.get::("created_at"), + row.get::("updated_at"), + ) + }) + .collect(); + + batch_insert_rows( + dst_pool, + "graph_nodes", + &[ + "id", + "name", + "entity_type", + "properties", + "embedding", + "file_id", + "kb_id", + "is_public", + "created_at", + "updated_at", + ], + &values, + |b, (id, name, entity_type, properties, embedding, file_id, kb_id, is_public, created_at, updated_at)| { + b.push_bind(id) + .push_bind(name) + .push_bind(entity_type) + .push_bind(properties) + .push_bind(embedding) + .push_bind(file_id) + .push_bind(kb_id) + .push_bind(is_public) + .push_bind(created_at) + .push_bind(updated_at); + }, + false, + ) + .await?; + } + Ok(()) +} + +async fn export_graph_edges(src_pool: &SqlitePool, dst_pool: &SqlitePool) -> anyhow::Result<()> { + // Get exported node IDs from dst_pool + let node_ids: Vec = sqlx::query_scalar("SELECT id FROM graph_nodes").fetch_all(dst_pool).await?; + if node_ids.is_empty() { + return Ok(()); + } + let node_set: std::collections::HashSet = node_ids.iter().copied().collect(); + + for chunk in node_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT id, source_node_id, target_node_id, relation_type, properties, weight, file_id, created_at \ + FROM graph_edges WHERE source_node_id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let rows = qb.build().fetch_all(src_pool).await?; + if rows.is_empty() { + continue; + } + + let values: Vec = rows + .iter() + .filter_map(|row| { + let target_id: i64 = row.get("target_node_id"); + if node_set.contains(&target_id) { + Some(( + row.get::("id"), + row.get::("source_node_id"), + target_id, + row.get::("relation_type"), + row.get::, _>("properties"), + row.get::, _>("weight"), + row.get::, _>("file_id"), + row.get::("created_at"), + )) + } else { + None + } + }) + .collect(); + + if values.is_empty() { + continue; + } + + batch_insert_rows( + dst_pool, + "graph_edges", + &[ + "id", + "source_node_id", + "target_node_id", + "relation_type", + "properties", + "weight", + "file_id", + "created_at", + ], + &values, + |b, (id, source, target, relation_type, properties, weight, file_id, created_at)| { + b.push_bind(id) + .push_bind(source) + .push_bind(target) + .push_bind(relation_type) + .push_bind(properties) + .push_bind(weight) + .push_bind(file_id) + .push_bind(created_at); + }, + false, + ) + .await?; + } + Ok(()) +} + +async fn export_entity_mentions(src_pool: &SqlitePool, dst_pool: &SqlitePool) -> anyhow::Result<()> { + // Get exported node IDs and materialized slice mappings from dst_pool. + sqlx::query( + "CREATE TEMP TABLE IF NOT EXISTS export_slice_id_map ( + target_file_id INTEGER NOT NULL, source_slice_id INTEGER NOT NULL, export_slice_id INTEGER NOT NULL, + PRIMARY KEY(target_file_id, source_slice_id) + )", + ) + .execute(dst_pool) + .await?; + let node_ids: Vec = sqlx::query_scalar("SELECT id FROM graph_nodes").fetch_all(dst_pool).await?; + let mappings: Vec<(i64, i64, i64)> = + sqlx::query_as("SELECT target_file_id, source_slice_id, export_slice_id FROM export_slice_id_map") + .fetch_all(dst_pool) + .await?; + if node_ids.is_empty() || mappings.is_empty() { + return Ok(()); + } + let mut slice_map: HashMap> = HashMap::new(); + for (target_file_id, source_slice_id, export_slice_id) in mappings { + slice_map.entry(source_slice_id).or_default().push((target_file_id, export_slice_id)); + } + let node_files: HashMap> = + sqlx::query_as::<_, (i64, Option)>("SELECT id, file_id FROM graph_nodes") + .fetch_all(dst_pool) + .await? + .into_iter() + .collect(); + + for chunk in node_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT id, node_id, slice_id, start_offset, end_offset, context, created_at \ + FROM entity_mentions WHERE node_id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let rows = qb.build().fetch_all(src_pool).await?; + if rows.is_empty() { + continue; + } + + let values: Vec = rows + .iter() + .filter_map(|row| { + let slice_id: i64 = row.get("slice_id"); + let target_file_id = node_files.get(&row.get::("node_id")).copied().flatten(); + let mapped_slice_id = slice_map.get(&slice_id).and_then(|entries| { + target_file_id + .and_then(|file_id| entries.iter().find(|(target, _)| *target == file_id).map(|(_, id)| *id)) + .or_else(|| entries.first().map(|(_, id)| *id)) + }); + mapped_slice_id.map(|mapped_slice_id| { + ( + row.get::("id"), + row.get::("node_id"), + mapped_slice_id, + row.get::, _>("start_offset"), + row.get::, _>("end_offset"), + row.get::, _>("context"), + row.get::("created_at"), + ) + }) + }) + .collect(); + + if values.is_empty() { + continue; + } + + batch_insert_rows( + dst_pool, + "entity_mentions", + &["id", "node_id", "slice_id", "start_offset", "end_offset", "context", "created_at"], + &values, + |b, (id, node_id, slice_id, start_offset, end_offset, context, created_at)| { + b.push_bind(id) + .push_bind(node_id) + .push_bind(slice_id) + .push_bind(start_offset) + .push_bind(end_offset) + .push_bind(context) + .push_bind(created_at); + }, + false, + ) + .await?; + } + Ok(()) +} + +async fn export_graph_snapshots(src_pool: &SqlitePool, dst_pool: &SqlitePool, kb_ids: &[i64]) -> anyhow::Result<()> { + if kb_ids.is_empty() { + return Ok(()); + } + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT id, kb_id, graph_data, node_count, edge_count, version, created_at \ + FROM graph_snapshots WHERE kb_id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let rows = qb.build().fetch_all(src_pool).await?; + if rows.is_empty() { + continue; + } + + let values: Vec = rows + .iter() + .map(|row| { + ( + row.get::("id"), + row.get::, _>("kb_id"), + row.get::, _>("graph_data"), + row.get::, _>("node_count"), + row.get::, _>("edge_count"), + row.get::, _>("version"), + row.get::("created_at"), + ) + }) + .collect(); + + batch_insert_rows( + dst_pool, + "graph_snapshots", + &["id", "kb_id", "graph_data", "node_count", "edge_count", "version", "created_at"], + &values, + |b, (id, kb_id, graph_data, node_count, edge_count, version, created_at)| { + b.push_bind(id) + .push_bind(kb_id) + .push_bind(graph_data) + .push_bind(node_count) + .push_bind(edge_count) + .push_bind(version) + .push_bind(created_at); + }, + false, + ) + .await?; + } + Ok(()) +} + +async fn copy_files( + file_ids: &[i64], files_dir: &Path, pdfs_dir: &Path, images_dir: &Path, contents_dir: &Path, pool: &SqlitePool, +) -> anyhow::Result<()> { + let cfg = config::get(); + + // Get file paths from database + let mut file_paths = Vec::new(); + + for chunk in file_ids.chunks(1000) { + let mut qb = QueryBuilder::::new( + "SELECT f.id, f.path, COALESCE(pa.source_file_id, f.id) AS parse_source_id \ + FROM files f LEFT JOIN parse_artifacts pa ON pa.id = f.artifact_id WHERE f.id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let rows = qb.build().fetch_all(pool).await?; + for row in rows { + let id: i64 = row.get("id"); + let path: String = row.get("path"); + let parse_source_id: i64 = row.get("parse_source_id"); + file_paths.push((id, path, parse_source_id)); + } + } + + let total_files = file_paths.len(); + let copied_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let skipped_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + + // Copy original files concurrently via spawn_blocking + let file_handles: Vec<_> = file_paths + .iter() + .cloned() + .map(|(_file_id, src_path, _parse_source_id)| { + let files_dir = files_dir.to_path_buf(); + let copied = copied_count.clone(); + let skipped = skipped_count.clone(); + tokio::task::spawn_blocking(move || { + let src = Path::new(&src_path); + if !src.exists() { + warn!("Source file not found: {}", src_path); + skipped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return; + } + let filename = src.file_name().unwrap_or(std::ffi::OsStr::new("unknown")); + let dst = files_dir.join(filename); + if let Err(e) = std::fs::copy(src, &dst) { + warn!("Failed to copy file {} to {}: {}", src_path, dst.display(), e); + skipped.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } else { + debug!("Copied file {} to {}", src_path, dst.display()); + copied.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + }) + }) + .collect(); + futures::future::join_all(file_handles).await; + + // Copy PDFs concurrently via spawn_blocking + let pdf_handles: Vec<_> = file_paths + .iter() + .cloned() + .map(|(file_id, _src_path, parse_source_id)| { + let pdfs_dir = pdfs_dir.to_path_buf(); + let cfg_pdf_path = cfg.storage.pdf_path.clone(); + tokio::task::spawn_blocking(move || { + let src_pdf = Path::new(&cfg_pdf_path).join(format!("{}.pdf", parse_source_id)); + if src_pdf.exists() { + let dst_pdf = pdfs_dir.join(format!("{}.pdf", file_id)); + if let Err(e) = std::fs::copy(&src_pdf, &dst_pdf) { + warn!("Failed to copy PDF {} to {}: {}", src_pdf.display(), dst_pdf.display(), e); + } + } + }) + }) + .collect(); + futures::future::join_all(pdf_handles).await; + + // Copy content and PDF content files concurrently via spawn_blocking + let content_handles: Vec<_> = file_paths + .iter() + .cloned() + .map(|(file_id, _src_path, parse_source_id)| { + let contents_dir = contents_dir.to_path_buf(); + let cfg_contents_path = cfg.storage.contents_path.clone(); + let cfg_images_path = cfg.storage.images_path.clone(); + tokio::task::spawn_blocking(move || { + let src_content = Path::new(&cfg_contents_path).join(format!("{}.txt", parse_source_id)); + if src_content.exists() { + let dst_content = contents_dir.join(format!("{}.txt", file_id)); + if let Err(e) = std::fs::copy(&src_content, &dst_content) { + warn!( + "Failed to copy content file {} to {}: {}", + src_content.display(), + dst_content.display(), + e + ); + } + } + let src_pdf_content = Path::new(&cfg_contents_path).join(format!("{}.json", parse_source_id)); + if src_pdf_content.exists() { + let dst_pdf_content = contents_dir.join(format!("{}.json", file_id)); + let result = (|| -> anyhow::Result<()> { + let bytes = std::fs::read(&src_pdf_content)?; + let mut rows: Vec = serde_json::from_slice(&bytes)?; + let images_prefix = format!("{}/", cfg_images_path); + for row in &mut rows { + if let Some(path) = row.img_path.as_mut() + && path.starts_with(&images_prefix) + { + *path = path[images_prefix.len()..].to_string(); + } + } + std::fs::write(&dst_pdf_content, serde_json::to_vec(&rows)?)?; + Ok(()) + })(); + if let Err(e) = result { + warn!( + "Failed to copy PDF content file {} to {}: {}", + src_pdf_content.display(), + dst_pdf_content.display(), + e + ); + } + } + }) + }) + .collect(); + futures::future::join_all(content_handles).await; + + let copied = copied_count.load(std::sync::atomic::Ordering::Relaxed); + let skipped = skipped_count.load(std::sync::atomic::Ordering::Relaxed); + info!("Copied {}/{} original files, skipped {}", copied, total_files, skipped); + + // Copy images referenced by PDF content JSON + let s = std::time::Instant::now(); + copy_images(pool, images_dir, file_ids).await?; + info!(" [files] Copy images: {}ms", s.elapsed().as_millis()); + + Ok(()) +} + +async fn copy_images(pool: &SqlitePool, images_dir: &Path, file_ids: &[i64]) -> anyhow::Result<()> { + if file_ids.is_empty() { + return Ok(()); + } + + let cfg = config::get(); + let data_root = Path::new(&cfg.storage.images_path).parent(); + + let mut effective_ids = file_ids.to_vec(); + for chunk in file_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT DISTINCT pa.source_file_id FROM files f JOIN parse_artifacts pa ON pa.id = f.artifact_id \ + WHERE f.id IN (", + ); + crate::db::push_i64_list(&mut qb, chunk); + qb.push(")"); + effective_ids.extend(qb.build_query_scalar::().fetch_all(pool).await?); + } + effective_ids.sort_unstable(); + effective_ids.dedup(); + let all_img_paths = collect_image_raw_paths_for_files(pool, &effective_ids).await?; + + if all_img_paths.is_empty() { + return Ok(()); + } + + // Copy all images concurrently via spawn_blocking + let img_handles: Vec<_> = all_img_paths + .into_iter() + .map(|trimmed| { + let images_dir = images_dir.to_path_buf(); + let cfg_images_path = cfg.storage.images_path.clone(); + let data_root = data_root.map(|p| p.to_path_buf()); + + tokio::task::spawn_blocking(move || { + let use_data_root = trimmed.contains('/') || trimmed.contains('\\'); + let src = if use_data_root { + let via_images_path = Path::new(&cfg_images_path).join(&trimmed); + if via_images_path.exists() { + via_images_path + } else if let Some(root) = data_root { + root.join(&trimmed) + } else { + via_images_path + } + } else { + Path::new(&cfg_images_path).join(&trimmed) + }; + + if !src.exists() { + warn!("Source image not found: {}", src.display()); + return false; + } + + let dst = if use_data_root { + images_dir.parent().unwrap_or(&images_dir).join(&trimmed) + } else { + images_dir.join(&trimmed) + }; + + if let Some(parent) = dst.parent() { + std::fs::create_dir_all(parent).ok(); + } + + if let Err(e) = std::fs::copy(&src, &dst) { + warn!("Failed to copy image {} to {}: {}", src.display(), dst.display(), e); + false + } else { + debug!("Copied image {} to {}", src.display(), dst.display()); + true + } + }) + }) + .collect(); + + let results = futures::future::join_all(img_handles).await; + let copied = results.iter().filter(|r| r.as_ref().is_ok_and(|v| *v)).count(); + let failed = results.len() - copied; + info!("Copied {} images, {} failed", copied, failed); + + Ok(()) +} + +async fn export_tantivy_index(src_path: &str, dst_path: &str, kb_ids: &[i64]) -> anyhow::Result { + if kb_ids.is_empty() { + return Ok(0); + } + + let src_path = src_path.to_string(); + let dst_path = dst_path.to_string(); + let kb_ids = kb_ids.to_vec(); + + tokio::task::spawn_blocking(move || { + let step_start = std::time::Instant::now(); + let src_index = tantivy::Index::open_in_dir(&src_path) + .with_context(|| format!("Failed to open source Tantivy index: {}", src_path))?; + let src_schema = src_index.schema(); + let kb_id_field = src_schema.get_field("kb_id")?; + + let reader = src_index.reader()?; + let searcher = reader.searcher(); + + // Build a BooleanQuery with TermQuery for each kb_id — directly hits the inverted index + let kb_queries: Vec<(Occur, Box)> = kb_ids + .iter() + .map(|kb_id| { + ( + Occur::Should, + Box::new(TermQuery::new(Term::from_field_i64(kb_id_field, *kb_id), IndexRecordOption::Basic)) + as Box, + ) + }) + .collect(); + let kb_query = BooleanQuery::new(kb_queries); + let doc_limit = 10_000_000; + let top_docs = searcher.search(&kb_query, &TopDocs::with_limit(doc_limit))?; + + if top_docs.len() == doc_limit { + warn!("Tantivy export hit document limit of {}", doc_limit); + } + let scan_ms = step_start.elapsed().as_millis(); + info!(" [tantivy] Matched {} docs (scan {}ms)", top_docs.len(), scan_ms); + + // Create destination index + let s = std::time::Instant::now(); + let (dst_schema, dst_index) = tantivy_engine::init_with_path(&dst_path)?; + let writer_memory = config::get().search.tantivy_memory_mb * 1_000_000; + let mut writer = dst_index.writer(writer_memory)?; + writer.set_merge_policy(Box::new(tantivy::merge_policy::LogMergePolicy::default())); + + let id_field = src_schema.get_field("id")?; + let file_id_field = src_schema.get_field("file_id")?; + let content_field = src_schema.get_field("content")?; + + let dst_id_field = dst_schema.get_field("id")?; + let dst_file_id_field = dst_schema.get_field("file_id")?; + let dst_content_field = dst_schema.get_field("content")?; + let dst_kb_id_field = dst_schema.get_field("kb_id")?; + + let mut count = 0; + for (_, doc_address) in top_docs { + let doc: TantivyDocument = searcher.doc(doc_address)?; + let doc_kb_id = doc.get_first(kb_id_field).and_then(|v| v.as_i64()); + + let id = doc.get_first(id_field).and_then(|v| v.as_i64()).unwrap_or(0); + let file_id = doc.get_first(file_id_field).and_then(|v| v.as_i64()).unwrap_or(0); + let content = doc.get_first(content_field).and_then(|v| v.as_str()).unwrap_or(""); + + let mut new_doc = doc! { + dst_id_field => id, + dst_file_id_field => file_id, + dst_content_field => content, + }; + if let Some(kb_id) = doc_kb_id { + new_doc.add_i64(dst_kb_id_field, kb_id); + } + writer.add_document(new_doc)?; + count += 1; + } + + writer.commit()?; + info!( + "Exported {} Tantivy documents to {} (scan {}ms, write {}ms, total {}ms)", + count, + dst_path, + scan_ms, + s.elapsed().as_millis(), + step_start.elapsed().as_millis() + ); + anyhow::Ok(count) + }) + .await + .map_err(|e| anyhow::anyhow!("Tantivy export task panicked: {}", e))? +} + +async fn export_lancedb(src_path: &str, dst_path: &str, kb_ids: &[i64]) -> anyhow::Result { + use lancedb::query::{ExecutableQuery, QueryBase}; + + if kb_ids.is_empty() { + return Ok(0); + } + + let step_start = std::time::Instant::now(); + + let src_conn = lancedb::connect(src_path).execute().await?; + let src_table = src_conn.open_table("documents").execute().await?; + let schema = src_table.schema().await?; + + let dst_conn = lancedb::connect(dst_path).execute().await?; + + let mut total_count = 0; + // 流式写出：每读到一个 batch 立即写入目标表，避免把全部数据（含向量）累积到内存。 + let mut dst_table: Option = None; + + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let predicate = if chunk.len() == 1 { + format!("kb_id = {}", chunk[0]) + } else { + format!("kb_id IN ({})", chunk.iter().map(|id| id.to_string()).collect::>().join(", ")) + }; + + let s = std::time::Instant::now(); + let mut stream = src_table.query().only_if(&predicate).limit(1_000_000).execute().await?; + + let mut chunk_count = 0; + while let Some(batch_result) = stream.next().await { + let batch = batch_result?; + chunk_count += batch.num_rows(); + total_count += batch.num_rows(); + match &dst_table { + None => { + dst_conn.create_table("documents", batch).execute().await?; + dst_table = Some(dst_conn.open_table("documents").execute().await?); + } + Some(table) => { + table.add(batch).execute().await?; + } + } + } + info!(" [lancedb] Query chunk ({} kb_ids, {} rows): {}ms", chunk.len(), chunk_count, s.elapsed().as_millis()); + } + + if dst_table.is_none() { + // 没有任何数据：创建带正确 schema 的空表 + let empty_batch = create_empty_batch_for_schema(&schema)?; + dst_conn.create_table("documents", empty_batch).execute().await?; + } + + info!("Exported {} LanceDB rows to {} (total {}ms)", total_count, dst_path, step_start.elapsed().as_millis()); + Ok(total_count) +} + +fn create_empty_batch_for_schema(schema: &Arc) -> anyhow::Result { + let mut arrays: Vec = Vec::with_capacity(schema.fields().len()); + + for field in schema.fields() { + let array: ArrayRef = match field.data_type() { + DataType::Int64 => Arc::new(Int64Array::from(Vec::::new())), + DataType::Utf8 => Arc::new(StringArray::from(Vec::::new())), + DataType::Boolean => Arc::new(BooleanArray::from(Vec::::new())), + DataType::FixedSizeList(inner_field, dim) => match inner_field.data_type() { + DataType::Float32 => { + let value_builder = Float32Builder::new(); + let mut list_builder = FixedSizeListBuilder::new(value_builder, *dim); + Arc::new(list_builder.finish()) + } + _ => anyhow::bail!("Unsupported inner type in FixedSizeList: {:?}", inner_field.data_type()), + }, + _ => anyhow::bail!("Unsupported data type for empty batch: {:?}", field.data_type()), + }; + arrays.push(array); + } + + Ok(RecordBatch::try_new(schema.clone(), arrays)?) +} + +async fn count_slices(pool: &SqlitePool, file_ids: &[i64]) -> anyhow::Result { + if file_ids.is_empty() { + return Ok(0); + } + let mut total = 0_i64; + for chunk in file_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new("SELECT COUNT(*) FROM slices WHERE file_id IN ("); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let count: i64 = qb.build_query_scalar().fetch_one(pool).await?; + total += count; + } + Ok(total as usize) +} + +async fn count_graph_data(pool: &SqlitePool, kb_ids: &[i64]) -> anyhow::Result<(usize, usize, usize, usize)> { + if kb_ids.is_empty() { + return Ok((0, 0, 0, 0)); + } + + let mut node_count = 0_i64; + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new("SELECT COUNT(*) FROM graph_nodes WHERE kb_id IN ("); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let cnt: i64 = qb.build_query_scalar().fetch_one(pool).await?; + node_count += cnt; + } + + let mut edge_count = 0_i64; + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT COUNT(*) FROM graph_edges e WHERE EXISTS (SELECT 1 FROM graph_nodes n WHERE n.id = e.source_node_id AND n.kb_id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push("))"); + let cnt: i64 = qb.build_query_scalar().fetch_one(pool).await?; + edge_count += cnt; + } + + let mut mention_count = 0_i64; + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new( + "SELECT COUNT(*) FROM entity_mentions m WHERE EXISTS (SELECT 1 FROM graph_nodes n WHERE n.id = m.node_id AND n.kb_id IN (", + ); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push("))"); + let cnt: i64 = qb.build_query_scalar().fetch_one(pool).await?; + mention_count += cnt; + } + + let mut snapshot_count = 0_i64; + for chunk in kb_ids.chunks(SQLITE_BATCH_SIZE) { + let mut qb = QueryBuilder::::new("SELECT COUNT(*) FROM graph_snapshots WHERE kb_id IN ("); + let mut separated = qb.separated(", "); + for id in chunk { + separated.push_bind(id); + } + qb.push(")"); + let cnt: i64 = qb.build_query_scalar().fetch_one(pool).await?; + snapshot_count += cnt; + } + + Ok((node_count as usize, edge_count as usize, mention_count as usize, snapshot_count as usize)) +} + +#[cfg(test)] +mod tests { + use super::*; + + async fn memory_db() -> anyhow::Result { + let pool = SqlitePoolOptions::new().max_connections(1).connect("sqlite::memory:").await?; + init_export_schema(&pool).await?; + Ok(pool) + } + + #[tokio::test] + async fn shared_artifact_is_materialized_during_export() -> anyhow::Result<()> { + let src = memory_db().await.context("source schema")?; + let dst = memory_db().await.context("destination schema")?; + sqlx::query("INSERT INTO knowledge_bases(id,user_id,user_name,name,kb_type) VALUES(1,'u','n','kb','analysis')") + .execute(&src) + .await + .context("insert kb")?; + for id in [1_i64, 2] { + sqlx::query( + "INSERT INTO files(id,user_id,user_name,hash,filename,path,size,tags,status,log,slice_type,kb_id, + is_public,artifact_id) VALUES(?,'u','n','hash','a.txt','/tmp/a',1,'',1,'','text',1,0,1)", + ) + .bind(id) + .execute(&src) + .await + .context("insert file")?; + } + sqlx::query( + "INSERT INTO parse_artifacts(id,artifact_key,content_hash,slice_type,parser_version,config_hash, + source_file_id,full_content) VALUES(1,'key','hash','text','v1','cfg',1,'shared')", + ) + .execute(&src) + .await + .context("insert artifact")?; + let source_slice: i64 = sqlx::query_scalar("INSERT INTO slices(file_id) VALUES(1) RETURNING id") + .fetch_one(&src) + .await + .context("insert slice")?; + sqlx::query("INSERT INTO slice_positions(slice_id,page_idx,x1,y1,x2,y2) VALUES(?,0,1,2,3,4)") + .bind(source_slice) + .execute(&src) + .await + .context("insert position")?; + let node_id: i64 = sqlx::query_scalar( + "INSERT INTO graph_nodes(name,entity_type,file_id,kb_id) VALUES('e','t',2,1) RETURNING id", + ) + .fetch_one(&src) + .await?; + sqlx::query("INSERT INTO entity_mentions(node_id,slice_id,context) VALUES(?,?,'ctx')") + .bind(node_id) + .bind(source_slice) + .execute(&src) + .await?; + + export_sqlite_data(&src, &dst, &[1], &[1], None).await.context("export sqlite")?; + + let slices: Vec<(i64, i64)> = + sqlx::query_as("SELECT id,file_id FROM slices ORDER BY file_id").fetch_all(&dst).await?; + assert_eq!(slices.len(), 2); + assert_eq!(slices.iter().map(|(_, file_id)| *file_id).collect::>(), vec![1, 2]); + assert_ne!(slices[0].0, slices[1].0); + assert_eq!(sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM slice_positions").fetch_one(&dst).await?, 2); + let mention_file: i64 = + sqlx::query_scalar("SELECT s.file_id FROM entity_mentions m JOIN slices s ON s.id=m.slice_id LIMIT 1") + .fetch_one(&dst) + .await?; + assert_eq!(mention_file, 2); + Ok(()) + } +} diff --git a/src/file_content.rs b/src/file_content.rs new file mode 100644 index 0000000..4eedd5d --- /dev/null +++ b/src/file_content.rs @@ -0,0 +1,62 @@ +use std::path::PathBuf; + +use anyhow::Context; + +use crate::config; + +/// 返回指定 file_id 对应的 content 文件路径。 +fn content_path(file_id: i64) -> PathBuf { + PathBuf::from(&config::get().storage.contents_path).join(format!("{}.txt", file_id)) +} + +/// 读取文件的完整 content。 +/// +/// - 文件存在：返回 `Some(内容)`。 +/// - 文件不存在：返回 `None`（对应数据库中 `content IS NULL` 的语义）。 +/// - 其他 IO 错误：返回 `Err`。 +pub async fn read(file_id: i64) -> anyhow::Result> { + let path = content_path(file_id); + match tokio::fs::read_to_string(&path).await { + Ok(content) => Ok(Some(content)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e).with_context(|| format!("failed to read content file {}", path.display())), + } +} + +/// 写入文件的完整 content。 +/// +/// 使用临时文件 + rename 保证原子性，避免崩溃后出现半写文件。 +pub async fn write(file_id: i64, content: &str) -> anyhow::Result<()> { + if content.is_empty() { + // 空内容与 NULL 语义统一：删除文件，避免留下空文件。 + return delete(file_id).await; + } + + let path = content_path(file_id); + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent) + .await + .with_context(|| format!("failed to create contents directory {}", parent.display()))?; + } + + let tmp_path = path.with_extension("txt.tmp"); + tokio::fs::write(&tmp_path, content.as_bytes()) + .await + .with_context(|| format!("failed to write tmp content file {}", tmp_path.display()))?; + tokio::fs::rename(&tmp_path, &path) + .await + .with_context(|| format!("failed to rename content file to {}", path.display()))?; + Ok(()) +} + +/// 删除文件的 content。 +/// +/// 文件不存在时视为已成功删除。 +pub async fn delete(file_id: i64) -> anyhow::Result<()> { + let path = content_path(file_id); + match tokio::fs::remove_file(&path).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).with_context(|| format!("failed to delete content file {}", path.display())), + } +} diff --git a/src/frontend.rs b/src/frontend.rs new file mode 100644 index 0000000..e703493 --- /dev/null +++ b/src/frontend.rs @@ -0,0 +1,44 @@ +use axum::{ + Router, body::Body, http::{StatusCode, Uri, header}, response::{IntoResponse, Response}, routing::get +}; +use rust_embed::Embed; + +#[derive(Embed)] +#[folder = "frontend/dist/"] +struct Assets; + +async fn static_handler(uri: Uri) -> impl IntoResponse { + let path = uri.path().trim_start_matches('/'); + + // 如果路径为空，返回 index.html + let path = if path.is_empty() { "index.html" } else { path }; + + serve_file(path) +} + +async fn index_handler() -> impl IntoResponse { + serve_file("index.html") +} + +fn serve_file(path: &str) -> Response { + if path == "favicon.ico" { + return Response::builder().status(StatusCode::NO_CONTENT).body(Body::empty()).unwrap(); + } + + match Assets::get(path) { + Some(content) => { + let mime = mime_guess::from_path(path).first_or_octet_stream().to_string(); + + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, mime) + .body(Body::from(content.data.into_owned())) + .unwrap() + } + None => Response::builder().status(StatusCode::NOT_FOUND).body(Body::from("Not Found")).unwrap(), + } +} + +pub fn router() -> Router { + Router::new().route("/", get(index_handler)).route("/{*path}", get(static_handler)) +} diff --git a/src/graph/graph_manager.rs b/src/graph/graph_manager.rs new file mode 100644 index 0000000..d36ebb7 --- /dev/null +++ b/src/graph/graph_manager.rs @@ -0,0 +1,299 @@ +use std::collections::HashMap; + +use anyhow::Result; +use petgraph::graph::{DiGraph, NodeIndex}; +use sqlx::SqlitePool; + +use super::{Edge, Entity, EntityType, Node, Relation, RelationType}; + +/// 知识图谱管理器 +pub struct KnowledgeGraph { + graph: DiGraph, + node_index: HashMap, + node_id_to_index: HashMap, + pool: SqlitePool, + kb_id: Option, +} + +impl KnowledgeGraph { + /// 从数据库加载图 + pub async fn load_from_db(pool: SqlitePool, kb_id: Option) -> Result { + let mut graph = DiGraph::new(); + let mut node_index = HashMap::new(); + let mut node_id_to_index = HashMap::new(); + + // 加载节点 + let nodes: Vec<(i64, String, String, Option)> = if let Some(kb_id) = kb_id { + sqlx::query_as("SELECT id, name, entity_type, properties FROM graph_nodes WHERE kb_id = ?") + .bind(kb_id) + .fetch_all(&pool) + .await? + } else { + sqlx::query_as("SELECT id, name, entity_type, properties FROM graph_nodes").fetch_all(&pool).await? + }; + + for (id, name, entity_type_str, properties_json) in nodes { + let entity_type = EntityType::Custom(entity_type_str); + let properties: HashMap = if let Some(json) = properties_json { + serde_json::from_str(&json).unwrap_or_default() + } else { + HashMap::new() + }; + + let node = Node { id, name: name.clone(), entity_type, properties }; + + let idx = graph.add_node(node); + node_index.insert(name, idx); + node_id_to_index.insert(id, idx); + } + + // 加载边（使用 n1.kb_id 避免歧义） + let edges: Vec<(i64, i64, i64, String, Option, f32)> = if let Some(kb_id) = kb_id { + sqlx::query_as( + "SELECT e.id, e.source_node_id, e.target_node_id, e.relation_type, e.properties, e.weight \ + FROM graph_edges e \ + INNER JOIN graph_nodes n1 ON e.source_node_id = n1.id \ + INNER JOIN graph_nodes n2 ON e.target_node_id = n2.id \ + WHERE n1.kb_id = ?", + ) + .bind(kb_id) + .fetch_all(&pool) + .await? + } else { + sqlx::query_as( + "SELECT e.id, e.source_node_id, e.target_node_id, e.relation_type, e.properties, e.weight \ + FROM graph_edges e \ + INNER JOIN graph_nodes n1 ON e.source_node_id = n1.id \ + INNER JOIN graph_nodes n2 ON e.target_node_id = n2.id", + ) + .fetch_all(&pool) + .await? + }; + + for (id, source_id, target_id, relation_type_str, properties_json, weight) in edges { + if let (Some(&source_idx), Some(&target_idx)) = + (node_id_to_index.get(&source_id), node_id_to_index.get(&target_id)) + { + let relation_type = RelationType::Custom(relation_type_str); + let properties: HashMap = if let Some(json) = properties_json { + serde_json::from_str(&json).unwrap_or_default() + } else { + HashMap::new() + }; + + let edge = Edge { id, relation_type, weight, properties }; + graph.add_edge(source_idx, target_idx, edge); + } + } + + Ok(Self { graph, node_index, node_id_to_index, pool, kb_id }) + } + + /// 添加节点（在给定事务连接上执行，调用方负责提交） + async fn add_node(&mut self, entity: &Entity, conn: &mut sqlx::SqliteConnection) -> Result { + if let Some(&idx) = self.node_index.get(&entity.name) { + return Ok(idx); + } + + let entity_type_str = entity.entity_type.as_str(); + let properties_json = serde_json::to_string(&entity.properties)?; + + let sql = "INSERT INTO graph_nodes (name, entity_type, properties, file_id, kb_id) \ + VALUES (?, ?, ?, ?, ?) \ + ON CONFLICT(name, entity_type, kb_id) DO UPDATE SET \ + properties = excluded.properties, \ + updated_at = strftime('%s','now') \ + RETURNING id"; + + let id: (i64,) = sqlx::query_as(sql) + .bind(&entity.name) + .bind(entity_type_str) + .bind(&properties_json) + .bind(entity.file_id) + .bind(entity.kb_id.or(self.kb_id)) + .fetch_one(&mut *conn) + .await?; + + let node = Node::from_entity(entity, id.0); + let idx = self.graph.add_node(node); + self.node_index.insert(entity.name.clone(), idx); + self.node_id_to_index.insert(id.0, idx); + + Ok(idx) + } + + /// 添加边（在给定事务连接上执行，调用方负责提交） + async fn add_edge( + &mut self, source_name: &str, target_name: &str, relation: &Relation, conn: &mut sqlx::SqliteConnection, + ) -> Result> { + let source_idx = match self.node_index.get(source_name) { + Some(&idx) => idx, + None => return Ok(None), + }; + + let target_idx = match self.node_index.get(target_name) { + Some(&idx) => idx, + None => return Ok(None), + }; + + let source_id = self.graph[source_idx].id; + let target_id = self.graph[target_idx].id; + + let relation_type_str = relation.relation_type.as_str(); + let properties_json = serde_json::to_string(&relation.properties)?; + + let sql = "INSERT INTO graph_edges (source_node_id, target_node_id, relation_type, properties, weight, file_id) \ + VALUES (?, ?, ?, ?, ?, ?) \ + RETURNING id"; + + let id: (i64,) = sqlx::query_as(sql) + .bind(source_id) + .bind(target_id) + .bind(relation_type_str) + .bind(&properties_json) + .bind(relation.weight) + .bind(relation.file_id) + .fetch_one(&mut *conn) + .await?; + + let edge = Edge::from_relation(relation, id.0); + let edge_idx = self.graph.add_edge(source_idx, target_idx, edge); + + Ok(Some(edge_idx)) + } + + /// 增量更新：所有实体/关系的写入在单个事务内完成，避免逐条提交的往返开销 + pub async fn incremental_update(&mut self, entities: Vec, relations: Vec) -> Result<()> { + let mut tx = self.pool.begin().await?; + + for entity in &entities { + self.add_node(entity, &mut tx).await?; + } + + for relation in &relations { + self.add_edge(&relation.source_name, &relation.target_name, relation, &mut tx).await?; + } + + tx.commit().await?; + Ok(()) + } + + /// 不加载整个内存图，直接在数据库中做增量 upsert（用于后台文件处理）。 + pub async fn incremental_update_direct( + pool: SqlitePool, kb_id: Option, entities: Vec, relations: Vec, + ) -> Result<()> { + let mut tx = pool.begin().await?; + let mut name_to_id: HashMap = HashMap::with_capacity(entities.len()); + + for entity in entities { + let entity_type_str = entity.entity_type.as_str(); + let properties_json = serde_json::to_string(&entity.properties)?; + let id: (i64,) = sqlx::query_as( + "INSERT INTO graph_nodes (name, entity_type, properties, file_id, kb_id) \ + VALUES (?, ?, ?, ?, ?) \ + ON CONFLICT(name, entity_type, kb_id) DO UPDATE SET \ + properties = excluded.properties, \ + updated_at = strftime('%s','now') \ + RETURNING id", + ) + .bind(&entity.name) + .bind(entity_type_str) + .bind(&properties_json) + .bind(entity.file_id) + .bind(entity.kb_id.or(kb_id)) + .fetch_one(&mut *tx) + .await?; + name_to_id.insert(entity.name, id.0); + } + + for relation in relations { + let Some(&source_id) = name_to_id.get(&relation.source_name) else { + continue; + }; + let Some(&target_id) = name_to_id.get(&relation.target_name) else { + continue; + }; + let relation_type_str = relation.relation_type.as_str(); + let properties_json = serde_json::to_string(&relation.properties)?; + sqlx::query( + "INSERT INTO graph_edges (source_node_id, target_node_id, relation_type, properties, weight, file_id) \ + VALUES (?, ?, ?, ?, ?, ?)", + ) + .bind(source_id) + .bind(target_id) + .bind(relation_type_str) + .bind(&properties_json) + .bind(relation.weight) + .bind(relation.file_id) + .execute(&mut *tx) + .await?; + } + + tx.commit().await?; + Ok(()) + } + + /// 保存图快照 + pub async fn save_snapshot(&self) -> Result<()> { + let node_count = self.graph.node_count() as i64; + let edge_count = self.graph.edge_count() as i64; + + let delete_sql = "DELETE FROM graph_snapshots WHERE kb_id IS ?"; + sqlx::query(delete_sql).bind(self.kb_id).execute(&self.pool).await?; + + let insert_sql = "INSERT INTO graph_snapshots (kb_id, graph_data, node_count, edge_count) \ + VALUES (?, ?, ?, ?)"; + sqlx::query(insert_sql) + .bind(self.kb_id) + .bind(Vec::::new()) + .bind(node_count) + .bind(edge_count) + .execute(&self.pool) + .await?; + + Ok(()) + } + + /// 不加载内存图，直接统计数据库中当前 KB 的节点/边数量并保存快照。 + pub async fn save_snapshot_direct(pool: &SqlitePool, kb_id: Option) -> Result<()> { + let (node_count,): (i64,) = if let Some(kb_id) = kb_id { + sqlx::query_as("SELECT COUNT(*) FROM graph_nodes WHERE kb_id = ?").bind(kb_id).fetch_one(pool).await? + } else { + sqlx::query_as("SELECT COUNT(*) FROM graph_nodes").fetch_one(pool).await? + }; + + let (edge_count,): (i64,) = if let Some(kb_id) = kb_id { + sqlx::query_as( + "SELECT COUNT(*) FROM graph_edges e \ + INNER JOIN graph_nodes n1 ON e.source_node_id = n1.id \ + WHERE n1.kb_id = ?", + ) + .bind(kb_id) + .fetch_one(pool) + .await? + } else { + sqlx::query_as( + "SELECT COUNT(*) FROM graph_edges e \ + INNER JOIN graph_nodes n1 ON e.source_node_id = n1.id", + ) + .fetch_one(pool) + .await? + }; + + let mut tx = pool.begin().await?; + sqlx::query("DELETE FROM graph_snapshots WHERE kb_id IS ?").bind(kb_id).execute(&mut *tx).await?; + sqlx::query( + "INSERT INTO graph_snapshots (kb_id, graph_data, node_count, edge_count) \ + VALUES (?, ?, ?, ?)", + ) + .bind(kb_id) + .bind(Vec::::new()) + .bind(node_count) + .bind(edge_count) + .execute(&mut *tx) + .await?; + tx.commit().await?; + + Ok(()) + } +} diff --git a/src/graph/llm_extractor.rs b/src/graph/llm_extractor.rs new file mode 100644 index 0000000..c11cc66 --- /dev/null +++ b/src/graph/llm_extractor.rs @@ -0,0 +1,315 @@ +use std::time::Duration; + +use anyhow::{Result, anyhow}; +use once_cell::sync::Lazy; +use reqwest::Client; +use serde::{Deserialize, Serialize}; + +use super::{Entity, EntityType, Relation, RelationType}; +use crate::config; + +/// 复用的 LLM HTTP client（连接池）。 +static LLM_HTTP_CLIENT: Lazy = Lazy::new(|| { + #[cfg(test)] + let builder = Client::builder().timeout(Duration::from_secs(120)).no_proxy(); + #[cfg(not(test))] + let builder = Client::builder().timeout(Duration::from_secs(120)); + builder.build().expect("build llm extractor http client") +}); + +/// LLM API响应格式（OpenAI 兼容格式） +#[derive(Debug, Deserialize)] +struct LLMApiResponse { + choices: Vec, +} + +#[derive(Debug, Deserialize)] +struct Choice { + message: ResponseMessage, +} + +#[derive(Debug, Deserialize)] +struct ResponseMessage { + content: String, +} + +/// 知识图谱提取结果（完全由LLM定义） +#[derive(Debug, Deserialize)] +struct KnowledgeGraphResponse { + #[serde(default)] + entities: Vec, + #[serde(default)] + relations: Vec, +} + +#[derive(Debug, Deserialize)] +struct LLMEntity { + name: String, + #[serde(rename = "type")] + entity_type: String, + #[serde(default)] + description: Option, +} + +#[derive(Debug, Deserialize)] +struct LLMRelation { + source: String, + target: String, + #[serde(rename = "type")] + relation_type: String, + #[serde(default)] + description: Option, +} + +/// LLM请求格式（OpenAI 兼容格式） +#[derive(Debug, Serialize)] +struct LLMRequest { + model: String, + messages: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + max_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + temperature: Option, +} + +#[derive(Debug, Serialize)] +struct Message { + role: String, + content: String, +} + +/// LLM知识图谱提取器（完全由LLM生成实体和关系） +pub struct LLMGraphExtractor { + client: Client, + api_url: String, + api_key: Option, + model: String, + enabled: bool, +} + +impl LLMGraphExtractor { + /// 从配置创建 + pub fn from_env() -> Self { + let cfg = config::get(); + let llm_config = &cfg.llm; + + let enabled = llm_config.is_enabled(); + let api_url = llm_config.api_url.clone().unwrap_or_default(); + let api_key = llm_config.api_key.clone(); + let model = llm_config.model.clone(); + + // 复用全局 client，避免每次构造都新建连接池 + let client = LLM_HTTP_CLIENT.clone(); + + Self { client, api_url, api_key, model, enabled } + } + + /// 检查是否已启用 + pub fn is_enabled(&self) -> bool { + self.enabled + } + + /// 提取知识图谱（实体和关系一次性提取，类型完全由LLM定义） + pub async fn extract_knowledge_graph(&self, text: &str, context: &str) -> Result<(Vec, Vec)> { + if !self.enabled { + return Err(anyhow!("LLM未启用")); + } + + if text.trim().is_empty() { + return Ok((Vec::new(), Vec::new())); + } + + let prompt = self.build_extraction_prompt(text, context); + let response = self.call_llm(&prompt).await?; + + // 转换实体 + let entities: Vec = response + .entities + .into_iter() + .map(|e| { + // 实体类型完全由LLM定义，直接使用Custom类型 + let entity_type = EntityType::Custom(e.entity_type); + let mut entity = Entity::new(e.name, entity_type); + if let Some(desc) = e.description { + entity = entity.with_property("description".to_string(), desc); + } + entity + }) + .collect(); + + // 转换关系 + let relations: Vec = response + .relations + .into_iter() + .map(|r| { + // 关系类型完全由LLM定义，直接使用Custom类型 + let relation_type = RelationType::Custom(r.relation_type); + let mut relation = Relation::new(r.source, r.target, relation_type); + if let Some(desc) = r.description { + relation = relation.with_property("description".to_string(), desc); + } + relation + }) + .collect(); + + Ok((entities, relations)) + } + + /// 构建知识图谱提取提示词 + fn build_extraction_prompt(&self, text: &str, context: &str) -> String { + format!( + r#"你是一个专业的知识图谱构建专家。请仔细阅读以下文本，从中提取实体和实体之间的关系，构建知识图谱。 + +文档信息: {} + +文本内容: +{} + +## 任务要求 + +### 1. 实体提取 +请识别文本中的所有重要实体，包括但不限于： +- 人名、组织名、公司名 +- 技术名词、编程语言、框架、工具 +- 项目名称、产品名称 +- 地点、时间 +- 专业术语、概念 +- 其他你认为重要的实体 + +对于每个实体，请自定义一个最合适的类型名称（用中文），如： +- 人物、公司、技术、框架、编程语言、数据库、工具、项目、地点、时间段、概念、技能、职位 等 + +### 2. 关系提取 +请识别实体之间的所有有意义的关系。关系类型请自定义（用中文），可以包括但不限于： +- 任职于、工作于、属于 +- 使用、掌握、熟悉、精通 +- 负责、开发、设计、实现 +- 包含、组成、依赖 +- 位于、发生于 +- 关联、相关 +- 其他你认为合适的关系类型 + +### 3. 输出要求 + +请严格按照以下JSON格式输出，不要输出任何其他内容： + +```json +{{ + "entities": [ + {{ + "name": "实体名称", + "type": "实体类型（自定义中文类型）", + "description": "简短描述（可选）" + }} + ], + "relations": [ + {{ + "source": "源实体名称", + "target": "目标实体名称", + "type": "关系类型（自定义中文类型）", + "description": "关系描述（可选）" + }} + ] +}} +``` + +### 4. 注意事项 +- 实体名称要准确，与文本中的表述一致 +- 关系的source和target必须是entities中存在的实体名称 +- 尽量提取所有有意义的实体和关系，不要遗漏 +- 避免提取过于笼统或无意义的实体（如"系统"、"项目"等太泛化的词） +- 关系要基于文本内容，不要臆测 +- 只输出JSON，不要有任何其他文字 + +请开始提取："#, + context, text + ) + } + + /// 调用LLM API + async fn call_llm(&self, prompt: &str) -> Result { + let request = LLMRequest { + model: self.model.clone(), + messages: vec![Message { role: "user".to_string(), content: prompt.to_string() }], + max_tokens: Some(4000), + temperature: Some(0.1), // 低温度，更确定性的输出 + }; + + let mut req_builder = self.client.post(&self.api_url).json(&request); + + if let Some(ref key) = self.api_key { + req_builder = req_builder.header("Authorization", format!("Bearer {}", key)); + } + + let response = req_builder.send().await.map_err(|e| anyhow!("LLM API请求失败: {}", e))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow!("LLM API返回错误 {}: {}", status, body)); + } + + let body = response.text().await.map_err(|e| anyhow!("读取响应失败: {}", e))?; + + // 解析 OpenAI 格式的响应 + let api_response: LLMApiResponse = + serde_json::from_str(&body).map_err(|e| anyhow!("解析LLM API响应失败: {}. 响应内容: {}", e, body))?; + + // 从第一个 choice 的 message 中提取内容 + let content = + api_response.choices.first().ok_or_else(|| anyhow!("LLM响应中没有choices"))?.message.content.clone(); + + // 尝试清理可能的markdown代码块标记 + let clean_content = self.clean_json_response(&content); + + // 解析 JSON 内容 + serde_json::from_str::(&clean_content) + .map_err(|e| anyhow!("解析知识图谱JSON失败: {}. 内容: {}", e, clean_content)) + } + + /// 清理LLM响应中可能的markdown代码块标记 + fn clean_json_response(&self, content: &str) -> String { + let content = content.trim(); + + // 移除可能的 ```json 和 ``` 标记 + let content = if content.starts_with("```json") { + content.strip_prefix("```json").unwrap_or(content) + } else if content.starts_with("```") { + content.strip_prefix("```").unwrap_or(content) + } else { + content + }; + + let content = if content.ends_with("```") { content.strip_suffix("```").unwrap_or(content) } else { content }; + + content.trim().to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_llm_graph_extractor_creation() { + let extractor = LLMGraphExtractor::from_env(); + // 如果没有设置环境变量，应该是禁用状态 + // assert!(!extractor.is_enabled()); + println!("LLM enabled: {}", extractor.is_enabled()); + } + + #[test] + fn test_clean_json_response() { + let extractor = LLMGraphExtractor::from_env(); + + let input = r#"```json +{"entities": [], "relations": []} +```"#; + let cleaned = extractor.clean_json_response(input); + assert_eq!(cleaned, r#"{"entities": [], "relations": []}"#); + + let input2 = r#"{"entities": [], "relations": []}"#; + let cleaned2 = extractor.clean_json_response(input2); + assert_eq!(cleaned2, r#"{"entities": [], "relations": []}"#); + } +} diff --git a/src/graph/mod.rs b/src/graph/mod.rs new file mode 100644 index 0000000..67ac456 --- /dev/null +++ b/src/graph/mod.rs @@ -0,0 +1,121 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +pub mod graph_manager; +pub mod llm_extractor; + +/// 实体类型枚举 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum EntityType { + // 领域特定（完全由LLM定义） + Custom(String), +} + +impl EntityType { + /// 转换为字符串 + pub fn as_str(&self) -> String { + match self { + EntityType::Custom(s) => s.clone(), + } + } +} + +/// 关系类型枚举 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum RelationType { + // 自定义（完全由LLM定义） + Custom(String), +} + +impl RelationType { + /// 转换为字符串 + pub fn as_str(&self) -> String { + match self { + RelationType::Custom(s) => s.clone(), + } + } +} + +/// 实体结构 +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Entity { + pub name: String, + pub entity_type: EntityType, + pub properties: HashMap, + pub file_id: Option, + pub kb_id: Option, +} + +impl Entity { + pub fn new(name: String, entity_type: EntityType) -> Self { + Self { name, entity_type, properties: HashMap::new(), file_id: None, kb_id: None } + } + + pub fn with_property(mut self, key: String, value: String) -> Self { + self.properties.insert(key, value); + self + } +} + +/// 关系结构 +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Relation { + pub source_name: String, + pub target_name: String, + pub relation_type: RelationType, + pub properties: HashMap, + pub weight: f32, + pub file_id: Option, +} + +impl Relation { + pub fn new(source_name: String, target_name: String, relation_type: RelationType) -> Self { + Self { source_name, target_name, relation_type, properties: HashMap::new(), weight: 1.0, file_id: None } + } + + pub fn with_property(mut self, key: String, value: String) -> Self { + self.properties.insert(key, value); + self + } +} + +/// 图节点（用于petgraph） +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Node { + pub id: i64, + pub name: String, + pub entity_type: EntityType, + pub properties: HashMap, +} + +impl Node { + pub fn from_entity(entity: &Entity, id: i64) -> Self { + Self { + id, + name: entity.name.clone(), + entity_type: entity.entity_type.clone(), + properties: entity.properties.clone(), + } + } +} + +/// 图边（用于petgraph） +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Edge { + pub id: i64, + pub relation_type: RelationType, + pub weight: f32, + pub properties: HashMap, +} + +impl Edge { + pub fn from_relation(relation: &Relation, id: i64) -> Self { + Self { + id, + relation_type: relation.relation_type.clone(), + weight: relation.weight, + properties: relation.properties.clone(), + } + } +} diff --git a/src/image_description.rs b/src/image_description.rs new file mode 100644 index 0000000..baf9030 --- /dev/null +++ b/src/image_description.rs @@ -0,0 +1,129 @@ +//! 图片描述持久化模块 +//! +//! 将外部图片文本化接口返回的描述和原始响应保存到 SQLite， +//! 支持索引重建/恢复时无需再次调用外部服务。 + +use std::collections::HashMap; + +use anyhow::{Context, Result}; +use sqlx::{FromRow, QueryBuilder, Sqlite, SqlitePool}; + +#[derive(Debug, Clone, FromRow)] +pub struct ImageDescription { + pub id: i64, + pub file_id: i64, + pub image_filename: String, + pub description: String, + pub jpg_filename: Option, + pub raw_response: String, + pub source: String, + pub created_at: i64, + pub updated_at: i64, +} + +/// 保存或更新单条图片描述 +pub async fn save( + pool: &SqlitePool, file_id: i64, image_filename: &str, description: &str, raw_response: &str, source: &str, jpg_filename: Option<&str>, +) -> Result<()> { + sqlx::query( + "INSERT INTO image_descriptions (file_id, image_filename, description, raw_response, source, jpg_filename) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(file_id, image_filename) DO UPDATE SET + description = excluded.description, + raw_response = excluded.raw_response, + source = excluded.source, + jpg_filename = excluded.jpg_filename, + updated_at = strftime('%s','now')", + ) + .bind(file_id) + .bind(image_filename) + .bind(description) + .bind(raw_response) + .bind(source) + .bind(jpg_filename) + .execute(pool) + .await + .with_context(|| format!("failed to save image_description for file_id={} filename={}", file_id, image_filename))?; + Ok(()) +} + +/// 按文件 ID + 图片文件名查询 +pub async fn get(pool: &SqlitePool, file_id: i64, image_filename: &str) -> Result> { + let row = sqlx::query_as::<_, ImageDescription>( + "SELECT id, file_id, image_filename, description, raw_response, source, jpg_filename, created_at, updated_at + FROM image_descriptions WHERE file_id = ? AND image_filename = ?", + ) + .bind(file_id) + .bind(image_filename) + .fetch_optional(pool) + .await + .with_context(|| format!("failed to get image_description for file_id={} filename={}", file_id, image_filename))?; + Ok(row) +} + +/// 删除指定图片的已缓存描述。用于清理旧版本误写入的服务错误文本。 +pub async fn delete(pool: &SqlitePool, file_id: i64, image_filename: &str) -> Result<()> { + sqlx::query("DELETE FROM image_descriptions WHERE file_id = ? AND image_filename = ?") + .bind(file_id) + .bind(image_filename) + .execute(pool) + .await + .with_context(|| format!("failed to delete image_description for file_id={} filename={}", file_id, image_filename))?; + Ok(()) +} + +/// 按文件 ID 列出所有图片描述（filename -> description） +pub async fn list_by_file(pool: &SqlitePool, file_id: i64) -> Result> { + let rows: Vec = sqlx::query_as::<_, ImageDescription>( + "SELECT id, file_id, image_filename, description, raw_response, source, jpg_filename, created_at, updated_at + FROM image_descriptions WHERE file_id = ?", + ) + .bind(file_id) + .fetch_all(pool) + .await + .with_context(|| format!("failed to list image_descriptions for file_id={}", file_id))?; + Ok(rows.into_iter().map(|row| (row.image_filename, row.description)).collect()) +} + +#[derive(Debug, sqlx::FromRow)] +struct FileImageDescriptionRow { + file_id: i64, + image_filename: String, + description: String, + jpg_filename: Option, +} + +/// 批量查询多个文件的图片描述，返回 (file_id, image_filename) -> description 映射。 +pub async fn list_by_file_ids(pool: &SqlitePool, file_ids: &[i64]) -> Result)>> { + if file_ids.is_empty() { + return Ok(HashMap::new()); + } + + let mut query_builder: QueryBuilder<'_, Sqlite> = + QueryBuilder::new("SELECT file_id, image_filename, description, jpg_filename FROM image_descriptions WHERE file_id IN ("); + let mut separated = query_builder.separated(", "); + for file_id in file_ids { + separated.push_bind(file_id); + } + separated.push_unseparated(")"); + + let rows: Vec = query_builder.build_query_as().fetch_all(pool).await?; + Ok(rows.into_iter().map(|row| ((row.file_id, row.image_filename), (row.description, row.jpg_filename))).collect()) +} + +/// 复制源文件的所有图片描述记录到目标文件（旧版解析结果复用场景） +pub async fn copy_for_file(pool: &SqlitePool, src_file_id: i64, dst_file_id: i64) -> Result<()> { + let rows: Vec = sqlx::query_as::<_, ImageDescription>( + "SELECT id, file_id, image_filename, description, raw_response, source, jpg_filename, created_at, updated_at + FROM image_descriptions WHERE file_id = ?", + ) + .bind(src_file_id) + .fetch_all(pool) + .await + .with_context(|| format!("failed to copy image_descriptions from file_id={}", src_file_id))?; + + for row in rows { + save(pool, dst_file_id, &row.image_filename, &row.description, &row.raw_response, &row.source, row.jpg_filename.as_deref()).await?; + } + Ok(()) +} diff --git a/src/image_parse.rs b/src/image_parse.rs new file mode 100644 index 0000000..0ed5d13 --- /dev/null +++ b/src/image_parse.rs @@ -0,0 +1,239 @@ +//! 外部图片文本化服务客户端 +//! +//! 当配置了 `HTKNOW_IMAGE_PARSE_URL` 时，处理器会把遇到的每张图片发送到该服务， +//! 获取文本化描述并持久化。本模块负责构造请求、解析多种可能的响应格式。 + +use std::time::Duration; + +use anyhow::{Context, Result}; +use base64::{Engine, engine::general_purpose::STANDARD}; +use once_cell::sync::Lazy; +use reqwest::Client; +use serde::Serialize; +use serde_json::Value; + +use crate::config; + +static HTTP_CLIENT: Lazy = Lazy::new(Client::new); + +#[derive(Debug, Serialize)] +struct ParseImageRequest { + #[serde(skip_serializing_if = "Option::is_none")] + content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + original_filename: Option, + filename: String, + image_base64: String, +} + +/// 图片文本化结果 +#[derive(Debug, Clone)] +pub struct ImageParseResponse { + /// 提取后的描述文本 + pub description: String, + /// 服务返回的原始响应（JSON 或纯文本），用于恢复/排查 + pub raw_response: String, + /// 解析服务返回的 JPG 文件名 + pub jpg_filename: Option, +} + +/// 从本地图片文件调用文本化服务 +pub async fn parse_image_file( + path: &std::path::Path, filename: &str, surrounding_content: Option<&str>, +) -> Result { + let bytes = + tokio::fs::read(path).await.with_context(|| format!("failed to read image file: {}", path.display()))?; + let base64 = STANDARD.encode(&bytes); + parse_image_base64(filename, &base64, surrounding_content, None).await +} + +/// 从 base64 编码的图片内容调用文本化服务 +pub async fn parse_image_base64( + filename: &str, image_base64: &str, surrounding_content: Option<&str>, original_filename: Option<&str>, +) -> Result { + let cfg = config::get(); + let url = + cfg.services.image_parse_url.as_deref().ok_or_else(|| anyhow::anyhow!("image parse URL is not configured"))?; + + // 兼容 data:image/xxx;base64, 前缀以及空白字符 + let payload = image_base64 + .find("base64,") + .map(|idx| &image_base64[idx + "base64,".len()..]) + .unwrap_or(image_base64) + .chars() + .filter(|c| !c.is_whitespace()) + .collect(); + + let request = ParseImageRequest { + content: surrounding_content.map(|s| s.to_string()), + original_filename: original_filename.map(|s| s.to_string()), + filename: filename.to_string(), + image_base64: payload, + }; + + let response = HTTP_CLIENT + .post(url) + .timeout(Duration::from_secs(cfg.services.image_parse_timeout_secs)) + .json(&request) + .send() + .await + .with_context(|| format!("image parse request failed: url={}, filename={}", url, filename))?; + + if !response.status().is_success() { + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + anyhow::bail!("image parse API error: {} - {}", status, text); + } + + let raw_response = response.text().await.unwrap_or_default(); + if is_error_response(&raw_response) { + anyhow::bail!("image parse API returned an error response: {}", raw_response.trim()); + } + let description = extract_description(&raw_response); + let jpg_filename = extract_string_field(&raw_response, "jpg_filename"); + Ok(ImageParseResponse { description, raw_response, jpg_filename }) +} + +/// 判断文本化服务是否在 HTTP 200 中返回了业务错误。 +/// +/// 部分服务会用 `Error: ...` 或 JSON 的 `error` 字段表示失败。此类响应不能作为图片描述写入索引。 +pub fn is_error_response(raw_response: &str) -> bool { + let trimmed = raw_response.trim(); + if trimmed.is_empty() { + return false; + } + + let lower = trimmed.to_ascii_lowercase(); + if lower.starts_with("error:") || lower.starts_with("connection error") || lower.starts_with("failed:") { + return true; + } + + let Ok(value) = serde_json::from_str::(trimmed) else { + return false; + }; + let Some(object) = value.as_object() else { + return false; + }; + + if object.get("error").is_some() { + return true; + } + if object.get("code").and_then(Value::as_i64).is_some_and(|code| code >= 400) { + return true; + } + object + .get("status") + .and_then(Value::as_str) + .is_some_and(|status| matches!(status.to_ascii_lowercase().as_str(), "error" | "failed" | "failure")) +} + +/// 从服务原始响应中提取描述文本，兼容多种 JSON 结构，解析失败时返回原字符串 +pub fn extract_description(raw_response: &str) -> String { + let trimmed = raw_response.trim(); + if trimmed.is_empty() { + return String::new(); + } + + let value: Value = match serde_json::from_str(trimmed) { + Ok(v) => v, + Err(_) => return trimmed.to_string(), + }; + + if let Some(desc) = find_description_value(&value) { + return desc.trim().to_string(); + } + + trimmed.to_string() +} + +fn extract_string_field(raw_response: &str, field: &str) -> Option { + let value = serde_json::from_str::(raw_response.trim()).ok()?; + find_string_field(&value, field) +} + +fn find_string_field(value: &Value, field: &str) -> Option { + if let Some(result) = value.get(field).and_then(Value::as_str) { + return Some(result.trim().to_string()); + } + if let Some(object) = value.as_object() { + for nested in object.values() { + if let Some(result) = find_string_field(nested, field) { + return Some(result); + } + } + } + if let Some(items) = value.as_array() { + for item in items { + if let Some(result) = find_string_field(item, field) { + return Some(result); + } + } + } + None +} + +fn find_description_value(value: &Value) -> Option { + // 优先顶层常见字段 + for key in &["image_content", "description", "text", "content", "result"] { + if let Some(v) = value.get(key).and_then(|v| v.as_str()) { + return Some(v.to_string()); + } + } + + // 尝试 data / result 嵌套 + for nested_key in &["data", "result"] { + if let Some(nested) = value.get(nested_key) { + if let Some(v) = nested.as_str() { + return Some(v.to_string()); + } + for key in &["image_content", "description", "text", "content", "result"] { + if let Some(v) = nested.get(key).and_then(|v| v.as_str()) { + return Some(v.to_string()); + } + } + } + } + + // 本身就是字符串 + value.as_str().map(|s| s.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_description_variants() { + assert_eq!(extract_description(r#"{"description": " a picture "}"#), "a picture"); + assert_eq!(extract_description(r#"{"data": {"text": "hello"}}"#), "hello"); + assert_eq!(extract_description(r#"{"result": {"description": "world"}}"#), "world"); + assert_eq!( + extract_description(r#"{"code":200,"message":"ok","data":{"image_content":"返回内容","filename":""}}"#), + "返回内容" + ); + assert_eq!(extract_description("plain text"), "plain text"); + assert_eq!(extract_description(""), ""); + } + + #[test] + fn test_error_responses_are_not_descriptions() { + assert!(is_error_response("Error: Connection error.")); + assert!(is_error_response(r#"{"error":"Connection error"}"#)); + assert!(is_error_response(r#"{"code":500,"message":"Connection error"}"#)); + assert!(!is_error_response(r#"{"description":"A cat on a sofa"}"#)); + assert!(!is_error_response("A picture of a connection cable")); + } + + #[test] + fn test_normalize_base64_payload() { + let raw = "data:image/png;base64, iVBORw0KGgo=\n"; + let normalized: String = raw + .find("base64,") + .map(|idx| &raw[idx + "base64,".len()..]) + .unwrap_or(raw) + .chars() + .filter(|c| !c.is_whitespace()) + .collect(); + assert_eq!(normalized, "iVBORw0KGgo="); + } +} diff --git a/src/init.sql b/src/init.sql new file mode 100644 index 0000000..cf57dde --- /dev/null +++ b/src/init.sql @@ -0,0 +1,256 @@ +CREATE TABLE IF NOT EXISTS files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT, -- 归属用户ID + user_name TEXT, -- 归属用户名 + hash TEXT NOT NULL, -- 文件HASH + filename TEXT NOT NULL, -- 文件名称 + path TEXT NOT NULL, -- 文件路径 + size INTEGER NOT NULL DEFAULT 0, -- 文件大小(字节) + tags TEXT NOT NULL DEFAULT '', -- 标签 + status INTEGER NOT NULL DEFAULT 0, -- 状态: 0-未处理, 1-已处理, 2-处理中, 3-不解析, -1-处理失败 + parse_run_id TEXT DEFAULT NULL, -- 当前解析批次，防止并发解析和旧任务提交结果 + artifact_id INTEGER DEFAULT NULL, -- 共享解析产物 + log TEXT DEFAULT '', -- 日志 + slice_type TEXT DEFAULT '', -- 切片类型 + kb_id INTEGER DEFAULT NULL, -- 知识库ID + parse_priority INTEGER NOT NULL DEFAULT 50, -- 解析队列优先级快照，避免领取待解析文件时跨表排序 + is_public INTEGER NOT NULL DEFAULT 0, -- 是否公开: 0-私有, 1-公开 + meta TEXT DEFAULT NULL, -- 元数据，存储任意JSON数据 + summary TEXT DEFAULT NULL, -- 文件摘要 + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')) +); + +CREATE TABLE IF NOT EXISTS knowledge_bases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT, -- 归属用户ID + user_name TEXT, -- 归属用户名 + name TEXT NOT NULL, -- 知识库名称 + description TEXT DEFAULT '', -- 知识库描述 + kb_type TEXT NOT NULL DEFAULT 'analysis', -- 知识库类型: analysis-分析型, storage-存储型 + parent_id INTEGER, -- 父级知识库ID + is_public INTEGER NOT NULL DEFAULT 0, -- 是否公开: 0-私有, 1-公开 + parse_priority INTEGER NOT NULL DEFAULT 50, -- 解析优先级: 0-100，越大越优先 + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')), + FOREIGN KEY(parent_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS slices ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL, -- 文件ID + parse_run_id TEXT DEFAULT NULL, -- 生成该切片的解析批次；历史数据为空 + ordinal INTEGER DEFAULT NULL, -- 切片在本次解析中的稳定顺序 + is_image INTEGER NOT NULL DEFAULT 0, -- 是否为图片切片: 0-否, 1-是 + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')), + FOREIGN KEY(file_id) REFERENCES files(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) +); + +CREATE TABLE IF NOT EXISTS image_descriptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL, + image_filename TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + raw_response TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + jpg_filename TEXT DEFAULT NULL, + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')), + UNIQUE(file_id, image_filename) +); +CREATE INDEX IF NOT EXISTS idx_image_descriptions_file_id ON image_descriptions(file_id); +CREATE INDEX IF NOT EXISTS idx_image_descriptions_file_filename ON image_descriptions(file_id, image_filename); + +CREATE TABLE IF NOT EXISTS parse_artifacts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + artifact_key TEXT NOT NULL UNIQUE, + content_hash TEXT NOT NULL, + slice_type TEXT NOT NULL, + parser_version TEXT NOT NULL, + config_hash TEXT NOT NULL, + source_file_id INTEGER NOT NULL, + full_content TEXT DEFAULT NULL, + summary TEXT DEFAULT NULL, + created_at INTEGER NOT NULL DEFAULT (strftime('%s','now')), + updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) +); +CREATE INDEX IF NOT EXISTS idx_files_artifact_id ON files(artifact_id); + +CREATE TABLE IF NOT EXISTS slice_positions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + slice_id INTEGER NOT NULL, -- 切片ID + page_idx INTEGER NOT NULL, -- 所在页码（Excel 中表示 sheet 索引） + x1 INTEGER NOT NULL, + y1 INTEGER NOT NULL, + x2 INTEGER NOT NULL, + y2 INTEGER NOT NULL, + sheet_name TEXT DEFAULT NULL, -- Excel 中所在 sheet 名称 + row_num INTEGER DEFAULT NULL, -- Excel 中所在行号（1-based） + created_at INTEGER DEFAULT (strftime('%s','now')), + FOREIGN KEY (slice_id) REFERENCES slices(id) ON DELETE CASCADE +); + +-- 知识图谱相关表 + +-- 图节点表（实体） +CREATE TABLE IF NOT EXISTS graph_nodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, -- 实体名称 + entity_type TEXT NOT NULL, -- 实体类型 + properties TEXT, -- JSON格式的属性 + embedding BLOB, -- 实体embedding向量 + file_id INTEGER, -- 来源文件ID + kb_id INTEGER, -- 所属知识库ID + is_public INTEGER NOT NULL DEFAULT 0, -- 是否公开: 0-私有, 1-公开 + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')), + UNIQUE(name, entity_type, kb_id) -- 同一知识库内，同类型实体名称唯一 +); + +-- 图边表（关系） +CREATE TABLE IF NOT EXISTS graph_edges ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_node_id INTEGER NOT NULL, -- 源节点ID + target_node_id INTEGER NOT NULL, -- 目标节点ID + relation_type TEXT NOT NULL, -- 关系类型 + properties TEXT, -- JSON格式的属性 + weight REAL DEFAULT 1.0, -- 关系权重 + file_id INTEGER, -- 来源文件ID + created_at INTEGER DEFAULT (strftime('%s','now')), + FOREIGN KEY (source_node_id) REFERENCES graph_nodes(id) ON DELETE CASCADE, + FOREIGN KEY (target_node_id) REFERENCES graph_nodes(id) ON DELETE CASCADE +); + +-- 实体提及表（实体在文档中的出现位置） +CREATE TABLE IF NOT EXISTS entity_mentions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + node_id INTEGER NOT NULL, -- 节点ID + slice_id INTEGER NOT NULL, -- 切片ID + start_offset INTEGER, -- 起始位置 + end_offset INTEGER, -- 结束位置 + context TEXT, -- 上下文片段 + created_at INTEGER DEFAULT (strftime('%s','now')), + FOREIGN KEY (node_id) REFERENCES graph_nodes(id) ON DELETE CASCADE, + FOREIGN KEY (slice_id) REFERENCES slices(id) ON DELETE CASCADE +); + +-- 图快照表（存储序列化的图结构） +CREATE TABLE IF NOT EXISTS graph_snapshots ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kb_id INTEGER, -- 知识库ID（NULL表示全局） + graph_data BLOB NOT NULL, -- 序列化的petgraph图 + node_count INTEGER, -- 节点数量 + edge_count INTEGER, -- 边数量 + version INTEGER DEFAULT 1, -- 版本号 + created_at INTEGER DEFAULT (strftime('%s','now')), + UNIQUE(kb_id, version) +); + +-- 索引 +CREATE INDEX IF NOT EXISTS idx_knowledge_bases_parent_id ON knowledge_bases(parent_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_bases_user_public_parent ON knowledge_bases(user_id, is_public, parent_id); +CREATE INDEX IF NOT EXISTS idx_files_kb_id ON files(kb_id); +CREATE INDEX IF NOT EXISTS idx_files_kb_id_created_at ON files(kb_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_files_kb_id_filename ON files(kb_id, filename); +CREATE INDEX IF NOT EXISTS idx_files_kb_id_updated_at ON files(kb_id, updated_at DESC); +CREATE INDEX IF NOT EXISTS idx_files_kb_id_status_updated_at ON files(kb_id, status, updated_at DESC); +CREATE INDEX IF NOT EXISTS idx_files_status_updated_at ON files(status, updated_at DESC); +CREATE INDEX IF NOT EXISTS idx_files_status_id ON files(status, id); +CREATE INDEX IF NOT EXISTS idx_files_id_kb_id ON files(id, kb_id); +CREATE INDEX IF NOT EXISTS idx_files_pending_created_at ON files(created_at) WHERE status = 0; +CREATE INDEX IF NOT EXISTS idx_files_user_id_created_at ON files(user_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_files_hash_slice_type_status_updated_at ON files(hash, slice_type, status, updated_at DESC); +CREATE INDEX IF NOT EXISTS idx_slices_file_id_id ON slices(file_id, id); +CREATE UNIQUE INDEX IF NOT EXISTS idx_slices_file_run_ordinal +ON slices(file_id, parse_run_id, ordinal) +WHERE parse_run_id IS NOT NULL AND ordinal IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_nodes_type ON graph_nodes(entity_type); +CREATE INDEX IF NOT EXISTS idx_nodes_kb ON graph_nodes(kb_id); +CREATE INDEX IF NOT EXISTS idx_nodes_file ON graph_nodes(file_id); +CREATE INDEX IF NOT EXISTS idx_nodes_name ON graph_nodes(name); +CREATE INDEX IF NOT EXISTS idx_edges_source ON graph_edges(source_node_id); +CREATE INDEX IF NOT EXISTS idx_edges_target ON graph_edges(target_node_id); +CREATE INDEX IF NOT EXISTS idx_edges_relation ON graph_edges(relation_type); +CREATE INDEX IF NOT EXISTS idx_mentions_node ON entity_mentions(node_id); +CREATE INDEX IF NOT EXISTS idx_mentions_slice ON entity_mentions(slice_id); +CREATE INDEX IF NOT EXISTS idx_slice_positions_slice ON slice_positions(slice_id); +CREATE INDEX IF NOT EXISTS idx_files_meta ON files(meta); + +-- 搜索词表（分词增强） +CREATE TABLE IF NOT EXISTS search_lexicon ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + term TEXT NOT NULL UNIQUE, -- 词条 + freq INTEGER DEFAULT NULL, -- 词频 + tag TEXT DEFAULT NULL, -- 词性 + enabled INTEGER NOT NULL DEFAULT 1, -- 是否启用 + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')) +); + +CREATE INDEX IF NOT EXISTS idx_search_lexicon_enabled ON search_lexicon(enabled); + +-- 搜索同义词词典（查询扩展） +CREATE TABLE IF NOT EXISTS search_synonyms ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + term TEXT NOT NULL, -- 原词 + synonym TEXT NOT NULL, -- 同义词 + weight REAL NOT NULL DEFAULT 1.0, -- 行级权重 + bidirectional INTEGER NOT NULL DEFAULT 1, -- 是否双向 + enabled INTEGER NOT NULL DEFAULT 1, -- 是否启用 + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')), + UNIQUE(term, synonym) +); + +CREATE INDEX IF NOT EXISTS idx_search_synonyms_term ON search_synonyms(term); +CREATE INDEX IF NOT EXISTS idx_search_synonyms_synonym ON search_synonyms(synonym); + +-- 压缩文件条目表 +CREATE TABLE IF NOT EXISTS archive_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL, + entry_path TEXT NOT NULL, + size INTEGER, + is_directory INTEGER DEFAULT 0, + created_at INTEGER DEFAULT (strftime('%s','now')), + FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_archive_entries_file_id ON archive_entries(file_id); + +-- 索引重建任务状态 +CREATE TABLE IF NOT EXISTS index_rebuild_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + status TEXT NOT NULL DEFAULT 'idle', -- idle/running/completed/failed + phase TEXT NOT NULL DEFAULT '', -- 当前阶段 + total_docs INTEGER NOT NULL DEFAULT 0, -- 总文档数（切片 + 全文等） + processed_docs INTEGER NOT NULL DEFAULT 0,-- 已处理文档数 + started_at INTEGER DEFAULT NULL, -- 开始时间（秒级时间戳） + updated_at INTEGER DEFAULT (strftime('%s','now')), + finished_at INTEGER DEFAULT NULL, -- 完成时间 + error TEXT DEFAULT NULL, -- 错误信息 + meta TEXT DEFAULT NULL -- 扩展信息（JSON） +); + +CREATE INDEX IF NOT EXISTS idx_index_rebuild_jobs_status ON index_rebuild_jobs(status); +CREATE INDEX IF NOT EXISTS idx_index_rebuild_jobs_updated_at ON index_rebuild_jobs(updated_at); + +-- 知识库成员权限表 +CREATE TABLE IF NOT EXISTS kb_permissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kb_id INTEGER NOT NULL, + user_id TEXT NOT NULL, + permission TEXT NOT NULL DEFAULT 'viewer', -- 'viewer' | 'editor' | 'admin' + created_at INTEGER DEFAULT (strftime('%s','now')), + updated_at INTEGER DEFAULT (strftime('%s','now')), + FOREIGN KEY(kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE, + UNIQUE(kb_id, user_id) +); +CREATE INDEX IF NOT EXISTS idx_kb_permissions_kb_id ON kb_permissions(kb_id); +CREATE INDEX IF NOT EXISTS idx_kb_permissions_user_id ON kb_permissions(user_id); diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..12b486e --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,62 @@ +use axum::{ + body::Body, http::{Request, StatusCode}, middleware::Next, response::{IntoResponse, Response} +}; +use base64::{Engine, engine::general_purpose::STANDARD}; + +pub mod api; +pub mod archive; +pub mod config; +pub mod db; +pub mod export; +pub mod file_content; +pub mod frontend; +pub mod graph; +pub mod image_description; +pub mod image_parse; +pub mod log4rs; +pub mod pdf_content; +pub mod pdf_highlight; +pub mod processor; +pub mod search; +pub mod slice_content; + +/// User authentication info extracted from request headers +#[derive(Clone, Debug)] +pub struct AuthUser { + pub user_id: String, + pub user_name: String, + pub role: String, +} + +impl AuthUser { + pub fn is_admin(&self) -> bool { + self.role == "admin" + } +} + +fn decode_base64_if_utf8(value: &str) -> String { + match STANDARD.decode(value) { + Ok(bytes) => String::from_utf8(bytes).unwrap_or_else(|_| value.to_owned()), + Err(_) => value.to_owned(), + } +} + +/// Auth middleware: extract `x-user-id` and `x-role` from headers and put them +/// into request extensions as `AuthUser` so handlers can extract `Extension`. +/// Returns 401 if required headers are missing or invalid. +pub async fn auth(mut req: Request, next: Next) -> Response { + let user_id_opt = req.headers().get("x-user-id").and_then(|v| v.to_str().ok()).map(|s| s.to_owned()); + let user_name_opt = req.headers().get("x-user-name").and_then(|v| v.to_str().ok()).map(decode_base64_if_utf8); + let role_opt = req.headers().get("x-role").and_then(|v| v.to_str().ok()).map(|s| s.to_owned()); + + match (user_id_opt, role_opt) { + (Some(user_id), Some(role)) => { + let user_name = user_name_opt.unwrap_or_default(); + let auth_user = AuthUser { user_id, user_name, role }; + req.extensions_mut().insert(auth_user); + next.run(req).await + } + (None, _) => (StatusCode::UNAUTHORIZED, "Missing x-user-id header").into_response(), + (_, None) => (StatusCode::UNAUTHORIZED, "Missing x-role header").into_response(), + } +} diff --git a/src/log4rs.rs b/src/log4rs.rs new file mode 100644 index 0000000..7dcc814 --- /dev/null +++ b/src/log4rs.rs @@ -0,0 +1,74 @@ +use std::{env, str::FromStr}; + +use log4rs::{ + append::console::ConsoleAppender, config::{Appender, Config, Logger, Root}, encode::pattern::PatternEncoder +}; + +fn parse_rust_log(rust_log: &str) -> (Option, Vec<(String, log::LevelFilter)>) { + let mut default_level = None; + let mut loggers = Vec::new(); + + for directive in rust_log.split(',') { + let directive = directive.trim(); + if directive.is_empty() { + continue; + } + + let (target, level_str) = match directive.split_once('=') { + Some((target, level)) => (Some(target.trim()), level.trim()), + None => (None, directive), + }; + + let Ok(level) = log::LevelFilter::from_str(level_str) else { + continue; + }; + + match target { + Some(target) if !target.is_empty() => { + loggers.push((target.to_string(), level)); + } + _ => { + default_level = Some(level); + } + } + } + + (default_level, loggers) +} + +pub fn init() { + // 创建一个控制台 Appender + let stdout = ConsoleAppender::builder() + .encoder(Box::new(PatternEncoder::new("{d(%Y-%m-%d %H:%M:%S)} [{t}] {h({l})} - {m}{n}"))) + .build(); + + let mut root_level = log::LevelFilter::Info; + let mut logger_levels = Vec::new(); + if let Ok(rust_log) = env::var("RUST_LOG") { + let (default_level, parsed_loggers) = parse_rust_log(&rust_log); + logger_levels = parsed_loggers; + if let Some(level) = default_level { + root_level = level; + } else if !logger_levels.is_empty() { + root_level = log::LevelFilter::Off; + } + } + + // Lance 的逐次 I/O span 在 INFO 级别会产生大量日志。显式配置相同 target 时仍允许覆盖。 + for (target, level) in [("lance", log::LevelFilter::Warn), ("tracing::span", log::LevelFilter::Warn)] { + if !logger_levels.iter().any(|(configured_target, _)| configured_target == target) { + logger_levels.push((target.to_string(), level)); + } + } + + // 构建 log4rs 配置，仅输出到控制台 + let mut config_builder = Config::builder().appender(Appender::builder().build("stdout", Box::new(stdout))); + for (target, level) in logger_levels { + config_builder = config_builder.logger(Logger::builder().build(target, level)); + } + let config = + config_builder.build(Root::builder().appender("stdout").build(root_level)).expect("构建 log4rs 配置失败"); + + // 初始化日志器 + log4rs::init_config(config).expect("初始化 log4rs 失败"); +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..805dbe3 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,191 @@ +#[cfg(debug_assertions)] +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[cfg(all(debug_assertions, feature = "profiling"))] +#[unsafe(export_name = "_rjem_malloc_conf")] +// lg_prof_sample:0 表示每次分配都采样(最详细,但性能开销大) +// lg_prof_sample:10 表示每 1KB 采样一次(推荐) +// lg_prof_sample:19 表示每 512KB 采样一次(默认值) +pub static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:10\0"; + +use std::{net::SocketAddr, sync::Arc}; + +use axum::{Router, extract::DefaultBodyLimit, middleware, response::Html, routing::get}; +use chrono::Local; +use htknow::{api, auth, config, db, frontend, log4rs, processor, search}; +use tokio::net::TcpListener; +use tokio_cron_scheduler::{Job, JobScheduler}; +use utoipa_swagger_ui::SwaggerUi; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + log4rs::init(); + + // 加载配置 + let cfg = config::get(); + log::info!("Configuration loaded: server={}:{}", cfg.server.host, cfg.server.port); + + let pool = db::init().await?; + log::info!("Initializing search engine..."); + let search_init_started_at = std::time::Instant::now(); + let search_engine = search::SearchEngine::init().await.with_pool(pool.clone()); + log::info!("Search engine initialized in {}ms", search_init_started_at.elapsed().as_millis()); + if let Err(err) = search_engine.maybe_rebuild_lancedb_from_db().await { + log::warn!("Failed to reconcile LanceDB from SQLite at startup; continuing with current index: {}", err); + } + if let Err(err) = search_engine.maybe_rebuild_tantivy_from_db().await { + log::warn!("Failed to reconcile Tantivy from SQLite at startup; continuing with current index: {}", err); + } + search::schedule_startup_index_maintenance(); + match search_engine.reload_lexicon().await { + Ok(loaded) => log::info!("Search lexicon loaded: {} words", loaded), + Err(e) => log::warn!("Failed to load search lexicon at startup: {}", e), + } + + let processor = + processor::FileProcessor::new(pool.clone(), search_engine.clone(), cfg.server.process_interval_secs); + if cfg.server.parse_enabled { + processor.start(); + } else { + log::warn!("HTKNOW_PARSE_ENABLED=false，后台文件解析已禁用，仅支持即时解析"); + } + let cron = cfg.server.lancedb_compact_cron.trim(); + let _lancedb_compact_scheduler = if !cron.is_empty() + && !cron.eq_ignore_ascii_case("off") + && !cron.eq_ignore_ascii_case("disabled") + && cron != "0" + { + let compact_lock = Arc::new(tokio::sync::Mutex::new(())); + let search_engine = search_engine.clone(); + let sched = JobScheduler::new().await?; + let job_lock = compact_lock.clone(); + + let job = Job::new_async_tz(cron, Local, move |_uuid, _lock| { + let search_engine = search_engine.clone(); + let job_lock = job_lock.clone(); + Box::pin(async move { + let _guard = job_lock.lock().await; + match search_engine.compact_lancedb().await { + Ok(stats) => { + log::info!( + "LanceDB auto-compact done: deleted_rows={}, total_rows_before={}, total_rows_after={}, size_before_bytes={}, size_after_bytes={}", + stats.deleted_rows, + stats.total_rows_before, + stats.total_rows_after, + stats.size_before_bytes, + stats.size_after_bytes, + ); + + match search_engine.force_merge_tantivy_indexes().await { + Ok((index_stats, full_index_stats)) => { + log::info!( + "Tantivy auto force-merge done: index_segments={}->{}, full_index_segments={}->{}", + index_stats.before_segments, + index_stats.after_segments, + full_index_stats.before_segments, + full_index_stats.after_segments, + ); + } + Err(e) => { + log::error!("Tantivy auto force-merge failed: {}", e); + } + } + } + Err(e) => { + log::error!("LanceDB auto-compact failed: {}", e); + } + } + }) + })?; + + sched.add(job).await?; + sched.start().await?; + log::warn!("LanceDB auto-compact enabled; avoid concurrent writes during compaction"); + log::info!("LanceDB auto-compact cron: {}", cron); + Some(sched) + } else { + None + }; + + // API 路由需要认证 + let upload_limit = cfg.server.upload_limit_mb * 1024 * 1024; + let api_router = Router::new() + .nest("/api/v1/knowledge/", api::app(pool, search_engine)) + .layer(middleware::from_fn(auth)) + .layer(DefaultBodyLimit::max(upload_limit)); + + // Swagger 路由（不需要认证） + let swagger = Router::new() + .route("/docs", get(swagger_ui_handler)) + .merge(SwaggerUi::new("/swagger-ui-assets").url("/api-docs/openapi.json", api::openapi())); + + // 合并路由：前端和 Swagger 不需要认证，API 需要认证 + let app = Router::new().merge(swagger).merge(api_router).merge(frontend::router()); + + let addr = format!("{}:{}", cfg.server.host, cfg.server.port); + let listener = TcpListener::bind(&addr).await?; + log::info!("Server listening on {}", addr); + log::info!("Swagger UI available at http://{}/docs", addr); + axum::serve(listener, app.into_make_service_with_connect_info::()).await?; + Ok(()) +} + +/// Swagger UI HTML handler +async fn swagger_ui_handler() -> Html<&'static str> { + Html( + r#" + + + + + + HTKnow API Documentation + + + +
+ + + +
+ + + + +
+ + + + "#, + ) +} diff --git a/src/pdf_content.rs b/src/pdf_content.rs new file mode 100644 index 0000000..8ddd852 --- /dev/null +++ b/src/pdf_content.rs @@ -0,0 +1,85 @@ +use std::path::PathBuf; + +use anyhow::Context; +use serde::{Deserialize, Serialize}; + +use crate::config; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct PdfContent { + pub page_idx: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bbox: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text_level: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub img_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub table_body: Option, +} + +pub fn path(file_id: i64) -> PathBuf { + PathBuf::from(&config::get().storage.contents_path).join(format!("{}.json", file_id)) +} + +pub async fn read(file_id: i64) -> anyhow::Result> { + let path = path(file_id); + let bytes = match tokio::fs::read(&path).await { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(e).with_context(|| format!("failed to read PDF content file {}", path.display())), + }; + serde_json::from_slice(&bytes).with_context(|| format!("failed to parse PDF content file {}", path.display())) +} + +pub async fn write(file_id: i64, contents: &[PdfContent]) -> anyhow::Result<()> { + if contents.is_empty() { + return delete(file_id).await; + } + + let path = path(file_id); + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent) + .await + .with_context(|| format!("failed to create contents directory {}", parent.display()))?; + } + let bytes = serde_json::to_vec(contents)?; + let tmp_path = path.with_extension("json.tmp"); + tokio::fs::write(&tmp_path, bytes) + .await + .with_context(|| format!("failed to write tmp PDF content file {}", tmp_path.display()))?; + tokio::fs::rename(&tmp_path, &path) + .await + .with_context(|| format!("failed to rename PDF content file to {}", path.display()))?; + Ok(()) +} + +pub async fn delete(file_id: i64) -> anyhow::Result<()> { + let path = path(file_id); + match tokio::fs::remove_file(&path).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).with_context(|| format!("failed to delete PDF content file {}", path.display())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_round_trip() { + let rows = vec![PdfContent { + page_idx: 1, + bbox: Some("[1,2,3,4]".to_string()), + text: Some("text".to_string()), + text_level: Some(2), + img_path: None, + table_body: None, + }]; + let json = serde_json::to_string(&rows).unwrap(); + assert_eq!(serde_json::from_str::>(&json).unwrap(), rows); + } +} diff --git a/src/pdf_highlight.rs b/src/pdf_highlight.rs new file mode 100644 index 0000000..f52e0cf --- /dev/null +++ b/src/pdf_highlight.rs @@ -0,0 +1,411 @@ +//! PDF 高亮标注模块 +//! +//! 用于在 PDF 文件中添加高亮标注 + +use anyhow::Result; +use log::info; +use lopdf::{ + Document, Object, ObjectId, Stream, content::{Content, Operation}, dictionary +}; +use serde::{Deserialize, Serialize}; + +/// 高亮位置信息 +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HighlightPosition { + /// 页码（0-based） + pub page_idx: i32, + /// 边界框 [x1, y1, x2, y2] + pub bbox: [i32; 4], +} + +/// 页面坐标系边界（用于缩放转换） +#[derive(Debug, Clone, Copy)] +pub struct PageCoordBounds { + pub min_x: f32, + pub min_y: f32, + pub max_x: f32, + pub max_y: f32, +} + +/// 高亮颜色配置（RGB，范围 0.0-1.0） +const HIGHLIGHT_COLOR: [f32; 3] = [1.0, 0.8, 0.0]; // 橙黄色 +/// 高亮透明度 +const HIGHLIGHT_OPACITY: f32 = 0.3; + +/// 为 PDF 添加高亮标注 +/// +/// # Arguments +/// * `pdf_bytes` - 原始 PDF 字节数据 +/// * `positions` - 高亮位置列表 +/// +/// # Returns +/// 带高亮标注的 PDF 字节数据 +pub fn add_highlights_to_pdf(pdf_bytes: &[u8], positions: &[HighlightPosition]) -> Result> { + add_highlights_to_pdf_with_bounds(pdf_bytes, positions, None) +} + +/// 为 PDF 添加高亮标注（可选页面坐标边界用于缩放） +pub fn add_highlights_to_pdf_with_bounds( + pdf_bytes: &[u8], positions: &[HighlightPosition], + coord_bounds_by_page: Option<&std::collections::HashMap>, +) -> Result> { + if positions.is_empty() { + return Ok(pdf_bytes.to_vec()); + } + + info!("Adding {} highlights to PDF", positions.len()); + + let mut doc = Document::load_mem(pdf_bytes)?; + + // 获取所有页面 + let pages = doc.get_pages(); + info!("PDF has {} pages", pages.len()); + + // 按页码分组高亮位置 + let mut positions_by_page: std::collections::HashMap> = + std::collections::HashMap::new(); + for pos in positions { + positions_by_page.entry(pos.page_idx).or_default().push(pos); + } + + // 为每个页面添加高亮 + for (page_idx, page_positions) in positions_by_page { + // PDF 页码从 1 开始，positions 的 page_idx 从 0 开始 + let page_num = (page_idx + 1) as u32; + + if let Some(&page_id) = pages.get(&page_num) { + // 获取页面的 MediaBox/CropBox 来确定页面尺寸与偏移（用于坐标转换） + let page_box = + get_page_box(&doc, page_id).unwrap_or(PageBox { x0: 0.0, y0: 0.0, width: 595.0, height: 842.0 }); + info!( + "Page {} (idx {}): size = {}x{}, origin = ({}, {}), {} highlights", + page_num, + page_idx, + page_box.width, + page_box.height, + page_box.x0, + page_box.y0, + page_positions.len() + ); + + let coord_bounds = coord_bounds_by_page + .and_then(|m| m.get(&page_idx)) + .copied() + .or_else(|| estimate_bounds_from_positions(&page_positions)); + if let Some(bounds) = coord_bounds { + info!( + "Page {} (idx {}): coord bounds = ({:.2}, {:.2}) -> ({:.2}, {:.2})", + page_num, page_idx, bounds.min_x, bounds.min_y, bounds.max_x, bounds.max_y + ); + } else { + info!("Page {} (idx {}): coord bounds missing, no scaling/offset", page_num, page_idx); + } + let bbox_transform = calc_bbox_transform_from_bounds(&page_box, coord_bounds); + info!( + "Page {} (idx {}): bbox transform scale x={:.4} y={:.4}", + page_num, page_idx, bbox_transform.scale_x, bbox_transform.scale_y + ); + + // 方法：添加 Highlight 注释（类似 PyMuPDF 的 add_highlight_annot） + add_highlight_annotations_to_page(&mut doc, page_id, &page_positions, &page_box, bbox_transform)?; + } else { + info!("Page {} not found in PDF", page_num); + } + } + + // 保存修改后的 PDF 到内存 + let mut output = Vec::new(); + doc.save_to(&mut output)?; + info!("Generated highlighted PDF: {} bytes", output.len()); + Ok(output) +} + +/// 页面尺寸与原点 +#[derive(Debug, Clone, Copy)] +struct PageBox { + x0: f32, + y0: f32, + width: f32, + height: f32, +} + +/// 获取页面 MediaBox/CropBox 的尺寸与原点 +fn get_page_box(doc: &Document, page_id: ObjectId) -> Option { + let page_dict = doc.get_dictionary(page_id).ok()?; + + if let Some(page_box) = + get_box_from_dict(page_dict, b"MediaBox").or_else(|| get_box_from_dict(page_dict, b"CropBox")) + { + return Some(page_box); + } + + // 如果没有，尝试从父页面获取（可继承） + if let Ok(parent_ref) = page_dict.get(b"Parent") + && let Ok(parent_id) = parent_ref.as_reference() + && let Ok(parent_dict) = doc.get_dictionary(parent_id) + && let Some(page_box) = + get_box_from_dict(parent_dict, b"MediaBox").or_else(|| get_box_from_dict(parent_dict, b"CropBox")) + { + return Some(page_box); + } + + None +} + +fn get_box_from_dict(dict: &lopdf::Dictionary, name: &[u8]) -> Option { + let obj = dict.get(name).ok()?; + let arr = obj.as_array().ok()?; + if arr.len() < 4 { + return None; + } + + let x1 = get_number(&arr[0]).unwrap_or(0.0); + let y1 = get_number(&arr[1]).unwrap_or(0.0); + let x2 = get_number(&arr[2]).unwrap_or(0.0); + let y2 = get_number(&arr[3]).unwrap_or(0.0); + + let x_min = x1.min(x2); + let x_max = x1.max(x2); + let y_min = y1.min(y2); + let y_max = y1.max(y2); + + Some(PageBox { x0: x_min, y0: y_min, width: x_max - x_min, height: y_max - y_min }) +} + +fn estimate_bounds_from_positions(positions: &[&HighlightPosition]) -> Option { + if positions.is_empty() { + return None; + } + + let first = positions[0]; + let mut min_x = first.bbox[0].min(first.bbox[2]) as f32; + let mut min_y = first.bbox[1].min(first.bbox[3]) as f32; + let mut max_x = first.bbox[0].max(first.bbox[2]) as f32; + let mut max_y = first.bbox[1].max(first.bbox[3]) as f32; + + for pos in positions.iter().skip(1) { + let x1 = pos.bbox[0] as f32; + let y1 = pos.bbox[1] as f32; + let x2 = pos.bbox[2] as f32; + let y2 = pos.bbox[3] as f32; + + min_x = min_x.min(x1.min(x2)); + min_y = min_y.min(y1.min(y2)); + max_x = max_x.max(x1.max(x2)); + max_y = max_y.max(y1.max(y2)); + } + + Some(PageCoordBounds { min_x, min_y, max_x, max_y }) +} + +#[derive(Debug, Clone, Copy)] +struct BboxTransform { + scale_x: f32, + scale_y: f32, + offset_x: f32, + offset_y: f32, +} + +fn calc_bbox_transform_from_bounds(page_box: &PageBox, bounds: Option) -> BboxTransform { + let Some(bounds) = bounds else { + return BboxTransform { scale_x: 1.0, scale_y: 1.0, offset_x: 0.0, offset_y: 0.0 }; + }; + + if !bounds.min_x.is_finite() || !bounds.min_y.is_finite() || !bounds.max_x.is_finite() || !bounds.max_y.is_finite() + { + return BboxTransform { scale_x: 1.0, scale_y: 1.0, offset_x: 0.0, offset_y: 0.0 }; + } + + // MinerU docs show two possible normalized coordinate ranges: + // - [0, 1] (percent) + // - [0, 1000] (normalized to 1000) + let max_x = bounds.max_x.max(bounds.min_x); + let max_y = bounds.max_y.max(bounds.min_y); + + if max_x <= 1.5 && max_y <= 1.5 { + return BboxTransform { scale_x: page_box.width, scale_y: page_box.height, offset_x: 0.0, offset_y: 0.0 }; + } + + if max_x <= 1000.0 && max_y <= 1000.0 { + return BboxTransform { + scale_x: page_box.width / 1000.0, + scale_y: page_box.height / 1000.0, + offset_x: 0.0, + offset_y: 0.0, + }; + } + + // Fallback: treat as absolute coords or pixels; shrink only if larger than page. + let mut scale_x = 1.0; + let mut scale_y = 1.0; + + if max_x > page_box.width * 1.02 { + scale_x = page_box.width / max_x; + } + if max_y > page_box.height * 1.02 { + scale_y = page_box.height / max_y; + } + + BboxTransform { scale_x, scale_y, offset_x: 0.0, offset_y: 0.0 } +} + +/// 从 Object 获取数字值 +fn get_number(obj: &Object) -> Option { + match obj { + Object::Integer(i) => Some(*i as f32), + Object::Real(r) => Some(*r), + _ => None, + } +} + +/// 在页面上添加 Highlight 注释 +fn add_highlight_annotations_to_page( + doc: &mut Document, page_id: ObjectId, positions: &[&HighlightPosition], page_box: &PageBox, + bbox_transform: BboxTransform, +) -> Result<()> { + // 为每个位置添加 Highlight 注释 + for pos in positions { + // 先减去偏移量，将坐标归一化到从 0 开始，然后缩放，最后加上页面原点 + let x1 = (pos.bbox[0] as f32 - bbox_transform.offset_x) * bbox_transform.scale_x + page_box.x0; + let y1_mineru = (pos.bbox[1] as f32 - bbox_transform.offset_y) * bbox_transform.scale_y; + let x2 = (pos.bbox[2] as f32 - bbox_transform.offset_x) * bbox_transform.scale_x + page_box.x0; + let y2_mineru = (pos.bbox[3] as f32 - bbox_transform.offset_y) * bbox_transform.scale_y; + + // 转换 Y 坐标（MinerU 是从顶部向下，PDF 是从底部向上） + let y1 = page_box.y0 + (page_box.height - y2_mineru); + let y2 = page_box.y0 + (page_box.height - y1_mineru); + + let (x1, x2) = if x1 <= x2 { (x1, x2) } else { (x2, x1) }; + let (y1, y2) = if y1 <= y2 { (y1, y2) } else { (y2, y1) }; + + let width = x2 - x1; + let height = y2 - y1; + + info!( + "Bbox raw: [{}, {}, {}, {}] -> PDF coords: ({:.2}, {:.2}) size {:.2}x{:.2}", + pos.bbox[0], pos.bbox[1], pos.bbox[2], pos.bbox[3], x1, y1, width, height + ); + + // 检查坐标是否在合理范围内 + if width <= 0.0 || height <= 0.0 { + info!("Skipping invalid highlight: width={}, height={}", width, height); + continue; + } + if x1 < page_box.x0 - 100.0 + || x2 > page_box.x0 + page_box.width + 100.0 + || y1 < page_box.y0 - 100.0 + || y2 > page_box.y0 + page_box.height + 100.0 + { + info!("Warning: highlight outside page bounds: ({:.2}, {:.2}) -> ({:.2}, {:.2})", x1, y1, x2, y2); + } + + // Highlight 注释需要 Rect + QuadPoints（PDF 坐标系，原点在左下） + let rect = Object::Array(vec![x1.into(), y1.into(), x2.into(), y2.into()]); + let quad_points = + Object::Array(vec![x1.into(), y2.into(), x2.into(), y2.into(), x2.into(), y1.into(), x1.into(), y1.into()]); + + let appearance_id = create_highlight_appearance(doc, width, height)?; + + let annot = dictionary! { + "Type" => Object::Name(b"Annot".to_vec()), + "Subtype" => Object::Name(b"Highlight".to_vec()), + "Rect" => rect, + "QuadPoints" => quad_points, + "C" => Object::Array(vec![HIGHLIGHT_COLOR[0].into(), HIGHLIGHT_COLOR[1].into(), HIGHLIGHT_COLOR[2].into()]), + "CA" => Object::Real(HIGHLIGHT_OPACITY), + "F" => Object::Integer(4), // Print + "P" => Object::Reference(page_id), + "AP" => dictionary! { "N" => Object::Reference(appearance_id) }, + }; + + let annot_id = doc.add_object(annot); + add_annot_to_page(doc, page_id, annot_id)?; + } + + Ok(()) +} + +/// 将注释添加到页面的 Annots 列表 +fn add_annot_to_page(doc: &mut Document, page_id: ObjectId, annot_id: ObjectId) -> Result<()> { + let page_obj = doc.get_object(page_id)?.clone(); + if let Object::Dictionary(page_dict) = &page_obj { + match page_dict.get(b"Annots") { + Ok(Object::Reference(annots_ref)) => { + let annots_obj = doc.get_object(*annots_ref)?.clone(); + let mut new_arr = match annots_obj { + Object::Array(arr) => arr, + _ => Vec::new(), + }; + new_arr.push(Object::Reference(annot_id)); + let annots_obj_mut = doc.get_object_mut(*annots_ref)?; + *annots_obj_mut = Object::Array(new_arr); + } + Ok(Object::Array(arr)) => { + let mut new_arr = arr.clone(); + new_arr.push(Object::Reference(annot_id)); + let page_obj_mut = doc.get_object_mut(page_id)?; + if let Object::Dictionary(page_dict_mut) = page_obj_mut { + page_dict_mut.set("Annots", Object::Array(new_arr)); + } + } + _ => { + let page_obj_mut = doc.get_object_mut(page_id)?; + if let Object::Dictionary(page_dict_mut) = page_obj_mut { + page_dict_mut.set("Annots", Object::Array(vec![Object::Reference(annot_id)])); + } + } + } + } + + Ok(()) +} + +fn create_highlight_appearance(doc: &mut Document, width: f32, height: f32) -> Result { + let gs_dict = dictionary! { + "Type" => Object::Name(b"ExtGState".to_vec()), + "ca" => Object::Real(HIGHLIGHT_OPACITY), + "CA" => Object::Real(HIGHLIGHT_OPACITY), + }; + let gs_id = doc.add_object(gs_dict); + + let resources = dictionary! { + "ExtGState" => dictionary! { + "GS1" => Object::Reference(gs_id), + }, + }; + + let operations = vec![ + Operation::new("q", vec![]), + Operation::new("gs", vec![Object::Name(b"GS1".to_vec())]), + Operation::new("rg", vec![HIGHLIGHT_COLOR[0].into(), HIGHLIGHT_COLOR[1].into(), HIGHLIGHT_COLOR[2].into()]), + Operation::new("re", vec![0.into(), 0.into(), width.into(), height.into()]), + Operation::new("f", vec![]), + Operation::new("Q", vec![]), + ]; + + let content = Content { operations }; + let stream = Stream::new( + dictionary! { + "Type" => Object::Name(b"XObject".to_vec()), + "Subtype" => Object::Name(b"Form".to_vec()), + "BBox" => Object::Array(vec![0.into(), 0.into(), width.into(), height.into()]), + "Resources" => Object::Dictionary(resources), + }, + content.encode()?, + ); + + Ok(doc.add_object(stream)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_highlight_position_serialization() { + let pos = HighlightPosition { page_idx: 0, bbox: [100, 200, 300, 250] }; + let json = serde_json::to_string(&pos).unwrap(); + assert!(json.contains("page_idx")); + assert!(json.contains("bbox")); + } +} diff --git a/src/processor.rs b/src/processor.rs new file mode 100644 index 0000000..e01e190 --- /dev/null +++ b/src/processor.rs @@ -0,0 +1,4165 @@ +use std::{ + collections::{HashMap, HashSet}, + future::Future, + path::Path, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use aho_corasick::{AhoCorasick, MatchKind}; +use base64::{Engine, engine::general_purpose::STANDARD}; +use futures::stream::StreamExt; +use log::{debug, error, info, warn}; +use lopdf::Document; +use once_cell::sync::Lazy; +use reqwest::multipart; +use serde::{Deserialize, Deserializer, Serialize}; +use sqlx::{QueryBuilder, Sqlite, SqlitePool}; +use tokio::{ + fs, + io::{AsyncBufReadExt, AsyncReadExt}, + process::Command, + time, +}; + +use crate::{ + api::{ + FILE_COLS_NO_CONTENT, File, collect_image_paths_for_files, collect_image_raw_paths_for_files, + effective_parse_file_id, find_reusable_parsed_file, remove_image_files, resolve_image_storage_path, + update_file_custom_image_meta, + }, + archive, config, + graph::{graph_manager::KnowledgeGraph, llm_extractor::LLMGraphExtractor}, + image_description, image_parse, + search::{self, SearchEngine, content_looks_like_image_reference, tantivy_engine}, +}; + +/// 将本地文件构造为流式 multipart Part，避免大文件全量读入内存。 +async fn stream_file_part(path: &str, filename: &str, mime_type: &str) -> anyhow::Result { + let file = fs::File::open(path).await?; + let len = file.metadata().await?.len(); + Ok(reqwest::multipart::Part::stream_with_length(file, len).file_name(filename.to_string()).mime_str(mime_type)?) +} + +/// 将 HTTP 响应体流式写入文件，并返回写入的字节数。 +async fn stream_response_to_file(response: &mut reqwest::Response, path: &std::path::Path) -> anyhow::Result { + let mut file = fs::File::create(path).await?; + let mut total: u64 = 0; + while let Some(chunk) = response.chunk().await? { + total += chunk.len() as u64; + tokio::io::AsyncWriteExt::write_all(&mut file, &chunk).await?; + } + Ok(total) +} + +/// 判断文件是否以指定字节开头。 +async fn file_starts_with(path: &std::path::Path, prefix: &[u8]) -> bool { + let mut file = match fs::File::open(path).await { + Ok(f) => f, + Err(_) => return false, + }; + let mut buf = vec![0u8; prefix.len()]; + match file.read(&mut buf).await { + Ok(n) if n == prefix.len() => buf == prefix, + _ => false, + } +} + +/// 读取文件开头若干字节并转换为文本摘要，用于错误日志。 +async fn summarize_file_start(path: &std::path::Path) -> String { + match fs::read(path).await { + Ok(bytes) => summarize_http_body(&bytes), + Err(e) => format!("", e), + } +} + +/// 复用的外部服务 HTTP client（连接池），超时取自启动时配置。 +static SERVICES_HTTP_CLIENT: Lazy = Lazy::new(|| { + let timeout = Duration::from_secs(config::get().services.request_timeout_secs); + reqwest::Client::builder().timeout(timeout).build().expect("build services http client") +}); + +#[derive(Debug, Deserialize, Serialize)] +struct Result { + #[serde(default)] + content_list: String, + #[serde(default)] + images: HashMap, +} + +#[allow(clippy::type_complexity)] +type RawImageJobs = (Vec<(String, String)>, HashMap, Vec); + +/// 并发调用外部图片文本化接口，保存结果并返回 filename -> description 映射。 +async fn parse_images_to_descriptions( + pool: &SqlitePool, file_id: i64, images: &HashMap, source: &str, original_filename: Option<&str>, +) -> anyhow::Result> { + let cfg = config::get(); + let Some(parse_url) = cfg.services.image_parse_url.as_deref() else { + return Ok(HashMap::new()); + }; + if images.is_empty() || parse_url.is_empty() { + return Ok(HashMap::new()); + } + let concurrency = cfg.services.image_parse_concurrency.max(1); + + let jobs: Vec<(String, String)> = images.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let original_filename = original_filename.map(str::to_owned); + let mut stream = futures::stream::iter(jobs.into_iter().map(|(filename, base64)| { + let source = source.to_string(); + let pool = pool.clone(); + let original_filename = original_filename.clone(); + async move { + match image_parse::parse_image_base64(&filename, &base64, None, original_filename.as_deref()).await { + Ok(resp) => { + if let Err(err) = image_description::save( + &pool, + file_id, + &filename, + &resp.description, + &resp.raw_response, + &source, + resp.jpg_filename.as_deref(), + ) + .await + { + warn!("Failed to save image description for {}: {}", filename, err); + } + Some((filename, resp.description)) + } + Err(err) => { + warn!("Image parse failed for {}: {}", filename, err); + None + } + } + } + })) + .buffer_unordered(concurrency); + + let mut map = HashMap::new(); + while let Some(result) = stream.next().await { + if let Some((filename, description)) = result { + map.insert(filename, description); + } + } + Ok(map) +} + +/// 从数据库加载图片描述；对未命中的图片尝试从已保存的文件补全。 +async fn load_or_parse_image_descriptions( + pool: &SqlitePool, file_id: i64, image_names: &[String], source: &str, +) -> anyhow::Result> { + let mut desc_map = image_description::list_by_file(pool, file_id).await?; + let stale_error_descriptions: Vec = desc_map + .iter() + .filter(|(_, description)| image_parse::is_error_response(description)) + .map(|(filename, _)| filename.clone()) + .collect(); + for filename in stale_error_descriptions { + desc_map.remove(&filename); + if let Err(err) = image_description::delete(pool, file_id, &filename).await { + warn!("Failed to remove stale image parse error for {}: {}", filename, err); + } + } + if config::get().services.image_parse_url.is_none() { + return Ok(desc_map); + } + + for name in image_names { + if desc_map.contains_key(name) { + continue; + } + let Some(path_str) = resolve_image_storage_path(name) else { continue }; + let path = Path::new(&path_str); + if !path.exists() { + continue; + } + match image_parse::parse_image_file(path, name, None).await { + Ok(resp) => { + if let Err(err) = + image_description::save(pool, file_id, name, &resp.description, &resp.raw_response, source, resp.jpg_filename.as_deref()).await + { + warn!("Failed to save image description for {}: {}", name, err); + } + desc_map.insert(name.clone(), resp.description); + } + Err(err) => warn!("Failed to parse image {} from disk: {}", name, err), + } + } + Ok(desc_map) +} + +fn enrich_content_list_with_image_descriptions(content_list: &mut [ContentItem], desc_map: &HashMap) { + for item in content_list.iter_mut() { + if item.typ == "image" + && let Some(img_name) = item.img_path.as_deref() + && let Some(description) = desc_map.get(img_name).filter(|s| !s.trim().is_empty()) + { + let caption = description.clone(); + let captions = item.image_caption.get_or_insert_with(Vec::new); + if !captions.iter().any(|existing| existing == &caption) { + captions.push(caption); + } + } + } +} + +async fn claim_pending_file_by_id(pool: &SqlitePool, file_id: i64) -> anyhow::Result> { + let sql = format!( + "UPDATE files + SET status = 2, parse_run_id = lower(hex(randomblob(16))), updated_at = strftime('%s','now') + WHERE id = ? AND status = 0 + RETURNING {FILE_COLS_NO_CONTENT}" + ); + Ok(sqlx::query_as::<_, File>(&sql).bind(file_id).fetch_optional(pool).await?) +} + +/// MinerU API 返回的结果结构 +#[derive(Debug, Deserialize)] +struct MinerUResponse { + results: MinerUResults, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum MinerUResults { + Map(HashMap), + List(Vec), +} + +#[derive(Debug, Deserialize)] +struct MinerUResultItem { + #[serde(default)] + status: String, + #[serde(default)] + error: String, + #[serde(default)] + filename: String, + #[serde(default)] + content_list: String, + #[serde(default)] + images: HashMap, +} + +#[derive(Debug, Deserialize)] +struct AnalyzePdfResponse { + code: i32, + #[serde(default)] + message: String, + data: Option, +} + +#[derive(Debug, Deserialize)] +struct AnalyzePdfData { + #[serde(default)] + content_list: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +struct AudioTranscriptionResponse { + #[serde(default)] + text: String, + #[serde(default)] + language: String, +} + +#[derive(Debug, Deserialize)] +struct CustomParseResponse { + #[serde(default)] + code: i32, + #[serde(default)] + message: String, + data: Option, +} + +#[derive(Debug, Deserialize)] +struct CustomParseData { + #[serde(default)] + slices: Vec, + #[serde(default)] + full_content: Option, + #[serde(default)] + summary: Option, + #[serde(default)] + images: Option>, + #[serde(default)] + content_list: Option>, +} + +#[derive(Debug)] +struct NormalizedCustomParseData { + slices: Vec, + full_content: Option, + summary: Option, + images: HashMap, + content_list: Option>, + image_paths: Vec, +} + +#[derive(Debug, Serialize)] +struct CustomParseReuseRequest<'a> { + pdf_contents: &'a [PdfContentRow], +} + +#[derive(Debug, Deserialize)] +struct CustomSlice { + content: String, + #[serde(default, deserialize_with = "deserialize_slice_positions")] + positions: Vec, +} +#[derive(Debug, Clone, Deserialize, Serialize)] +struct ContentItem { + #[serde(default, rename = "type")] + typ: String, + #[serde(default)] + bbox: Vec, + #[serde(default)] + page_idx: i32, + #[serde(default)] + text: Option, + #[serde(default)] + text_level: Option, + #[serde(default)] + text_format: Option, // latex + #[serde(default)] + img_path: Option, + #[serde(default)] + image_caption: Option>, + #[serde(default)] + table_body: Option, + #[serde(default)] + table_caption: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] +struct SlicePosition { + page_idx: i32, + bbox: [i32; 4], + #[serde(default)] + sheet_name: Option, + #[serde(default)] + row_num: Option, +} + +#[derive(Debug, Deserialize)] +struct RawSlicePosition { + page_idx: i32, + #[serde(default)] + bbox: Vec, + #[serde(default)] + sheet_name: Option, + #[serde(default)] + row_num: Option, +} + +fn deserialize_slice_positions<'de, D>(deserializer: D) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let raw_positions: Option> = Option::deserialize(deserializer)?; + let mut positions = Vec::new(); + if let Some(raw_positions) = raw_positions { + for raw in raw_positions { + if raw.bbox.len() == 4 { + positions.push(SlicePosition { + page_idx: raw.page_idx, + bbox: [raw.bbox[0], raw.bbox[1], raw.bbox[2], raw.bbox[3]], + sheet_name: raw.sheet_name, + row_num: raw.row_num, + }); + } else { + debug!("Dropping custom slice position on page {} due to invalid bbox {:?}", raw.page_idx, raw.bbox); + } + } + } + Ok(positions) +} + +#[derive(Debug, Clone)] +struct SliceWithPositions { + content: String, + positions: Vec, + is_image: bool, +} + +#[derive(Debug, Clone)] +struct Segment { + start: usize, + end: usize, + positions: Vec, +} + +type PdfContentRow = crate::pdf_content::PdfContent; + +#[derive(Debug, Clone, sqlx::FromRow)] +struct SliceRow { + id: i64, + content: String, +} + +#[derive(Debug, Clone, sqlx::FromRow)] +struct SlicePositionRecord { + slice_id: i64, + page_idx: i32, + x1: i32, + y1: i32, + x2: i32, + y2: i32, + sheet_name: Option, + row_num: Option, +} + +#[derive(Debug, Clone)] +struct ClonedSlice { + old_id: i64, + new_id: i64, + content: String, +} + +/// 文件处理器：定时从数据库读取未处理的文件并处理 +pub struct FileProcessor { + pool: SqlitePool, + search_engine: search::SearchEngine, + interval: Duration, +} + +static PARSE_PAUSED: AtomicBool = AtomicBool::new(false); +static PARSE_TIMING_RUN_SEQ: AtomicU64 = AtomicU64::new(1); + +struct ParseTimingCtx { + file_id: i64, + filename: String, + run_id: String, + pipeline: &'static str, + run_started: Instant, + step_seq: u32, +} + +impl ParseTimingCtx { + fn new(file: &File, pipeline: &'static str) -> Self { + let seq = PARSE_TIMING_RUN_SEQ.fetch_add(1, Ordering::Relaxed); + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_millis()).unwrap_or_default(); + let run_id = format!("f{}-{}-{}", file.id, now_ms, seq); + let ctx = Self { + file_id: file.id, + filename: file.filename.clone(), + run_id, + pipeline, + run_started: Instant::now(), + step_seq: 0, + }; + debug!( + target: "parse_timing", + "[parse_timing] event=run_start file_id={} filename={:?} run_id={} pipeline={}", + ctx.file_id, + ctx.filename, + ctx.run_id, + ctx.pipeline + ); + ctx + } + + fn set_pipeline(&mut self, pipeline: &'static str) { + if self.pipeline == pipeline { + return; + } + self.pipeline = pipeline; + debug!( + target: "parse_timing", + "[parse_timing] event=pipeline_switch file_id={} run_id={} pipeline={}", + self.file_id, + self.run_id, + self.pipeline + ); + } + + fn step_start(&mut self, step: &'static str) -> (u32, Instant) { + self.step_seq += 1; + let seq = self.step_seq; + debug!( + target: "parse_timing", + "[parse_timing] event=step_start file_id={} run_id={} pipeline={} seq={} step={}", + self.file_id, + self.run_id, + self.pipeline, + seq, + step + ); + (seq, Instant::now()) + } + + fn step_ok(&self, step: &'static str, seq: u32, started_at: Instant) { + debug!( + target: "parse_timing", + "[parse_timing] event=step_end file_id={} run_id={} pipeline={} seq={} step={} status=ok duration_ms={}", + self.file_id, + self.run_id, + self.pipeline, + seq, + step, + started_at.elapsed().as_millis() + ); + } + + fn step_err(&self, step: &'static str, seq: u32, started_at: Instant, err: &anyhow::Error) { + let err_detail = format!("{:#}", err); + debug!( + target: "parse_timing", + "[parse_timing] event=step_end file_id={} run_id={} pipeline={} seq={} step={} status=err duration_ms={} err={:?}", + self.file_id, + self.run_id, + self.pipeline, + seq, + step, + started_at.elapsed().as_millis(), + err_detail + ); + } + + fn finish(&self, result: &anyhow::Result<()>) { + match result { + Ok(_) => { + debug!( + target: "parse_timing", + "[parse_timing] event=run_end file_id={} filename={:?} run_id={} pipeline={} status=ok total_duration_ms={} steps={}", + self.file_id, + self.filename, + self.run_id, + self.pipeline, + self.run_started.elapsed().as_millis(), + self.step_seq + ); + } + Err(err) => { + let err_detail = format!("{:#}", err); + debug!( + target: "parse_timing", + "[parse_timing] event=run_end file_id={} filename={:?} run_id={} pipeline={} status=err total_duration_ms={} steps={} err={:?}", + self.file_id, + self.filename, + self.run_id, + self.pipeline, + self.run_started.elapsed().as_millis(), + self.step_seq, + err_detail + ); + } + } + } + + async fn step(&mut self, step: &'static str, fut: Fut) -> anyhow::Result + where + Fut: Future>, + { + let (seq, started_at) = self.step_start(step); + match fut.await { + Ok(value) => { + self.step_ok(step, seq, started_at); + Ok(value) + } + Err(err) => { + self.step_err(step, seq, started_at, &err); + Err(err) + } + } + } +} + +async fn timed_step_opt(timing: Option<&mut ParseTimingCtx>, step: &'static str, fut: Fut) -> anyhow::Result +where + Fut: Future>, +{ + match timing { + Some(ctx) => ctx.step(step, fut).await, + None => fut.await, + } +} + +fn summarize_http_body(body_bytes: &[u8]) -> String { + if body_bytes.is_empty() { + return "".to_string(); + } + + let raw = String::from_utf8_lossy(body_bytes).trim().to_string(); + if raw.is_empty() { + return "".to_string(); + } + + const MAX_CHARS: usize = 800; + if raw.chars().count() > MAX_CHARS { + let preview: String = raw.chars().take(MAX_CHARS).collect(); + format!("{}...(truncated, {} bytes)", preview, body_bytes.len()) + } else { + raw + } +} + +pub fn set_parse_paused(paused: bool) { + PARSE_PAUSED.store(paused, Ordering::SeqCst); + if paused { + warn!("File parsing has been paused for index maintenance"); + } else { + info!("File parsing resumed"); + } +} + +pub fn is_parse_paused() -> bool { + PARSE_PAUSED.load(Ordering::SeqCst) +} + +impl FileProcessor { + fn artifact_key(file: &File) -> String { + let cfg = config::get(); + format!( + "{}:{}:builtin-v1:{}:{}", + file.hash, file.slice_type, cfg.slice.smart_slice_max_chars, cfg.slice.fixed_slice_overlap_chars + ) + } + + async fn ensure_parse_artifact( + &self, file: &File, full_content: &str, summary: Option<&str>, + ) -> anyhow::Result { + let key = Self::artifact_key(file); + let cfg = config::get(); + let config_hash = format!("{}:{}", cfg.slice.smart_slice_max_chars, cfg.slice.fixed_slice_overlap_chars); + sqlx::query( + "INSERT OR IGNORE INTO parse_artifacts + (artifact_key, content_hash, slice_type, parser_version, config_hash, source_file_id, full_content, summary) + VALUES (?, ?, ?, 'builtin-v1', ?, ?, ?, ?)", + ) + .bind(&key) + .bind(&file.hash) + .bind(&file.slice_type) + .bind(config_hash) + .bind(file.id) + .bind(full_content) + .bind(summary) + .execute(&self.pool) + .await?; + let artifact_id: i64 = sqlx::query_scalar("SELECT id FROM parse_artifacts WHERE artifact_key = ?") + .bind(key) + .fetch_one(&self.pool) + .await?; + let updated_source_summary = sqlx::query( + "UPDATE parse_artifacts + SET summary = ?, updated_at = strftime('%s','now') + WHERE id = ? AND source_file_id = ?", + ) + .bind(summary) + .bind(artifact_id) + .bind(file.id) + .execute(&self.pool) + .await? + .rows_affected(); + if updated_source_summary == 0 && summary.is_some() { + sqlx::query( + "UPDATE parse_artifacts + SET summary = COALESCE(NULLIF(summary, ''), ?), updated_at = strftime('%s','now') + WHERE id = ? AND (summary IS NULL OR summary = '')", + ) + .bind(summary) + .bind(artifact_id) + .execute(&self.pool) + .await?; + } + sqlx::query("UPDATE files SET artifact_id = ? WHERE id = ?") + .bind(artifact_id) + .bind(file.id) + .execute(&self.pool) + .await?; + Ok(artifact_id) + } + + async fn adopt_existing_artifact_if_needed( + &self, file: &File, artifact_id: i64, full_content: &str, summary: Option<&str>, + ) -> anyhow::Result<()> { + let source_file_id: i64 = sqlx::query_scalar("SELECT source_file_id FROM parse_artifacts WHERE id = ?") + .bind(artifact_id) + .fetch_one(&self.pool) + .await?; + if source_file_id == file.id { + return Ok(()); + } + + // 两个相同内容的文件并发完成时，artifact 唯一键只允许一个源文件。 + // 丢弃当前批次的重复持久化数据，并把搜索投影切换到共享源切片。 + self.cleanup_processing_file_data_with_retry(file.id, 3).await?; + let rows = self.fetch_slice_rows(source_file_id).await?; + let shared: Vec = + rows.into_iter().map(|row| ClonedSlice { old_id: row.id, new_id: row.id, content: row.content }).collect(); + let mut source = file.clone(); + source.id = source_file_id; + source.content = Some(full_content.to_string()); + source.summary = summary.map(str::to_string); + self.reindex_cloned_slices(file, &shared, &source).await?; + Ok(()) + } + /// 创建新的文件处理器 + /// + /// # Arguments + /// * `pool` - 数据库连接池 + /// * `search_engine` - 搜索引擎实例 + /// * `interval_secs` - 处理间隔（秒） + pub fn new(pool: SqlitePool, search_engine: search::SearchEngine, interval_secs: u64) -> Self { + Self { pool, search_engine, interval: Duration::from_secs(interval_secs) } + } + + fn services_http_client(&self) -> anyhow::Result { + // 复用全局 client，避免每次请求重建连接池（reqwest::Client 内部 Arc，clone 廉价） + Ok(SERVICES_HTTP_CLIENT.clone()) + } + + /// 启动后台处理任务 + pub fn start(self) { + let processor = Arc::new(self); + tokio::spawn(async move { + info!("File processor started with interval: {:?}", processor.interval); + + if let Err(e) = processor.reset_processing_files().await { + error!("Failed to reset in-progress files: {}", e); + } + + // 自适应轮询：队列空闲时逐步拉长检查间隔，有任务到达后立即回到基础间隔， + // 避免长期无文件时仍以固定频率空转查询数据库。上限取基础间隔的 3 倍， + // 兼顾空闲负载与新文件的响应延迟。 + let base_interval = processor.interval; + let max_idle_interval = base_interval * 3; + let mut idle_interval = base_interval; + + loop { + if is_parse_paused() { + debug!("File processor paused, sleeping for {:?}", base_interval); + time::sleep(base_interval).await; + continue; + } + // 持续处理直到没有待处理的文件 + let mut did_work = false; + loop { + if is_parse_paused() { + debug!("File processor paused while processing queue"); + break; + } + match processor.process_pending_files().await { + Ok(has_more) => { + if !has_more { + // 没有更多文件需要处理，退出内循环 + debug!("No more pending files, waiting for next check"); + break; + } + // 有更多文件，继续处理 + did_work = true; + debug!("More files pending, continuing processing"); + } + Err(e) => { + error!("Error processing files: {}", e); + break; + } + } + } + + // 根据是否处理过文件调整下次检查的等待时间 + if did_work { + idle_interval = base_interval; + } else { + idle_interval = (idle_interval * 2).min(max_idle_interval); + } + time::sleep(idle_interval).await; + } + }); + } + + /// 重置异常退出时处于“处理中”的文件状态 + async fn reset_processing_files(&self) -> anyhow::Result<()> { + let file_ids: Vec = + sqlx::query_scalar("SELECT id FROM files WHERE status = 2").fetch_all(&self.pool).await?; + + if file_ids.is_empty() { + return Ok(()); + } + + info!("Found {} in-progress files from previous run, resetting", file_ids.len()); + + for file_id in file_ids { + if let Err(e) = self.reset_processing_file_data(file_id).await { + error!("Failed to reset processing file {}: {}", file_id, e); + } + } + + Ok(()) + } + + async fn reset_processing_file_data(&self, file_id: i64) -> anyhow::Result<()> { + let image_paths = collect_image_paths_for_files(&self.pool, std::slice::from_ref(&file_id)).await?; + let mut tx = self.pool.begin().await?; + + self.delete_processing_file_data(&mut tx, file_id).await?; + sqlx::query( + "UPDATE files SET status = 0, parse_run_id = NULL, log = '', updated_at = strftime('%s','now') WHERE id = ?", + ) + .bind(file_id) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + self.delete_processing_file_content(file_id).await; + remove_image_files(image_paths).await; + + if let Err(e) = self.search_engine.delete(Some(file_id), None).await { + warn!("Failed to delete search index for file {}: {}", file_id, e); + } + + Ok(()) + } + + async fn cleanup_processing_file_data(&self, file_id: i64) -> anyhow::Result<()> { + let image_paths = collect_image_paths_for_files(&self.pool, std::slice::from_ref(&file_id)).await?; + let mut tx = self.pool.begin().await?; + self.delete_processing_file_data(&mut tx, file_id).await?; + tx.commit().await?; + + self.delete_processing_file_content(file_id).await; + remove_image_files(image_paths).await; + + if let Err(e) = self.search_engine.delete(Some(file_id), None).await { + warn!("Failed to delete search index for file {}: {}", file_id, e); + } + + Ok(()) + } + + async fn cleanup_processing_file_data_with_retry(&self, file_id: i64, max_attempts: usize) -> anyhow::Result<()> { + let max_attempts = max_attempts.max(1); + let mut attempt = 1usize; + + loop { + match self.cleanup_processing_file_data(file_id).await { + Ok(()) => return Ok(()), + Err(err) => { + if attempt >= max_attempts || !Self::is_pool_timeout_error(&err) { + return Err(err); + } + + let retry_in_ms = (attempt as u64) * 200; + warn!( + "Cleanup for file {} failed due to DB pool timeout (attempt {}/{}), retrying in {}ms", + file_id, attempt, max_attempts, retry_in_ms + ); + time::sleep(Duration::from_millis(retry_in_ms)).await; + attempt += 1; + } + } + } + } + + fn is_pool_timeout_error(err: &anyhow::Error) -> bool { + err.chain().any(|cause| { + cause.downcast_ref::().is_some_and(|sqlx_err| matches!(sqlx_err, sqlx::Error::PoolTimedOut)) + }) + } + + async fn delete_processing_file_data( + &self, tx: &mut sqlx::Transaction<'_, Sqlite>, file_id: i64, + ) -> anyhow::Result<()> { + sqlx::query("DELETE FROM entity_mentions WHERE slice_id IN (SELECT id FROM slices WHERE file_id = ?)") + .bind(file_id) + .execute(&mut **tx) + .await?; + sqlx::query("DELETE FROM slice_positions WHERE slice_id IN (SELECT id FROM slices WHERE file_id = ?)") + .bind(file_id) + .execute(&mut **tx) + .await?; + sqlx::query("DELETE FROM slices WHERE file_id = ?").bind(file_id).execute(&mut **tx).await?; + sqlx::query("UPDATE files SET summary = NULL WHERE id = ?").bind(file_id).execute(&mut **tx).await?; + Ok(()) + } + + // Keep filesystem cleanup outside the SQLite transaction so disk I/O cannot hold the writer lock. + async fn delete_processing_file_content(&self, file_id: i64) { + if let Err(e) = crate::slice_content::delete(file_id).await { + warn!("Failed to delete slice content file for processing cleanup of file {}: {}", file_id, e); + } + if let Err(e) = crate::pdf_content::delete(file_id).await { + warn!("Failed to delete PDF content file for processing cleanup of file {}: {}", file_id, e); + } + if let Err(e) = crate::file_content::delete(file_id).await { + warn!("Failed to delete content file for processing cleanup of file {}: {}", file_id, e); + } + } + + async fn persist_file_summary( + &self, file_id: i64, kb_id: Option, summary: Option<&str>, + ) -> anyhow::Result<()> { + let summary = summary.and_then(|value| { + let trimmed = value.trim(); + if trimmed.is_empty() { None } else { Some(trimmed.to_string()) } + }); + if let Some(summary) = summary { + self.search_engine.write_summary(file_id, kb_id, summary).await?; + } else { + self.search_engine.delete_summary_by_file(file_id).await?; + } + Ok(()) + } + + /// 处理所有待处理的文件 + /// 返回是否还有更多文件需要处理 + async fn process_pending_files(self: &Arc) -> anyhow::Result { + if is_parse_paused() { + return Ok(false); + } + + let cfg = config::get(); + let configured_concurrency = cfg.server.process_concurrency.max(1); + let db_safe_concurrency = (cfg.database.max_connections as usize).saturating_sub(2).max(1); + let concurrency = configured_concurrency.min(db_safe_concurrency); + if concurrency < configured_concurrency { + warn!( + "Reducing file processing concurrency from {} to {} to avoid DB pool exhaustion (max_connections={})", + configured_concurrency, concurrency, cfg.database.max_connections + ); + } + info!("Processing pending queue with dynamic claiming (concurrency: {})", concurrency); + + let mut handles = Vec::with_capacity(concurrency); + for worker_idx in 0..concurrency { + let processor = Arc::clone(self); + handles.push(tokio::spawn(async move { processor.run_pending_worker(worker_idx).await })); + } + + let mut claimed_total = 0usize; + for handle in handles { + let worker_claimed = handle.await.map_err(|e| anyhow::anyhow!("pending worker join failed: {}", e))??; + claimed_total += worker_claimed; + } + + Ok(claimed_total > 0) + } + + async fn run_pending_worker(self: Arc, worker_idx: usize) -> anyhow::Result { + let mut claimed = 0usize; + + loop { + if is_parse_paused() { + break; + } + + let Some(file) = self.claim_next_pending_file().await? else { + break; + }; + claimed += 1; + + info!("Pending worker {} claimed file {} (kb_id={:?})", worker_idx, file.id, file.kb_id); + + if let Err(e) = self.process_file_claimed(&file).await { + error!("Failed to process file {}: {}", file.id, e); + if let Err(cleanup_err) = self.cleanup_processing_file_data_with_retry(file.id, 3).await { + error!("Failed to cleanup processing data for file {}: {}", file.id, cleanup_err); + } + self.mark_file_failed(file.id, &e.to_string()).await?; + } else { + info!("Successfully processed file {}", file.id); + } + } + + Ok(claimed) + } + + /// 原子领取一个待处理文件，按知识库解析优先级排序 + async fn claim_next_pending_file(&self) -> anyhow::Result> { + let sql = format!( + "UPDATE files + SET status = 2, parse_run_id = lower(hex(randomblob(16))), updated_at = strftime('%s','now') + WHERE id = ( + SELECT id + FROM files + WHERE status = 0 + ORDER BY parse_priority DESC, created_at ASC, id ASC + LIMIT 1 + ) + AND status = 0 + RETURNING {FILE_COLS_NO_CONTENT}" + ); + let file = sqlx::query_as::<_, File>(&sql).fetch_optional(&self.pool).await?; + Ok(file) + } + + /// 所有指定文件的即时解析也必须经过同一个 compare-and-set 领取入口。 + /// 返回 `None` 表示该文件已经被后台 worker 或另一个请求领取。 + async fn claim_file_by_id(&self, file_id: i64) -> anyhow::Result> { + claim_pending_file_by_id(&self.pool, file_id).await + } + + async fn try_reuse_existing_data(&self, file: &File) -> anyhow::Result { + if !config::get().server.reuse_duplicate_files { + return Ok(false); + } + let Some(source_file) = + find_reusable_parsed_file(&self.pool, &file.hash, &file.slice_type, Some(file.id)).await? + else { + return Ok(false); + }; + + info!("Found reusable parsed file {} for new file {} (hash={})", source_file.id, file.id, file.hash); + + match self.clone_file_data(&source_file, file).await { + Ok(_) => Ok(true), + Err(err) => { + error!("Failed to reuse parsed data from file {} for file {}: {}", source_file.id, file.id, err); + if let Err(clean_err) = self.cleanup_processing_file_data_with_retry(file.id, 3).await { + warn!("Failed to cleanup file {} after reuse error: {}", file.id, clean_err); + } + sqlx::query("UPDATE files SET log = ?, updated_at = strftime('%s','now') WHERE id = ? AND status = 2") + .bind(format!("Reuse failed: {}", err)) + .bind(file.id) + .execute(&self.pool) + .await?; + Ok(false) + } + } + } + + async fn process_file_claimed(&self, file: &File) -> anyhow::Result<()> { + self.process_file_inner(file, true, false).await + } + + async fn process_file_claimed_skip_reuse(&self, file: &File) -> anyhow::Result<()> { + self.process_file_inner(file, true, true).await + } + + async fn process_file_inner(&self, file: &File, already_claimed: bool, skip_reuse: bool) -> anyhow::Result<()> { + let mut timing = ParseTimingCtx::new(file, "dispatch"); + let result = async { + if is_parse_paused() { + anyhow::bail!("parse is paused for index maintenance"); + } + info!("Processing file: {} ({})", file.filename, file.id); + if !self.ensure_file_exists(file.id, "process start").await? { + return Ok(()); + } + + let current_status: Option = timing + .step("status_check", async { + Ok(sqlx::query_scalar("SELECT status FROM files WHERE id = ?") + .bind(file.id) + .fetch_optional(&self.pool) + .await?) + }) + .await?; + if let Some(status) = current_status { + if !already_claimed && status != 0 { + info!("File {} status changed to {}, skipping processing", file.id, status); + return Ok(()); + } + if already_claimed && status != 2 { + info!("Claimed file {} status changed to {}, skipping processing", file.id, status); + return Ok(()); + } + } + + let is_storage = timing.step("storage_kb_check", self.is_storage_kb(file.kb_id)).await?; + if is_storage { + info!("Skipping parsing for storage knowledge base file {}", file.id); + timing.step("mark_storage_skipped", self.mark_file_storage_skipped(file.id)).await?; + return Ok(()); + } + + if !skip_reuse && timing.step("reuse_check", self.try_reuse_existing_data(file)).await? { + timing.set_pipeline("reuse"); + info!("File {} reused existing parsed data, skipping processing pipeline", file.id); + return Ok(()); + } + + timing + .step("set_processing_status", async { + let sql = "UPDATE files SET status = 2, updated_at = strftime('%s','now') WHERE id = ?"; + sqlx::query(sql).bind(file.id).execute(&self.pool).await?; + Ok(()) + }) + .await?; + + // 检查文件是否为 PDF 或图片 + let filename_lower = file.filename.to_lowercase(); + let is_pdf = filename_lower.ends_with(".pdf"); + let is_image = crate::search::is_image_file(&filename_lower); + let is_audio = Self::is_audio_file(&filename_lower); + + // 检查文件是否为 Office 文档 + let is_word = filename_lower.ends_with(".doc") || filename_lower.ends_with(".docx"); + let is_presentation = filename_lower.ends_with(".ppt") || filename_lower.ends_with(".pptx"); + let is_excel = filename_lower.ends_with(".xls") || filename_lower.ends_with(".xlsx"); + + let cfg = config::get(); + let custom_url = cfg.services.custom_parse_url.as_deref(); + let custom_reuse_url = cfg.services.custom_parse_reuse_url.as_deref(); + + if let Some(reuse_url) = custom_reuse_url { + info!("Custom parse reuse enabled"); + match self.process_file_with_custom_reuse_parser(file, reuse_url, Some(&mut timing)).await { + Ok(true) => { + timing.set_pipeline("custom_reuse"); + info!( + "Custom parse reuse enabled, reused existing pdf_contents for file {} via {}", + file.id, reuse_url + ); + return Ok(()); + } + Ok(false) => {} + Err(err) => { + error!("Custom parse reuse failed for file {}: {}", file.id, err); + } + } + } + + if is_excel { + timing.set_pipeline("excel"); + if !self.ensure_file_exists(file.id, "before excel processing").await? { + return Ok(()); + } + info!("Detected Excel document, parsing directly: {}", file.filename); + self.process_excel_file(file, Some(&mut timing)).await?; + return Ok(()); + } + + if is_word || is_presentation { + timing.set_pipeline("office_pdf"); + if !self.ensure_file_exists(file.id, "before office conversion").await? { + return Ok(()); + } + info!("Detected Office document, converting to PDF: {}", file.filename); + + if let Some(custom_url) = custom_url { + let stored_pdf_path = + timing.step("convert_office_to_pdf", self.convert_office_to_pdf(file)).await?; + let mut temp_file = file.clone(); + temp_file.path = stored_pdf_path.to_string_lossy().to_string(); + temp_file.filename = format!("{}.pdf", file.id); + + timing.set_pipeline("custom_parser"); + info!("Custom parse enabled, routing converted PDF for file {} to {}", file.id, custom_url); + self.process_file_with_custom_parser(&temp_file, custom_url, Some(&mut timing)).await?; + return Ok(()); + } + + self.convert_office_to_pdf_and_process(file, Some(&mut timing)).await?; + return Ok(()); + } + + // 压缩文件：不解析，直接标记为跳过 + if archive::is_archive_file(&filename_lower) { + timing.set_pipeline("archive"); + info!("Detected archive file, skipping parsing: {}", file.filename); + timing + .step("mark_archive_skipped", async { + let sql = + "UPDATE files SET status = 3, parse_run_id = NULL, log = ?, updated_at = strftime('%s','now') WHERE id = ?"; + sqlx::query(sql).bind("Archive file: not parsed").bind(file.id).execute(&self.pool).await?; + Ok(()) + }) + .await?; + return Ok(()); + } + + if let Some(custom_url) = custom_url + && is_pdf + { + timing.set_pipeline("custom_parser"); + info!("Custom parse enabled, routing file {} to {}", file.id, custom_url); + self.process_file_with_custom_parser(file, custom_url, Some(&mut timing)).await?; + return Ok(()); + } + + if is_pdf { + timing.set_pipeline("pdf"); + if !self.ensure_file_exists(file.id, "before pdf processing").await? { + return Ok(()); + } + // 处理 PDF 或图片文件 + self.process_pdf_file(file, None, false, None, Some(&mut timing)).await?; + } else if is_image { + timing.set_pipeline("image"); + if !self.ensure_file_exists(file.id, "before image embedding").await? { + return Ok(()); + } + let image_embedding = if search::embedding::image_embedding_enabled() { + match timing + .step( + "get_image_embedding", + search::embedding::get_image_embedding_from_path(&file.path, Some(&file.filename)), + ) + .await + { + Ok(embedding) => Some(Arc::new(embedding)), + Err(err) => { + // 图片向量服务不一定支持所有可上传的图片格式；解析和入库不应因此失败。 + warn!("Failed to get image embedding for {}, skipping it: {}", file.filename, err); + None + } + } + } else { + info!("Image embedding URL not configured, skipping image embedding for file {}", file.id); + None + }; + if Self::requires_mineru_image_conversion(&filename_lower) { + match Self::convert_image_for_mineru(file).await { + Ok(converted_path) => { + let mut converted_file = file.clone(); + converted_file.path = converted_path.to_string_lossy().to_string(); + converted_file.filename = format!("{}.png", file.filename); + let result = self + .process_pdf_file(&converted_file, image_embedding, true, Some(&file.filename), Some(&mut timing)) + .await; + if let Err(err) = fs::remove_file(&converted_path).await { + if err.kind() != std::io::ErrorKind::NotFound { + warn!("Failed to remove converted image {}: {}", converted_path.display(), err); + } + } + result?; + } + Err(err) => { + // 保证文件可检索；但正常情况下应优先保留 MinerU 产生的 content_list。 + warn!( + "Failed to convert image {} for MinerU, falling back to a single direct slice: {}", + file.filename, err + ); + self.process_uploaded_image_file(file, image_embedding, Some(&mut timing)).await?; + } + } + } else { + self.process_pdf_file(file, image_embedding, true, None, Some(&mut timing)).await?; + } + } else if is_audio { + timing.set_pipeline("audio"); + if !self.ensure_file_exists(file.id, "before audio processing").await? { + return Ok(()); + } + self.process_audio_file(file, Some(&mut timing)).await?; + } else { + timing.set_pipeline("text"); + if !self.ensure_file_exists(file.id, "before text processing").await? { + return Ok(()); + } + // 处理普通文本文件 + self.process_text_file(file, Some(&mut timing)).await?; + } + + Ok(()) + } + .await; + timing.finish(&result); + result + } + + async fn process_file_with_custom_parser( + &self, file: &File, custom_url: &str, mut timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result<()> { + if !self.ensure_file_exists(file.id, "custom parse start").await? { + return Ok(()); + } + + let parse_data = + timed_step_opt(timing.as_deref_mut(), "custom_parse_api", self.call_custom_parse_api(file, custom_url)) + .await?; + let normalized = Self::normalize_custom_parse_data(parse_data)?; + + if let Some(content_list) = normalized.content_list.as_ref() { + self.insert_custom_pdf_contents(file.id, content_list, timing.as_deref_mut()).await?; + } else { + timed_step_opt(timing.as_deref_mut(), "custom_update_image_meta", async { + update_file_custom_image_meta(&self.pool, file.id, &normalized.image_paths, "custom_parse_images") + .await?; + Ok(()) + }) + .await?; + } + + if !normalized.images.is_empty() { + self.save_custom_images(&normalized.images, timing.as_deref_mut()).await?; + } + + parse_images_to_descriptions(&self.pool, file.id, &normalized.images, "custom", Some(&file.filename)).await?; + + self.save_custom_slices( + file, + &normalized.slices, + normalized.full_content.as_deref(), + normalized.summary.as_deref(), + timing, + ) + .await?; + + Ok(()) + } + + async fn process_file_with_custom_reuse_parser( + &self, file: &File, custom_reuse_url: &str, mut timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result { + let pdf_rows = + timed_step_opt(timing.as_deref_mut(), "fetch_pdf_content_rows", self.fetch_pdf_content_rows(file.id)) + .await?; + if pdf_rows.is_empty() { + debug!("Custom parse reuse skipped for file {} because pdf_contents are empty", file.id); + return Ok(false); + } + + let payload = CustomParseReuseRequest { pdf_contents: &pdf_rows }; + let client = self.services_http_client()?; + let response = timed_step_opt(timing.as_deref_mut(), "custom_parse_reuse_api", async { + Ok(client.post(custom_reuse_url).json(&payload).send().await?) + }) + .await?; + let response_url = response.url().to_string(); + let status = response.status(); + let request_id = + response.headers().get("x-request-id").and_then(|v| v.to_str().ok()).unwrap_or("-").to_string(); + let body_bytes = response.bytes().await?; + if !status.is_success() { + return Err(anyhow::anyhow!( + "Custom parse reuse API failed (status={}, url={}, request_id={}, body_len={}): {}", + status.as_u16(), + response_url, + request_id, + body_bytes.len(), + summarize_http_body(&body_bytes) + )); + } + + let parsed: CustomParseResponse = serde_json::from_slice(&body_bytes).map_err(|e| { + anyhow::anyhow!( + "Custom parse reuse API response decode failed (status={}, url={}, body_len={}): {} - {}", + status.as_u16(), + response_url, + body_bytes.len(), + e, + summarize_http_body(&body_bytes) + ) + })?; + + if parsed.code != 200 { + let msg = if parsed.message.is_empty() { "" } else { parsed.message.as_str() }; + return Err(anyhow::anyhow!( + "Custom parse reuse API returned error code={} message={} url={}", + parsed.code, + msg, + response_url + )); + } + + let data = parsed.data.ok_or_else(|| anyhow::anyhow!("Custom parse reuse API returned empty data"))?; + if data.slices.is_empty() && data.summary.as_deref().unwrap_or("").trim().is_empty() { + return Err(anyhow::anyhow!("Custom parse reuse API returned empty slices and summary")); + } + + self.save_custom_slices(file, &data.slices, data.full_content.as_deref(), data.summary.as_deref(), timing) + .await?; + + Ok(true) + } + + fn normalize_custom_parse_data(data: CustomParseData) -> anyhow::Result { + let mut image_mapping: HashMap = HashMap::new(); + let images = data.images.unwrap_or_default(); + + for image_name in images.keys() { + Self::ensure_safe_image_name(image_name)?; + image_mapping.entry(image_name.clone()).or_insert_with(|| image_name.clone()); + } + + if let Some(content_list) = data.content_list.as_ref() { + for item in content_list { + if let Some(img_path) = item.img_path.as_deref() { + Self::ensure_safe_image_name(img_path)?; + let mapped_path = image_mapping.get(img_path).cloned().or_else(|| { + Self::basename_for_path(img_path).and_then(|basename| image_mapping.get(&basename).cloned()) + }); + let mapped_path = mapped_path.unwrap_or_else(|| img_path.to_string()); + image_mapping.entry(img_path.to_string()).or_insert(mapped_path); + } + } + } + + let image_mapping = Self::expand_image_mapping_aliases(image_mapping); + let mut referenced_image_paths = Vec::new(); + for slice in &data.slices { + for image_path in Self::extract_custom_image_refs(&slice.content) { + if Self::lookup_image_mapping(&image_mapping, &image_path).is_none() + && resolve_image_storage_path(&image_path).is_some() + && !referenced_image_paths.iter().any(|existing| existing == &image_path) + { + referenced_image_paths.push(image_path); + } + } + } + if let Some(full_content) = data.full_content.as_deref() { + for image_path in Self::extract_custom_image_refs(full_content) { + if Self::lookup_image_mapping(&image_mapping, &image_path).is_none() + && resolve_image_storage_path(&image_path).is_some() + && !referenced_image_paths.iter().any(|existing| existing == &image_path) + { + referenced_image_paths.push(image_path); + } + } + } + + let mut normalized_images = HashMap::new(); + for (image_name, image_base64) in images { + let new_name = image_mapping + .get(&image_name) + .cloned() + .or_else(|| Self::basename_for_path(&image_name).and_then(|name| image_mapping.get(&name).cloned())) + .unwrap_or_else(|| image_name.clone()); + normalized_images.insert(new_name, image_base64); + } + + let mut content_list = data.content_list; + if let Some(items) = content_list.as_mut() { + for item in items { + if let Some(img_path) = item.img_path.as_deref() + && let Some(new_path) = Self::lookup_image_mapping(&image_mapping, img_path) + { + item.img_path = Some(new_path); + } + } + } + + let slices = data + .slices + .into_iter() + .map(|mut slice| { + slice.content = Self::rewrite_custom_image_refs(&slice.content, &image_mapping); + slice + }) + .collect(); + let full_content = data.full_content.map(|content| Self::rewrite_custom_image_refs(&content, &image_mapping)); + let summary = data.summary.and_then(|summary| { + let trimmed = summary.trim(); + if trimmed.is_empty() { None } else { Some(trimmed.to_string()) } + }); + + let mut image_paths = Vec::new(); + let mut seen = HashSet::new(); + for new_path in image_mapping.values() { + let trimmed = new_path.trim(); + if !trimmed.is_empty() && seen.insert(trimmed.to_string()) { + image_paths.push(trimmed.to_string()); + } + } + for path in referenced_image_paths { + let trimmed = path.trim(); + if !trimmed.is_empty() && seen.insert(trimmed.to_string()) { + image_paths.push(trimmed.to_string()); + } + } + + Ok(NormalizedCustomParseData { + slices, + full_content, + summary, + images: normalized_images, + content_list, + image_paths, + }) + } + + fn ensure_safe_image_name(image_name: &str) -> anyhow::Result<()> { + anyhow::ensure!( + resolve_image_storage_path(image_name).is_some(), + "Custom parse returned unsafe image path: {}", + image_name + ); + Ok(()) + } + + fn lookup_image_mapping(mapping: &HashMap, image_path: &str) -> Option { + mapping + .get(image_path) + .cloned() + .or_else(|| Self::basename_for_path(image_path).and_then(|basename| mapping.get(&basename).cloned())) + } + + fn basename_for_path(path: &str) -> Option { + std::path::Path::new(path).file_name().and_then(|name| name.to_str()).map(|name| name.to_string()) + } + + fn expand_image_mapping_aliases(mut mapping: HashMap) -> HashMap { + let aliases: Vec<(String, String)> = mapping + .iter() + .filter_map(|(old_path, new_path)| { + let basename = Self::basename_for_path(old_path)?; + if basename == *old_path { None } else { Some((basename, new_path.clone())) } + }) + .collect(); + for (alias, new_path) in aliases { + mapping.entry(alias).or_insert(new_path); + } + mapping + } + + fn extract_custom_image_refs(content: &str) -> Vec { + let api_re = regex::Regex::new(r"/api/v1/knowledge/files/([^\s'\)>]+)").expect("valid image reference regex"); + let markdown_re = regex::Regex::new(r"!\[[^\]]*\]\(([^)]+)\)").expect("valid markdown image regex"); + let mut refs = Vec::new(); + for cap in api_re.captures_iter(content) { + if let Some(path) = cap.get(1).map(|m| m.as_str().trim()).filter(|path| !path.is_empty()) + && !refs.iter().any(|existing| existing == path) + { + refs.push(path.to_string()); + } + } + for cap in markdown_re.captures_iter(content) { + let Some(path) = cap.get(1).map(|m| m.as_str().trim()).filter(|path| !path.is_empty()) else { + continue; + }; + if path.starts_with("http://") || path.starts_with("https://") || path.starts_with("data:") { + continue; + } + let normalized = path.strip_prefix("/api/v1/knowledge/files/").unwrap_or(path).trim(); + if !normalized.is_empty() && !refs.iter().any(|existing| existing == normalized) { + refs.push(normalized.to_string()); + } + } + refs + } + + fn fallback_rewrite_custom_image_refs(content: &str, mapping: &HashMap) -> String { + let mut rewritten = content.to_string(); + let mut pairs: Vec<(&String, &String)> = mapping.iter().collect(); + pairs.sort_by_key(|(right, _)| std::cmp::Reverse(right.len())); + for (old_path, new_path) in pairs { + rewritten = rewritten.replace( + &format!("/api/v1/knowledge/files/{}", old_path), + &format!("/api/v1/knowledge/files/{}", new_path), + ); + rewritten = rewritten.replace(&format!("]({})", old_path), &format!("]({})", new_path)); + } + rewritten + } + + fn rewrite_custom_image_refs(content: &str, mapping: &HashMap) -> String { + if mapping.is_empty() || content.is_empty() { + return content.to_string(); + } + + let mut patterns = Vec::with_capacity(mapping.len() * 2); + let mut replacements = Vec::with_capacity(mapping.len() * 2); + for (old_path, new_path) in mapping { + patterns.push(format!("/api/v1/knowledge/files/{old_path}")); + replacements.push(format!("/api/v1/knowledge/files/{new_path}")); + patterns.push(format!("]({old_path})")); + replacements.push(format!("]({new_path})")); + } + + match AhoCorasick::builder().match_kind(MatchKind::LeftmostLongest).build(&patterns) { + Ok(ac) => ac.replace_all(content, &replacements), + Err(e) => { + warn!("Failed to build aho-corasick automaton: {e}, fallback to loop"); + Self::fallback_rewrite_custom_image_refs(content, mapping) + } + } + } + + async fn call_custom_parse_api(&self, file: &File, custom_url: &str) -> anyhow::Result { + let mime_type = mime_guess::from_path(&file.filename).first_or_octet_stream().essence_str().to_string(); + let file_part = stream_file_part(&file.path, &file.filename, &mime_type).await?; + let form = multipart::Form::new().part("file", file_part); + + let client = self.services_http_client()?; + let response = client.post(custom_url).multipart(form).send().await?; + let response_url = response.url().to_string(); + let status = response.status(); + let request_id = + response.headers().get("x-request-id").and_then(|v| v.to_str().ok()).unwrap_or("-").to_string(); + let body_bytes = response.bytes().await?; + if !status.is_success() { + return Err(anyhow::anyhow!( + "Custom parse API failed (status={}, url={}, request_id={}, body_len={}): {}", + status.as_u16(), + response_url, + request_id, + body_bytes.len(), + summarize_http_body(&body_bytes) + )); + } + + let parsed: CustomParseResponse = serde_json::from_slice(&body_bytes).map_err(|e| { + anyhow::anyhow!( + "Custom parse API response decode failed (status={}, url={}, body_len={}): {} - {}", + status.as_u16(), + response_url, + body_bytes.len(), + e, + summarize_http_body(&body_bytes) + ) + })?; + + if parsed.code != 200 { + let msg = if parsed.message.is_empty() { "" } else { parsed.message.as_str() }; + return Err(anyhow::anyhow!( + "Custom parse API returned error code={} message={} url={}", + parsed.code, + msg, + response_url + )); + } + + let data = parsed.data.ok_or_else(|| anyhow::anyhow!("Custom parse API returned empty data"))?; + if data.slices.is_empty() && data.summary.as_deref().unwrap_or("").trim().is_empty() { + return Err(anyhow::anyhow!("Custom parse API returned empty slices and summary")); + } + + Ok(data) + } + + async fn save_custom_slices( + &self, file: &File, slices: &[CustomSlice], full_content: Option<&str>, summary: Option<&str>, + timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result<()> { + if !self.ensure_file_exists(file.id, "before writing custom slices").await? { + return Ok(()); + } + + let derived_full_content = match full_content { + Some(content) if !content.trim().is_empty() => content.to_string(), + _ => { + let mut combined = String::new(); + for (idx, slice) in slices.iter().enumerate() { + if idx > 0 { + combined.push_str("\n\n"); + } + combined.push_str(&slice.content); + } + combined + } + }; + + let wrapped: Vec = slices + .iter() + .map(|slice| { + let content = slice.content.clone(); + SliceWithPositions { + content, + positions: slice.positions.clone(), + is_image: content_looks_like_image_reference(&slice.content), + } + }) + .collect(); + let embeddings = vec![None; wrapped.len()]; + self.finish_file_processing( + file, + wrapped, + embeddings, + &derived_full_content, + "Custom parse processed successfully", + None, + summary, + timing, + ) + .await + } + + async fn insert_custom_pdf_contents( + &self, file_id: i64, content_list: &[ContentItem], timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result<()> { + let valid_content_items: Vec = + content_list.iter().filter(|item| item.typ != "discarded").cloned().collect(); + timed_step_opt(timing, "custom_write_pdf_contents", async { + let rows = Self::content_items_to_pdf_rows(&valid_content_items); + crate::pdf_content::write(file_id, &rows).await + }) + .await + } + + fn content_items_to_pdf_rows(items: &[ContentItem]) -> Vec { + items + .iter() + .map(|item| PdfContentRow { + page_idx: item.page_idx, + bbox: if item.bbox.is_empty() { None } else { serde_json::to_string(&item.bbox).ok() }, + text: item.text.clone(), + text_level: item.text_level, + img_path: item.img_path.clone(), + table_body: item.table_body.clone(), + }) + .collect() + } + + async fn save_custom_images( + &self, images: &HashMap, timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result<()> { + timed_step_opt(timing, "custom_save_images", async { + let cfg = config::get(); + fs::create_dir_all(&cfg.storage.images_path).await?; + info!("custom image count: {}", images.len()); + for (img_name, img_base64) in images { + let payload = match img_base64.find("base64,") { + Some(idx) => &img_base64[idx + "base64,".len()..], + None => img_base64.as_str(), + }; + let preview: String = payload.chars().take(32).collect(); + debug!("Decoding custom image {} (len={}, preview=\"{}\")", img_name, payload.len(), preview); + let bytes = STANDARD.decode(payload).map_err(|err| { + error!( + "Failed to decode custom image {} (len={}, preview=\"{}\"): {}", + img_name, + payload.len(), + preview, + err + ); + anyhow::anyhow!(err) + })?; + let Some(image_path) = resolve_image_storage_path(img_name) else { + anyhow::bail!("Custom parse returned unsafe image path: {}", img_name); + }; + if let Some(parent) = std::path::Path::new(&image_path).parent() { + fs::create_dir_all(parent).await?; + } + fs::write(image_path, bytes).await?; + } + Ok(()) + }) + .await + } + + /// 调用外部服务将 Word/PPT 文档转换为 PDF + async fn convert_office_to_pdf(&self, file: &File) -> anyhow::Result { + let cfg = config::get(); + let pdf_dir = std::path::Path::new(&cfg.storage.pdf_path); + fs::create_dir_all(pdf_dir).await?; + + let pdf_filename = format!("{}.pdf", file.id); + let stored_pdf_path = pdf_dir.join(&pdf_filename); + let temp_pdf_path = pdf_dir.join(format!(".{}.pdf.tmp", file.id)); + let mime_type = mime_guess::from_path(&file.filename).first_or_octet_stream().essence_str().to_string(); + let mut convert_url = reqwest::Url::parse(&cfg.services.office_convert_url)?; + if !convert_url.query_pairs().any(|(key, _)| key == "target_format") { + convert_url.query_pairs_mut().append_pair("target_format", "pdf"); + } + + let mut last_error: Option = None; + for attempt in 0..2 { + // 每次重试重新打开文件流，避免流式 body 不可复用。 + let file_part = stream_file_part(&file.path, &file.filename, &mime_type).await?; + let form = multipart::Form::new().part("file", file_part); + + let client = self.services_http_client()?; + let mut response = match client.post(convert_url.clone()).multipart(form).send().await { + Ok(response) => response, + Err(err) => { + last_error = Some(format!("Office convert API request failed (url={}): {}", convert_url, err)); + if attempt == 0 { + warn!( + "Office convert API request failed, retrying in 3s: {}", + last_error.as_deref().unwrap_or("unknown error") + ); + time::sleep(Duration::from_secs(3)).await; + } + continue; + } + }; + + let response_url = response.url().to_string(); + let status = response.status(); + let body_len = stream_response_to_file(&mut response, &temp_pdf_path).await?; + if !status.is_success() { + let body_text = summarize_file_start(&temp_pdf_path).await; + last_error = Some(format!( + "Office convert API failed (status={}, url={}, body_len={}): {}", + status.as_u16(), + response_url, + body_len, + body_text + )); + } else if body_len == 0 { + last_error = Some(format!("Office convert API returned empty PDF body (url={})", response_url)); + } else if !file_starts_with(&temp_pdf_path, b"%PDF-").await { + let body_text = summarize_file_start(&temp_pdf_path).await; + last_error = Some(format!( + "Office convert API returned non-PDF body (status={}, url={}, body_len={}): {}", + status.as_u16(), + response_url, + body_len, + body_text + )); + } else { + fs::rename(&temp_pdf_path, &stored_pdf_path).await?; + return Ok(stored_pdf_path); + } + + let _ = fs::remove_file(&temp_pdf_path).await; + if attempt == 0 { + warn!( + "Office convert API failed, retrying in 3s: {}", + last_error.as_deref().unwrap_or("unknown error") + ); + time::sleep(Duration::from_secs(3)).await; + } + } + + let error_msg = last_error.unwrap_or_else(|| "unknown error".to_string()); + Err(anyhow::anyhow!("Failed to convert office document to PDF: {}", error_msg)) + } + + /// 将 Word/PPT 文档转换为 PDF 并处理 + async fn convert_office_to_pdf_and_process( + &self, file: &File, mut timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result<()> { + let stored_pdf_path = + timed_step_opt(timing.as_deref_mut(), "convert_office_to_pdf", self.convert_office_to_pdf(file)).await?; + + // 创建临时 File 结构用于处理 PDF + let mut temp_file = file.clone(); + temp_file.path = stored_pdf_path.to_string_lossy().to_string(); + temp_file.filename = format!("{}.pdf", file.id); + + // 使用 process_pdf_file 处理转换后的 PDF + self.process_pdf_file(&temp_file, None, false, Some(file.filename.as_str()), timing).await + } + + /// 处理 Excel 文件，按 sheet+行 生成切片 + async fn process_excel_file(&self, file: &File, mut timing: Option<&mut ParseTimingCtx>) -> anyhow::Result<()> { + if !self.ensure_file_exists(file.id, "excel processing start").await? { + return Ok(()); + } + info!("Processing Excel file: {}", file.filename); + + let slices = + timed_step_opt(timing.as_deref_mut(), "parse_excel", async { self.parse_excel_to_slices(file).await }) + .await?; + + if slices.is_empty() { + timed_step_opt(timing.as_deref_mut(), "finalize_empty_excel", async { + let sql = "UPDATE files SET status = 1, parse_run_id = NULL, log = ?, updated_at = strftime('%s','now') WHERE id = ?"; + sqlx::query(sql).bind("Excel processed successfully (empty)").bind(file.id).execute(&self.pool).await?; + crate::file_content::delete(file.id).await?; + Ok(()) + }) + .await?; + return Ok(()); + } + + let slice_count = slices.len(); + let full_content = slices.iter().map(|s| s.content.as_str()).collect::>().join("\n\n"); + let embeddings = vec![None; slice_count]; + self.finish_file_processing( + file, + slices, + embeddings, + &full_content, + "Excel processed successfully", + None, + None, + timing, + ) + .await + } + + /// 解析 Excel 文件，按 sheet+行 生成切片 + async fn parse_excel_to_slices(&self, file: &File) -> anyhow::Result> { + // calamine 打开/解析 Excel 是同步阻塞的 CPU+IO，放到阻塞线程池，避免阻塞 async 运行时 + let path = file.path.clone(); + let file_id = file.id; + tokio::task::spawn_blocking(move || -> anyhow::Result> { + use calamine::{Reader, open_workbook_auto}; + + let path = std::path::Path::new(&path); + let mut workbook: calamine::Sheets> = + open_workbook_auto(path).map_err(|e| anyhow::anyhow!("Failed to open Excel file: {}", e))?; + + let mut slices = Vec::new(); + let sheet_names = workbook.sheet_names().to_vec(); + + for (sheet_idx, sheet_name) in sheet_names.iter().enumerate() { + let range = match workbook.worksheet_range(sheet_name) { + Ok(r) => r, + Err(e) => { + warn!("Failed to read sheet '{}' in file {}: {}", sheet_name, file_id, e); + continue; + } + }; + + let rows: Vec<_> = range.rows().collect(); + if rows.len() < 2 { + continue; + } + + let header: Vec = rows[0] + .iter() + .enumerate() + .map(|(col_idx, cell)| { + let s = cell.to_string().trim().to_string(); + if s.is_empty() { format!("列{}", col_idx + 1) } else { s } + }) + .collect(); + + for (row_idx, row) in rows.iter().enumerate().skip(1) { + let mut lines = Vec::new(); + let mut has_data = false; + + for (col_idx, cell) in row.iter().enumerate() { + let value = cell.to_string().trim().to_string(); + if !value.is_empty() { + let header_label = + header.get(col_idx).cloned().unwrap_or_else(|| format!("列{}", col_idx + 1)); + lines.push(format!("{}: {}", header_label, value)); + has_data = true; + } + } + + if !has_data { + continue; + } + + let mut content = format!("Sheet: {}\n", sheet_name); + content.push_str(&lines.join("\n")); + + let positions = vec![SlicePosition { + page_idx: sheet_idx as i32, + bbox: [0, 0, 0, 0], + sheet_name: Some(sheet_name.clone()), + row_num: Some((row_idx + 1) as i32), + }]; + + slices.push(SliceWithPositions { content, positions, is_image: false }); + } + } + + Ok(slices) + }) + .await? + } + + /// MinerU 不支持的图片格式需要先转为 PNG；仅在转换失败时使用轻量级入库兜底。 + fn requires_mineru_image_conversion(filename_lower: &str) -> bool { + filename_lower.ends_with(".svg") || filename_lower.ends_with(".ico") || filename_lower.ends_with(".avif") + } + + /// 将 MinerU 不支持的图片临时规范化为 PNG。 + /// + /// - SVG 使用 rsvg-convert，保留矢量图的渲染结果； + /// - ICO/AVIF 使用 ImageMagick，`[0]` 选取第一帧/图层； + /// - 成功后调用方必须删除返回的临时文件。 + async fn convert_image_for_mineru(file: &File) -> anyhow::Result { + let cfg = config::get(); + let temp_dir = std::path::Path::new(&cfg.storage.temp_path); + fs::create_dir_all(temp_dir).await?; + let output_path = temp_dir.join(format!("mineru-image-{}-{}.png", file.id, uuid::Uuid::new_v4())); + let output = output_path.to_string_lossy().to_string(); + let filename_lower = file.filename.to_ascii_lowercase(); + + let result = if filename_lower.ends_with(".svg") { + Self::run_image_converter( + "rsvg-convert", + &[ + "--format".to_string(), + "png".to_string(), + "--background-color".to_string(), + "white".to_string(), + "--output".to_string(), + output.clone(), + file.path.clone(), + ], + ) + .await + } else { + let input = format!("{}[0]", file.path); + let args = vec![ + input, + "-background".to_string(), + "white".to_string(), + "-alpha".to_string(), + "remove".to_string(), + "-alpha".to_string(), + "off".to_string(), + output.clone(), + ]; + // Debian 的 ImageMagick 6 提供 convert，7 提供 magick。优先兼容两者。 + match Self::run_image_converter("magick", &args).await { + Ok(()) => Ok(()), + Err(magick_err) => { + #[cfg(not(windows))] + { + Self::run_image_converter("convert", &args) + .await + .map_err(|convert_err| anyhow::anyhow!("magick: {}; convert: {}", magick_err, convert_err)) + } + #[cfg(windows)] + { + Err(magick_err) + } + } + } + }; + + if let Err(err) = result { + let _ = fs::remove_file(&output_path).await; + return Err(err); + } + if !fs::try_exists(&output_path).await? { + anyhow::bail!("image converter did not create {}", output_path.display()); + } + Ok(output_path) + } + + async fn run_image_converter(program: &str, args: &[String]) -> anyhow::Result<()> { + let output = Command::new(program).args(args).output().await?; + if output.status.success() { + return Ok(()); + } + anyhow::bail!( + "{} exited with {}: {}", + program, + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + async fn process_uploaded_image_file( + &self, file: &File, image_embedding: Option>>, timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result<()> { + let mut description = String::new(); + if config::get().services.image_parse_url.is_some() { + match image_parse::parse_image_file(Path::new(&file.path), &file.filename, None).await { + Ok(response) => { + description = response.description; + if let Err(err) = image_description::save( + &self.pool, + file.id, + &file.filename, + &description, + &response.raw_response, + "upload", + response.jpg_filename.as_deref(), + ) + .await + { + warn!("Failed to save image description for {}: {}", file.filename, err); + } + } + Err(err) => { + // 图片文本化是增强能力。部分格式可能不受下游模型支持，仍应保留文件并完成索引。 + warn!("Failed to parse uploaded image {}, continuing without description: {}", file.filename, err); + } + } + } + + let content = if description.trim().is_empty() { + format!("图片文件：{}", file.filename) + } else { + format!("图片文件：{}\n{}", file.filename, description) + }; + let slices = vec![SliceWithPositions { content: content.clone(), positions: Vec::new(), is_image: true }]; + self.finish_file_processing( + file, + slices, + vec![image_embedding], + &content, + "Image processed successfully (direct format)", + None, + None, + timing, + ) + .await + } + + /// 处理 PDF 文件，调用 MinerU API + async fn process_pdf_file( + &self, file: &File, image_embedding: Option>>, is_image: bool, index_filename: Option<&str>, + mut timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result<()> { + if !self.ensure_file_exists(file.id, "pdf processing start").await? { + return Ok(()); + } + info!("Processing PDF file: {}", file.filename); + + let existing_rows = timed_step_opt(timing.as_deref_mut(), "pdf_existing_content_check", async { + crate::pdf_content::read(file.id).await + }) + .await?; + + let mut content_list: Vec; + let mut mineru_images: HashMap = HashMap::new(); + + if !existing_rows.is_empty() + && existing_rows.iter().any(|row| row.bbox.as_deref().is_some_and(|v| !v.is_empty())) + { + info!("Found existing PDF contents for file {}, skipping MinerU API call", file.id); + + content_list = timed_step_opt(timing.as_deref_mut(), "load_existing_pdf_content", async { + let content_list = existing_rows + .iter() + .map(|row| { + let typ = if row.text.is_some() { + "text".to_string() + } else if row.img_path.is_some() { + "image".to_string() + } else if row.table_body.is_some() { + "table".to_string() + } else { + "unknown".to_string() + }; + + let bbox = row + .bbox + .as_ref() + .and_then(|bbox| serde_json::from_str::>(bbox).ok()) + .unwrap_or_default(); + + ContentItem { + typ, + bbox, + page_idx: row.page_idx, + text: row.text.clone(), + text_level: row.text_level, + text_format: None, + img_path: row.img_path.clone(), + image_caption: None, + table_body: row.table_body.clone(), + table_caption: None, + } + }) + .collect(); + Ok(content_list) + }) + .await?; + } else { + // 没有数据，调用 MinerU API + let mineru_result = self.call_mineru_api(file, is_image, timing.as_deref_mut()).await?; + mineru_images = mineru_result.images.clone(); + content_list = timed_step_opt(timing.as_deref_mut(), "parse_mineru_content_list", async { + Ok(serde_json::from_str(&mineru_result.content_list)?) + }) + .await?; + if !self.ensure_file_exists(file.id, "before writing pdf contents").await? { + return Ok(()); + } + + // 提取文本内容并过滤掉 discarded 项 + let valid_content_items: Vec = + content_list.iter().filter(|item| item.typ != "discarded").cloned().collect(); + + // 只有在有有效内容时才插入数据库 + if !valid_content_items.is_empty() { + timed_step_opt(timing.as_deref_mut(), "write_pdf_contents", async { + let rows = Self::content_items_to_pdf_rows(&valid_content_items); + crate::pdf_content::write(file.id, &rows).await + }) + .await?; + } + + // 保存图片到本地 + timed_step_opt(timing.as_deref_mut(), "save_mineru_images", async { + let cfg = config::get(); + fs::create_dir_all(&cfg.storage.images_path).await?; + info!("image count: {}", mineru_result.images.len()); + info!("images: {:?}", mineru_result.images.keys()); + for (img_name, img_base64) in &mineru_result.images { + // 保存图片，如果以 data:image/jpeg;base64, 开头就去掉，没有也不报错 + let base64_marker = "base64,"; + let (prefix, payload) = match img_base64.find(base64_marker) { + Some(idx) => { + (&img_base64[..idx + base64_marker.len()], &img_base64[idx + base64_marker.len()..]) + } + None => ("(raw)", img_base64.as_str()), + }; + let preview: String = payload.chars().take(32).collect(); + debug!( + "Decoding mineru image {} for file {} (prefix={}, len={}, preview=\"{}\")", + img_name, + file.id, + prefix, + payload.len(), + preview + ); + let bytes = STANDARD.decode(payload).map_err(|err| { + error!( + "Failed to decode mineru image {} for file {} (prefix={}, len={}, preview=\"{}\"): {}", + img_name, + file.id, + prefix, + payload.len(), + preview, + err + ); + anyhow::anyhow!(err) + })?; + fs::write(format!("{}/{}", cfg.storage.images_path, img_name), bytes).await?; + } + Ok(()) + }) + .await?; + } + + // 图片文本化：解析 MinerU/历史图片，持久化描述并 enrich content_list。 + let image_names: Vec = content_list + .iter() + .filter_map(|item| if item.typ == "image" { item.img_path.clone() } else { None }) + .collect(); + let mut desc_map = if existing_rows.is_empty() { + parse_images_to_descriptions( + &self.pool, + file.id, + &mineru_images, + "mineru", + (!is_image).then_some(file.filename.as_str()), + ) + .await? + } else { + load_or_parse_image_descriptions(&self.pool, file.id, &image_names, "existing").await? + }; + + // 上传的图片文件本身也需要被文本化，并生成一个图片切片。 + if is_image && config::get().services.image_parse_url.is_some() && !file.filename.is_empty() { + if !desc_map.contains_key(&file.filename) { + match image_parse::parse_image_file(Path::new(&file.path), &file.filename, None).await { + Ok(resp) => { + if let Err(err) = image_description::save( + &self.pool, + file.id, + &file.filename, + &resp.description, + &resp.raw_response, + "upload", + resp.jpg_filename.as_deref(), + ) + .await + { + warn!("Failed to save upload image description for {}: {}", file.filename, err); + } + desc_map.insert(file.filename.clone(), resp.description); + } + Err(err) => warn!("Failed to parse upload image {}: {}", file.filename, err), + } + } + if !content_list.iter().any(|item| item.typ == "image" && item.img_path.as_deref() == Some(&file.filename)) + { + let caption = desc_map.get(&file.filename).cloned().unwrap_or_default(); + content_list.push(ContentItem { + typ: "image".to_string(), + bbox: vec![], + page_idx: 0, + text: None, + text_level: None, + text_format: None, + img_path: Some(file.filename.clone()), + image_caption: if caption.is_empty() { + None + } else { + Some(vec![caption]) + }, + table_body: None, + table_caption: None, + }); + } + } + + enrich_content_list_with_image_descriptions(&mut content_list, &desc_map); + + // 构建全文与切片均为 CPU 密集操作，合并放到阻塞线程池执行，避免阻塞异步运行时。 + // content_list 在此之后不再使用，直接移入闭包。 + let slice_type = file.slice_type.clone(); + let (full_content, slices) = timed_step_opt(timing.as_deref_mut(), "slice_build", async { + tokio::task::spawn_blocking(move || -> anyhow::Result<(String, Vec)> { + let (full_content, full_segments) = Self::build_full_content_and_segments(&content_list); + let slices = if slice_type == "smart" || slice_type.is_empty() { + // 智能切片：使用 content_list + Self::smart_slice_content_with_positions(&content_list)? + } else if slice_type == "fixed" { + Self::fixed_slice_content_with_positions(&full_content, &full_segments)? + } else { + Self::slice_content(&full_content, &slice_type)? + .into_iter() + .map(|content| { + let is_image = content_looks_like_image_reference(&content); + SliceWithPositions { content, positions: vec![], is_image } + }) + .collect() + }; + Ok((full_content, slices)) + }) + .await + .map_err(|e| anyhow::anyhow!("slice task failed: {e}"))? + }) + .await?; + + if !self.ensure_file_exists(file.id, "before writing slices").await? { + return Ok(()); + } + + let slice_count = slices.len(); + let embeddings = vec![image_embedding.clone(); slice_count]; + self.finish_file_processing( + file, + slices, + embeddings, + &full_content, + "PDF processed successfully", + index_filename, + None, + timing, + ) + .await + } + + async fn call_mineru_api( + &self, file: &File, is_image: bool, mut timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result { + let cfg = config::get(); + + if !is_image && cfg.services.mineru_max_pages > 0 { + // lopdf 解析整个 PDF 是同步阻塞操作（仅用于获取页数），放到阻塞线程池 + let path = file.path.clone(); + let page_count = + tokio::task::spawn_blocking(move || Document::load(&path).map(|doc| doc.get_pages().len())).await?; + match page_count { + Ok(total_pages) => { + if total_pages > cfg.services.mineru_max_pages { + return timed_step_opt(timing.as_deref_mut(), "mineru_api_in_ranges", async { + self.call_mineru_api_in_ranges(file, total_pages, cfg.services.mineru_max_pages).await + }) + .await; + } + } + Err(e) => { + warn!( + "Failed to read PDF page count for {}: {}, falling back to single MinerU request", + file.filename, e + ); + } + } + } + + timed_step_opt(timing, "mineru_api", async { + self.call_mineru_api_with_path(&file.path, &file.filename, is_image, None, None).await + }) + .await + } + + async fn call_mineru_api_in_ranges( + &self, file: &File, total_pages: usize, max_pages: usize, + ) -> anyhow::Result { + let total_parts = total_pages.div_ceil(max_pages); + info!( + "PDF {} has {} pages, calling MinerU in {} ranges (max {} pages per range)", + file.filename, total_pages, total_parts, max_pages + ); + + let mut merged_items: Vec = Vec::new(); + let mut merged_images: HashMap = HashMap::new(); + + for (part_index, range_start) in (0..total_pages).step_by(max_pages).enumerate() { + let range_end = std::cmp::min(range_start + max_pages, total_pages) - 1; + let chunk_filename = format!("{}_part_{}.pdf", file.filename, part_index + 1); + + let chunk_result = self + .call_mineru_api_with_path(&file.path, &chunk_filename, false, Some(range_start), Some(range_end)) + .await?; + let mut chunk_items: Vec = serde_json::from_str(&chunk_result.content_list)?; + + let image_prefix = format!("part{}_", part_index + 1); + let min_page_idx = chunk_items.iter().map(|item| item.page_idx).min(); + let needs_offset = min_page_idx.is_some_and(|min| min < range_start as i32); + for item in &mut chunk_items { + if needs_offset { + item.page_idx += range_start as i32; + } + if let Some(img_path) = item.img_path.as_deref() { + item.img_path = Some(Self::prefix_image_path(img_path, &image_prefix)); + } + } + + for (img_name, img_base64) in chunk_result.images { + let prefixed = Self::prefix_image_path(&img_name, &image_prefix); + merged_images.insert(prefixed, img_base64); + } + + merged_items.extend(chunk_items); + } + + let content_list = serde_json::to_string(&merged_items)?; + Ok(Result { content_list, images: merged_images }) + } + + fn prefix_image_path(img_path: &str, prefix: &str) -> String { + match img_path.rsplit_once('/') { + Some((dir, name)) => format!("{}/{}{}", dir, prefix, name), + None => format!("{}{}", prefix, img_path), + } + } + + async fn call_mineru_api_with_path( + &self, file_path: &str, filename: &str, is_image: bool, start_page: Option, end_page: Option, + ) -> anyhow::Result { + let cfg = config::get(); + let mineru_url = cfg.services.mineru_url.trim_end_matches('/'); + + let client = self.services_http_client()?; + if mineru_url.ends_with("/file_parse") { + // 构建 multipart form + let mime_type = if is_image { + mime_guess::from_path(filename).first_or_octet_stream().essence_str().to_string() + } else { + "application/pdf".to_string() + }; + + let file_part = stream_file_part(file_path, filename, &mime_type).await?; + let mut form = multipart::Form::new() + .text("return_middle_json", "false") + .text("return_model_output", "false") + .text("return_md", "false") + .text("return_images", "true") + .text("parse_method", "auto") + .text("lang_list", "ch") + .text("output_dir", "") + .text("server_url", "string") + .text("return_content_list", "true") + .text("backend", "pipeline") + .text("table_enable", "true") + .text("response_format_zip", "false") + .text("formula_enable", "true") + .part("files", file_part); + let start_page_id = start_page.unwrap_or(0); + let end_page_id = end_page.map(|v| v.to_string()).unwrap_or_else(|| "99999".to_string()); + form = form.text("start_page_id", start_page_id.to_string()).text("end_page_id", end_page_id); + + // 调用 MinerU API + let response = client.post(mineru_url).multipart(form).send().await?; + let status = response.status(); + let body_bytes = response.bytes().await?; + if !status.is_success() { + let error_text = String::from_utf8_lossy(&body_bytes); + return Err(anyhow::anyhow!("MinerU API failed: {}", error_text)); + } + + let mineru_response: MinerUResponse = serde_json::from_slice(&body_bytes).map_err(|e| { + let body_text = String::from_utf8_lossy(&body_bytes); + anyhow::anyhow!("MinerU API response decode failed: {} - {}", e, body_text) + })?; + let mineru_result = match mineru_response.results { + MinerUResults::Map(results) => { + results.into_values().next().ok_or_else(|| anyhow::anyhow!("MinerU API returned empty results"))? + } + MinerUResults::List(results) => { + if let Some(item) = results.iter().find(|item| item.status == "error") { + let mut error_msg = item.error.clone(); + if error_msg.is_empty() { + error_msg = "MinerU API returned error result".to_string(); + } + if !item.filename.is_empty() { + error_msg = format!("{} (file: {})", error_msg, item.filename); + } + return Err(anyhow::anyhow!("MinerU API failed: {}", error_msg)); + } + let first = results + .into_iter() + .next() + .ok_or_else(|| anyhow::anyhow!("MinerU API returned empty results"))?; + Result { content_list: first.content_list, images: first.images } + } + }; + return Ok(mineru_result); + } + + let mut url = reqwest::Url::parse(mineru_url)?; + { + let mut query = url.query_pairs_mut(); + query.append_pair("backend", "pipeline"); + query.append_pair("method", "auto"); + query.append_pair("lang", "ch"); + query.append_pair("start_page", &start_page.unwrap_or(0).to_string()); + if let Some(end_page) = end_page { + query.append_pair("end_page", &end_page.to_string()); + } + query.append_pair("formula_enable", "true"); + query.append_pair("table_enable", "true"); + } + + let mime_type = mime_guess::from_path(filename).first_or_octet_stream().essence_str().to_string(); + let file_part = stream_file_part(file_path, filename, &mime_type).await?; + let form = multipart::Form::new().part("file", file_part); + + let response = client.post(url.clone()).multipart(form).send().await?; + if !response.status().is_success() { + let error_text = response.text().await?; + return Err(anyhow::anyhow!("MinerU API failed: {}", error_text)); + } + + let analyze_response: AnalyzePdfResponse = response.json().await?; + if analyze_response.code != 200 { + return Err(anyhow::anyhow!("MinerU API failed: {}", analyze_response.message)); + } + + let Some(data) = analyze_response.data else { + return Err(anyhow::anyhow!("MinerU API returned empty data")); + }; + + let host = url.host_str().ok_or_else(|| anyhow::anyhow!("MinerU API URL missing host"))?; + let origin = if let Some(port) = url.port() { + format!("{}://{}:{}", url.scheme(), host, port) + } else { + format!("{}://{}", url.scheme(), host) + }; + + let mut content_list = data.content_list; + + // 去重收集需要下载的图片（filename -> url） + let mut to_download: HashMap = HashMap::new(); + for item in &content_list { + let Some(img_path) = item.img_path.as_deref() else { continue }; + let filename = std::path::Path::new(img_path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(img_path) + .to_string(); + to_download + .entry(filename) + .or_insert_with(|| format!("{}/output/{}", origin, img_path.trim_start_matches('/'))); + } + + // 并发下载图片（限制并发数，避免打爆 MinerU），各图片之间无依赖。 + use futures::stream::{StreamExt, TryStreamExt}; + let images: HashMap = + futures::stream::iter(to_download.into_iter().map(|(filename, image_url)| { + let client = client.clone(); + async move { + let image_response = client.get(&image_url).send().await?; + if !image_response.status().is_success() { + let error_text = image_response.text().await?; + return Err(anyhow::anyhow!("MinerU image download failed: {}", error_text)); + } + let image_bytes = image_response.bytes().await?; + let image_mime = mime_guess::from_path(&filename).first_or_octet_stream().essence_str().to_string(); + let image_base64 = STANDARD.encode(image_bytes); + anyhow::Ok((filename, format!("data:{};base64,{}", image_mime, image_base64))) + } + })) + .buffer_unordered(8) + .try_collect() + .await?; + + // 回填 img_path 为去掉目录的文件名 + for item in &mut content_list { + if let Some(img_path) = item.img_path.as_deref() { + let filename = std::path::Path::new(img_path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(img_path) + .to_string(); + item.img_path = Some(filename); + } + } + + let content_list_json = serde_json::to_string(&content_list)?; + Ok(Result { content_list: content_list_json, images }) + } + + /// 流式读取文本文件，边读边分片，避免一次性把整个文件载入内存。 + async fn stream_text_file(path: &str, slice_type: &str) -> anyhow::Result<(String, Vec)> { + use tokio::io::BufReader; + + let file = tokio::fs::File::open(path).await?; + let mut reader = BufReader::new(file); + let mut full_content = String::new(); + let mut slices = Vec::new(); + + match slice_type { + "paragraph" => { + let mut current = String::new(); + loop { + let mut line = String::new(); + let n = reader.read_line(&mut line).await?; + if n == 0 { + break; + } + + let (content, term) = if line.ends_with("\r\n") { + let c = line.drain(..line.len() - 2).collect::(); + (c, "\r\n") + } else if line.ends_with('\n') { + let c = line.drain(..line.len() - 1).collect::(); + (c, "\n") + } else { + (line, "") + }; + + full_content.push_str(&content); + full_content.push_str(term); + + if content.trim().is_empty() { + if !current.trim().is_empty() { + slices.push(current.trim().to_string()); + current.clear(); + } + } else { + if !current.is_empty() { + current.push_str(term); + } + current.push_str(&content); + } + } + if !current.trim().is_empty() { + slices.push(current.trim().to_string()); + } + } + "sentence" => { + const SENTENCE_END: [char; 6] = ['。', '.', '?', '!', '？', '！']; + let mut buf = String::new(); + loop { + let mut line = String::new(); + let n = reader.read_line(&mut line).await?; + if n == 0 { + break; + } + + let (content, term) = if line.ends_with("\r\n") { + let c = line.drain(..line.len() - 2).collect::(); + (c, "\r\n") + } else if line.ends_with('\n') { + let c = line.drain(..line.len() - 1).collect::(); + (c, "\n") + } else { + (line, "") + }; + + full_content.push_str(&content); + full_content.push_str(term); + + for ch in content.chars().chain(term.chars()) { + if SENTENCE_END.contains(&ch) { + if !buf.trim().is_empty() { + slices.push(buf.trim().to_string()); + } + buf.clear(); + } else { + buf.push(ch); + } + } + } + if !buf.trim().is_empty() { + slices.push(buf.trim().to_string()); + } + } + _ => { + let cfg = config::get(); + let chunk_size = cfg.slice.smart_slice_max_chars; + let overlap = cfg.slice.fixed_slice_overlap_chars; + let mut buf: Vec = Vec::new(); + + loop { + let mut line = String::new(); + let n = reader.read_line(&mut line).await?; + if n == 0 { + break; + } + + let (content, term) = if line.ends_with("\r\n") { + let c = line.drain(..line.len() - 2).collect::(); + (c, "\r\n") + } else if line.ends_with('\n') { + let c = line.drain(..line.len() - 1).collect::(); + (c, "\n") + } else { + (line, "") + }; + + full_content.push_str(&content); + full_content.push_str(term); + + for ch in content.chars().chain(term.chars()) { + buf.push(ch); + if buf.len() >= chunk_size { + let slice: String = buf[..chunk_size].iter().collect(); + slices.push(slice); + if overlap >= chunk_size || overlap == 0 { + buf.clear(); + } else { + let keep = buf.len() - overlap; + buf.drain(0..keep); + } + } + } + } + + while !buf.is_empty() { + let end = buf.len().min(chunk_size); + let slice: String = buf[..end].iter().collect(); + slices.push(slice); + if end == buf.len() { + break; + } + if overlap >= end || overlap == 0 { + break; + } + let keep = buf.len() - overlap; + buf.drain(0..keep); + } + } + } + + Ok((full_content, slices)) + } + + /// 处理普通文本文件 + async fn process_text_file(&self, file: &File, mut timing: Option<&mut ParseTimingCtx>) -> anyhow::Result<()> { + if !self.ensure_file_exists(file.id, "before reading text").await? { + return Ok(()); + } + let (content, slices) = timed_step_opt(timing.as_deref_mut(), "read_text_file", async { + Self::stream_text_file(&file.path, &file.slice_type).await + }) + .await?; + self.process_plain_text_content(file, content, slices, "Processing completed successfully", timing).await + } + + async fn process_audio_file(&self, file: &File, mut timing: Option<&mut ParseTimingCtx>) -> anyhow::Result<()> { + info!("Processing audio file: {}", file.filename); + + if !self.ensure_file_exists(file.id, "before reading audio").await? { + return Ok(()); + } + let mime_type = mime_guess::from_path(&file.filename).first_or_octet_stream().essence_str().to_string(); + let file_part = timed_step_opt( + timing.as_deref_mut(), + "open_audio_file", + stream_file_part(&file.path, &file.filename, &mime_type), + ) + .await?; + + let form = multipart::Form::new().part("file", file_part); + + let client = self.services_http_client()?; + let cfg = config::get(); + let mut req_builder = client.post(&cfg.services.audio_transcription_url).multipart(form); + if let Some(key) = &cfg.services.audio_transcription_key + && !key.is_empty() + { + req_builder = req_builder.header("Authorization", format!("Bearer {}", key)); + } + + let response = + timed_step_opt(timing.as_deref_mut(), "audio_transcription_api", async { Ok(req_builder.send().await?) }) + .await?; + if !response.status().is_success() { + let error_text = response.text().await?; + return Err(anyhow::anyhow!("Audio transcription API failed: {}", error_text)); + } + + let AudioTranscriptionResponse { text, language } = response.json().await?; + let log_message = if language.is_empty() { + "Audio processed successfully".to_string() + } else { + format!("Audio processed successfully (language: {})", language) + }; + + let text = timed_step_opt(timing.as_deref_mut(), "audio_transcription", async { Ok(text) }).await?; + let slices = timed_step_opt(timing.as_deref_mut(), "slice_build", async { + let content_owned = text.clone(); + let slice_type = file.slice_type.clone(); + tokio::task::spawn_blocking(move || Self::slice_content(&content_owned, &slice_type)) + .await + .map_err(|e| anyhow::anyhow!("slice_content task failed: {e}"))? + }) + .await?; + + self.process_plain_text_content(file, text, slices, &log_message, timing).await + } + + async fn process_plain_text_content( + &self, file: &File, content: String, slices: Vec, log_message: &str, + timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result<()> { + let wrapped: Vec = slices + .into_iter() + .map(|content| { + let is_image = content_looks_like_image_reference(&content); + SliceWithPositions { content, positions: vec![], is_image } + }) + .collect(); + let embeddings = vec![None; wrapped.len()]; + self.finish_file_processing(file, wrapped, embeddings, &content, log_message, None, None, timing).await + } + + /// 标记文件处理失败 + async fn mark_file_failed(&self, file_id: i64, error_msg: &str) -> anyhow::Result<()> { + let sql = "UPDATE files SET status = -1, parse_run_id = NULL, log = ?, updated_at = strftime('%s','now') WHERE id = ?"; + sqlx::query(sql).bind(error_msg).bind(file_id).execute(&self.pool).await?; + Ok(()) + } + + async fn mark_file_storage_skipped(&self, file_id: i64) -> anyhow::Result<()> { + let sql = + "UPDATE files SET status = 3, parse_run_id = NULL, log = ?, updated_at = strftime('%s','now') WHERE id = ?"; + sqlx::query(sql).bind("Storage mode: not parsed").bind(file_id).execute(&self.pool).await?; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + async fn finish_file_processing( + &self, file: &File, slices: Vec, embeddings: Vec>>>, + full_content: &str, log_message: &str, index_name_override: Option<&str>, summary: Option<&str>, + mut timing: Option<&mut ParseTimingCtx>, + ) -> anyhow::Result<()> { + if !self.ensure_file_exists(file.id, "before writing slices").await? { + return Ok(()); + } + let slice_count = slices.len(); + anyhow::ensure!( + embeddings.len() == slice_count, + "embeddings count {} does not match slice count {}", + embeddings.len(), + slice_count + ); + + let parse_run_id: String = sqlx::query_scalar( + "SELECT parse_run_id FROM files WHERE id = ? AND status = 2 AND parse_run_id IS NOT NULL", + ) + .bind(file.id) + .fetch_one(&self.pool) + .await?; + + let (search_docs, search_embeddings) = timed_step_opt(timing.as_deref_mut(), "insert_slices", async { + let persisted = self.insert_slices_and_positions(file.id, &parse_run_id, slices).await?; + let mut docs = Vec::with_capacity(persisted.len()); + let mut embs = Vec::with_capacity(persisted.len()); + for (idx, (id, content, is_image)) in persisted.into_iter().enumerate() { + docs.push(tantivy_engine::Document::new(id, file.id, file.kb_id, content).with_is_image(is_image)); + embs.push(embeddings.get(idx).cloned().flatten()); + } + Ok((docs, embs)) + }) + .await?; + + if !search_docs.is_empty() { + timed_step_opt(timing.as_deref_mut(), "write_search_batch", async { + self.search_engine.write_batch(search_docs, search_embeddings).await?; + Ok(()) + }) + .await?; + } + + if !self.ensure_file_exists(file.id, "before writing full index").await? { + return Ok(()); + } + let index_name = index_name_override.unwrap_or(&file.filename); + let index_full_content = format!("{}\n\n{}", index_name, full_content); + timed_step_opt(timing.as_deref_mut(), "write_full_index", async { + self.search_engine + .write_full(tantivy_engine::Document::new(file.id, file.id, file.kb_id, index_full_content)) + .await?; + Ok(()) + }) + .await?; + + if !self.ensure_file_exists(file.id, "before updating status").await? { + return Ok(()); + } + timed_step_opt(timing.as_deref_mut(), "publish_parse_artifact", async { + let artifact_id = self.ensure_parse_artifact(file, full_content, summary).await?; + self.adopt_existing_artifact_if_needed(file, artifact_id, full_content, summary).await?; + Ok(()) + }) + .await?; + let normalized_summary = summary.and_then(|value| { + let trimmed = value.trim(); + if trimmed.is_empty() { None } else { Some(trimmed.to_string()) } + }); + timed_step_opt(timing.as_deref_mut(), "write_summary_index", async { + self.persist_file_summary(file.id, file.kb_id, normalized_summary.as_deref()).await + }) + .await?; + timed_step_opt(timing.as_deref_mut(), "finalize_file_status", async { + let sql = "UPDATE files SET status = 1, parse_run_id = NULL, log = ?, summary = ?, updated_at = strftime('%s','now') \ + WHERE id = ? AND status = 2 AND parse_run_id = ?"; + let updated = sqlx::query(sql) + .bind(log_message) + .bind(normalized_summary.as_deref()) + .bind(file.id) + .bind(&parse_run_id) + .execute(&self.pool) + .await? + .rows_affected(); + anyhow::ensure!(updated == 1, "parse run for file {} is no longer current", file.id); + crate::file_content::write(file.id, full_content).await?; + Ok(()) + }) + .await?; + + info!("File {} processed successfully with {} slices", file.id, slice_count); + + self.search_engine.reload_readers()?; + + timed_step_opt(timing, "build_knowledge_graph", async { + self.maybe_build_knowledge_graph(file).await; + Ok(()) + }) + .await?; + + Ok(()) + } + + /// 返回每个 slice 的 (id, content, is_image) 供后续构建搜索文档使用。 + async fn insert_slices_and_positions( + &self, file_id: i64, parse_run_id: &str, slices: Vec, + ) -> anyhow::Result> { + if slices.is_empty() { + return Ok(Vec::new()); + } + // 每批最多 500 条 slice 一个事务，避免长时间持有 SQLite 写锁阻塞其他写操作 + const SLICE_TX_BATCH: usize = 500; + let mut persisted: Vec<(i64, String, bool)> = Vec::with_capacity(slices.len()); + for (chunk_idx, slice_chunk) in slices.chunks(SLICE_TX_BATCH).enumerate() { + let mut tx = self.pool.begin().await?; + let mut chunk_persisted: Vec<(i64, String, bool)> = Vec::with_capacity(slice_chunk.len()); + let mut position_rows: Vec<(i64, SlicePosition)> = Vec::new(); + let binds_per_row = 4_usize; + let max_vars = 999_usize; + let batch_size = std::cmp::max(1, max_vars / binds_per_row); + for (insert_chunk_idx, insert_chunk) in slice_chunk.chunks(batch_size).enumerate() { + let insert_offset = insert_chunk_idx * batch_size; + let mut slice_sql = + QueryBuilder::::new("INSERT INTO slices (file_id, parse_run_id, ordinal, is_image) "); + slice_sql.push_values(insert_chunk.iter().enumerate(), |mut b, (idx, slice)| { + let ordinal = chunk_idx * SLICE_TX_BATCH + insert_offset + idx; + b.push_bind(file_id) + .push_bind(parse_run_id) + .push_bind(ordinal as i64) + .push_bind(i64::from(slice.is_image)); + }); + slice_sql.push( + " ON CONFLICT(file_id, parse_run_id, ordinal) WHERE parse_run_id IS NOT NULL AND ordinal IS NOT NULL \ + DO UPDATE SET updated_at = strftime('%s','now') RETURNING id", + ); + + let inserted_ids: Vec<(i64,)> = slice_sql.build_query_as().fetch_all(&mut *tx).await?; + anyhow::ensure!( + inserted_ids.len() == insert_chunk.len(), + "inserted slice row count mismatch: expected {}, got {}", + insert_chunk.len(), + inserted_ids.len() + ); + + for (slice, (id,)) in insert_chunk.iter().zip(inserted_ids) { + for position in &slice.positions { + position_rows.push((id, position.clone())); + } + chunk_persisted.push((id, slice.content.clone(), slice.is_image)); + } + } + if !position_rows.is_empty() { + let binds_per_row = 8_usize; + let max_vars = 999_usize; + let batch_size = std::cmp::max(1, max_vars / binds_per_row); + for chunk in position_rows.chunks(batch_size) { + let mut pos_sql = QueryBuilder::::new( + "insert into slice_positions(slice_id, page_idx, x1, y1, x2, y2, sheet_name, row_num) ", + ); + pos_sql.push_values(chunk.iter(), |mut b, (slice_id, position)| { + b.push_bind(slice_id) + .push_bind(position.page_idx) + .push_bind(position.bbox[0]) + .push_bind(position.bbox[1]) + .push_bind(position.bbox[2]) + .push_bind(position.bbox[3]) + .push_bind(&position.sheet_name) + .push_bind(position.row_num); + }); + pos_sql.build().execute(&mut *tx).await?; + } + } + tx.commit().await?; + crate::slice_content::upsert_many( + file_id, + &chunk_persisted.iter().map(|(id, content, _)| (*id, content.clone())).collect::>(), + ) + .await?; + persisted.extend(chunk_persisted); + } + Ok(persisted) + } + + async fn ensure_file_exists(&self, file_id: i64, stage: &str) -> anyhow::Result { + let exists = sqlx::query_scalar::<_, i64>("SELECT 1 FROM files WHERE id = ? LIMIT 1") + .bind(file_id) + .fetch_optional(&self.pool) + .await? + .is_some(); + if !exists { + info!("File {} no longer exists during {}, skipping further processing", file_id, stage); + } + Ok(exists) + } + + async fn is_storage_kb(&self, kb_id: Option) -> anyhow::Result { + let Some(kb_id) = kb_id else { return Ok(false) }; + let kb_type: Option = sqlx::query_scalar("SELECT kb_type FROM knowledge_bases WHERE id = ?") + .bind(kb_id) + .fetch_optional(&self.pool) + .await?; + Ok(matches!(kb_type.as_deref(), Some("storage"))) + } + + fn is_audio_file(filename_lower: &str) -> bool { + filename_lower.ends_with(".wav") + || filename_lower.ends_with(".mp3") + || filename_lower.ends_with(".m4a") + || filename_lower.ends_with(".aac") + || filename_lower.ends_with(".flac") + || filename_lower.ends_with(".ogg") + || filename_lower.ends_with(".opus") + || filename_lower.ends_with(".wma") + || filename_lower.ends_with(".amr") + || filename_lower.ends_with(".aiff") + || filename_lower.ends_with(".aif") + || filename_lower.ends_with(".alac") + || filename_lower.ends_with(".webm") + } + + /// 根据 slice_type 对内容进行分片 + fn slice_content(content: &str, slice_type: &str) -> anyhow::Result> { + match slice_type { + "paragraph" => { + // 按段落分片（以双换行符分隔） + Ok(content.split("\n\n").map(|s| s.trim()).filter(|s| !s.is_empty()).map(|s| s.to_string()).collect()) + } + "sentence" => { + // 按句子分片（简单实现：以句号、问号、感叹号分隔） + Ok(content + .split(['。', '.', '?', '!', '？', '！']) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect()) + } + _ => { + // 固定长度分片（每8000字符，重叠100字符） + let cfg = config::get(); + let chunk_size = cfg.slice.smart_slice_max_chars; + let overlap = cfg.slice.fixed_slice_overlap_chars; + + let chars: Vec = content.chars().collect(); + let mut slices = Vec::new(); + let mut start = 0; + + while start < chars.len() { + let end = std::cmp::min(start + chunk_size, chars.len()); + let slice: String = chars[start..end].iter().collect(); + slices.push(slice); + + if end >= chars.len() { + break; + } + + // 下一个切片从 end - overlap 开始，以实现重叠 + start = end.saturating_sub(overlap); + if start >= end { + break; + } + } + + Ok(slices) + } + } + } + + fn build_full_content_and_segments(content_list: &[ContentItem]) -> (String, Vec) { + let mut full_content = String::new(); + let mut segments = Vec::new(); + let mut current_len = 0usize; + + for pdf_content in content_list { + if pdf_content.typ == "discarded" { + continue; + } + let mut item_content = String::new(); + if pdf_content.typ == "text" { + if let Some(text) = &pdf_content.text { + if let Some(lv) = pdf_content.text_level { + item_content.push_str("#".repeat(lv as usize).as_str()); + item_content.push(' '); + } + item_content.push_str(text); + } + } else if pdf_content.typ == "image" { + if let Some(img_name) = &pdf_content.img_path { + item_content.push_str(&format!("![{}](/api/v1/knowledge/files/{})", img_name, img_name)); + } + if let Some(captions) = &pdf_content.image_caption { + for caption in captions { + item_content.push_str(&caption.to_string()); + } + } + } else if pdf_content.typ == "table" { + if let Some(table_caption) = &pdf_content.table_caption { + for caption in table_caption { + item_content.push_str(&caption.to_string()); + } + } + if let Some(table_body) = &pdf_content.table_body { + item_content.push_str(table_body); + } + } + + if item_content.is_empty() { + continue; + } + + let start = current_len; + full_content.push_str(&item_content); + current_len += item_content.chars().count(); + let end = current_len; + + let positions = Self::positions_from_item(pdf_content); + if !positions.is_empty() { + segments.push(Segment { start, end, positions }); + } + + full_content.push_str("\n\n"); + current_len += 2; + } + + (full_content, segments) + } + + fn fixed_slice_content_with_positions( + content: &str, segments: &[Segment], + ) -> anyhow::Result> { + let cfg = config::get(); + let chunk_size = cfg.slice.smart_slice_max_chars; + let overlap = cfg.slice.fixed_slice_overlap_chars; + + let chars: Vec = content.chars().collect(); + let mut slices = Vec::new(); + let mut start = 0; + + while start < chars.len() { + let end = std::cmp::min(start + chunk_size, chars.len()); + let slice: String = chars[start..end].iter().collect(); + let positions = Self::positions_for_range(segments, start, end); + slices.push(SliceWithPositions { + content: slice.clone(), + positions, + is_image: content_looks_like_image_reference(&slice), + }); + + if end >= chars.len() { + break; + } + + start = end.saturating_sub(overlap); + if start >= end { + break; + } + } + + Ok(slices) + } + + fn smart_slice_content_with_positions(content_list: &[ContentItem]) -> anyhow::Result> { + let cfg = config::get(); + let max_chars = cfg.slice.smart_slice_max_chars; + + let mut slices = Vec::new(); + let mut current_slice = String::new(); + let mut current_segments: Vec = Vec::new(); + let mut current_len = 0usize; + let mut current_header = String::new(); // 当前所在的标题 + let mut current_header_positions: Vec = Vec::new(); + + for item in content_list { + if item.typ == "discarded" { + continue; + } + + let mut item_content = String::new(); + let item_positions = Self::positions_from_item(item); + + // 处理文本内容 + if item.typ == "text" { + if let Some(text) = &item.text { + // 如果是标题，更新当前标题 + if let Some(level) = item.text_level { + // 这是一个标题 + let header_text = format!("{} {}", "#".repeat(level as usize), text); + + // 如果当前有累积的内容，先保存 + if !current_slice.trim().is_empty() { + Self::flush_slice_with_positions( + &mut slices, + &mut current_slice, + &mut current_segments, + max_chars, + ); + current_len = 0; + } + + // 更新当前标题 + current_header = header_text; + current_header_positions = item_positions.clone(); + // 开始新的切片，包含标题 + current_slice.clear(); + current_segments.clear(); + Self::append_segment( + &mut current_slice, + &mut current_len, + &mut current_segments, + ¤t_header, + current_header_positions.clone(), + ); + Self::append_separator(&mut current_slice, &mut current_len); + continue; + } else { + // 普通文本 + item_content.push_str(text); + } + } + } else if item.typ == "image" { + if let Some(img_name) = &item.img_path { + item_content.push_str(&format!("![{}](/api/v1/knowledge/files/{})", img_name, img_name)); + } + if let Some(captions) = &item.image_caption { + for caption in captions { + item_content.push_str(&caption.to_string()); + } + } + } else if item.typ == "table" { + if let Some(table_caption) = &item.table_caption { + for caption in table_caption { + item_content.push_str(&caption.to_string()); + } + } + if let Some(table_body) = &item.table_body { + item_content.push_str( + &table_body + .replace(" colspan=\"1\"", "") + .replace(" rowspan=\"1\"", "") + .replace(" colspan='1'", "") + .replace(" rowspan='1'", "") + .replace(" colspan=1", "") + .replace(" rowspan=1", ""), + ); + } + } + + if item_content.is_empty() { + continue; + } + + // 检查加入这个内容后是否超过字数限制 + let test_len = if current_slice.is_empty() { + // 如果当前切片为空，但有标题，先加上标题 + if !current_header.is_empty() { + current_header.chars().count() + 2 + item_content.chars().count() + } else { + item_content.chars().count() + } + } else { + current_len + item_content.chars().count() + 2 + }; + + if test_len > max_chars { + // 超过限制，保存当前切片 + if !current_slice.trim().is_empty() { + Self::flush_slice_with_positions(&mut slices, &mut current_slice, &mut current_segments, max_chars); + current_len = 0; + } + + // 开始新的切片 + current_slice.clear(); + current_segments.clear(); + if !current_header.is_empty() { + Self::append_segment( + &mut current_slice, + &mut current_len, + &mut current_segments, + ¤t_header, + current_header_positions.clone(), + ); + Self::append_separator(&mut current_slice, &mut current_len); + } + Self::append_segment( + &mut current_slice, + &mut current_len, + &mut current_segments, + &item_content, + item_positions.clone(), + ); + Self::append_separator(&mut current_slice, &mut current_len); + } else { + // 没有超过限制，继续累积 + if current_slice.is_empty() { + current_slice.clear(); + current_segments.clear(); + current_len = 0; + if !current_header.is_empty() { + Self::append_segment( + &mut current_slice, + &mut current_len, + &mut current_segments, + ¤t_header, + current_header_positions.clone(), + ); + Self::append_separator(&mut current_slice, &mut current_len); + } + Self::append_segment( + &mut current_slice, + &mut current_len, + &mut current_segments, + &item_content, + item_positions.clone(), + ); + } else { + Self::append_segment( + &mut current_slice, + &mut current_len, + &mut current_segments, + &item_content, + item_positions.clone(), + ); + Self::append_separator(&mut current_slice, &mut current_len); + } + } + } + + // 保存最后的切片 + if !current_slice.trim().is_empty() { + Self::flush_slice_with_positions(&mut slices, &mut current_slice, &mut current_segments, max_chars); + } + + Ok(slices) + } + + fn flush_slice_with_positions( + slices: &mut Vec, current_slice: &mut String, segments: &mut Vec, max_chars: usize, + ) { + if current_slice.trim().is_empty() { + return; + } + let content = std::mem::take(current_slice); + let segment_data = std::mem::take(segments); + let mut new_slices = Self::split_slice_with_positions(&content, &segment_data, max_chars); + slices.append(&mut new_slices); + } + + fn split_slice_with_positions(content: &str, segments: &[Segment], max_chars: usize) -> Vec { + let char_count = content.chars().count(); + if char_count == 0 { + return Vec::new(); + } + + let ranges = if char_count <= max_chars { + vec![(0, char_count)] + } else { + let sentence_ranges = Self::sentence_ranges(content); + if sentence_ranges.is_empty() { + vec![(0, char_count)] + } else { + let mut ranges = Vec::new(); + let mut slice_start = sentence_ranges[0].0; + let mut slice_end = sentence_ranges[0].1; + for (start, end) in sentence_ranges.iter().skip(1) { + if end - slice_start > max_chars && slice_end > slice_start { + ranges.push((slice_start, slice_end)); + slice_start = *start; + } + slice_end = *end; + } + ranges.push((slice_start, slice_end)); + ranges + } + }; + + let chars: Vec = content.chars().collect(); + ranges + .into_iter() + .filter_map(|(start, end)| { + if start >= end || end > chars.len() { + return None; + } + let slice: String = chars[start..end].iter().collect(); + let positions = Self::positions_for_range(segments, start, end); + Some(SliceWithPositions { + content: slice.clone(), + positions, + is_image: content_looks_like_image_reference(&slice), + }) + }) + .collect() + } + + fn sentence_ranges(content: &str) -> Vec<(usize, usize)> { + let mut ranges = Vec::new(); + let mut start = 0usize; + for (idx, ch) in content.chars().enumerate() { + if matches!(ch, '。' | '.' | '?' | '!' | '？' | '！') { + let end = idx + 1; + if end > start { + ranges.push((start, end)); + } + start = end; + } + } + let total = content.chars().count(); + if start < total { + ranges.push((start, total)); + } + ranges + } + + fn append_segment( + current_slice: &mut String, current_len: &mut usize, segments: &mut Vec, text: &str, + positions: Vec, + ) { + if text.is_empty() { + return; + } + let start = *current_len; + current_slice.push_str(text); + *current_len += text.chars().count(); + let end = *current_len; + if !positions.is_empty() { + segments.push(Segment { start, end, positions }); + } + } + + fn append_separator(current_slice: &mut String, current_len: &mut usize) { + current_slice.push_str("\n\n"); + *current_len += 2; + } + + fn positions_from_item(item: &ContentItem) -> Vec { + if item.bbox.len() == 4 { + let bbox = [item.bbox[0], item.bbox[1], item.bbox[2], item.bbox[3]]; + vec![SlicePosition { page_idx: item.page_idx, bbox, sheet_name: None, row_num: None }] + } else { + Vec::new() + } + } + + fn positions_for_range(segments: &[Segment], start: usize, end: usize) -> Vec { + let mut set: HashSet = HashSet::new(); + for segment in segments { + if segment.end > start && segment.start < end { + for position in &segment.positions { + set.insert(position.clone()); + } + } + } + set.into_iter().collect() + } + + async fn clone_file_data(&self, source: &File, target: &File) -> anyhow::Result<()> { + if source.id == target.id { + anyhow::bail!("Source and target file ids are identical"); + } + if source.status != 1 { + anyhow::bail!("Source file {} is not processed", source.id); + } + + // 新结构只复制搜索投影，SQLite 中的切片/PDF 解析结果由 artifact 共享。 + if config::get().server.reuse_duplicate_files { + let source_file_id = effective_parse_file_id(&self.pool, source.id).await?; + let mut source_for_reindex = source.clone(); + let full_content = if let Some(artifact_id) = source.artifact_id { + sqlx::query_scalar::<_, Option>("SELECT full_content FROM parse_artifacts WHERE id = ?") + .bind(artifact_id) + .fetch_optional(&self.pool) + .await? + .flatten() + .unwrap_or_default() + } else { + crate::file_content::read(source_file_id).await?.unwrap_or_default() + }; + let summary = if let Some(artifact_id) = source.artifact_id { + sqlx::query_scalar::<_, Option>("SELECT summary FROM parse_artifacts WHERE id = ?") + .bind(artifact_id) + .fetch_optional(&self.pool) + .await? + .flatten() + } else { + source.summary.clone() + }; + source_for_reindex.content = Some(full_content.clone()); + source_for_reindex.summary = summary.clone(); + let artifact_id = + self.ensure_parse_artifact(&source_for_reindex, &full_content, summary.as_deref()).await?; + let rows = self.fetch_slice_rows(source_file_id).await?; + let shared_slices: Vec = rows + .into_iter() + .map(|row| ClonedSlice { old_id: row.id, new_id: row.id, content: row.content }) + .collect(); + + sqlx::query( + "UPDATE files SET status = 2, artifact_id = ?, log = ?, updated_at = strftime('%s','now') WHERE id = ?", + ) + .bind(artifact_id) + .bind(format!("Reusing shared parse artifact {}", artifact_id)) + .bind(target.id) + .execute(&self.pool) + .await?; + let indexed_content = self.reindex_cloned_slices(target, &shared_slices, &source_for_reindex).await?; + sqlx::query( + "UPDATE files SET status = 1, parse_run_id = NULL, artifact_id = ?, log = ?, summary = ?, \ + updated_at = strftime('%s','now') WHERE id = ?", + ) + .bind(artifact_id) + .bind(format!("Reused shared parse artifact {} from file {}", artifact_id, source_file_id)) + .bind(summary.as_deref()) + .bind(target.id) + .execute(&self.pool) + .await?; + crate::file_content::write(target.id, &indexed_content).await?; + let mut updated_file = target.clone(); + updated_file.artifact_id = Some(artifact_id); + updated_file.content = Some(indexed_content); + updated_file.summary = summary.clone(); + self.maybe_build_knowledge_graph(&updated_file).await; + return Ok(()); + } + + let reuse_log = format!("Reusing parsed data from file {}", source.id); + sqlx::query("UPDATE files SET status = 2, log = ?, updated_at = strftime('%s','now') WHERE id = ?") + .bind(&reuse_log) + .bind(target.id) + .execute(&self.pool) + .await?; + + self.cleanup_processing_file_data_with_retry(target.id, 3).await?; + + let pdf_rows = self.fetch_pdf_content_rows(source.id).await?; + let mut slice_rows = self.fetch_slice_rows(source.id).await?; + let slice_ids: Vec = slice_rows.iter().map(|row| row.id).collect(); + let slice_positions = self.fetch_slice_position_rows(&slice_ids).await?; + let (image_jobs, image_mapping) = self.prepare_image_jobs(&pdf_rows, source.id); + let meta_image_paths = if pdf_rows.is_empty() { + collect_image_raw_paths_for_files(&self.pool, &[source.id]).await? + } else { + Vec::new() + }; + let (meta_image_jobs, meta_image_mapping, target_meta_image_paths) = + Self::prepare_raw_image_jobs(&meta_image_paths, source.id); + + // 兼容历史数据：复用时剥离旧的 f{file_id}_ 图片前缀，并同步重写内容中的引用。 + let mut combined_image_mapping = image_mapping.clone(); + combined_image_mapping.extend(meta_image_mapping.clone()); + if !combined_image_mapping.is_empty() { + for row in &mut slice_rows { + row.content = Self::rewrite_custom_image_refs(&row.content, &combined_image_mapping); + } + } + let mut source_for_reindex = source.clone(); + if !combined_image_mapping.is_empty() + && let Some(content) = crate::file_content::read(source.id).await? + { + source_for_reindex.content = Some(Self::rewrite_custom_image_refs(&content, &combined_image_mapping)); + } + + let mut tx = self.pool.begin().await?; + self.insert_pdf_rows(&mut tx, target.id, &pdf_rows, &image_mapping).await?; + let cloned_slices = self.insert_slice_rows(&mut tx, target.id, &slice_rows).await?; + self.insert_slice_positions(&mut tx, &cloned_slices, &slice_positions).await?; + tx.commit().await?; + + self.copy_image_files(&image_jobs).await?; + self.copy_image_files(&meta_image_jobs).await?; + self.copy_converted_pdf(source.id, target.id).await?; + + if pdf_rows.is_empty() { + update_file_custom_image_meta(&self.pool, target.id, &target_meta_image_paths, "reuse_custom_images") + .await?; + } + + let full_content = self.reindex_cloned_slices(target, &cloned_slices, &source_for_reindex).await?; + + self.search_engine.reload_readers()?; + + let final_log = format!("Reused parsed data from file {}", source.id); + sqlx::query( + "UPDATE files SET status = 1, parse_run_id = NULL, log = ?, summary = ?, updated_at = strftime('%s','now') WHERE id = ?", + ) + .bind(&final_log) + .bind(source.summary.as_deref()) + .bind(target.id) + .execute(&self.pool) + .await?; + crate::file_content::write(target.id, &full_content).await?; + + let mut updated_file = target.clone(); + updated_file.content = Some(full_content.clone()); + updated_file.summary = source.summary.clone(); + self.maybe_build_knowledge_graph(&updated_file).await; + + Ok(()) + } + + async fn fetch_pdf_content_rows(&self, file_id: i64) -> anyhow::Result> { + crate::pdf_content::read(file_id).await + } + + async fn fetch_slice_rows(&self, file_id: i64) -> anyhow::Result> { + let ids: Vec = sqlx::query_scalar("SELECT id FROM slices WHERE file_id = ? ORDER BY id") + .bind(file_id) + .fetch_all(&self.pool) + .await?; + let contents = crate::slice_content::read_all(file_id).await?; + Ok(ids.into_iter().map(|id| SliceRow { id, content: contents.get(&id).cloned().unwrap_or_default() }).collect()) + } + + async fn fetch_slice_position_rows(&self, slice_ids: &[i64]) -> anyhow::Result> { + if slice_ids.is_empty() { + return Ok(Vec::new()); + } + let mut all_rows = Vec::new(); + let chunk_size = 400; + for chunk in slice_ids.chunks(chunk_size) { + let mut qb = QueryBuilder::::new( + "SELECT slice_id, page_idx, x1, y1, x2, y2, sheet_name, row_num FROM slice_positions WHERE slice_id IN (", + ); + let mut separated = qb.separated(", "); + for slice_id in chunk { + separated.push_bind(slice_id); + } + qb.push(") ORDER BY slice_id, id"); + let rows = qb.build_query_as::().fetch_all(&self.pool).await?; + all_rows.extend(rows); + } + Ok(all_rows) + } + + fn prepare_image_jobs( + &self, rows: &[PdfContentRow], source_id: i64, + ) -> (Vec<(String, String)>, HashMap) { + let mut jobs = Vec::new(); + let mut mapping = HashMap::new(); + for row in rows { + if let Some(path) = &row.img_path { + if mapping.contains_key(path) { + continue; + } + let new_path = Self::remove_image_id_prefix(path, source_id); + if new_path != *path { + jobs.push((path.clone(), new_path.clone())); + mapping.insert(path.clone(), new_path); + } + } + } + (jobs, mapping) + } + + fn prepare_raw_image_jobs(paths: &[String], source_id: i64) -> RawImageJobs { + let mut jobs = Vec::new(); + let mut mapping = HashMap::new(); + let mut target_paths = Vec::new(); + let mut seen = HashSet::new(); + for path in paths { + let trimmed = path.trim(); + if trimmed.is_empty() || !seen.insert(trimmed.to_string()) { + continue; + } + let new_path = Self::remove_image_id_prefix(trimmed, source_id); + if new_path != trimmed { + mapping.insert(trimmed.to_string(), new_path.clone()); + jobs.push((trimmed.to_string(), new_path.clone())); + } + target_paths.push(new_path); + } + (jobs, mapping, target_paths) + } + + async fn insert_pdf_rows( + &self, _tx: &mut sqlx::Transaction<'_, Sqlite>, target_file_id: i64, rows: &[PdfContentRow], + image_mapping: &HashMap, + ) -> anyhow::Result<()> { + let rows = rows + .iter() + .map(|row| { + let new_img_path = row + .img_path + .as_ref() + .and_then(|path| image_mapping.get(path)) + .cloned() + .or_else(|| row.img_path.clone()); + PdfContentRow { img_path: new_img_path, ..row.clone() } + }) + .collect::>(); + crate::pdf_content::write(target_file_id, &rows).await + } + + async fn insert_slice_rows( + &self, tx: &mut sqlx::Transaction<'_, Sqlite>, target_file_id: i64, rows: &[SliceRow], + ) -> anyhow::Result> { + if rows.is_empty() { + return Ok(Vec::new()); + } + + let binds_per_row = 2_usize; + let max_vars = 999_usize; + let batch_size = std::cmp::max(1, max_vars / binds_per_row); + let mut cloned = Vec::with_capacity(rows.len()); + for chunk in rows.chunks(batch_size) { + let mut qb = QueryBuilder::::new("INSERT INTO slices (file_id) "); + qb.push_values(chunk.iter(), |mut b, _row| { + b.push_bind(target_file_id); + }); + qb.push(" RETURNING id"); + + let inserted_ids: Vec<(i64,)> = qb.build_query_as().fetch_all(&mut **tx).await?; + anyhow::ensure!( + inserted_ids.len() == chunk.len(), + "inserted cloned slice row count mismatch: expected {}, got {}", + chunk.len(), + inserted_ids.len() + ); + + for (row, (new_id,)) in chunk.iter().zip(inserted_ids) { + cloned.push(ClonedSlice { old_id: row.id, new_id, content: row.content.clone() }); + } + } + + crate::slice_content::upsert_many( + target_file_id, + &cloned.iter().map(|row| (row.new_id, row.content.clone())).collect::>(), + ) + .await?; + Ok(cloned) + } + + async fn insert_slice_positions( + &self, tx: &mut sqlx::Transaction<'_, Sqlite>, cloned_slices: &[ClonedSlice], positions: &[SlicePositionRecord], + ) -> anyhow::Result<()> { + if positions.is_empty() || cloned_slices.is_empty() { + return Ok(()); + } + let id_map: HashMap = cloned_slices.iter().map(|slice| (slice.old_id, slice.new_id)).collect(); + let filtered: Vec<&SlicePositionRecord> = + positions.iter().filter(|row| id_map.contains_key(&row.slice_id)).collect(); + if filtered.is_empty() { + return Ok(()); + } + let binds_per_row = 8; + let batch_size = std::cmp::max(1, 999 / binds_per_row); + for chunk in filtered.chunks(batch_size) { + let mut qb = QueryBuilder::::new( + "INSERT INTO slice_positions(slice_id, page_idx, x1, y1, x2, y2, sheet_name, row_num) ", + ); + qb.push_values(chunk.iter(), |mut b, row| { + let new_id = id_map[&row.slice_id]; + b.push_bind(new_id) + .push_bind(row.page_idx) + .push_bind(row.x1) + .push_bind(row.y1) + .push_bind(row.x2) + .push_bind(row.y2) + .push_bind(&row.sheet_name) + .push_bind(row.row_num); + }); + qb.build().execute(&mut **tx).await?; + } + Ok(()) + } + + async fn copy_image_files(&self, jobs: &[(String, String)]) -> anyhow::Result<()> { + for (old_path, new_path) in jobs { + let Some(src_abs) = resolve_image_storage_path(old_path) else { + warn!("Unable to resolve source image path {}, skipping reuse copy", old_path); + continue; + }; + let Some(dst_abs) = resolve_image_storage_path(new_path) else { + warn!("Unable to resolve destination image path {}, skipping reuse copy", new_path); + continue; + }; + if let Some(parent) = std::path::Path::new(&dst_abs).parent() { + fs::create_dir_all(parent).await?; + } + fs::copy(&src_abs, &dst_abs).await?; + } + Ok(()) + } + + async fn copy_converted_pdf(&self, source_id: i64, target_id: i64) -> anyhow::Result<()> { + let cfg = config::get(); + let pdf_dir = std::path::Path::new(&cfg.storage.pdf_path); + let src_pdf = pdf_dir.join(format!("{}.pdf", source_id)); + match fs::try_exists(&src_pdf).await { + Ok(true) => {} + Ok(false) => return Ok(()), + Err(err) => { + warn!("Failed to check converted PDF existence for file {}: {}, skipping copy", source_id, err); + return Ok(()); + } + } + fs::create_dir_all(pdf_dir).await?; + let dst_pdf = pdf_dir.join(format!("{}.pdf", target_id)); + fs::copy(&src_pdf, &dst_pdf).await?; + Ok(()) + } + + async fn reindex_cloned_slices( + &self, target: &File, cloned_slices: &[ClonedSlice], source: &File, + ) -> anyhow::Result { + let mut search_docs = Vec::new(); + for slice in cloned_slices { + let is_image = content_looks_like_image_reference(&slice.content); + search_docs.push( + tantivy_engine::Document::new(slice.new_id, target.id, target.kb_id, slice.content.clone()) + .with_is_image(is_image), + ); + } + + if !search_docs.is_empty() { + let filename_lower = target.filename.to_lowercase(); + let is_image_file = crate::search::is_image_file(&filename_lower); + let embeddings = if is_image_file && search::embedding::image_embedding_enabled() { + let embedding = + search::embedding::get_image_embedding_from_path(&target.path, Some(&target.filename)).await?; + let arc_embedding = Arc::new(embedding); + (0..search_docs.len()).map(|_| Some(Arc::clone(&arc_embedding))).collect() + } else { + if is_image_file { + info!("Image embedding URL not configured, skipping image embedding for reused file {}", target.id); + } + vec![None; search_docs.len()] + }; + self.search_engine.write_batch(search_docs, embeddings).await?; + } + + let full_content = if let Some(content) = source.content.clone() { + content + } else if cloned_slices.is_empty() { + String::new() + } else { + cloned_slices + .iter() + .map(|slice| slice.content.as_str()) + .filter(|content| !content.trim().is_empty()) + .map(|content| content.to_string()) + .collect::>() + .join("\n\n") + }; + + let index_full_content = if full_content.is_empty() { + target.filename.clone() + } else { + format!("{}\n\n{}", target.filename, full_content) + }; + self.search_engine + .write_full(tantivy_engine::Document::new(target.id, target.id, target.kb_id, index_full_content)) + .await?; + self.persist_file_summary(target.id, target.kb_id, source.summary.as_deref()).await?; + + Ok(full_content) + } + + fn remove_image_id_prefix(original: &str, source_id: i64) -> String { + let path = std::path::Path::new(original); + let filename = path.file_name().and_then(|name| name.to_str()).unwrap_or(original); + let prefix = format!("f{}_", source_id); + let stripped = if filename.starts_with(&prefix) { + filename.trim_start_matches(&prefix).to_string() + } else { + filename.to_string() + }; + if let Some(parent) = path.parent() { parent.join(stripped).to_string_lossy().to_string() } else { stripped } + } + + async fn maybe_build_knowledge_graph(&self, file: &File) { + if !config::get().server.build_knowledge_graph { + debug!("Knowledge graph building disabled by config, skipping file {}", file.id); + return; + } + + if let Err(e) = self.build_knowledge_graph(file).await { + error!("Failed to build knowledge graph for file {}: {}", file.id, e); + // 不影响主流程，仅记录错误 + } + } + + /// 构建知识图谱（完全由LLM生成） + async fn build_knowledge_graph(&self, file: &File) -> anyhow::Result<()> { + info!("Building knowledge graph for file {}", file.id); + + // 1. 初始化LLM图谱提取器 + let llm_extractor = LLMGraphExtractor::from_env(); + + if !llm_extractor.is_enabled() { + warn!("LLM not enabled, skipping knowledge graph building for file {}", file.id); + return Ok(()); + } + + info!("Using LLM to generate knowledge graph for file {}", file.id); + + // 2. 获取文件的所有切片 + let source_file_id = effective_parse_file_id(&self.pool, file.id).await?; + let slices = self + .fetch_slice_rows(source_file_id) + .await? + .into_iter() + .map(|row| (row.id, row.content)) + .collect::>(); + + if slices.is_empty() { + debug!("No slices found for file {}, skipping graph building", file.id); + return Ok(()); + } + + // 3. 合并所有切片内容（限制长度避免超出LLM上下文） + let mut combined_content = String::new(); + let max_content_length = 8000; // 限制总长度 + + for (_, content) in &slices { + if combined_content.len() + content.len() > max_content_length { + break; + } + combined_content.push_str(content); + combined_content.push_str("\n\n"); + } + + if combined_content.trim().is_empty() { + debug!("No content to process for file {}", file.id); + return Ok(()); + } + + // 4. 调用LLM提取知识图谱 + let context = format!("文件名: {}", file.filename); + + let (mut entities, mut relations) = + match llm_extractor.extract_knowledge_graph(&combined_content, &context).await { + Ok(result) => result, + Err(e) => { + error!("LLM knowledge graph extraction failed for file {}: {}", file.id, e); + return Err(e); + } + }; + + info!("LLM extracted {} entities and {} relations from file {}", entities.len(), relations.len(), file.id); + + // 5. 为实体和关系添加文件信息 + for entity in &mut entities { + entity.file_id = Some(file.id); + entity.kb_id = file.kb_id; + } + + for relation in &mut relations { + relation.file_id = Some(file.id); + } + + // 6. 增量直写知识图谱（避免把整个 KB 的图加载到内存） + KnowledgeGraph::incremental_update_direct(self.pool.clone(), file.kb_id, entities, relations).await?; + + // 7. 保存图快照 + KnowledgeGraph::save_snapshot_direct(&self.pool, file.kb_id).await?; + + info!("Knowledge graph updated successfully for file {} (LLM-generated)", file.id); + + Ok(()) + } +} + +pub async fn process_file_immediate(pool: SqlitePool, search_engine: SearchEngine, file_id: i64) -> anyhow::Result<()> { + if is_parse_paused() { + anyhow::bail!("parse is paused for index maintenance"); + } + let processor = FileProcessor::new(pool, search_engine, 0); + let Some(file) = processor.claim_file_by_id(file_id).await? else { + info!("File {} is not pending; immediate parse claim skipped", file_id); + return Ok(()); + }; + let result = processor.process_file_claimed(&file).await; + if let Err(err) = &result { + if let Err(cleanup_err) = processor.cleanup_processing_file_data_with_retry(file_id, 3).await { + error!("Failed to cleanup immediate parse data for file {}: {}", file_id, cleanup_err); + } + processor.mark_file_failed(file_id, &err.to_string()).await?; + } + result +} + +pub async fn process_file_immediate_skip_reuse( + pool: SqlitePool, search_engine: SearchEngine, file_id: i64, +) -> anyhow::Result<()> { + if is_parse_paused() { + anyhow::bail!("parse is paused for index maintenance"); + } + let processor = FileProcessor::new(pool, search_engine, 0); + let Some(file) = processor.claim_file_by_id(file_id).await? else { + info!("File {} is not pending; immediate parse claim skipped", file_id); + return Ok(()); + }; + let result = processor.process_file_claimed_skip_reuse(&file).await; + if let Err(err) = &result { + if let Err(cleanup_err) = processor.cleanup_processing_file_data_with_retry(file_id, 3).await { + error!("Failed to cleanup immediate parse data for file {}: {}", file_id, cleanup_err); + } + processor.mark_file_failed(file_id, &err.to_string()).await?; + } + result +} + +pub async fn try_reuse_file_with_file( + pool: SqlitePool, search_engine: SearchEngine, file: File, +) -> anyhow::Result { + if is_parse_paused() { + return Ok(false); + } + let processor = FileProcessor::new(pool, search_engine, 0); + let Some(claimed_file) = processor.claim_file_by_id(file.id).await? else { + return Ok(false); + }; + let result = processor.try_reuse_existing_data(&claimed_file).await; + match result { + Ok(true) => Ok(true), + Ok(false) => { + sqlx::query( + "UPDATE files SET status = 0, parse_run_id = NULL, updated_at = strftime('%s','now') \ + WHERE id = ? AND status = 2", + ) + .bind(file.id) + .execute(&processor.pool) + .await?; + Ok(false) + } + Err(err) => { + sqlx::query( + "UPDATE files SET status = 0, parse_run_id = NULL, log = ?, updated_at = strftime('%s','now') \ + WHERE id = ? AND status = 2", + ) + .bind(format!("Reuse failed: {}", err)) + .bind(file.id) + .execute(&processor.pool) + .await?; + Err(err) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn immediate_parse_claim_is_atomic() -> anyhow::Result<()> { + let temp = tempfile::NamedTempFile::new()?; + let url = format!("sqlite://{}", temp.path().display()); + let pool = sqlx::sqlite::SqlitePoolOptions::new().max_connections(4).connect(&url).await?; + sqlx::query( + "CREATE TABLE files ( + id INTEGER PRIMARY KEY, + user_id TEXT NOT NULL, user_name TEXT NOT NULL, hash TEXT NOT NULL, + filename TEXT NOT NULL, path TEXT NOT NULL, size INTEGER NOT NULL, + tags TEXT NOT NULL, status INTEGER NOT NULL, log TEXT NOT NULL, + slice_type TEXT NOT NULL, kb_id INTEGER, is_public INTEGER NOT NULL, + meta TEXT, summary TEXT, parse_priority INTEGER NOT NULL, parse_run_id TEXT, artifact_id INTEGER, + created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL + )", + ) + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO files VALUES (1, 'u', 'n', 'h', 'f.txt', '/tmp/f', 1, '', 0, '', 'text', NULL, 0, NULL, NULL, 50, NULL, NULL, 1, 1)", + ) + .execute(&pool) + .await?; + + let (first, second) = tokio::join!(claim_pending_file_by_id(&pool, 1), claim_pending_file_by_id(&pool, 1)); + let claimed = usize::from(first?.is_some()) + usize::from(second?.is_some()); + assert_eq!(claimed, 1); + + let (status, run_id): (i32, Option) = + sqlx::query_as("SELECT status, parse_run_id FROM files WHERE id = 1").fetch_one(&pool).await?; + assert_eq!(status, 2); + assert!(run_id.is_some()); + Ok(()) + } + + fn image_content_item(img_path: &str) -> ContentItem { + ContentItem { + typ: "image".to_string(), + bbox: vec![1, 2, 3, 4], + page_idx: 0, + text: None, + text_level: None, + text_format: None, + img_path: Some(img_path.to_string()), + image_caption: None, + table_body: None, + table_caption: None, + } + } + + #[test] + fn custom_parse_normalization_preserves_image_names() -> anyhow::Result<()> { + let mut images = HashMap::new(); + images.insert("img.png".to_string(), "raw-base64".to_string()); + let data = CustomParseData { + slices: vec![CustomSlice { + content: "see ![x](/api/v1/knowledge/files/img.png)".to_string(), + positions: Vec::new(), + }], + full_content: Some("full /api/v1/knowledge/files/img.png".to_string()), + summary: None, + images: Some(images), + content_list: Some(vec![image_content_item("images/img.png")]), + }; + + let normalized = FileProcessor::normalize_custom_parse_data(data)?; + + assert!(normalized.images.contains_key("img.png")); + assert_eq!(normalized.content_list.as_ref().unwrap()[0].img_path.as_deref(), Some("img.png")); + assert!(normalized.slices[0].content.contains("/api/v1/knowledge/files/img.png")); + assert!(normalized.full_content.as_ref().unwrap().contains("/api/v1/knowledge/files/img.png")); + assert_eq!(normalized.image_paths, vec!["img.png".to_string()]); + Ok(()) + } + + #[test] + fn custom_parse_normalization_preserves_unmapped_slice_refs() -> anyhow::Result<()> { + let data = CustomParseData { + slices: vec![CustomSlice { + content: "legacy ![x](/api/v1/knowledge/files/legacy.png)".to_string(), + positions: Vec::new(), + }], + full_content: None, + summary: None, + images: None, + content_list: None, + }; + + let normalized = FileProcessor::normalize_custom_parse_data(data)?; + + assert!(normalized.images.is_empty()); + assert!(normalized.slices[0].content.contains("/api/v1/knowledge/files/legacy.png")); + assert_eq!(normalized.image_paths, vec!["legacy.png".to_string()]); + Ok(()) + } + + #[test] + fn reuse_removes_legacy_image_id_prefix() { + assert_eq!(FileProcessor::remove_image_id_prefix("f42_img.png", 42), "img.png"); + assert_eq!(FileProcessor::remove_image_id_prefix("images/f42_img.png", 42), "images/img.png"); + assert_eq!(FileProcessor::remove_image_id_prefix("img.png", 42), "img.png"); + assert_eq!(FileProcessor::remove_image_id_prefix("f7_img.png", 42), "f7_img.png"); + } + + #[test] + fn unsupported_mineru_image_formats_require_conversion() { + for filename in ["icon.ico", "diagram.svg", "photo.avif"] { + assert!(crate::search::is_image_file(filename)); + assert!(FileProcessor::requires_mineru_image_conversion(filename)); + } + assert!(!FileProcessor::requires_mineru_image_conversion("photo.png")); + } + + #[test] + fn image_descriptions_are_only_appended_once() { + let mut items = vec![image_content_item("upload.gif")]; + let descriptions = HashMap::from([("upload.gif".to_string(), "animated image".to_string())]); + + enrich_content_list_with_image_descriptions(&mut items, &descriptions); + enrich_content_list_with_image_descriptions(&mut items, &descriptions); + + let captions = items[0].image_caption.as_ref().expect("image caption should be present"); + assert_eq!(captions.len(), 1); + assert_eq!(captions[0], "animated image"); + } +} diff --git a/src/search/advanced.rs b/src/search/advanced.rs new file mode 100644 index 0000000..e1d7a85 --- /dev/null +++ b/src/search/advanced.rs @@ -0,0 +1,427 @@ +use std::{collections::HashSet, time::Duration}; + +use anyhow::{Context, Result, anyhow}; +use log::{info, warn}; +use once_cell::sync::Lazy; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use sqlx::SqlitePool; + +use super::SearchResultItem; +use crate::config; + +const MAX_NEIGHBOR_SLICES: i64 = 30; + +/// 复用的 LLM HTTP client（连接池）。LLM 调用超时较长，固定 600s。 +static LLM_HTTP_CLIENT: Lazy = + Lazy::new(|| Client::builder().timeout(Duration::from_secs(600)).build().expect("build reqwest client")); + +#[derive(Clone)] +pub struct LlmClient { + client: Client, + api_url: Option, + api_key: Option, + model: String, +} + +impl Default for LlmClient { + fn default() -> Self { + Self::new() + } +} + +impl LlmClient { + pub fn new() -> Self { + let cfg = config::get(); + let llm = cfg.llm.clone(); + // 复用全局 client，避免每次构造都新建连接池 + Self { client: LLM_HTTP_CLIENT.clone(), api_url: llm.api_url, api_key: llm.api_key, model: llm.model } + } + + pub fn is_enabled(&self) -> bool { + self.api_url.is_some() + } + + pub async fn chat_json( + &self, prompt: &str, max_tokens: usize, temperature: f32, + ) -> Result { + let content = self.chat(prompt, max_tokens, temperature).await?; + let clean = clean_json_like(&content); + let parsed = serde_json::from_str::(&clean) + .map_err(|e| anyhow!("Failed to parse LLM JSON response: {}. content={}", e, clean))?; + Ok(parsed) + } + + pub async fn chat(&self, prompt: &str, max_tokens: usize, temperature: f32) -> Result { + let url = self.api_url.as_ref().ok_or_else(|| anyhow!("LLM not configured"))?; + let request = ChatRequest { + model: self.model.clone(), + messages: vec![ + Message { + role: "system".to_string(), + content: "Do not output chain-of-thought. Only output final answer.".to_string(), + }, + Message { role: "user".to_string(), content: prompt.to_string() }, + ], + max_tokens: Some(max_tokens), + temperature: Some(temperature), + }; + + let mut builder = self.client.post(url).json(&request); + if let Some(api_key) = &self.api_key { + builder = builder.header("Authorization", format!("Bearer {}", api_key)); + } + + let response = builder.send().await.context("LLM HTTP request failed")?; + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow!("LLM HTTP error {}: {}", status, body)); + } + let body = response.text().await.context("Failed to read LLM body")?; + let parsed: ChatResponse = + serde_json::from_str(&body).map_err(|e| anyhow!("Failed to decode LLM response: {} body={}", e, body))?; + let choice = + parsed.choices.first().ok_or_else(|| anyhow!("LLM response missing choices"))?.message.content.clone(); + Ok(choice) + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "snake_case")] +pub enum PlanAction { + RecentDocuments, + DocumentStructure, + PageContent, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct PlanStep { + pub action: PlanAction, + #[serde(default)] + pub comment: String, +} + +pub struct QueryPlanner { + _llm: LlmClient, +} + +impl QueryPlanner { + pub fn new(llm: LlmClient) -> Self { + Self { _llm: llm } + } + + pub async fn plan(&self, _query: &str, _max_steps: usize) -> Vec { + default_plan(2) + } +} + +fn default_plan(max_steps: usize) -> Vec { + let mut steps = vec![ + PlanStep { + action: PlanAction::RecentDocuments, comment: "搜索相关文档并选出候选切片".to_string() + }, + PlanStep { + action: PlanAction::PageContent, + comment: "按候选顺序补充上下文并由 LLM 判断可回答性，命中即停止".to_string(), + }, + ]; + steps.truncate(max_steps.min(steps.len())); + steps +} + +pub struct RelevanceJudge { + llm: LlmClient, +} + +#[derive(Debug, Clone, Serialize)] +pub struct JudgeOutcome { + pub is_relevant: bool, + pub score: f32, + pub reason: String, +} + +impl RelevanceJudge { + pub fn new(llm: LlmClient) -> Self { + Self { llm } + } + + pub async fn judge(&self, question: &str, context: &str) -> JudgeOutcome { + if question.trim().is_empty() || context.trim().is_empty() { + return JudgeOutcome { is_relevant: false, score: 0.0, reason: "输入为空".to_string() }; + } + if !self.llm.is_enabled() { + return JudgeOutcome { is_relevant: true, score: 1.0, reason: "LLM 未启用，默认通过".to_string() }; + } + + let prompt = format!( + r#" +你是一名检索质量评估助手。判断候选内容是否可以帮助回答问题，只输出 JSON： +{{"relevant":true/false, "score":0-1之间的小数, "reason":"简要说明"}} + +问题： +{} + +候选内容： +{} +"#, + question, context + ); + + match self.llm.chat_json::(&prompt, 50000, 0.2).await { + Ok(resp) => { + let mut score = resp.score.unwrap_or(0.5); + if !(0.0..=1.0).contains(&score) { + score = score.clamp(0.0, 1.0); + } + info!("result: {}\nscore: {}", context, score); + JudgeOutcome { + is_relevant: resp.relevant, + score, + reason: resp.reason.unwrap_or_else(|| "LLM 无理由".to_string()), + } + } + Err(err) => { + warn!("LLM judge failed: {}", err); + JudgeOutcome { is_relevant: true, score: 0.6, reason: "LLM 判定失败，默认通过".to_string() } + } + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct ChunkSegment { + pub slice_id: i64, + pub text: String, +} + +#[derive(Debug, Serialize)] +pub struct ContextChunk { + pub slice_ids: Vec, + pub content: String, + pub segments: Vec, +} + +#[derive(Clone)] +pub struct ChunkRefiner { + llm: LlmClient, +} + +impl ChunkRefiner { + pub fn new(llm: LlmClient) -> Self { + Self { llm } + } + + pub async fn refine(&self, question: &str, segments: &[ChunkSegment]) -> Result { + if segments.is_empty() { + return Ok(RefineOutcome { segments: Vec::new(), reason: None }); + } + if !self.llm.is_enabled() { + return Ok(RefineOutcome { segments: segments.to_vec(), reason: None }); + } + + let limited_segments: Vec = segments + .iter() + .map(|seg| RefineSegmentInput { slice_id: seg.slice_id, text: truncate_text(&seg.text, 400) }) + .collect(); + + let prompt = format!( + r#" +你会得到一个问题和若干切片。请保留对问题最有帮助的切片，其余丢弃。 +输出 JSON：{{"keep_slice_ids":[1,2], "reason":"简短说明"}}。至少保留一个切片。 + +问题： +{} + +切片： +{} +"#, + question, + serde_json::to_string(&limited_segments)? + ); + + match self.llm.chat_json::(&prompt, 50000, 0.2).await { + Ok(resp) => { + let keep_ids: HashSet = if resp.keep_slice_ids.is_empty() { + segments.iter().map(|seg| seg.slice_id).collect() + } else { + resp.keep_slice_ids.into_iter().collect() + }; + let filtered: Vec = + segments.iter().filter(|seg| keep_ids.contains(&seg.slice_id)).cloned().collect(); + let final_segments = if filtered.is_empty() { segments.to_vec() } else { filtered }; + Ok(RefineOutcome { segments: final_segments, reason: resp.reason }) + } + Err(err) => { + warn!("LLM chunk refine failed: {}", err); + Ok(RefineOutcome { segments: segments.to_vec(), reason: None }) + } + } + } +} + +pub struct RefineOutcome { + pub segments: Vec, + pub reason: Option, +} + +pub async fn assemble_context_chunk( + pool: &SqlitePool, center: &SearchResultItem, per_side_chars: usize, +) -> Result { + // 前后切片两条查询相互独立，并发执行 + let (before_rows, after_rows) = tokio::try_join!( + fetch_before_slices(pool, center.file_id, center.id), + fetch_after_slices(pool, center.file_id, center.id), + )?; + + let before = take_with_limit(before_rows, per_side_chars, true); + let after = take_with_limit(after_rows, per_side_chars, false); + + let mut segments: Vec = Vec::new(); + for row in &before { + segments.push(ChunkSegment { slice_id: row.id, text: row.content.clone() }); + } + segments.push(ChunkSegment { slice_id: center.id, text: center.content.clone() }); + for row in &after { + segments.push(ChunkSegment { slice_id: row.id, text: row.content.clone() }); + } + + let slice_ids = segments.iter().map(|seg| seg.slice_id).collect(); + let combined = segments.iter().map(|seg| seg.text.as_str()).collect::>().join("\n"); + Ok(ContextChunk { slice_ids, content: combined, segments }) +} + +#[derive(sqlx::FromRow, Clone)] +struct SliceRow { + id: i64, + content: String, +} + +#[derive(Debug, Serialize)] +struct RefineSegmentInput { + slice_id: i64, + text: String, +} + +fn truncate_text(text: &str, max_chars: usize) -> String { + if text.chars().count() <= max_chars { + return text.to_string(); + } + let mut result = String::new(); + for (idx, ch) in text.chars().enumerate() { + if idx >= max_chars { + result.push_str("..."); + break; + } + result.push(ch); + } + result +} + +fn take_with_limit(items: Vec, char_limit: usize, reverse_after: bool) -> Vec { + if reverse_after { + // 当前 items 为倒序 + let mut selected = Vec::new(); + let mut total = 0; + for row in items.into_iter() { + total += row.content.chars().count(); + selected.push(row); + if total >= char_limit { + break; + } + } + selected.reverse(); + selected + } else { + let mut selected = Vec::new(); + let mut total = 0; + for row in items.into_iter() { + total += row.content.chars().count(); + selected.push(row); + if total >= char_limit { + break; + } + } + selected + } +} + +async fn fetch_before_slices(pool: &SqlitePool, file_id: i64, center_id: i64) -> Result> { + let sql = + format!("SELECT id FROM slices WHERE file_id = ? AND id < ? ORDER BY id DESC LIMIT {}", MAX_NEIGHBOR_SLICES); + let ids: Vec = sqlx::query_scalar(&sql).bind(file_id).bind(center_id).fetch_all(pool).await?; + let contents = crate::slice_content::read_all(file_id).await?; + Ok(ids.into_iter().map(|id| SliceRow { id, content: contents.get(&id).cloned().unwrap_or_default() }).collect()) +} + +async fn fetch_after_slices(pool: &SqlitePool, file_id: i64, center_id: i64) -> Result> { + let sql = + format!("SELECT id FROM slices WHERE file_id = ? AND id > ? ORDER BY id ASC LIMIT {}", MAX_NEIGHBOR_SLICES); + let ids: Vec = sqlx::query_scalar(&sql).bind(file_id).bind(center_id).fetch_all(pool).await?; + let contents = crate::slice_content::read_all(file_id).await?; + Ok(ids.into_iter().map(|id| SliceRow { id, content: contents.get(&id).cloned().unwrap_or_default() }).collect()) +} + +#[derive(Debug, Deserialize)] +struct RefineResponse { + #[serde(default)] + keep_slice_ids: Vec, + #[serde(default)] + reason: Option, +} + +#[derive(Debug, Deserialize)] +struct JudgeResponse { + relevant: bool, + #[serde(default)] + score: Option, + #[serde(default)] + reason: Option, +} + +#[derive(Debug, Serialize)] +struct ChatRequest { + model: String, + messages: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + max_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + temperature: Option, +} + +#[derive(Debug, Serialize)] +struct Message { + role: String, + content: String, +} + +#[derive(Debug, Deserialize)] +struct ChatResponse { + choices: Vec, +} + +#[derive(Debug, Deserialize)] +struct ChatChoice { + message: ChatMessage, +} + +#[derive(Debug, Deserialize)] +struct ChatMessage { + content: String, +} + +fn clean_json_like(input: &str) -> String { + let mut text = input.trim(); + if let Some(rest) = text.split_once("") { + text = rest.1; + } + if let Some(stripped) = text.strip_prefix("```json") { + text = stripped.trim_start(); + } else if let Some(stripped) = text.strip_prefix("```") { + text = stripped.trim_start(); + } + if let Some(stripped) = text.strip_suffix("```") { + text = stripped.trim_end(); + } + text.trim().to_string() +} diff --git a/src/search/chinese_tokenizer.rs b/src/search/chinese_tokenizer.rs new file mode 100644 index 0000000..6750735 --- /dev/null +++ b/src/search/chinese_tokenizer.rs @@ -0,0 +1,206 @@ +//! 中文分词器模块 - 提供高性能的中英文混合分词功能 + +use std::{collections::HashSet, sync::RwLock}; + +use anyhow::anyhow; +use jieba_rs::{Jieba, TokenizeMode}; +use lazy_static::lazy_static; +use log::info; +use tantivy::tokenizer::{Token, TokenStream, Tokenizer}; + +#[derive(Debug, Clone)] +pub struct LexiconEntry { + pub term: String, + pub freq: Option, + pub tag: Option, +} + +// 使用全局静态 Jieba 实例，避免重复初始化 +lazy_static! { + static ref JIEBA: RwLock = RwLock::new(Jieba::new()); + // 基础停用词集合，可按需扩展 + static ref STOP_WORDS: HashSet<&'static str> = { + let mut set = HashSet::new(); + let content = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/stopwords.txt")); + for line in content.lines() { + let word = line.trim(); + if word.is_empty() || word.starts_with('#') { + continue; + } + set.insert(word); + } + set + }; +} + +pub fn reload_custom_words(entries: &[LexiconEntry]) -> anyhow::Result { + let mut jieba = Jieba::new(); + let mut loaded = 0usize; + for entry in entries { + let term = entry.term.trim(); + if term.is_empty() { + continue; + } + let tag = entry.tag.as_deref().map(str::trim).filter(|s| !s.is_empty()); + let freq = entry.freq.filter(|f| *f > 0); + jieba.add_word(term, freq, tag); + loaded += 1; + } + + let mut guard = JIEBA.write().map_err(|_| anyhow!("failed to lock jieba for writing"))?; + *guard = jieba; + info!("Reloaded Jieba custom lexicon with {} words", loaded); + Ok(loaded) +} + +/// 分词模式 +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum SegmentationMode { + /// 搜索模式：适合搜索引擎（召回率高但较慢） + #[default] + Search, + /// 全模式：速度较慢，但能识别更多的词汇 + All, +} + +/// 快速中文分词器 - 针对性能优化 +#[derive(Clone)] +pub struct FastChineseTokenizer { + mode: SegmentationMode, +} + +impl FastChineseTokenizer { + /// 创建新的分词器 + pub fn new(mode: SegmentationMode) -> Self { + FastChineseTokenizer { mode } + } + /// 创建全模式分词器 + pub fn all() -> Self { + Self::new(SegmentationMode::All) + } + + /// 执行分词 + pub fn segment(&self, text: &str) -> Vec { + let jieba = JIEBA.read().unwrap_or_else(|e| e.into_inner()); + let words = match self.mode { + SegmentationMode::Search => { + // 搜索模式：使用 cut_for_search，召回率高 + jieba + .cut_for_search(text, false) + .into_iter() + .filter(|s| { + let word = s.trim(); + !word.is_empty() && !STOP_WORDS.contains(word) + }) + .map(|s| s.to_string()) + .collect() + } + SegmentationMode::All => jieba + .cut_all(text) + .into_iter() + .filter(|s| { + let word = s.trim(); + !word.is_empty() && !STOP_WORDS.contains(word) + }) + .map(|s| s.to_string()) + .collect(), + }; + info!("Segmented mode: {:?}, text: {}, words: {:?}", self.mode, text, words); + words + } + + fn tokenize_with_offsets(&self, text: &str) -> Vec { + let mode = match self.mode { + SegmentationMode::Search => TokenizeMode::Search, + SegmentationMode::All => TokenizeMode::Default, + }; + let jieba = JIEBA.read().unwrap_or_else(|e| e.into_inner()); + jieba + .tokenize(text, mode, false) + .into_iter() + .filter(|t| { + let word = t.word.trim(); + !word.is_empty() && !STOP_WORDS.contains(word) + }) + .map(|t| TokenInfo { text: t.word.to_string(), start: t.start, end: t.end }) + .collect() + } +} + +impl Tokenizer for FastChineseTokenizer { + type TokenStream<'a> = ChineseTokenStream<'a>; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + let tokens = self.tokenize_with_offsets(text); + ChineseTokenStream::new(text, tokens) + } +} + +/// Token stream for Chinese text +pub struct ChineseTokenStream<'a> { + tokens: Vec, + current_index: usize, + token: Token, + char_to_byte: Vec, + _phantom: std::marker::PhantomData<&'a ()>, +} + +struct TokenInfo { + text: String, + start: usize, + end: usize, +} + +impl<'a> ChineseTokenStream<'a> { + fn new(text: &'a str, tokens: Vec) -> Self { + let mut char_to_byte = Vec::with_capacity(text.chars().count() + 1); + for (byte_idx, _) in text.char_indices() { + char_to_byte.push(byte_idx); + } + char_to_byte.push(text.len()); + ChineseTokenStream { + tokens, + current_index: 0, + token: Token::default(), + char_to_byte, + _phantom: std::marker::PhantomData, + } + } +} + +impl<'a> TokenStream for ChineseTokenStream<'a> { + fn advance(&mut self) -> bool { + while self.current_index < self.tokens.len() { + let token_info = &self.tokens[self.current_index]; + self.current_index += 1; + + // 跳过空的token + if token_info.text.trim().is_empty() { + continue; + } + + let char_len = self.char_to_byte.len().saturating_sub(1); + if token_info.start >= char_len || token_info.end > char_len || token_info.start >= token_info.end { + continue; + } + let start = self.char_to_byte[token_info.start]; + let end = self.char_to_byte[token_info.end]; + + self.token.text.clear(); + self.token.text.push_str(&token_info.text); + self.token.offset_from = start; + self.token.offset_to = end; + self.token.position = self.token.position.wrapping_add(1); + return true; + } + false + } + + fn token(&self) -> &Token { + &self.token + } + + fn token_mut(&mut self) -> &mut Token { + &mut self.token + } +} diff --git a/src/search/embedding.rs b/src/search/embedding.rs new file mode 100644 index 0000000..937b9c6 --- /dev/null +++ b/src/search/embedding.rs @@ -0,0 +1,209 @@ +use std::time::Duration; + +use anyhow::{Context, Result}; +use once_cell::sync::Lazy; +use reqwest::Client; +use serde::{Deserialize, Serialize}; + +use crate::config; + +/// 判断图片 embedding 服务是否已配置。 +pub fn image_embedding_enabled() -> bool { + config::get().services.image_embedding_url.is_some() +} + +static HTTP_CLIENT: Lazy = Lazy::new(Client::new); + +#[derive(Debug, Serialize)] +struct EmbeddingRequest { + model: String, + input: Vec, +} + +#[derive(Debug, Deserialize)] +struct EmbeddingResponse { + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct EmbeddingData { + embedding: Vec, +} + +/// 获取图片的 embedding 向量（从文件路径） +pub async fn get_image_embedding_from_path(path: &str, text: Option<&str>) -> Result> { + let file_name = std::path::Path::new(path).file_name().and_then(|name| name.to_str()).unwrap_or("image"); + let mime = mime_guess::from_path(path).first_or_octet_stream(); + let cfg = config::get(); + let url = cfg + .services + .image_embedding_url + .as_deref() + .ok_or_else(|| anyhow::anyhow!("image embedding URL is not configured"))?; + + let part = reqwest::multipart::Part::file(path) + .await + .with_context(|| format!("failed to open image file for embedding: {}", path))? + .file_name(file_name.to_string()) + .mime_str(mime.essence_str())?; + let form = reqwest::multipart::Form::new().part("file", part).text("text", text.unwrap_or(file_name).to_string()); + + let response = HTTP_CLIENT + .post(url) + .timeout(Duration::from_secs(cfg.search.embedding_timeout_secs)) + .multipart(form) + .send() + .await + .with_context(|| { + format!( + "image embedding request failed: url={}, file={}, timeout={}s", + url, file_name, cfg.search.embedding_timeout_secs + ) + })?; + + handle_image_embedding_response(response).await +} + +/// 获取图片的 embedding 向量（从文件内容） +pub async fn get_image_embedding_from_bytes( + file_name: &str, content_type: Option<&str>, bytes: Vec, text: Option<&str>, +) -> Result> { + let cfg = config::get(); + let url = cfg + .services + .image_embedding_url + .as_deref() + .ok_or_else(|| anyhow::anyhow!("image embedding URL is not configured"))?; + let mut part = reqwest::multipart::Part::bytes(bytes).file_name(file_name.to_string()); + if let Some(content_type) = content_type { + part = part.mime_str(content_type)?; + } + let form = reqwest::multipart::Form::new().part("file", part).text("text", text.unwrap_or(file_name).to_string()); + + let response = HTTP_CLIENT + .post(url) + .timeout(Duration::from_secs(cfg.search.embedding_timeout_secs)) + .multipart(form) + .send() + .await + .with_context(|| { + format!( + "image embedding request failed: url={}, file={}, timeout={}s", + url, file_name, cfg.search.embedding_timeout_secs + ) + })?; + + handle_image_embedding_response(response).await +} + +async fn handle_image_embedding_response(response: reqwest::Response) -> Result> { + if !response.status().is_success() { + let status = response.status(); + let error_text = response.text().await.unwrap_or_default(); + anyhow::bail!("Image embedding API error: {} - {}", status, error_text); + } + + let embedding_response: EmbeddingResponse = + response.json().await.context("image embedding response decode failed")?; + + embedding_response + .data + .into_iter() + .next() + .map(|data| data.embedding) + .ok_or_else(|| anyhow::anyhow!("No image embedding returned")) +} + +/// 获取文本的 embedding 向量 +pub async fn get_embedding(text: &str) -> Result> { + let cfg = config::get(); + let query = text.trim(); + if query.is_empty() { + anyhow::bail!("Embedding query cannot be empty"); + } + let request = EmbeddingRequest { model: cfg.ai.embedding_model.clone(), input: vec![query.to_string()] }; + + let response = HTTP_CLIENT + .post(&cfg.services.embedding_url) + .timeout(Duration::from_secs(cfg.search.embedding_timeout_secs)) + .json(&request) + .send() + .await + .with_context(|| { + format!( + "embedding request failed: url={}, input_chars={}, timeout={}s", + cfg.services.embedding_url, + text.chars().count(), + cfg.search.embedding_timeout_secs + ) + })?; + + if !response.status().is_success() { + let status = response.status(); + let error_text = response.text().await.unwrap_or_default(); + anyhow::bail!("Embedding API error: {} - {}, input_chars={}", status, error_text, text.chars().count()); + } + + let embedding_response: EmbeddingResponse = response.json().await.context("embedding response decode failed")?; + + let embedding = embedding_response + .data + .into_iter() + .next() + .map(|data| data.embedding) + .ok_or_else(|| anyhow::anyhow!("No embedding returned"))?; + Ok(embedding) +} + +/// 批量获取文本的 embedding 向量（自动分批） +pub async fn get_embeddings(texts: &[String]) -> Result>> { + if texts.is_empty() { + return Ok(Vec::new()); + } + + let batch_size = config::get().ai.embedding_batch_size; + let mut all_embeddings = Vec::with_capacity(texts.len()); + for chunk in texts.chunks(batch_size) { + let batch = get_embeddings_single_batch(chunk).await?; + all_embeddings.extend(batch); + } + Ok(all_embeddings) +} + +async fn get_embeddings_single_batch(texts: &[String]) -> Result>> { + let cfg = config::get(); + let request = EmbeddingRequest { model: cfg.ai.embedding_model.clone(), input: texts.to_vec() }; + + let response = HTTP_CLIENT + .post(&cfg.services.embedding_url) + .timeout(Duration::from_secs(cfg.search.embedding_timeout_secs)) + .json(&request) + .send() + .await + .with_context(|| { + format!( + "batch embedding request failed: url={}, batch_size={}, total_chars={}, timeout={}s", + cfg.services.embedding_url, + texts.len(), + texts.iter().map(|t| t.chars().count()).sum::(), + cfg.search.embedding_timeout_secs + ) + })?; + + if !response.status().is_success() { + let status = response.status(); + let error_text = response.text().await.unwrap_or_default(); + anyhow::bail!( + "Embedding API error: {} - {}, batch_size={}, total_chars={}", + status, + error_text, + texts.len(), + texts.iter().map(|text| text.chars().count()).sum::() + ); + } + + let embedding_response: EmbeddingResponse = + response.json().await.context("batch embedding response decode failed")?; + + Ok(embedding_response.data.into_iter().map(|data| data.embedding).collect()) +} diff --git a/src/search/lancedb.rs b/src/search/lancedb.rs new file mode 100644 index 0000000..c1f5eda --- /dev/null +++ b/src/search/lancedb.rs @@ -0,0 +1,1411 @@ +use std::{ + collections::HashSet, + path::Path, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result}; +use arrow_array::{ + Array, ArrayRef, BooleanArray, Float32Array, Int64Array, RecordBatch, StringArray, + builder::{FixedSizeListBuilder, Float32Builder}, +}; +use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use futures::stream::StreamExt; +use lancedb::{ + Connection, Table, connect, + index::{ + Index, + scalar::{BTreeIndexBuilder, BitmapIndexBuilder}, + }, + query::{ExecutableQuery, QueryBase, Select}, + table::{CompactionOptions, NewColumnTransform, OptimizeAction, OptimizeOptions}, +}; +use log::{debug, info, warn}; +use once_cell::sync::OnceCell; + +use super::{SummarySearchResultItem, embedding, tantivy_engine::SearchResultItem}; +use crate::config; + +static LANCEDB: OnceCell> = OnceCell::new(); +static LANCEDB_TABLE: OnceCell> = OnceCell::new(); +static SUMMARY_LANCEDB_TABLE: OnceCell> = OnceCell::new(); +static TABLE_NAME: &str = "documents"; +static SUMMARY_TABLE_NAME: &str = "file_summaries"; +static IS_DELETED_COLUMN: &str = "is_deleted"; +static SEARCH_SELECT_COLUMNS: &[&str] = &["id", "file_id", "kb_id", "content", "_distance"]; +static SUMMARY_SEARCH_SELECT_COLUMNS: &[&str] = &["file_id", "kb_id", "summary", "_distance"]; +static VECTOR_FAST_SEARCH_ENABLED: AtomicBool = AtomicBool::new(false); +static IMAGE_FAST_SEARCH_ENABLED: AtomicBool = AtomicBool::new(false); +static SUMMARY_FAST_SEARCH_ENABLED: AtomicBool = AtomicBool::new(false); + +/// IVF-PQ uses 256 centroids for its 8-bit product quantizer. Tiny datasets +/// are both faster with exact search and cannot provide enough training data. +const MIN_VECTORS_FOR_INDEX: usize = 512; + +/// 增量索引优化与 compact 之间的互斥锁。 +static OPTIMIZE_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); +/// 单调递增版本号，用于防抖：只有最新版本才实际执行优化。 +static OPTIMIZE_VERSION: AtomicU64 = AtomicU64::new(0); +/// 写操作后触发增量索引优化的防抖间隔。 +const OPTIMIZE_DEBOUNCE_MS: u64 = 5000; +const SUMMARY_RECALL_MULTIPLIER: usize = 5; +const SUMMARY_RECALL_MAX: usize = 100; + +#[derive(Debug, Clone)] +pub struct CompactStats { + pub deleted_rows: u64, + pub deleted_rows_before: u64, + pub deleted_rows_after: u64, + pub total_rows_before: u64, + pub total_rows_after: u64, + pub size_before_bytes: u64, + pub size_after_bytes: u64, +} + +#[derive(Clone)] +pub struct Document { + pub id: i64, // 切片 ID + pub file_id: i64, // 文件 ID + pub kb_id: Option, // 知识库 ID + pub content: String, // 内容 + pub is_image: bool, // 是否为图片文件 + pub image_embedding: Option>>, // 图片 embedding +} + +impl Document { + pub fn new(id: i64, file_id: i64, kb_id: Option, content: String) -> Self { + Document { id, file_id, kb_id, content, is_image: false, image_embedding: None } + } + + pub fn with_is_image(mut self, is_image: bool) -> Self { + self.is_image = is_image; + self + } + + pub fn with_image_embedding(mut self, embedding: Arc>) -> Self { + self.is_image = true; + self.image_embedding = Some(embedding); + self + } +} + +#[derive(Debug, Clone)] +pub struct SummaryDocument { + pub file_id: i64, + pub kb_id: Option, + pub summary: String, +} + +impl SummaryDocument { + pub fn new(file_id: i64, kb_id: Option, summary: String) -> Self { + Self { file_id, kb_id, summary } + } +} + +/// 初始化 LanceDB。返回 `true` 表示表是新建或从损坏中恢复的，调用方应考虑从 SQLite 回填数据。 +pub async fn init() -> Result { + let cfg = config::get(); + let storage_path = &cfg.storage.lancedb_path; + tokio::fs::create_dir_all(storage_path).await?; + + let t0 = Instant::now(); + let db = connect(storage_path).execute().await?; + info!("LanceDB init substep: connect() took {}ms", t0.elapsed().as_millis()); + LANCEDB.set(Arc::new(db)).map_err(|_| anyhow::anyhow!("Failed to initialize LanceDB"))?; + + // 创建表的 schema + let schema = create_schema(); + let summary_schema = create_summary_schema(); + + // 检查表是否存在 + let conn = get_connection()?; + let t1 = Instant::now(); + let table_exists = conn + .table_names() + .execute() + .await + .map(|table_names| table_names.contains(&TABLE_NAME.to_string())) + .unwrap_or(false); + info!("LanceDB init substep: table_names() took {}ms", t1.elapsed().as_millis()); + + let t2 = Instant::now(); + let (table, was_recreated) = if table_exists { + match conn.open_table(TABLE_NAME).execute().await { + Ok(table) => (table, false), + Err(err) => { + warn!( + "LanceDB table '{}' exists but cannot be opened (likely corrupted): {}. Attempting recovery...", + TABLE_NAME, err + ); + (recover_table(storage_path, &schema).await?, true) + } + } + } else { + (create_empty_table(&schema).await?, true) + }; + info!( + "LanceDB init substep: open/create/recover table took {}ms (recreated={})", + t2.elapsed().as_millis(), + was_recreated + ); + + let t3 = Instant::now(); + ensure_is_deleted_column(&table).await?; + info!("LanceDB init substep: ensure_is_deleted_column() took {}ms", t3.elapsed().as_millis()); + + LANCEDB_TABLE.set(Arc::new(table)).map_err(|_| anyhow::anyhow!("Failed to cache LanceDB table"))?; + + let t4 = Instant::now(); + let summary_table_names = conn.table_names().execute().await.unwrap_or_default(); + let summary_table_exists = summary_table_names.contains(&SUMMARY_TABLE_NAME.to_string()); + let (summary_table, summary_was_recreated) = if summary_table_exists { + match conn.open_table(SUMMARY_TABLE_NAME).execute().await { + Ok(table) => (table, false), + Err(err) => { + warn!( + "LanceDB table '{}' exists but cannot be opened (likely corrupted): {}. Attempting recovery...", + SUMMARY_TABLE_NAME, err + ); + (recover_named_table(storage_path, SUMMARY_TABLE_NAME, &summary_schema).await?, true) + } + } + } else { + (create_empty_named_table(SUMMARY_TABLE_NAME, &summary_schema).await?, true) + }; + ensure_is_deleted_column(&summary_table).await?; + SUMMARY_LANCEDB_TABLE + .set(Arc::new(summary_table)) + .map_err(|_| anyhow::anyhow!("Failed to cache LanceDB summary table"))?; + info!( + "LanceDB init substep: open/create/recover summary table took {}ms (recreated={})", + t4.elapsed().as_millis(), + summary_was_recreated + ); + + Ok(was_recreated) +} + +/// Reconcile must finish before this is called so index construction cannot +/// race startup repairs. Searches use exact search while maintenance runs. +pub fn schedule_startup_index_maintenance() { + tokio::spawn(async move { + let _guard = OPTIMIZE_LOCK.lock().await; + let started_at = Instant::now(); + let result = match (get_table(), get_summary_table()) { + (Ok(table), Ok(summary_table)) => { + async { + maintain_search_indices(&table).await?; + maintain_summary_search_indices(&summary_table).await?; + Ok(()) + } + .await + } + (Err(err), _) | (_, Err(err)) => Err(err), + }; + if let Err(err) = result { + warn!("LanceDB background index initialization failed: {}", err); + } + info!("LanceDB background index initialization took {}ms", started_at.elapsed().as_millis()); + }); +} + +async fn create_empty_table(schema: &Arc) -> Result { + let conn = get_connection()?; + let empty_batch = create_empty_batch(schema)?; + conn.create_table(TABLE_NAME, empty_batch) + .execute() + .await + .with_context(|| format!("Failed to create LanceDB table '{}'", TABLE_NAME)) +} + +async fn create_empty_named_table(table_name: &str, schema: &Arc) -> Result
{ + let conn = get_connection()?; + let empty_batch = if table_name == SUMMARY_TABLE_NAME { + create_summary_empty_batch(schema)? + } else { + create_empty_batch(schema)? + }; + conn.create_table(table_name, empty_batch) + .execute() + .await + .with_context(|| format!("Failed to create LanceDB table '{}'", table_name)) +} + +async fn recover_table(storage_path: &str, schema: &Arc) -> Result
{ + let conn = get_connection()?; + + // 先尝试用 LanceDB API 优雅删除损坏的表 + if let Err(err) = conn.drop_table(TABLE_NAME, &[]).await { + warn!("LanceDB drop_table failed (will remove filesystem directory instead): {}", err); + } + + // 清理文件系统残留，并备份到带时间戳的目录 + let table_dir = Path::new(storage_path).join(format!("{}.lance", TABLE_NAME)); + if table_dir.exists() { + let backup_dir = { + let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + table_dir.with_file_name(format!("documents.lance.corrupted.{}", timestamp)) + }; + warn!("Moving corrupted LanceDB table from {} to {}", table_dir.display(), backup_dir.display()); + tokio::fs::rename(&table_dir, &backup_dir) + .await + .with_context(|| format!("Failed to move corrupted LanceDB table to {}", backup_dir.display()))?; + } + + create_empty_table(schema).await +} + +async fn recover_named_table(storage_path: &str, table_name: &str, schema: &Arc) -> Result
{ + let conn = get_connection()?; + + if let Err(err) = conn.drop_table(table_name, &[]).await { + warn!("LanceDB drop_table failed (will remove filesystem directory instead): {}", err); + } + + let table_dir = Path::new(storage_path).join(format!("{}.lance", table_name)); + if table_dir.exists() { + let backup_dir = { + let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + table_dir.with_file_name(format!("{}.lance.corrupted.{}", table_name, timestamp)) + }; + warn!("Moving corrupted LanceDB table from {} to {}", table_dir.display(), backup_dir.display()); + tokio::fs::rename(&table_dir, &backup_dir) + .await + .with_context(|| format!("Failed to move corrupted LanceDB table to {}", backup_dir.display()))?; + } + + create_empty_named_table(table_name, schema).await +} + +pub async fn write_documents(doc: Document) -> Result<()> { + let table = get_table()?; + let schema = create_schema(); + let batch = create_record_batch(vec![doc], &schema).await?; + table.add(batch).execute().await?; + on_write_may_need_optimize(); + Ok(()) +} + +/// 批量写入文档，一次性获取 embeddings 并写入 +pub async fn write_documents_batch(docs: Vec) -> Result<()> { + write_documents_batch_inner(docs, true).await +} + +/// 重建期间批量写入，由调用方在全部完成后统一触发索引刷新。 +pub async fn write_documents_batch_for_rebuild(docs: Vec) -> Result<()> { + write_documents_batch_inner(docs, false).await +} + +pub async fn write_summary(doc: SummaryDocument) -> Result<()> { + let summary = doc.summary.trim(); + if summary.is_empty() { + delete_summary_by_file(doc.file_id).await?; + return Ok(()); + } + + let file_id = doc.file_id; + let table = get_summary_table()?; + let schema = create_summary_schema(); + let batch = create_summary_record_batch( + vec![SummaryDocument { file_id, kb_id: doc.kb_id, summary: summary.to_string() }], + &schema, + ) + .await?; + delete_summary_by_file(file_id).await?; + table.add(batch).execute().await?; + on_write_may_need_optimize(); + Ok(()) +} + +pub async fn replace_summaries_batch_for_rebuild(docs: Vec) -> Result<()> { + let docs: Vec = docs.into_iter().filter(|doc| !doc.summary.trim().is_empty()).collect(); + if docs.is_empty() { + return Ok(()); + } + + let file_ids: Vec = docs.iter().map(|doc| doc.file_id).collect(); + let table = get_summary_table()?; + let schema = create_summary_schema(); + let batch = create_summary_record_batch(docs, &schema).await?; + delete_summary_by_predicate(build_in_predicate("file_id", &file_ids), false).await?; + table.add(batch).execute().await?; + set_fast_search_enabled(false, &SUMMARY_FAST_SEARCH_ENABLED); + Ok(()) +} + +async fn write_documents_batch_inner(docs: Vec, schedule_optimize: bool) -> Result<()> { + if docs.is_empty() { + return Ok(()); + } + let table = get_table()?; + let schema = create_schema(); + let batch = create_record_batch(docs, &schema).await?; + table.add(batch).execute().await?; + if schedule_optimize { + on_write_may_need_optimize(); + } else { + set_fast_search_enabled(false, &VECTOR_FAST_SEARCH_ENABLED); + set_fast_search_enabled(false, &IMAGE_FAST_SEARCH_ENABLED); + set_fast_search_enabled(false, &SUMMARY_FAST_SEARCH_ENABLED); + } + Ok(()) +} + +/// 重建写入完成后触发一次防抖索引刷新。 +pub fn schedule_optimize_after_rebuild() { + on_write_may_need_optimize(); +} + +pub async fn search( + query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, +) -> Result> { + if has_empty_scope(file_ids, kb_ids) { + return Ok(Vec::new()); + } + + let cfg = config::get(); + let table = get_table()?; + + // 获取查询文本的 embedding + let embedding_start = std::time::Instant::now(); + let query_vector = embedding::get_embedding(query).await?; + debug!("LanceDB embedding {}ms", embedding_start.elapsed().as_millis()); + + // 使用向量搜索 + let fast_search = vector_fast_search_enabled(); + let mut query_builder = + table.query().nearest_to(query_vector)?.column("vector").select(Select::columns(SEARCH_SELECT_COLUMNS)); + if fast_search { + query_builder = query_builder.fast_search(); + } + + // 应用过滤条件 + let filter_conditions = build_filter_conditions(false, file_ids, kb_ids); + + if !filter_conditions.is_empty() { + query_builder = query_builder.only_if(filter_conditions.join(" AND ")); + } + + let execute_start = std::time::Instant::now(); + let mut result_stream = query_builder.limit(cfg.search.limit).execute().await?; + debug!("LanceDB execute {}ms fast_search={}", execute_start.elapsed().as_millis(), fast_search); + + let mut search_results = Vec::with_capacity(cfg.search.limit); + + // 从 stream 中读取数据 + let stream_start = std::time::Instant::now(); + let mut first_batch_ms = None; + let mut batch_count = 0usize; + let mut row_count = 0usize; + let mut decode_ms = 0u128; + while let Some(batch_result) = result_stream.next().await { + if first_batch_ms.is_none() { + first_batch_ms = Some(stream_start.elapsed().as_millis()); + } + let batch = batch_result?; + row_count += batch.num_rows(); + let decode_start = std::time::Instant::now(); + decode_search_batch(&batch, &mut search_results)?; + decode_ms += decode_start.elapsed().as_millis(); + batch_count += 1; + } + + // 按分数排序 + search_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + debug!( + "LanceDB stream total={}ms first_batch={}ms decode={}ms batches={} rows={}", + stream_start.elapsed().as_millis(), + first_batch_ms.unwrap_or(0), + decode_ms, + batch_count, + row_count + ); + + Ok(search_results) +} + +pub async fn search_image_by_text( + query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, +) -> Result> { + if has_empty_scope(file_ids, kb_ids) { + return Ok(Vec::new()); + } + + let cfg = config::get(); + let table = get_table()?; + + let query_vector = embedding::get_embedding(query).await?; + let fast_search = vector_fast_search_enabled(); + let mut query_builder = + table.query().nearest_to(query_vector)?.column("vector").select(Select::columns(SEARCH_SELECT_COLUMNS)); + if fast_search { + query_builder = query_builder.fast_search(); + } + + let filter_conditions = build_filter_conditions(true, file_ids, kb_ids); + if !filter_conditions.is_empty() { + query_builder = query_builder.only_if(filter_conditions.join(" AND ")); + } + + let mut result_stream = query_builder.limit(cfg.search.limit).execute().await?; + let mut search_results = Vec::with_capacity(cfg.search.limit); + while let Some(batch_result) = result_stream.next().await { + let batch = batch_result?; + decode_search_batch(&batch, &mut search_results)?; + } + + search_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + Ok(search_results) +} + +pub async fn search_image( + query_vector: Vec, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, +) -> Result> { + if has_empty_scope(file_ids, kb_ids) { + return Ok(Vec::new()); + } + + let cfg = config::get(); + let table = get_table()?; + + let image_vector_dim = config::get().ai.image_embedding_dim; + if query_vector.len() != image_vector_dim as usize { + anyhow::bail!("Image embedding dimension mismatch: expected {}, got {}", image_vector_dim, query_vector.len()); + } + + let mut query_builder = + table.query().nearest_to(query_vector)?.column("image_vector").select(Select::columns(SEARCH_SELECT_COLUMNS)); + + if image_fast_search_enabled() { + query_builder = query_builder.fast_search(); + } + + let filter_conditions = build_filter_conditions(true, file_ids, kb_ids); + + if !filter_conditions.is_empty() { + query_builder = query_builder.only_if(filter_conditions.join(" AND ")); + } + + let mut result_stream = query_builder.limit(cfg.search.limit).execute().await?; + + let mut search_results = Vec::with_capacity(cfg.search.limit); + while let Some(batch_result) = result_stream.next().await { + let batch = batch_result?; + decode_search_batch(&batch, &mut search_results)?; + } + + search_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + + Ok(search_results) +} + +pub async fn search_summary( + query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, +) -> Result> { + if has_empty_scope(file_ids, kb_ids) { + return Ok(Vec::new()); + } + + let cfg = config::get(); + let recall_limit = summary_recall_limit(cfg.search.limit); + let table = get_summary_table()?; + let query_vector = embedding::get_embedding(query).await?; + let fast_search = summary_fast_search_enabled(); + let mut query_builder = + table.query().nearest_to(query_vector)?.column("vector").select(Select::columns(SUMMARY_SEARCH_SELECT_COLUMNS)); + if fast_search { + query_builder = query_builder.fast_search(); + } + + let filter_conditions = build_summary_filter_conditions(file_ids, kb_ids); + if !filter_conditions.is_empty() { + query_builder = query_builder.only_if(filter_conditions.join(" AND ")); + } + + let mut result_stream = query_builder.limit(recall_limit).execute().await?; + let mut search_results = Vec::with_capacity(recall_limit); + while let Some(batch_result) = result_stream.next().await { + let batch = batch_result?; + decode_summary_search_batch(&batch, &mut search_results)?; + } + + search_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + Ok(search_results) +} + +fn summary_recall_limit(final_limit: usize) -> usize { + let final_limit = final_limit.max(1); + let max_recall = SUMMARY_RECALL_MAX.max(final_limit); + final_limit.saturating_mul(SUMMARY_RECALL_MULTIPLIER).min(max_recall) +} + +fn build_in_predicate(column: &str, ids: &[i64]) -> String { + if ids.len() == 1 { + format!("{column} = {}", ids[0]) + } else { + let ids = ids.iter().map(|id| id.to_string()).collect::>().join(", "); + format!("{column} IN ({ids})") + } +} + +async fn delete_by_predicate(predicate: String) -> Result<()> { + delete_by_predicate_inner(predicate, true).await +} + +async fn delete_by_predicate_inner(predicate: String, schedule_optimize: bool) -> Result<()> { + let table = get_table()?; + table.update().only_if(predicate).column(IS_DELETED_COLUMN, "true").execute().await?; + if schedule_optimize { + on_write_may_need_optimize(); + } else { + set_fast_search_enabled(false, &VECTOR_FAST_SEARCH_ENABLED); + set_fast_search_enabled(false, &IMAGE_FAST_SEARCH_ENABLED); + } + Ok(()) +} + +pub async fn delete_by_file(file_id: i64) -> Result<()> { + delete_by_files(&[file_id]).await +} + +pub async fn delete_by_files(file_ids: &[i64]) -> Result<()> { + if file_ids.is_empty() { + return Ok(()); + } + delete_by_predicate(build_in_predicate("file_id", file_ids)).await +} + +pub async fn delete_summary_by_file(file_id: i64) -> Result<()> { + delete_summaries_by_files(&[file_id]).await +} + +pub async fn delete_summaries_by_files(file_ids: &[i64]) -> Result<()> { + if file_ids.is_empty() { + return Ok(()); + } + delete_summary_by_predicate(build_in_predicate("file_id", file_ids), true).await +} + +pub async fn delete_summaries_by_files_for_rebuild(file_ids: &[i64]) -> Result<()> { + if file_ids.is_empty() { + return Ok(()); + } + delete_summary_by_predicate(build_in_predicate("file_id", file_ids), false).await +} + +pub async fn delete_by_slices(slice_ids: &[i64]) -> Result<()> { + if slice_ids.is_empty() { + return Ok(()); + } + delete_by_predicate(build_in_predicate("id", slice_ids)).await +} + +/// 重建期间批量软删除多余切片，由调用方在全部完成后统一刷新索引。 +pub async fn delete_by_slices_for_rebuild(slice_ids: &[i64]) -> Result<()> { + if slice_ids.is_empty() { + return Ok(()); + } + delete_by_predicate_inner(build_in_predicate("id", slice_ids), false).await +} + +pub async fn delete_by_kb(kb_id: i64) -> Result<()> { + delete_by_kbs(&[kb_id]).await +} + +pub async fn delete_by_kbs(kb_ids: &[i64]) -> Result<()> { + if kb_ids.is_empty() { + return Ok(()); + } + delete_by_predicate(build_in_predicate("kb_id", kb_ids)).await +} + +pub async fn delete_summaries_by_kbs(kb_ids: &[i64]) -> Result<()> { + if kb_ids.is_empty() { + return Ok(()); + } + delete_summary_by_predicate(build_in_predicate("kb_id", kb_ids), true).await +} + +async fn delete_summary_by_predicate(predicate: String, schedule_optimize: bool) -> Result<()> { + let table = get_summary_table()?; + table.update().only_if(predicate).column(IS_DELETED_COLUMN, "true").execute().await?; + if schedule_optimize { + on_write_may_need_optimize(); + } else { + set_fast_search_enabled(false, &SUMMARY_FAST_SEARCH_ENABLED); + } + Ok(()) +} + +/// 清理已删除的记录，释放磁盘和内存空间 +pub async fn compact() -> Result { + let _guard = OPTIMIZE_LOCK.lock().await; + + // 先禁用 fast_search，避免在 compact 期间使用未就绪的索引 + VECTOR_FAST_SEARCH_ENABLED.store(false, Ordering::Relaxed); + IMAGE_FAST_SEARCH_ENABLED.store(false, Ordering::Relaxed); + SUMMARY_FAST_SEARCH_ENABLED.store(false, Ordering::Relaxed); + + let table = get_table()?; + let storage_path = config::get().storage.lancedb_path.clone(); + + let size_before_bytes = { + let path = storage_path.clone(); + tokio::task::spawn_blocking(move || dir_size_bytes(Path::new(&path))).await?? + }; + let total_rows_before = table.count_rows(None).await? as u64; + let deleted_rows_before = table.count_rows(Some(format!("{} = true", IS_DELETED_COLUMN))).await? as u64; + + // 删除标记为已删除的记录 + let delete_predicate = format!("{} = true", IS_DELETED_COLUMN); + table.delete(&delete_predicate).await?; + + // 执行 compact / index / prune，回收空间并强制清理旧版本 + table.optimize(OptimizeAction::Compact { options: CompactionOptions::default(), remap_options: None }).await?; + table.optimize(OptimizeAction::Index(OptimizeOptions::default())).await?; + table + .optimize(OptimizeAction::Prune { + older_than: Some(lancedb::table::Duration::minutes(10)), + delete_unverified: Some(false), + error_if_tagged_old_versions: Some(true), + }) + .await?; + + refresh_fast_search_state_for_column(&table, "vector", &VECTOR_FAST_SEARCH_ENABLED).await?; + refresh_fast_search_state_for_column(&table, "image_vector", &IMAGE_FAST_SEARCH_ENABLED).await?; + if let Ok(summary_table) = get_summary_table() { + refresh_fast_search_state_for_column(&summary_table, "vector", &SUMMARY_FAST_SEARCH_ENABLED).await?; + } + + let total_rows_after = table.count_rows(None).await? as u64; + let deleted_rows_after = table.count_rows(Some(format!("{} = true", IS_DELETED_COLUMN))).await? as u64; + let size_after_bytes = { + let path = storage_path.clone(); + tokio::task::spawn_blocking(move || dir_size_bytes(Path::new(&path))).await?? + }; + + Ok(CompactStats { + deleted_rows: deleted_rows_before.saturating_sub(deleted_rows_after), + deleted_rows_before, + deleted_rows_after, + total_rows_before, + total_rows_after, + size_before_bytes, + size_after_bytes, + }) +} + +fn get_connection() -> Result> { + LANCEDB.get().cloned().ok_or_else(|| anyhow::anyhow!("LanceDB not initialized")) +} + +fn get_table() -> Result> { + LANCEDB_TABLE.get().cloned().ok_or_else(|| anyhow::anyhow!("LanceDB table not initialized")) +} + +fn get_summary_table() -> Result> { + SUMMARY_LANCEDB_TABLE.get().cloned().ok_or_else(|| anyhow::anyhow!("LanceDB summary table not initialized")) +} + +/// 清空 LanceDB 中所有文档，用于从 SQLite 完全重建。 +pub async fn clear_all_documents() -> Result<()> { + let table = get_table()?; + table.delete("true").await?; + on_write_may_need_optimize(); + Ok(()) +} + +pub async fn load_existing_summaries(expected_count: u64) -> Result> { + let table = get_summary_table()?; + let predicate = format!("({} = false OR {} IS NULL)", IS_DELETED_COLUMN, IS_DELETED_COLUMN); + let started_at = std::time::Instant::now(); + + info!("Loading existing LanceDB file summaries: expected={}", expected_count); + let capacity = usize::try_from(expected_count).unwrap_or(0); + let mut existing = Vec::with_capacity(capacity); + let mut stream = + table.query().select(Select::columns(&["file_id", "kb_id", "summary"])).only_if(predicate).execute().await?; + + while let Some(batch_result) = stream.next().await { + let batch = batch_result?; + let file_id_array = batch + .column_by_name("file_id") + .and_then(|col| col.as_any().downcast_ref::()) + .ok_or_else(|| anyhow::anyhow!("Missing or invalid file_id column"))?; + let kb_id_array = batch.column_by_name("kb_id").and_then(|col| col.as_any().downcast_ref::()); + let summary_array = batch + .column_by_name("summary") + .and_then(|col| col.as_any().downcast_ref::()) + .ok_or_else(|| anyhow::anyhow!("Missing or invalid summary column"))?; + + for i in 0..batch.num_rows() { + let file_id = file_id_array.value(i); + let kb_id = kb_id_array.and_then(|arr| if arr.is_null(i) { None } else { Some(arr.value(i)) }); + existing.push(SummaryDocument::new(file_id, kb_id, summary_array.value(i).to_string())); + } + } + + info!( + "Loaded existing LanceDB file summaries: count={} elapsed={}ms", + existing.len(), + started_at.elapsed().as_millis() + ); + Ok(existing) +} + +/// 一次性加载 LanceDB 中所有未软删除的文档 id。 +pub async fn load_existing_ids(expected_count: u64) -> Result> { + let table = get_table()?; + let predicate = format!("({} = false OR {} IS NULL)", IS_DELETED_COLUMN, IS_DELETED_COLUMN); + let started_at = std::time::Instant::now(); + + info!("Loading existing LanceDB document ids: expected={}", expected_count); + let capacity = usize::try_from(expected_count).unwrap_or(0); + let mut existing = HashSet::with_capacity(capacity); + let mut stream = table.query().select(Select::columns(&["id"])).only_if(predicate).execute().await?; + let mut heartbeat = tokio::time::interval(std::time::Duration::from_secs(10)); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + heartbeat.tick().await; + + loop { + let next_batch = stream.next(); + tokio::pin!(next_batch); + let batch_result = loop { + tokio::select! { + batch_result = &mut next_batch => break batch_result, + _ = heartbeat.tick() => { + info!( + "Loading existing LanceDB document ids: loaded={}/{} elapsed={}s (waiting for next batch)", + existing.len(), + expected_count, + started_at.elapsed().as_secs() + ); + } + } + }; + + let Some(batch_result) = batch_result else { + break; + }; + let batch = batch_result?; + let id_array = batch + .column_by_name("id") + .and_then(|col| col.as_any().downcast_ref::()) + .ok_or_else(|| anyhow::anyhow!("Missing or invalid id column"))?; + for i in 0..batch.num_rows() { + existing.insert(id_array.value(i)); + } + let percent = if expected_count == 0 { 100.0 } else { existing.len() as f64 * 100.0 / expected_count as f64 }; + info!( + "Loading existing LanceDB document ids: loaded={}/{} ({:.1}%) elapsed={}s", + existing.len(), + expected_count, + percent, + started_at.elapsed().as_secs() + ); + } + + info!( + "Loaded existing LanceDB document ids: count={} elapsed={}ms", + existing.len(), + started_at.elapsed().as_millis() + ); + Ok(existing) +} + +fn vector_fast_search_enabled() -> bool { + VECTOR_FAST_SEARCH_ENABLED.load(Ordering::Relaxed) +} + +fn image_fast_search_enabled() -> bool { + IMAGE_FAST_SEARCH_ENABLED.load(Ordering::Relaxed) +} + +fn summary_fast_search_enabled() -> bool { + SUMMARY_FAST_SEARCH_ENABLED.load(Ordering::Relaxed) +} + +fn set_fast_search_enabled(enabled: bool, flag: &AtomicBool) { + flag.store(enabled, Ordering::Relaxed); +} + +/// 写/删操作后立即调用：先禁用 fast_search 防止读到未索引数据，再触发一次防抖增量优化。 +fn on_write_may_need_optimize() { + set_fast_search_enabled(false, &VECTOR_FAST_SEARCH_ENABLED); + set_fast_search_enabled(false, &IMAGE_FAST_SEARCH_ENABLED); + set_fast_search_enabled(false, &SUMMARY_FAST_SEARCH_ENABLED); + + let version = OPTIMIZE_VERSION.fetch_add(1, Ordering::Relaxed).wrapping_add(1); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(OPTIMIZE_DEBOUNCE_MS)).await; + run_debounced_optimize(version).await; + }); +} + +async fn run_debounced_optimize(version: u64) { + // 如果版本号已被后续写操作超过，跳过本次优化 + if OPTIMIZE_VERSION.load(Ordering::Relaxed) != version { + return; + } + + let _guard = OPTIMIZE_LOCK.lock().await; + + // 再次检查版本，避免在等锁期间又有新写入 + if OPTIMIZE_VERSION.load(Ordering::Relaxed) != version { + return; + } + + let table = match get_table() { + Ok(t) => t, + Err(e) => { + warn!("LanceDB debounced optimize failed to get table: {}", e); + return; + } + }; + + let indices_after_ensure = match ensure_search_indices(&table).await { + Ok(indices) => Some(indices), + Err(e) => { + warn!("LanceDB ensure indices after write failed: {}", e); + None + } + }; + + let optimize_start = std::time::Instant::now(); + match table.optimize(OptimizeAction::Index(OptimizeOptions::default())).await { + Ok(stats) => { + debug!("LanceDB debounced optimize_index done in {}ms: {:?}", optimize_start.elapsed().as_millis(), stats); + let indices = match indices_after_ensure { + Some(indices) => Ok(indices), + None => table.list_indices().await, + }; + match indices { + Ok(indices) => { + if let Err(e) = + refresh_fast_search_state_from_indices(&table, &indices, "vector", &VECTOR_FAST_SEARCH_ENABLED) + .await + { + warn!("LanceDB refresh vector fast_search after optimize failed: {}", e); + } + if let Err(e) = refresh_fast_search_state_from_indices( + &table, + &indices, + "image_vector", + &IMAGE_FAST_SEARCH_ENABLED, + ) + .await + { + warn!("LanceDB refresh image_vector fast_search after optimize failed: {}", e); + } + if let Ok(summary_table) = get_summary_table() + && let Err(e) = maintain_summary_search_indices(&summary_table).await + { + warn!("LanceDB maintain summary vector index after optimize failed: {}", e); + } + } + Err(e) => warn!("LanceDB list indices after optimize failed: {}", e), + } + } + Err(e) => { + warn!("LanceDB debounced optimize_index failed: {} ({}ms)", e, optimize_start.elapsed().as_millis()); + } + } +} + +async fn refresh_fast_search_state_for_column(table: &Table, column: &str, flag: &AtomicBool) -> Result<()> { + let indices = table.list_indices().await?; + refresh_fast_search_state_from_indices(table, &indices, column, flag).await +} + +async fn refresh_fast_search_state_from_indices( + table: &Table, indices: &[lancedb::index::IndexConfig], column: &str, flag: &AtomicBool, +) -> Result<()> { + let Some(index_config) = indices.iter().find(|idx| idx.columns.iter().any(|c| c == column)) else { + set_fast_search_enabled(false, flag); + warn!("LanceDB index missing on column={}; fast_search disabled", column); + return Ok(()); + }; + + let Some(stats) = table.index_stats(&index_config.name).await? else { + set_fast_search_enabled(false, flag); + warn!("LanceDB index stats missing for index={}; fast_search disabled", index_config.name); + return Ok(()); + }; + + let fast_search = stats.num_unindexed_rows == 0 && stats.num_indexed_rows > 0; + set_fast_search_enabled(fast_search, flag); + info!( + "LanceDB index ready: column={} name={} type={:?} indexed_rows={} unindexed_rows={} fast_search={}", + column, index_config.name, stats.index_type, stats.num_indexed_rows, stats.num_unindexed_rows, fast_search + ); + + Ok(()) +} + +async fn maintain_search_indices(table: &Table) -> Result<()> { + let indices = ensure_search_indices(table).await?; + refresh_fast_search_state_from_indices(table, &indices, "vector", &VECTOR_FAST_SEARCH_ENABLED).await?; + refresh_fast_search_state_from_indices(table, &indices, "image_vector", &IMAGE_FAST_SEARCH_ENABLED).await?; + Ok(()) +} + +async fn maintain_summary_search_indices(table: &Table) -> Result<()> { + let indices = ensure_summary_search_indices(table).await?; + refresh_fast_search_state_from_indices(table, &indices, "vector", &SUMMARY_FAST_SEARCH_ENABLED).await?; + Ok(()) +} + +fn has_empty_scope(file_ids: Option<&Vec>, kb_ids: Option<&Vec>) -> bool { + matches!(file_ids, Some(ids) if ids.is_empty()) || matches!(kb_ids, Some(ids) if ids.is_empty()) +} + +fn build_filter_conditions(is_image: bool, file_ids: Option<&Vec>, kb_ids: Option<&Vec>) -> Vec { + let mut filter_conditions = vec![ + format!("({} = false OR {} IS NULL)", IS_DELETED_COLUMN, IS_DELETED_COLUMN), + format!("is_image = {}", is_image), + ]; + + if let Some(fids) = file_ids { + let ids_str = fids.iter().map(|id| id.to_string()).collect::>().join(", "); + filter_conditions.push(format!("file_id IN ({})", ids_str)); + } + if let Some(kids) = kb_ids { + let ids_str = kids.iter().map(|id| id.to_string()).collect::>().join(", "); + filter_conditions.push(format!("kb_id IN ({})", ids_str)); + } + + filter_conditions +} + +fn build_summary_filter_conditions(file_ids: Option<&Vec>, kb_ids: Option<&Vec>) -> Vec { + let mut filter_conditions = vec![format!("({} = false OR {} IS NULL)", IS_DELETED_COLUMN, IS_DELETED_COLUMN)]; + + if let Some(fids) = file_ids { + let ids_str = fids.iter().map(|id| id.to_string()).collect::>().join(", "); + filter_conditions.push(format!("file_id IN ({})", ids_str)); + } + if let Some(kids) = kb_ids { + let ids_str = kids.iter().map(|id| id.to_string()).collect::>().join(", "); + filter_conditions.push(format!("kb_id IN ({})", ids_str)); + } + + filter_conditions +} + +fn decode_search_batch(batch: &RecordBatch, search_results: &mut Vec) -> Result<()> { + let num_rows = batch.num_rows(); + + let id_array = batch + .column_by_name("id") + .and_then(|col| col.as_any().downcast_ref::()) + .ok_or_else(|| anyhow::anyhow!("Missing or invalid id column"))?; + + let file_id_array = batch + .column_by_name("file_id") + .and_then(|col| col.as_any().downcast_ref::()) + .ok_or_else(|| anyhow::anyhow!("Missing or invalid file_id column"))?; + + let kb_id_array = batch.column_by_name("kb_id").and_then(|col| col.as_any().downcast_ref::()); + + let content_array = batch + .column_by_name("content") + .and_then(|col| col.as_any().downcast_ref::()) + .ok_or_else(|| anyhow::anyhow!("Missing or invalid content column"))?; + + let distance_array = batch.column_by_name("_distance").and_then(|col| col.as_any().downcast_ref::()); + + for i in 0..num_rows { + let id = id_array.value(i); + let file_id = file_id_array.value(i); + let kb_id = kb_id_array.and_then(|arr| if arr.is_null(i) { None } else { Some(arr.value(i)) }); + let content = content_array.value(i).to_string(); + let score = distance_array.map_or(0.5, |arr| distance_to_score(arr.value(i))); + + search_results.push(SearchResultItem { id, file_id, kb_id, content, score }); + } + + Ok(()) +} + +fn decode_summary_search_batch(batch: &RecordBatch, search_results: &mut Vec) -> Result<()> { + let num_rows = batch.num_rows(); + + let file_id_array = batch + .column_by_name("file_id") + .and_then(|col| col.as_any().downcast_ref::()) + .ok_or_else(|| anyhow::anyhow!("Missing or invalid file_id column"))?; + + let kb_id_array = batch.column_by_name("kb_id").and_then(|col| col.as_any().downcast_ref::()); + + let summary_array = batch + .column_by_name("summary") + .and_then(|col| col.as_any().downcast_ref::()) + .ok_or_else(|| anyhow::anyhow!("Missing or invalid summary column"))?; + + let distance_array = batch.column_by_name("_distance").and_then(|col| col.as_any().downcast_ref::()); + + for i in 0..num_rows { + let file_id = file_id_array.value(i); + let kb_id = kb_id_array.and_then(|arr| if arr.is_null(i) { None } else { Some(arr.value(i)) }); + let summary = summary_array.value(i).to_string(); + let score = distance_array.map_or(0.5, |arr| distance_to_score(arr.value(i))); + + search_results.push(SummarySearchResultItem { file_id, kb_id, summary, score }); + } + + Ok(()) +} + +fn distance_to_score(distance: f32) -> f32 { + (1.0 / (1.0 + distance)).clamp(0.0, 1.0) +} + +fn dir_size_bytes(path: &Path) -> Result { + if !path.exists() { + return Ok(0); + } + let mut total = 0u64; + for entry in std::fs::read_dir(path)? { + let entry = entry?; + let metadata = entry.metadata()?; + if metadata.is_dir() { + total = total.saturating_add(dir_size_bytes(&entry.path())?); + } else { + total = total.saturating_add(metadata.len()); + } + } + Ok(total) +} + +fn create_schema() -> Arc { + let vector_dim = config::get().ai.embedding_dim; + let image_vector_dim = config::get().ai.image_embedding_dim; + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("file_id", DataType::Int64, false), + Field::new("kb_id", DataType::Int64, true), + Field::new("content", DataType::Utf8, false), + Field::new("is_image", DataType::Boolean, false), + Field::new(IS_DELETED_COLUMN, DataType::Boolean, true), + // 向量字段 - 从配置获取维度 + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), vector_dim), + false, + ), + Field::new( + "image_vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), image_vector_dim), + true, + ), + ])) +} + +fn create_summary_schema() -> Arc { + let vector_dim = config::get().ai.embedding_dim; + Arc::new(ArrowSchema::new(vec![ + Field::new("file_id", DataType::Int64, false), + Field::new("kb_id", DataType::Int64, true), + Field::new("summary", DataType::Utf8, false), + Field::new(IS_DELETED_COLUMN, DataType::Boolean, true), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), vector_dim), + false, + ), + ])) +} + +fn create_empty_batch(schema: &Arc) -> Result { + let vector_dim = config::get().ai.embedding_dim; + let image_vector_dim = config::get().ai.image_embedding_dim; + let id_array: ArrayRef = Arc::new(Int64Array::from(Vec::::new())); + let file_id_array: ArrayRef = Arc::new(Int64Array::from(Vec::::new())); + let kb_id_array: ArrayRef = Arc::new(Int64Array::from(Vec::>::new())); + let content_array: ArrayRef = Arc::new(StringArray::from(Vec::::new())); + let is_image_array: ArrayRef = Arc::new(BooleanArray::from(Vec::::new())); + let is_deleted_array: ArrayRef = Arc::new(BooleanArray::from(Vec::::new())); + + // 创建空的向量数组 + let value_builder = Float32Builder::new(); + let mut list_builder = FixedSizeListBuilder::new(value_builder, vector_dim); + let vector_array: ArrayRef = Arc::new(list_builder.finish()); + + let image_value_builder = Float32Builder::new(); + let mut image_list_builder = FixedSizeListBuilder::new(image_value_builder, image_vector_dim); + let image_vector_array: ArrayRef = Arc::new(image_list_builder.finish()); + + Ok(RecordBatch::try_new( + schema.clone(), + vec![ + id_array, + file_id_array, + kb_id_array, + content_array, + is_image_array, + is_deleted_array, + vector_array, + image_vector_array, + ], + )?) +} + +fn create_summary_empty_batch(schema: &Arc) -> Result { + let vector_dim = config::get().ai.embedding_dim; + let file_id_array: ArrayRef = Arc::new(Int64Array::from(Vec::::new())); + let kb_id_array: ArrayRef = Arc::new(Int64Array::from(Vec::>::new())); + let summary_array: ArrayRef = Arc::new(StringArray::from(Vec::::new())); + let is_deleted_array: ArrayRef = Arc::new(BooleanArray::from(Vec::::new())); + + let value_builder = Float32Builder::new(); + let mut list_builder = FixedSizeListBuilder::new(value_builder, vector_dim); + let vector_array: ArrayRef = Arc::new(list_builder.finish()); + + Ok(RecordBatch::try_new( + schema.clone(), + vec![file_id_array, kb_id_array, summary_array, is_deleted_array, vector_array], + )?) +} + +async fn create_record_batch(docs: Vec, schema: &Arc) -> Result { + let vector_dim = config::get().ai.embedding_dim; + let image_vector_dim = config::get().ai.image_embedding_dim; + let ids: Vec = docs.iter().map(|d| d.id).collect(); + let file_ids: Vec = docs.iter().map(|d| d.file_id).collect(); + let kb_ids: Vec> = docs.iter().map(|d| d.kb_id).collect(); + let contents: Vec = docs.iter().map(|d| d.content.clone()).collect(); + let is_images: Vec = docs.iter().map(|d| d.is_image).collect(); + let is_deleted: Vec = vec![false; docs.len()]; + + // 先用 contents 计算 embedding，再把所有权转移给 StringArray，避免多一次 clone + let embeddings = embedding::get_embeddings(&contents).await?; + + let id_array: ArrayRef = Arc::new(Int64Array::from(ids)); + let file_id_array: ArrayRef = Arc::new(Int64Array::from(file_ids)); + let kb_id_array: ArrayRef = Arc::new(Int64Array::from(kb_ids)); + let content_array: ArrayRef = Arc::new(StringArray::from(contents)); + let is_image_array: ArrayRef = Arc::new(BooleanArray::from(is_images)); + let is_deleted_array: ArrayRef = Arc::new(BooleanArray::from(is_deleted)); + + let value_builder = Float32Builder::new(); + let mut list_builder = FixedSizeListBuilder::new(value_builder, vector_dim); + + for embedding_vec in embeddings { + if embedding_vec.len() != vector_dim as usize { + anyhow::bail!("Embedding dimension mismatch: expected {}, got {}", vector_dim, embedding_vec.len()); + } + + let values_builder = list_builder.values(); + for &value in &embedding_vec { + values_builder.append_value(value); + } + list_builder.append(true); + } + + let vector_array: ArrayRef = Arc::new(list_builder.finish()); + + let image_value_builder = Float32Builder::new(); + let mut image_list_builder = FixedSizeListBuilder::new(image_value_builder, image_vector_dim); + for doc in &docs { + let values_builder = image_list_builder.values(); + if let Some(image_embedding) = &doc.image_embedding { + if image_embedding.len() != image_vector_dim as usize { + anyhow::bail!( + "Image embedding dimension mismatch: expected {}, got {}", + image_vector_dim, + image_embedding.len() + ); + } + for &value in image_embedding.as_ref() { + values_builder.append_value(value); + } + image_list_builder.append(true); + } else { + for _ in 0..image_vector_dim { + values_builder.append_value(0.0); + } + image_list_builder.append(false); + } + } + let image_vector_array: ArrayRef = Arc::new(image_list_builder.finish()); + + Ok(RecordBatch::try_new( + schema.clone(), + vec![ + id_array, + file_id_array, + kb_id_array, + content_array, + is_image_array, + is_deleted_array, + vector_array, + image_vector_array, + ], + )?) +} + +async fn create_summary_record_batch(docs: Vec, schema: &Arc) -> Result { + let vector_dim = config::get().ai.embedding_dim; + let file_ids: Vec = docs.iter().map(|d| d.file_id).collect(); + let kb_ids: Vec> = docs.iter().map(|d| d.kb_id).collect(); + let summaries: Vec = docs.iter().map(|d| d.summary.trim().to_string()).collect(); + let is_deleted: Vec = vec![false; docs.len()]; + let embeddings = embedding::get_embeddings(&summaries).await?; + + let file_id_array: ArrayRef = Arc::new(Int64Array::from(file_ids)); + let kb_id_array: ArrayRef = Arc::new(Int64Array::from(kb_ids)); + let summary_array: ArrayRef = Arc::new(StringArray::from(summaries)); + let is_deleted_array: ArrayRef = Arc::new(BooleanArray::from(is_deleted)); + + let value_builder = Float32Builder::new(); + let mut list_builder = FixedSizeListBuilder::new(value_builder, vector_dim); + + for embedding_vec in embeddings { + if embedding_vec.len() != vector_dim as usize { + anyhow::bail!("Summary embedding dimension mismatch: expected {}, got {}", vector_dim, embedding_vec.len()); + } + + let values_builder = list_builder.values(); + for &value in &embedding_vec { + values_builder.append_value(value); + } + list_builder.append(true); + } + + let vector_array: ArrayRef = Arc::new(list_builder.finish()); + + Ok(RecordBatch::try_new( + schema.clone(), + vec![file_id_array, kb_id_array, summary_array, is_deleted_array, vector_array], + )?) +} + +async fn ensure_is_deleted_column(table: &lancedb::Table) -> Result<()> { + let schema = table.schema().await?; + if schema.field_with_name(IS_DELETED_COLUMN).is_err() { + table + .add_columns( + NewColumnTransform::SqlExpressions(vec![(IS_DELETED_COLUMN.to_string(), "false".to_string())]), + None, + ) + .await?; + } + Ok(()) +} + +async fn ensure_search_indices(table: &Table) -> Result> { + let existing_indices = table.list_indices().await?; + let row_count = table.count_rows(None).await?; + let mut created_index = false; + + if row_count >= MIN_VECTORS_FOR_INDEX { + if !has_index_on_column(&existing_indices, "vector") { + info!("Creating LanceDB vector index on column=vector (vectors={})", row_count); + table.create_index(&["vector"], Index::Auto).replace(false).execute().await?; + created_index = true; + } + } else { + info!( + "Deferring LanceDB text vector index: vectors={} minimum={} (exact search remains enabled)", + row_count, MIN_VECTORS_FOR_INDEX + ); + } + + let scalar_indices = [ + ("id", Index::BTree(BTreeIndexBuilder::default())), + ("file_id", Index::BTree(BTreeIndexBuilder::default())), + ("kb_id", Index::BTree(BTreeIndexBuilder::default())), + ("is_deleted", Index::Bitmap(BitmapIndexBuilder::default())), + ("is_image", Index::Bitmap(BitmapIndexBuilder::default())), + ]; + for (column, index) in scalar_indices { + if !has_index_on_column(&existing_indices, column) { + info!("Creating LanceDB scalar index on column={}", column); + table.create_index(&[column], index).replace(false).execute().await?; + created_index = true; + } + } + + if row_count > 0 && !has_index_on_column(&existing_indices, "image_vector") { + let image_count = table + .count_rows(Some(format!( + "is_image = true AND image_vector IS NOT NULL AND ({} = false OR {} IS NULL)", + IS_DELETED_COLUMN, IS_DELETED_COLUMN + ))) + .await?; + if image_count >= MIN_VECTORS_FOR_INDEX { + info!("Creating LanceDB vector index on column=image_vector (vectors={})", image_count); + table.create_index(&["image_vector"], Index::Auto).replace(false).execute().await?; + created_index = true; + } else { + info!( + "Deferring LanceDB image vector index: vectors={} minimum={} (exact search remains enabled)", + image_count, MIN_VECTORS_FOR_INDEX + ); + } + } + + let refreshed_indices = if created_index { table.list_indices().await? } else { existing_indices }; + debug!( + "LanceDB indices: {}", + refreshed_indices + .iter() + .map(|idx| format!("{}:{:?}:{:?}", idx.name, idx.index_type, idx.columns)) + .collect::>() + .join(", ") + ); + + Ok(refreshed_indices) +} + +async fn ensure_summary_search_indices(table: &Table) -> Result> { + let existing_indices = table.list_indices().await?; + let row_count = table.count_rows(None).await?; + let mut created_index = false; + + if row_count >= MIN_VECTORS_FOR_INDEX { + if !has_index_on_column(&existing_indices, "vector") { + info!("Creating LanceDB summary vector index on column=vector (vectors={})", row_count); + table.create_index(&["vector"], Index::Auto).replace(false).execute().await?; + created_index = true; + } + } else { + info!( + "Deferring LanceDB summary vector index: vectors={} minimum={} (exact search remains enabled)", + row_count, MIN_VECTORS_FOR_INDEX + ); + } + + let scalar_indices = [ + ("file_id", Index::BTree(BTreeIndexBuilder::default())), + ("kb_id", Index::BTree(BTreeIndexBuilder::default())), + ("is_deleted", Index::Bitmap(BitmapIndexBuilder::default())), + ]; + for (column, index) in scalar_indices { + if !has_index_on_column(&existing_indices, column) { + info!("Creating LanceDB summary scalar index on column={}", column); + table.create_index(&[column], index).replace(false).execute().await?; + created_index = true; + } + } + + let refreshed_indices = if created_index { table.list_indices().await? } else { existing_indices }; + debug!( + "LanceDB summary indices: {}", + refreshed_indices + .iter() + .map(|idx| format!("{}:{:?}:{:?}", idx.name, idx.index_type, idx.columns)) + .collect::>() + .join(", ") + ); + + Ok(refreshed_indices) +} + +fn has_index_on_column(indices: &[lancedb::index::IndexConfig], column: &str) -> bool { + indices.iter().any(|idx| idx.columns.iter().any(|candidate| candidate == column)) +} diff --git a/src/search/mod.rs b/src/search/mod.rs new file mode 100644 index 0000000..ee2899c --- /dev/null +++ b/src/search/mod.rs @@ -0,0 +1,1969 @@ +use std::{ + collections::{HashMap, HashSet}, + fs, + path::Path, + sync::Arc, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, anyhow}; +use log::{debug, info, warn}; +use once_cell::sync::Lazy; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use sqlx::{QueryBuilder, Sqlite, SqlitePool}; +use tantivy::{Index, IndexReader, ReloadPolicy, schema::Schema}; +use tokio::sync::Mutex; + +use crate::config; + +pub mod advanced; +mod chinese_tokenizer; +pub mod embedding; +mod lancedb; +pub mod tantivy_engine; + +pub use lancedb::schedule_startup_index_maintenance; +pub use tantivy_engine::{FullSearchResultItem, SearchResultItem}; + +const FULL_SNIPPET_MAX_CHARS: usize = 400; +const MAX_QUERY_TERMS_FOR_SYNONYM_LOOKUP: usize = 100; +const DEFAULT_REBUILD_BATCH_SIZE: i64 = 100; +static RERANK_HTTP_CLIENT: Lazy = Lazy::new(Client::new); + +/// /v1/rerank 格式的请求体 +#[derive(Debug, Serialize)] +struct RerankRequest { + model: String, + query: String, + documents: Vec, +} + +/// /v1/rerank 格式的响应体 +#[derive(Debug, Deserialize)] +struct RerankResponse { + results: Vec, +} + +#[derive(Debug, Deserialize)] +struct RerankResult { + index: usize, + relevance_score: f32, +} + +/// /rerank 格式的请求体 +#[derive(Debug, Serialize)] +struct SimpleRerankRequest { + query: String, + texts: Vec, +} + +/// /rerank 格式的响应体（数组） +#[derive(Debug, Deserialize)] +struct SimpleRerankResult { + index: usize, + score: f32, +} + +#[derive(Debug, sqlx::FromRow)] +struct SynonymRow { + term: String, + synonym: String, + weight: f32, + bidirectional: i64, +} + +/// 同义词表 TTL 缓存：避免每次查询都扫库。变更会在 TTL 窗口内生效。 +const SYNONYM_CACHE_TTL: Duration = Duration::from_secs(60); + +struct SynonymCache { + loaded_at: Instant, + rows: Vec, + /// row.term -> rows 下标 + by_term: HashMap>, + /// row.synonym -> rows 下标 + by_synonym: HashMap>, +} + +impl SynonymCache { + fn build(rows: Vec) -> Self { + let mut by_term: HashMap> = HashMap::new(); + let mut by_synonym: HashMap> = HashMap::new(); + for (idx, row) in rows.iter().enumerate() { + by_term.entry(row.term.clone()).or_default().push(idx); + by_synonym.entry(row.synonym.clone()).or_default().push(idx); + } + Self { loaded_at: Instant::now(), rows, by_term, by_synonym } + } +} + +#[derive(Debug, sqlx::FromRow)] +struct LexiconRow { + term: String, + freq: Option, + tag: Option, +} + +#[derive(Debug, sqlx::FromRow)] +struct RebuildSliceRow { + id: i64, + source_file_id: i64, + file_id: i64, + kb_id: Option, +} + +#[derive(Debug, sqlx::FromRow)] +struct RebuildLanceDbSliceRow { + id: i64, + source_file_id: i64, + file_id: i64, + kb_id: Option, + content: String, + filename: String, + path: String, +} + +#[derive(Debug, sqlx::FromRow)] +struct RebuildFullMetaRow { + id: i64, + kb_id: Option, + filename: String, +} + +#[derive(Debug, Clone)] +pub struct RebuildProgress { + pub phase: String, + pub total_docs: i64, + pub processed_docs: i64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SummarySearchResultItem { + pub file_id: i64, + pub kb_id: Option, + pub summary: String, + pub score: f32, +} + +/// 根据文件名后缀判断是否为图片文件。 +pub fn is_image_file(filename: &str) -> bool { + let lower = filename.to_lowercase(); + lower.ends_with(".jpg") + || lower.ends_with(".jpeg") + || lower.ends_with(".png") + || lower.ends_with(".gif") + || lower.ends_with(".bmp") + || lower.ends_with(".webp") + || lower.ends_with(".tiff") + || lower.ends_with(".tif") + || lower.ends_with(".svg") + || lower.ends_with(".ico") + || lower.ends_with(".avif") + || lower.ends_with(".heic") + || lower.ends_with(".heif") +} + +/// 根据切片内容判断是否包含图片引用。 +pub fn content_looks_like_image_reference(content: &str) -> bool { + content.contains("![") && content.contains("](/api/v1/knowledge/files/") +} + +async fn fetch_rebuild_lancedb_rows( + pool: &SqlitePool, slice_ids: &[i64], +) -> anyhow::Result> { + if slice_ids.is_empty() { + return Ok(Vec::new()); + } + + let mut query_builder: QueryBuilder<'_, Sqlite> = QueryBuilder::new( + "SELECT s.id, s.file_id AS source_file_id, COALESCE(ref.id, source.id) AS file_id, COALESCE(ref.kb_id, source.kb_id) AS kb_id, \ + COALESCE(ref.filename, source.filename) AS filename, \ + COALESCE(ref.path, source.path) AS path \ + FROM slices s JOIN files source ON source.id = s.file_id \ + LEFT JOIN parse_artifacts pa ON pa.source_file_id = source.id \ + LEFT JOIN files ref ON ref.artifact_id = pa.id WHERE s.id IN (", + ); + let mut separated = query_builder.separated(", "); + for slice_id in slice_ids { + separated.push_bind(slice_id); + } + separated.push_unseparated(") ORDER BY s.id ASC"); + let mut rows: Vec = query_builder.build_query_as().fetch_all(pool).await?; + let mut contents = HashMap::new(); + for source_file_id in rows.iter().map(|row| row.source_file_id).collect::>() { + contents.insert(source_file_id, crate::slice_content::read_all(source_file_id).await?); + } + for row in &mut rows { + row.content = contents.get(&row.source_file_id).and_then(|v| v.get(&row.id)).cloned().unwrap_or_default(); + } + Ok(rows) +} + +#[derive(Clone)] +pub struct SearchEngine { + schema: Schema, + index_reader: IndexReader, + index_write_lock: Arc>, + index_writer: Arc, + full_schema: Schema, + full_index_reader: IndexReader, + full_index_write_lock: Arc>, + full_index_writer: Arc, + rebuild_lock: Arc>, + pool: Option, + synonym_cache: Arc>>, + /// LanceDB 在本次启动时是否被新建或从损坏中恢复，true 表示需要从 SQLite 回填向量。 + lancedb_recreated: bool, +} + +impl SearchEngine { + pub async fn init() -> Self { + let t0 = Instant::now(); + let lancedb_recreated = lancedb::init().await.expect("init lancedb failed"); + info!("Search init substep: lancedb::init() took {}ms", t0.elapsed().as_millis()); + + let t1 = Instant::now(); + let (schema, index) = tantivy_engine::init().unwrap(); + info!("Search init substep: tantivy_engine::init() took {}ms", t1.elapsed().as_millis()); + + let t2 = Instant::now(); + let (full_schema, full_index) = tantivy_engine::init_full().unwrap(); + info!("Search init substep: tantivy_engine::init_full() took {}ms", t2.elapsed().as_millis()); + + let t3 = Instant::now(); + let index_reader = build_reader(&index, "index"); + info!("Search init substep: build_reader(index) took {}ms", t3.elapsed().as_millis()); + + let t4 = Instant::now(); + let full_index_reader = build_reader(&full_index, "full_index"); + info!("Search init substep: build_reader(full_index) took {}ms", t4.elapsed().as_millis()); + + let t5 = Instant::now(); + let index_writer = tantivy_engine::IndexWriterHandle::open(index, schema.clone(), "index".to_string()) + .await + .expect("open tantivy index writer failed"); + info!("Search init substep: open index writer took {}ms", t5.elapsed().as_millis()); + + let t6 = Instant::now(); + let full_index_writer = + tantivy_engine::IndexWriterHandle::open(full_index, full_schema.clone(), "full_index".to_string()) + .await + .expect("open tantivy full index writer failed"); + info!("Search init substep: open full index writer took {}ms", t6.elapsed().as_millis()); + + Self { + schema, + index_reader, + index_write_lock: Arc::new(Mutex::new(())), + index_writer, + full_schema, + full_index_reader, + full_index_write_lock: Arc::new(Mutex::new(())), + full_index_writer, + rebuild_lock: Arc::new(Mutex::new(())), + pool: None, + synonym_cache: Arc::new(tokio::sync::RwLock::new(None)), + lancedb_recreated, + } + } + + pub fn with_pool(mut self, pool: SqlitePool) -> Self { + self.pool = Some(pool); + self + } + + /// 检查 SQLite 的切片数量与 LanceDB 有效文档数是否一致，不一致则增量补齐缺失的切片。 + /// 图片文件会重新调用图片 embedding 服务恢复 image_vector。 + pub async fn maybe_rebuild_lancedb_from_db(&self) -> anyhow::Result<()> { + let Some(pool) = &self.pool else { + return Err(anyhow!("search engine db pool not set")); + }; + + // 只统计仍有对应 file 的有效 slice；孤儿切片会在后续被清理出 LanceDB。 + let total_slices: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM slices s WHERE EXISTS (SELECT 1 FROM files f WHERE f.id = s.file_id)", + ) + .fetch_one(pool) + .await?; + info!("Checking LanceDB document ids against {} valid SQLite slices...", total_slices); + let t_load_ids = Instant::now(); + let mut unmatched_lancedb_ids = if self.lancedb_recreated { + HashSet::new() + } else { + lancedb::load_existing_ids(total_slices as u64).await? + }; + info!("LanceDB rebuild substep: load_existing_ids() took {}ms", t_load_ids.elapsed().as_millis()); + let lancedb_count = unmatched_lancedb_ids.len(); + + if total_slices == 0 { + if lancedb_count > 0 { + warn!("SQLite has no valid slices but LanceDB has {} documents; clearing LanceDB", lancedb_count); + lancedb::clear_all_documents().await?; + } + self.sync_lancedb_summaries_from_db().await?; + return Ok(()); + } + + // 加载所有 slice id 并分离出孤儿切片，确保 LanceDB 不保留无效向量。 + let all_slice_rows: Vec<(i64, i64)> = + sqlx::query_as("SELECT s.id, s.file_id FROM slices s ORDER BY s.id ASC").fetch_all(pool).await?; + let valid_file_ids: HashSet = + sqlx::query_scalar("SELECT id FROM files").fetch_all(pool).await?.into_iter().collect(); + let mut sqlite_ids = Vec::with_capacity(all_slice_rows.len()); + let mut orphan_ids = Vec::new(); + for (slice_id, file_id) in all_slice_rows { + if valid_file_ids.contains(&file_id) { + sqlite_ids.push(slice_id); + } else { + orphan_ids.push(slice_id); + } + } + + let mut missing_ids = Vec::with_capacity(total_slices.saturating_sub(lancedb_count as i64).max(0) as usize); + for id in sqlite_ids { + if !unmatched_lancedb_ids.remove(&id) { + missing_ids.push(id); + } + } + // LanceDB 中多出的 id 包括：历史遗留向量、以及本次识别出的孤儿切片向量。 + let mut extra_ids: Vec = unmatched_lancedb_ids.into_iter().collect(); + extra_ids.sort_unstable(); + info!( + "LanceDB comparison completed: sqlite_valid={} lancedb={} missing={} extra={} orphan={}", + total_slices, + lancedb_count, + missing_ids.len(), + extra_ids.len(), + orphan_ids.len() + ); + + let cfg = config::get(); + let batch_size = cfg.search.lancedb_rebuild_batch_size.max(1); + let rebuild_started_at = Instant::now(); + + // 先把孤儿切片向量标记为删除，避免它们继续占用索引并干扰后续比较。 + let mut orphan_removed = 0usize; + for id_batch in orphan_ids.chunks(batch_size) { + lancedb::delete_by_slices_for_rebuild(id_batch).await?; + orphan_removed += id_batch.len(); + info!( + "LanceDB orphan cleanup progress: removed={}/{} total={}s", + orphan_removed, + orphan_ids.len(), + rebuild_started_at.elapsed().as_secs() + ); + } + + if missing_ids.is_empty() && extra_ids.is_empty() && orphan_ids.is_empty() { + self.sync_lancedb_summaries_from_db().await?; + return Ok(()); + } + + if self.lancedb_recreated { + info!("LanceDB was recreated; rebuilding vectors from SQLite slices..."); + } else { + warn!( + "LanceDB differs from SQLite; restoring {} missing slices and removing {} extra slices...", + missing_ids.len(), + extra_ids.len() + ); + } + + let cfg = config::get(); + let batch_size = cfg.search.lancedb_rebuild_batch_size.max(1); + let rebuild_started_at = Instant::now(); + + let mut removed = 0usize; + for id_batch in extra_ids.chunks(batch_size) { + lancedb::delete_by_slices_for_rebuild(id_batch).await?; + removed += id_batch.len(); + info!( + "LanceDB cleanup progress: removed={}/{} total={}s", + removed, + extra_ids.len(), + rebuild_started_at.elapsed().as_secs() + ); + } + + // 只为缺失切片所属的图片文件重建 image_embedding。 + let mut image_embeddings: HashMap>> = HashMap::new(); + let mut attempted_image_embeddings = HashSet::new(); + let mut processed = 0usize; + + for id_batch in missing_ids.chunks(batch_size) { + let t_fetch = Instant::now(); + let rows = fetch_rebuild_lancedb_rows(pool, id_batch).await?; + info!( + "LanceDB rebuild substep: fetch_rebuild_lancedb_rows(ids={}-{}) took {}ms", + id_batch.first().copied().unwrap_or_default(), + id_batch.last().copied().unwrap_or_default(), + t_fetch.elapsed().as_millis() + ); + if rows.len() != id_batch.len() { + return Err(anyhow!( + "Failed to load all missing SQLite slices: requested={}, loaded={}, ids={}-{}", + id_batch.len(), + rows.len(), + id_batch.first().copied().unwrap_or_default(), + id_batch.last().copied().unwrap_or_default() + )); + } + let batch_first_id = id_batch.first().copied().unwrap_or_default(); + let batch_last_id = id_batch.last().copied().unwrap_or_default(); + let batch_started_at = Instant::now(); + + for row in &rows { + if embedding::image_embedding_enabled() + && attempted_image_embeddings.insert(row.file_id) + && is_image_file(&row.filename) + { + match embedding::get_image_embedding_from_path(&row.path, Some(&row.filename)).await { + Ok(image_embedding) => { + image_embeddings.insert(row.file_id, Arc::new(image_embedding)); + } + Err(err) => { + warn!( + "Failed to rebuild image embedding for file {} ({} at {}): {}", + row.file_id, row.filename, row.path, err + ); + } + } + } + } + + let docs: Vec = rows + .into_iter() + .map(|row| { + let mut doc = lancedb::Document::new(row.id, row.file_id, row.kb_id, row.content); + let is_image = content_looks_like_image_reference(&doc.content) || is_image_file(&row.filename); + doc = doc.with_is_image(is_image); + if let Some(image_embedding) = image_embeddings.get(&row.file_id) { + doc = doc.with_image_embedding(image_embedding.clone()); + } + doc + }) + .collect(); + let restored = docs.len(); + lancedb::write_documents_batch_for_rebuild(docs).await?; + processed += restored; + info!( + "LanceDB rebuild progress: restored={}/{} ids={}-{} batch={}ms total={}s", + processed, + missing_ids.len(), + batch_first_id, + batch_last_id, + batch_started_at.elapsed().as_millis(), + rebuild_started_at.elapsed().as_secs() + ); + } + + // 全部批次写完后只触发一次索引刷新。 + lancedb::schedule_optimize_after_rebuild(); + info!( + "LanceDB reconciliation completed: restored={} removed={} elapsed={}s", + processed, + removed, + rebuild_started_at.elapsed().as_secs() + ); + self.sync_lancedb_summaries_from_db().await?; + Ok(()) + } + + async fn sync_lancedb_summaries_from_db(&self) -> anyhow::Result<()> { + let Some(pool) = &self.pool else { + return Err(anyhow!("search engine db pool not set")); + }; + + #[derive(Debug, sqlx::FromRow)] + struct SummaryRow { + id: i64, + kb_id: Option, + summary: String, + } + + let sqlite_rows: Vec = sqlx::query_as( + "SELECT id, kb_id, summary FROM files \ + WHERE status = 1 AND summary IS NOT NULL AND trim(summary) != '' \ + ORDER BY id ASC", + ) + .fetch_all(pool) + .await?; + let mut sqlite_by_file_id = HashMap::with_capacity(sqlite_rows.len()); + for row in sqlite_rows { + let summary = row.summary.trim(); + if !summary.is_empty() { + sqlite_by_file_id.insert(row.id, lancedb::SummaryDocument::new(row.id, row.kb_id, summary.to_string())); + } + } + + let existing_summaries = lancedb::load_existing_summaries(sqlite_by_file_id.len() as u64).await?; + let mut existing_counts: HashMap = HashMap::with_capacity(existing_summaries.len()); + let mut replace_file_ids: HashSet = HashSet::new(); + let mut delete_file_ids: HashSet = HashSet::new(); + + for existing in existing_summaries { + *existing_counts.entry(existing.file_id).or_insert(0) += 1; + match sqlite_by_file_id.get(&existing.file_id) { + Some(expected) if expected.kb_id == existing.kb_id && expected.summary == existing.summary.trim() => {} + Some(_) => { + replace_file_ids.insert(existing.file_id); + } + None => { + delete_file_ids.insert(existing.file_id); + } + } + } + + for (&file_id, &count) in &existing_counts { + if count > 1 { + if sqlite_by_file_id.contains_key(&file_id) { + replace_file_ids.insert(file_id); + } else { + delete_file_ids.insert(file_id); + } + } + } + + for &file_id in sqlite_by_file_id.keys() { + if !existing_counts.contains_key(&file_id) { + replace_file_ids.insert(file_id); + } + } + + for file_id in &replace_file_ids { + delete_file_ids.remove(file_id); + } + + if replace_file_ids.is_empty() && delete_file_ids.is_empty() { + info!("LanceDB file summary vectors are consistent with SQLite: {} summaries", sqlite_by_file_id.len()); + return Ok(()); + } + + let cfg = config::get(); + let batch_size = cfg.search.lancedb_rebuild_batch_size.max(1); + let started_at = Instant::now(); + let replace_docs: Vec = + replace_file_ids.iter().filter_map(|id| sqlite_by_file_id.get(id).cloned()).collect(); + let mut replaced = 0usize; + for docs in replace_docs.chunks(batch_size) { + lancedb::replace_summaries_batch_for_rebuild(docs.to_vec()).await?; + replaced += docs.len(); + } + + let delete_ids: Vec = delete_file_ids.into_iter().collect(); + let mut deleted = 0usize; + for ids in delete_ids.chunks(batch_size) { + lancedb::delete_summaries_by_files_for_rebuild(ids).await?; + deleted += ids.len(); + } + + lancedb::schedule_optimize_after_rebuild(); + info!( + "LanceDB file summary vectors synced from SQLite: total={} replaced={} deleted={} elapsed={}s", + sqlite_by_file_id.len(), + replaced, + deleted, + started_at.elapsed().as_secs() + ); + Ok(()) + } + + /// 检查 SQLite 切片数量与 Tantivy 默认索引文档数是否一致，不一致则从 SQLite 重建默认索引。 + /// 用于 schema 变更（新增 is_image 字段）或索引损坏后的自动恢复。 + pub async fn maybe_rebuild_tantivy_from_db(&self) -> anyhow::Result<()> { + let Some(pool) = &self.pool else { + return Err(anyhow!("search engine db pool not set")); + }; + + // 只统计仍有对应 file 的有效 slice，避免孤儿切片导致无限重建。 + let total_slices: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM slices s WHERE EXISTS (SELECT 1 FROM files f WHERE f.id = s.file_id)", + ) + .fetch_one(pool) + .await?; + let index_docs = self.index_reader.searcher().num_docs() as i64; + if total_slices == index_docs { + info!("Tantivy default index is consistent with SQLite: {} valid docs", total_slices); + return Ok(()); + } + + info!( + "Tantivy default index mismatch: sqlite_valid_slices={} index_docs={}, rebuilding...", + total_slices, index_docs + ); + + #[derive(Debug, sqlx::FromRow)] + struct SliceMeta { + id: i64, + source_file_id: i64, + file_id: i64, + kb_id: Option, + is_image: i64, + } + + let rows: Vec = sqlx::query_as( + "SELECT s.id, s.file_id AS source_file_id, f.id AS file_id, f.kb_id, s.is_image \ + FROM slices s JOIN files f ON f.id = s.file_id \ + ORDER BY s.id ASC", + ) + .fetch_all(pool) + .await?; + + let all_ids: Vec = rows.iter().map(|row| row.id).collect(); + if all_ids.is_empty() { + return Ok(()); + } + + let cfg = config::get(); + let batch_size = cfg.search.tantivy_rebuild_batch_size.max(1); + + let _guard = self.index_write_lock.lock().await; + for chunk in all_ids.chunks(batch_size) { + self.index_writer.delete_by_field("id", chunk).await?; + } + + let mut contents = HashMap::new(); + for source_file_id in rows.iter().map(|row| row.source_file_id).collect::>() { + contents.insert(source_file_id, crate::slice_content::read_all(source_file_id).await?); + } + + for chunk in rows.chunks(batch_size) { + let docs: Vec = chunk + .iter() + .map(|row| { + let content = + contents.get(&row.source_file_id).and_then(|m| m.get(&row.id)).cloned().unwrap_or_default(); + let is_image = row.is_image != 0 || content_looks_like_image_reference(&content); + tantivy_engine::Document::new(row.id, row.file_id, row.kb_id, content).with_is_image(is_image) + }) + .collect(); + self.index_writer.write_batch(docs).await?; + } + reload_reader(&self.index_reader, "index")?; + + info!("Tantivy default index rebuilt from SQLite: {} docs", all_ids.len()); + Ok(()) + } + + pub async fn reload_lexicon(&self) -> anyhow::Result { + let Some(pool) = &self.pool else { + return Ok(0); + }; + + let rows: Vec = + sqlx::query_as("SELECT term, freq, tag FROM search_lexicon WHERE enabled = 1 ORDER BY id") + .fetch_all(pool) + .await?; + + let entries: Vec = rows + .into_iter() + .map(|row| chinese_tokenizer::LexiconEntry { + term: row.term, + freq: row.freq.and_then(|value| usize::try_from(value).ok()).filter(|value| *value > 0), + tag: row.tag, + }) + .collect(); + + chinese_tokenizer::reload_custom_words(&entries) + } + + pub async fn rebuild_tantivy_indexes(&self, job_tag: &str, mut on_progress: F) -> anyhow::Result<()> + where + F: FnMut(RebuildProgress) -> Fut + Send, + Fut: std::future::Future + Send, + { + let Some(pool) = &self.pool else { + return Err(anyhow!("search engine db pool not set")); + }; + let _rebuild_guard = self.rebuild_lock.lock().await; + + let total_slices: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM slices s WHERE EXISTS (SELECT 1 FROM files f WHERE f.id = s.file_id)", + ) + .fetch_one(pool) + .await?; + let total_full_docs: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM files WHERE status = 1").fetch_one(pool).await?; + let total_docs = total_slices + total_full_docs; + let mut processed_docs = 0_i64; + on_progress(RebuildProgress { phase: "prepare".to_string(), total_docs, processed_docs }).await; + + let cfg = config::get(); + let rebuild_batch_size = i64::try_from(cfg.search.tantivy_rebuild_batch_size) + .ok() + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_REBUILD_BATCH_SIZE); + let tag = sanitize_job_tag(job_tag); + let slice_live_path = cfg.search.tantivy_index_path.clone(); + let full_live_path = cfg.search.tantivy_full_index_path.clone(); + let slice_temp_path = format!("{}.rebuild.{}", slice_live_path, tag); + let full_temp_path = format!("{}.rebuild.{}", full_live_path, tag); + let slice_backup_path = format!("{}.backup.{}", slice_live_path, tag); + let full_backup_path = format!("{}.backup.{}", full_live_path, tag); + + cleanup_dir_if_exists(&slice_temp_path)?; + cleanup_dir_if_exists(&full_temp_path)?; + cleanup_dir_if_exists(&slice_backup_path)?; + cleanup_dir_if_exists(&full_backup_path)?; + + let rebuild_result: anyhow::Result<()> = async { + let (slice_schema, slice_temp_index) = tantivy_engine::init_with_path(&slice_temp_path) + .with_context(|| format!("init temp slice index failed: {}", slice_temp_path))?; + let (full_schema, full_temp_index) = tantivy_engine::init_with_path(&full_temp_path) + .with_context(|| format!("init temp full index failed: {}", full_temp_path))?; + let mut slice_writer = tantivy_engine::create_rebuild_writer(&slice_temp_index, "rebuild_slice") + .await + .context("create temp slice writer failed")?; + let mut full_writer = tantivy_engine::create_rebuild_writer(&full_temp_index, "rebuild_full") + .await + .context("create temp full writer failed")?; + let mut total_slice_docs = 0_usize; + let mut total_full_docs = 0_usize; + + on_progress(RebuildProgress { phase: "build_slice".to_string(), total_docs, processed_docs }).await; + let mut last_slice_id = 0_i64; + loop { + let rows: Vec = sqlx::query_as( + "SELECT s.id, s.file_id AS source_file_id, COALESCE(ref.id, source.id) AS file_id, \ + COALESCE(ref.kb_id, source.kb_id) AS kb_id \ + FROM slices s JOIN files source ON source.id = s.file_id \ + LEFT JOIN parse_artifacts pa ON pa.source_file_id = source.id \ + LEFT JOIN files ref ON ref.artifact_id = pa.id \ + WHERE s.id > ? \ + ORDER BY s.id ASC \ + LIMIT ?", + ) + .bind(last_slice_id) + .bind(rebuild_batch_size) + .fetch_all(pool) + .await?; + if rows.is_empty() { + break; + } + last_slice_id = rows.last().map(|row| row.id).unwrap_or(last_slice_id); + let batch_size = rows.len() as i64; + let mut contents = HashMap::new(); + for source_file_id in rows.iter().map(|row| row.source_file_id).collect::>() { + contents.insert(source_file_id, crate::slice_content::read_all(source_file_id).await?); + } + let docs: Vec = rows + .into_iter() + .map(|row| { + let content = + contents.get(&row.source_file_id).and_then(|v| v.get(&row.id)).cloned().unwrap_or_default(); + let is_image = content_looks_like_image_reference(&content); + tantivy_engine::Document::new(row.id, row.file_id, row.kb_id, content).with_is_image(is_image) + }) + .collect(); + total_slice_docs += tantivy_engine::add_documents(&mut slice_writer, &slice_schema, docs)?; + processed_docs += batch_size; + on_progress(RebuildProgress { phase: "build_slice".to_string(), total_docs, processed_docs }).await; + } + tantivy_engine::commit_writer(&mut slice_writer, "rebuild_slice", total_slice_docs) + .context("commit temp slice writer failed")?; + + on_progress(RebuildProgress { phase: "build_full".to_string(), total_docs, processed_docs }).await; + let mut last_file_id = 0_i64; + loop { + let meta_rows: Vec = sqlx::query_as( + "SELECT id, kb_id, filename \ + FROM files \ + WHERE status = 1 AND id > ? \ + ORDER BY id ASC \ + LIMIT ?", + ) + .bind(last_file_id) + .bind(rebuild_batch_size) + .fetch_all(pool) + .await?; + if meta_rows.is_empty() { + break; + } + last_file_id = meta_rows.last().map(|row| row.id).unwrap_or(last_file_id); + let batch_size = meta_rows.len() as i64; + + let ids: Vec = meta_rows.iter().map(|row| row.id).collect(); + let content_by_id = fetch_file_contents_by_ids(pool, &ids).await?; + + let docs: Vec = meta_rows + .into_iter() + .map(|row| { + let full_content = content_by_id.get(&row.id).cloned().unwrap_or_default(); + let index_content = if full_content.trim().is_empty() { + row.filename + } else { + format!("{}\n\n{}", row.filename, full_content) + }; + tantivy_engine::Document::new(row.id, row.id, row.kb_id, index_content) + }) + .collect(); + total_full_docs += tantivy_engine::add_documents(&mut full_writer, &full_schema, docs)?; + processed_docs += batch_size; + on_progress(RebuildProgress { phase: "build_full".to_string(), total_docs, processed_docs }).await; + } + tantivy_engine::commit_writer(&mut full_writer, "rebuild_full", total_full_docs) + .context("commit temp full writer failed")?; + + drop(slice_writer); + drop(full_writer); + + drop(slice_temp_index); + drop(full_temp_index); + + let _slice_write_guard = self.index_write_lock.lock().await; + let _full_write_guard = self.full_index_write_lock.lock().await; + on_progress(RebuildProgress { phase: "swap".to_string(), total_docs, processed_docs }).await; + + if let Err(err) = swap_index_dir(&slice_live_path, &slice_temp_path, &slice_backup_path) { + return Err(err.context("swap slice index failed")); + } + + if let Err(err) = swap_index_dir(&full_live_path, &full_temp_path, &full_backup_path) { + if let Err(rb_err) = restore_backup_dir(&slice_live_path, &slice_backup_path) { + warn!("rollback slice index failed after full swap failure: {}", rb_err); + } + return Err(err.context("swap full index failed")); + } + + if let Err(err) = reload_reader(&self.index_reader, "index") { + let _ = restore_backup_dir(&full_live_path, &full_backup_path); + let _ = restore_backup_dir(&slice_live_path, &slice_backup_path); + return Err(err).context("reload slice reader after swap failed"); + } + if let Err(err) = reload_reader(&self.full_index_reader, "full_index") { + let _ = restore_backup_dir(&full_live_path, &full_backup_path); + let _ = restore_backup_dir(&slice_live_path, &slice_backup_path); + return Err(err).context("reload full reader after swap failed"); + } + + cleanup_dir_if_exists(&slice_backup_path)?; + cleanup_dir_if_exists(&full_backup_path)?; + processed_docs = total_docs; + on_progress(RebuildProgress { phase: "completed".to_string(), total_docs, processed_docs }).await; + Ok(()) + } + .await; + + if rebuild_result.is_err() { + // 清理临时重建目录（重建过程中的中间产物） + let _ = cleanup_dir_if_exists(&slice_temp_path); + let _ = cleanup_dir_if_exists(&full_temp_path); + // 保留 backup 目录不删除——如果 swap 后 reload 失败且 restore 也失败， + // backup 是恢复到上一次可用索引的唯一手段。 + // 这些 backup 会在下次成功重建后被覆盖，或通过手动清理。 + } + + rebuild_result + } + + pub async fn write( + &self, doc: tantivy_engine::Document, image_embedding: Option>>, + ) -> anyhow::Result<()> { + { + let _guard = self.index_write_lock.lock().await; + self.index_writer.write_batch(vec![doc.clone()]).await?; + reload_reader(&self.index_reader, "index")?; + } + + let mut lancedb_doc = lancedb::Document::new(doc.id, doc.file_id, doc.kb_id, doc.content); + lancedb_doc = lancedb_doc.with_is_image(doc.is_image); + if let Some(image_embedding) = image_embedding { + lancedb_doc = lancedb_doc.with_image_embedding(image_embedding); + } + lancedb::write_documents(lancedb_doc).await?; + Ok(()) + } + + /// 批量写入切片到默认索引与 LanceDB。 + /// + /// 注意：写入后不会自动 reload reader，调用方需在完成全部写入后调用 [`reload_readers`]， + /// 避免批量处理时每次写入都重建 reader。 + pub async fn write_batch( + &self, docs: Vec, image_embeddings: Vec>>>, + ) -> anyhow::Result<()> { + if docs.is_empty() { + return Ok(()); + } + + { + let _guard = self.index_write_lock.lock().await; + self.index_writer.write_batch(docs.clone()).await?; + } + + let lancedb_docs: Vec = docs + .iter() + .zip(image_embeddings.iter()) + .map(|(doc, image_embedding)| { + let mut lancedb_doc = lancedb::Document::new(doc.id, doc.file_id, doc.kb_id, doc.content.clone()); + lancedb_doc = lancedb_doc.with_is_image(doc.is_image); + if let Some(embedding) = image_embedding { + lancedb_doc = lancedb_doc.with_image_embedding(embedding.clone()); + } + lancedb_doc + }) + .collect(); + + lancedb::write_documents_batch(lancedb_docs).await?; + Ok(()) + } + + /// 写入全文索引。注意：写入后不会自动 reload reader，调用方需在完成全部写入后调用 [`reload_readers`]。 + pub async fn write_full(&self, doc: tantivy_engine::Document) -> anyhow::Result<()> { + { + let _guard = self.full_index_write_lock.lock().await; + self.full_index_writer.write_batch(vec![doc]).await?; + } + Ok(()) + } + + pub async fn write_summary(&self, file_id: i64, kb_id: Option, summary: String) -> anyhow::Result<()> { + lancedb::write_summary(lancedb::SummaryDocument::new(file_id, kb_id, summary)).await + } + + pub async fn delete_summary_by_file(&self, file_id: i64) -> anyhow::Result<()> { + lancedb::delete_summary_by_file(file_id).await + } + + /// 更新指定切片在默认索引与 LanceDB 中的内容。 + /// + /// 内部会先删除旧 slice 文档/向量，再写入新内容，并 reload 默认索引 reader。 + /// LanceDB 采用软删除，旧向量记录会被标记为 `is_deleted=true`，查询时不可见。 + pub async fn update_slices( + &self, file_id: i64, kb_id: Option, updates: Vec<(i64, String)>, + ) -> anyhow::Result<()> { + if updates.is_empty() { + return Ok(()); + } + + let slice_ids: Vec = updates.iter().map(|(id, _)| *id).collect(); + let docs: Vec = updates + .into_iter() + .map(|(id, content)| { + let is_image = content_looks_like_image_reference(&content); + tantivy_engine::Document::new(id, file_id, kb_id, content).with_is_image(is_image) + }) + .collect(); + + // 1. 默认索引：删除旧 slice 文档并写入新文档，随后 reload reader + { + let _guard = self.index_write_lock.lock().await; + self.index_writer.delete_by_field("id", &slice_ids).await?; + self.index_writer.write_batch(docs.clone()).await?; + reload_reader(&self.index_reader, "index")?; + } + + // 2. LanceDB：软删除旧向量并写入新向量（LanceDB 会自动为 content 生成 embedding） + lancedb::delete_by_slices(&slice_ids).await?; + let lancedb_docs: Vec = docs + .into_iter() + .map(|doc| lancedb::Document::new(doc.id, doc.file_id, doc.kb_id, doc.content).with_is_image(doc.is_image)) + .collect(); + lancedb::write_documents_batch(lancedb_docs).await?; + + Ok(()) + } + + /// 更新指定文件在全文索引中的内容。 + /// + /// 会先删除该 file_id 对应的旧全文文档，再写入 `filename\n\nfull_content`。 + pub async fn update_full_index_for_file( + &self, file_id: i64, kb_id: Option, filename: String, full_content: String, + ) -> anyhow::Result<()> { + let index_content = + if full_content.trim().is_empty() { filename } else { format!("{}\n\n{}", filename, full_content) }; + + { + let _guard = self.full_index_write_lock.lock().await; + self.full_index_writer.delete_by_field("file_id", &[file_id]).await?; + self.full_index_writer + .write_batch(vec![tantivy_engine::Document::new(file_id, file_id, kb_id, index_content)]) + .await?; + reload_reader(&self.full_index_reader, "full_index")?; + } + Ok(()) + } + + pub fn reload_readers(&self) -> anyhow::Result<()> { + reload_reader(&self.index_reader, "index")?; + reload_reader(&self.full_index_reader, "full_index")?; + Ok(()) + } + + pub async fn delete(&self, file_id: Option, kb_id: Option) -> anyhow::Result<()> { + let file_buf = file_id.map(|id| [id]); + let kb_buf = kb_id.map(|id| [id]); + self.delete_batch(file_buf.as_ref().map(|ids| &ids[..]), kb_buf.as_ref().map(|ids| &ids[..])).await + } + + pub async fn delete_batch(&self, file_ids: Option<&[i64]>, kb_ids: Option<&[i64]>) -> anyhow::Result<()> { + let overall_start = Instant::now(); + if let Some(file_ids) = file_ids.filter(|ids| !ids.is_empty()) { + let tantivy_delete = async { + let lock_wait_start = Instant::now(); + { + let _guard = self.index_write_lock.lock().await; + let locked_at = Instant::now(); + debug!( + "search_delete file_count={} tantivy_lock_wait_ms={}", + file_ids.len(), + lock_wait_start.elapsed().as_millis() + ); + self.index_writer.delete_by_field("file_id", file_ids).await?; + debug!( + "search_delete file_count={} tantivy_inner_ms={}", + file_ids.len(), + locked_at.elapsed().as_millis() + ); + } + reload_reader(&self.index_reader, "index")?; + debug!( + "search_delete file_count={} tantivy {}ms", + file_ids.len(), + lock_wait_start.elapsed().as_millis() + ); + anyhow::Ok(()) + }; + + let lancedb_delete = async { + let step_start = Instant::now(); + if file_ids.len() == 1 { + lancedb::delete_by_file(file_ids[0]).await?; + } else { + lancedb::delete_by_files(file_ids).await?; + } + lancedb::delete_summaries_by_files(file_ids).await?; + debug!("search_delete file_count={} lancedb {}ms", file_ids.len(), step_start.elapsed().as_millis()); + anyhow::Ok(()) + }; + + let tantivy_full_delete = async { + let lock_wait_start = Instant::now(); + { + let _guard = self.full_index_write_lock.lock().await; + let locked_at = Instant::now(); + debug!( + "search_delete file_count={} tantivy_full_lock_wait_ms={}", + file_ids.len(), + lock_wait_start.elapsed().as_millis() + ); + self.full_index_writer.delete_by_field("file_id", file_ids).await?; + debug!( + "search_delete file_count={} tantivy_full_inner_ms={}", + file_ids.len(), + locked_at.elapsed().as_millis() + ); + } + reload_reader(&self.full_index_reader, "full_index")?; + debug!( + "search_delete file_count={} tantivy_full {}ms", + file_ids.len(), + lock_wait_start.elapsed().as_millis() + ); + anyhow::Ok(()) + }; + + let (tantivy_result, lancedb_result, tantivy_full_result) = + tokio::join!(tantivy_delete, lancedb_delete, tantivy_full_delete); + tantivy_result?; + lancedb_result?; + tantivy_full_result?; + } + if let Some(kb_ids) = kb_ids.filter(|ids| !ids.is_empty()) { + let tantivy_delete = async { + let lock_wait_start = Instant::now(); + { + let _guard = self.index_write_lock.lock().await; + let locked_at = Instant::now(); + debug!( + "search_delete kb_count={} tantivy_lock_wait_ms={}", + kb_ids.len(), + lock_wait_start.elapsed().as_millis() + ); + self.index_writer.delete_by_field("kb_id", kb_ids).await?; + debug!( + "search_delete kb_count={} tantivy_inner_ms={}", + kb_ids.len(), + locked_at.elapsed().as_millis() + ); + } + reload_reader(&self.index_reader, "index")?; + debug!("search_delete kb_count={} tantivy {}ms", kb_ids.len(), lock_wait_start.elapsed().as_millis()); + anyhow::Ok(()) + }; + + let lancedb_delete = async { + let step_start = Instant::now(); + if kb_ids.len() == 1 { + lancedb::delete_by_kb(kb_ids[0]).await?; + } else { + lancedb::delete_by_kbs(kb_ids).await?; + } + lancedb::delete_summaries_by_kbs(kb_ids).await?; + debug!("search_delete kb_count={} lancedb {}ms", kb_ids.len(), step_start.elapsed().as_millis()); + anyhow::Ok(()) + }; + + let tantivy_full_delete = async { + let lock_wait_start = Instant::now(); + { + let _guard = self.full_index_write_lock.lock().await; + let locked_at = Instant::now(); + debug!( + "search_delete kb_count={} tantivy_full_lock_wait_ms={}", + kb_ids.len(), + lock_wait_start.elapsed().as_millis() + ); + self.full_index_writer.delete_by_field("kb_id", kb_ids).await?; + debug!( + "search_delete kb_count={} tantivy_full_inner_ms={}", + kb_ids.len(), + locked_at.elapsed().as_millis() + ); + } + reload_reader(&self.full_index_reader, "full_index")?; + debug!( + "search_delete kb_count={} tantivy_full {}ms", + kb_ids.len(), + lock_wait_start.elapsed().as_millis() + ); + anyhow::Ok(()) + }; + + let (tantivy_result, lancedb_result, tantivy_full_result) = + tokio::join!(tantivy_delete, lancedb_delete, tantivy_full_delete); + tantivy_result?; + lancedb_result?; + tantivy_full_result?; + } + debug!( + "search_delete total {}ms file_count={:?} kb_count={:?}", + overall_start.elapsed().as_millis(), + file_ids.map(|ids| ids.len()), + kb_ids.map(|ids| ids.len()) + ); + Ok(()) + } + + pub async fn search( + &self, query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + ) -> anyhow::Result> { + let total_start = Instant::now(); + debug!("Searching for query: {}", query); + + let synonym_start = Instant::now(); + let synonym_map = match self.load_query_synonyms(query).await { + Ok(map) => map, + Err(e) => { + warn!("Failed to load query synonyms for '{}': {}", query, e); + HashMap::new() + } + }; + debug!( + "Search synonym lookup {}ms count={}", + synonym_start.elapsed().as_millis(), + synonym_map.values().map(Vec::len).sum::() + ); + + let index_reader = self.index_reader.clone(); + let schema = self.schema.clone(); + let tantivy_query = query.to_string(); + let tantivy_file_ids = file_ids.cloned(); + let tantivy_kb_ids = kb_ids.cloned(); + let tantivy_synonym_map = synonym_map.clone(); + let tantivy_started = Instant::now(); + let tantivy_task = tokio::task::spawn_blocking(move || { + let synonym_ref = if tantivy_synonym_map.is_empty() { None } else { Some(&tantivy_synonym_map) }; + let results = tantivy_engine::search_sync( + &index_reader, + &schema, + &tantivy_query, + tantivy_file_ids.as_ref(), + tantivy_kb_ids.as_ref(), + None, + synonym_ref, + )?; + debug!("Tantivy branch total {}ms", tantivy_started.elapsed().as_millis()); + anyhow::Ok(results) + }); + + let lancedb_started = Instant::now(); + let lancedb_result = lancedb::search(query, file_ids, kb_ids).await; + debug!("LanceDB branch total {}ms", lancedb_started.elapsed().as_millis()); + let tantivy_result = tantivy_task.await.map_err(|err| anyhow!("Tantivy search task failed: {}", err))?; + + // 使用 tantivy 搜索 + let tantivy_results = tantivy_result?; + debug!("Tantivy results count: {}", tantivy_results.len()); + + // 使用 lancedb 搜索 + let lancedb_results = match lancedb_result { + Ok(results) => { + debug!("LanceDB results count: {}", results.len()); + results + } + Err(err) => { + warn!("Vector search failed for query {:?}, falling back to Tantivy-only results: {}", query, err); + Vec::new() + } + }; + + // 合并结果：使用 HashMap 按 id 去重，保留最高分数，同时去除内容为空的结果 + let mut merged_map: HashMap = HashMap::new(); + + for result in tantivy_results { + // 跳过空内容（包括仅有空白的情况） + if result.content.trim().is_empty() { + continue; + } + merged_map.insert(result.id, result); + } + + for result in lancedb_results { + // 跳过空内容（包括仅有空白的情况） + if result.content.trim().is_empty() { + continue; + } + + merged_map + .entry(result.id) + .and_modify(|e| { + // 如果已存在，取两者中分数较高的 + if result.score > e.score { + *e = result.clone(); + } + }) + .or_insert(result); + } + + // 转换为 Vec 并按分数降序排序 + let mut merged_results: Vec = merged_map.into_values().collect(); + merged_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + + info!("Merged results count: {}", merged_results.len()); + + // 如果结果为空，直接返回 + if merged_results.is_empty() { + return Ok(merged_results); + } + + // 使用 BGE-Rerank 重排序（失败时内部回退为原结果，无需预先 clone 整个结果集） + let mut final_results = self.rerank(query, merged_results).await; + let limit = config::get().search.limit.max(1); + if final_results.len() > limit { + final_results.truncate(limit); + } + + debug!("Search total {}ms", total_start.elapsed().as_millis()); + Ok(final_results) + } + + pub async fn search_full( + &self, query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + ) -> anyhow::Result> { + let synonym_map = match self.load_query_synonyms(query).await { + Ok(map) => map, + Err(e) => { + warn!("Failed to load query synonyms for '{}': {}", query, e); + HashMap::new() + } + }; + let synonym_ref = if synonym_map.is_empty() { None } else { Some(&synonym_map) }; + tantivy_engine::search_with_snippet( + &self.full_index_reader, + &self.full_schema, + query, + file_ids, + kb_ids, + None, + FULL_SNIPPET_MAX_CHARS, + synonym_ref, + ) + .await + } + + pub async fn search_summary( + &self, query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + ) -> anyhow::Result> { + let total_start = Instant::now(); + let vector_results = lancedb::search_summary(query, file_ids, kb_ids).await?; + debug!("Summary vector search returned {} candidates", vector_results.len()); + if vector_results.is_empty() { + return Ok(vector_results); + } + + let mut final_results = self.rerank_summaries(query, vector_results).await; + let limit = config::get().search.limit.max(1); + if final_results.len() > limit { + final_results.truncate(limit); + } + + debug!("Summary search total {}ms", total_start.elapsed().as_millis()); + Ok(final_results) + } + + pub async fn search_image_by_text( + &self, query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + ) -> anyhow::Result> { + let total_start = Instant::now(); + debug!("Searching images by text for query: {}", query); + + let synonym_start = Instant::now(); + let synonym_map = match self.load_query_synonyms(query).await { + Ok(map) => map, + Err(e) => { + warn!("Failed to load query synonyms for '{}': {}", query, e); + HashMap::new() + } + }; + debug!( + "Image-by-text synonym lookup {}ms count={}", + synonym_start.elapsed().as_millis(), + synonym_map.values().map(Vec::len).sum::() + ); + + let index_reader = self.index_reader.clone(); + let schema = self.schema.clone(); + let tantivy_query = query.to_string(); + let tantivy_file_ids = file_ids.cloned(); + let tantivy_kb_ids = kb_ids.cloned(); + let tantivy_synonym_map = synonym_map.clone(); + let tantivy_started = Instant::now(); + let tantivy_task = tokio::task::spawn_blocking(move || { + let synonym_ref = if tantivy_synonym_map.is_empty() { None } else { Some(&tantivy_synonym_map) }; + let results = tantivy_engine::search_sync( + &index_reader, + &schema, + &tantivy_query, + tantivy_file_ids.as_ref(), + tantivy_kb_ids.as_ref(), + Some(true), + synonym_ref, + )?; + debug!("Tantivy image-by-text branch total {}ms", tantivy_started.elapsed().as_millis()); + anyhow::Ok(results) + }); + + let lancedb_started = Instant::now(); + let lancedb_result = lancedb::search_image_by_text(query, file_ids, kb_ids).await; + debug!("LanceDB image-by-text branch total {}ms", lancedb_started.elapsed().as_millis()); + let tantivy_result = + tantivy_task.await.map_err(|err| anyhow!("Tantivy image-by-text search task failed: {}", err))?; + + let tantivy_results = tantivy_result?; + debug!("Tantivy image-by-text results count: {}", tantivy_results.len()); + + let lancedb_results = match lancedb_result { + Ok(results) => { + debug!("LanceDB image-by-text results count: {}", results.len()); + results + } + Err(err) => { + warn!( + "Vector image-by-text search failed for query {:?}, falling back to Tantivy-only: {}", + query, err + ); + Vec::new() + } + }; + + let mut merged_map: HashMap = HashMap::new(); + for result in tantivy_results { + if result.content.trim().is_empty() { + continue; + } + merged_map.insert(result.id, result); + } + for result in lancedb_results { + if result.content.trim().is_empty() { + continue; + } + merged_map + .entry(result.id) + .and_modify(|e| { + if result.score > e.score { + *e = result.clone(); + } + }) + .or_insert(result); + } + + let mut merged_results: Vec = merged_map.into_values().collect(); + merged_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + info!("Image-by-text merged results count: {}", merged_results.len()); + + if merged_results.is_empty() { + return Ok(merged_results); + } + + let mut final_results = self.rerank(query, merged_results).await; + let limit = config::get().search.limit.max(1); + if final_results.len() > limit { + final_results.truncate(limit); + } + + debug!("Image-by-text search total {}ms", total_start.elapsed().as_millis()); + Ok(final_results) + } + + pub async fn search_image( + &self, image_embedding: Vec, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + ) -> anyhow::Result> { + lancedb::search_image(image_embedding, file_ids, kb_ids).await + } + + /// 计算每个结果（按输入顺序对齐）的 rerank 分数。 + /// 仅借用 results，失败时不消耗它，使调用方可零拷贝回退。 + async fn compute_rerank_scores( + &self, query: &str, results: &[SearchResultItem], + ) -> anyhow::Result>> { + let documents: Vec = results.iter().map(|result| result.content.clone()).collect(); + self.compute_rerank_scores_for_texts(query, &documents).await + } + + async fn compute_rerank_scores_for_texts( + &self, query: &str, source_documents: &[String], + ) -> anyhow::Result>> { + let cfg = config::get(); + // 提取所有文档内容用于重排序，并做去重（按内容借用，去重映射只需单次 clone 进 documents） + let mut documents: Vec = Vec::new(); + let mut document_index_map: Vec = Vec::with_capacity(source_documents.len()); + let mut document_index_by_content: HashMap<&str, usize> = HashMap::new(); + for document in source_documents.iter() { + if let Some(&idx) = document_index_by_content.get(document.as_str()) { + document_index_map.push(idx); + continue; + } + let idx = documents.len(); + documents.push(document.clone()); + document_index_by_content.insert(document.as_str(), idx); + document_index_map.push(idx); + } + if documents.is_empty() { + return Ok(Vec::new()); + } + + // 根据 URL 后缀判断使用哪种 rerank 接口格式 + let use_v1_format = cfg.services.rerank_url.ends_with("/v1/rerank"); + + // 调用 BGE-Rerank API + let rerank_http_start = Instant::now(); + let response = if use_v1_format { + let rerank_request = RerankRequest { + model: cfg.ai.rerank_model.clone(), + query: query.to_string(), + documents: documents.clone(), + }; + RERANK_HTTP_CLIENT + .post(&cfg.services.rerank_url) + .timeout(Duration::from_secs(cfg.search.rerank_timeout_secs)) + .json(&rerank_request) + .send() + .await? + } else { + let rerank_request = SimpleRerankRequest { query: query.to_string(), texts: documents.clone() }; + RERANK_HTTP_CLIENT + .post(&cfg.services.rerank_url) + .timeout(Duration::from_secs(cfg.search.rerank_timeout_secs)) + .json(&rerank_request) + .send() + .await? + }; + debug!("Rerank HTTP request {}ms", rerank_http_start.elapsed().as_millis()); + + if !response.status().is_success() { + let status = response.status(); + let error_text = response.text().await.unwrap_or_default(); + anyhow::bail!("Rerank API failed with status {}: {}; documents={}", status, error_text, documents.len()); + } + + // 先获取响应文本用于调试 + let rerank_read_start = Instant::now(); + let response_text = response.text().await?; + debug!("Rerank response read {}ms", rerank_read_start.elapsed().as_millis()); + + // 解析 JSON 响应 + let rerank_parse_start = Instant::now(); + let mut rerank_scores: Vec> = vec![None; documents.len()]; + if use_v1_format { + let rerank_response: RerankResponse = serde_json::from_str(&response_text)?; + debug!("Rerank response parse {}ms", rerank_parse_start.elapsed().as_millis()); + if rerank_response.results.len() != documents.len() { + anyhow::bail!( + "Rerank results count mismatch: expected {}, got {}", + documents.len(), + rerank_response.results.len() + ); + } + for rerank_result in &rerank_response.results { + if let Some(score) = rerank_scores.get_mut(rerank_result.index) { + *score = Some(rerank_result.relevance_score); + } + } + } else { + let simple_results: Vec = serde_json::from_str(&response_text)?; + debug!("Rerank response parse {}ms", rerank_parse_start.elapsed().as_millis()); + if simple_results.len() != documents.len() { + anyhow::bail!( + "Rerank results count mismatch: expected {}, got {}", + documents.len(), + simple_results.len() + ); + } + for result in &simple_results { + if let Some(score) = rerank_scores.get_mut(result.index) { + *score = Some(result.score); + } + } + } + + // 将去重后的分数映射回每个结果（按输入顺序） + let per_result: Vec> = + document_index_map.iter().map(|&doc_idx| rerank_scores.get(doc_idx).copied().flatten()).collect(); + Ok(per_result) + } + + async fn rerank(&self, query: &str, results: Vec) -> Vec { + let rerank_total_start = Instant::now(); + let cfg = config::get(); + + // 计算分数；失败则原样返回（无需 clone 回退） + let scores = match self.compute_rerank_scores(query, &results).await { + Ok(scores) => scores, + Err(err) => { + warn!("Rerank failed for query {:?}, returning merged search results without rerank: {}", query, err); + return results; + } + }; + + // 使用重排序分数更新结果 + let mut reranked_results: Vec = results + .into_iter() + .enumerate() + .map(|(i, mut result)| { + if let Some(Some(score)) = scores.get(i) { + result.score = *score; + } + result + }) + .collect(); + + // 按新分数降序排序 + reranked_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + let threshold = cfg.ai.rerank_threshold; + let filter_results: Vec = + reranked_results.into_iter().filter(|f| f.score >= threshold).collect(); + info!("Reranked results count: {}", filter_results.len()); + debug!("Rerank total {}ms", rerank_total_start.elapsed().as_millis()); + filter_results + } + + async fn rerank_summaries( + &self, query: &str, results: Vec, + ) -> Vec { + let rerank_total_start = Instant::now(); + let cfg = config::get(); + let summaries: Vec = results.iter().map(|result| result.summary.clone()).collect(); + + let scores = match self.compute_rerank_scores_for_texts(query, &summaries).await { + Ok(scores) => scores, + Err(err) => { + warn!( + "Summary rerank failed for query {:?}, returning vector summary results without rerank: {}", + query, err + ); + return results; + } + }; + + let mut reranked_results: Vec = results + .into_iter() + .enumerate() + .map(|(i, mut result)| { + if let Some(Some(score)) = scores.get(i) { + result.score = *score; + } + result + }) + .collect(); + + reranked_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + let threshold = cfg.ai.rerank_threshold; + let filter_results: Vec = + reranked_results.into_iter().filter(|result| result.score >= threshold).collect(); + info!("Reranked summary results count: {}", filter_results.len()); + debug!("Summary rerank total {}ms", rerank_total_start.elapsed().as_millis()); + filter_results + } + + /// 使用知识图谱扩展查询 + /// 从查询中识别实体，并查找相关实体来扩展查询 + pub async fn expand_query_with_graph(&self, query: &str, kb_ids: Option<&Vec>) -> anyhow::Result> { + let pool = match &self.pool { + Some(p) => p, + None => return Ok(vec![query.to_string()]), // 如果没有数据库连接，直接返回原查询 + }; + + let mut expanded_queries = vec![query.to_string()]; + + // 1. 在知识图谱中搜索匹配的实体 + let mut qb = QueryBuilder::new("SELECT DISTINCT name, entity_type FROM graph_nodes WHERE name LIKE "); + qb.push_bind(format!("%{}%", query)); + + if let Some(ids) = kb_ids + && !ids.is_empty() + { + qb.push(" AND kb_id IN ("); + let mut separated = qb.separated(", "); + for id in ids { + separated.push_bind(id); + } + qb.push(")"); + } + qb.push(" LIMIT 10"); + let entities: Vec<(String, String)> = qb.build_query_as().fetch_all(pool).await?; + + // 2. 对于每个匹配的实体，查找相关实体 + for (entity_name, _) in entities.iter().take(3) { + // 限制为前3个实体 + // 查找与该实体相关的其他实体（通过边连接） + let related_sql = r#" + SELECT DISTINCT n.name + FROM graph_nodes n + JOIN graph_edges e ON (n.id = e.target_node_id OR n.id = e.source_node_id) + JOIN graph_nodes source ON (source.id = e.source_node_id OR source.id = e.target_node_id) + WHERE source.name = ? + AND n.name != ? + LIMIT 5 + "#; + + let related_entities: Vec<(String,)> = + sqlx::query_as(related_sql).bind(entity_name).bind(entity_name).fetch_all(pool).await?; + + // 添加相关实体到扩展查询 + for (related_name,) in related_entities { + if !expanded_queries.contains(&related_name) { + expanded_queries.push(related_name); + } + } + } + + info!("Query expansion: '{}' -> {:?}", query, expanded_queries); + Ok(expanded_queries) + } + + /// 清理 LanceDB 已删除的记录，释放空间 + pub async fn compact_lancedb(&self) -> anyhow::Result { + lancedb::compact().await + } + + /// 强制合并 Tantivy segment，减少碎片与已删除文档 tombstone。 + pub async fn force_merge_tantivy_indexes( + &self, + ) -> anyhow::Result<(tantivy_engine::ForceMergeStats, tantivy_engine::ForceMergeStats)> { + let _rebuild_guard = self.rebuild_lock.lock().await; + let (slice_stats, full_stats) = { + let _slice_write_guard = self.index_write_lock.lock().await; + let _full_write_guard = self.full_index_write_lock.lock().await; + let slice_stats = self.index_writer.force_merge().await?; + let full_stats = self.full_index_writer.force_merge().await?; + (slice_stats, full_stats) + }; + + reload_reader(&self.index_reader, "index")?; + reload_reader(&self.full_index_reader, "full_index")?; + + Ok((slice_stats, full_stats)) + } + + /// 确保同义词缓存新鲜（TTL 内复用，过期则重载全部 enabled 行）。 + async fn ensure_synonym_cache(&self, pool: &SqlitePool) -> anyhow::Result<()> { + { + let guard = self.synonym_cache.read().await; + if let Some(cache) = guard.as_ref() + && cache.loaded_at.elapsed() < SYNONYM_CACHE_TTL + { + return Ok(()); + } + } + let mut guard = self.synonym_cache.write().await; + // 双检：可能已有其他任务在等待写锁期间刷新过。 + if let Some(cache) = guard.as_ref() + && cache.loaded_at.elapsed() < SYNONYM_CACHE_TTL + { + return Ok(()); + } + let rows: Vec = + sqlx::query_as("SELECT term, synonym, weight, bidirectional FROM search_synonyms WHERE enabled = 1") + .fetch_all(pool) + .await?; + *guard = Some(SynonymCache::build(rows)); + Ok(()) + } + + /// 主动失效同义词缓存（同义词增删改后调用，使变更立即生效）。 + pub async fn invalidate_synonym_cache(&self) { + *self.synonym_cache.write().await = None; + } + + async fn load_query_synonyms(&self, query: &str) -> anyhow::Result { + let cfg = config::get(); + if !cfg.search.synonym_enabled { + return Ok(HashMap::new()); + } + let Some(pool) = &self.pool else { + return Ok(HashMap::new()); + }; + + let terms = extract_query_terms(query); + if terms.is_empty() { + return Ok(HashMap::new()); + } + + self.ensure_synonym_cache(pool).await?; + let cache_guard = self.synonym_cache.read().await; + let Some(cache) = cache_guard.as_ref() else { + return Ok(HashMap::new()); + }; + if cache.rows.is_empty() { + return Ok(HashMap::new()); + } + + // 收集与查询词相关的候选行下标（命中 term 或 synonym 列）。 + let mut candidate_idx: HashSet = HashSet::new(); + for term in &terms { + if let Some(idxs) = cache.by_term.get(term) { + candidate_idx.extend(idxs.iter().copied()); + } + if let Some(idxs) = cache.by_synonym.get(term) { + candidate_idx.extend(idxs.iter().copied()); + } + } + if candidate_idx.is_empty() { + return Ok(HashMap::new()); + } + + let input_terms: HashSet<&str> = terms.iter().map(String::as_str).collect(); + let mut synonym_map: tantivy_engine::SynonymMap = HashMap::new(); + let max_per_term = cfg.search.max_synonyms_per_term.max(1); + let max_total = cfg.search.max_total_synonyms.max(1); + let boost_factor = cfg.search.synonym_boost.max(0.0); + let mut total_inserted = 0usize; + + for idx in candidate_idx { + let row = &cache.rows[idx]; + let boost = row.weight.max(0.0) * boost_factor; + if boost <= 0.0 { + continue; + } + + if input_terms.contains(row.term.as_str()) + && insert_synonym(&mut synonym_map, row.term.as_str(), row.synonym.as_str(), boost, max_per_term) + { + total_inserted += 1; + if total_inserted >= max_total { + break; + } + } + + if row.bidirectional != 0 + && input_terms.contains(row.synonym.as_str()) + && insert_synonym(&mut synonym_map, row.synonym.as_str(), row.term.as_str(), boost, max_per_term) + { + total_inserted += 1; + if total_inserted >= max_total { + break; + } + } + } + + Ok(synonym_map) + } + + /// 使用图谱增强的搜索 + /// 先扩展查询，然后对每个扩展查询进行搜索，最后合并去重结果 + pub async fn search_with_graph_expansion( + &self, query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + ) -> anyhow::Result> { + // 1. 扩展查询 + let expanded_queries = self.expand_query_with_graph(query, kb_ids).await?; + + if expanded_queries.len() == 1 { + // 没有扩展，直接使用原查询 + return self.search(query, file_ids, kb_ids).await; + } + + // 2. 对每个扩展查询并发搜索（彼此独立，无需串行） + let mut all_results: HashMap = HashMap::new(); + + let search_futures = expanded_queries.iter().enumerate().map(|(idx, expanded_query)| { + // 原始查询的结果权重更高 + let weight = if idx == 0 { 1.0 } else { 0.7 }; + async move { + let results = self.search(expanded_query, file_ids, kb_ids).await; + (weight, results) + } + }); + let per_query = futures::future::join_all(search_futures).await; + + for (weight, results) in per_query { + for mut result in results? { + result.score *= weight; + + all_results + .entry(result.id) + .and_modify(|e| { + // 如果已存在，取两者中分数较高的 + if result.score > e.score { + *e = result.clone(); + } + }) + .or_insert(result); + } + } + + // 3. 转换为Vec并按分数排序 + let mut merged_results: Vec = all_results.into_values().collect(); + merged_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); + + info!("Graph-expanded search returned {} results", merged_results.len()); + Ok(merged_results) + } +} + +async fn fetch_file_contents_by_ids(_pool: &SqlitePool, ids: &[i64]) -> anyhow::Result> { + if ids.is_empty() { + return Ok(HashMap::new()); + } + + let mut map = HashMap::with_capacity(ids.len()); + for id in ids { + if let Some(content) = crate::file_content::read(*id).await? { + map.insert(*id, content); + } + } + Ok(map) +} + +fn sanitize_job_tag(input: &str) -> String { + let trimmed = input.trim(); + if !trimmed.is_empty() { + let sanitized: String = trimmed + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' { ch } else { '_' }) + .collect(); + if !sanitized.is_empty() { + return sanitized; + } + } + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_millis()).unwrap_or_default(); + now_ms.to_string() +} + +fn cleanup_dir_if_exists(path: &str) -> anyhow::Result<()> { + let path_ref = Path::new(path); + if !path_ref.exists() { + return Ok(()); + } + fs::remove_dir_all(path_ref).with_context(|| format!("remove dir failed: {}", path))?; + Ok(()) +} + +fn swap_index_dir(active_path: &str, staged_path: &str, backup_path: &str) -> anyhow::Result<()> { + let active = Path::new(active_path); + let staged = Path::new(staged_path); + let backup = Path::new(backup_path); + if !staged.exists() { + return Err(anyhow!("staged index path not found: {}", staged_path)); + } + + if let Some(parent) = active.parent() { + fs::create_dir_all(parent).with_context(|| format!("create parent dir failed: {}", parent.display()))?; + } + + if backup.exists() { + fs::remove_dir_all(backup).with_context(|| format!("remove backup dir failed: {}", backup.display()))?; + } + + let moved_old = if active.exists() { + fs::rename(active, backup) + .with_context(|| format!("rename active->backup failed: {} -> {}", active_path, backup_path))?; + true + } else { + false + }; + + if let Err(err) = fs::rename(staged, active) { + if moved_old && backup.exists() { + let _ = fs::rename(backup, active); + } + return Err(err).with_context(|| format!("rename staged->active failed: {} -> {}", staged_path, active_path)); + } + + Ok(()) +} + +fn restore_backup_dir(active_path: &str, backup_path: &str) -> anyhow::Result<()> { + let active = Path::new(active_path); + let backup = Path::new(backup_path); + if !backup.exists() { + return Ok(()); + } + if active.exists() { + fs::remove_dir_all(active).with_context(|| format!("remove active dir failed: {}", active_path))?; + } + fs::rename(backup, active).with_context(|| format!("restore backup failed: {} -> {}", backup_path, active_path))?; + Ok(()) +} + +fn build_reader(index: &Index, label: &str) -> IndexReader { + let start = Instant::now(); + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::Manual) + .try_into() + .unwrap_or_else(|e| panic!("failed to create tantivy {} reader: {}", label, e)); + debug!("Tantivy {} index.reader init {}ms", label, start.elapsed().as_millis()); + reader +} + +fn extract_query_terms(query: &str) -> Vec { + let mut terms = + chinese_tokenizer::FastChineseTokenizer::new(chinese_tokenizer::SegmentationMode::Search).segment(query); + terms.push(query.trim().to_string()); + terms.retain(|t| !t.trim().is_empty()); + terms.sort(); + terms.dedup(); + if terms.len() > MAX_QUERY_TERMS_FOR_SYNONYM_LOOKUP { + terms.truncate(MAX_QUERY_TERMS_FOR_SYNONYM_LOOKUP); + } + terms +} + +fn insert_synonym( + synonym_map: &mut tantivy_engine::SynonymMap, source_term: &str, synonym_term: &str, boost: f32, + max_per_term: usize, +) -> bool { + let source = source_term.trim(); + let synonym = synonym_term.trim(); + if source.is_empty() || synonym.is_empty() || source == synonym { + return false; + } + + let entry = synonym_map.entry(source.to_string()).or_default(); + if entry.iter().any(|candidate| candidate.term == synonym) { + return false; + } + if entry.len() >= max_per_term { + return false; + } + + entry.push(tantivy_engine::SynonymTerm { term: synonym.to_string(), boost }); + true +} + +fn reload_reader(reader: &IndexReader, label: &str) -> tantivy::Result<()> { + let start = Instant::now(); + reader.reload()?; + debug!("Tantivy {} reader reload {}ms", label, start.elapsed().as_millis()); + Ok(()) +} diff --git a/src/search/tantivy_engine.rs b/src/search/tantivy_engine.rs new file mode 100644 index 0000000..0031f4b --- /dev/null +++ b/src/search/tantivy_engine.rs @@ -0,0 +1,999 @@ +use std::{ + collections::{HashMap, HashSet}, path::Path, sync::{ + Arc, mpsc::{self, Sender} + }, thread, time::{Duration, Instant, SystemTime, UNIX_EPOCH} +}; + +use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind}; +use log::{debug, info, warn}; +use serde::Serialize; +use tantivy::{ + Index, IndexReader, Result, TantivyDocument, TantivyError, Term, collector::TopDocs, directory::error::LockError, doc, merge_policy::LogMergePolicy, query::{BooleanQuery, BoostQuery, Occur, Query, TermQuery}, schema::{FAST, Field, INDEXED, IndexRecordOption, STORED, Schema, TextFieldIndexing, TextOptions, Value as _} +}; + +use super::chinese_tokenizer; +use crate::config; + +const ALL_TOKENIZER: &str = "all"; +const INDEX_WRITER_LOCK_RETRY_MAX_ATTEMPTS: usize = 8; +const INDEX_WRITER_LOCK_RETRY_BASE_MS: u64 = 40; + +#[derive(Debug, Clone)] +pub struct SynonymTerm { + pub term: String, + pub boost: f32, +} + +pub type SynonymMap = HashMap>; + +#[derive(Clone)] +pub struct Document { + pub id: i64, // 切片 ID + pub file_id: i64, // 文件 ID + pub kb_id: Option, // 知识库 ID + pub content: String, // 内容 + pub is_image: bool, // 是否为图片切片 +} + +/// 搜索结果项 +#[derive(Debug, Clone, Serialize)] +pub struct SearchResultItem { + pub id: i64, // 切片 ID + pub file_id: i64, // 文件 ID + pub kb_id: Option, // 知识库 ID + pub content: String, // 内容 + pub score: f32, // 搜索得分 +} + +/// 全文索引搜索结果项 +#[derive(Debug, Clone, Serialize)] +pub struct FullSearchResultItem { + pub file_id: i64, // 文件 ID + pub kb_id: Option, // 知识库 ID + pub snippet: String, // 高亮片段 + pub score: f32, // 搜索得分 +} + +#[derive(Debug, Clone, Serialize)] +pub struct ForceMergeStats { + pub before_segments: usize, + pub after_segments: usize, + pub before_docs: u64, + pub after_docs: u64, + pub before_deleted_docs: u64, + pub after_deleted_docs: u64, + pub gc_deleted_files: usize, + pub gc_failed_files: usize, + pub skipped: bool, + pub duration_ms: u128, +} + +impl Document { + pub fn new(id: i64, file_id: i64, kb_id: Option, content: String) -> Self { + Document { id, file_id, kb_id, content, is_image: false } + } + + pub fn with_is_image(mut self, is_image: bool) -> Self { + self.is_image = is_image; + self + } +} + +/// create new index +/// +/// delete if it exists +pub fn init() -> Result<(Schema, Index)> { + let cfg = config::get(); + init_with_path(&cfg.search.tantivy_index_path) +} + +pub fn init_full() -> Result<(Schema, Index)> { + let cfg = config::get(); + init_with_path(&cfg.search.tantivy_full_index_path) +} + +pub fn init_with_path(path: &str) -> Result<(Schema, Index)> { + let t0 = Instant::now(); + let schema = build_schema(); + info!("Tantivy init substep: build_schema() took {}ms", t0.elapsed().as_millis()); + + let path = Path::new(path); + let t1 = Instant::now(); + let index = if path.exists() { + match Index::open_in_dir(path) { + Ok(idx) => { + // 如果 schema 缺少 is_image 字段，按时间戳备份旧索引并重建 + if !schema_has_is_image(idx.schema()) { + warn!( + "Tantivy index at '{}' is missing is_image field; backing up and creating fresh index.", + path.display() + ); + backup_index_dir(path)?; + std::fs::create_dir_all(path)?; + Index::create_in_dir(path, schema.clone())? + } else { + idx + } + } + Err(e) => { + warn!( + "Tantivy index at '{}' is corrupted or unreadable: {}. Removing and creating fresh index.", + path.display(), + e + ); + std::fs::remove_dir_all(path)?; + std::fs::create_dir_all(path)?; + Index::create_in_dir(path, schema.clone())? + } + } + } else { + std::fs::create_dir_all(path)?; + Index::create_in_dir(path, schema.clone())? + }; + info!("Tantivy init substep: open/create index at '{}' took {}ms", path.display(), t1.elapsed().as_millis()); + + let t2 = Instant::now(); + register_tokenizers(&index); + info!("Tantivy init substep: register_tokenizers() took {}ms", t2.elapsed().as_millis()); + + Ok((schema, index)) +} + +fn schema_has_is_image(schema: Schema) -> bool { + schema.get_field("is_image").is_ok() +} + +fn backup_index_dir(path: &Path) -> std::io::Result<()> { + let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + let backup_path = path.with_file_name(format!( + "{}.backup.{}", + path.file_name().and_then(|n| n.to_str()).unwrap_or("tantivy_index"), + timestamp + )); + warn!("Backing up old Tantivy index from {} to {}", path.display(), backup_path.display()); + std::fs::rename(path, &backup_path)?; + Ok(()) +} + +fn is_lock_busy_error(err: &TantivyError) -> bool { + matches!(err, TantivyError::LockFailure(LockError::LockBusy, _)) +} + +/// Retry opening a Tantivy index writer with exponential backoff. +/// +/// `$sleep` is a closure that performs the actual sleep for the given number of +/// milliseconds. +macro_rules! retry_open_writer { + ($index:expr, $sleep:expr) => {{ + let mut delay_ms = INDEX_WRITER_LOCK_RETRY_BASE_MS; + let mut attempt = 1; + loop { + match open_writer($index) { + Ok(writer) => break Ok(writer), + Err(err) if is_lock_busy_error(&err) && attempt < INDEX_WRITER_LOCK_RETRY_MAX_ATTEMPTS => { + warn!( + "Tantivy index writer lock busy (attempt {}/{}), retry in {}ms", + attempt, INDEX_WRITER_LOCK_RETRY_MAX_ATTEMPTS, delay_ms + ); + $sleep(delay_ms); + delay_ms = (delay_ms * 2).min(800); + attempt += 1; + } + Err(err) => break Err(err), + } + } + }}; +} + +async fn create_writer(index: &Index) -> tantivy::Result { + retry_open_writer!(index, |ms| std::thread::sleep(Duration::from_millis(ms))) +} + +fn create_writer_blocking(index: &Index) -> tantivy::Result { + retry_open_writer!(index, |ms| std::thread::sleep(Duration::from_millis(ms))) +} + +fn open_writer(index: &Index) -> tantivy::Result { + let writer_memory = config::get().search.tantivy_memory_mb * 1_000_000; + let writer = index.writer(writer_memory)?; + // 设置 merge policy，减少小 segment 数量 + writer.set_merge_policy(Box::new(LogMergePolicy::default())); + Ok(writer) +} + +pub async fn create_writer_with_timing(index: &Index, label: &str) -> tantivy::Result { + let start = Instant::now(); + let writer = create_writer(index).await?; + debug!("tantivy_writer_create label={} duration_ms={}", label, start.elapsed().as_millis()); + Ok(writer) +} + +fn create_writer_with_timing_blocking(index: &Index, label: &str) -> tantivy::Result { + let start = Instant::now(); + let writer = create_writer_blocking(index)?; + debug!("tantivy_writer_create label={} duration_ms={}", label, start.elapsed().as_millis()); + Ok(writer) +} + +/// 每个 Tantivy 索引对应一个长期存活的 writer 线程。 +/// +/// `IndexWriter` 不是 `Send`/`Sync`，不能放在 `Arc>` 里跨越 await，因此在线程内部持有 writer， +/// 外部通过 channel 发送写/删除/merge 任务，并通过 oneshot 等待结果。每次操作后仍执行 commit，保留原 +/// 有的数据安全语义。 +pub struct IndexWriterHandle { + tx: Sender, + thread: Option>, +} + +struct WriterJob { + op: WriterOp, + reply: Option>, +} + +enum WriterOp { + WriteBatch(Vec), + DeleteTerms { field_name: &'static str, ids: Vec }, + ForceMerge, + Shutdown, +} + +enum WriterResult { + Ok, + WriteBatch, + ForceMerge(ForceMergeStats), + Err(String), +} + +impl IndexWriterHandle { + pub async fn open(index: Index, schema: Schema, label: String) -> anyhow::Result> { + let (init_tx, init_rx) = tokio::sync::oneshot::channel::>(); + let (tx, rx) = mpsc::channel::(); + let thread_name = format!("tantivy-writer-{}", label); + let thread = thread::Builder::new().name(thread_name).spawn(move || { + let mut writer = match create_writer_with_timing_blocking(&index, &label) { + Ok(w) => { + let _ = init_tx.send(Ok(())); + Some(w) + } + Err(e) => { + let _ = init_tx.send(Err(anyhow::anyhow!(e))); + return; + } + }; + + while let Ok(job) = rx.recv() { + let current_writer = match writer.take() { + Some(w) => w, + None => { + let result = WriterResult::Err("tantivy writer is not available".to_string()); + if let Some(reply) = job.reply { + let _ = reply.send(result); + } + continue; + } + }; + + let (result, maybe_writer) = match job.op { + WriterOp::Shutdown => { + drop(current_writer); + break; + } + WriterOp::WriteBatch(docs) => { + if docs.is_empty() { + (WriterResult::WriteBatch, Some(current_writer)) + } else { + let mut writer = current_writer; + let result = match add_documents(&mut writer, &schema, docs) { + Ok(count) => match commit_writer(&mut writer, &label, count) { + Ok(()) => WriterResult::WriteBatch, + Err(e) => WriterResult::Err(e.to_string()), + }, + Err(e) => WriterResult::Err(e.to_string()), + }; + (result, Some(writer)) + } + } + WriterOp::DeleteTerms { field_name, ids } => { + if ids.is_empty() { + (WriterResult::Ok, Some(current_writer)) + } else { + let mut writer = current_writer; + let field = get_field(&schema, field_name); + let delete_start = Instant::now(); + for id in &ids { + let term = Term::from_field_i64(field, *id); + writer.delete_term(term); + } + debug!( + "tantivy_delete_terms target={} count={} duration_ms={}", + field_name, + ids.len(), + delete_start.elapsed().as_millis() + ); + let result = match commit_writer(&mut writer, &label, ids.len()) { + Ok(()) => WriterResult::Ok, + Err(e) => WriterResult::Err(e.to_string()), + }; + (result, Some(writer)) + } + } + WriterOp::ForceMerge => match force_merge_with_writer(&index, current_writer, &label) { + Ok((stats, new_writer)) => (WriterResult::ForceMerge(stats), Some(new_writer)), + Err(e) => (WriterResult::Err(e.to_string()), None), + }, + }; + + writer = maybe_writer; + if let Some(reply) = job.reply { + let _ = reply.send(result); + } + // writer 为 None 时说明 force_merge 失败且未能重建，退出循环 + if writer.is_none() { + warn!("tantivy writer thread {} exiting after force_merge failure", label); + break; + } + } + // writer 在这里 drop，释放目录锁 + })?; + + init_rx.await??; + Ok(Arc::new(Self { tx, thread: Some(thread) })) + } + + pub async fn write_batch(&self, docs: Vec) -> anyhow::Result<()> { + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + self.tx + .send(WriterJob { op: WriterOp::WriteBatch(docs), reply: Some(reply_tx) }) + .map_err(|e| anyhow::anyhow!("tantivy writer channel closed: {e}"))?; + match reply_rx.await? { + WriterResult::Ok | WriterResult::WriteBatch => Ok(()), + WriterResult::ForceMerge(_) => Err(anyhow::anyhow!("unexpected force_merge result from write_batch")), + WriterResult::Err(e) => Err(anyhow::anyhow!("tantivy write_batch failed: {e}")), + } + } + + pub async fn delete_by_field(&self, field_name: &'static str, ids: &[i64]) -> anyhow::Result<()> { + if ids.is_empty() { + return Ok(()); + } + let ids = ids.to_vec(); + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + self.tx + .send(WriterJob { op: WriterOp::DeleteTerms { field_name, ids }, reply: Some(reply_tx) }) + .map_err(|e| anyhow::anyhow!("tantivy writer channel closed: {e}"))?; + match reply_rx.await? { + WriterResult::Ok => Ok(()), + WriterResult::WriteBatch => Err(anyhow::anyhow!("unexpected write result from delete")), + WriterResult::ForceMerge(_) => Err(anyhow::anyhow!("unexpected force_merge result from delete")), + WriterResult::Err(e) => Err(anyhow::anyhow!("tantivy delete failed: {e}")), + } + } + + pub async fn force_merge(&self) -> anyhow::Result { + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + self.tx + .send(WriterJob { op: WriterOp::ForceMerge, reply: Some(reply_tx) }) + .map_err(|e| anyhow::anyhow!("tantivy writer channel closed: {e}"))?; + match reply_rx.await? { + WriterResult::ForceMerge(stats) => Ok(stats), + WriterResult::Ok => Err(anyhow::anyhow!("unexpected ok result from force_merge")), + WriterResult::WriteBatch => Err(anyhow::anyhow!("unexpected write result from force_merge")), + WriterResult::Err(e) => Err(anyhow::anyhow!("tantivy force_merge failed: {e}")), + } + } +} + +impl Drop for IndexWriterHandle { + fn drop(&mut self) { + if let Some(thread) = self.thread.take() { + let _ = self.tx.send(WriterJob { op: WriterOp::Shutdown, reply: None }); + let _ = thread.join(); + } + } +} + +fn force_merge_with_writer( + index: &Index, mut writer: tantivy::IndexWriter, label: &str, +) -> anyhow::Result<(ForceMergeStats, tantivy::IndexWriter)> { + let start = Instant::now(); + let before_metas = index.searchable_segment_metas()?; + let before_segments = before_metas.len(); + let before_docs = segment_docs(&before_metas); + let before_deleted_docs = segment_deleted_docs(&before_metas); + + if before_segments < 2 { + let (gc_deleted_files, gc_failed_files) = garbage_collect_index_files(&writer, "force_merge_skipped")?; + writer.wait_merging_threads()?; + let new_writer = create_writer_with_timing_blocking(index, &format!("{}_gc", label))?; + return Ok(( + ForceMergeStats { + before_segments, + after_segments: before_segments, + before_docs, + after_docs: before_docs, + before_deleted_docs, + after_deleted_docs: before_deleted_docs, + gc_deleted_files, + gc_failed_files, + skipped: true, + duration_ms: start.elapsed().as_millis(), + }, + new_writer, + )); + } + + let segment_ids = before_metas.iter().map(|meta| meta.id()).collect::>(); + writer.merge(&segment_ids).wait()?; + writer.wait_merging_threads()?; + let new_writer = create_writer_with_timing_blocking(index, &format!("{}_gc", label))?; + let (gc_deleted_files, gc_failed_files) = garbage_collect_index_files(&new_writer, "force_merge")?; + + let after_metas = index.searchable_segment_metas()?; + Ok(( + ForceMergeStats { + before_segments, + after_segments: after_metas.len(), + before_docs, + after_docs: segment_docs(&after_metas), + before_deleted_docs, + after_deleted_docs: segment_deleted_docs(&after_metas), + gc_deleted_files, + gc_failed_files, + skipped: false, + duration_ms: start.elapsed().as_millis(), + }, + new_writer, + )) +} + +pub async fn create_rebuild_writer(index: &Index, label: &str) -> tantivy::Result { + create_writer_with_timing(index, label).await +} + +pub fn add_documents( + index_writer: &mut tantivy::IndexWriter, schema: &Schema, docs: impl IntoIterator, +) -> tantivy::Result { + let mut count = 0_usize; + for doc in docs { + index_writer.add_document(create_document(doc, schema))?; + count += 1; + } + Ok(count) +} + +pub fn commit_writer(index_writer: &mut tantivy::IndexWriter, label: &str, doc_count: usize) -> tantivy::Result<()> { + let commit_start = Instant::now(); + index_writer.commit()?; + debug!("tantivy_commit label={} docs={} duration_ms={}", label, doc_count, commit_start.elapsed().as_millis()); + Ok(()) +} + +pub fn search_sync( + reader: &IndexReader, schema: &Schema, query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + filter_is_image: Option, synonym_map: Option<&SynonymMap>, +) -> anyhow::Result> { + let cfg = config::get(); + let searcher = reader.searcher(); + let tantivy_query = build_query(query, file_ids, kb_ids, filter_is_image, schema, synonym_map)?; + let search_start = Instant::now(); + let top_docs = searcher.search(&tantivy_query, &TopDocs::with_limit(cfg.search.limit))?; + debug!("Tantivy searcher.search {}ms", search_start.elapsed().as_millis()); + + let mut results = vec![]; + let mut doc_total_ms: u128 = 0; + let mut doc_max_ms: u128 = 0; + let mut doc_count: usize = 0; + for (score, doc_address) in top_docs { + let doc_start = Instant::now(); + let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?; + let doc_elapsed = doc_start.elapsed().as_millis(); + doc_total_ms += doc_elapsed; + if doc_elapsed > doc_max_ms { + doc_max_ms = doc_elapsed; + } + doc_count += 1; + let id = retrieved_doc.get_first(get_field(schema, "id")).and_then(|v| v.as_i64()).unwrap_or(0); + let file_id = retrieved_doc.get_first(get_field(schema, "file_id")).and_then(|v| v.as_i64()).unwrap_or(0); + let kb_id = retrieved_doc.get_first(get_field(schema, "kb_id")).and_then(|v| v.as_i64()); + let content = retrieved_doc + .get_first(get_field(schema, "content")) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .unwrap_or_default(); + results.push(SearchResultItem { id, file_id, kb_id, content, score }); + } + debug!("Tantivy searcher.doc total={}ms max={}ms count={}", doc_total_ms, doc_max_ms, doc_count); + + Ok(results) +} + +pub async fn search( + reader: &IndexReader, schema: &Schema, query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + filter_is_image: Option, synonym_map: Option<&SynonymMap>, +) -> anyhow::Result> { + search_sync(reader, schema, query, file_ids, kb_ids, filter_is_image, synonym_map) +} + +#[allow(clippy::too_many_arguments)] +pub fn search_with_snippet_sync( + reader: &IndexReader, schema: &Schema, query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + filter_is_image: Option, max_chars: usize, synonym_map: Option<&SynonymMap>, +) -> anyhow::Result> { + let cfg = config::get(); + let searcher = reader.searcher(); + let tantivy_query = build_query(query, file_ids, kb_ids, filter_is_image, schema, synonym_map)?; + let search_start = Instant::now(); + let top_docs = searcher.search(&tantivy_query, &TopDocs::with_limit(cfg.search.limit))?; + debug!("Tantivy full searcher.search {}ms", search_start.elapsed().as_millis()); + + let terms = build_snippet_terms(query, synonym_map); + let matcher = build_matcher(&terms); + + let mut results = vec![]; + let mut doc_total_ms: u128 = 0; + let mut doc_max_ms: u128 = 0; + let mut doc_count: usize = 0; + for (score, doc_address) in top_docs { + let doc_start = Instant::now(); + let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?; + let doc_elapsed = doc_start.elapsed().as_millis(); + doc_total_ms += doc_elapsed; + if doc_elapsed > doc_max_ms { + doc_max_ms = doc_elapsed; + } + doc_count += 1; + let file_id = retrieved_doc.get_first(get_field(schema, "file_id")).and_then(|v| v.as_i64()).unwrap_or(0); + let kb_id = retrieved_doc.get_first(get_field(schema, "kb_id")).and_then(|v| v.as_i64()); + let content = retrieved_doc.get_first(get_field(schema, "content")).and_then(|v| v.as_str()).unwrap_or(""); + let snippet = build_best_snippet(content, matcher.as_ref(), terms.len(), max_chars); + results.push(FullSearchResultItem { file_id, kb_id, snippet, score }); + } + debug!("Tantivy full searcher.doc total={}ms max={}ms count={}", doc_total_ms, doc_max_ms, doc_count); + + Ok(results) +} + +#[allow(clippy::too_many_arguments)] +pub async fn search_with_snippet( + reader: &IndexReader, schema: &Schema, query: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, + filter_is_image: Option, max_chars: usize, synonym_map: Option<&SynonymMap>, +) -> anyhow::Result> { + // 搜索 + 取文档 + 生成 snippet 均为 CPU 密集的同步操作，放到阻塞线程池执行，避免阻塞异步运行时。 + let reader = reader.clone(); + let schema = schema.clone(); + let query = query.to_string(); + let file_ids = file_ids.cloned(); + let kb_ids = kb_ids.cloned(); + let synonym_map = synonym_map.cloned(); + tokio::task::spawn_blocking(move || { + search_with_snippet_sync( + &reader, + &schema, + &query, + file_ids.as_ref(), + kb_ids.as_ref(), + filter_is_image, + max_chars, + synonym_map.as_ref(), + ) + }) + .await + .map_err(|e| anyhow::anyhow!("search_with_snippet task panicked: {e}"))? +} + +fn garbage_collect_index_files(writer: &tantivy::IndexWriter, label: &str) -> anyhow::Result<(usize, usize)> { + let start = Instant::now(); + let gc_result = writer.garbage_collect_files().wait()?; + let deleted_files = gc_result.deleted_files.len(); + let failed_files = gc_result.failed_to_delete_files.len(); + if failed_files > 0 { + warn!( + "tantivy_gc label={} deleted_files={} failed_files={} duration_ms={}", + label, + deleted_files, + failed_files, + start.elapsed().as_millis() + ); + } else { + debug!( + "tantivy_gc label={} deleted_files={} failed_files={} duration_ms={}", + label, + deleted_files, + failed_files, + start.elapsed().as_millis() + ); + } + Ok((deleted_files, failed_files)) +} + +fn segment_docs(metas: &[tantivy::SegmentMeta]) -> u64 { + metas.iter().map(|meta| u64::from(meta.num_docs())).sum() +} + +fn segment_deleted_docs(metas: &[tantivy::SegmentMeta]) -> u64 { + metas.iter().map(|meta| u64::from(meta.num_deleted_docs())).sum() +} + +fn build_query( + input: &str, file_ids: Option<&Vec>, kb_ids: Option<&Vec>, filter_is_image: Option, + schema: &Schema, synonym_map: Option<&SynonymMap>, +) -> tantivy::Result> { + let mut subqueries: Vec<(Occur, Box)> = Vec::new(); + let query_terms = build_query_terms(input, synonym_map); + + // content 至少命中一个 + let mut content_queries = Vec::new(); + for query_term in query_terms { + let tq = TermQuery::new( + Term::from_field_text(get_field(schema, "content"), query_term.term.as_str()), + IndexRecordOption::Basic, + ); + if (query_term.boost - 1.0).abs() > f32::EPSILON { + content_queries + .push((Occur::Should, Box::new(BoostQuery::new(Box::new(tq), query_term.boost)) as Box)); + } else { + content_queries.push((Occur::Should, Box::new(tq) as Box)); + } + } + let content_bool = BooleanQuery::new(content_queries); + subqueries.push((Occur::Must, Box::new(content_bool))); + + if let Some(ids) = file_ids + && !ids.is_empty() + { + let mut file_id_queries = Vec::new(); + let file_id_field = get_field(schema, "file_id"); + for file_id in ids { + let file_id_query = TermQuery::new(Term::from_field_i64(file_id_field, *file_id), IndexRecordOption::Basic); + file_id_queries.push((Occur::Should, Box::new(file_id_query) as Box)); + } + if !file_id_queries.is_empty() { + let file_ids_bool_query = BooleanQuery::new(file_id_queries); + subqueries.push((Occur::Must, Box::new(file_ids_bool_query))); + } + } + + if let Some(ids) = kb_ids + && !ids.is_empty() + { + let mut kb_id_queries = Vec::new(); + let kb_id_field = get_field(schema, "kb_id"); + for kb_id in ids { + let kb_id_query = TermQuery::new(Term::from_field_i64(kb_id_field, *kb_id), IndexRecordOption::Basic); + kb_id_queries.push((Occur::Should, Box::new(kb_id_query) as Box)); + } + if !kb_id_queries.is_empty() { + let kb_ids_bool_query = BooleanQuery::new(kb_id_queries); + subqueries.push((Occur::Must, Box::new(kb_ids_bool_query))); + } + } + + if let Some(is_image) = filter_is_image { + let is_image_field = get_field(schema, "is_image"); + let term = Term::from_field_i64(is_image_field, i64::from(is_image)); + subqueries.push((Occur::Must, Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box)); + } + + Ok(Box::new(BooleanQuery::new(subqueries))) +} + +fn create_document(doc: Document, schema: &Schema) -> TantivyDocument { + let mut document = doc! { + get_field(schema, "id") => doc.id, + get_field(schema, "file_id") => doc.file_id, + get_field(schema, "content") => doc.content, + get_field(schema, "is_image") => i64::from(doc.is_image), + }; + if let Some(kb_id) = doc.kb_id { + document.add_i64(get_field(schema, "kb_id"), kb_id); + } + document +} + +fn get_field(schema: &Schema, field: &str) -> Field { + schema.get_field(field).unwrap_or_else(|_| panic!("Field '{}' not found in schema", field)) +} + +fn perform_segmentation(text: &str, mode: chinese_tokenizer::SegmentationMode) -> Vec { + chinese_tokenizer::FastChineseTokenizer::new(mode).segment(text) +} + +fn register_tokenizers(index: &Index) { + let all_tokenizer = chinese_tokenizer::FastChineseTokenizer::all(); + index.tokenizers().register(ALL_TOKENIZER, all_tokenizer); +} + +fn build_schema() -> Schema { + let mut schema_builder = Schema::builder(); + schema_builder.add_i64_field("id", INDEXED | STORED | FAST); // 切片 id + schema_builder.add_i64_field("file_id", INDEXED | STORED | FAST); // 文件 id + schema_builder.add_i64_field("kb_id", INDEXED | STORED | FAST); // 知识库 id + schema_builder.add_i64_field("is_image", INDEXED | FAST); // 是否为图片切片 + + let text_options = TextOptions::default() + .set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(ALL_TOKENIZER) + .set_index_option(IndexRecordOption::WithFreqsAndPositions), + ) + .set_stored(); + schema_builder.add_text_field("content", text_options); // 内容 + schema_builder.build() +} + +#[derive(Debug, Clone, Copy)] +struct MatchInfo { + start_char: usize, + end_char: usize, + term_idx: usize, +} + +fn build_snippet_terms(query: &str, synonym_map: Option<&SynonymMap>) -> Vec { + let mut terms = Vec::new(); + let mut seen = HashSet::new(); + let segmented = perform_segmentation(query, chinese_tokenizer::SegmentationMode::Search); + for term in segmented.into_iter().chain(std::iter::once(query.trim().to_string())) { + let trimmed = term.trim(); + if trimmed.is_empty() { + continue; + } + if seen.insert(trimmed.to_string()) { + terms.push(trimmed.to_string()); + } + if let Some(map) = synonym_map + && let Some(synonyms) = map.get(trimmed) + { + for synonym in synonyms { + let syn = synonym.term.trim(); + if syn.is_empty() { + continue; + } + if seen.insert(syn.to_string()) { + terms.push(syn.to_string()); + } + } + } + } + terms.sort_by_key(|b| std::cmp::Reverse(b.len())); + terms +} + +#[derive(Debug, Clone)] +struct QueryTerm { + term: String, + boost: f32, +} + +fn build_query_terms(input: &str, synonym_map: Option<&SynonymMap>) -> Vec { + let mut terms = Vec::new(); + let mut seen = HashSet::new(); + let segmented = perform_segmentation(input, chinese_tokenizer::SegmentationMode::Search); + for term in segmented { + let trimmed = term.trim(); + if trimmed.is_empty() { + continue; + } + if seen.insert(trimmed.to_string()) { + terms.push(QueryTerm { term: trimmed.to_string(), boost: 1.0 }); + } + if let Some(map) = synonym_map + && let Some(synonyms) = map.get(trimmed) + { + for synonym in synonyms { + let syn = synonym.term.trim(); + if syn.is_empty() || syn == trimmed { + continue; + } + if seen.insert(syn.to_string()) { + terms.push(QueryTerm { term: syn.to_string(), boost: synonym.boost.max(0.01) }); + } + } + } + } + + if terms.is_empty() { + let raw = input.trim(); + if !raw.is_empty() { + terms.push(QueryTerm { term: raw.to_string(), boost: 1.0 }); + if let Some(map) = synonym_map + && let Some(synonyms) = map.get(raw) + { + for synonym in synonyms { + let syn = synonym.term.trim(); + if syn.is_empty() || syn == raw { + continue; + } + if seen.insert(syn.to_string()) { + terms.push(QueryTerm { term: syn.to_string(), boost: synonym.boost.max(0.01) }); + } + } + } + } + } + + terms +} + +fn build_matcher(terms: &[String]) -> Option { + if terms.is_empty() { + return None; + } + AhoCorasickBuilder::new().ascii_case_insensitive(true).match_kind(MatchKind::LeftmostLongest).build(terms).ok() +} + +fn build_best_snippet(content: &str, matcher: Option<&AhoCorasick>, term_count: usize, max_chars: usize) -> String { + if content.is_empty() { + return String::new(); + } + let char_to_byte = build_char_index(content); + let char_len = char_to_byte.len().saturating_sub(1); + let max_chars = max_chars.max(1); + + if char_len <= max_chars { + return highlight_range(content, matcher, &char_to_byte, 0, char_len, false, false); + } + + let matches = + if let Some(matcher) = matcher { collect_matches(content, matcher, &char_to_byte) } else { Vec::new() }; + + if matches.is_empty() { + return highlight_range(content, matcher, &char_to_byte, 0, max_chars, false, true); + } + + let (best_left, best_right) = select_best_window(&matches, term_count, max_chars); + let left_start = matches[best_left].start_char; + let right_end = matches[best_right].end_char; + let span = right_end.saturating_sub(left_start); + let extra = max_chars.saturating_sub(span); + let desired_start = left_start.saturating_sub(extra / 2); + let lower_bound = right_end.saturating_sub(max_chars); + let upper_bound = left_start.min(char_len.saturating_sub(max_chars)); + let start = desired_start.clamp(lower_bound, upper_bound); + let end = start.saturating_add(max_chars).min(char_len); + + highlight_range(content, matcher, &char_to_byte, start, end, start > 0, end < char_len) +} + +fn collect_matches(content: &str, matcher: &AhoCorasick, char_to_byte: &[usize]) -> Vec { + let mut matches = Vec::new(); + for m in matcher.find_iter(content) { + let start_char = byte_to_char_start(char_to_byte, m.start()); + let end_char = byte_to_char_end(char_to_byte, m.end()); + if end_char > start_char { + matches.push(MatchInfo { start_char, end_char, term_idx: m.pattern().as_usize() }); + } + } + matches +} + +fn select_best_window(matches: &[MatchInfo], term_count: usize, max_chars: usize) -> (usize, usize) { + let mut counts = vec![0usize; term_count]; + let mut unique = 0usize; + let mut right = 0usize; + let mut best_left = 0usize; + let mut best_right = 0usize; + let mut best_unique = 0usize; + let mut best_total = 0usize; + let mut best_span = usize::MAX; + + for left in 0..matches.len() { + if right < left { + right = left; + } + let window_start = matches[left].start_char; + let window_end = window_start.saturating_add(max_chars); + while right < matches.len() && matches[right].end_char <= window_end { + let term_idx = matches[right].term_idx; + if term_idx < term_count { + if counts[term_idx] == 0 { + unique += 1; + } + counts[term_idx] += 1; + } + right += 1; + } + let total = right.saturating_sub(left); + if total > 0 { + let rightmost_end = matches[right - 1].end_char; + let span = rightmost_end.saturating_sub(matches[left].start_char); + let better = unique > best_unique + || (unique == best_unique && total > best_total) + || (unique == best_unique && total == best_total && span < best_span) + || (unique == best_unique + && total == best_total + && span == best_span + && matches[left].start_char < matches[best_left].start_char); + if better { + best_unique = unique; + best_total = total; + best_span = span; + best_left = left; + best_right = right - 1; + } + } + let term_idx = matches[left].term_idx; + if term_idx < term_count { + counts[term_idx] = counts[term_idx].saturating_sub(1); + if counts[term_idx] == 0 { + unique = unique.saturating_sub(1); + } + } + } + + (best_left, best_right) +} + +fn build_char_index(text: &str) -> Vec { + let mut char_to_byte = Vec::with_capacity(text.chars().count() + 1); + for (byte_idx, _) in text.char_indices() { + char_to_byte.push(byte_idx); + } + char_to_byte.push(text.len()); + char_to_byte +} + +fn byte_to_char_start(char_to_byte: &[usize], byte_idx: usize) -> usize { + match char_to_byte.binary_search(&byte_idx) { + Ok(idx) => idx, + Err(idx) => idx.saturating_sub(1), + } +} + +fn byte_to_char_end(char_to_byte: &[usize], byte_idx: usize) -> usize { + match char_to_byte.binary_search(&byte_idx) { + Ok(idx) => idx, + Err(idx) => idx, + } +} + +fn highlight_range( + content: &str, matcher: Option<&AhoCorasick>, char_to_byte: &[usize], start_char: usize, end_char: usize, + prefix: bool, suffix: bool, +) -> String { + let start = *char_to_byte.get(start_char).unwrap_or(&0); + let end = *char_to_byte.get(end_char).unwrap_or(&content.len()); + let slice = &content[start..end.min(content.len())]; + + let mut output = String::new(); + if prefix { + output.push_str("..."); + } + if let Some(matcher) = matcher { + let mut last = 0usize; + for m in matcher.find_iter(slice) { + let start_idx = m.start(); + let end_idx = m.end(); + if start_idx > last { + output.push_str(&escape_html(&slice[last..start_idx])); + } + output.push_str(""); + output.push_str(&escape_html(&slice[start_idx..end_idx])); + output.push_str(""); + last = end_idx; + } + if last < slice.len() { + output.push_str(&escape_html(&slice[last..])); + } + } else { + output.push_str(&escape_html(slice)); + } + if suffix { + output.push_str("..."); + } + output +} + +fn escape_html(input: &str) -> String { + let mut escaped = String::with_capacity(input.len()); + for ch in input.chars() { + match ch { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + _ => escaped.push(ch), + } + } + escaped +} diff --git a/src/slice_content.rs b/src/slice_content.rs new file mode 100644 index 0000000..26da14b --- /dev/null +++ b/src/slice_content.rs @@ -0,0 +1,54 @@ +use std::{collections::HashMap, path::PathBuf}; + +use anyhow::Context; + +use crate::config; + +fn content_path(file_id: i64) -> PathBuf { + PathBuf::from(&config::get().storage.slice_contents_path).join(format!("{}.json", file_id)) +} + +/// 一个源文件的全部切片正文。正文不再放入 SQLite，以减少 WAL 和主库体积。 +pub async fn read_all(file_id: i64) -> anyhow::Result> { + let path = content_path(file_id); + match tokio::fs::read(&path).await { + Ok(bytes) => { + serde_json::from_slice(&bytes).with_context(|| format!("invalid slice content file {}", path.display())) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(HashMap::new()), + Err(e) => Err(e).with_context(|| format!("failed to read slice content file {}", path.display())), + } +} + +pub async fn upsert_many(file_id: i64, rows: &[(i64, String)]) -> anyhow::Result<()> { + if rows.is_empty() { + return Ok(()); + } + let mut contents = read_all(file_id).await?; + for (id, content) in rows { + contents.insert(*id, content.clone()); + } + write_all(file_id, &contents).await +} + +pub async fn write_all(file_id: i64, contents: &HashMap) -> anyhow::Result<()> { + let path = content_path(file_id); + if contents.is_empty() { + return delete(file_id).await; + } + let parent = path.parent().expect("slice content path has parent"); + tokio::fs::create_dir_all(parent).await?; + let bytes = serde_json::to_vec(contents)?; + let tmp = path.with_extension("json.tmp"); + tokio::fs::write(&tmp, bytes).await.with_context(|| format!("failed to write {}", tmp.display()))?; + tokio::fs::rename(&tmp, &path).await.with_context(|| format!("failed to rename {}", path.display())) +} + +pub async fn delete(file_id: i64) -> anyhow::Result<()> { + let path = content_path(file_id); + match tokio::fs::remove_file(&path).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).with_context(|| format!("failed to delete slice content file {}", path.display())), + } +} diff --git a/stopwords.txt b/stopwords.txt new file mode 100644 index 0000000..fc3c61d --- /dev/null +++ b/stopwords.txt @@ -0,0 +1,1395 @@ +-- +? +“ +” +》 +－－ +able +about +above +according +accordingly +across +actually +after +afterwards +again +against +ain't +all +allow +allows +almost +alone +along +already +also +although +always +am +among +amongst +an +and +another +any +anybody +anyhow +anyone +anything +anyway +anyways +anywhere +apart +appear +appreciate +appropriate +are +aren't +around +as +a's +aside +ask +asking +associated +at +available +away +awfully +be +became +because +become +becomes +becoming +been +before +beforehand +behind +being +believe +below +beside +besides +best +better +between +beyond +both +brief +but +by +came +can +cannot +cant +can't +cause +causes +certain +certainly +changes +clearly +c'mon +co +com +come +comes +concerning +consequently +consider +considering +contain +containing +contains +corresponding +could +couldn't +course +c's +currently +definitely +described +despite +did +didn't +different +do +does +doesn't +doing +done +don't +down +downwards +during +each +edu +eg +eight +either +else +elsewhere +enough +entirely +especially +et +etc +even +ever +every +everybody +everyone +everything +everywhere +ex +exactly +example +except +far +few +fifth +first +five +followed +following +follows +for +former +formerly +forth +four +from +further +furthermore +get +gets +getting +given +gives +go +goes +going +gone +got +gotten +greetings +had +hadn't +happens +hardly +has +hasn't +have +haven't +having +he +hello +help +hence +her +here +hereafter +hereby +herein +here's +hereupon +hers +herself +he's +hi +him +himself +his +hither +hopefully +how +howbeit +however +i'd +ie +if +ignored +i'll +i'm +immediate +in +inasmuch +inc +indeed +indicate +indicated +indicates +inner +insofar +instead +into +inward +is +isn't +it +it'd +it'll +its +it's +itself +i've +just +keep +keeps +kept +know +known +knows +last +lately +later +latter +latterly +least +less +lest +let +let's +like +liked +likely +little +look +looking +looks +ltd +mainly +many +may +maybe +me +mean +meanwhile +merely +might +more +moreover +most +mostly +much +must +my +myself +name +namely +nd +near +nearly +necessary +need +needs +neither +never +nevertheless +new +next +nine +no +nobody +non +none +noone +nor +normally +not +nothing +novel +now +nowhere +obviously +of +off +often +oh +ok +okay +old +on +once +one +ones +only +onto +or +other +others +otherwise +ought +our +ours +ourselves +out +outside +over +overall +own +particular +particularly +per +perhaps +placed +please +plus +possible +presumably +probably +provides +que +quite +qv +rather +rd +re +really +reasonably +regarding +regardless +regards +relatively +respectively +right +said +same +saw +say +saying +says +second +secondly +see +seeing +seem +seemed +seeming +seems +seen +self +selves +sensible +sent +serious +seriously +seven +several +shall +she +should +shouldn't +since +six +so +some +somebody +somehow +someone +something +sometime +sometimes +somewhat +somewhere +soon +sorry +specified +specify +specifying +still +sub +such +sup +sure +take +taken +tell +tends +th +than +thank +thanks +thanx +that +thats +that's +the +their +theirs +them +themselves +then +thence +there +thereafter +thereby +therefore +therein +theres +there's +thereupon +these +they +they'd +they'll +they're +they've +think +third +this +thorough +thoroughly +those +though +three +through +throughout +thru +thus +to +together +too +took +toward +towards +tried +tries +truly +try +trying +t's +twice +two +un +under +unfortunately +unless +unlikely +until +unto +up +upon +us +use +used +useful +uses +using +usually +value +various +very +via +viz +vs +want +wants +was +wasn't +way +we +we'd +welcome +well +we'll +went +were +we're +weren't +we've +what +whatever +what's +when +whence +whenever +where +whereafter +whereas +whereby +wherein +where's +whereupon +wherever +whether +which +while +whither +who +whoever +whole +whom +who's +whose +why +will +willing +wish +with +within +without +wonder +won't +would +wouldn't +yes +yet +you +you'd +you'll +your +you're +yours +yourself +yourselves +you've +zero +zt +ZT +zz +ZZ +一 +一下 +一些 +一切 +一则 +一天 +一定 +一方面 +一旦 +一时 +一来 +一样 +一次 +一片 +一直 +一致 +一般 +一起 +一边 +一面 +万一 +上下 +上升 +上去 +上来 +上述 +上面 +下列 +下去 +下来 +下面 +不一 +不久 +不仅 +不会 +不但 +不光 +不单 +不变 +不只 +不可 +不同 +不够 +不如 +不得 +不怕 +不惟 +不成 +不拘 +不敢 +不断 +不是 +不比 +不然 +不特 +不独 +不管 +不能 +不要 +不论 +不足 +不过 +不问 +与 +与其 +与否 +与此同时 +专门 +且 +两者 +严格 +严重 +个 +个人 +个别 +中小 +中间 +丰富 +临 +为 +为主 +为了 +为什么 +为什麽 +为何 +为着 +主张 +主要 +举行 +乃 +乃至 +么 +之 +之一 +之前 +之后 +之後 +之所以 +之类 +乌乎 +乎 +乘 +也 +也好 +也是 +也罢 +了 +了解 +争取 +于 +于是 +于是乎 +云云 +互相 +产生 +人们 +人家 +什么 +什么样 +什麽 +今后 +今天 +今年 +今後 +仍然 +从 +从事 +从而 +他 +他人 +他们 +他的 +代替 +以 +以上 +以下 +以为 +以便 +以免 +以前 +以及 +以后 +以外 +以後 +以来 +以至 +以至于 +以致 +们 +任 +任何 +任凭 +任务 +企图 +伟大 +似乎 +似的 +但 +但是 +何 +何况 +何处 +何时 +作为 +你 +你们 +你的 +使得 +使用 +例如 +依 +依照 +依靠 +促进 +保持 +俺 +俺们 +倘 +倘使 +倘或 +倘然 +倘若 +假使 +假如 +假若 +做到 +像 +允许 +充分 +先后 +先後 +先生 +全部 +全面 +兮 +共同 +关于 +其 +其一 +其中 +其二 +其他 +其余 +其它 +其实 +其次 +具体 +具体地说 +具体说来 +具有 +再者 +再说 +冒 +冲 +决定 +况且 +准备 +几 +几乎 +几时 +凭 +凭借 +出去 +出来 +出现 +分别 +则 +别 +别的 +别说 +到 +前后 +前者 +前进 +前面 +加之 +加以 +加入 +加强 +十分 +即 +即令 +即使 +即便 +即或 +即若 +却不 +原来 +又 +及 +及其 +及时 +及至 +双方 +反之 +反应 +反映 +反过来 +反过来说 +取得 +受到 +变成 +另 +另一方面 +另外 +只是 +只有 +只要 +只限 +叫 +叫做 +召开 +叮咚 +可 +可以 +可是 +可能 +可见 +各 +各个 +各人 +各位 +各地 +各种 +各级 +各自 +合理 +同 +同一 +同时 +同样 +后来 +后面 +向 +向着 +吓 +吗 +否则 +吧 +吧哒 +吱 +呀 +呃 +呕 +呗 +呜 +呜呼 +呢 +周围 +呵 +呸 +呼哧 +咋 +和 +咚 +咦 +咱 +咱们 +咳 +哇 +哈 +哈哈 +哉 +哎 +哎呀 +哎哟 +哗 +哟 +哦 +哩 +哪 +哪个 +哪些 +哪儿 +哪天 +哪年 +哪怕 +哪样 +哪边 +哪里 +哼 +哼唷 +唉 +啊 +啐 +啥 +啦 +啪达 +喂 +喏 +喔唷 +嗡嗡 +嗬 +嗯 +嗳 +嘎 +嘎登 +嘘 +嘛 +嘻 +嘿 +因 +因为 +因此 +因而 +固然 +在 +在下 +地 +坚决 +坚持 +基本 +处理 +复杂 +多 +多少 +多数 +多次 +大力 +大多数 +大大 +大家 +大批 +大约 +大量 +失去 +她 +她们 +她的 +好的 +好象 +如 +如上所述 +如下 +如何 +如其 +如果 +如此 +如若 +存在 +宁 +宁可 +宁愿 +宁肯 +它 +它们 +它们的 +它的 +安全 +完全 +完成 +实现 +实际 +宣布 +容易 +密切 +对 +对于 +对应 +将 +少数 +尔后 +尚且 +尤其 +就 +就是 +就是说 +尽 +尽管 +属于 +岂但 +左右 +巨大 +巩固 +己 +已经 +帮助 +常常 +并 +并不 +并不是 +并且 +并没有 +广大 +广泛 +应当 +应用 +应该 +开外 +开始 +开展 +引起 +强烈 +强调 +归 +当 +当前 +当时 +当然 +当着 +形成 +彻底 +彼 +彼此 +往 +往往 +待 +後来 +後面 +得 +得出 +得到 +心里 +必然 +必要 +必须 +怎 +怎么 +怎么办 +怎么样 +怎样 +怎麽 +总之 +总是 +总的来看 +总的来说 +总的说来 +总结 +总而言之 +恰恰相反 +您 +意思 +愿意 +慢说 +成为 +我 +我们 +我的 +或 +或是 +或者 +战斗 +所 +所以 +所有 +所谓 +打 +扩大 +把 +抑或 +拿 +按 +按照 +换句话说 +换言之 +据 +掌握 +接着 +接著 +故 +故此 +整个 +方便 +方面 +旁人 +无宁 +无法 +无论 +既 +既是 +既然 +时候 +明显 +明确 +是 +是否 +是的 +显然 +显著 +普通 +普遍 +更加 +曾经 +替 +最后 +最大 +最好 +最後 +最近 +最高 +有 +有些 +有关 +有利 +有力 +有所 +有效 +有时 +有点 +有的 +有着 +有著 +望 +朝 +朝着 +本 +本着 +来 +来着 +极了 +构成 +果然 +果真 +某 +某个 +某些 +根据 +根本 +欢迎 +正在 +正如 +正常 +此 +此外 +此时 +此间 +毋宁 +每 +每个 +每天 +每年 +每当 +比 +比如 +比方 +比较 +毫不 +没有 +沿 +沿着 +注意 +深入 +清楚 +满足 +漫说 +焉 +然则 +然后 +然後 +然而 +照 +照着 +特别是 +特殊 +特点 +现代 +现在 +甚么 +甚而 +甚至 +用 +由 +由于 +由此可见 +的 +的话 +目前 +直到 +直接 +相似 +相信 +相反 +相同 +相对 +相对而言 +相应 +相当 +相等 +省得 +看出 +看到 +看来 +看看 +看见 +真是 +真正 +着 +着呢 +矣 +知道 +确定 +离 +积极 +移动 +突出 +突然 +立即 +第 +等 +等等 +管 +紧接着 +纵 +纵令 +纵使 +纵然 +练习 +组成 +经 +经常 +经过 +结合 +结果 +给 +绝对 +继续 +继而 +维持 +综上所述 +罢了 +考虑 +者 +而 +而且 +而况 +而外 +而已 +而是 +而言 +联系 +能 +能否 +能够 +腾 +自 +自个儿 +自从 +自各儿 +自家 +自己 +自身 +至 +至于 +良好 +若 +若是 +若非 +范围 +莫若 +获得 +虽 +虽则 +虽然 +虽说 +行为 +行动 +表明 +表示 +被 +要 +要不 +要不是 +要不然 +要么 +要是 +要求 +规定 +觉得 +认为 +认真 +认识 +让 +许多 +论 +设使 +设若 +该 +说明 +诸位 +谁 +谁知 +赶 +起 +起来 +起见 +趁 +趁着 +越是 +跟 +转动 +转变 +转贴 +较 +较之 +边 +达到 +迅速 +过 +过去 +过来 +运用 +还是 +还有 +这 +这个 +这么 +这么些 +这么样 +这么点儿 +这些 +这会儿 +这儿 +这就是说 +这时 +这样 +这点 +这种 +这边 +这里 +这麽 +进入 +进步 +进而 +进行 +连 +连同 +适应 +适当 +适用 +逐步 +逐渐 +通常 +通过 +造成 +遇到 +遭到 +避免 +那 +那个 +那么 +那么些 +那么样 +那些 +那会儿 +那儿 +那时 +那样 +那边 +那里 +那麽 +部分 +鄙人 +采取 +里面 +重大 +重新 +重要 +鉴于 +问题 +防止 +阿 +附近 +限制 +除 +除了 +除此之外 +除非 +随 +随着 +随著 +集中 +需要 +非但 +非常 +非徒 +靠 +顺 +顺着 +首先 +高兴 +是不是 +说说 diff --git a/test_result.md b/test_result.md new file mode 100644 index 0000000..2402e98 --- /dev/null +++ b/test_result.md @@ -0,0 +1,28 @@ +# Load Testing with 990 files test + +## knowledge_base +测试时间：26-03-26 + hey -n 1000 -c 50 -H 'x-user-id: 1' -H 'x-role: admin' http://192.168.0.46:11000/api/v1/knowledge/knowledge_base/ + +Summary: + Total: 1.0624 secs + Slowest: 0.1404 secs + Fastest: 0.0050 secs + Average: 0.0501 secs + Requests/sec: 941.3084 + + Total data: 1646000 bytes + Size/request: 1646 bytes + +测试时间：26-03-26 + hey -n 1000 -c 50 -m POST -H 'Content-Type: application/json' -H 'x-user-id: 1' -H 'x-role: admin' -d '{"description": "压测", "is_public": true, "kb_type": "analysis", "name": "压测知识库", "parent_id": 1}' http://192.168.0.46:11000/api/v1/knowledge/knowledge_base/ + + + hey -n 1000 -c 50 -H 'x-user-id: 1' -H 'x-role: admin' http://192.168.0.46:11000/api/v1/knowledge/knowledge_base/2 + +Summary: + Total: 1.9395 secs + Slowest: 0.3394 secs + Fastest: 0.0091 secs + Average: 0.0915 secs + Requests/sec: 515.5856 diff --git a/tests/api_concurrency.rs b/tests/api_concurrency.rs new file mode 100644 index 0000000..ac404d5 --- /dev/null +++ b/tests/api_concurrency.rs @@ -0,0 +1,247 @@ +mod common; +use std::{ + fs, time::{Duration, Instant} +}; + +use axum::http::StatusCode; +use common::*; +use futures::future::join_all; + +#[derive(Debug, Clone)] +struct RequestResult { + status: StatusCode, + duration: Duration, +} + +#[derive(Debug)] +struct ConcurrencyMetrics { + total: usize, + success: usize, + failed: usize, + mean_ms: f64, + p50_ms: f64, + p95_ms: f64, + p99_ms: f64, + rps: f64, + total_duration_ms: f64, +} + +impl std::fmt::Display for ConcurrencyMetrics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "total={} success={} failed={} rps={:.1} mean={:.1}ms p50={:.1}ms p95={:.1}ms p99={:.1}ms total_time={:.1}ms", + self.total, + self.success, + self.failed, + self.rps, + self.mean_ms, + self.p50_ms, + self.p95_ms, + self.p99_ms, + self.total_duration_ms + ) + } +} + +fn percentile(sorted: &[f64], p: f64) -> f64 { + if sorted.is_empty() { + return 0.0; + } + let idx = ((sorted.len() - 1) as f64 * p) as usize; + sorted[idx] +} + +fn compute_metrics(results: Vec, total_duration: Duration) -> ConcurrencyMetrics { + let total = results.len(); + let success = results.iter().filter(|r| r.status.is_success()).count(); + let failed = total.saturating_sub(success); + + let mut durations: Vec = results.iter().map(|r| r.duration.as_secs_f64() * 1000.0).collect(); + durations.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + let mean_ms = durations.iter().sum::() / total.max(1) as f64; + let p50_ms = percentile(&durations, 0.5); + let p95_ms = percentile(&durations, 0.95); + let p99_ms = percentile(&durations, 0.99); + let total_duration_ms = total_duration.as_secs_f64() * 1000.0; + let rps = total as f64 / total_duration.as_secs_f64().max(f64::EPSILON); + + ConcurrencyMetrics { total, success, failed, mean_ms, p50_ms, p95_ms, p99_ms, rps, total_duration_ms } +} + +fn kb_create_body(name: &str) -> Value { + serde_json::json!({ + "name": name, + "description": "perf test knowledge base", + "kb_type": "analysis", + "parent_id": null, + "is_public": false + }) +} + +#[tokio::test] +async fn concurrent_kb_list_read_perf() { + let app = app().await; + let user = TestUser::new("perf-kb-list"); + + // Seed a few KBs so the list endpoint has real work to do. + for _ in 0..5 { + let req = authed_json_request( + "POST", + "/api/v1/knowledge/knowledge_base/", + &user, + kb_create_body(&format!("Seed KB {}", next_seq())), + ); + let _ = app.clone().oneshot(req).await.unwrap(); + } + + const REQUEST_COUNT: usize = 200; + const CONCURRENCY: usize = 50; + + let start = Instant::now(); + let futures: Vec<_> = (0..REQUEST_COUNT) + .map(|i| { + let app = app.clone(); + let user = TestUser::with_role(&format!("perf-kb-list-user-{}", i), "admin"); + async move { + let req_start = Instant::now(); + let req = authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/", &user); + let res = app.oneshot(req).await.unwrap(); + RequestResult { status: res.status(), duration: req_start.elapsed() } + } + }) + .collect(); + + let results = join_all(futures).await; + let metrics = compute_metrics(results, start.elapsed()); + println!("concurrent_kb_list_read_perf ({} requests, concurrency={}): {}", REQUEST_COUNT, CONCURRENCY, metrics); + + assert_eq!(metrics.success, metrics.total, "all KB list requests should succeed"); + assert!(metrics.p95_ms < 500.0, "p95 latency should be below 500ms, got {:.1}ms", metrics.p95_ms); +} + +#[tokio::test] +async fn concurrent_kb_create_write_perf() { + let app = app().await; + + const REQUEST_COUNT: usize = 100; + const CONCURRENCY: usize = 20; + + let start = Instant::now(); + let futures: Vec<_> = (0..REQUEST_COUNT) + .map(|i| { + let app = app.clone(); + let user = TestUser::with_role(&format!("perf-kb-create-user-{}", i), "admin"); + async move { + let req_start = Instant::now(); + let body = kb_create_body(&format!("Perf KB {}", next_seq())); + let req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &user, body); + let res = app.oneshot(req).await.unwrap(); + RequestResult { status: res.status(), duration: req_start.elapsed() } + } + }) + .collect(); + + let results = join_all(futures).await; + let metrics = compute_metrics(results, start.elapsed()); + println!("concurrent_kb_create_write_perf ({} requests, concurrency={}): {}", REQUEST_COUNT, CONCURRENCY, metrics); + + assert_eq!(metrics.success, metrics.total, "all KB create requests should succeed"); + assert!(metrics.p95_ms < 1000.0, "p95 latency should be below 1000ms, got {:.1}ms", metrics.p95_ms); +} + +#[tokio::test] +async fn concurrent_file_list_read_perf() { + let app = app().await; + let pool = get_pool().await; + let env = setup_env(); + let user = TestUser::new("perf-file-list"); + + let kb_id = insert_kb(&pool, &user, "Perf File KB", "analysis", None, false).await; + + // Seed a few files so the file list endpoint has real work to do. + let file_dir = env.data_dir.join("files"); + fs::create_dir_all(&file_dir).unwrap(); + for _ in 0..5 { + let path = file_dir.join(format!("perf-file-{}.txt", next_seq())); + fs::write(&path, b"perf test content").unwrap(); + insert_file(&pool, &user, "perf.txt", &path, Some(kb_id), vec!["perf".to_string()], false).await; + } + + const REQUEST_COUNT: usize = 150; + const CONCURRENCY: usize = 30; + + let start = Instant::now(); + let futures: Vec<_> = (0..REQUEST_COUNT) + .map(|i| { + let app = app.clone(); + let user = TestUser::with_role(&format!("perf-file-list-user-{}", i), "admin"); + async move { + let req_start = Instant::now(); + let req = authed_empty_request("GET", "/api/v1/knowledge/files/", &user); + let res = app.oneshot(req).await.unwrap(); + RequestResult { status: res.status(), duration: req_start.elapsed() } + } + }) + .collect(); + + let results = join_all(futures).await; + let metrics = compute_metrics(results, start.elapsed()); + println!("concurrent_file_list_read_perf ({} requests, concurrency={}): {}", REQUEST_COUNT, CONCURRENCY, metrics); + + assert_eq!(metrics.success, metrics.total, "all file list requests should succeed"); + assert!(metrics.p95_ms < 500.0, "p95 latency should be below 500ms, got {:.1}ms", metrics.p95_ms); +} + +#[tokio::test] +async fn concurrent_mixed_read_write_perf() { + let app = app().await; + let user = TestUser::new("perf-mixed"); + + // Seed a few KBs. + for _ in 0..3 { + let req = authed_json_request( + "POST", + "/api/v1/knowledge/knowledge_base/", + &user, + kb_create_body(&format!("Seed {}", next_seq())), + ); + let _ = app.clone().oneshot(req).await.unwrap(); + } + + const REQUEST_COUNT: usize = 200; + const CONCURRENCY: usize = 40; + // 70% read, 30% write. + const WRITE_RATIO: usize = 7; + + let start = Instant::now(); + let futures: Vec<_> = (0..REQUEST_COUNT) + .map(|i| { + let app = app.clone(); + let user = TestUser::with_role(&format!("perf-mixed-user-{}", i), "admin"); + async move { + let req_start = Instant::now(); + let is_read = i % 10 < WRITE_RATIO; + let req = if is_read { + authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/", &user) + } else { + let body = kb_create_body(&format!("Mixed Perf KB {}", next_seq())); + authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &user, body) + }; + let res = app.oneshot(req).await.unwrap(); + RequestResult { status: res.status(), duration: req_start.elapsed() } + } + }) + .collect(); + + let results = join_all(futures).await; + let metrics = compute_metrics(results, start.elapsed()); + println!( + "concurrent_mixed_read_write_perf ({} requests, concurrency={}, 70% read / 30% write): {}", + REQUEST_COUNT, CONCURRENCY, metrics + ); + + assert!(metrics.success >= metrics.total * 99 / 100, "success rate should be >= 99%"); + assert!(metrics.p95_ms < 1000.0, "p95 latency should be below 1000ms, got {:.1}ms", metrics.p95_ms); +} diff --git a/tests/api_integration.rs b/tests/api_integration.rs new file mode 100644 index 0000000..a955316 --- /dev/null +++ b/tests/api_integration.rs @@ -0,0 +1,918 @@ +mod common; +use std::fs; + +use common::*; + +#[tokio::test] +async fn knowledge_base_flow() { + let app = app().await; + let user = TestUser::new("kb-flow"); + + let unauth_req = + Request::builder().method("GET").uri("/api/v1/knowledge/knowledge_base/").body(Body::empty()).unwrap(); + let unauth_res = app.clone().oneshot(unauth_req).await.unwrap(); + assert_eq!(unauth_res.status(), StatusCode::UNAUTHORIZED); + + let create_body = serde_json::json!({ + "name": "Integration KB", + "description": "kb for integration tests", + "kb_type": "analysis", + "parent_id": null, + "is_public": false + }); + let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &user, create_body); + let create_res = app.clone().oneshot(create_req).await.unwrap(); + assert_eq!(create_res.status(), StatusCode::OK); + let created = response_json(create_res).await; + let kb_id = created["id"].as_i64().expect("created kb id"); + + let list_req = authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/", &user); + let list_res = app.clone().oneshot(list_req).await.unwrap(); + assert_eq!(list_res.status(), StatusCode::OK); + let list_bytes = list_res.into_body().collect().await.unwrap().to_bytes(); + let list: Vec = serde_json::from_slice(&list_bytes).unwrap(); + assert!(list.iter().any(|kb| kb["id"].as_i64() == Some(kb_id) && kb["name"].as_str() == Some("Integration KB"))); + + let update_body = serde_json::json!({ + "parent_id": kb_id + }); + let update_req = + authed_json_request("PUT", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &user, update_body); + let update_res = app.clone().oneshot(update_req).await.unwrap(); + assert_eq!(update_res.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn knowledge_base_public_and_reparse() { + let app = app().await; + let user = TestUser::new("kb-public"); + + let create_body = serde_json::json!({ + "name": "Public KB", + "description": "kb for public tests", + "kb_type": "analysis", + "parent_id": null, + "is_public": false + }); + let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &user, create_body); + let create_res = app.clone().oneshot(create_req).await.unwrap(); + assert_eq!(create_res.status(), StatusCode::OK); + let created = response_json(create_res).await; + let kb_id = created["id"].as_i64().expect("created kb id"); + + let public_req = authed_json_request( + "PUT", + format!("/api/v1/knowledge/knowledge_base/{}", kb_id), + &user, + serde_json::json!({ "is_public": true }), + ); + let public_res = app.clone().oneshot(public_req).await.unwrap(); + assert_eq!(public_res.status(), StatusCode::OK); + let public_json = response_json(public_res).await; + assert_eq!(public_json["is_public"].as_bool(), Some(true)); + + let reparse_req = authed_empty_request("POST", "/api/v1/knowledge/knowledge_base/reparse", &user); + let reparse_res = app.clone().oneshot(reparse_req).await.unwrap(); + assert_eq!(reparse_res.status(), StatusCode::OK); + let reparse_json = response_json(reparse_res).await; + assert_eq!(reparse_json["kb_count"].as_i64(), Some(1)); + assert_eq!(reparse_json["file_count"].as_i64(), Some(0)); +} + +#[tokio::test] +async fn knowledge_base_tree_and_detail_flow() { + let app = app().await; + let pool = get_pool().await; + let env = setup_env(); + let user = TestUser::new("kb-tree"); + + let root_kb_id = insert_kb(&pool, &user, "Root KB", "analysis", None, false).await; + let child_kb_id = insert_kb(&pool, &user, "Child KB", "analysis", Some(root_kb_id), false).await; + + let file_dir = env.data_dir.join("files"); + fs::create_dir_all(&file_dir).unwrap(); + let root_file_path = file_dir.join(format!("root-file-{}.txt", next_seq())); + let child_file_path = file_dir.join(format!("child-file-{}.txt", next_seq())); + fs::write(&root_file_path, b"root file").unwrap(); + fs::write(&child_file_path, b"child file").unwrap(); + + let root_file_id = + insert_file(&pool, &user, "root-file.txt", &root_file_path, Some(root_kb_id), vec!["root".to_string()], false) + .await; + let child_file_id = insert_file( + &pool, + &user, + "child-file.txt", + &child_file_path, + Some(child_kb_id), + vec!["child".to_string()], + false, + ) + .await; + + let tree_req = + authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/tree?kb_id={}", root_kb_id), &user); + let tree_res = app.clone().oneshot(tree_req).await.unwrap(); + assert_eq!(tree_res.status(), StatusCode::OK); + let tree_json = response_json(tree_res).await; + let tree_nodes = tree_json.as_array().expect("tree nodes"); + assert_eq!(tree_nodes.len(), 1); + let root_node = &tree_nodes[0]; + assert_eq!(root_node["id"].as_i64(), Some(root_kb_id)); + assert!(root_node["files"].as_array().unwrap().iter().any(|file| file["id"].as_i64() == Some(root_file_id))); + assert!(root_node["children"].as_array().unwrap().iter().any(|child| { + child["id"].as_i64() == Some(child_kb_id) + && child["files"].as_array().unwrap().iter().any(|file| file["id"].as_i64() == Some(child_file_id)) + })); + + let detail_req = authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}", root_kb_id), &user); + let detail_res = app.clone().oneshot(detail_req).await.unwrap(); + assert_eq!(detail_res.status(), StatusCode::OK); + let detail_json = response_json(detail_res).await; + assert_eq!(detail_json["id"].as_i64(), Some(root_kb_id)); + assert!(detail_json["files"].is_null(), "detail should no longer return files"); + + // New paginated files endpoint + let kb_files_req = authed_empty_request( + "GET", + format!("/api/v1/knowledge/knowledge_base/{}/files?filename=root-file", root_kb_id), + &user, + ); + let kb_files_res = app.clone().oneshot(kb_files_req).await.unwrap(); + assert_eq!(kb_files_res.status(), StatusCode::OK); + let kb_files_json = response_json(kb_files_res).await; + let kb_files_items = kb_files_json["items"].as_array().expect("kb files items"); + assert!(kb_files_items.iter().any(|file| file["id"].as_i64() == Some(root_file_id))); + assert!(kb_files_json["total"].as_i64().unwrap_or(0) >= 1); + + // Pagination + let kb_files_page_req = authed_empty_request( + "GET", + format!("/api/v1/knowledge/knowledge_base/{}/files?size=1&page=1", root_kb_id), + &user, + ); + let kb_files_page_res = app.clone().oneshot(kb_files_page_req).await.unwrap(); + assert_eq!(kb_files_page_res.status(), StatusCode::OK); + let kb_files_page_json = response_json(kb_files_page_res).await; + assert!(kb_files_page_json["total"].as_i64().unwrap_or(0) >= 1); + assert!(kb_files_page_json["items"].as_array().expect("items").len() <= 1); + + // Tag filter + let kb_files_tag_req = + authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/files?tag=root", root_kb_id), &user); + let kb_files_tag_res = app.clone().oneshot(kb_files_tag_req).await.unwrap(); + assert_eq!(kb_files_tag_res.status(), StatusCode::OK); + let kb_files_tag_json = response_json(kb_files_tag_res).await; + assert!( + kb_files_tag_json["items"] + .as_array() + .expect("tag items") + .iter() + .all(|file| { file["id"].as_i64() == Some(root_file_id) }) + ); +} + +#[tokio::test] +async fn knowledge_base_tag_stats_respect_scope_data_quality_and_permissions() { + let app = app().await; + let pool = get_pool().await; + let env = setup_env(); + let owner = TestUser::with_role("tag-owner", "user"); + let viewer = TestUser::with_role("tag-viewer", "user"); + + let root_kb_id = insert_kb(&pool, &owner, "Tag Root", "analysis", None, true).await; + let child_kb_id = insert_kb(&pool, &owner, "Tag Child", "analysis", Some(root_kb_id), false).await; + let file_dir = env.data_dir.join("files"); + fs::create_dir_all(&file_dir).unwrap(); + + let private_path = file_dir.join(format!("tag-private-{}.txt", next_seq())); + let public_path = file_dir.join(format!("tag-public-{}.txt", next_seq())); + let child_path = file_dir.join(format!("tag-child-{}.txt", next_seq())); + let invalid_path = file_dir.join(format!("tag-invalid-{}.txt", next_seq())); + fs::write(&private_path, b"private").unwrap(); + fs::write(&public_path, b"public").unwrap(); + fs::write(&child_path, b"child").unwrap(); + fs::write(&invalid_path, b"invalid").unwrap(); + + insert_file( + &pool, + &owner, + "tag-private.txt", + &private_path, + Some(root_kb_id), + vec!["alpha".to_string(), "shared".to_string(), "shared".to_string()], + false, + ) + .await; + insert_file( + &pool, + &owner, + "tag-public.txt", + &public_path, + Some(root_kb_id), + vec!["alpha".to_string(), "".to_string(), "beta".to_string()], + true, + ) + .await; + insert_file( + &pool, + &owner, + "tag-child.txt", + &child_path, + Some(child_kb_id), + vec!["child".to_string(), "shared".to_string()], + true, + ) + .await; + let invalid_file_id = + insert_file(&pool, &owner, "tag-invalid.txt", &invalid_path, Some(root_kb_id), vec![], false).await; + sqlx::query("UPDATE files SET tags = 'not-json' WHERE id = ?").bind(invalid_file_id).execute(&pool).await.unwrap(); + + let direct_req = + authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/tags", root_kb_id), &owner); + let direct_res = app.clone().oneshot(direct_req).await.unwrap(); + assert_eq!(direct_res.status(), StatusCode::OK); + let direct_json = response_json(direct_res).await; + assert_eq!(direct_json["kb_id"].as_i64(), Some(root_kb_id)); + assert_eq!( + direct_json["tags"], + serde_json::json!([ + {"tag": "alpha", "file_count": 2}, + {"tag": "beta", "file_count": 1}, + {"tag": "shared", "file_count": 1} + ]) + ); + + let descendants_req = authed_empty_request( + "GET", + format!("/api/v1/knowledge/knowledge_base/{}/tags?include_descendants=true", root_kb_id), + &owner, + ); + let descendants_res = app.clone().oneshot(descendants_req).await.unwrap(); + assert_eq!(descendants_res.status(), StatusCode::OK); + let descendants_json = response_json(descendants_res).await; + assert_eq!( + descendants_json["tags"], + serde_json::json!([ + {"tag": "alpha", "file_count": 2}, + {"tag": "shared", "file_count": 2}, + {"tag": "beta", "file_count": 1}, + {"tag": "child", "file_count": 1} + ]) + ); + + let viewer_req = authed_empty_request( + "GET", + format!("/api/v1/knowledge/knowledge_base/{}/tags?include_descendants=true", root_kb_id), + &viewer, + ); + let viewer_res = app.clone().oneshot(viewer_req).await.unwrap(); + assert_eq!(viewer_res.status(), StatusCode::OK); + let viewer_json = response_json(viewer_res).await; + assert_eq!( + viewer_json["tags"], + serde_json::json!([ + {"tag": "alpha", "file_count": 1}, + {"tag": "beta", "file_count": 1} + ]) + ); + + let inaccessible_kb_id = insert_kb(&pool, &owner, "Tag Private", "analysis", None, false).await; + let inaccessible_req = + authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/tags", inaccessible_kb_id), &viewer); + let inaccessible_res = app.clone().oneshot(inaccessible_req).await.unwrap(); + assert_eq!(inaccessible_res.status(), StatusCode::NOT_FOUND); + + let admin = TestUser::new("tag-admin"); + let missing_req = authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/9223372036854775807/tags", &admin); + let missing_res = app.oneshot(missing_req).await.unwrap(); + assert_eq!(missing_res.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn file_endpoints_flow() { + let app = app().await; + let pool = get_pool().await; + let env = setup_env(); + let user = TestUser::new("file"); + + let kb_id = insert_kb(&pool, &user, "File KB", "analysis", None, false).await; + + let file_suffix = next_seq(); + let file_dir = env.data_dir.join("files"); + fs::create_dir_all(&file_dir).unwrap(); + let file_path = file_dir.join(format!("file-{}.txt", file_suffix)); + let file_contents = b"file contents"; + fs::write(&file_path, file_contents).unwrap(); + + let file_id = insert_file( + &pool, + &user, + "file.txt", + &file_path, + Some(kb_id), + vec!["tag1".to_string(), "tag2".to_string()], + false, + ) + .await; + + let slice_id = insert_slice(&pool, file_id, "slice content").await; + insert_slice_position(&pool, slice_id, 1, [1, 2, 3, 4]).await; + + let list_req = authed_empty_request("GET", "/api/v1/knowledge/files/", &user); + let list_res = app.clone().oneshot(list_req).await.unwrap(); + assert_eq!(list_res.status(), StatusCode::OK); + let list_json = response_json(list_res).await; + let list = list_json["items"].as_array().expect("file list items"); + assert!(list_json["total"].as_i64().unwrap_or(0) >= 1); + assert!(list.iter().any(|f| f["id"].as_i64() == Some(file_id))); + + let list_tag_req = authed_empty_request("GET", "/api/v1/knowledge/files/?tag=tag1", &user); + let list_tag_res = app.clone().oneshot(list_tag_req).await.unwrap(); + assert_eq!(list_tag_res.status(), StatusCode::OK); + let list_tag_json = response_json(list_tag_res).await; + let list_tag = list_tag_json["items"].as_array().expect("file list items"); + assert!(list_tag.iter().any(|f| f["id"].as_i64() == Some(file_id))); + + let list_kb_req = authed_empty_request("GET", format!("/api/v1/knowledge/files/?kb_id={}", kb_id), &user); + let list_kb_res = app.clone().oneshot(list_kb_req).await.unwrap(); + assert_eq!(list_kb_res.status(), StatusCode::OK); + let list_kb_json = response_json(list_kb_res).await; + let list_kb = list_kb_json["items"].as_array().expect("file list items"); + assert!(list_kb.iter().any(|f| f["id"].as_i64() == Some(file_id))); + + let get_req = authed_empty_request("GET", format!("/api/v1/knowledge/files/{}", file_id), &user); + let get_res = app.clone().oneshot(get_req).await.unwrap(); + assert_eq!(get_res.status(), StatusCode::OK); + let get_json = response_json(get_res).await; + assert_eq!(get_json["id"].as_i64(), Some(file_id)); + + let update_body = serde_json::json!({ + "filename": "renamed.txt", + "tags": ["tag3"] + }); + let update_req = authed_json_request("PUT", format!("/api/v1/knowledge/files/{}", file_id), &user, update_body); + let update_res = app.clone().oneshot(update_req).await.unwrap(); + assert_eq!(update_res.status(), StatusCode::OK); + let update_json = response_json(update_res).await; + assert_eq!(update_json["filename"].as_str(), Some("renamed.txt")); + let updated_tags: Vec = serde_json::from_str(update_json["tags"].as_str().unwrap()).unwrap(); + assert_eq!(updated_tags, vec!["tag3".to_string()]); + + let download_req = authed_empty_request("GET", format!("/api/v1/knowledge/files/{}/download", file_id), &user); + let download_res = app.clone().oneshot(download_req).await.unwrap(); + assert_eq!(download_res.status(), StatusCode::OK); + let disposition = + download_res.headers().get(header::CONTENT_DISPOSITION).and_then(|v| v.to_str().ok()).unwrap_or(""); + assert!(disposition.contains("renamed.txt")); + let download_bytes = download_res.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(download_bytes.as_ref(), file_contents); + + let slices_req = authed_empty_request("GET", format!("/api/v1/knowledge/files/{}/slices", file_id), &user); + let slices_res = app.clone().oneshot(slices_req).await.unwrap(); + assert_eq!(slices_res.status(), StatusCode::OK); + let slices_json = response_json(slices_res).await; + let slices = slices_json.as_array().expect("slice list"); + let slice = slices.iter().find(|s| s["id"].as_i64() == Some(slice_id)).expect("slice entry"); + assert_eq!(slice["file_id"].as_i64(), Some(file_id)); + assert_eq!(slice["content"].as_str(), Some("slice content")); + assert!(slice["positions"].is_null(), "slices endpoint should not return positions"); + + let delete_req = authed_empty_request("DELETE", format!("/api/v1/knowledge/files/{}", file_id), &user); + let delete_res = app.clone().oneshot(delete_req).await.unwrap(); + assert_eq!(delete_res.status(), StatusCode::OK); + assert!(!file_path.exists()); +} + +#[tokio::test] +async fn slice_highlight_by_id() { + let app = app().await; + let pool = get_pool().await; + let env = setup_env(); + let user = TestUser::new("slice_highlight"); + + let kb_id = insert_kb(&pool, &user, "Slice Highlight KB", "analysis", None, false).await; + + let file_suffix = next_seq(); + let file_dir = env.data_dir.join("files"); + fs::create_dir_all(&file_dir).unwrap(); + let file_path = file_dir.join(format!("highlight-{}.txt", file_suffix)); + fs::write(&file_path, b"highlight content").unwrap(); + + let file_id = insert_file(&pool, &user, "highlight.txt", &file_path, Some(kb_id), Vec::new(), false).await; + + let slice_id = insert_slice(&pool, file_id, "slice content").await; + insert_slice_position(&pool, slice_id, 2, [10, 20, 100, 30]).await; + + // 查询切片高亮信息 + let highlight_req = authed_empty_request( + "GET", + format!("/api/v1/knowledge/files/{}/slices/{}/highlight", file_id, slice_id), + &user, + ); + let highlight_res = app.clone().oneshot(highlight_req).await.unwrap(); + assert_eq!(highlight_res.status(), StatusCode::OK); + let highlight_json = response_json(highlight_res).await; + let positions = highlight_json["positions"].as_array().expect("positions array"); + assert_eq!(positions.len(), 1); + assert_eq!(positions[0]["page_idx"].as_i64(), Some(2)); + let bbox = positions[0]["bbox"].as_array().expect("bbox array"); + assert_eq!(bbox.len(), 4); + assert_eq!(bbox[0].as_i64(), Some(10)); + assert_eq!(bbox[3].as_i64(), Some(30)); + + // 切片不属于该文件应返回 400 + let other_file_id = insert_file(&pool, &user, "other.txt", &file_path, Some(kb_id), Vec::new(), false).await; + let bad_req = authed_empty_request( + "GET", + format!("/api/v1/knowledge/files/{}/slices/{}/highlight", other_file_id, slice_id), + &user, + ); + let bad_res = app.clone().oneshot(bad_req).await.unwrap(); + assert_eq!(bad_res.status(), StatusCode::BAD_REQUEST); + + // 不存在的切片应返回 404 + let missing_req = + authed_empty_request("GET", format!("/api/v1/knowledge/files/{}/slices/{}/highlight", file_id, 999999), &user); + let missing_res = app.clone().oneshot(missing_req).await.unwrap(); + assert_eq!(missing_res.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn slice_highlight_page_by_id() { + let app = app().await; + let pool = get_pool().await; + let env = setup_env(); + let user = TestUser::new("slice_highlight_page"); + + let kb_id = insert_kb(&pool, &user, "Slice Highlight Page KB", "analysis", None, false).await; + + let file_suffix = next_seq(); + let file_dir = env.data_dir.join("files"); + fs::create_dir_all(&file_dir).unwrap(); + let file_path = file_dir.join(format!("highlight-page-{}.txt", file_suffix)); + fs::write(&file_path, b"highlight content").unwrap(); + + let file_id = insert_file(&pool, &user, "highlight-page.txt", &file_path, Some(kb_id), Vec::new(), false).await; + + // 为文件写入 pdf_content，使 highlight-page 能按内容字数判断 + let pdf_rows = vec![ + htknow::pdf_content::PdfContent { + page_idx: 0, + bbox: Some("[10,20,100,30]".to_string()), + text: Some("a".to_string()), + text_level: None, + img_path: None, + table_body: None, + }, + htknow::pdf_content::PdfContent { + page_idx: 1, + bbox: Some("[10,20,100,30]".to_string()), + text: Some("a".to_string()), + text_level: None, + img_path: None, + table_body: None, + }, + ]; + htknow::pdf_content::write(file_id, &pdf_rows).await.unwrap(); + + // 场景 1：第一页内容字数少于阈值（19 * 1 = 19 < 20）且存在第二页，应返回第二页 + let slice_id_a = insert_slice(&pool, file_id, "slice a").await; + for _ in 0..19 { + insert_slice_position(&pool, slice_id_a, 0, [10, 20, 100, 30]).await; + } + insert_slice_position(&pool, slice_id_a, 1, [10, 20, 100, 30]).await; + + let req_a = authed_empty_request( + "GET", + format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", file_id, slice_id_a), + &user, + ); + let res_a = app.clone().oneshot(req_a).await.unwrap(); + assert_eq!(res_a.status(), StatusCode::OK); + let json_a = response_json(res_a).await; + assert_eq!(json_a["page_idx"].as_i64(), Some(1)); + + // 场景 2：第一页内容字数达到阈值（21 * 1 = 21 >= 20），应返回第一页 + let slice_id_b = insert_slice(&pool, file_id, "slice b").await; + for _ in 0..21 { + insert_slice_position(&pool, slice_id_b, 0, [10, 20, 100, 30]).await; + } + insert_slice_position(&pool, slice_id_b, 1, [10, 20, 100, 30]).await; + + let req_b = authed_empty_request( + "GET", + format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", file_id, slice_id_b), + &user, + ); + let res_b = app.clone().oneshot(req_b).await.unwrap(); + assert_eq!(res_b.status(), StatusCode::OK); + let json_b = response_json(res_b).await; + assert_eq!(json_b["page_idx"].as_i64(), Some(0)); + + // 场景 3：只有一页且内容字数少于阈值（无 pdf_content 匹配，按位置数量兜底为 1），仍返回第一页 + let slice_id_c = insert_slice(&pool, file_id, "slice c").await; + insert_slice_position(&pool, slice_id_c, 2, [10, 20, 100, 30]).await; + + let req_c = authed_empty_request( + "GET", + format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", file_id, slice_id_c), + &user, + ); + let res_c = app.clone().oneshot(req_c).await.unwrap(); + assert_eq!(res_c.status(), StatusCode::OK); + let json_c = response_json(res_c).await; + assert_eq!(json_c["page_idx"].as_i64(), Some(2)); + + // 场景 4：切片没有高亮位置应返回 404 + let slice_id_d = insert_slice(&pool, file_id, "slice d").await; + let req_d = authed_empty_request( + "GET", + format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", file_id, slice_id_d), + &user, + ); + let res_d = app.clone().oneshot(req_d).await.unwrap(); + assert_eq!(res_d.status(), StatusCode::NOT_FOUND); + + // 场景 5：切片不属于该文件应返回 400 + let other_file_id = insert_file(&pool, &user, "other.txt", &file_path, Some(kb_id), Vec::new(), false).await; + let req_bad = authed_empty_request( + "GET", + format!("/api/v1/knowledge/files/{}/slices/{}/highlight-page", other_file_id, slice_id_a), + &user, + ); + let res_bad = app.clone().oneshot(req_bad).await.unwrap(); + assert_eq!(res_bad.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn graph_endpoints_flow() { + let app = app().await; + let pool = get_pool().await; + let env = setup_env(); + let user = TestUser::new("graph"); + + let kb_id = insert_kb(&pool, &user, "Graph KB", "analysis", None, false).await; + + let file_suffix = next_seq(); + let file_dir = env.data_dir.join("files"); + fs::create_dir_all(&file_dir).unwrap(); + let file_path = file_dir.join(format!("graph-{}.txt", file_suffix)); + fs::write(&file_path, b"graph file").unwrap(); + + let file_id = insert_file(&pool, &user, "graph.txt", &file_path, Some(kb_id), Vec::new(), false).await; + let slice_id = insert_slice(&pool, file_id, "mention context").await; + + let node_a_id = insert_graph_node( + &pool, + "Alpha", + "device", + Some(serde_json::json!({ "origin": "test" })), + Some(file_id), + Some(kb_id), + ) + .await; + let node_b_id = insert_graph_node(&pool, "Beta", "component", None, Some(file_id), Some(kb_id)).await; + insert_graph_edge(&pool, node_a_id, node_b_id, "related_to", Some(file_id)).await; + insert_entity_mention(&pool, node_a_id, slice_id, "Alpha mention").await; + + let search_req = + authed_empty_request("GET", format!("/api/v1/knowledge/graph/entities?kb_id={}&q=Alpha", kb_id), &user); + let search_res = app.clone().oneshot(search_req).await.unwrap(); + assert_eq!(search_res.status(), StatusCode::OK); + let search_json = response_json(search_res).await; + let search_list = search_json.as_array().expect("entity list"); + assert!(search_list.iter().any(|entity| entity["id"].as_i64() == Some(node_a_id))); + + let entity_req = authed_empty_request("GET", format!("/api/v1/knowledge/graph/entities/{}", node_a_id), &user); + let entity_res = app.clone().oneshot(entity_req).await.unwrap(); + assert_eq!(entity_res.status(), StatusCode::OK); + let entity_json = response_json(entity_res).await; + assert_eq!(entity_json["entity"]["id"].as_i64(), Some(node_a_id)); + let neighbors = entity_json["neighbors"].as_array().expect("neighbors"); + assert_eq!(neighbors.len(), 1); + assert_eq!(neighbors[0]["entity"]["id"].as_i64(), Some(node_b_id)); + assert_eq!(neighbors[0]["direction"].as_str(), Some("outgoing")); + let mentions = entity_json["mentions"].as_array().expect("mentions"); + assert_eq!(mentions.len(), 1); + assert_eq!(mentions[0]["file_id"].as_i64(), Some(file_id)); + + let stats_req = authed_empty_request("GET", format!("/api/v1/knowledge/graph/stats?kb_id={}", kb_id), &user); + let stats_res = app.clone().oneshot(stats_req).await.unwrap(); + assert_eq!(stats_res.status(), StatusCode::OK); + let stats_json = response_json(stats_res).await; + assert_eq!(stats_json["node_count"].as_i64(), Some(2)); + assert_eq!(stats_json["edge_count"].as_i64(), Some(1)); + let entity_types = stats_json["entity_types"].as_object().expect("entity types"); + assert_eq!(entity_types.get("device").and_then(|v| v.as_i64()), Some(1)); + assert_eq!(entity_types.get("component").and_then(|v| v.as_i64()), Some(1)); + let relation_types = stats_json["relation_types"].as_object().expect("relation types"); + assert_eq!(relation_types.get("related_to").and_then(|v| v.as_i64()), Some(1)); +} + +#[tokio::test] +async fn search_full_empty_and_image_requires_file() { + let app = app().await; + let user = TestUser::new("search"); + + let full_req = authed_empty_request("GET", "/api/v1/knowledge/search/full?query=missing", &user); + let full_res = app.clone().oneshot(full_req).await.unwrap(); + assert_eq!(full_res.status(), StatusCode::OK); + let full_json = response_json(full_res).await; + assert_eq!(full_json["results"].as_array().map(|v| v.len()), Some(0)); + + let boundary = format!("boundary-{}", next_seq()); + let body = multipart_body(&boundary, &[("text", "sample")]); + let image_req = authed_multipart_request("POST", "/api/v1/knowledge/search/image", &user, &boundary, body); + let image_res = app.clone().oneshot(image_req).await.unwrap(); + assert_eq!(image_res.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn kb_permission_viewer_can_read_but_not_modify() { + let app = app().await; + let owner = TestUser::with_role("kb-perm-owner", "user"); + let viewer = TestUser::with_role("kb-perm-viewer", "user"); + + // Owner creates a private KB + let create_body = serde_json::json!({ + "name": "Permission Test KB", + "description": "kb for permission tests", + "kb_type": "analysis", + "parent_id": null, + "is_public": false + }); + let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &owner, create_body); + let create_res = app.clone().oneshot(create_req).await.unwrap(); + assert_eq!(create_res.status(), StatusCode::OK); + let created = response_json(create_res).await; + let kb_id = created["id"].as_i64().expect("created kb id"); + assert_eq!(created["current_user_permission"].as_str(), Some("admin")); + + // Owner grants viewer permission + let grant_req = authed_json_request( + "POST", + format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), + &owner, + serde_json::json!({ "user_id": viewer.id, "permission": "viewer" }), + ); + let grant_res = app.clone().oneshot(grant_req).await.unwrap(); + assert_eq!(grant_res.status(), StatusCode::OK); + + // Viewer can list the KB + let list_req = authed_empty_request("GET", "/api/v1/knowledge/knowledge_base/", &viewer); + let list_res = app.clone().oneshot(list_req).await.unwrap(); + assert_eq!(list_res.status(), StatusCode::OK); + let list_bytes = list_res.into_body().collect().await.unwrap().to_bytes(); + let list: Vec = serde_json::from_slice(&list_bytes).unwrap(); + let kb = list.iter().find(|k| k["id"].as_i64() == Some(kb_id)).expect("viewer sees the kb"); + assert_eq!(kb["current_user_permission"].as_str(), Some("viewer")); + + // Viewer can get detail + let get_req = authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &viewer); + let get_res = app.clone().oneshot(get_req).await.unwrap(); + assert_eq!(get_res.status(), StatusCode::OK); + let get_json = response_json(get_res).await; + assert_eq!(get_json["current_user_permission"].as_str(), Some("viewer")); + + // Viewer cannot update the KB + let update_req = authed_json_request( + "PUT", + format!("/api/v1/knowledge/knowledge_base/{}", kb_id), + &viewer, + serde_json::json!({ "name": "Hacked" }), + ); + let update_res = app.clone().oneshot(update_req).await.unwrap(); + assert_eq!(update_res.status(), StatusCode::FORBIDDEN); + + // Viewer cannot reparse + let reparse_req = + authed_empty_request("POST", format!("/api/v1/knowledge/knowledge_base/{}/reparse", kb_id), &viewer); + let reparse_res = app.clone().oneshot(reparse_req).await.unwrap(); + assert_eq!(reparse_res.status(), StatusCode::FORBIDDEN); + + // Viewer cannot delete + let delete_req = authed_empty_request("DELETE", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &viewer); + let delete_res = app.clone().oneshot(delete_req).await.unwrap(); + assert_eq!(delete_res.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn kb_permission_editor_can_upload_but_not_delete() { + let app = app().await; + let env = setup_env(); + let owner = TestUser::with_role("kb-editor-owner", "user"); + let editor = TestUser::with_role("kb-editor-editor", "user"); + + // Owner creates a private storage KB (so we can upload without parse complications) + let create_body = serde_json::json!({ + "name": "Editor Test KB", + "description": "kb for editor permission tests", + "kb_type": "storage", + "parent_id": null, + "is_public": false + }); + let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &owner, create_body); + let create_res = app.clone().oneshot(create_req).await.unwrap(); + assert_eq!(create_res.status(), StatusCode::OK); + let kb_id = response_json(create_res).await["id"].as_i64().unwrap(); + + // Owner grants editor permission + let grant_req = authed_json_request( + "POST", + format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), + &owner, + serde_json::json!({ "user_id": editor.id, "permission": "editor" }), + ); + let grant_res = app.clone().oneshot(grant_req).await.unwrap(); + assert_eq!(grant_res.status(), StatusCode::OK); + + // Editor can update the KB name/description + let update_req = authed_json_request( + "PUT", + format!("/api/v1/knowledge/knowledge_base/{}", kb_id), + &editor, + serde_json::json!({ "name": "Renamed by Editor", "description": "updated" }), + ); + let update_res = app.clone().oneshot(update_req).await.unwrap(); + assert_eq!(update_res.status(), StatusCode::OK); + + // Editor cannot change visibility + let vis_req = authed_json_request( + "PUT", + format!("/api/v1/knowledge/knowledge_base/{}", kb_id), + &editor, + serde_json::json!({ "is_public": true }), + ); + let vis_res = app.clone().oneshot(vis_req).await.unwrap(); + assert_eq!(vis_res.status(), StatusCode::FORBIDDEN); + + // Editor can upload a file + let file_dir = env.data_dir.join("files"); + fs::create_dir_all(&file_dir).unwrap(); + let test_file = file_dir.join(format!("editor-upload-{}.txt", next_seq())); + fs::write(&test_file, b"editor upload test").unwrap(); + + let boundary = format!("boundary-{}", next_seq()); + let upload_req = authed_multipart_request_with_file( + "POST", + "/api/v1/knowledge/files/", + &editor, + &boundary, + &[("kb_id", &kb_id.to_string()), ("slice_type", "text")], + "file", + "test.txt", + b"editor upload test", + ); + let upload_res = app.clone().oneshot(upload_req).await.unwrap(); + assert_eq!(upload_res.status(), StatusCode::OK); + + // Editor cannot delete the KB + let delete_req = authed_empty_request("DELETE", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &editor); + let delete_res = app.clone().oneshot(delete_req).await.unwrap(); + assert_eq!(delete_res.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn kb_permission_admin_can_manage_permissions() { + let app = app().await; + let owner = TestUser::with_role("kb-admin-owner", "user"); + let viewer = TestUser::with_role("kb-admin-viewer", "user"); + + // Owner creates KB + let create_body = serde_json::json!({ + "name": "Admin Perm Test KB", + "description": "test", + "kb_type": "analysis", + "is_public": false + }); + let create_req = authed_json_request("POST", "/api/v1/knowledge/knowledge_base/", &owner, create_body); + let create_res = app.clone().oneshot(create_req).await.unwrap(); + let kb_id = response_json(create_res).await["id"].as_i64().unwrap(); + + // Owner lists permissions (should be empty initially except no explicit rows) + let list_req = + authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), &owner); + let list_res = app.clone().oneshot(list_req).await.unwrap(); + assert_eq!(list_res.status(), StatusCode::OK); + + // Owner grants viewer permission + let grant_req = authed_json_request( + "POST", + format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), + &owner, + serde_json::json!({ "user_id": viewer.id, "permission": "viewer" }), + ); + let grant_res = app.clone().oneshot(grant_req).await.unwrap(); + assert_eq!(grant_res.status(), StatusCode::OK); + let grant_json = response_json(grant_res).await; + assert_eq!(grant_json["user_id"].as_str(), Some(viewer.id.as_str())); + assert_eq!(grant_json["permission"].as_str(), Some("viewer")); + + // Viewer cannot call permission APIs + let viewer_list_req = + authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), &viewer); + let viewer_list_res = app.clone().oneshot(viewer_list_req).await.unwrap(); + assert_eq!(viewer_list_res.status(), StatusCode::FORBIDDEN); + + // Owner upgrades viewer to admin + let upgrade_req = authed_json_request( + "POST", + format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), + &owner, + serde_json::json!({ "user_id": viewer.id, "permission": "admin" }), + ); + let upgrade_res = app.clone().oneshot(upgrade_req).await.unwrap(); + assert_eq!(upgrade_res.status(), StatusCode::OK); + + // Now viewer (as KB admin) can list permissions + let list2_req = + authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), &viewer); + let list2_res = app.clone().oneshot(list2_req).await.unwrap(); + assert_eq!(list2_res.status(), StatusCode::OK); + + // Owner removes viewer permission + let remove_req = authed_empty_request( + "DELETE", + format!("/api/v1/knowledge/knowledge_base/{}/permissions/{}", kb_id, viewer.id), + &owner, + ); + let remove_res = app.clone().oneshot(remove_req).await.unwrap(); + assert_eq!(remove_res.status(), StatusCode::OK); + + // After removal, viewer can no longer access the KB + let get_req = authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &viewer); + let get_res = app.clone().oneshot(get_req).await.unwrap(); + assert_eq!(get_res.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn kb_permission_search_filters_unauthorized_kb() { + let app = app().await; + let pool = get_pool().await; + let owner = TestUser::with_role("kb-search-owner", "user"); + let other = TestUser::with_role("kb-search-other", "user"); + + // Owner creates a private KB with a file + let kb_id = insert_kb(&pool, &owner, "Search Permission KB", "analysis", None, false).await; + let file_dir = setup_env().data_dir.join("files"); + fs::create_dir_all(&file_dir).unwrap(); + let test_file = file_dir.join(format!("search-perm-{}.txt", next_seq())); + fs::write(&test_file, b"secret content about dragons").unwrap(); + let file_id = insert_file(&pool, &owner, "secret-perm.txt", &test_file, Some(kb_id), vec![], false).await; + + // Set file as completed so it appears in full search + sqlx::query("UPDATE files SET status = 1 WHERE id = ?").bind(file_id).execute(&pool).await.unwrap(); + + // Owner can use full-search filename filter and find the file + let owner_search_req = + authed_empty_request("GET", "/api/v1/knowledge/search/full?filename=secret-perm.txt", &owner); + let owner_search_res = app.clone().oneshot(owner_search_req).await.unwrap(); + assert_eq!(owner_search_res.status(), StatusCode::OK); + let owner_json = response_json(owner_search_res).await; + let owner_results = owner_json["results"].as_array().expect("results"); + assert!( + owner_results.iter().any(|r| r["file"]["id"].as_i64() == Some(file_id)), + "owner should find their own file" + ); + + // Other user without permission cannot see the result + let other_search_req = + authed_empty_request("GET", "/api/v1/knowledge/search/full?filename=secret-perm.txt", &other); + let other_search_res = app.clone().oneshot(other_search_req).await.unwrap(); + assert_eq!(other_search_res.status(), StatusCode::OK); + let other_json = response_json(other_search_res).await; + let other_results = other_json["results"].as_array().expect("results"); + assert!( + !other_results.iter().any(|r| r["file"]["id"].as_i64() == Some(file_id)), + "other user should not see unauthorized file" + ); + + // Grant viewer permission and verify search now returns result + let grant_req = authed_json_request( + "POST", + format!("/api/v1/knowledge/knowledge_base/{}/permissions", kb_id), + &owner, + serde_json::json!({ "user_id": other.id, "permission": "viewer" }), + ); + let grant_res = app.clone().oneshot(grant_req).await.unwrap(); + assert_eq!(grant_res.status(), StatusCode::OK); + + // Verify other can now access the KB detail + let get_req = authed_empty_request("GET", format!("/api/v1/knowledge/knowledge_base/{}", kb_id), &other); + let get_res = app.clone().oneshot(get_req).await.unwrap(); + assert_eq!(get_res.status(), StatusCode::OK); + let get_json = response_json(get_res).await; + assert_eq!(get_json["current_user_permission"].as_str(), Some("viewer")); + + let granted_search_req = + authed_empty_request("GET", "/api/v1/knowledge/search/full?filename=secret-perm.txt", &other); + let granted_search_res = app.clone().oneshot(granted_search_req).await.unwrap(); + assert_eq!(granted_search_res.status(), StatusCode::OK); + let granted_json = response_json(granted_search_res).await; + let granted_results = granted_json["results"].as_array().expect("results"); + assert!( + granted_results.iter().any(|r| r["file"]["id"].as_i64() == Some(file_id)), + "viewer should now find the file" + ); +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..cd21534 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,293 @@ +#![allow(dead_code, unused_imports)] + +use std::{ + fs, path::PathBuf, sync::{ + OnceLock, atomic::{AtomicUsize, Ordering} + }, time::{SystemTime, UNIX_EPOCH} +}; + +use axum::{Router, extract::DefaultBodyLimit, middleware}; +pub use axum::{ + body::Body, http::{Request, Response, StatusCode, header} +}; +use htknow::{api, auth, db, search::SearchEngine, slice_content}; +pub use http_body_util::BodyExt; +pub use serde_json::Value; +use sqlx::SqlitePool; +use tokio::sync::OnceCell; +pub use tower::ServiceExt; + +pub struct TestEnv { + pub data_dir: PathBuf, +} + +static TEST_ENV: OnceLock = OnceLock::new(); +static APP: OnceCell = OnceCell::const_new(); +static TEST_SEQ: AtomicUsize = AtomicUsize::new(1); + +pub fn next_seq() -> usize { + TEST_SEQ.fetch_add(1, Ordering::Relaxed) +} + +pub fn setup_env() -> &'static TestEnv { + TEST_ENV.get_or_init(|| { + let nonce = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let base_dir = std::env::temp_dir().join(format!("htknow-it-{}", nonce)); + std::fs::create_dir_all(&base_dir).expect("create test data dir"); + + let db_path = base_dir.join("test.sqlite"); + unsafe { + std::env::set_var("HTKNOW_DATA_DIR", base_dir.to_string_lossy().as_ref()); + std::env::set_var("DATABASE_URL", format!("sqlite://{}", db_path.display())); + std::env::set_var("HTKNOW_DB_MAX_CONNECTIONS", "10"); + std::env::set_var("HTKNOW_DB_INIT_DEFAULT_KBS", "false"); + std::env::set_var("HTKNOW_SERVER_UPLOAD_LIMIT_MB", "1"); + std::env::set_var("HTKNOW_SEARCH_LIMIT", "5"); + } + + TestEnv { data_dir: base_dir } + }) +} + +pub async fn build_app() -> Router { + setup_env(); + + let pool = db::init().await.expect("init db"); + let search_engine = SearchEngine::init().await.with_pool(pool.clone()); + let upload_limit = htknow::config::get().server.upload_limit_mb * 1024 * 1024; + + Router::new() + .nest("/api/v1/knowledge/", api::app(pool, search_engine)) + .layer(middleware::from_fn(auth)) + .layer(DefaultBodyLimit::max(upload_limit)) +} + +pub async fn app() -> Router { + APP.get_or_init(|| async { build_app().await }).await.clone() +} + +pub async fn get_pool() -> SqlitePool { + let _ = app().await; + db::init().await.expect("init db") +} + +pub struct TestUser { + pub id: String, + pub name: String, + pub role: String, +} + +impl TestUser { + pub fn new(prefix: &str) -> Self { + Self::with_role(prefix, "admin") + } + + pub fn with_role(prefix: &str, role: &str) -> Self { + let seq = next_seq(); + Self { id: format!("{}-{}", prefix, seq), name: format!("{}-name-{}", prefix, seq), role: role.to_string() } + } +} + +pub fn authed_request( + method: &str, uri: String, user: &TestUser, body: Body, content_type: Option, +) -> Request { + let mut builder = Request::builder().method(method).uri(uri); + builder = builder.header("x-user-id", &user.id).header("x-user-name", &user.name).header("x-role", &user.role); + if let Some(content_type) = content_type { + builder = builder.header(header::CONTENT_TYPE, content_type); + } + builder.body(body).unwrap() +} + +pub fn authed_empty_request(method: &str, uri: impl Into, user: &TestUser) -> Request { + authed_request(method, uri.into(), user, Body::empty(), None) +} + +pub fn authed_json_request(method: &str, uri: impl Into, user: &TestUser, body: Value) -> Request { + authed_request(method, uri.into(), user, Body::from(body.to_string()), Some("application/json".to_string())) +} + +pub fn authed_multipart_request( + method: &str, uri: impl Into, user: &TestUser, boundary: &str, body: Vec, +) -> Request { + let content_type = format!("multipart/form-data; boundary={}", boundary); + authed_request(method, uri.into(), user, Body::from(body), Some(content_type)) +} + +pub async fn response_json(res: Response) -> Value { + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + if bytes.is_empty() { + panic!("response body is empty"); + } + serde_json::from_slice(&bytes).unwrap_or_else(|e| { + panic!("failed to parse JSON response: {}\nbody: {}", e, String::from_utf8_lossy(&bytes)); + }) +} + +pub async fn insert_kb( + pool: &SqlitePool, user: &TestUser, name: &str, kb_type: &str, parent_id: Option, is_public: bool, +) -> i64 { + let is_public = if is_public { 1 } else { 0 }; + sqlx::query( + "INSERT INTO knowledge_bases (user_id, user_name, name, description, kb_type, parent_id, is_public) \ + VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&user.id) + .bind(&user.name) + .bind(name) + .bind("test knowledge base") + .bind(kb_type) + .bind(parent_id) + .bind(is_public) + .execute(pool) + .await + .unwrap() + .last_insert_rowid() +} + +pub async fn insert_file( + pool: &SqlitePool, user: &TestUser, filename: &str, path: &PathBuf, kb_id: Option, tags: Vec, + is_public: bool, +) -> i64 { + let is_public = if is_public { 1 } else { 0 }; + let tags_json = serde_json::to_string(&tags).unwrap(); + let hash = format!("hash-{}", next_seq()); + let size = fs::metadata(path).map(|meta| meta.len() as i64).unwrap_or(0); + sqlx::query( + "INSERT INTO files (user_id, user_name, hash, filename, path, size, slice_type, kb_id, is_public, tags, status, log) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&user.id) + .bind(&user.name) + .bind(hash) + .bind(filename) + .bind(path.to_string_lossy().as_ref()) + .bind(size) + .bind("text") + .bind(kb_id) + .bind(is_public) + .bind(tags_json) + .bind(0) + .bind("") + .execute(pool) + .await + .unwrap() + .last_insert_rowid() +} + +pub async fn insert_slice(pool: &SqlitePool, file_id: i64, content: &str) -> i64 { + let id = sqlx::query("INSERT INTO slices (file_id) VALUES (?)") + .bind(file_id) + .execute(pool) + .await + .unwrap() + .last_insert_rowid(); + slice_content::upsert_many(file_id, &[(id, content.to_string())]).await.unwrap(); + id +} + +pub async fn insert_slice_position(pool: &SqlitePool, slice_id: i64, page_idx: i32, bbox: [i32; 4]) { + insert_slice_position_full(pool, slice_id, page_idx, bbox, None, None).await; +} + +pub async fn insert_slice_position_full( + pool: &SqlitePool, slice_id: i64, page_idx: i32, bbox: [i32; 4], sheet_name: Option<&str>, row_num: Option, +) { + sqlx::query( + "INSERT INTO slice_positions (slice_id, page_idx, x1, y1, x2, y2, sheet_name, row_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ) + .bind(slice_id) + .bind(page_idx) + .bind(bbox[0]) + .bind(bbox[1]) + .bind(bbox[2]) + .bind(bbox[3]) + .bind(sheet_name) + .bind(row_num) + .execute(pool) + .await + .unwrap(); +} + +pub async fn insert_graph_node( + pool: &SqlitePool, name: &str, entity_type: &str, properties: Option, file_id: Option, + kb_id: Option, +) -> i64 { + let props = properties.map(|value| value.to_string()); + sqlx::query( + "INSERT INTO graph_nodes (name, entity_type, properties, file_id, kb_id, is_public) \ + VALUES (?, ?, ?, ?, ?, 0)", + ) + .bind(name) + .bind(entity_type) + .bind(props) + .bind(file_id) + .bind(kb_id) + .execute(pool) + .await + .unwrap() + .last_insert_rowid() +} + +pub async fn insert_graph_edge( + pool: &SqlitePool, source_node_id: i64, target_node_id: i64, relation_type: &str, file_id: Option, +) -> i64 { + sqlx::query("INSERT INTO graph_edges (source_node_id, target_node_id, relation_type, file_id) VALUES (?, ?, ?, ?)") + .bind(source_node_id) + .bind(target_node_id) + .bind(relation_type) + .bind(file_id) + .execute(pool) + .await + .unwrap() + .last_insert_rowid() +} + +pub async fn insert_entity_mention(pool: &SqlitePool, node_id: i64, slice_id: i64, context: &str) -> i64 { + sqlx::query("INSERT INTO entity_mentions (node_id, slice_id, context) VALUES (?, ?, ?)") + .bind(node_id) + .bind(slice_id) + .bind(context) + .execute(pool) + .await + .unwrap() + .last_insert_rowid() +} + +pub fn multipart_body(boundary: &str, fields: &[(&str, &str)]) -> Vec { + let mut body = Vec::new(); + for (name, value) in fields { + body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes()); + body.extend_from_slice(format!("Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name).as_bytes()); + body.extend_from_slice(value.as_bytes()); + body.extend_from_slice(b"\r\n"); + } + body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes()); + body +} + +#[allow(clippy::too_many_arguments)] +// Helper: build multipart request that includes an actual file field +pub fn authed_multipart_request_with_file( + method: &str, uri: impl Into, user: &TestUser, boundary: &str, extra_fields: &[(&str, &str)], + field_name: &str, file_name: &str, file_content: &[u8], +) -> Request { + let mut body = Vec::new(); + for (name, value) in extra_fields { + body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes()); + body.extend_from_slice(format!("Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name).as_bytes()); + body.extend_from_slice(value.as_bytes()); + body.extend_from_slice(b"\r\n"); + } + body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes()); + body.extend_from_slice( + format!("Content-Disposition: form-data; name=\"{}\"; filename=\"{}\"\r\n", field_name, file_name).as_bytes(), + ); + body.extend_from_slice(b"Content-Type: text/plain\r\n\r\n"); + body.extend_from_slice(file_content); + body.extend_from_slice(b"\r\n"); + body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes()); + + let content_type = format!("multipart/form-data; boundary={}", boundary); + authed_request(method, uri.into(), user, Body::from(body), Some(content_type)) +} diff --git a/todo.md b/todo.md new file mode 100644 index 0000000..fdf0ed9 --- /dev/null +++ b/todo.md @@ -0,0 +1,3 @@ +文档：122-06A0014-B01001_发动机-维修手册.pdf +搜索：保养主推进柴油机的步骤 +搜索方式：高级搜索